[R14 integration] Full Python port stack + live R cache + source-faithful fixes (DRAFT)

.github/workflows/inspect-r-api-update.yml

@@ -3,7 +3,7 @@ name: Regenerate R parity cache
 on:
   # One-shot integration audit: run against the complete Python PR stack.
   push:
-    branches: [integration/r14-parity-on-python-stack-20261106]
+    branches: [integration/r14-parity-on-python-stack-20261106, automation/r14-stack-cache-20261106]
     paths: [.github/workflows/inspect-r-api-update.yml]
   repository_dispatch:
     types: [nns-r-package-updated]
@@ -158,7 +158,7 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          Rscript -e "options(repos=c(CRAN='https://cloud.r-project.org')); install.packages(c('remotes','jsonlite')); remotes::install_deps('upstream/package', dependencies=NA, upgrade='never')"
+          Rscript -e "options(repos=c(CRAN='https://cloud.r-project.org')); install.packages(c('remotes','jsonlite','mlbench')); remotes::install_deps('upstream/package', dependencies=NA, upgrade='never')"
 
       - name: Install exact NNS source package
         shell: bash
@@ -174,7 +174,7 @@ jobs:
           set -euo pipefail
           python -m pip install -U pip
           python -m pip install build scikit-build-core nanobind pytest ruff==0.16.3 mypy==2.3.1 "numpy<2.5" scipy
-          python -m pip install hypothesis pytest-benchmark pytest-xdist pandas
+          python -m pip install hypothesis pytest-benchmark pytest-xdist pandas matplotlib
           python -m pip install -e . --force-reinstall --no-deps
 
       - name: Regenerate complete live-R parity cache
@@ -333,8 +333,26 @@ jobs:
             committed-cache-verification.log
           if-no-files-found: warn
 
+      # Rebuild directly on the existing #192 branch.  Do not let the
+      # create-pull-request action reset this branch from #191 and discard
+      # the independent R14 Python implementation fixes already on #192.
+      - name: Commit regenerated R14 cache to existing audit branch
+        if: ${{ always() && github.event_name == 'push' && github.ref_name == 'automation/r14-stack-cache-20261106' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git add -A tests/_r.py tests/_r_cache sync/nns_source.json
+          if ! git diff --cached --quiet; then
+            git commit -m 'Rebuild R 14.0 parity fixtures with heterogeneous-list decoder'
+            # Fast-forward only; concurrent branch edits must never be overwritten.
+            git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}"
+          else
+            echo 'R cache is unchanged after fresh regeneration.'
+          fi
       - name: Open or update R cache regeneration PR
-        if: always()
+        if: ${{ always() && github.ref_name != 'automation/r14-stack-cache-20261106' }}
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }}
@@ -371,7 +389,7 @@ jobs:
         run: |
           echo "Live regeneration outcome: ${{ steps.regenerate.outcome }}"
           echo "Cache-only verification outcome: ${{ steps.verify.outcome }}"
-          echo "The regenerated R cache has been preserved in its pull request."
+          echo "The regenerated R cache has been preserved on the audit branch or in its PR."
           if [ "${{ steps.regenerate.outcome }}" != "success" ] || [ "${{ steps.verify.outcome }}" != "success" ]; then
             echo "::error::The oracle regeneration or its offline verification failed; generated cache is for review, not release."
             exit 1

.github/workflows/native-backend-ci.yml

@@ -46,10 +46,14 @@ jobs:
         run: python -c "import nns._nnscore as c; print(c.lpm(2.0, 0.0, [-2.0, -1.0, 0.5, 3.0]))"
 
       - name: Run invariants without Rscript
-        run: python -m pytest -q tests/invariants
+        id: invariants
+        continue-on-error: true
+        run: python -m pytest -q tests/invariants --junitxml=artifacts/invariants.xml
 
       - name: Run parity from committed R cache
-        run: NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity
+        id: parity
+        continue-on-error: true
+        run: NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity --junitxml=artifacts/parity.xml
 
       - name: Run invariants and cached parity with the pure-Python fallbacks
         # NNS_DISABLE_NATIVE=1 makes nns._native report no extension, so every
@@ -83,3 +87,23 @@ jobs:
 
       - name: Build distributions
         run: python -m build
+
+      # Diagnostics do not make a failing R14 parity check green.
+      - name: Archive R14 per-test diagnostics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: r14-parity-python-${{ matrix.python-version }}
+          path: artifacts/*.xml
+          if-no-files-found: warn
+
+      - name: Enforce R14 invariant and parity gates
+        if: always()
+        shell: bash
+        run: |
+          echo "Invariants: ${{ steps.invariants.outcome }}"
+          echo "R14 parity: ${{ steps.parity.outcome }}"
+          if [ "${{ steps.invariants.outcome }}" != "success" ] || [ "${{ steps.parity.outcome }}" != "success" ]; then
+            echo "::error::R14 source-faithful parity or invariants did not pass."
+            exit 1
+          fi

.github/workflows/r14-reproduction.yml

@@ -0,0 +1,72 @@
+name: R14 parity reproduction bundle
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [automation/r14-stack-cache-20261106]
+    paths: [.github/workflows/r14-reproduction.yml]
+
+permissions:
+  contents: read
+
+jobs:
+  bundle:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Archive tracked source and pinned R package
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p reproduction/wheels
+          git archive --format=tar.gz --output=reproduction/python-source.tar.gz HEAD
+          git rev-parse HEAD > reproduction/python-commit.txt
+          curl --fail --location --retry 3 \
+            https://raw.githubusercontent.com/OVVO-Financial/NNS/7b07a837c130ba7220dbe16c584221508757d5ae/NNS_14.0.tar.gz \
+            --output reproduction/NNS_14.0.tar.gz
+          sha256sum reproduction/*.tar.gz > reproduction/SHA256SUMS
+          python -m pip download --only-binary=:all: --dest reproduction/wheels \
+            hypothesis pytest-benchmark pytest-xdist nanobind scikit-build-core ruff mypy
+      - uses: actions/upload-artifact@v4
+        with:
+          name: r14-reproduction-source
+          path: reproduction/
+          retention-days: 7
+          if-no-files-found: error
+
+  oracle-runtime:
+    runs-on: ubuntu-latest
+    container: debian:trixie
+    steps:
+      - name: Build exact R source oracle
+        shell: bash
+        run: |
+          set -euo pipefail
+          apt-get update -qq
+          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+            ca-certificates curl r-base-core r-base-dev r-cran-rcpp \
+            r-cran-rcppparallel r-cran-jsonlite r-cran-mass r-cran-mlbench
+          curl --fail --location --retry 3 \
+            https://raw.githubusercontent.com/OVVO-Financial/NNS/7b07a837c130ba7220dbe16c584221508757d5ae/NNS_14.0.tar.gz \
+            --output /tmp/NNS_14.0.tar.gz
+          echo '5060f6ec426c889501740bf8a585d41f37a746554cb6fdf81f301ca41232cd5b  /tmp/NNS_14.0.tar.gz' | sha256sum -c -
+          MAKEFLAGS=-j2 R CMD INSTALL /tmp/NNS_14.0.tar.gz
+          mkdir -p oracle-runtime/dependent-libraries
+          Rscript -e 'library(NNS); stopifnot(packageVersion("NNS") == "14.0"); sessionInfo()' > oracle-runtime/session-info.txt
+          find /usr/lib/R /usr/local/lib/R -name '*.so' -print0 | \
+            xargs -0 -r ldd 2>/dev/null | awk '/=> \/.* \(/{print $3}' | sort -u > oracle-runtime/library-paths.txt
+          while IFS= read -r lib; do
+            cp -L "${lib}" oracle-runtime/dependent-libraries/
+          done < oracle-runtime/library-paths.txt
+          tar -h -C / -czf oracle-runtime/r14-runtime.tar.gz \
+            usr/bin/R usr/bin/Rscript usr/lib/R usr/share/R etc/R usr/local/lib/R
+          sha256sum oracle-runtime/r14-runtime.tar.gz > oracle-runtime/SHA256SUMS
+      - uses: actions/upload-artifact@v4
+        with:
+          name: r14-executable-oracle-debian13
+          path: oracle-runtime/
+          retention-days: 7
+          if-no-files-found: error

docs/R14_PARITY_INTEGRATION_GATE.md

@@ -0,0 +1,50 @@
+# NNS-Python: R 14.0 parity integration gate
+
+> **Do not merge, squash, force-push, or close the open function-port PRs while the cumulative R 14.0 gate is red.** This is a coordination ledger for the existing implementation, not a proposal to rewrite it.
+
+## Authoritative source and branch structure
+
+- R behavior: `OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae`, NNS 14.0.
+- The 41 still-open port PRs #148–#189 (excluding #177, which is not open) are **a single linear stack**: each change's base is the immediately preceding open head. All their code is present in `rasheed/py-co-moment-lengths` (#189).
+- `main → #189`: **223 commits, 36 Python source files** (comparison at initial R14 triage).
+- #191 (`integration/r14-parity-on-python-stack-20261106`) is **6 commits on top of #189, with no Python source changes**; it imports and audits R14 reference fixtures.
+- #192 (`automation/r14-stack-cache-20261106`) is based on #191; it contains rebuilt R14 fixtures **and source-level Python repairs**. It is the only active integration/fix head for this audit.
+- #190 (`automation/r-cache-nns-14.0`, based on `main`) is a separate initial cache-generation PR. Its cache-only update **cannot be landed into `main` separately** from the cumulative Python implementations.
+- Do not let the cache-generation workflow replace #192 from #191: that discards its Python repairs. Regenerate fixtures **in place** and preserve `sync/nns_source.json`.
+
+## Verified R14 baseline (before current code-fix cycle)
+
+The corrected live R14 run generated **2,745 source-authored entries in 78 shards**. Cache-only run on the same refreshed head: **88 failed, 2,021 passed, 12 skipped**; independent invariants: **14 failed, 694 passed, 1 skipped, 3 xfailed**. These are gate failures, not permission to weaken or drop tests.
+
+| Cluster | Remaining cache-only parity failures | Original feature PRs to preserve | Current R14 work |
+|---|---:|---|---|
+| `NNS.stack` | 21 | #154, #161, #168, #176 | #192 nominal-class routing and shared regression; inspect OOF candidate selection |
+| `NNS.boost` | 13 | #154, #161 | Depends on both `nns_reg_engine` and `nns_stack`; isolate independent sampling/frequency mismatches |
+| Label/factor dispatch | 11 | #155, #163 | #192 regression categorical exact-cell correction; resolve live-R fixture exceptions separately |
+| `NNS.ARMA` / optimizer | 11 | #150, #159, #166, #173, #174, #182 | #192 complete R14 optimizer return and split-only bias policy |
+| `NNS.VAR` | 11 | #159, #180 | Audit shared stack/ARMA forecast path, then VAR-specific lag and ensemble logic |
+| `NNS.reg` | 10 | #151, #154, #155, #157, #169 | #192 R14 exact joint-cell rules; investigate factor and multiclass fit separately |
+| `NNS.diff` / derivatives | 4 | #151, #161, #168, #186 | Remains independent |
+| Other (including R13 smoke) | 7 | Feature-specific | Do not relabel legacy-version failures as current R14 success |
+
+**These counts are by test file, not proven unique root causes.** Several downstream failures can disappear together; do not claim that without rerunning the same R14 fixtures.
+
+## Execution / landing policy
+
+1. Treat the **R 14.0 code and its generated fixture payload as the oracle**. No old cache carry-forward, invented expected output, threshold relaxation, suppressing checks or artificial xfail/skip.
+2. For each cluster, compare the **earliest differing intermediate**: predictor metadata → encoded design → regression points / RPM → fitted values → OOF predictions → candidate selection → final output. Fix the underlying function, not each failure assertion.
+3. Isolate regression and nominal classification first; rerun labels, regression, multivariate fast-path invariants, stack, boost. Then ARMA optimizer and VAR. Finally diff and native-vs-fallback invariants.
+4. Separate harness defects from algorithmic defects. A missing R fixture must be traced to the live R exception or codec; a heterogeneous output must be compared recursively rather than coerced blindly into `float64`.
+5. Keep the fix on #192 during R14 integration so all existing source work is preserved. Record the originating feature PR numbers above. **Do not transplant fixes into individual old heads and trigger 41 cascading merge conflicts mid-audit**.
+6. Once strict R14 native, fallback, cached parity and invariant gates pass, decide a controlled **ancestry-preserving** landing sequence for the existing stack (or a reviewed integration merge) based on repository merge policy. Never squash an upstream stack commit without rebasing downstream heads; never report draft or failing work as merged.
+7. Each CI pass must report failures by function and Python version, with explicit before/after comparisons. Keep legacy R13 compatibility cases visible as separately versioned tests.
+
+## Current source changes on #192
+
+- `tests/_r.py`: heterogeneous nested R lists are preserved rather than rejected by NumPy coercion.
+- `tests/invariants/test_r_cache_codec.py`: round-trip regression tests for codec and source provenance.
+- `src/nns/_reg_engine.py`: a fully categorical predictor design uses R14's maximum-order exact joint cells with default neighbor count one.
+- `src/nns/arma.py`: R14 `NNS.ARMA.optim` now exposes `raw.results`, `validation.forecast`, `bias.validation` and an explicit `bias_correction` policy (`none` or `split`).
+- Invariant coverage: numeric multivariate max-order fast path is tested separately from R14 nominal-class posterior output; optimizer tests assert the revised return contract.
+
+**Status:** work in progress, PR #192 remains draft. Counts above are a reference baseline prior to these newest source fixes, not a claim of all-green CI.

src/nns/_caus_r14.py

@@ -0,0 +1,95 @@
+"""R 14.0 directional association (NNS.caus / Uni.caus).
+
+Ground truth: NNS@7b07a837 R/Uni_Causation.R, R/Causal_matrix.R.
+Each pair is ranked to its own empirical lower-tail copula coordinate;
+the raw scores combine variance explained within equal-frequency cells and
+unfloored mean-of-partition-level dependence (not NNS.dep's maximum).
+"""
+from __future__ import annotations
+
+import math
+
+import numpy as np
+from numpy.typing import NDArray
+
+from nns._dep_r14 import dep_levels_r14
+
+Array = NDArray[np.float64]
+
+
+def _ranks(v: Array) -> Array:
+    """R LPM.ratio(0, v, v): inclusive empirical CDF of every observation."""
+    sorted_values = np.sort(v)
+    return np.searchsorted(sorted_values, v, side="right").astype(np.float64) / v.size
+
+
+def _variance(v: Array) -> float:
+    """R (LPM(2,mean,v) + UPM(2,mean,v)) * m/(m-1), length >=2."""
+    m = int(v.size)
+    if m < 2:
+        return float("nan")
+    # Population 2nd partial moments about the *same* sample mean.
+    mean = float(np.mean(v))
+    centered = v - mean
+    return float(np.mean(centered * centered) * (m / (m - 1)))
+
+
+def _explained(cause_u: Array, effect_u: Array) -> float:
+    """R .nns_caus_P: conditional share of effect variance resolved by cause."""
+    n = int(cause_u.size)
+    if n < 3:
+        return 0.0
+    groups = np.maximum(1, np.ceil(cause_u * max(2, math.floor(math.sqrt(n))))).astype(int)
+    total = _variance(effect_u)
+    if not math.isfinite(total) or total <= 0.0:
+        return 0.0
+    numerator, denominator = 0.0, 0
+    for group in np.unique(groups):
+        sample = effect_u[groups == group]
+        d = _variance(sample)
+        if math.isfinite(d):
+            numerator += sample.size * d
+            denominator += sample.size
+    if denominator == 0:
+        return 0.0
+    within = numerator / denominator
+    return min(1.0, max(0.0, 1.0 - within / total))
+
+
+def _rho(cause_u: Array, effect_u: Array) -> float:
+    """R .nns_caus_rho: average, not maximum, of unfloored NNS.dep levels."""
+    try:
+        value = float(np.mean(dep_levels_r14(cause_u, effect_u)))
+        return value if math.isfinite(value) else 0.0
+    except (ValueError, ArithmeticError, OverflowError):
+        # Mirrors R's tryCatch(error=function(e) 0).
+        return 0.0
+
+
+def raw_scores(x: Array, y: Array, tau: int = 0) -> tuple[float, float]:
+    """R .nns_caus_raw: paired, equally lagged directional raw scores."""
+    n = int(x.size)
+    k = n - tau
+    x_lag, y_lag = _ranks(x[:k]), _ranks(y[:k])
+    x_lead, y_lead = _ranks(x[tau:]), _ranks(y[tau:])
+    p_xy = _explained(x_lag, y_lead)
+    p_yx = _explained(y_lag, x_lead)
+    rho_xy = _rho(x_lag, y_lead)
+    rho_yx = _rho(y_lag, x_lead)
+    raw_xy = (p_xy + max(rho_xy - rho_yx, 0.0)) / 2.0
+    raw_yx = (p_yx + max(rho_yx - rho_xy, 0.0)) / 2.0
+    return raw_xy, raw_yx
+
+
+def index(x: Array, y: Array, tau: int = 0) -> tuple[float, float]:
+    """R .nns_caus_index: complementary directional indices, neutral 0.5."""
+    xy, yx = raw_scores(x, y, tau)
+    k = 1.0 / math.sqrt(float(x.size - tau))
+    forward = (xy + k) / (xy + yx + 2.0 * k)
+    return forward, 1.0 - forward
+
+
+def predictor_weight(response: Array, predictor: Array) -> float:
+    """R Uni.caus(response, predictor, tau): aligned raw predictor->response."""
+    n = min(response.size, predictor.size)
+    return raw_scores(predictor[:n], response[:n], 0)[0]