diff --git a/.github/workflows/inspect-r-api-update.yml b/.github/workflows/inspect-r-api-update.yml index f039c9ee..af975d77 100644 --- a/.github/workflows/inspect-r-api-update.yml +++ b/.github/workflows/inspect-r-api-update.yml @@ -1,6 +1,10 @@ name: Regenerate R parity cache on: + # One-shot integration audit: run against the complete Python PR stack. + push: + branches: [integration/r14-parity-on-python-stack-20261106, automation/r14-stack-cache-20261106] + paths: [.github/workflows/inspect-r-api-update.yml] repository_dispatch: types: [nns-r-package-updated] workflow_dispatch: @@ -55,6 +59,13 @@ jobs: r_version="${{ github.event.client_payload.r_version }}" source_tarball="${{ github.event.client_payload.source_tarball }}" windows_binary="${{ github.event.client_payload.windows_binary }}" + elif [ "${{ github.event_name }}" = "push" ]; then + # The pinned upstream release for the full-stack parity audit. + r_repo="OVVO-Financial/NNS" + r_commit="7b07a837c130ba7220dbe16c584221508757d5ae" + r_version="14.0" + source_tarball="NNS_14.0.tar.gz" + windows_binary="NNS_14.0.zip" else r_repo="OVVO-Financial/NNS" r_commit="${{ inputs.r_commit }}" @@ -147,7 +158,7 @@ jobs: shell: bash run: | set -euo pipefail - Rscript -e "options(repos=c(CRAN='https://cloud.r-project.org')); install.packages(c('remotes','jsonlite')); remotes::install_deps('upstream/package', dependencies=NA, upgrade='never')" + Rscript -e "options(repos=c(CRAN='https://cloud.r-project.org')); install.packages(c('remotes','jsonlite','mlbench')); remotes::install_deps('upstream/package', dependencies=NA, upgrade='never')" - name: Install exact NNS source package shell: bash @@ -162,8 +173,8 @@ jobs: run: | set -euo pipefail python -m pip install -U pip - python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy - python -m pip install hypothesis pytest-benchmark pytest-xdist + python -m pip install build scikit-build-core nanobind pytest ruff==0.16.3 mypy==2.3.1 "numpy<2.5" scipy + python -m pip install hypothesis pytest-benchmark pytest-xdist pandas matplotlib python -m pip install -e . --force-reinstall --no-deps - name: Regenerate complete live-R parity cache @@ -322,17 +333,39 @@ jobs: committed-cache-verification.log if-no-files-found: warn + # Rebuild directly on the existing #192 branch. Do not let the + # create-pull-request action reset this branch from #191 and discard + # the independent R14 Python implementation fixes already on #192. + - name: Commit regenerated R14 cache to existing audit branch + if: ${{ always() && github.event_name == 'push' && github.ref_name == 'automation/r14-stack-cache-20261106' }} + shell: bash + run: | + set -euo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add -A tests/_r.py tests/_r_cache sync/nns_source.json + if ! git diff --cached --quiet; then + git commit -m 'Rebuild R 14.0 parity fixtures with heterogeneous-list decoder' + # Fast-forward only; concurrent branch edits must never be overwritten. + git push origin "HEAD:refs/heads/${GITHUB_REF_NAME}" + else + echo 'R cache is unchanged after fresh regeneration.' + fi - name: Open or update R cache regeneration PR - if: always() + if: ${{ always() && github.ref_name != 'automation/r14-stack-cache-20261106' }} uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} - branch: automation/r-cache-nns-${{ steps.package.outputs.r_version }} + # A separate PR against the full-stack audit branch; never overwrite #190. + branch: ${{ github.event_name == 'push' && 'automation/r14-stack-cache-20261106' || format('automation/r-cache-nns-{0}', steps.package.outputs.r_version) }} + base: ${{ github.event_name == 'push' && 'integration/r14-parity-on-python-stack-20261106' || github.event.repository.default_branch }} delete-branch: true commit-message: Regenerate R parity cache for NNS ${{ steps.package.outputs.r_version }} title: Regenerate R parity cache for NNS ${{ steps.package.outputs.r_version }} body: | Automatic parity-cache refresh from the R source of truth. + For the one-shot stack audit, this PR targets draft #191 (all port PRs + through #189). Do NOT merge until deterministic parity is verified. - R repository: `${{ steps.package.outputs.r_repo }}` - R commit: `${{ steps.package.outputs.r_commit }}` @@ -356,4 +389,8 @@ jobs: run: | echo "Live regeneration outcome: ${{ steps.regenerate.outcome }}" echo "Cache-only verification outcome: ${{ steps.verify.outcome }}" - echo "The regenerated R cache has been preserved in its pull request." + echo "The regenerated R cache has been preserved on the audit branch or in its PR." + if [ "${{ steps.regenerate.outcome }}" != "success" ] || [ "${{ steps.verify.outcome }}" != "success" ]; then + echo "::error::The oracle regeneration or its offline verification failed; generated cache is for review, not release." + exit 1 + fi diff --git a/.github/workflows/native-backend-ci.yml b/.github/workflows/native-backend-ci.yml index 6d7e3a0b..513cee9f 100644 --- a/.github/workflows/native-backend-ci.yml +++ b/.github/workflows/native-backend-ci.yml @@ -36,8 +36,8 @@ jobs: - name: Install build tools run: | python -m pip install -U pip - python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy - python -m pip install hypothesis pytest-benchmark pytest-xdist + python -m pip install build scikit-build-core nanobind pytest ruff==0.16.3 mypy==2.3.1 "numpy<2.5" scipy + python -m pip install hypothesis pytest-benchmark pytest-xdist pandas - name: Install package editable run: python -m pip install -e . @@ -46,10 +46,24 @@ jobs: run: python -c "import nns._nnscore as c; print(c.lpm(2.0, 0.0, [-2.0, -1.0, 0.5, 3.0]))" - name: Run invariants without Rscript - run: python -m pytest -q tests/invariants + id: invariants + continue-on-error: true + run: python -m pytest -q tests/invariants --junitxml=artifacts/invariants.xml - name: Run parity from committed R cache - run: NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity + id: parity + continue-on-error: true + run: NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity --junitxml=artifacts/parity.xml + + - name: Run invariants and cached parity with the pure-Python fallbacks + # NNS_DISABLE_NATIVE=1 makes nns._native report no extension, so every + # public function takes its NumPy fallback, as an install without the + # compiled extension would. Tests that call nns._nnscore + # directly still run against the extension. + env: + NNS_DISABLE_NATIVE: "1" + NNS_R_CACHE_ONLY: "1" + run: python -m pytest -q tests/invariants tests/parity - name: Run property tests run: python -m pytest -q tests/property @@ -73,3 +87,23 @@ jobs: - name: Build distributions run: python -m build + + # Diagnostics do not make a failing R14 parity check green. + - name: Archive R14 per-test diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: r14-parity-python-${{ matrix.python-version }} + path: artifacts/*.xml + if-no-files-found: warn + + - name: Enforce R14 invariant and parity gates + if: always() + shell: bash + run: | + echo "Invariants: ${{ steps.invariants.outcome }}" + echo "R14 parity: ${{ steps.parity.outcome }}" + if [ "${{ steps.invariants.outcome }}" != "success" ] || [ "${{ steps.parity.outcome }}" != "success" ]; then + echo "::error::R14 source-faithful parity or invariants did not pass." + exit 1 + fi diff --git a/.github/workflows/r14-reproduction.yml b/.github/workflows/r14-reproduction.yml new file mode 100644 index 00000000..75bc6e30 --- /dev/null +++ b/.github/workflows/r14-reproduction.yml @@ -0,0 +1,72 @@ +name: R14 parity reproduction bundle + +on: + workflow_dispatch: + push: + branches: [automation/r14-stack-cache-20261106] + paths: [.github/workflows/r14-reproduction.yml] + +permissions: + contents: read + +jobs: + bundle: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Archive tracked source and pinned R package + shell: bash + run: | + set -euo pipefail + mkdir -p reproduction/wheels + git archive --format=tar.gz --output=reproduction/python-source.tar.gz HEAD + git rev-parse HEAD > reproduction/python-commit.txt + curl --fail --location --retry 3 \ + https://raw.githubusercontent.com/OVVO-Financial/NNS/7b07a837c130ba7220dbe16c584221508757d5ae/NNS_14.0.tar.gz \ + --output reproduction/NNS_14.0.tar.gz + sha256sum reproduction/*.tar.gz > reproduction/SHA256SUMS + python -m pip download --only-binary=:all: --dest reproduction/wheels \ + hypothesis pytest-benchmark pytest-xdist nanobind scikit-build-core ruff mypy + - uses: actions/upload-artifact@v4 + with: + name: r14-reproduction-source + path: reproduction/ + retention-days: 7 + if-no-files-found: error + + oracle-runtime: + runs-on: ubuntu-latest + container: debian:trixie + steps: + - name: Build exact R source oracle + shell: bash + run: | + set -euo pipefail + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl r-base-core r-base-dev r-cran-rcpp \ + r-cran-rcppparallel r-cran-jsonlite r-cran-mass r-cran-mlbench + curl --fail --location --retry 3 \ + https://raw.githubusercontent.com/OVVO-Financial/NNS/7b07a837c130ba7220dbe16c584221508757d5ae/NNS_14.0.tar.gz \ + --output /tmp/NNS_14.0.tar.gz + echo '5060f6ec426c889501740bf8a585d41f37a746554cb6fdf81f301ca41232cd5b /tmp/NNS_14.0.tar.gz' | sha256sum -c - + MAKEFLAGS=-j2 R CMD INSTALL /tmp/NNS_14.0.tar.gz + mkdir -p oracle-runtime/dependent-libraries + Rscript -e 'library(NNS); stopifnot(packageVersion("NNS") == "14.0"); sessionInfo()' > oracle-runtime/session-info.txt + find /usr/lib/R /usr/local/lib/R -name '*.so' -print0 | \ + xargs -0 -r ldd 2>/dev/null | awk '/=> \/.* \(/{print $3}' | sort -u > oracle-runtime/library-paths.txt + while IFS= read -r lib; do + cp -L "${lib}" oracle-runtime/dependent-libraries/ + done < oracle-runtime/library-paths.txt + tar -h -C / -czf oracle-runtime/r14-runtime.tar.gz \ + usr/bin/R usr/bin/Rscript usr/lib/R usr/share/R etc/R usr/local/lib/R + sha256sum oracle-runtime/r14-runtime.tar.gz > oracle-runtime/SHA256SUMS + - uses: actions/upload-artifact@v4 + with: + name: r14-executable-oracle-debian13 + path: oracle-runtime/ + retention-days: 7 + if-no-files-found: error diff --git a/.github/workflows/sync-from-nns-core.yml b/.github/workflows/sync-from-nns-core.yml new file mode 100644 index 00000000..97d1cd19 --- /dev/null +++ b/.github/workflows/sync-from-nns-core.yml @@ -0,0 +1,193 @@ +name: Sync from NNS-core + +# Receiver side of the nns-core-updated contract (NNS-core +# .github/workflows/notify-python.yml and docs/r-source-sync.md). When an +# accepted NNS-core commit lands, NNS-core dispatches nns-core-updated with +# client_payload.{core_commit, r_repo, r_commit, r_version, r_src_tree_hash}. +# This workflow re-vendors extern/NNS-core from that commit, rebuilds the +# extension, runs the native and fallback test gates and opens a pull request. +# The PR is a draft unless every gate passed. +# +# This file must stay in workflow-baseline.yml's protected list, otherwise the +# baseline guard deletes it from pull requests and NNS-core's notify job stops +# dispatching (it checks for this receiver first). + +on: + repository_dispatch: + types: [nns-core-updated] + workflow_dispatch: + inputs: + core_commit: + description: Full 40-character OVVO-Financial/NNS-core commit SHA to vendor + required: true + type: string + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-from-nns-core + cancel-in-progress: false + +jobs: + sync-core: + runs-on: ubuntu-latest + + steps: + - name: Check out NNS-python + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve and validate payload + id: payload + shell: bash + # Event fields reach the shell only through env: and are validated + # before use (no ${{ }} expansion inside run:). + env: + EVENT_NAME: ${{ github.event_name }} + DISPATCH_CORE_COMMIT: ${{ github.event.client_payload.core_commit }} + DISPATCH_R_REPO: ${{ github.event.client_payload.r_repo }} + DISPATCH_R_COMMIT: ${{ github.event.client_payload.r_commit }} + DISPATCH_R_VERSION: ${{ github.event.client_payload.r_version }} + DISPATCH_R_HASH: ${{ github.event.client_payload.r_src_tree_hash }} + INPUT_CORE_COMMIT: ${{ inputs.core_commit }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "repository_dispatch" ]; then + core_commit="$DISPATCH_CORE_COMMIT" + r_repo="${DISPATCH_R_REPO:-}" + r_commit="${DISPATCH_R_COMMIT:-}" + r_version="${DISPATCH_R_VERSION:-}" + r_hash="${DISPATCH_R_HASH:-}" + else + core_commit="$INPUT_CORE_COMMIT" + r_repo=""; r_commit=""; r_version=""; r_hash="" + fi + [[ "$core_commit" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::core_commit must be a full lowercase 40-character SHA"; exit 1; } + [[ -z "$r_repo" || "$r_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::bad r_repo"; exit 1; } + [[ -z "$r_commit" || "$r_commit" =~ ^[0-9a-fA-F]{7,40}$ ]] || { echo "::error::bad r_commit"; exit 1; } + [[ -z "$r_version" || "$r_version" =~ ^[0-9A-Za-z._-]{1,32}$ ]] || { echo "::error::bad r_version"; exit 1; } + [[ -z "$r_hash" || "$r_hash" =~ ^[0-9a-fA-F]{0,64}$ ]] || { echo "::error::bad r_src_tree_hash"; exit 1; } + { + echo "core_commit=$core_commit" + echo "core_short=${core_commit:0:7}" + echo "r_repo=${r_repo:-unknown}" + echo "r_commit=${r_commit:-unknown}" + echo "r_version=${r_version:-unknown}" + echo "r_src_tree_hash=${r_hash:-unknown}" + } >> "$GITHUB_OUTPUT" + + - name: Fetch the NNS-core commit + shell: bash + env: + CORE_COMMIT: ${{ steps.payload.outputs.core_commit }} + run: | + set -euo pipefail + mkdir -p upstream/NNS-core + cd upstream/NNS-core + git init -q + git remote add origin https://github.com/OVVO-Financial/NNS-core.git + git fetch --depth 1 origin "$CORE_COMMIT" + git checkout -q FETCH_HEAD + test "$(git rev-parse HEAD)" = "$CORE_COMMIT" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Re-vendor extern/NNS-core + shell: bash + env: + CORE_COMMIT: ${{ steps.payload.outputs.core_commit }} + run: | + set -euo pipefail + # Only core_commit is recorded: the manifest's r_commit/r_version + # describe the R package the parity cache encodes, which an NNS-core + # update does not change. The R provenance of the core is listed in + # the PR body. + python scripts/sync_nns_core_snapshot.py \ + --core-checkout upstream/NNS-core \ + --core-commit "$CORE_COMMIT" + rm -rf upstream + git status --short + + - name: Install the package (builds the native extension) + id: build + continue-on-error: true + shell: bash + run: | + set -euo pipefail + { + python -m pip install -U pip && + python -m pip install build scikit-build-core nanobind pytest ruff==0.16.3 mypy==2.3.1 "numpy<2.5" scipy && + python -m pip install hypothesis pytest-benchmark pytest-xdist pandas && + python -m pip install -e . && + python -c "import nns._nnscore as c; print(c.lpm(2.0, 0.0, [-2.0, -1.0, 0.5, 3.0]))" + } 2>&1 | tee nns-core-sync-build.log + + - name: Run native and fallback gates + id: gates + if: steps.build.outcome == 'success' + continue-on-error: true + shell: bash + run: | + set -o pipefail + { + python -m pytest -q tests/invariants && + NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity && + NNS_DISABLE_NATIVE=1 NNS_R_CACHE_ONLY=1 python -m pytest -q tests/invariants tests/parity && + python -m pytest -q tests/tools + } 2>&1 | tee nns-core-sync-gates.log + + - name: Upload gate log + if: always() + uses: actions/upload-artifact@v4 + with: + name: nns-core-sync-${{ steps.payload.outputs.core_short }} + path: | + nns-core-sync-build.log + nns-core-sync-gates.log + if-no-files-found: warn + + - name: Open or update the NNS-core sync PR + id: cpr + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} + branch: automation/nns-core-${{ steps.payload.outputs.core_short }} + delete-branch: true + draft: ${{ steps.gates.outcome != 'success' }} + commit-message: "native: re-vendor NNS-core ${{ steps.payload.outputs.core_short }}" + title: "native: re-vendor NNS-core ${{ steps.payload.outputs.core_short }}" + body: | + Automatic re-vendor of `extern/NNS-core` from an accepted NNS-core commit. + + - NNS-core commit: `${{ steps.payload.outputs.core_commit }}` + - R source the core was ported from: `${{ steps.payload.outputs.r_repo }}@${{ steps.payload.outputs.r_commit }}` (NNS ${{ steps.payload.outputs.r_version }}, src tree `${{ steps.payload.outputs.r_src_tree_hash }}`) + - Build and native import: `${{ steps.build.outcome }}` + - Gates (invariants, cached parity, the same with `NNS_DISABLE_NATIVE=1`, tools): `${{ steps.gates.outcome }}` + + Only `extern/NNS-core` and `sync/nns_source.json` (`core_commit`) change here. If the build or a gate failed this PR is a draft: check the build and gate logs. Any parity shard whose R reference itself changed must be regenerated from R. + add-paths: | + extern/NNS-core/** + sync/nns_source.json + + - name: Match the existing PR's draft state to the gates + if: steps.cpr.outputs.pull-request-number != '' && steps.cpr.outputs.pull-request-operation != 'created' + shell: bash + env: + GH_TOKEN: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} + PR: ${{ steps.cpr.outputs.pull-request-number }} + READY: ${{ steps.gates.outcome == 'success' }} + run: | + set -euo pipefail + # create-pull-request leaves an existing PR's draft state as it was, even on an unchanged rerun. + draft=$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft) + if [ "$READY" = "true" ] && [ "$draft" = "true" ]; then + gh pr ready "$PR" --repo "$GITHUB_REPOSITORY" + elif [ "$READY" != "true" ] && [ "$draft" = "false" ]; then + gh pr ready "$PR" --repo "$GITHUB_REPOSITORY" --undo + fi diff --git a/.github/workflows/workflow-baseline.yml b/.github/workflows/workflow-baseline.yml index c9ba99f5..ffdc8d06 100644 --- a/.github/workflows/workflow-baseline.yml +++ b/.github/workflows/workflow-baseline.yml @@ -52,12 +52,20 @@ jobs: ".github/workflows/inspect-r-api-update.yml" ".github/workflows/docs.yml" ".github/workflows/release.yml" + ".github/workflows/sync-from-nns-core.yml" ) mkdir -p .github/workflows for path in "${protected_workflows[@]}"; do - git show "origin/${BASE_REF}:${path}" > "${path}" + # A protected workflow that the default branch does not have yet + # (for example a newly added receiver) is removed like any other + # unapproved workflow change. + if git cat-file -e "origin/${BASE_REF}:${path}" 2>/dev/null; then + git show "origin/${BASE_REF}:${path}" > "${path}" + else + rm -f "${path}" + fi done while IFS= read -r -d '' path; do @@ -66,7 +74,8 @@ jobs: ".github/workflows/native-backend-ci.yml"|\ ".github/workflows/inspect-r-api-update.yml"|\ ".github/workflows/docs.yml"|\ - ".github/workflows/release.yml") + ".github/workflows/release.yml"|\ + ".github/workflows/sync-from-nns-core.yml") ;; *) echo "Removing non-baseline workflow: ${path}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b2b4fcf..6a3c2c18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,18 @@ set(NNSCORE_BUILD_TESTS OFF CACHE BOOL "Build NNS-core tests" FORCE) set(NNSCORE_BUILD_PYTHON OFF CACHE BOOL "Build NNS-core Python bindings" FORCE) add_subdirectory(extern/NNS-core) +# Preserve the accepted NNS-core vendor tree; explicitly overlay the two +# R14 central-tendency kernels that its R13.3 snapshot does not yet supply. +get_target_property(NNS_CORE_SOURCES nnscore SOURCES) +set(NNS_R14_CORE_SOURCES "") +foreach(source IN LISTS NNS_CORE_SOURCES) + if(NOT source STREQUAL "src/central_tendencies.cpp") + list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/extern/NNS-core/${source}") + endif() +endforeach() +list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_central_tendencies.cpp") +set_property(TARGET nnscore PROPERTY SOURCES "${NNS_R14_CORE_SOURCES}") + # The Python extension is a shared object. The vendored NNS-core target builds # as a static library, so it must be compiled as position-independent code before # it can be linked into the nanobind module on ELF platforms. diff --git a/docs/R14_LOCAL_PARITY_REPAIR.md b/docs/R14_LOCAL_PARITY_REPAIR.md new file mode 100644 index 00000000..c6e26e48 --- /dev/null +++ b/docs/R14_LOCAL_PARITY_REPAIR.md @@ -0,0 +1,204 @@ +# R 14.0 source-faithful Python alignment: local verification + +Date: 2026-11-07. + +**Delivery status: locally implemented and tested; not pushed, merged, or released.** +The current GitHub connection exposes read-only actions. These changes are supplied +as an apply-ready patch for the existing PR #192 branch, not as a replacement for +its inherited function work. GitHub CI must run on the resulting commit before +any merge or release decision. + +## Exact source and ancestry + +- Python repository: `OVVO-Financial/NNS-python`. +- Target branch: `automation/r14-stack-cache-20261106` (draft PR #192). +- Expected remote HEAD: `71ae8e820a099c46f04c14a638610af49f810d62`. +- Imported source archive: `9ee110aa03dfde0e1d4ecd21429292305d6c3518`. + The GitHub comparison confirms that only `.github/workflows/r14-reproduction.yml` + differs between that archive and the target HEAD. This patch does not touch it. +- R behavior: `OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae`. +- Installed oracle package: NNS 14.0, compiled from `NNS_14.0.tar.gz`. +- Oracle archive SHA256: + `5060f6ec426c889501740bf8a585d41f37a746554cb6fdf81f301ca41232cd5b`. +- The cumulative implementation under #189 -> #191 -> #192 is preserved. + No original feature PR was merged, closed, or replaced by this repair. +- `extern/NNS-core` is unchanged. R14 native additions have separate, hashed + source attribution in `sync/nns_source.json`; the historical vendor hash is + not represented as the hash of the new algorithms. + +## Measured results + +The downloaded Python 3.13 CI baseline at `d2ea97b...` contained 77 failing +cached-parity tests and 9 failing invariant tests. Some failures were genuine +implementation differences; others asserted obsolete R13 contracts. The latter +were replaced by checks of the actual pinned R14 contract, not by skips, +expected-failure markers, or wider parity tolerances. + +| Verification gate | Passed | Failed / errors | Skipped | +| --- | ---: | ---: | ---: | +| Native: complete parity, invariants, properties, plotting, vignettes | 2,961 | 0 | 13 | +| Native invariant subset of that run | 721 | 0 | 1 | +| Native cached-parity subset of that run | 2,118 | 0 | 12 | +| Native property subset | 59 | 0 | 0 | +| Native plotting subset | 54 | 0 | 0 | +| Native vignette subset | 9 | 0 | 0 | +| Public pure-Python fallback: parity and invariants | 2,815 | 0 | 37 | +| Live installed-R practical examples | 9 | 0 | 2 | + +Rows overlap; their counts must not be added as unique tests. The fallback run +uses `NNS_DISABLE_NATIVE=1`, matching the repository's fallback gate. The compiled +extension remains installed, so tests explicitly calling it may still run. + +Additional checks: + +- `ruff check .`: passed with Ruff 0.17.4. +- `mypy`: passed, 54 source files, with mypy 2.4.1. +- `git diff --check`: passed. +- Editable native build: passed. +- Source distribution build: passed. +- Native wheel built from that source distribution, rather than the working tree: + passed. +- Isolated wheel smoke test, bypassing the editable import finder: passed for + partial moments, versioned R14 dependence, categorical exact-cell regression, + and pipeline VAR. The extension reports the pinned R source commit. +- Native provenance and dispatch checks: passed. + +### Skips and limits + +The native suite's 13 skips are the 11 intentionally live-R-only practical +examples, an existing empty example parameter set, and one already-skipped +obsolete seed-invariance assertion. The live-R run independently exercises nine +of those eleven examples; two require the optional R `caret` package, absent in +this environment. The fallback run additionally skips 24 native-only checks. +No skip or xfail was added to hide a mismatch. Three obsolete native-mode xfails +and two obsolete live-VAR xfails were removed only after they passed. + +This is Linux x86_64 / Python 3.13.5 verification with NumPy 2.3.5, SciPy 1.17.0, +pytest 9.0.2, Hypothesis 6.170.15, and R 4.5.0. The existing Hypothesis CI profile +was used without reducing its example counts. CI's exact older Ruff/mypy pins, +Python 3.11/3.12/3.14, and Windows/macOS were not run here. The downloaded R runtime +emits TBB loading warnings; reference calls use the serial `ncores=1` path. +There is no performance or all-input equivalence claim beyond the tested cases. + +## Actual implementation defects repaired + +### Shared regression, labels, and classification + +`R/Multivariate_Regression.R` retains both atoms of binary coordinates even at +finite order. Python previously merged a terminal cell. The shared partition +engine now uses the R14 support IDs. Default neighbor selection also averages +the copula summaries for `y` and `-y`, as R14 does, instead of allowing a response +sign convention to change the smoothing count. + +Unordered predictor levels are canonicalized while explicitly ordered levels +retain their order. Class response maps remain distinct from predictor metadata. +The univariate posterior interpolation uses R's fraction-first arithmetic; an +otherwise tiny rounding difference was changing tied class decisions. + +### Stack and Boost + +The regression component of Stack now receives the selected original predictor +coordinates. It previously received two copies of the reduced synthetic index, +so its cross-validation candidate was not the model R14 deployed. The old +response-offset workaround is gone; numeric internal fits explicitly request +REG rather than accidentally triggering nominal classification. + +Fold balancing follows R's first-observed class order and permits the singleton +case R accepts. Boost no longer decodes an already-decoded final Stack result; +that second conversion collapsed valid 0/1 predictions to zeros. Both the +ordinary numeric and factor/nominal test families are covered by the final gate. + +### ARMA, seasonality, and VAR + +Python's nonlinear ARMA shrinkage was missing. Its component-level shrink +operation, and the single-lag linear shortcut's shrink guard, now follow +`R/ARMA.R`. R14 optimizer scoring uses the matching mean/zero-denominator rule. +Seasonality modulo augmentation now applies the R14 baseline-CV rule for periods +already examined and rejected. + +VAR now defaults to R14 pipeline validation. Preprocessing, univariate forecasts, +and the lagged forecast predictors are reconstructed at the earlier validation +origin; held-out future observations do not enter those predictors. The explicit +conditional-validation path remains available. A regression guard perturbs only +held-out observations and verifies that training/predictor arrays do not move. +Trailing missing-value extrapolation uses R14's index-cell gradients. + +### Native kernels + +The recorded vendor tree contains older central-tendency and dependence kernels. +Explicit R14 overlays supply gravity/mode and the multilevel, null-corrected +dependence estimator. They are attributed to the exact R source files. Their +non-R numerical adaptations (vector marshalling, erfc, and hypergeometric +initialization) are documented, rather than described as byte-identical source. + +Versioned native entry points (`gravity_r14`, `mode_r14`, `gravity_exact_r14`, +`nns_dep_pair_r14`) prevent new Python code from silently binding an older native +extension. Public functions use the native R14 path when available and the +independent Python fallback otherwise. Both paths pass their comparison gates. + +## Cache and test-transport repairs + +The committed oracle cache now contains **2,764 entries across 80 shards**, up +from 2,745 entries across 78 shards. Added or changed reference values came from +the installed pinned R14 oracle, never from Python output. + +- Four missing label references were recovered by suppressing plotting in the + headless reference calls, not by substituting fabricated outputs. +- New shrinkage and multiclass-Boost interval references were generated by R14. +- Variable-length nominal prediction sets are compared recursively; coercing + them to a rectangular numeric array previously failed before values were + compared. +- Live curve-fitting inputs are exported with `digits=17`. The former 15-digit + JSON export moved observations across exact partition boundaries. +- Small integer-power R callbacks and derivative anchors were checked against + the actual oracle rather than excused by looser tolerances. + +## Test changes that reflect an R version change + +These are explicit contract updates, not algorithmic corrections disguised as +changed expected values: + +- R14 nominal regression returns posterior probabilities and prediction sets, + not ordinal confidence intervals. Properties now check normalization, valid + class membership, ordered sets, and the requested estimated probability mass. +- Stack/Boost retain binary interval compatibility; multiclass intervals are + NULL. A newly generated R14 Boost fixture verifies that distinction. +- Positive-degree multivariate cumulative hazard is rejected by R14 because the + generalized partial-moment ratio is not a joint distribution. Its property + test now asserts that exact rejection. +- The classification vignette displays R14's class-specific signed-index + equations instead of treating species labels as one ordinal response. +- Version-specific smoke cases keep their inputs but identify R14 behavior + explicitly. Historical numerical constants are not silently rebranded as + proof that both R13 and R14 behave identically. + +## Reproduction commands + +Install the package and ordinary test/build dependencies, including the native +extension. Ordinary cached tests require no R process. The verification environment +hid `Rscript` from PATH for cached gates, as ordinary GitHub CI does. + +```sh +python -m pip install -e . --no-build-isolation --no-deps +NNS_R_CACHE_ONLY=1 python -m pytest -q tests/invariants tests/parity \ + tests/property tests/plotting tests/docs/test_vignette_examples.py +NNS_R_CACHE_ONLY=1 NNS_DISABLE_NATIVE=1 python -m pytest -q \ + tests/invariants tests/parity +ruff check . +mypy +python -m build --sdist +python -m pip wheel --no-deps --no-build-isolation dist/ovvo_nns-2.1.0.tar.gz +``` + +For live-R practical examples, install the exact archive above, verify +`packageVersion("NNS") == "14.0"`, and clear all CI/offline-cache toggles before +running `tests/parity/test_practical_examples.py`. `caret` is needed for all eleven. +Do not regenerate the cache from an unpinned installed NNS version. + +## Landing discipline + +Apply the supplied patch only to the clean, named PR #192 head. Its helper checks +the branch, remote repository, exact HEAD, patch digest, and `git apply --check`. +It does not commit, push, merge, force-push, or close PRs. Keep the cumulative +ancestry and run the complete GitHub matrix after pushing the repair commit. +The green local gates do not change the remote branch's status until that happens. diff --git a/docs/R14_PARITY_INTEGRATION_GATE.md b/docs/R14_PARITY_INTEGRATION_GATE.md new file mode 100644 index 00000000..f06c18d2 --- /dev/null +++ b/docs/R14_PARITY_INTEGRATION_GATE.md @@ -0,0 +1,64 @@ +# NNS-Python: R 14.0 parity integration gate + +> **Do not merge, squash, force-push, or close the open function-port PRs while the cumulative R 14.0 gate is red.** This is a coordination ledger for the existing implementation, not a proposal to rewrite it. + +## Local repair verification (2026-11-07; not yet published) + +The repair patch described in [R14 local parity repair](R14_LOCAL_PARITY_REPAIR.md) +passes 2,961 native tests (13 existing skips), 2,815 public-fallback tests +(37 skips), and 9 live-R practical examples (2 missing optional-caret skips). +Ruff, mypy, the sdist-to-wheel build, and an isolated wheel smoke test also pass. +The cache now has 2,764 R-authored entries in 80 shards. `extern/NNS-core` is +unchanged; separately attributed native R14 overlays replace stale dispatch. + +These are **local** results for a patch targeting remote head `71ae8e8...`. +They are not a claim that the patch has been pushed or that GitHub's matrix is +green. The earlier baseline table below is preserved as historical evidence. +Do not merge until the repaired head passes remote CI. + +## Authoritative source and branch structure + +- R behavior: `OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae`, NNS 14.0. +- The 41 still-open port PRs #148–#189 (excluding #177, which is not open) are **a single linear stack**: each change's base is the immediately preceding open head. All their code is present in `rasheed/py-co-moment-lengths` (#189). +- `main → #189`: **223 commits, 36 Python source files** (comparison at initial R14 triage). +- #191 (`integration/r14-parity-on-python-stack-20261106`) is **6 commits on top of #189, with no Python source changes**; it imports and audits R14 reference fixtures. +- #192 (`automation/r14-stack-cache-20261106`) is based on #191; it contains rebuilt R14 fixtures **and source-level Python repairs**. It is the only active integration/fix head for this audit. +- #190 (`automation/r-cache-nns-14.0`, based on `main`) is a separate initial cache-generation PR. Its cache-only update **cannot be landed into `main` separately** from the cumulative Python implementations. +- Do not let the cache-generation workflow replace #192 from #191: that discards its Python repairs. Regenerate fixtures **in place** and preserve `sync/nns_source.json`. + +## Verified R14 baseline (before current code-fix cycle) + +The corrected live R14 run generated **2,745 source-authored entries in 78 shards**. Cache-only run on the same refreshed head: **88 failed, 2,021 passed, 12 skipped**; independent invariants: **14 failed, 694 passed, 1 skipped, 3 xfailed**. These are gate failures, not permission to weaken or drop tests. + +| Cluster | Remaining cache-only parity failures | Original feature PRs to preserve | Current R14 work | +|---|---:|---|---| +| `NNS.stack` | 21 | #154, #161, #168, #176 | #192 nominal-class routing and shared regression; inspect OOF candidate selection | +| `NNS.boost` | 13 | #154, #161 | Depends on both `nns_reg_engine` and `nns_stack`; isolate independent sampling/frequency mismatches | +| Label/factor dispatch | 11 | #155, #163 | #192 regression categorical exact-cell correction; resolve live-R fixture exceptions separately | +| `NNS.ARMA` / optimizer | 11 | #150, #159, #166, #173, #174, #182 | #192 complete R14 optimizer return and split-only bias policy | +| `NNS.VAR` | 11 | #159, #180 | Audit shared stack/ARMA forecast path, then VAR-specific lag and ensemble logic | +| `NNS.reg` | 10 | #151, #154, #155, #157, #169 | #192 R14 exact joint-cell rules; investigate factor and multiclass fit separately | +| `NNS.diff` / derivatives | 4 | #151, #161, #168, #186 | Remains independent | +| Other (including R13 smoke) | 7 | Feature-specific | Do not relabel legacy-version failures as current R14 success | + +**These counts are by test file, not proven unique root causes.** Several downstream failures can disappear together; do not claim that without rerunning the same R14 fixtures. + +## Execution / landing policy + +1. Treat the **R 14.0 code and its generated fixture payload as the oracle**. No old cache carry-forward, invented expected output, threshold relaxation, suppressing checks or artificial xfail/skip. +2. For each cluster, compare the **earliest differing intermediate**: predictor metadata → encoded design → regression points / RPM → fitted values → OOF predictions → candidate selection → final output. Fix the underlying function, not each failure assertion. +3. Isolate regression and nominal classification first; rerun labels, regression, multivariate fast-path invariants, stack, boost. Then ARMA optimizer and VAR. Finally diff and native-vs-fallback invariants. +4. Separate harness defects from algorithmic defects. A missing R fixture must be traced to the live R exception or codec; a heterogeneous output must be compared recursively rather than coerced blindly into `float64`. +5. Keep the fix on #192 during R14 integration so all existing source work is preserved. Record the originating feature PR numbers above. **Do not transplant fixes into individual old heads and trigger 41 cascading merge conflicts mid-audit**. +6. Once strict R14 native, fallback, cached parity and invariant gates pass, decide a controlled **ancestry-preserving** landing sequence for the existing stack (or a reviewed integration merge) based on repository merge policy. Never squash an upstream stack commit without rebasing downstream heads; never report draft or failing work as merged. +7. Each CI pass must report failures by function and Python version, with explicit before/after comparisons. Keep legacy R13 compatibility cases visible as separately versioned tests. + +## Current source changes on #192 + +- `tests/_r.py`: heterogeneous nested R lists are preserved rather than rejected by NumPy coercion. +- `tests/invariants/test_r_cache_codec.py`: round-trip regression tests for codec and source provenance. +- `src/nns/_reg_engine.py`: a fully categorical predictor design uses R14's maximum-order exact joint cells with default neighbor count one. +- `src/nns/arma.py`: R14 `NNS.ARMA.optim` now exposes `raw.results`, `validation.forecast`, `bias.validation` and an explicit `bias_correction` policy (`none` or `split`). +- Invariant coverage: numeric multivariate max-order fast path is tested separately from R14 nominal-class posterior output; optimizer tests assert the revised return contract. + +**Status:** work in progress, PR #192 remains draft. Counts above are a reference baseline prior to these newest source fixes, not a claim of all-green CI. diff --git a/docs/api_reference.md b/docs/api_reference.md index 86e05971..9dba554c 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -38,12 +38,12 @@ The generator introspects `nns.__all__`, imports each public object, records sig | Partial moments | `pm_matrix` | `PM.matrix` | implemented | Partial moment matrix helper. | | Co-moments | `co_lpm` | `Co.LPM` | implemented | Pairwise co-lower partial moment. | | Co-moments | `co_upm` | `Co.UPM` | implemented | Pairwise co-upper partial moment. | -| Co-moments | `d_lpm` | `D.LPM` | implemented | Pairwise distance lower partial moment. | -| Co-moments | `d_upm` | `D.UPM` | implemented | Pairwise distance upper partial moment. | +| Co-moments | `d_lpm` | `D.LPM` | implemented | Divergent lower partial moment (x above, y below target). | +| Co-moments | `d_upm` | `D.UPM` | implemented | Divergent upper partial moment (x below, y above target). | | Co-moments | `co_lpm_nd` | `Co.LPM_nD` | implemented | N-dimensional co-lower partial moment wrapper. | | Co-moments | `co_upm_nd` | `Co.UPM_nD` | implemented | N-dimensional co-upper partial moment wrapper. | -| Co-moments | `dpm_nd` | `DPM_nD` | implemented | N-dimensional directional partial moment wrapper. | -| Classical moments | `ecdf_pm` | ECDF partial-moment helper | implemented | Empirical distribution helper used by moment routines. | +| Co-moments | `dpm_nd` | `DPM_nD` | implemented | N-dimensional divergent partial moment wrapper. | +| Classical moments | `ecdf_pm` | ECDF partial-moment helper | implemented | Empirical CDF `lpm(0, points, x)` (standalone helper). | | Classical moments | `mean_pm` | `Mean` partial-moment helper | implemented | Mean via partial-moment conventions. | | Classical moments | `var_pm` | `Variance` partial-moment helper | implemented | Variance via partial-moment conventions. | | Classical moments | `skew_pm` | `Skewness` partial-moment helper | implemented | Skewness via partial-moment conventions. | @@ -51,13 +51,13 @@ The generator introspects `nns.__all__`, imports each public object, records sig | Classical moments | `nns_moments` | `NNS.moments` | implemented | Bundled NNS moment diagnostics. | | Dependence | `nns_dep` | `NNS.dep` | implemented | Nonlinear dependence and correlation. | | Dependence | `nns_cor` | `NNS.cor` | implemented | Nonlinear correlation convenience API. | -| Dependence | `nns_copula` | `NNS.copula` | implemented | Bivariate copula surface. | +| Dependence | `nns_copula` | `NNS.copula` | implemented | Bivariate or multivariate copula dependence. | | Causation | `nns_causation` | `NNS.caus` | implemented | Directional causation. | -| Causation | `causal_matrix` | `Causal.matrix` | implemented | Matrix form of causation relationships. | +| Causation | `causal_matrix` | `NNS.caus` (matrix input) | implemented | Matrix form of causation relationships. | | Regression | `nns_reg` | `NNS.reg` | implemented | Bivariate regression, classification, and interval surfaces. | -| Regression | `nns_m_reg` | `NNS.M.reg` | partial | Multivariate regression. Raw factor expansion is guarded. | +| Regression | `nns_m_reg` | `NNS.M.reg` | implemented | Multivariate regression, including raw factor predictors. | | Regression | `nns_stack` | `NNS.stack` | implemented | Stacked ensemble path. | -| Regression | `nns_boost` | `NNS.boost` | partial | Boosted ensemble path with one high-feature stochastic threshold guard. | +| Regression | `nns_boost` | `NNS.boost` | implemented | Boosted ensemble path. | | Regression helpers | `FactorDesign` | Python helper | implemented | Dataclass returned by factor-preparation helpers. | | Regression helpers | `prepare_factor_predictors` | Factor preparation path | implemented | Builds regression-ready factor design matrices. | | Categorical helpers | `encode_factor_codes` | Factor-code helper | implemented | Encodes categorical labels into deterministic numeric codes. | @@ -67,14 +67,14 @@ The generator introspects `nns.__all__`, imports each public object, records sig | Forecasting | `nns_arma` | `NNS.ARMA` | partial | Univariate ARMA-style forecast surface. | | Forecasting | `nns_arma_optim` | `NNS.ARMA.optim` | partial | ARMA optimizer surface. | | Forecasting | `nns_var` | `NNS.VAR` | partial | Multivariate VAR-style forecast surface. | -| Distribution and ANOVA | `nns_cdf` | `NNS.cdf` | implemented | NNS empirical CDF path. | +| Distribution and ANOVA | `nns_cdf` | `NNS.CDF` | implemented | NNS empirical CDF path. | | Distribution and ANOVA | `nns_anova` | `NNS.ANOVA` | implemented | Binary, multi-group, and pairwise ANOVA-style comparisons. | | Distribution and ANOVA | `nns_norm` | `NNS.norm` | implemented | NNS normalization helper. | | Distance and partitioning | `nns_distance` | `NNS.distance` | implemented | Single distance computation. | | Distance and partitioning | `nns_distance_bulk` | `NNS.distance.bulk` | implemented | Bulk distance computation. | | Distance and partitioning | `nns_part` | `NNS.part` | implemented | Partitioning helper returning Python-native structures. | | Central tendencies | `nns_gravity` | `NNS.gravity` | implemented | NNS gravity center helper. | -| Central tendencies | `nns_mode` | `NNS.mode` | implemented | NNS mode helper. | +| Central tendencies | `nns_mode` | `NNS.mode` | implemented | NNS mode helper. Default `multi=False` (R: `multi = TRUE`). | | Central tendencies | `nns_rescale` | `NNS.rescale` | implemented | Rescaling helper. | | Stochastic dominance | `fsd` | `NNS.FSD` | implemented | First-order stochastic dominance. | | Stochastic dominance | `ssd` | `NNS.SSD` | implemented | Second-order stochastic dominance. | @@ -90,8 +90,10 @@ The generator introspects `nns.__all__`, imports each public object, records sig | Differentiation | `nns_diff` | `NNS.diff` | implemented | Numerical differentiation. | | Differentiation | `dy_dx` | `dy.dx` | implemented | Scalar derivative helper. | | Differentiation | `dy_d` | `dy.d_` | partial | Multivariate derivative helper. | -| VaR helpers | `lpm_var` | `LPM.VaR` | implemented | Lower partial-moment VaR helper. | -| VaR helpers | `upm_var` | `UPM.VaR` | implemented | Upper partial-moment VaR helper. | +| VaR helpers | `lpm_var` | `LPM.VaR` | implemented | Lower partial-moment VaR; scalar or vector `percentile`. | +| VaR helpers | `upm_var` | `UPM.VaR` | implemented | Upper partial-moment VaR; scalar or vector `percentile`. | +| Native threads | `set_num_threads` | `ncores` (per call) | Python-only | Default native thread count; `None` = `NNS_NUM_THREADS` or the hardware count. See conventions, Threads. | +| Native threads | `get_num_threads` | - | Python-only | Current default native thread count. | ## Reference by API area @@ -133,6 +135,8 @@ Closest R API: `PM.matrix`. Builds a partial-moment matrix surface used by dependence and co-moment routines. +The package attribute `nns.pm_matrix` is this function, so it shadows the `nns.pm_matrix` submodule: `import nns.pm_matrix as m` binds the function. Use `from nns.pm_matrix import ...` to reach module-level names. + ### Co-moments and n-dimensional partial moments #### `co_lpm` @@ -151,23 +155,23 @@ Computes pairwise co-upper partial moments. Closest R API: `D.LPM`. -Computes pairwise lower directional partial moments. +Divergent lower partial moment: `mean(max(x - target_x, 0)^degree_upm * max(target_y - y, 0)^degree_lpm)`, the quadrant where `x` is above and `y` at or below its target. #### `d_upm` Closest R API: `D.UPM`. -Computes pairwise upper directional partial moments. +Divergent upper partial moment: `mean(max(target_x - x, 0)^degree_lpm * max(y - target_y, 0)^degree_upm)`, the quadrant where `x` is at or below and `y` above its target. #### `co_lpm_nd`, `co_upm_nd`, `dpm_nd` -N-dimensional wrappers around the public co-moment and directional partial-moment concepts. +N-dimensional co-lower, co-upper and divergent partial moments (R `Co.LPM_nD`, `Co.UPM_nD`, `DPM_nD`). At degree 0 they count rows at or below the target in every coordinate, strictly above it in every coordinate, and the rest, so the three shares sum to 1. ### Classical moment helpers #### `ecdf_pm` -Empirical CDF helper used by partial-moment-based moment routines. +Empirical CDF `lpm(0, points, x)` (the share of `x` at or below each point; sorted `x` when `points` is omitted). No other routine uses it. #### `mean_pm` @@ -195,7 +199,7 @@ Returns bundled NNS moment diagnostics. Closest R API: `NNS.dep`. -Returns NNS nonlinear dependence and correlation for bivariate input, or matrix-style dependence and correlation results for matrix input. +Returns NNS nonlinear dependence and correlation for bivariate input, or `Correlation` and `Dependence` matrices for a 2-D array or DataFrame (`y=None`). `asym=True` measures `y` as a function of `x` (matrix entry `[i, j]`: row variable -> column variable). #### `nns_cor` @@ -207,7 +211,7 @@ Convenience API for the NNS nonlinear correlation surface. Closest R API: `NNS.copula`. -Computes the bivariate scalar copula surface supported by NNS Python. +Computes R's co-partial-moment copula dependence in `[0, 1]` for two vectors (`nns_copula(x, y)`) or a matrix with any number of columns (`nns_copula(X)`). #### `nns_causation` @@ -217,9 +221,9 @@ Computes directional causation relationships, including supported numeric lag an #### `causal_matrix` -Closest R API: `Causal.matrix`. +Closest R API: `NNS.caus` with a matrix argument (internally `NNS.caus.matrix`; there is no `Causal.matrix`). -Computes a matrix representation of causation relationships. +Computes the antisymmetric net-causation matrix; entry `[i, j]` is the net causation of column `j` on column `i` (positive: `j` causes `i`). DataFrame input returns a labelled DataFrame. ### Regression, classification, and ensembles @@ -227,7 +231,7 @@ Computes a matrix representation of causation relationships. Closest R API: `NNS.reg`. -Bivariate NNS regression and classification. `dist=None` is the default and selects the native blended NNS distance; `dist="NNS"` is the explicit alias. Supports numeric, class-code, confidence interval, smoothing, dimension-reduction, and public factor-expansion paths. +Bivariate NNS regression and classification. `dist=None` is the default and selects the native blended NNS distance; `dist="NNS"` is the explicit alias. Supports numeric, class-code, confidence interval, smoothing, dimension-reduction, and factor paths. pandas column names are kept, `point_est` columns are matched by name, and string/categorical columns are factors (declared or sorted level order; `class_levels=` / `factor_levels=` declare levels). Returns Python-native structures, generally dictionaries containing estimates, diagnostics, residuals, intervals, and related arrays. @@ -235,19 +239,19 @@ Returns Python-native structures, generally dictionaries containing estimates, d Closest R API: `NNS.M.reg`. -Multivariate regression and classification surface. Numeric and class paths are implemented. Direct raw-factor `factor_2_dummy=True` is guarded because the installed R direct raw-factor path errors; use `prepare_factor_predictors(...)` first. +Multivariate regression and classification surface (R `NNS.M.reg`). Numeric, class, and raw factor/character predictor paths are implemented; `factor_2_dummy=True` is the default, as in R 13.3, and pandas column names are kept. #### `nns_stack` Closest R API: `NNS.stack`. -Stacked ensemble API for numeric and classification workflows. `dist=None` is the default and selects the native blended NNS distance; `dist="NNS"` is the explicit alias. `NNS.dim.red.variables` (0-based predictor indices, ranked by absolute coefficient) and `NNS.dim.red.coefficients` report the dimension-reduction selection; with `stack=True` those predictors are the only ones behind `stack`. +Stacked ensemble API for numeric and classification workflows. `dist=None` is the default and selects the native blended NNS distance; `dist="NNS"` is the explicit alias. `NNS.dim.red.variables` (predictor names, ranked by absolute coefficient) and `NNS.dim.red.coefficients` (dict of encoded column name to coefficient) report the dimension-reduction selection; with `stack=True` those predictors are the only ones behind `stack`. #### `nns_boost` Closest R API: `NNS.boost`. -Boosted ensemble API. `dist=None` is the default and selects the native blended NNS distance; `dist="NNS"` is the explicit alias. Deterministic and stochastic structures are implemented. The high-feature stochastic threshold path is guarded to make an installed-R failure explicit. +Boosted ensemble API. `dist=None` is the default and selects the native blended NNS distance; `dist="NNS"` is the explicit alias. Seeded runs reproduce R's random stream; `feature.weights` / `feature.frequency` are keyed by predictor name (`V1..Vp` for an unnamed matrix). ### Categorical and factor helpers @@ -257,7 +261,7 @@ Python helper dataclass used to hold prepared factor design matrices and metadat #### `prepare_factor_predictors` -Builds a regression-ready full-rank factor design matrix before calling `nns_m_reg(...)`. +Builds a full-rank factor design matrix with the encoder `nns_reg` uses (bool columns are factors; dummy names such as `x_TRUE.` and `V1_a`). Not needed for `nns_reg` / `nns_m_reg`, which encode raw factor columns themselves. #### `encode_factor_codes` @@ -301,7 +305,7 @@ Nonparametric vector autoregression: univariate `nns_arma_optim` forecasts plus, #### `nns_cdf` -Closest R API: `NNS.cdf`. +Closest R API: `NNS.CDF`. NNS empirical CDF path. Deterministic non-plotting behavior is implemented. @@ -315,7 +319,8 @@ NNS ANOVA-style comparison helper covering binary, multi-group, pairwise, and de Closest R API: `NNS.norm`. -NNS normalization helper. Accepts a 2-D matrix or, like R's list input, a +NNS normalization helper. Accepts a 2-D matrix, a pandas DataFrame (returned +as a DataFrame with R's `" Normalized"` names) or, like R's list input, a sequence of 1-D vectors (one per variable). Equal-length vectors are column-stacked and normalized through the matrix path; unequal-length vectors force `linear=True` exactly as R does and return a list of scaled @@ -351,7 +356,7 @@ Computes the NNS gravity center helper. Closest R API: `NNS.mode`. -Computes the NNS mode helper. +Computes the NNS mode. The default is `multi=False` (a single float), whereas R's `NNS.mode` defaults to `multi = TRUE` and returns every detected mode; pass `multi=True` for R's default. Explicit `multi` values match R. #### `nns_rescale` @@ -433,13 +438,13 @@ Multivariate derivative helper. Scalar and vectorized point and distribution mod Closest R API: `LPM.VaR`. -Lower partial-moment VaR helper used by deterministic interval paths. +Returns the target `t` with `LPM.ratio(degree, t, x) = percentile`. `percentile` may be a scalar (returns a float) or array-like (returns an array of the same shape, like R's vectorised call); values are clamped to `[0, 1]`. Degree 0 is the type-7 sample quantile. #### `upm_var` Closest R API: `UPM.VaR`. -Upper partial-moment VaR helper used by deterministic interval paths. +Returns the target `t` with `UPM.ratio(degree, t, x) = percentile`; scalar or array-like `percentile` as for `lpm_var`. ## Documentation maintenance checklist diff --git a/docs/api_status.md b/docs/api_status.md index 93977940..1a34c2e3 100644 --- a/docs/api_status.md +++ b/docs/api_status.md @@ -13,11 +13,11 @@ data-frame quirk, or runtime side effect as a public Python API. Current release-relevant state: the core partial-moment APIs, deterministic regression/classification/forecasting surfaces, and scalar/vectorized multivariate derivative modes are parity-covered on focused fixtures. The -largest remaining API work is now mostly ergonomic: categorical predictor -preparation is explicit through `prepare_factor_predictors(...)`, while direct -raw-factor `nns_m_reg(..., factor_2_dummy=True)` remains guarded because the -installed R internal path errors. Named R data-frame factor ordering quirks are -documented as outside NNS Python' positional-column API boundary. Performance gaps +regression, stack and boost functions take pandas frames the way R takes data +frames: column names are kept and prediction columns are matched by name, +string and categorical columns are factors in their declared (or sorted) level +order, and `class_levels=` / `factor_levels=` declare levels explicitly. +Performance gaps remain mostly in large stochastic-dominance workloads where R uses compiled kernels. @@ -38,7 +38,7 @@ invariant, and property coverage. |---|---|---|---| | Core partial moments: `lpm`, `upm`, `lpm_ratio`, `upm_ratio` | implemented | high | Matches R partial-moment conventions, including degree-zero equality handling. | | Partial-moment matrices and n-dimensional wrappers: `pm_matrix`, `co_lpm_nd`, `co_upm_nd`, `dpm_nd` | implemented | high | Public matrix and n-dimensional partial-moment surfaces are covered. | -| Pairwise co-moments: `co_lpm`, `co_upm`, `d_lpm`, `d_upm` | implemented | high | Python raises on length mismatch instead of silently truncating like R. | +| Pairwise co-moments: `co_lpm`, `co_upm`, `d_lpm`, `d_upm` | implemented | high | Unequal or empty inputs and targets raise, as in R. | | Classical helpers: `ecdf_pm`, `mean_pm`, `var_pm`, `skew_pm`, `kurt_pm`, `nns_moments` | implemented | high | Population-normalized defaults are documented in `docs/conventions.md`. | | VaR helpers: `lpm_var`, `upm_var` | implemented | high | Used by deterministic confidence and prediction interval paths. | | Central tendencies: `nns_gravity`, `nns_mode`, `nns_rescale` | implemented | high | Public helper behavior is covered through direct and dependent tests. | @@ -48,17 +48,17 @@ invariant, and property coverage. | Distribution functions: `nns_cdf` | implemented | high | Deterministic non-plotting paths are implemented; plotting is ignored. | | Distance helpers: `nns_distance`, `nns_distance_bulk` | implemented | high | Numeric and classification conventions follow installed R behavior. | | Partitioning: `nns_part` | implemented | high | Returns plain dictionaries/arrays instead of R `data.table` objects. | -| Regression: `nns_reg` | implemented | high | Numeric, class-code, confidence interval, smoothing, dimension-reduction, and public factor-expansion paths are covered. | -| Multivariate regression: `nns_m_reg` | partial | medium-high | Numeric and class paths are implemented; use `prepare_factor_predictors(...)` for categorical design matrices before calling `nns_m_reg`. Direct raw factor expansion remains guarded. | -| Stack: `nns_stack` | implemented | medium | Numeric/class paths, intervals, factor expansion, and `ts_test` are covered; exact stochastic sample parity is not expected. | -| Boost: `nns_boost` | partial | medium | Deterministic and stochastic structures are implemented; one high-feature threshold path remains guarded to match installed-R failure behavior. | +| Regression: `nns_reg` | implemented | high | Numeric, class-code, confidence interval, smoothing, dimension-reduction, and factor paths are covered; `point_est` columns are matched by name, categorical/declared levels are honoured. | +| Multivariate regression: `nns_m_reg` | implemented | high | Numeric, class, and raw factor/character predictor paths (`factor_2_dummy=True` by default, as R 13.3) are covered; column names are kept. | +| Stack: `nns_stack` | implemented | medium-high | Numeric/class paths, intervals, factor predictors, `ts_test`, and R's seeded CV/balance draws (bit-for-bit) are covered; test columns are matched by name and selection outputs carry predictor names. | +| Boost: `nns_boost` | implemented | medium | Learner trials, epochs, and the final stack match R's seeded stream; feature weights/frequencies are named by column. | | Seasonality: `nns_seas` | implemented | high | Non-plotting installed-R path is implemented and cached defensively. | -| ARMA and VAR: `nns_arma`, `nns_arma_optim`, `nns_var` | partial | medium | Numeric forecasting and every VAR dimension-reduction path (`cor`, `NNS.dep`, `NNS.caus`, `all`) match R NNS 13.3 exactly on driven multi-variable fixtures, including own-lag weights, `objective="max"`, vector and list `tau`, and interior/trailing missing values. VAR's `relevant_variables` are the regressors the cross-validated dimension reduction threshold keeps, and `multivariate` is `nns_stack(method=1)` (`nns_reg`) on exactly those regressors. Explicit numeric multi-lag ARMA uses actual-lag weighting instead of installed R's position-based weighting quirk. Stochastic interval streams are structural/statistical parity only. | +| ARMA and VAR: `nns_arma`, `nns_arma_optim`, `nns_var` | partial | medium | Numeric forecasting and every VAR dimension-reduction path (`cor`, `NNS.dep`, `NNS.caus`, `all`) match R NNS 13.3 exactly on driven multi-variable fixtures, including own-lag weights, `objective="max"`, vector and list `tau`, and interior/trailing missing values. VAR's `relevant_variables` are the regressors the cross-validated dimension reduction threshold keeps, and `multivariate` is `nns_stack(method=1)` (`nns_reg`) on exactly those regressors. Stochastic interval streams are structural/statistical parity only. | | Bootstrap/Monte Carlo: `nns_meboot`, `nns_mc` | implemented | medium | Deterministic diagnostics are parity-tested; exact stochastic replicate parity with R is not expected. | | Stochastic dominance/superiority: `fsd`, `ssd`, `tsd`, `.uni` wrappers, `nns_ss`, `nns_sd_cluster`, `sd_efficient_set` | implemented | medium | Public structures and deterministic paths are covered. SD uses exact pure-NumPy prefix-pair kernels plus a degree-1 discrete order-statistic matrix path; R's C++ core remains faster on full finance fixtures. Stochastic intervals use NNS Python RNG. | | ANOVA: `nns_anova` | implemented | high | Binary, multi-group, pairwise, and degenerate `NaN` conventions are covered. | | Normalization: `nns_norm` | implemented | high | Numeric matrix path and R's list-of-vectors path are implemented; unequal-length vectors force linear scaling as in R. | -| Categorical helpers: `encode_factor_codes`, `factor_2_dummy`, `factor_2_dummy_fr`, `prepare_factor_predictors` | implemented | high | Explicit `levels=` / `factor_levels=` should be used to reproduce R factor ordering. `prepare_factor_predictors(...)` exposes the regression-ready full-rank design matrix path. | +| Categorical helpers: `encode_factor_codes`, `factor_2_dummy`, `factor_2_dummy_fr`, `prepare_factor_predictors` | implemented | high | Explicit `levels=` reproduces R factor ordering. `prepare_factor_predictors(...)` builds a full-rank dummy design; the regression functions encode raw factors themselves and do not need it. | | Scalar differentiation: `nns_diff`, `dy_dx` | implemented | high | `dy_dx(..., eval_point="overall")` and numeric evaluation points are covered. | | Multivariate differentiation: `dy_d` | partial | medium-high | Scalar and vectorized point/distribution modes are covered on focused fixtures. Mixed derivatives are supported for two-regressor inputs where defined; multi-row matrix mixed derivatives use pointwise Python semantics rather than R's order-dependent list-matrix packing quirk. | @@ -66,9 +66,7 @@ invariant, and property coverage. | Area | Path | Current behavior | Reason / next action | |---|---|---|---| -| Multivariate regression | direct `factor_2_dummy=True` raw predictor path | Guarded with `NotImplementedError` in direct `nns_m_reg(..., factor_2_dummy=True)`. | Installed R direct `NNS.M.reg` raw factor input errors. Use `prepare_factor_predictors(...)` first, or use the public `nns_reg(..., factor_2_dummy=True, factor_levels=...)` expansion path. | -| Boost | `threshold` on the `n_features > 10` stochastic path | Guarded with `NotImplementedError` on the high-feature stochastic epoch path. | Installed R errors because `test.features` is never built. NNS Python keeps this explicit. | -| Boost/factor predictors | named data-frame factor predictor ordering | Deferred, not represented as a named-column API. | NNS Python uses positional `X1`, `X2`, ... semantics. Installed R named data frames can reorder columns alphabetically before `data.matrix`. | +| (none) | | No production path is guarded with `NotImplementedError`. | | ## Intentional Design Boundaries @@ -96,8 +94,11 @@ examples include: - Empty numeric inputs raise `ValueError`; R NNS often returns `NaN`. - Co-moment length mismatches raise `ValueError`; R warns, truncates, and divides by the longer length. -- Factor and class labels are explicit. R factor levels become numeric codes; - NNS Python callers should pass `levels=` or `class_levels=` when ordering matters. +- R factor levels become numeric codes with `class.levels` as the label map. + pandas categoricals keep their declared order; plain string labels are + coded in sorted order (R's `factor()` order; R's `NNS.reg` character + predictors use first appearance, which depends on row order); + `class_levels=` / `factor_levels=` declare an order explicitly. - Public outputs use NumPy arrays and plain dictionaries instead of R `data.table` objects. - Some installed-R quirks are intentionally matched when they affect stable @@ -106,8 +107,8 @@ examples include: Current passing coverage includes partial-moment equivalences, curve fitting, regression residuals, Boston Housing, and the macro-like VAR univariate, multivariate, and ensemble outputs. Strict xfails track current installed-R - deviations in the Iris classification vignette and the documented ARMA - numeric multi-lag weighting divergence. The Iris + deviations in the Iris classification vignette; explicit numeric multi-lag + ARMA (e.g. `seasonal_factor=[132, 276]`) matches installed R. The Iris classification xfail mixes two different issues: NNS Python stack predicts the correct held-out class where installed R NNS 13.0 rounds the same borderline estimate down, while boost remains a true output disparity whose installed-R diff --git a/docs/conventions.md b/docs/conventions.md index b2cdf52d..f6f0e5e7 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -7,7 +7,43 @@ acceleration through `nns._nnscore` where available. Source builds use `scikit-build-core` and `nanobind` to compile the extension; published wheels should be preferred when available. Public APIs keep Python implementations and explicit fallback behavior, so native code remains a deliberate, -benchmark-backed implementation detail rather than a public API. +benchmark-backed implementation detail rather than a public API. Setting the +environment variable `NNS_DISABLE_NATIVE=1` before importing `nns` forces the +NumPy fallbacks, as when the extension is not built; CI runs the invariant and +cached parity tests that way too. + +## Threads + +The native kernels (vectorised partial moments, the PM matrix, the n-dimensional +co-moments and the copula) can use several threads. Inputs below a fixed amount of +work always run serially, and results never depend on the thread count. +`nns.set_num_threads(n)` sets the default; `n=None` restores the automatic choice, +which is the `NNS_NUM_THREADS` environment variable when it is a positive integer +and the number of hardware threads otherwise. `nns.get_num_threads()` returns the +current default. The `ncores` argument of `nns_reg`, `nns_m_reg`, `nns_stack`, +`nns_boost`, `nns_arma_optim` and `nns_var` caps the native threads for that call: +`None` keeps the default, `1` runs serially, and anything other than `None` or a +positive integer raises `ValueError`, as in R's `NNS.M.reg`. The cap applies per +Python thread. R also uses `ncores` for its parallel `NNS.ARMA.optim` workers; +NNS Python has no process-level parallelism. The heavy kernels release the GIL, +so Python threads can run them concurrently. + +## Plotting + +Compute functions never draw unless asked. Where R's default is `plot = TRUE` +(`NNS.reg`, `NNS.FSD`/`SSD`/`TSD`, `NNS.ANOVA`, ...), NNS Python defaults to +`plot=False` and returns the same values either way; `plot=True` additionally +draws the figure through the matching `nns.plotting.plot_*` function and never +calls `plt.show()`. The flag exists on `nns_reg`, `nns_m_reg`, `nns_cdf`, +`nns_seas`, `nns_arma`, `nns_arma_optim`, `nns_part` (R's `Voronoi`), +`fsd`, `ssd`, `tsd`, `nns_anova`, `nns_norm` (R's `chart.type = "l"`), +`nns_diff` and `nns_copula`. `nns_sd_cluster(dendrogram=True)` draws the +dendrogram, as R does, because the flag is also what returns the `hclust` data. +A requested figure that cannot be drawn raises a `UserWarning` instead of being +skipped silently: multivariate `nns_reg(plot=True)` with more than two +regressors (R draws its 3-D scatter for exactly two; `residual_plot=True` draws +the fitted-values plot for any number), `nns_copula(plot=True)` with more than +three variables, and `nns_norm(plot=True)` on unequal-length lists. ## Degree-Zero Boundary @@ -23,12 +59,12 @@ NNS Python raises `ValueError`. Rationale: empty arrays in Python are upstream bugs, and NumPy convention is to warn or fail on empty reductions rather than silently produce a meaningful statistic. -## Co-Moment Length Mismatch Divergence From R +## Co-Moment Lengths -R NNS warns when `x` and `y` lengths differ, computes over the shorter length, and divides by the longer length. -NNS Python raises `ValueError`. - -Rationale: mismatched co-moment inputs lose observations silently in R. Python callers should fix alignment before computing a bivariate statistic. +`co_lpm`, `co_upm`, `d_lpm` and `d_upm` raise `ValueError` unless `x` and `y` are +non-empty and the same length, and `target_x` and `target_y` are non-empty and the +same length or one of them length 1. R's `Co.LPM`, `Co.UPM`, `D.LPM` and `D.UPM` +check the same rules. ## PM Matrix Target Defaults @@ -38,6 +74,15 @@ scalar numeric target across all variables; R requires callers to pass an explicit vector such as `rep(0, ncol(variable))`. Target vectors whose length does not match the number of variables raise `ValueError`. +## Mode Default Divergence From R + +`nns_mode` defaults to `multi=False` and returns a single float, while R's +`NNS.mode` defaults to `multi = TRUE` and returns every detected mode (e.g. two +values for a bimodal sample). Pass `multi=True` for R's default output; explicit +`multi` values match R. The Python default is kept because the multi-mode peak +finder also reports spurious secondary modes on unimodal data (in R as well), +and the internal callers all ask for a single mode. + ## Classical Moment Normalization `mean_pm`, `var_pm`, `skew_pm`, and `kurt_pm` use population normalization by @@ -46,7 +91,9 @@ accepts `ddof` for NumPy-style variance scaling. `skew_pm` and `kurt_pm` do not apply SciPy's optional finite-sample bias correction. `nns_moments` is the public `NNS.moments` wrapper and returns R's dictionary -shape with `mean`, `variance`, `skewness`, and `kurtosis`. `nns_gravity` exposes +shape with `mean`, `variance`, `skewness`, and `kurtosis`. As in R, undefined +moments are `NaN` rather than errors (constant input; `population=False` with +n <= 3). `nns_gravity` exposes R's public `NNS.gravity` central-tendency helper. `fsd_uni`, `ssd_uni`, and `tsd_uni` are the unidirectional stochastic-dominance wrappers behind R's `.uni` exports. `co_lpm_nd`, `co_upm_nd`, and `dpm_nd` expose the public @@ -65,10 +112,14 @@ convenience for that stochastic path. peels `sd_efficient_set` results and returns a dictionary of `Cluster_1`, `Cluster_2`, ... memberships. The output contains variable names, not numeric cluster labels; when names are omitted, NNS Python uses R-style `X_1`, `X_2`, ... -names. `type="continuous"` is supported for first-degree efficient sets. +names, or the column names of a DataFrame input. `type="continuous"` is +supported for first-degree efficient sets, `fsd`, and `fsd_uni`; as in R, an +unrecognised `type` warns (and `fsd` then runs its continuous branch, while +`fsd_uni` and the efficient-set routines run the discrete one). `dendrogram=True` returns a plain dictionary mirroring R's `hclust` fields: `merge`, `height`, `order`, `labels`, `method`, `call`, and `dist.method`. -NNS Python does not plot the dendrogram; it only returns the object data. +With fewer than two clustered variables it warns, as R does, and returns the +clusters only. The dendrogram is drawn as well (see Plotting). The stochastic-dominance implementation is deliberately pure NumPy. It mirrors R's C++ SD core mathematically by sorting each column once, storing prefix sums, @@ -76,24 +127,32 @@ and evaluating dominance on each pair's merged threshold grid rather than on one global all-column grid. Pairwise tests (`fsd`, `ssd`, `tsd`, and the `*_uni` wrappers) accept samples of unequal length: each sample's curve is evaluated on the merged grid exactly as R's `NNS.FSD`/`NNS.SSD`/`NNS.TSD` compute -`LPM(degree, sort(c(x, y)), sample)` per sample. R's C++ `.uni` walkers assume -equal-length inputs, so for unequal lengths the Python `*_uni` wrappers are an -intentional extension carrying the same merged-grid semantics. The +`LPM(degree, sort(c(x, y)), sample)` per sample, and as the patched R `.uni` +walkers do (each sample walked with its own length). Every pairwise verdict +comes from one port of R's native pairwise test: for TSD and continuous FSD it +also checks the closed-form extremum of the curve difference strictly between +merged sample points (LPM2 is piecewise quadratic and the continuous ratio +linear-fractional, so the difference can peak there), as the patched R does. The matrix-based efficient-set and cluster routines operate on data columns and -therefore remain equal-length by construction. The full prefix-pair dominance matrix remains available -internally for verification and fallback. Large degree-1 discrete calls use an -exact order-statistic dominance matrix: with equal-length empirical samples, -one sample first-order stochastically dominates another exactly when every -sorted order statistic is at least as large and at least one is strictly larger. -Large degree-1 continuous and degree 2/3 calls use a lazy kept-only prefix scan. +therefore remain equal-length by construction. Every efficient-set and cluster +call, whatever the column count, tests each pair on that pair's own grid, so a +pair's verdict never depends on which other columns are present. Degree-1 +discrete calls use an exact order-statistic dominance matrix: with equal-length +empirical samples, one sample first-order stochastically dominates another +exactly when every sorted order statistic is at least as large and at least one +is strictly larger. Degree-1 continuous and degree 2/3 calls use a lazy +kept-only prefix scan. Columns are visited in R's LPM-at-global-maximum order with original-index tie breaks, and only already-kept candidates are tested against the current column. Each prefix pair check applies min/mean/identical guards before evaluating -curves, then exits as soon as dominance is disproved. - -These choices preserve exact R-style dominance semantics: no tolerances, -approximate equality, output reordering, or diagonal/identical-column behavior -changes are introduced. Polars is intentionally not used in this SD kernel +curves. + +Curve differences up to `64 * eps * S^d` (`S` the largest absolute value of +the two samples; `d` = 1 for SSD and the mean condition, 2 for TSD, a ratio +analogue for continuous FSD) are treated as ties, as in the patched R, so +rounding noise does not decide dominance; discrete FSD compares exact +empirical CDF fractions. No output reordering or diagonal/identical-column +behavior changes are introduced. Polars is intentionally not used in this SD kernel because the hot path is dense pairwise threshold evaluation rather than data-frame grouping or filtering. R remains faster on some large finance fixtures because its C++ path walks merged sorted thresholds in tight parallel @@ -107,31 +166,51 @@ uses R's lower-partial-moment frequency convention, and positive degrees use `LPM.ratio` deformation. Univariate output columns follow installed R (`x` plus `CDF`, `S(x)`, `h(x)`, or `H(x)`), while multivariate output keeps the final column named `CDF` for all types, including survival, hazard, and cumulative -hazard. Plotting is ignored. The univariate `NA`/`Inf` comparison quirks are -handled inside `nns_cdf` without loosening the global partial-moment APIs. +hazard. Input column names that repeat each other or `CDF` get R +`make.unique` suffixes (`CDF.1`, `a.1`), since a dict cannot hold the duplicate +names R returns. Plotting is ignored. As in the patched R (see [parity](parity.md)), missing +values are dropped first (complete rows for a matrix), so the CDF reaches 1, and +infinite values raise `ValueError`. For `d >= 3` columns the survival, hazard and +cumulative hazard use the joint survival `P(X_1 > t_1, ..., X_d > t_d)` by +inclusion-exclusion over all column subsets. ## Dependence -`nns_dep` follows R's `NNS.dep` bivariate path, including `NNS.gravity` handling -for zero-range inputs and non-positive or non-finite bin widths. NNS Python also caps -the internal gravity bin count at `4 * len(input)` to prevent pathological -allocations on inputs where R's C++ `int` conversion effectively collapses an -absurd bin count. `abs(Correlation) <= Dependence` is not guaranteed by +`nns_dep(x, y)` follows R's `NNS.dep` bivariate path. With `asym=True` the partition +is on `x` and the result measures `y` as a function of `x`. `nns_dep(X)` on a 2-D +array or DataFrame returns R's `Correlation` and `Dependence` matrices; with +`asym=True` entry `[i, j]` is `nns_dep(X[:, i], X[:, j], asym=True)` (row variable +-> column variable, which is what R returns although its manual states the +reverse). R's `p.value` option is not implemented. The bivariate path includes `NNS.gravity` handling +for zero-range inputs and non-positive or non-finite bin widths. The gravity +histogram (bin width `IQR / sqrt(n)`) is built from the occupied bins only, so a +single outlier that makes R allocate millions of empty bins costs nothing and gives +R's value. The compiled kernel (NNS-core `nns::gravity`) and the NumPy fallback +compute the same value bit for bit. They differ from R 13.3 only when the bin count +exceeds the C `int` range: R's cast overflows there and the histogram collapses to +one bin, while NNS Python (like the patched R) keeps the real histogram. +`abs(Correlation) <= Dependence` is not guaranteed by `NNS.dep`; both R and NNS Python can return signed correlation magnitudes above the dependence component for near-binary inputs. ## Copula -`nns_copula(x, y)` is the bivariate scalar form of R's `NNS.copula(cbind(x, y))`. -When targets are omitted, NNS Python uses column means, matching R's `target = NULL`. -The `target_x` and `target_y` arguments map to R's two-element target vector. +`nns_copula(x, y)` is the bivariate form of R's `NNS.copula(cbind(x, y))`, and +`nns_copula(X)` takes a 2-D matrix with any number of columns (at least two), as R +does; `target` gives the per-column targets of the matrix form. When targets are +omitted NNS Python uses the column means computed like R's `colMeans` (long-double +sequential sum), matching R's `target = NULL` to the last bit. The `target_x` and +`target_y` arguments map to R's two-element target vector. Empty or non-finite +input raises `ValueError` (R stops on missing values). ## Causation `nns_causation(x, y)` maps to R's `NNS.caus(x, y, tau = 0, p.value = FALSE)` numeric-vector path. It returns the two directional components and the named signed net log-ratio key selected by R, either `C(x--->y)` or `C(y--->x)`. -`causal_matrix` maps to R's `NNS.caus.matrix` antisymmetric matrix convention. +`causal_matrix` maps to R's `NNS.caus` on a matrix (internally `NNS.caus.matrix`): +an antisymmetric matrix whose entry `[i, j]` is the net causation of column `j` on +column `i` (positive: `j` causes `i`). A DataFrame input returns a DataFrame labelled with its columns. `tau='ts'` uses `nns_seas(... )["periods"]` exactly like installed R: the first period not exceeding `sqrt(length(x))` is selected per variable, including harmonics when R selects them. Inputs with no eligible selected period follow @@ -142,36 +221,73 @@ supported. `nns_part` maps to R's `NNS.part` but returns plain NumPy arrays instead of `data.table` objects: `"dt"` and `"regression.points"` are dictionaries of -arrays. Installed R 13.0 only distinguishes `type = NULL` from any non-null -`type`: `None` uses XY quadrant splits, while every non-`None` value uses -X-only splits. This differs from documentation that implies separate `"X"`, -`"Y"`, and `"XONLY"` modes. NNS Python matches the installed binary. -`order="max"` is rejected with `TypeError`; installed R coerces it to `NA` and -returns a useless zero-order map. All five `noise_reduction` modes are -supported: `"off"`, `"mean"`, `"median"`, `"mode"`, and `"mode_class"`. +arrays. As in R 13.3, `type` must be `None` (XY quadrant splits) or `"XONLY"` +(any case); other values raise `ValueError`. `order` is `None`, a positive +integer (NumPy integers and integral floats are accepted), or `"max"`, which +returns R's explicit maximum partition (one point per observation, or per unique +`x` for `XONLY`). `obs_req` must be a non-negative integer. The returned `order` +is the depth actually reached; it never exceeds `floor(log2(n))` and can be smaller +than requested. Pairs with `NaN` in either variable are dropped and infinities are +kept, as in R. All five `noise_reduction` modes are supported: `"off"`, `"mean"`, +`"median"`, `"mode"`, and `"mode_class"`. ## Regression -`nns_reg` maps to R's univariate numeric `NNS.reg` path with -`factor.2.dummy = FALSE` and plotting disabled. Return keys match R's list names, but data.table outputs are plain -dictionaries of NumPy arrays. `multivariate_call=True` returns R's internal -two-column regression-point structure as `{"x": ..., "y": ...}` for -`nns_m_reg`, including after dimension-reduction projection. Matrix `x` without -dimension reduction dispatches to `nns_m_reg`. +`nns_reg` maps to R's `NNS.reg` (plotting is a separate side effect). Return +keys match R's list names; data.table outputs are plain dictionaries of NumPy +arrays. `multivariate_call=True` returns R's internal two-column +regression-point structure as `{"x": ..., "y": ...}`. More than one encoded +predictor without dimension reduction runs the `NNS.M.reg` branch. + +Inputs follow R's data-frame rules (`.nns_reg_encode_predictors`): + +- pandas column names are kept as they are (R's `check.names = FALSE`; only + duplicates are renamed, by `make.unique`) as keys in `RPM`, `Fitted.xy` and + `equation`; factor dummy columns are `_`. And `point_est` columns are matched to the training + columns by name: the same set of names is reordered into training order, + names disjoint from the training names (or an unnamed array) are taken + positionally, and a partial overlap raises `ValueError`. A named row such as + `df.iloc[0]` is matched the same way. A single predictor is positional. +- A column is a factor when it is a pandas categorical, when `factor_levels=` + declares its levels, or when it holds labels: strings (including + numeric-looking strings such as `"1"`, `"10"`, as in R) or an object column + with any non-number. An object column of numbers (`df.to_numpy()` of a mixed + frame) is numeric. `bool` columns (and object columns of booleans) are + factors with levels `FALSE`/`TRUE`, as R treats logicals: dummies + `_FALSE.` / `_TRUE.` (R's `make.names`), so even a single + bool predictor runs the `NNS.M.reg` branch. +- Factor levels are the declared levels in their order (pandas categories, or + `factor_levels=`: a mapping from column name to levels, one entry per column + with `None` for numeric columns, or a flat level list for a single + predictor). Undeclared label columns use sorted levels. Divergence: R uses + first-appearance order for character predictors, which makes its fit depend + on the row order; sorted is R's `factor()` order. With the default + `factor_2_dummy=True` the order only permutes the one-hot columns (one per + level); with `factor_2_dummy=False` the level order is the integer code. +- Missing values in a factor column (training or `point_est`) raise + `ValueError`, as in R; `point_est` levels unseen in training raise too. +- The response is a factor when it is a pandas categorical or `class_levels=` + is given (codes `1..K` in that order, `class.levels` lists them, and + classification is implied), a set of labels when it holds strings (sorted + levels, R's `factor(y)`), and numeric otherwise; an object array of numbers + is a numeric response, not classification. Missing labels raise. + Classification returns numeric class codes with `class.levels` as the label + map. + `dist=None` is the default for `nns_reg`, `nns_m_reg`, `nns_stack`, and `nns_boost` and selects the native blended NNS distance, with `dist="NNS"` as the explicit alias. Continuous predictors use range-normalized coordinate differences and sum `abs(z) + z**2` across non-zero-range columns; explicit `dist="L1"`, `dist="L2"`, and `dist="FACTOR"` keep their legacy behavior. -Classification is supported for numeric/logical/factor-like class-code targets. `smooth=True` follows installed R's ordinary piecewise fallback for univariate inputs with fewer than four observations and for univariate `order="max"`; R does not call `smooth.spline` there. Spline-eligible inputs use a private fixed-`spar` cubic smoothing-spline adapter matching the `stats::smooth.spline` -subset used by `NNS.reg`: `spar = (dependence + 0.5) / 2`, R-style knots, and -R's interior-band trace ratio for lambda. -Factor predictor expansion is supported through the public `nns_reg` path. -When combined with dimension reduction, factor predictors are expanded with -R's full-rank dummy convention before synthetic `x.star` coefficients are -computed. For callers that want direct multivariate regression, use -`prepare_factor_predictors(...)` first and pass the returned numeric design -matrix into `nns_m_reg(...)`: +subset used by `NNS.reg`: `spar = (dependence + 0.5) / 2`, R's +`1e-6 * IQR(x)` tie grouping, `.nknots.smspl` knot thinning above 49 unique x, +the Fortran `sgram` penalty (with its `.3330` constant), R's interior-band +trace ratio for lambda, and a banded Cholesky solve. +With dimension reduction, factor predictors are encoded first (one dummy column +per level with `factor_2_dummy=True`) and the synthetic `x.star` coefficients +are computed on the encoded columns, as in R. For callers that want an explicit +numeric design matrix, `prepare_factor_predictors(...)` returns the same +full-rank dummy expansion: ```python from nns import nns_m_reg, prepare_factor_predictors @@ -185,9 +301,8 @@ design = prepare_factor_predictors( fit = nns_m_reg(design.x, y, point_est=design.point_est) ``` -`prepare_factor_predictors(...)` uses the same full-rank dummy expansion as -`nns_reg(..., factor_2_dummy=True)`, combines training `x` and `point_est` -before expansion, and returns deterministic feature names. +`prepare_factor_predictors(...)` combines training `x` and `point_est` before +expansion and returns deterministic feature names. Numeric dimension reduction is supported for `"cor"`, `"NNS.dep"`, `"NNS.caus"`, `"all"`, `"equal"`, and numeric coefficient vectors. The @@ -204,47 +319,35 @@ observed `y` values and `regression.points` is the sorted observed `(x, y)` map. The derivative table still comes from R's pre-reset regression-point construction, which NNS Python matches rather than recomputing adjacent slopes from all observations. -The `"mode"` and `"mode_class"` noise-reduction modes are accepted in the -univariate path and use the shared `nns_part`/`nns_mode` implementation. The -`"mode_class"` default-order path can produce segment `standard.errors` values -that differ from R at floating grouping granularity: installed R groups the -`gradient` column through data.table's numeric radix grouping, while NumPy keeps -near-identical binary floating values as separate groups. Regression points, -coefficients, fitted values, and point estimates still match R on that path. - -Regression confidence intervals are deterministic and use R's `LPM.VaR` / -`UPM.VaR` logic, not `nns_mc` / `nns_meboot`. In the univariate fitted table, -both `conf.int.pos` and `conf.int.neg` use `UPM.VaR(..., degree = 1)` on -segment residuals, matching installed R even though the lower side might look -like an `LPM` candidate. Univariate `point_est` prediction intervals use -`UPM.VaR(..., degree = 0)` for the upper column and `LPM.VaR(..., degree = 0)` -for the lower column. Below-range univariate point estimates follow R's -`findInterval`/data.table behavior: index `0` rows are dropped, so `pred.int` -can have fewer rows than `Point.est`. For class mode, fitted confidence columns -remain raw numeric values, while univariate `pred.int` columns are rounded with -R's `x %% 1 < 0.5` rule. Spline-eligible `smooth=True` interval tables use the -same deterministic residual VaR logic after smoothing, matching installed R. +`noise_reduction` accepts R's `"mean"`, `"median"`, `"mode"` and `"off"`; +classification always reduces by the class mode. + +Regression confidence intervals are deterministic residual bands: the +`alpha/2` and `1 - alpha/2` type-8 quantiles of the training residuals are +added to the fitted values (`conf.int.neg` / `conf.int.pos` in `Fitted.xy`) +and to each point estimate (`pred.int.neg` / `pred.int.pos`, one row per +`point_est` row, including points outside the training range). In class mode +`pred.int` bounds are snapped to the observed class codes; the fitted +confidence columns stay raw. ## Multivariate Regression -`nns_m_reg` maps to installed R's numeric `NNS.M.reg` path with -`factor.2.dummy = FALSE` and plotting disabled. -Outputs use R's keys (`R2`, `rhs.partitions`, `RPM`, `Point.est`, `pred.int`, -and `Fitted.xy`) with data.table objects represented as dictionaries of NumPy -arrays. Numeric and class confidence intervals are deterministic and use the -global residual `UPM.VaR(..., degree = 1)` offset from installed R. In class -mode, fitted predictions and point estimates are rounded/clamped to class codes, -but `pred.int` lower/upper bounds and fitted confidence columns remain raw -numeric values. Classification mode (`type="class"`) is supported for -numeric/logical/factor-like targets and returns numeric class codes. Direct -`nns_m_reg(..., factor_2_dummy=True)` remains rejected for raw factor -predictors because installed R errors on that path. This is an intentional API -boundary rather than a mathematical gap: `nns_m_reg` is the numeric -multivariate engine, while `prepare_factor_predictors(...)` performs the -R-compatible categorical design-matrix preparation. Public `nns_reg` factor -predictor expansion is also supported with `factor_2_dummy=True` and explicit -`factor_levels=` metadata; it combines training `x` and `point_est` before -full-rank dummy expansion, matching installed R's `factor_2_dummy_FR` path. +`nns_m_reg` maps to R 13.3's `NNS.M.reg`, including its defaults +(`factor_2_dummy=True`). Predictors and the response are prepared exactly as in +`nns_reg` (see Regression): raw string and categorical columns are encoded by +the function itself (one-hot dummies, or integer codes with +`factor_2_dummy=False`), pandas column names are kept as the `RPM`, +`rhs.partitions` and `Fitted.xy` keys (`V1..Vp` for an unnamed matrix), and +`point_est` columns are matched by name. A string, bool or categorical +response, or one given `class_levels=`, is a classification target with +`class.levels` set; `type="class"` forces classification of a numeric +response. At least two encoded predictor columns are required. Outputs use +R's keys (`R2`, `rhs.partitions`, `RPM`, `Point.est`, `pred.int`, +`Fitted.xy`, `n.best`, `dist`, `class.levels`) with data.table objects as +dictionaries of NumPy arrays; intervals are the residual-quantile bands +described under Regression. `prepare_factor_predictors(...)` is a separate +legacy helper that builds a full-rank dummy design; it is not needed for +either function. Point estimates match installed R, including the one-row outsider behavior in the multi-point path where R drops matrix dimensions before extrapolating. @@ -253,28 +356,49 @@ the regression-point matrix and defaulting `n.best` to 1. ## Stack -`nns_stack` maps to R's numeric and deterministic classification `NNS.stack` -paths using the real `nns_reg` dimension-reduction and multivariate-regression -internals. `type="class"` is supported for numeric/logical/factor-like targets -and returns numeric class codes, not labels. Use `class_levels=` to reproduce R -factor level ordering. Raw string labels remain rejected unless explicit levels -are supplied. `balance=True` is supported for classification and follows R's -`downSample` + `upSample` structure: each non-empty class is downsampled to the -minority count without replacement, each class is upsampled to the majority -count with replacement, and the downsampled rows are concatenated before the -upsampled rows. Exact sampled-row parity with R is not expected because NNS Python -uses NumPy's RNG; `random_seed` is a NNS Python-only reproducibility convenience. -Numeric and class prediction intervals are supported and are combined by -installed R's weighted data.table arithmetic. For class stacks, single-method -`method=1` and `method=2` return the delegated interval table unchanged; when -`method=(1,2)`, the weighted final interval table is rounded with R's -`x %% 1 < 0.5` rule. -`ts_test` is supported and follows installed R's split exactly: CV training uses -the tail `ts_test` rows, while CV testing uses the earlier rows -`1:(n - ts_test)`. This is intentionally not changed even though it is -counterintuitive. R's `CV.size = NULL` samples a random value between 0.2 and -1/3; NNS Python uses a deterministic default of `0.25`. Pass `cv_size` explicitly for -exact R parity. +`nns_stack` ports R 13.3's `NNS.stack`: out-of-fold selection of `n.best` +(Method 1) and of the dimension-reduction count (Method 2), classification +thresholds and ensemble weights, using the real `nns_reg` internals. + +Randomness: every split and balancing draw reproduces R's Mersenne-Twister +`sample()` stream (`set.seed(seed)`, default `seed=123`), so a seeded run +matches R bit for bit. `seed=None` draws a seed from OS entropy (R continues +its global stream). `random_seed=` is an alias for `seed`. + +Splits follow R: `cv_size=None` (R's `CV.size = NULL`) with the default +`folds=5` is stratified (classification) or plain k-fold cross-validation; +`folds=1` with `cv_size=None` draws one holdout size from `runif(0.2, 1/3)`; +an explicit `cv_size` gives `folds` repeated holdouts of that size. `ts_test` +uses rolling chronological folds: each validation block of `ts_test` rows is +predicted from all earlier rows. `balance=True` (classification) down- and +up-samples each fitting split as R's `.balance_indices` does. + +Inputs follow R's data-frame rules. pandas column names are kept (an unnamed +matrix is `V1..Vp`, a vector `X1`), and `ivs_test` columns are matched to the +training columns by name (same set reordered, disjoint names positional, +partial overlap an error). Factor predictors are pandas categoricals (all +declared levels, as `levels()`), columns given `factor_levels=` (same forms as +`nns_reg`), or label columns (strings, including numeric-looking strings), +which use sorted levels; each becomes one dummy per level, fitted on the full +training levels, named `__`. + +Classification (`type="class"`, or automatic for string, bool, categorical +and two-valued numeric responses) returns numeric class codes `1..K`, with +`class.levels` as the label map. Codes follow declared order for a pandas +categorical or `class_levels=` (unused levels dropped, as R's `droplevels`), +sorted order for strings, `False < True` for bool, and sorted values for a +numeric response. The sorted order is R's factor order, which R's `NNS.stack` +and `NNS.boost` use for character responses and predictors. + +`NNS.dim.red.variables` lists the names of the predictors carried by the +final dimension reduction, ranked by absolute coefficient, and +`NNS.dim.red.coefficients` is a dict from encoded column name to coefficient +(R's named vector). `NNS.reg.n.best` is `NaN` when the all-points candidate +wins with a single (encoded) predictor, as R's `NA`, and the RPM row count in +the multivariate case. + +`nns_stack_fit` / `nns_stack_predict` run the cross-validated selection once and +reuse it for new test sets. The installed-R 13.0 Iris classification vignette with `folds=1` is a documented stack disparity rather than a NNS Python correctness target. On the `141:150` holdout, @@ -287,70 +411,40 @@ threshold rounding differs. Since R default `folds=5` also returns class code `3`, NNS Python keeps the behavior that matches the practical classification result instead of forcing installed-R-13.0 `folds=1` parity. -Factor predictor expansion is supported for `nns_stack(method=1)` and -`nns_stack(method=2)` with explicit `factor_levels=` metadata. NNS Python expands -training and test predictors together using the same full-rank dummy convention -as installed R's aligned train/test builder. Pure factor-predictor `method=2` -and `method=(1,2)` match installed R's fallback to method 1. Mixed -factor/numeric `method=2` uses the expanded numeric design directly. Mixed -factor/numeric `method=(1,2)` is supported for parity-covered cases that use -explicit `factor_levels` expansion. - ## Boost -`nns_boost` maps to R's numeric and deterministic classification `NNS.boost` -paths and uses the real `nns_reg` and `nns_stack` implementations. The -small-feature path (`n_features <= 10`, where R evaluates all feature -combinations) is supported. For `n_features > 10`, NNS Python follows R's stochastic -epoch structure: it samples learner-trial feature sets, builds a weighted -survivor feature pool, then samples epoch feature counts and survivor features -from that pool. Exact sampled-feature parity with R is not expected because -NNS Python uses NumPy's RNG, and `random_seed` is NNS Python-only. Installed R errors for -`threshold=` on this path because the threshold short-circuit leaves -`test.features` undefined, so NNS Python keeps that guard. `ts_test` is supported on -the stochastic path and follows R's separate epoch holdout split: initial -learner trials test rows `1:(n - ts_test)`, while epochs test the final -`2 * ts_test + 1` rows. `type="class"` returns numeric class codes, not labels; use -`class_levels=` to reproduce R factor level ordering. Raw string labels remain -rejected unless explicit levels are supplied. `balance=True` is supported for -classification and uses the same R-style `downSample` + `upSample` structure as -`nns_stack`; exact sampled-row parity with R is not expected. -Explicit-level factor predictors are supported through `factor_levels=`. NNS Python -integer-codes those columns before deterministic feature selection, matching -installed R's `data.matrix` conversion under NNS Python' positional-column -convention. Pass `None` for numeric columns in mixed predictor matrices, for -example `factor_levels=(["low", "mid", "high"], None)`. Multiple explicit-level -factor predictor columns use positional `X1`, `X2`, ... semantics; installed R -data frames with semantic column names sort columns alphabetically before -fitting, so callers should order NNS Python columns explicitly when reproducing those -named-data-frame cases. Numeric `pred_int` is supported and -delegates to `nns_stack(pred_int=...)`, matching installed R; it is deterministic -and does not use MC/meboot. `features_only=True` returns before the final stack -fit and ignores `pred_int`, matching R. Classification `pred_int` is supported -and delegates to final stack `method=1`, so interval bounds remain raw numeric -values. `ts_test` is supported for deterministic and stochastic boost paths. R -requires usable column names for matrix inputs; NNS Python uses positional numeric columns. As with `nns_stack`, R -samples a random CV size when `CV.size = NULL`; NNS Python uses deterministic -`cv_size=0.25` unless specified. For classification boost, final predictions, -feature weights, and feature frequencies are parity-tested against installed R -when balance is disabled and structurally tested when balance sampling is -enabled. The public `n.best` value is structural-only because R's final internal -`NNS.stack` call samples its own `CV.size = NULL` split, while NNS Python keeps the -deterministic stack default. +`nns_boost` ports R 13.3's `NNS.boost`. Learner trials fit multivariate +`nns_reg` models on feature subsets (all `2^p - 1` subsets when that is at most +`learner_trials`, otherwise distinct random subsets) against fresh holdouts; +`threshold` is the probability passed to `LPM.VaR` over the learner-trial +objective distribution (`0.8` / `0.2` by default, `extreme=True` uses the best +trial); `epochs` (default `2 * n`) re-test the surviving subsets; the final +estimate replicates the keeper features by their relative frequencies into one +`nns_stack(method=1, stack=False)`. `depth` is passed as `order`. All random +draws reproduce R's Mersenne-Twister stream (default `seed=123`), so seeded +runs match R; `cv_size=None` draws the holdout size from `runif(0.2, 1/3)` as R +does. + +Inputs are prepared as in `nns_stack`: pandas column names are kept and name +`feature.weights` / `feature.frequency` (`V1..Vp` for an unnamed matrix), test +columns are matched by name, and factor predictors (categoricals, +`factor_levels=`, or label columns with sorted levels) stay factors in every +learner fit; declared levels absent from the training data are dropped +(R's `droplevels`). A string, bool or categorical response is classified and +returns class codes with `class.levels` (declared order, `class_levels=`, or +sorted labels); a numeric response with `type="class"` returns its original +class values. The installed-R 13.0 Iris boost vignette remains a true parity gap, but not a quality target for exact output matching. On the same all-class-`3` holdout, installed R 13.0 balanced boost returns class code `1` for every row, while NNS Python balanced boost returns class code `2` for every row; both are wrong for that -example. Installed R 13.0 also does not accept the `folds` argument shown in the -rendered upstream overview for `NNS.boost`, so this example is tracked as -R-version/upstream-example drift plus a boost parity gap rather than evidence -that NNS Python should copy the installed-R balanced output. +example. ## Seasonality -`nns_seas` maps to installed R's non-plotting `NNS.seas` path and ignores -`plot`, consistent with other NNS Python ports. Inputs shorter than five observations +`nns_seas` maps to installed R's `NNS.seas`; `plot=True` draws the figure as a +side effect (see Plotting). Inputs shorter than five observations return R's sentinel period `0`. For mean-zero data, R falls back from coefficient of variation to `abs(acf1) ** -1`; NNS Python follows the same fallback and non-finite handling. Installed R can report harmonics rather than the visually @@ -369,32 +463,35 @@ recursive: each estimate is appended before the next horizon step. Plot arguments are ignored. Prediction intervals use `nns_mc` / `nns_meboot`; exact stochastic parity with R is not expected because RNG streams differ. `random_seed` is a NNS Python-only convenience for reproducible interval tests. -No-`pred_int` deterministic forecasts are parity-tested except where NNS Python -intentionally uses a more direct seasonal-lag weighting convention. -`seasonal_factor=True` uses only the first detected period from `nns_seas`, -matching `ARMA.seas.weighting(TRUE, ...)`; `seasonal_factor=False` uses the -selected `best_periods` rows. `dynamic=True` with numeric seasonal factors -raises with R's static-seasonality error. Constant-series behavior follows -installed R, including zero forecasts for automatic seasonality paths and `NaN` -forecasts for some explicit numeric-lag paths. Character `weights` with numeric -multi-lag seasonal factors is rejected because installed R errors during numeric -multiplication on that path. - -For explicit numeric multi-lag seasonal factors such as -`seasonal_factor=[132, 276]`, NNS Python intentionally weights each candidate lag by -the coefficient of variation of that actual lag's reverse component series. -Installed R NNS instead computes the coefficient-of-variation term with reverse -steps `1:length(seasonal.factor)` while still applying the observation penalty -to the actual lag values. NNS Python keeps the actual-lag weighting because it better -matches the documented idea that each supplied seasonal factor is weighted by -its own seasonality strength and observation count. The R-compatible difference -is covered by a strict xfail practical test rather than hidden. +No-`pred_int` deterministic forecasts are parity-tested against R. +As in R, `seasonal_factor=True` with a non-`None` `best_periods` is treated as +`seasonal_factor=False`: it weights the `best_periods` best periods from +`nns_seas` (the default `best_periods=1` selects the best period). +`seasonal_factor=True, best_periods=None` uses the first detected period +(weight 1), and `seasonal_factor=False, best_periods=None` weights all periods. `dynamic=True` with numeric seasonal factors +raises with R's static-seasonality error. Where the lag weights are `0/0` (a +constant series, or selected periods whose CV equals the variable CV), the +weights fall back to equal weights as in the patched R, so a constant series +forecasts its constant. Character `weights` such as `"equal"` weight every lag +equally, including with numeric multi-lag seasonal factors. Explicit numeric multi-lag seasonal factors weight +each lag by the coefficient of variation of that lag's own reverse component +series, as installed R does (`seasonal.factor[i]`). + +Arguments are validated up front where R either errors or returns garbage: +`training_set` must lie in `1..n`, numeric lags in `1..n`, numeric `weights` +need one entry per lag (R recycles silently), and automatic seasonality needs at +least 5 observations. `seasonal_factor=None` behaves like R's `NULL`, i.e. like +`False`. With `h=1` and `pred_int`, where R fails, the band is the forecast +widened by the linear residual term because there is no forecast path for +`nns_mc` to bootstrap. `nns_arma_optim(pred_int=None)` returns empty bounds like +R. `nns_arma_optim` is supported for the installed-R optimizer path. It greedily selects seasonal factors, evaluates the default co-moment-normalized objective, -then applies the same equal-weight, bias-shift, shrink, and smooth-regressed -variable checks as R. The optimizer's prediction intervals are deterministic -VaR bands around the in-sample optimizer errors; they are separate from +then applies the same equal-weight, shrink, smooth-regressed variable and +bias-shift checks as R, the bias last, on the returned model. The optimizer's +prediction intervals add the `lpm_var`/`upm_var` of that model's validation +residuals (actual - forecast) to the results; they are separate from `nns_arma(pred_int=...)`, which uses the Monte Carlo path. Custom Python `obj_fn` callables may be supplied, but R expression objects are not part of the Python API. `nns_var` is implemented for numeric matrix-like inputs (a pandas @@ -409,8 +506,10 @@ variables are found as in NNS 11.6: the dimension reduction threshold is cross-validated on the last `max(2h, ceil(0.2 n))` lagged training rows, and the regressors whose coefficient meets it are kept. The multivariate estimate is `nns_stack(method=1, stack=False)` on exactly those regressors (`nns_reg` with -each its own dimension and a cross-validated `n_best`), evaluated on the last -`h` lagged training rows (R's `tail(IV, h)`). `relevant_variables` is that +each its own dimension and a cross-validated `n_best`), trained on the observed +lagged rows and evaluated on the `h` forecast-period rows of the lag matrix built +from the series extended with the univariate forecasts (as in the patched R; 13.3 +evaluated on the last `h` observed rows, an in-sample refit). `relevant_variables` is that selection, so it lists exactly the regressors each multivariate estimate is built from. `obj_fn` (a `(predicted, actual) -> float` callable) and `objective` apply to both stages; without `obj_fn` the default MSE / @@ -418,16 +517,21 @@ co-movement objective falls back to MSE when it is undefined for every Method 1 candidate, as in R. Series whose observed minimum is positive have their interpolated values floored at zero. The `h == 0` path is normalized to a Python dictionary containing `interpolated_and_extrapolated` and `names` -rather than R's bare data-frame return. `ncores` and `nowcast` (row labelling -only in R) are accepted for API compatibility. +rather than R's bare data-frame return. `nowcast` (row labelling only in R) is +accepted for API compatibility; `ncores` caps the native threads (see Threads). ## Meboot `nns_meboot` maps to R's `NNS.meboot` maximum-entropy bootstrap algorithm and returns plain Python dictionaries instead of R's vectorized list-matrix wrapper. -Scalar `rho` returns one result dictionary; vector `rho` returns a list of result -dictionaries in R's vectorized order. `rho=None` follows installed R's empty -output behavior, and length-one input returns only `{"x": x}`. +`rho`, `target_drift` and `target_drift_scale` are vectorized like R's +`Vectorize`/`mapply`: the given arguments are recycled elementwise (not crossed) +and a list of result dictionaries is returned in that order; when all of them are +scalars a single result dictionary is returned. `rho=None` follows installed R's empty +output behavior, and length-one input returns only `{"x": x}`. When the detrended +residuals are all exactly zero (a constant or exactly linear series) `rho` has no +effect, as in R: the replicates are copies of the trend with their means spread by +`force_clt`, and a constant series gives constant columns. Exact replicate parity with R is not expected because NNS Python uses NumPy's random number generator and SciPy's optimizer while R uses its global RNG and @@ -451,8 +555,8 @@ are covered as smoke tests, but installed R behavior remains the parity source. ## Normalization -`nns_norm(x, linear=False)` maps to R's numeric matrix `NNS.norm` path with -plotting disabled. NNS Python accepts finite 2D arrays. `linear=True` uses R's +`nns_norm(x, linear=False)` maps to R's numeric matrix `NNS.norm` path +(`plot=True` draws R's `chart.type = "l"` panels). NNS Python accepts finite 2D arrays. `linear=True` uses R's mean-ratio scaling, while `linear=False` additionally weights scaling by absolute correlation for fewer than 10 columns and NNS dependence for 10 or more columns. @@ -481,34 +585,41 @@ binary rather than the higher-level classification intent. ## Classification -R classification paths work with numeric class codes. R factors become -1-indexed numeric codes in factor-level order and predictions are returned as -codes rather than decoded labels. NNS Python provides `factor_2_dummy`, -`factor_2_dummy_fr`, `encode_factor_codes`, and `prepare_factor_predictors`; -pass explicit `levels=` / `factor_levels=` to reproduce R factor level order -because NumPy arrays do not carry factor metadata. - -`nns_reg(..., type="class")`, `nns_m_reg(..., type="class")`, and -`nns_stack(..., type="class")` are supported for numeric, logical, and -factor-like targets. Use `class_levels=` when passing string/object labels so -NNS Python can reproduce R factor codes explicitly. Raw string classification remains -rejected where installed R errors or produces unusable `NA` conversions. -Predictions and point estimates are numeric class codes, not original labels, -matching installed R. Class confidence intervals are supported in `nns_reg` and -`nns_m_reg`; stack/boost class `pred_int` is supported through those regression -interval tables. +R classification paths work with numeric class codes: factor levels become +1-indexed codes in level order and predictions are returned as codes rather +than labels, with `class.levels` as the label map. NNS Python follows that +contract. pandas categoricals carry their level order; `class_levels=` (for +the response) and `factor_levels=` (for predictors) declare it explicitly for +plain arrays; undeclared string labels are coded in sorted order. String +labels are accepted directly by `nns_reg`, `nns_m_reg`, `nns_stack` and +`nns_boost`. The standalone helpers `factor_2_dummy`, `factor_2_dummy_fr` and +`encode_factor_codes` take `levels=` for the same purpose. Class confidence +intervals are supported in `nns_reg` and `nns_m_reg`; stack/boost class +`pred_int` is supported through those regression interval tables. ## Differentiation -`nns_diff` maps to R's scalar callable `NNS.diff` path with plotting and trace -output disabled. It returns a dictionary keyed by R's matrix row names and +`nns_diff` maps to R's scalar callable `NNS.diff` path with trace +output disabled (`plot=True` draws the function and secant panel). It returns a dictionary keyed by R's matrix row names and rounds results to `digits`, matching R's default output convention. `dy_dx(..., eval_point="overall")` maps to R's `dy.dx(..., eval.point = "overall")` path and returns the mean fitted gradient from unsmoothed `nns_reg`. Numeric `dy_dx` evaluation points use R's finite-difference grid -around smooth `nns_reg` point estimates and return a table-like dictionary with -`eval.point`, `first.derivative`, and `second.derivative`. Boundary-point -quirks follow installed R where covered by parity tests. +around smooth `nns_reg` point estimates, stepping by +`gravity(diff(sort(unique(x)))) * h` for each bandwidth `h` in R's `h_s` (the +spacing of the sorted values, as in the patched R, so row order does not matter), +and return a +table-like dictionary with `eval.point`, `first.derivative`, and +`second.derivative`. At `min(x)` or `max(x)` the first derivative is R's +one-sided slope and the second derivative is NaN. As in R, every position in +`h_s` counts once, so a bandwidth that repeats (n < 49) is averaged in each +time it appears. +`eval_point` must be given (a number, a vector, or `"overall"`). +`nns_diff` finds the inferred step like R's `uniroot(extendInt = "yes")` with +`tol = tol / 10`, and declares convergence when the bisection bracket on the +intercept `B` is narrower than `tol * max(|point|, |f(point) - B|)`, a +derivative error of about `tol * max(1, |f'(point)|)` (as in the patched R), so +iterations and convergence flags match R. NNS Python derivative parity is defined at the public input/output level, while preserving R's cumulative finite-difference perturbation pattern for `dy_d`. @@ -539,11 +650,26 @@ boundary row. NNS Python mirrors those boundary quirks for parity. ## ANOVA -`nns_anova` maps to R's non-plotting `NNS.ANOVA` paths. Binary comparisons +`nns_anova` maps to R's `NNS.ANOVA` paths (`plot=False` by default; see Plotting). Binary comparisons return a dictionary keyed like R's list output, aggregate multi-group comparisons return `{"Certainty": value}`, and `pairwise=True` returns R's -symmetric certainty matrix. Confidence interval bootstrapping is structurally -identical to R but uses NumPy RNG instead of R's `sample()`, so exact per-call -parity is not achievable; numeric values converge to the same population CI. -Pass `random_seed` for reproducible NNS Python results. Degenerate zero-variance -groups preserve R's `NaN` CDF/certainty convention. +symmetric certainty matrix. The grand statistic is the mean (or, with +`medians=True`, the median) of all observations pooled, as in the patched R. +`NaN` values are dropped from every group, as R does with `na.omit`; `±Inf` raises a `ValueError` (R's binary path fails on it in +`LPM.VaR`). Confidence-interval bootstrapping draws from a +reproduction of R's Mersenne-Twister `sample()` stream, so +`random_seed=s` gives the same effect-size bounds as `set.seed(s)` followed by +the same `NNS.ANOVA` call in R; without a seed the stream starts from fresh +entropy. `robust=True` follows R's resampling exactly: 100 independent bootstrap +resamples of each group, each from its own full sample and size (as in the patched +R; 13.3 resampled only the first `min(n_control, n_treatment)` positions of both +groups). It returns `Robust Certainty Estimate`, the median of the finite +resampled certainties (R's `median(na.rm = TRUE)`), plus R's `Lower % CI` and `Upper % CI` bounds, and no bounds when +`confidence_interval=None`. R names the effect-size keys of the robust output +`Effect_Size_LB.2.5%` etc. (an `unlist()` artefact); NNS Python keeps +`Effect_Size_LB`/`Effect_Size_UB`. One-sided effect-size intervals follow the +patched R: `tails="left"` gives `(treatment_lower - control_upper, Inf)` and +`tails="right"` gives `(-Inf, treatment_upper - control_lower)` (R 13.3 failed +on them). Degenerate zero-variance groups preserve R's `NaN` CDF/certainty +convention; where every robust certainty is `NaN`, R's VaR stops while NNS Python +returns `NaN` bounds. diff --git a/docs/parity.md b/docs/parity.md index 2b69703c..16b36bea 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -107,6 +107,28 @@ proxy-restricted runners that cannot install R), the cache cannot be regenerated live; rerun the local-source install and `scripts/regenerate_r_cache.py` on a host with R. +## Cache provenance: R 13.3 + patches + +The committed shards in `tests/_r_cache/` were generated from R NNS 13.3 plus +the fixes in OVVO-Financial/NNS pull requests #73-#92, not from the unpatched +13.3 release. Every shard +was rebuilt from that R build with the full test suite; shards for functions the +patches do not touch came out unchanged. Where Python has not mirrored one of +those fixes yet, the parity tests that depend on it are marked xfail until it is. +The entries that tied discrete modes reach (NNS.mode, NNS.part, NNS.reg and +NNS.boost) also carry the discrete NNS.mode tie rule of pull request #111, +refilled from the same build plus that change. The two NNS.copula entries +where every margin lies on one side of its target carry pull request #143 the same way, +the two univariate hazard and cumulative hazard NNS.CDF target values +carry #123, and the NNS.SD.cluster entries where nothing is peeled and the +dendrogram's leaf order carry #126. The NNS.MC.grid helper builds the grid as +R main does after #146, and the three dy.dx entries with repeated bandwidths +carry #147. + +To reproduce, install the patched package and point `R_LIBS` at it when +regenerating (`R_LIBS= python scripts/regenerate_r_cache.py ...`); the +`nns_version` stamp stays `13.3`. + ## Automated parity check (with deferred autofix agent) `NNS-python` automates R-behavior fidelity detection. The automated **fixing diff --git a/docs/sync_contract.md b/docs/sync_contract.md index b6938c27..30798761 100644 --- a/docs/sync_contract.md +++ b/docs/sync_contract.md @@ -32,6 +32,17 @@ Native snapshots are vendored under: extern/NNS-core ``` +`.github/workflows/sync-from-nns-core.yml` receives NNS-core's +`nns-core-updated` repository dispatch (payload +`core_commit, r_repo, r_commit, r_version, r_src_tree_hash`). It can also be run +by hand with a `core_commit`. It re-vendors the commit with +`scripts/sync_nns_core_snapshot.py`, which copies the `CMakeLists.txt`, +`include/` and `src/` of that commit and records `core_commit`. It then builds the +extension, runs the invariant and cached parity gates with and without +`NNS_DISABLE_NATIVE=1`, and opens a pull request, as a draft when a gate fails. +The receiver is in `workflow-baseline.yml`'s protected list; NNS-core's +`notify-python.yml` checks that it exists before dispatching. + ## R behavior fidelity rule Python API behavior is tested directly against live `OVVO-Financial/NNS` at the @@ -55,6 +66,7 @@ A Python sync PR is not complete until these pass: python -m pip install -e . --force-reinstall python -m pytest -q tests/invariants NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity +NNS_DISABLE_NATIVE=1 NNS_R_CACHE_ONLY=1 python -m pytest -q tests/invariants tests/parity python -m pytest -q tests/parity/test_r13_smoke.py python -m pytest -q tests/docs/test_vignette_examples.py ruff check . diff --git a/examples/vignettes/06_comparing_distributions.py b/examples/vignettes/06_comparing_distributions.py index 06bf8d8b..a55d799e 100644 --- a/examples/vignettes/06_comparing_distributions.py +++ b/examples/vignettes/06_comparing_distributions.py @@ -54,14 +54,16 @@ def anova_figure( [control_stat, treatment_stat, grand], color=["steelblue", "darkorange", "gray"], ) - lower = result.get("Lower Bound Robust Certainty") - upper = result.get("Upper Bound Robust Certainty") + lower = result.get("Lower 95% CI") + upper = result.get("Upper 95% CI") robust = result.get("Robust Certainty Estimate") label = f"Certainty = {float(np.asarray(result['Certainty'], dtype=float)):.4f}" if robust is not None: robust_value = float(np.asarray(robust, dtype=float)) - bounds = (float(np.asarray(lower, dtype=float)), float(np.asarray(upper, dtype=float))) - label += f"\nrobust = {robust_value:.4f} [{bounds[0]:.4f}, {bounds[1]:.4f}]" + label += f"\nrobust = {robust_value:.4f}" + if lower is not None and upper is not None: + bounds = (float(np.asarray(lower, dtype=float)), float(np.asarray(upper, dtype=float))) + label += f" [{bounds[0]:.4f}, {bounds[1]:.4f}]" axes[0].set_title(label) axes[0].set_ylabel("group statistic") diff --git a/examples/vignettes/07_clustering_and_regression.py b/examples/vignettes/07_clustering_and_regression.py index 6221fce1..b07ee075 100644 --- a/examples/vignettes/07_clustering_and_regression.py +++ b/examples/vignettes/07_clustering_and_regression.py @@ -6,6 +6,7 @@ Run directly to print the important returned structures and write the figures to ``examples/vignettes/output/07_clustering_and_regression``. """ + from __future__ import annotations import itertools @@ -179,13 +180,19 @@ def main() -> None: iris_x, iris_y, levels = load_iris() feature_names = ["Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width"] dim_red = nns_reg(iris_x, iris_y, dim_red_method="cor", class_levels=levels) - equation = dim_red["equation"] - coefficients = np.asarray(equation["Coefficient"], dtype=float) - show("Dimension reduction equation", equation) - terms = " + ".join( - f"{coefficients[i]:.3f}*{feature_names[i]}" for i in range(len(feature_names)) - ) - print(f"\nSpecies = ({terms}) / {coefficients[-1]:g}") + + # R14 nominal classification builds one signed index per class indicator, + # rather than regressing the arbitrary numeric species codes themselves. + def show_class_indices(equations: dict) -> None: + for label, equation in equations.items(): + coefficients = np.asarray(equation["Coefficient"], dtype=float) + terms = " + ".join( + f"{coefficients[i]:.3f}*{feature_names[i]}" for i in range(len(feature_names)) + ) + print(f"\n{label} index = ({terms}) / {coefficients[-1]:g}") + + show("Class-specific dimension reduction equations", dim_red["equation"]) + show_class_indices(dim_red["equation"]) # %% [markdown] # ### Threshold @@ -194,13 +201,8 @@ def main() -> None: # below the requested level, reducing the denominator accordingly. subsection("Dimension reduction with threshold") reduced = nns_reg(iris_x, iris_y, dim_red_method="cor", threshold=0.75, class_levels=levels) - equation = reduced["equation"] - coefficients = np.asarray(equation["Coefficient"], dtype=float) - show("Thresholded dimension reduction equation", equation) - terms = " + ".join( - f"{coefficients[i]:.3f}*{feature_names[i]}" for i in range(len(feature_names)) - ) - print(f"\nSpecies = ({terms}) / {coefficients[-1]:g}") + show("Thresholded class-specific equations", reduced["equation"]) + show_class_indices(reduced["equation"]) point_est = nns_reg( iris_x, @@ -215,8 +217,8 @@ def main() -> None: # %% [markdown] # # Classification # - # ``type = "CLASS"`` rounds estimates to class values. The base category of - # the response must be 1, not 0. + # ``type = "CLASS"`` selects nominal classes from estimated probabilities. + # Declared factor levels give the code-to-label map used in this example. subsection("Classification") classified = nns_reg(iris_x, iris_y, type="CLASS", point_est=iris_x[:10], class_levels=levels) show( diff --git a/extern/NNS-core/CMakeLists.txt b/extern/NNS-core/CMakeLists.txt index 818cc236..cb51bba0 100644 --- a/extern/NNS-core/CMakeLists.txt +++ b/extern/NNS-core/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.18) # Project name and version lockstep with NNS releases -project(nnscore VERSION 13.0.0 LANGUAGES CXX) +project(nnscore VERSION 13.3.0 LANGUAGES CXX) # Enforce C++17 Standard (Required for std::optional and modern standard library features) set(CMAKE_CXX_STANDARD 17) @@ -19,6 +19,9 @@ add_library(nnscore src/seasonality.cpp src/fast_lm.cpp src/internal_functions.cpp + src/mreg_setup.cpp + src/mreg_predict.cpp + src/stack_fast.cpp ) # Set the public include directories so consumers can `#include "nns/nns.hpp"` @@ -37,12 +40,17 @@ target_link_libraries(nnscore PUBLIC Threads::Threads) option(NNSCORE_BUILD_TESTS "Build Catch2 unit and conformance tests" ON) option(NNSCORE_BUILD_PYTHON "Build Python bindings via nanobind" OFF) +option(NNSCORE_BUILD_FIDELITY "Build the live-R fidelity evaluation helper (nns_eval)" OFF) if(NNSCORE_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/cpp/CMakeLists.txt") enable_testing() add_subdirectory(tests/cpp) endif() +if(NNSCORE_BUILD_FIDELITY AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fidelity/CMakeLists.txt") + add_subdirectory(tests/fidelity) +endif() + if(NNSCORE_BUILD_PYTHON) add_subdirectory(bindings/python) endif() \ No newline at end of file diff --git a/extern/NNS-core/include/nns/internal_functions.hpp b/extern/NNS-core/include/nns/internal_functions.hpp index 5b1211cc..d40483a7 100644 --- a/extern/NNS-core/include/nns/internal_functions.hpp +++ b/extern/NNS-core/include/nns/internal_functions.hpp @@ -58,7 +58,11 @@ struct ForecastVectors { std::vector> forecast_index; }; -TimeSeriesVectors generate_vectors(const double* x, std::size_t n, const int* lags, std::size_t num_lags); +/// Port of generate.vectors(x, l, len). `len` (R 13.3) lets a caller treat +/// only the first `len` elements of a pre-allocated buffer as the series; +/// len < 0 (default) uses all n elements. Throws if len > n. +TimeSeriesVectors generate_vectors(const double* x, std::size_t n, const int* lags, std::size_t num_lags, + long long len = -1); ForecastVectors generate_lin_vectors(const double* x, std::size_t n, int l, int h = 1); // --- ARMA Seasonality Weighting --- @@ -70,6 +74,7 @@ struct ARMAWeights { /// Computes ARMA seasonality weighting. /// Replaces the dynamic R DataFrame lookup with explicit arrays. +/// Weights are equal when (varcovar - covar) / sqrt(period) sums to 0. ARMAWeights arma_seas_weighting(const double* periods, const double* covar, const double* varcovar, std::size_t m); // --- Maximum Entropy Bootstrap (MEBoot) --- diff --git a/extern/NNS-core/include/nns/mreg.hpp b/extern/NNS-core/include/nns/mreg.hpp new file mode 100644 index 00000000..39bb2d67 --- /dev/null +++ b/extern/NNS-core/include/nns/mreg.hpp @@ -0,0 +1,117 @@ +// include/nns/mreg.hpp +// +// Portable ports of the R 13.1-13.3 NNS.M.reg / NNS.stack native kernels: +// src/NNS_mreg_setup.cpp -> duplicate_column_map, mreg_setup +// src/NNS_mreg_predict.cpp -> mreg_predict, mreg_predict_path +// src/NNS_stack_fast.cpp -> xstar_path +// +// All matrices are column-major (element (i, j) at i + j * nrow), like R. +// Indices returned by these functions are 0-based (R returns 1-based ones). +// +// SPDX-License-Identifier: GPL-3.0-only +#ifndef NNS_MREG_HPP +#define NNS_MREG_HPP + +#include +#include +#include + +namespace nns { + +/// Port of NNS_duplicate_column_map_cpp. For each column j of the n x p +/// matrix X, the index of the first column whose values are byte-identical +/// to column j (j itself if there is none). Columns containing NaN never +/// match another column, exactly as upstream. +std::vector duplicate_column_map(const double* X, std::size_t n, + std::size_t p); + +/// Reducer codes of NNS_mreg_setup_cpp (R: noise.reduction). +enum MregReducer : int { + kMregMean = 0, // "mean" + kMregMedian = 1, // "median" + kMregMode = 2, // "mode": NNS.mode (class responses: most frequent value) + kMregGravity = 3, // "off": NNS.gravity +}; + +struct MregSetupResult { + std::size_t n_groups = 0; + std::size_t p = 0; + /// n_groups x (p + 1) column-major regression point matrix: the reduced + /// X1..Xp of each group, then the reduced y. + std::vector rpm; + /// Group id of every input row ("i1.i2...ip", findInterval ids). + std::vector ids; + /// Group id of every RPM row, sorted lexicographically (as upstream). + std::vector row_ids; +}; + +/// Port of NNS_mreg_setup_cpp. X is n x p, y has length n, and +/// boundaries[j] holds the sorted partition boundaries of column j. Each row +/// is assigned the id built from findInterval(X[i, j], boundaries[j], +/// rightmost.closed = TRUE) over the columns; rows sharing an id are reduced +/// with `reducer_code` (y uses the exact-frequency mode when is_class). +/// Non-finite values are dropped before reducing (NaN if none remain). +MregSetupResult mreg_setup(const double* X, std::size_t n, std::size_t p, + const double* y, + const std::vector>& boundaries, + int reducer_code, bool is_class); + +/// Distance codes shared by the M.reg kernels (R: dist). +enum MregDistance : int { + kMregDistNNS = 0, // sum(|z| + z^2) over range-normalised columns + kMregDistL2 = 1, // sqrt(sum(z^2)) + kMregDistL1 = 2, // sum(|z|) + kMregDistFactor = 3, // Hamming distance over encoded columns +}; + +/// Port of NNS_mreg_predict_v2_cpp: the k-nearest-neighbour prediction for +/// each of the m rows of Xtest (m x p) against the regression points rpm_x +/// (n x p) with fitted values yhat (length n). k = 1 aggregates all exact +/// nearest-distance ties (gravity, or the class mode); k > 1 uses the +/// eight-component ensemble weights. mins/maxs (length p) give the column +/// ranges used for normalisation. k is clipped to n. +/// +/// The density components use closed forms instead of Rmath (dt is used up +/// to its normalising constant, which cancels), so results agree with R to +/// floating-point rounding (about 1e-15), not bit for bit. +/// +/// n_threads follows nns::parallel_for (<= 0: library default). +/// Throws std::invalid_argument on inconsistent arguments. +std::vector mreg_predict(const double* rpm_x, std::size_t n, std::size_t p, + const double* yhat, const double* Xtest, + std::size_t m, int k, int dist_code, + const double* mins, const double* maxs, + bool is_class, int n_threads = -1); + +/// Port of NNS_mreg_predict_path_v2_cpp: predictions for every k = 1..kmax +/// (kmax clipped to n). Returns an m x kmax column-major matrix; column k-1 +/// equals mreg_predict(..., k, ...). +std::vector mreg_predict_path(const double* rpm_x, std::size_t n, std::size_t p, + const double* yhat, const double* Xtest, + std::size_t m, int kmax, int dist_code, + const double* mins, const double* maxs, + bool is_class, int n_threads = -1); + +struct XstarPathResult { + std::size_t n_train = 0, n_test = 0, p = 0; + std::vector train; // n_train x p, column-major + std::vector test; // n_test x p, column-major + std::vector denominator; // length p + std::vector column_order; // 0-based, as passed in + std::vector representative; // 0-based first identical column +}; + +/// Port of NNS_xstar_path_cpp (NNS.stack X* projection path). Column m of +/// the output is the running sum over the first m + 1 columns in +/// `column_order` (0-based) of design * coefficient, divided by the number of +/// non-zero coefficients so far (all coefficients count as 1 when every one +/// is zero). representative[m] is the first earlier output column identical +/// to column m in both train and test. +XstarPathResult xstar_path(const double* train_design, std::size_t n_train, + const double* test_design, std::size_t n_test, + std::size_t p, const double* coefficients, + const std::size_t* column_order, int n_threads = -1); + +} // namespace nns + +#endif // NNS_MREG_HPP diff --git a/extern/NNS-core/include/nns/nns.hpp b/extern/NNS-core/include/nns/nns.hpp index 55b9281a..b9361fd8 100644 --- a/extern/NNS-core/include/nns/nns.hpp +++ b/extern/NNS-core/include/nns/nns.hpp @@ -18,5 +18,6 @@ #include "nns/seasonality.hpp" #include "nns/fast_lm.hpp" #include "nns/internal_functions.hpp" +#include "nns/mreg.hpp" #endif // NNS_UMBRELLA_HPP diff --git a/extern/NNS-core/include/nns/parallel.hpp b/extern/NNS-core/include/nns/parallel.hpp index e91d7222..e8a7235a 100644 --- a/extern/NNS-core/include/nns/parallel.hpp +++ b/extern/NNS-core/include/nns/parallel.hpp @@ -4,31 +4,87 @@ #ifndef NNS_PARALLEL_HPP #define NNS_PARALLEL_HPP +#include #include +#include +#include +#include #include #include namespace nns { +namespace detail { + +// 0 = automatic (NNS_NUM_THREADS environment variable, else the hardware +// concurrency). Shared by every translation unit (inline function static). +inline std::atomic& thread_setting() { + static std::atomic setting{0}; + return setting; +} + +inline int automatic_thread_count() { + static const int count = [] { + if (const char* env = std::getenv("NNS_NUM_THREADS")) { + const long v = std::strtol(env, nullptr, 10); + if (v > 0) return static_cast(v); + } + const unsigned hw = std::thread::hardware_concurrency(); + return hw > 0 ? static_cast(hw) : 1; + }(); + return count; +} + +} // namespace detail + +/// Total estimated work (items x work_per_item, roughly in inner-loop +/// iterations) below which parallel_for runs serially. Spawning and joining a +/// thread costs tens of microseconds, which dominated small calls. +constexpr std::size_t kMinParallelWork = 200000; + +/// Set the default number of worker threads used when a kernel is called with +/// n_threads <= 0 (the default everywhere). n <= 0 restores the automatic +/// choice: the NNS_NUM_THREADS environment variable if set to a positive +/// integer, else std::thread::hardware_concurrency(). Thread-safe; intended +/// for bindings that expose an `ncores` / `set_num_threads` option, and for +/// running inside process pools without oversubscribing the machine. +inline void set_num_threads(int n) { detail::thread_setting().store(n > 0 ? n : 0); } + +/// The number of worker threads used for n_threads <= 0 (always >= 1). +inline int get_num_threads() { + const int n = detail::thread_setting().load(); + return n > 0 ? n : detail::automatic_thread_count(); +} + /// A lightweight lambda-driven static parallel iteration system. -/// Replaces RcppParallel::parallelFor. +/// Replaces RcppParallel::parallelFor. Every index is processed exactly once +/// by `func(chunk_begin, chunk_end)`; kernels write disjoint output slots, so +/// results never depend on the thread count. An exception thrown by `func` +/// on any thread is rethrown on the calling thread once every worker has +/// joined (the first one if several throw). /// /// @param begin Starting loop index (inclusive). /// @param end Ending loop index (exclusive). /// @param func A callable matching void(std::size_t worker_begin, std::size_t worker_end). -/// @param n_threads Number of requested worker threads. If <= 0, hardware concurrency is used. +/// @param n_threads Number of requested worker threads. If <= 0, get_num_threads() is used. +/// @param work_per_item Estimated cost of one index (e.g. the inner loop +/// length). When > 0 and (end - begin) * work_per_item < kMinParallelWork, +/// the loop runs serially on the calling thread. 0 means unknown (no +/// serial fallback beyond the thread count). template -void parallel_for(std::size_t begin, std::size_t end, Func&& func, int n_threads = -1) { - std::size_t total_elements = end - begin; - if (total_elements == 0) return; +void parallel_for(std::size_t begin, std::size_t end, Func&& func, int n_threads = -1, + std::size_t work_per_item = 0) { + if (end <= begin) return; + const std::size_t total_elements = end - begin; - // Determine available worker count - unsigned int hw = std::thread::hardware_concurrency(); - std::size_t worker_count = (n_threads <= 0) ? (hw > 0 ? hw : 1) : static_cast(n_threads); + std::size_t worker_count = + static_cast(n_threads > 0 ? n_threads : get_num_threads()); // Prevent over-threading on tiny tasks - if (worker_count > total_elements) { - worker_count = total_elements; + if (worker_count > total_elements) worker_count = total_elements; + if (work_per_item > 0 && total_elements < kMinParallelWork && + total_elements * work_per_item < kMinParallelWork) { + worker_count = 1; } // Fallback cleanly to synchronous serial loop if single-threaded @@ -37,34 +93,51 @@ void parallel_for(std::size_t begin, std::size_t end, Func&& func, int n_threads return; } - std::size_t chunk_size = total_elements / worker_count; - std::size_t remainder = total_elements % worker_count; + const std::size_t chunk_size = total_elements / worker_count; + const std::size_t remainder = total_elements % worker_count; + + std::exception_ptr error; + std::mutex error_mu; + auto run = [&error, &error_mu](auto& f, std::size_t b, std::size_t e) { + try { + f(b, e); + } catch (...) { + std::lock_guard lock(error_mu); + if (!error) error = std::current_exception(); + } + }; std::vector threads; threads.reserve(worker_count - 1); + // A joinable std::thread terminates on destruction, so join on every exit path. + struct Joiner { + std::vector& threads; + void join() { + for (auto& t : threads) + if (t.joinable()) t.join(); + } + ~Joiner() { join(); } + } joiner{threads}; std::size_t current_begin = begin; - for (std::size_t i = 0; i < worker_count; ++i) { - std::size_t current_end = current_begin + chunk_size + (i < remainder ? 1 : 0); - + const std::size_t current_end = current_begin + chunk_size + (i < remainder ? 1 : 0); // Main thread executes the final piece directly to eliminate thread spawn latency if (i == worker_count - 1) { - func(current_begin, current_end); + run(func, current_begin, current_end); } else { - threads.emplace_back(func, current_begin, current_end); + // Each worker gets its own copy of func, as std::thread(func, ...) made. + threads.emplace_back([&run, f = func, current_begin, current_end]() mutable { + run(f, current_begin, current_end); + }); current_begin = current_end; } } - // Collect active workers - for (auto& t : threads) { - if (t.joinable()) { - t.join(); - } - } + joiner.join(); + if (error) std::rethrow_exception(error); } -} // namespace nns +} // namespace nns -#endif // NNS_PARALLEL_HPP \ No newline at end of file +#endif // NNS_PARALLEL_HPP diff --git a/extern/NNS-core/include/nns/partial_moments.hpp b/extern/NNS-core/include/nns/partial_moments.hpp index 2ba51ef8..adf7eaf0 100644 --- a/extern/NNS-core/include/nns/partial_moments.hpp +++ b/extern/NNS-core/include/nns/partial_moments.hpp @@ -1,6 +1,11 @@ // include/nns/partial_moments.hpp // // SPDX-License-Identifier: GPL-3.0-only + +// ---- NNS-CORE PROVENANCE (managed by scripts/sync_adapter.py; do not hand-edit) ---- +// NNS-R-SOURCE: src/partial_moments.cpp @ 82a92eb45c1a1892acce82e66175cf2e29434b65 +// NNS-R-SOURCE: src/partial_moments.h @ 82a92eb45c1a1892acce82e66175cf2e29434b65 +// ---- end NNS-CORE PROVENANCE ---- #ifndef NNS_PARTIAL_MOMENTS_HPP #define NNS_PARTIAL_MOMENTS_HPP @@ -18,6 +23,9 @@ struct PMMatrixResult { std::size_t dim = 0; }; +// All degree arguments must be >= 0; every function below throws +// std::invalid_argument for a negative or NaN degree. + double lpm(double degree, double target, const double* x, std::size_t n); double upm(double degree, double target, const double* x, std::size_t n); diff --git a/extern/NNS-core/include/nns/stochastic_dominance.hpp b/extern/NNS-core/include/nns/stochastic_dominance.hpp index 93d78020..a38b44c3 100644 --- a/extern/NNS-core/include/nns/stochastic_dominance.hpp +++ b/extern/NNS-core/include/nns/stochastic_dominance.hpp @@ -10,25 +10,36 @@ namespace nns { // --- Univariate Dominance Tests --- -// Faithful ports of NNS_FSD_uni_cpp / NNS_SSD_uni_cpp / NNS_TSD_uni_cpp. -// Return 1 if x dominates y, otherwise 0. Identical samples never dominate. +// Ports of NNS_FSD_uni_cpp / NNS_SSD_uni_cpp / NNS_TSD_uni_cpp. +// Return 1 if x dominates y, otherwise 0. Identical samples, and samples +// whose curves coincide, never dominate. x and y may differ in length. // NaN (missing) values raise std::invalid_argument, matching the upstream // "You have some missing values, please address." stop; +/-Inf is permitted. +// An empty sample also raises std::invalid_argument. /// First-degree Stochastic Dominance (Univariate) /// @param x Pointer to the first array. +/// @param n_x Length of x. /// @param y Pointer to the second array. -/// @param n Length of the arrays. +/// @param n_y Length of y. /// @param discrete Treat data as discrete (ECDF compare) or continuous /// (degree-1 LPM-ratio compare), matching upstream type = /// "discrete"/"continuous". -int fsd_uni(const double* x, const double* y, std::size_t n, bool discrete); +int fsd_uni(const double* x, std::size_t n_x, const double* y, std::size_t n_y, + bool discrete); /// Second-degree Stochastic Dominance (Univariate) -int ssd_uni(const double* x, const double* y, std::size_t n); +int ssd_uni(const double* x, std::size_t n_x, const double* y, std::size_t n_y); /// Third-degree Stochastic Dominance (Univariate) -int tsd_uni(const double* x, const double* y, std::size_t n); +int tsd_uni(const double* x, std::size_t n_x, const double* y, std::size_t n_y); + +/// Equal-length forms, x and y both of length n. +inline int fsd_uni(const double* x, const double* y, std::size_t n, bool discrete) { + return fsd_uni(x, n, y, n, discrete); +} +inline int ssd_uni(const double* x, const double* y, std::size_t n) { return ssd_uni(x, n, y, n); } +inline int tsd_uni(const double* x, const double* y, std::size_t n) { return tsd_uni(x, n, y, n); } // --- Pairwise Dominance Matrix --- diff --git a/extern/NNS-core/include/nns/version.hpp b/extern/NNS-core/include/nns/version.hpp index 9e1fbc83..ffeefb9e 100644 --- a/extern/NNS-core/include/nns/version.hpp +++ b/extern/NNS-core/include/nns/version.hpp @@ -5,9 +5,9 @@ #define NNS_VERSION_HPP #define NNS_CORE_VERSION_MAJOR 13 -#define NNS_CORE_VERSION_MINOR 0 +#define NNS_CORE_VERSION_MINOR 3 #define NNS_CORE_VERSION_PATCH 0 -#define NNS_CORE_VERSION "13.0.0" +#define NNS_CORE_VERSION "13.3.0" #endif // NNS_VERSION_HPP \ No newline at end of file diff --git a/extern/NNS-core/src/central_tendencies.cpp b/extern/NNS-core/src/central_tendencies.cpp index 36a743e1..adc8fb4e 100644 --- a/extern/NNS-core/src/central_tendencies.cpp +++ b/extern/NNS-core/src/central_tendencies.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -73,26 +74,66 @@ void quartiles_like_R_code(const std::vector& xs, double& q1, double& q2 } } -// Minimal replacement for NNS_bin used by mode/gravity +// Minimal replacement for NNS_bin used by mode/gravity. Bins are +// [origin + k*width, origin + (k+1)*width), k = 0 .. nbins-1, with +// nbins = floor((max - origin) / width + 1e-12) + 1, as in NNS 13.3. +// +// Gravity and mode only read counts within 6 bins of a non-empty bin, so runs +// of more than kMaxEmptyRun empty bins keep only their first and last +// kMaxEmptyRun / 2 bins: results are bit-identical and memory is O(n). A width +// so small that the bin count exceeds 2^52 is widened to (max - origin) / 2^52. +constexpr std::int64_t kMaxEmptyRun = 16; +constexpr double kMaxBinIndex = 4503599627370496.0; // 2^52 + void simple_bin_counts(const std::vector& xs, double width, double origin, std::vector& bin_names, std::vector& counts) { - const int l = static_cast(xs.size()); - if (l == 0) { bin_names.clear(); counts.clear(); return; } - + bin_names.clear(); + counts.clear(); + if (xs.empty()) return; + const double xmax = xs.back(); - int nbins = static_cast(std::floor((xmax - origin) / width + 1e-12)) + 1; - if (nbins < 1) nbins = 1; - - bin_names.resize(nbins); - for (int k = 0; k < nbins; ++k) bin_names[k] = origin + k * width; - - counts.assign(nbins, 0); + double last = std::floor((xmax - origin) / width + 1e-12); + if (!(last < kMaxBinIndex)) { + width = (xmax - origin) / kMaxBinIndex; + last = std::floor((xmax - origin) / width + 1e-12); + if (!(last < kMaxBinIndex)) last = kMaxBinIndex; + } + const std::int64_t nbins = (last >= 0.0) ? static_cast(last) + 1 : 1; + const std::int64_t cap = static_cast(xs.size()) * (kMaxEmptyRun + 1) + kMaxEmptyRun; + bin_names.reserve(static_cast(std::min(nbins, cap))); + counts.reserve(static_cast(std::min(nbins, cap))); + + auto push_bin = [&](std::int64_t k, int c) { + bin_names.push_back(origin + static_cast(k) * width); + counts.push_back(c); + }; + auto push_empty_run = [&](std::int64_t first, std::int64_t last_bin) { + const std::int64_t len = last_bin - first + 1; + if (len <= 0) return; + if (len <= kMaxEmptyRun) { + for (std::int64_t k = first; k <= last_bin; ++k) push_bin(k, 0); + return; + } + const std::int64_t half = kMaxEmptyRun / 2; + for (std::int64_t k = first; k < first + half; ++k) push_bin(k, 0); + for (std::int64_t k = last_bin - half + 1; k <= last_bin; ++k) push_bin(k, 0); + }; + + // xs is sorted, so bin indices are non-decreasing. + const double top = static_cast(nbins - 1); + std::int64_t prev = -1; for (double v : xs) { - int idx = static_cast(std::floor((v - origin) / width)); - if (idx < 0) idx = 0; - if (idx >= nbins) idx = nbins - 1; - counts[idx] += 1; + const double q = std::floor((v - origin) / width); + const std::int64_t k = (q < 0.0) ? 0 : (q >= top ? nbins - 1 : static_cast(q)); + if (k == prev) { + ++counts.back(); + continue; + } + push_empty_run(prev + 1, k - 1); + push_bin(k, 1); + prev = k; } + push_empty_run(prev + 1, nbins - 1); } // Triangular smoothing helper: 7-tap [1,2,3,4,3,2,1] with mirrored edges @@ -103,9 +144,12 @@ void smooth_counts_tri7(const std::vector& counts, std::vector& smo smooth.assign(n, 0.0); if (n == 0) return; + // Mirror at the edges. The clamp only binds for n <= 3 bins, whose smoothed + // values are unused (the peak scan needs n >= 7). auto at = [&](int idx) -> int { - if (idx < 0) return counts[-idx]; - if (idx >= n) return counts[2 * n - 2 - idx]; + if (idx < 0) idx = -idx; + if (idx >= n) idx = 2 * n - 2 - idx; + idx = std::min(std::max(idx, 0), n - 1); return counts[idx]; }; @@ -286,24 +330,24 @@ std::vector mode(const double* x_in, std::size_t n, bool discrete, bool return {nearest_int_half_up(med)}; } - std::unordered_map freq; + // Key on the rounded value itself: NNS 13.3's int key overflowed beyond the + // int range. "+ 0.0" folds -0.0 into the 0 key. + std::unordered_map freq; freq.reserve(l * 2u); - for (double v : xnum) ++freq[static_cast(nearest_int_half_up(v))]; + for (double v : xnum) ++freq[nearest_int_half_up(v) + 0.0]; int maxf = 0; for (const auto& kv : freq) if (kv.second > maxf) maxf = kv.second; - std::vector modes_int; + std::vector modes_int; for (const auto& kv : freq) if (kv.second == maxf) modes_int.push_back(kv.first); std::sort(modes_int.begin(), modes_int.end()); if (multi) { - std::vector out(modes_int.size()); - for (std::size_t i = 0; i < modes_int.size(); ++i) out[i] = static_cast(modes_int[i]); - return out; + return modes_int; } else { long double sum = 0.0L; - for (int m : modes_int) sum += static_cast(m); + for (double m : modes_int) sum += static_cast(m); double mean_modes = modes_int.empty() ? kNaN : static_cast(sum / static_cast(modes_int.size())); return {mean_modes}; } @@ -365,7 +409,8 @@ std::vector mode(const double* x_in, std::size_t n, bool discrete, bool } if (!peak_idx.empty()) { - std::sort(peak_idx.begin(), peak_idx.end(), [&](int a, int b){ return cs[a] > cs[b]; }); + // Equal heights keep bin order, so the kept peaks don't depend on the sort implementation. + std::stable_sort(peak_idx.begin(), peak_idx.end(), [&](int a, int b){ return cs[a] > cs[b]; }); std::vector kept; for (int idx : peak_idx) { bool too_close = false; @@ -394,11 +439,14 @@ std::vector mode(const double* x_in, std::size_t n, bool discrete, bool std::sort(out.begin(), out.end()); return out; } else { - int best_t = 0; - for (std::size_t t = 1; t < kept.size(); ++t) { - if (cs[kept[t]] > cs[kept[best_t]]) best_t = t; - } - return {centers[best_t]}; + // Equally high peaks are averaged, as the tied-bin fallback below does. + double best_h = cs[kept[0]]; + for (std::size_t t = 1; t < kept.size(); ++t) best_h = std::max(best_h, cs[kept[t]]); + long double sum = 0.0L; + int nt = 0; + for (std::size_t t = 0; t < kept.size(); ++t) + if (cs[kept[t]] == best_h) { sum += static_cast(centers[t]); ++nt; } + return {static_cast(sum / nt)}; } } } diff --git a/extern/NNS-core/src/dependence.cpp b/extern/NNS-core/src/dependence.cpp index dd2ff4d4..e26a4451 100644 --- a/extern/NNS-core/src/dependence.cpp +++ b/extern/NNS-core/src/dependence.cpp @@ -1,6 +1,6 @@ // src/dependence.cpp // -// Implementation extracted from NNS 13.0 NNS_dep.cpp. Decoupled from Rcpp. +// Implementation extracted from NNS NNS_dep.cpp (synced to R 13.3, 82a92eb). Decoupled from Rcpp. // // SPDX-License-Identifier: GPL-3.0-only #include "nns/dependence.hpp" @@ -61,7 +61,9 @@ double copula_signed(const std::vector& xv, const std::vector& y if (dx > 0.0 && dy > 0.0) d0_cupm += 1.0; if (dx <= 0.0 && dy <= 0.0) d0_clpm += 1.0; - if (!((dx < 0.0 && dy < 0.0) || (dx > 0.0 && dy > 0.0))) + // Discordant = neither co-lower (<= target) nor co-upper (> target), so + // the degree-0 counts partition the cell. + if (!((dx <= 0.0 && dy <= 0.0) || (dx > 0.0 && dy > 0.0))) dpm_d0_count += 1.0; if (dx >= 0.0 && dy >= 0.0) { @@ -77,8 +79,12 @@ double copula_signed(const std::vector& xv, const std::vector& y } double inv_n = 1.0 / static_cast(n); - double d0_Co = (d0_cupm + d0_clpm) * inv_n; - if (d0_Co == 1.0 || d0_Co == 0.0) return 1.0; + // Compare counts, not count * (1/n): 49 * (1/49) != 1. All + // concordant: perfect positive dependence; all discordant: negative. + double d0_concordant = d0_cupm + d0_clpm; + double d0_Co = d0_concordant * inv_n; + if (d0_concordant == static_cast(n)) return 1.0; + if (d0_concordant == 0.0) return -1.0; double c1_total = c1_cupm + c1_clpm + c1_dpm; double co_d1 = c1_total > 0.0 ? (c1_cupm + c1_clpm) / c1_total : 0.0; @@ -86,7 +92,10 @@ double copula_signed(const std::vector& xv, const std::vector& y double dpm_d1 = c1_total > 0.0 ? c1_dpm / c1_total : 0.0; constexpr double indep_Co = 0.5; - constexpr double indep_D = 0.75; + // Independence null for the discordant partial moment (R 13.1): a point is + // concordant when all-below OR all-above the target, so under independence + // P(discordant) = 1 - 2 * 0.5^2 = 0.5, not 0.75. + constexpr double indep_D = 0.5; double discrete_dep = std::min(1.0, std::max(0.0, std::abs(d0_Co - indep_Co) / indep_Co)); double continuous_dep = std::min(1.0, std::max(0.0, std::abs(co_d1 - indep_Co) / indep_Co)); @@ -113,7 +122,9 @@ double copula_degree0_unsigned(const std::vector& xv, const std::vector< double dy = yv[i] - ty; if (dx > 0.0 && dy > 0.0) d0_cupm += 1.0; if (dx <= 0.0 && dy <= 0.0) d0_clpm += 1.0; - if (!((dx < 0.0 && dy < 0.0) || (dx > 0.0 && dy > 0.0))) + // Discordant = neither co-lower (<= target) nor co-upper (> target), so + // the degree-0 counts partition the cell. + if (!((dx <= 0.0 && dy <= 0.0) || (dx > 0.0 && dy > 0.0))) dpm_d0_count += 1.0; } @@ -122,7 +133,10 @@ double copula_degree0_unsigned(const std::vector& xv, const std::vector< double dpm_d0 = dpm_d0_count * inv_n; constexpr double indep_Co = 0.5; - constexpr double indep_D = 0.75; + // Independence null for the discordant partial moment (R 13.1): a point is + // concordant when all-below OR all-above the target, so under independence + // P(discordant) = 1 - 2 * 0.5^2 = 0.5, not 0.75. + constexpr double indep_D = 0.5; double disc_dep = std::min(1.0, std::max(0.0, std::abs(d0_Co - indep_Co) / indep_Co)); double nd_disc = std::abs(dpm_d0 - indep_D) / indep_D; @@ -260,7 +274,7 @@ DepMatrixResult dep_matrix(const double* X, std::size_t n, std::size_t p, } all_quads[j] = std::move(quad); } - }, nthreads); + }, nthreads, n * 8U); // ~log2(n) hashing passes over n rows // Phase 2: Compute Pairwise Dependence std::vector corr_upper(n_pairs, 0.0); @@ -304,7 +318,7 @@ DepMatrixResult dep_matrix(const double* X, std::size_t n, std::size_t p, dep_lower[idx] = dep_upper[idx]; } } - }, nthreads); + }, nthreads, n * 16U); // dep_pair: several passes over n rows // Phase 3: Construct the final Column-Major output matrices DepMatrixResult result; diff --git a/extern/NNS-core/src/distance.cpp b/extern/NNS-core/src/distance.cpp index 6c81a24a..3207418f 100644 --- a/extern/NNS-core/src/distance.cpp +++ b/extern/NNS-core/src/distance.cpp @@ -1,6 +1,7 @@ // src/distance.cpp // -// Implementation extracted from NNS 13.0 NNS_distance.cpp. Decoupled from Rcpp. +// Implementation extracted from NNS NNS_distance.cpp (synced to R 13.3, 82a92eb; the +// 13.3 column-major indexing fix needs no change here). Decoupled from Rcpp. // // SPDX-License-Identifier: GPL-3.0-only #include "nns/distance.hpp" @@ -482,7 +483,7 @@ std::vector distance_path_parallel(const double* RPM, std::size_t l, std } } } - }, nthreads); + }, nthreads, l * n + static_cast(kmax) * static_cast(kmax)); return out; } @@ -619,7 +620,7 @@ std::vector distance_path_single_parallel(const double* RPM, std::size_t out[r] = mode_class_weighted(topY, w); } } - }, nthreads); + }, nthreads, l * n + static_cast(k)); return out; } diff --git a/extern/NNS-core/src/fast_lm.cpp b/extern/NNS-core/src/fast_lm.cpp index bcc90b1f..dc2dcc5b 100644 --- a/extern/NNS-core/src/fast_lm.cpp +++ b/extern/NNS-core/src/fast_lm.cpp @@ -1,6 +1,6 @@ // src/fast_lm.cpp // -// Pure C++ port of NNS 13.0 fast_lm.cpp. Decoupled from Rcpp. +// Pure C++ port of NNS fast_lm.cpp (unchanged through R 13.3, 82a92eb). Decoupled from Rcpp. // // This file preserves the numerical rules and return payloads of the original // Rcpp functions: @@ -35,11 +35,20 @@ inline double design_value(const double* x, std::size_t n, std::size_t row, return (col_with_intercept == 0) ? 1.0 : at(x, n, row, col_with_intercept - 1U); } +// Port of Rcpp sugar mean(), which the R kernels call: long-double sum plus one +// correction pass, as in R's summary.c. inline double mean_vec(const double* x, std::size_t n) { if (n == 0U) return kNaN; - double s = 0.0; + const long double nd = static_cast(n); + long double s = 0.0L; for (std::size_t i = 0; i < n; ++i) s += x[i]; - return s / static_cast(n); + s /= nd; + if (std::isfinite(static_cast(s))) { + long double t = 0.0L; + for (std::size_t i = 0; i < n; ++i) t += x[i] - s; + s += t / nd; + } + return static_cast(s); } std::string dim_msg(const char* prefix, std::size_t a, std::size_t b) { diff --git a/extern/NNS-core/src/internal_functions.cpp b/extern/NNS-core/src/internal_functions.cpp index c14ddc6d..67f0e7ef 100644 --- a/extern/NNS-core/src/internal_functions.cpp +++ b/extern/NNS-core/src/internal_functions.cpp @@ -191,25 +191,32 @@ bool is_discrete(const double* x, std::size_t n) { // ---------- Vector Generation ---------- -TimeSeriesVectors generate_vectors(const double* x, std::size_t n, const int* lags, std::size_t num_lags) { +TimeSeriesVectors generate_vectors(const double* x, std::size_t n, const int* lags, std::size_t num_lags, + long long len) { + if (len > static_cast(n)) + throw std::invalid_argument("generate_vectors: `len` must not exceed the length of x"); + if (len >= 0) n = static_cast(len); TimeSeriesVectors res; res.series.resize(num_lags); res.index.resize(num_lags); + // Signed arithmetic: with size_t n, `n - start` underflows for lag > n. + const long long nn = static_cast(n); for (std::size_t t = 0; t < num_lags; ++t) { - int lag = lags[t]; + const long long lag = lags[t]; if (lag <= 0) continue; - int start = (n % lag) + 1; - int m = ((n - start) / lag) + 1; + // For lag > n, start = n + 1 and the component is empty. + const long long start = (nn % lag) + 1; + const long long m = (start > nn) ? 0 : ((nn - start) / lag) + 1; - std::vector s(m); - std::vector idx(m); + std::vector s(static_cast(m)); + std::vector idx(static_cast(m)); - int pos = start; - for (int i = 0; i < m; ++i, pos += lag) { - s[i] = x[pos - 1]; - idx[i] = i + 1; + long long pos = start; + for (long long i = 0; i < m; ++i, pos += lag) { + s[static_cast(i)] = x[pos - 1]; + idx[static_cast(i)] = static_cast(i + 1); } res.series[t] = std::move(s); res.index[t] = std::move(idx); @@ -218,20 +225,25 @@ TimeSeriesVectors generate_vectors(const double* x, std::size_t n, const int* la } ForecastVectors generate_lin_vectors(const double* x, std::size_t n, int l, int h) { + // l is a modulus below (l = 0 is SIGFPE in R 13.3); h sizes the result. + if (l < 1) throw std::invalid_argument("generate_lin_vectors: `l` must be a positive integer"); + if (h < 0) throw std::invalid_argument("generate_lin_vectors: `h` must be non-negative"); + const long long nn = static_cast(n); int max_fcast = std::min(h, l); ForecastVectors res; res.series.resize(max_fcast); res.index.resize(max_fcast); for (int i = 1; i <= max_fcast; ++i) { - int start = ((n + i - 1) % l) + 1; - int m = ((n - start) / l) + 1; - std::vector s(m); - std::vector idx(m); - int pos = start; - for (int k = 0; k < m; ++k, pos += l) { - s[k] = x[pos - 1]; - idx[k] = k + 1; + const long long start = ((nn + i - 1) % l) + 1; + // start > n (l > n): empty component, see generate_vectors. + const long long m = (start > nn) ? 0 : ((nn - start) / l) + 1; + std::vector s(static_cast(m)); + std::vector idx(static_cast(m)); + long long pos = start; + for (long long k = 0; k < m; ++k, pos += l) { + s[static_cast(k)] = x[pos - 1]; + idx[static_cast(k)] = static_cast(k + 1); } res.series[i - 1] = std::move(s); res.index[i - 1] = std::move(idx); @@ -275,7 +287,8 @@ ARMAWeights arma_seas_weighting(const double* periods, const double* covar, cons ARMAWeights res; res.lags.assign(periods, periods + m); if (denom == 0.0) { - res.weights.assign(m, 0.0); + // All-zero weights forced a forecast of 0; NNS.ARMA uses equal weights. + res.weights.assign(m, 1.0 / static_cast(m)); } else { res.weights.resize(m); for (std::size_t i = 0; i < m; ++i) res.weights[i] = wprod[i] / denom; diff --git a/extern/NNS-core/src/mreg_predict.cpp b/extern/NNS-core/src/mreg_predict.cpp new file mode 100644 index 00000000..dd56d779 --- /dev/null +++ b/extern/NNS-core/src/mreg_predict.cpp @@ -0,0 +1,539 @@ +// src/mreg_predict.cpp +// +// Pure C++ port of R NNS 13.3 src/NNS_mreg_predict.cpp +// (NNS_mreg_predict_v2_cpp / NNS_mreg_predict_path_v2_cpp, i.e. the +// PredictPathWorker kernel). Decoupled from Rcpp and Rmath: +// - range-normalised NNS / L2 / L1 or Hamming (FACTOR) distances, +// - stable ascending order with index tie-break via partial selection, +// - k = 1 aggregates all exact nearest-distance ties (gravity / class mode), +// - k > 1 uses the eight-component ensemble weights, +// - exact weighted class mode (max total, tie -> smallest value), +// - the O(log n + k) 1-D fast path for duplicated coordinates. +// +// The R kernel evaluates its density components with Rmath (Rf_dt, Rf_dexp, +// Rf_dlnorm, Rf_dnorm4) inside RcppParallel workers. Here dexp, dlnorm and +// dnorm are the same closed forms Rmath uses, and dt follows Rmath's +// factorisation with lgamma for the constant. No R API is touched from worker +// threads, and results agree with R to floating-point rounding. +// +// SPDX-License-Identifier: GPL-3.0-only +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nns/central_tendencies.hpp" +#include "nns/mreg.hpp" +#include "nns/parallel.hpp" + +namespace nns { + +namespace { + +constexpr double kNaN = std::numeric_limits::quiet_NaN(); +constexpr double kInf = std::numeric_limits::infinity(); +constexpr double kLnSqrt2Pi = 0.918938533204672741780329736406; // log(sqrt(2*pi)) +constexpr double k1SqrtTwoPi = 0.398942280401432677939946059934; // 1/sqrt(2*pi) +constexpr double kLn2 = 0.693147180559945309417232121458; + +// ---- densities (closed forms; see file comment) ----------------------------- + +// The df-only term t of Rmath's dt(x, df). +inline double dt_log_const(double df) { + return std::lgamma(0.5 * (df + 1.0)) - std::lgamma(0.5 * df) + 0.5 * std::log(2.0 / df); +} + +// Rmath dt(x, df, give_log = 0) for finite df > 0 with t = dt_log_const(df), +// factored as Rmath does so that far-away weights underflow where R's do. +inline double dt0(double x, double df, double t) { + if (!std::isfinite(x)) return 0.0; + const double ax = std::fabs(x), x2n = x * x / df; + const bool large = x2n > 1.0 / DBL_EPSILON; + const double l_x2n = large ? std::log(ax) - 0.5 * std::log(df) + : 0.5 * (x2n > 0.2 ? std::log(1.0 + x2n) : std::log1p(x2n)); + const double scale = large ? std::sqrt(df) / ax : std::exp(-l_x2n); + return std::exp(t - df * l_x2n) * k1SqrtTwoPi * scale; +} + +// Rmath dexp(x, scale, give_log = 0). +inline double dexp_scale(double x, double scale) { + return (x < 0.0) ? 0.0 : std::exp(-x / scale) / scale; +} + +// Rmath dlnorm(x, meanlog = 0, sdlog, give_log = 1) for x > 0, sdlog > 0. +inline double dlnorm_log(double x, double sdlog) { + const double y = std::log(x) / sdlog; + return -(kLnSqrt2Pi + 0.5 * y * y + std::log(x * sdlog)); +} + +// Rmath dnorm4(x, mu = 0, sigma, give_log = 0) for finite sigma > 0. +inline double dnorm0(double x, double sigma) { + x = x / sigma; + if (!std::isfinite(x)) return 0.0; + x = std::fabs(x); + if (x >= 2.0 * std::sqrt(DBL_MAX)) return 0.0; + if (x < 5.0) return k1SqrtTwoPi * std::exp(-0.5 * x * x) / sigma; + if (x > std::sqrt(-2.0 * kLn2 * (DBL_MIN_EXP + 1 - DBL_MANT_DIG))) return 0.0; + const double x1 = std::ldexp(std::nearbyint(std::ldexp(x, 16)), -16); + const double x2 = x - x1; + return k1SqrtTwoPi / sigma * (std::exp(-0.5 * x1 * x1) * std::exp((-0.5 * x2 - x1) * x2)); +} + +// ---- weight helpers ---------------------------------------------------------- + +inline void normalize_weights(std::vector& w) { + const std::size_t k = w.size(); + double s = 0.0; + for (double& v : w) { + if (!std::isfinite(v) || v < 0.0) v = 0.0; + s += v; + } + if (s <= 0.0) { + for (double& v : w) v = 1.0 / static_cast(k); + } else { + for (double& v : w) v /= s; + } +} + +inline double sample_sd(const std::vector& v) { + const std::size_t n = v.size(); + if (n < 2) return kNaN; + double mu = 0.0; + for (double x : v) mu += x; + mu /= static_cast(n); + double acc = 0.0; + for (double x : v) { + const double d = x - mu; + acc += d * d; + } + return std::sqrt(acc / static_cast(n - 1)); +} + +// Totals per distinct value, max total, tie -> smallest value. +inline double weighted_mode(const std::vector& vals, const std::vector& w) { + std::vector> items; + items.reserve(vals.size()); + for (std::size_t i = 0; i < vals.size(); ++i) { + if (std::isfinite(vals[i]) && std::isfinite(w[i]) && w[i] >= 0.0) + items.push_back(std::make_pair(vals[i], w[i])); + } + if (items.empty()) return kNaN; + std::sort(items.begin(), items.end()); + double best_val = items[0].first; + double best_tot = -1.0; + std::size_t i = 0; + while (i < items.size()) { + const double v = items[i].first; + double tot = 0.0; + while (i < items.size() && items[i].first == v) { + tot += items[i].second; + ++i; + } + if (tot > best_tot) { + best_tot = tot; + best_val = v; + } + } + return best_val; +} + +void rank_components(int k, std::vector& uniform, std::vector& exponential, + std::vector& lognormal, std::vector& power) { + uniform.assign(k, 1.0 / static_cast(k)); + exponential.resize(k); + lognormal.assign(k, 0.0); + power.resize(k); + const double kk = static_cast(k); + for (int i = 0; i < k; ++i) exponential[i] = dexp_scale(static_cast(i + 1), kk); + normalize_weights(exponential); + std::vector ranks(k); + for (int i = 0; i < k; ++i) ranks[i] = static_cast(i + 1); + const double rank_sd = sample_sd(ranks); + if (std::isfinite(rank_sd) && rank_sd > 0.0) { + for (int i = 0; i < k; ++i) lognormal[i] = std::fabs(dlnorm_log(ranks[i], rank_sd)); + normalize_weights(lognormal); + std::reverse(lognormal.begin(), lognormal.end()); + } + for (int i = 0; i < k; ++i) { + const double r = static_cast(i + 1); + power[i] = 1.0 / (r * r); + } + normalize_weights(power); +} + +// Rank weights for k in [kmin, kmax]; every k up to kmax is O(kmax^2) memory. +struct RankComponents { + std::vector> uniform, exponential, lognormal, power; + std::vector dt_const; + RankComponents(int kmin, int kmax) + : uniform(kmax + 1), exponential(kmax + 1), lognormal(kmax + 1), power(kmax + 1), + dt_const(kmax + 1, 0.0) { + for (int k = std::max(kmin, 2); k <= kmax; ++k) { + rank_components(k, uniform[k], exponential[k], lognormal[k], power[k]); + dt_const[k] = dt_log_const(static_cast(k)); + } + } +}; + +inline void normalize_prefix(std::vector& values, int k) { + double s = 0.0; + for (int i = 0; i < k; ++i) { + if (!std::isfinite(values[i]) || values[i] < 0.0) values[i] = 0.0; + s += values[i]; + } + if (s <= 0.0) { + const double u = 1.0 / static_cast(k); + for (int i = 0; i < k; ++i) values[i] = u; + } else { + for (int i = 0; i < k; ++i) values[i] /= s; + } +} + +inline double weighted_mode_prefix(const std::vector& yk, + const std::vector& weights, int k, + std::vector& cls, std::vector& totals) { + int ncls = 0; + for (int i = 0; i < k; ++i) { + const double v = yk[i], w = weights[i]; + if (!std::isfinite(v) || !std::isfinite(w) || w < 0.0) continue; + int pos = -1; + for (int c = 0; c < ncls; ++c) + if (cls[c] == v) { + pos = c; + break; + } + if (pos < 0) { + pos = ncls++; + cls[pos] = v; + totals[pos] = 0.0; + } + totals[pos] += w; + } + if (ncls == 0) return kNaN; + double best_val = cls[0], best_tot = totals[0]; + for (int c = 1; c < ncls; ++c) { + if (totals[c] > best_tot || (totals[c] == best_tot && cls[c] < best_val)) { + best_tot = totals[c]; + best_val = cls[c]; + } + } + return best_val; +} + +struct Scratch { + std::vector student, inverse, normal, rbf, total, cls, cls_totals; + explicit Scratch(int kmax) + : student(kmax), inverse(kmax), normal(kmax), rbf(kmax), total(kmax), cls(kmax), + cls_totals(kmax) {} +}; + +double predict_sorted_k(const std::vector& dk, const std::vector& yk, int k, + bool is_class, const RankComponents& rank, Scratch& s) { + const double kk = static_cast(k); + for (int i = 0; i < k; ++i) s.student[i] = dt0(dk[i], kk, rank.dt_const[k]); + normalize_prefix(s.student, k); + for (int i = 0; i < k; ++i) s.inverse[i] = 1.0 / std::max(dk[i], 1e-12); + normalize_prefix(s.inverse, k); + + double mu = 0.0; + for (int i = 0; i < k; ++i) mu += dk[i]; + mu /= kk; + double acc = 0.0; + for (int i = 0; i < k; ++i) { + const double z = dk[i] - mu; + acc += z * z; + } + const double sd = (k > 1) ? std::sqrt(acc / static_cast(k - 1)) : kNaN; + const double var = std::isfinite(sd) ? sd * sd : kNaN; + if (std::isfinite(sd) && sd > 0.0) { + for (int i = 0; i < k; ++i) s.normal[i] = dnorm0(dk[i], sd); + normalize_prefix(s.normal, k); + } else { + for (int i = 0; i < k; ++i) s.normal[i] = 0.0; + } + if (std::isfinite(var) && var > 0.0) { + for (int i = 0; i < k; ++i) s.rbf[i] = std::exp(-dk[i] / (2.0 * var)); + normalize_prefix(s.rbf, k); + } else { + for (int i = 0; i < k; ++i) s.rbf[i] = 0.0; + } + + for (int i = 0; i < k; ++i) { + s.total[i] = rank.uniform[k][i] + s.student[i] + s.inverse[i] + rank.exponential[k][i] + + rank.lognormal[k][i] + rank.power[k][i] + s.normal[i] + s.rbf[i]; + } + normalize_prefix(s.total, k); + + if (is_class) return weighted_mode_prefix(yk, s.total, k, s.cls, s.cls_totals); + double dot = 0.0; + for (int i = 0; i < k; ++i) dot += yk[i] * s.total[i]; + return dot; +} + +// ---- the prediction kernel (port of PredictPathWorker) ---------------------- + +class PredictKernel { + public: + PredictKernel(const double* rpm_x, std::size_t n, std::size_t p, const double* yhat, + const double* Xtest, std::size_t m, int kmax, int dist_code, + const double* mins, const double* maxs, bool is_class, + const RankComponents& rank, double* out, int only_k) + : rpm_x_(rpm_x), yhat_(yhat), Xtest_(Xtest), out_(out), n_(static_cast(n)), + p_(static_cast(p)), m_(m), kmax_(kmax), dist_code_(dist_code), + only_k_(only_k), is_class_(is_class), rank_(rank) { + for (int j = 0; j < p_; ++j) { + const double range = maxs[j] - mins[j]; + if (std::isfinite(range) && range > 0.0) { + active_.push_back(j); + col_range_.push_back(range); + } + } + // Effectively 1-D design: all active columns identical in rpm_x and Xtest + // (FACTOR / Hamming excluded). + if (dist_code_ != kMregDistFactor && !active_.empty()) { + axis_ = active_[0]; + bool identical = true; + for (std::size_t a = 1; a < active_.size() && identical; ++a) { + const int j = active_[a]; + for (int i = 0; i < n_; ++i) + if (rx(i, j) != rx(i, axis_)) { + identical = false; + break; + } + for (std::size_t i = 0; identical && i < m_; ++i) + if (xt(i, j) != xt(i, axis_)) { + identical = false; + break; + } + } + if (identical) { + is_1d_ = true; + std::vector coord(n_); + for (int i = 0; i < n_; ++i) coord[i] = rx(i, axis_); + so_.resize(n_); + for (int i = 0; i < n_; ++i) so_[i] = i; + std::sort(so_.begin(), so_.end(), [&coord](int a, int b) { + return coord[a] < coord[b] || (coord[a] == coord[b] && a < b); + }); + sc_.resize(n_); + for (int i = 0; i < n_; ++i) sc_[i] = coord[so_[i]]; + } + } + } + + void operator()(std::size_t begin, std::size_t end) const { + std::vector d(n_), dk(kmax_), yk(kmax_), tied, one_weights; + std::vector idx(n_), tk; + Scratch s(kmax_); + auto cmp = [&d](int a, int b) { return d[a] < d[b] || (d[a] == d[b] && a < b); }; + auto out = [&](std::size_t r, std::size_t c) -> double& { return out_[r + c * m_]; }; + for (std::size_t r = begin; r < end; ++r) { + if (is_1d_) { + const double t = xt(r, axis_); + if (only_k_ > 0) { + if (only_k_ == 1) { + out(r, 0) = min_ties_1d(r, t, tied, one_weights); + } else { + topk_1d(r, t, only_k_, tk, dk); + for (int i = 0; i < only_k_; ++i) yk[i] = yhat_[tk[i]]; + out(r, 0) = predict_sorted_k(dk, yk, only_k_, is_class_, rank_, s); + } + } else { + out(r, 0) = min_ties_1d(r, t, tied, one_weights); + if (kmax_ >= 2) { + topk_1d(r, t, kmax_, tk, dk); + for (int i = 0; i < kmax_; ++i) yk[i] = yhat_[tk[i]]; + for (int k = 2; k <= kmax_; ++k) + out(r, static_cast(k - 1)) = + predict_sorted_k(dk, yk, k, is_class_, rank_, s); + } + } + continue; + } + distances_for_row(r, d); + for (int i = 0; i < n_; ++i) idx[i] = i; + if (kmax_ < n_) { + std::nth_element(idx.begin(), idx.begin() + kmax_, idx.end(), cmp); + std::sort(idx.begin(), idx.begin() + kmax_, cmp); + } else { + std::sort(idx.begin(), idx.end(), cmp); + } + for (int i = 0; i < kmax_; ++i) { + dk[i] = d[idx[i]]; + yk[i] = yhat_[idx[i]]; + } + if (only_k_ > 0) { + out(r, 0) = (only_k_ == 1) ? min_ties(d, tied, one_weights) + : predict_sorted_k(dk, yk, only_k_, is_class_, rank_, s); + } else { + out(r, 0) = min_ties(d, tied, one_weights); + for (int k = 2; k <= kmax_; ++k) + out(r, static_cast(k - 1)) = + predict_sorted_k(dk, yk, k, is_class_, rank_, s); + } + } + } + + private: + double rx(int i, int j) const { return rpm_x_[static_cast(i) + static_cast(j) * n_]; } + double xt(std::size_t i, int j) const { return Xtest_[i + static_cast(j) * m_]; } + + // Range-normalised distance of training row i to query r. The 1-D path uses + // it too, so it selects neighbours and ties exactly as the generic path does. + double distance(int i, std::size_t r) const { + double acc = 0.0; + for (std::size_t a = 0; a < active_.size(); ++a) { + const int j = active_[a]; + // Divide as R's reference does; a precomputed 1/range can flip exact ties. + const double z = (rx(i, j) - xt(r, j)) / col_range_[a]; + if (dist_code_ == kMregDistL1) acc += std::fabs(z); + else if (dist_code_ == kMregDistNNS) acc += std::fabs(z) + z * z; + else acc += z * z; + } + return dist_code_ == kMregDistL2 ? std::sqrt(acc) : acc; + } + + // distance() is non-decreasing in |coord - t|, so walking outward in that + // order and extending over distance() ties at the boundary finds the k nearest. + void topk_1d(std::size_t row, double t, int k, std::vector& out_idx, + std::vector& out_d) const { + const int nn = static_cast(sc_.size()); + k = std::min(k, nn); + int r = static_cast(std::lower_bound(sc_.begin(), sc_.end(), t) - sc_.begin()); + int lo = r - 1; + std::vector> win; + while (static_cast(win.size()) < k && (lo >= 0 || r < nn)) { + const double dl = (lo >= 0) ? std::fabs(sc_[lo] - t) : kInf; + const double dr = (r < nn) ? std::fabs(sc_[r] - t) : kInf; + const int pos = (dl <= dr) ? lo-- : r++; + win.emplace_back(distance(so_[pos], row), so_[pos]); + } + double thr = 0.0; + for (const auto& w : win) thr = std::max(thr, w.first); + for (double dd; lo >= 0 && (dd = distance(so_[lo], row)) <= thr; --lo) + win.emplace_back(dd, so_[lo]); + for (double dd; r < nn && (dd = distance(so_[r], row)) <= thr; ++r) + win.emplace_back(dd, so_[r]); + std::sort(win.begin(), win.end()); // (distance, original index) + out_idx.clear(); + for (int i = 0; i < k; ++i) { + out_idx.push_back(win[i].second); + out_d[i] = win[i].first; + } + } + + double min_ties_1d(std::size_t row, double t, std::vector& tied, + std::vector& weights) const { + const int nn = static_cast(sc_.size()); + const int r = static_cast(std::lower_bound(sc_.begin(), sc_.end(), t) - sc_.begin()); + const int lo = r - 1; + const double dl = (lo >= 0) ? distance(so_[lo], row) : kInf; + const double dr = (r < nn) ? distance(so_[r], row) : kInf; + const double mn = std::min(dl, dr); + tied.clear(); + for (int li = lo; li >= 0 && distance(so_[li], row) == mn; --li) + if (std::isfinite(yhat_[so_[li]])) tied.push_back(yhat_[so_[li]]); + for (int ri = r; ri < nn && distance(so_[ri], row) == mn; ++ri) + if (std::isfinite(yhat_[so_[ri]])) tied.push_back(yhat_[so_[ri]]); + if (is_class_) { + weights.assign(tied.size(), 1.0); + return weighted_mode(tied, weights); + } + return gravity(tied.data(), tied.size(), false); + } + + void distances_for_row(std::size_t r, std::vector& d) const { + if (dist_code_ == kMregDistFactor) { + for (int i = 0; i < n_; ++i) { + double mismatches = 0.0; + for (int j = 0; j < p_; ++j) + if (rx(i, j) != xt(r, j)) mismatches += 1.0; + d[i] = mismatches / static_cast(p_); + } + } else if (active_.empty()) { + std::fill(d.begin(), d.end(), 0.0); + } else { + for (int i = 0; i < n_; ++i) d[i] = distance(i, r); + } + } + + double min_ties(const std::vector& d, std::vector& tied, + std::vector& weights) const { + double dmin = d[0]; + for (int i = 1; i < n_; ++i) + if (d[i] < dmin) dmin = d[i]; + tied.clear(); + for (int i = 0; i < n_; ++i) + if (d[i] == dmin && std::isfinite(yhat_[i])) tied.push_back(yhat_[i]); + if (is_class_) { + weights.assign(tied.size(), 1.0); + return weighted_mode(tied, weights); + } + return gravity(tied.data(), tied.size(), false); + } + + const double* rpm_x_; + const double* yhat_; + const double* Xtest_; + double* out_; + int n_, p_; + std::size_t m_; + int kmax_, dist_code_, only_k_; + bool is_class_; + const RankComponents& rank_; + std::vector active_; + std::vector col_range_; + bool is_1d_ = false; + int axis_ = -1; + std::vector sc_; + std::vector so_; +}; + +void validate(std::size_t n, std::size_t p, std::size_t m, int k, int dist_code, + const double* mins, const double* maxs, const char* kname) { + (void)m; + if (n == 0) throw std::invalid_argument("mreg_predict: rpm_x must have at least one row"); + if (p == 0) throw std::invalid_argument("mreg_predict: rpm_x must have at least one column"); + if (mins == nullptr || maxs == nullptr) + throw std::invalid_argument("mreg_predict: mins/maxs must have one value per column"); + if (k < 1) throw std::invalid_argument(std::string("mreg_predict: ") + kname + " must be >= 1"); + if (dist_code < 0 || dist_code > 3) + throw std::invalid_argument("mreg_predict: unknown NNS multivariate distance code"); +} + +} // namespace + +std::vector mreg_predict(const double* rpm_x, std::size_t n, std::size_t p, + const double* yhat, const double* Xtest, std::size_t m, + int k, int dist_code, const double* mins, const double* maxs, + bool is_class, int n_threads) { + validate(n, p, m, k, dist_code, mins, maxs, "k"); + if (static_cast(k) > n) k = static_cast(n); + RankComponents rank(k, k); + std::vector out(m); + PredictKernel kernel(rpm_x, n, p, yhat, Xtest, m, k, dist_code, mins, maxs, is_class, rank, + out.data(), /*only_k=*/k); + parallel_for(0, m, kernel, n_threads, n * p + static_cast(k) * 16U); + return out; +} + +std::vector mreg_predict_path(const double* rpm_x, std::size_t n, std::size_t p, + const double* yhat, const double* Xtest, std::size_t m, + int kmax, int dist_code, const double* mins, + const double* maxs, bool is_class, int n_threads) { + validate(n, p, m, kmax, dist_code, mins, maxs, "kmax"); + if (static_cast(kmax) > n) kmax = static_cast(n); + RankComponents rank(2, kmax); + std::vector out(m * static_cast(kmax)); + PredictKernel kernel(rpm_x, n, p, yhat, Xtest, m, kmax, dist_code, mins, maxs, is_class, + rank, out.data(), /*only_k=*/0); + parallel_for(0, m, kernel, n_threads, + n * p + static_cast(kmax) * static_cast(kmax) * 8U); + return out; +} + +} // namespace nns diff --git a/extern/NNS-core/src/mreg_setup.cpp b/extern/NNS-core/src/mreg_setup.cpp new file mode 100644 index 00000000..f658da07 --- /dev/null +++ b/extern/NNS-core/src/mreg_setup.cpp @@ -0,0 +1,151 @@ +// src/mreg_setup.cpp +// +// Pure C++ port of R NNS 13.3 src/NNS_mreg_setup.cpp +// (NNS_duplicate_column_map_cpp, NNS_mreg_setup_cpp). Decoupled from Rcpp. +// The reducers, findInterval ids and lexicographic RPM row order are +// preserved verbatim, except that the "mode" reducer is NNS.mode for numeric +// responses; class responses keep the exact-frequency mode. +// +// SPDX-License-Identifier: GPL-3.0-only +#include "nns/mreg.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "nns/central_tendencies.hpp" + +namespace nns { + +namespace { + +constexpr double kNaN = std::numeric_limits::quiet_NaN(); + +double reduce_numeric(std::vector& v, int reducer, bool is_class) { + v.erase(std::remove_if(v.begin(), v.end(), [](double x) { return !std::isfinite(x); }), + v.end()); + if (v.empty()) return kNaN; + std::sort(v.begin(), v.end()); + // Same estimator as the pure-R reducer (.nns_reg_reduce_value). + if (!is_class && reducer == kMregMode) { + const std::vector m = nns::mode(v.data(), v.size(), false, false); + return m.empty() ? kNaN : m.front(); + } + if (is_class) { + double best = v[0]; + int bestn = 0; + for (std::size_t i = 0; i < v.size();) { + const double val = v[i]; + int cnt = 0; + while (i < v.size() && v[i] == val) { + ++i; + ++cnt; + } + if (cnt > bestn) { + bestn = cnt; + best = val; + } + } + return best; + } + if (reducer == kMregMedian) { + const std::size_t n = v.size(); + return (n % 2) ? v[n / 2] : (v[n / 2 - 1] + v[n / 2]) / 2.0; + } + if (reducer == kMregGravity) return gravity(v.data(), v.size(), false); + double s = 0.0; + for (double x : v) s += x; + return s / static_cast(v.size()); +} + +// One-based findInterval(x, b, left.open = FALSE, rightmost.closed = TRUE): +// below the first boundary -> 0; exactly the last boundary -> n - 1; above +// the last -> n. The ids are later sorted as strings, so parity here +// preserves the RPM row order. +int find_interval_one(double x, const std::vector& b) { + const int n = static_cast(b.size()); + if (n == 0) return 0; + if (x < b[0]) return 0; + if (x == b[static_cast(n - 1)]) return std::max(0, n - 1); + return static_cast(std::upper_bound(b.begin(), b.end(), x) - b.begin()); +} + +} // namespace + +std::vector duplicate_column_map(const double* X, std::size_t n, + std::size_t p) { + std::vector rep(p); + std::unordered_map> seen; + for (std::size_t j = 0; j < p; ++j) { + const double* col = X + j * n; + std::string key(reinterpret_cast(col), n * sizeof(double)); + std::size_t r = j; + auto it = seen.find(key); + if (it != seen.end()) { + for (std::size_t cand : it->second) { + const double* other = X + cand * n; + bool same = true; + for (std::size_t i = 0; i < n && same; ++i) same = (col[i] == other[i]); + if (same) { + r = cand; + break; + } + } + it->second.push_back(j); + } else { + seen.emplace(std::move(key), std::vector(1, j)); + } + rep[j] = r; + } + return rep; +} + +MregSetupResult mreg_setup(const double* X, std::size_t n, std::size_t p, + const double* y, + const std::vector>& boundaries, + int reducer_code, bool is_class) { + if (boundaries.size() != p) + throw std::invalid_argument("mreg_setup: need one boundary vector per column"); + + MregSetupResult out; + out.p = p; + out.ids.resize(n); + std::unordered_map> groups; + groups.reserve(n * 2); + for (std::size_t i = 0; i < n; ++i) { + std::string id; + for (std::size_t j = 0; j < p; ++j) { + if (j) id += '.'; + id += std::to_string(find_interval_one(X[i + j * n], boundaries[j])); + } + groups[id].push_back(i); + out.ids[i] = std::move(id); + } + + out.row_ids.reserve(groups.size()); + for (const auto& kv : groups) out.row_ids.push_back(kv.first); + std::sort(out.row_ids.begin(), out.row_ids.end()); + + const std::size_t g_n = out.row_ids.size(); + out.n_groups = g_n; + out.rpm.assign(g_n * (p + 1), 0.0); + std::vector vals; + for (std::size_t g = 0; g < g_n; ++g) { + const std::vector& idx = groups[out.row_ids[g]]; + for (std::size_t j = 0; j < p; ++j) { + vals.clear(); + for (std::size_t row : idx) vals.push_back(X[row + j * n]); + out.rpm[g + j * g_n] = reduce_numeric(vals, reducer_code, false); + } + vals.clear(); + for (std::size_t row : idx) vals.push_back(y[row]); + out.rpm[g + p * g_n] = reduce_numeric(vals, reducer_code, is_class); + } + return out; +} + +} // namespace nns diff --git a/extern/NNS-core/src/partial_moments.cpp b/extern/NNS-core/src/partial_moments.cpp index d082233c..70cb57f3 100644 --- a/extern/NNS-core/src/partial_moments.cpp +++ b/extern/NNS-core/src/partial_moments.cpp @@ -2,10 +2,17 @@ // // Implementation extracted from NNS 13.0 src/partial_moments.{h,cpp}. // Every numerical rule (>= vs > boundaries, integer-degree fast powers, -// median shift in the prefix backend, min/max-length recycling, population -// adjustment gating, crossed DUPM/DLPM mirroring) is preserved verbatim. +// min/max-length recycling, population adjustment gating, crossed DUPM/DLPM +// mirroring) is preserved verbatim. The vectorised (> 32 target) backend uses +// a cancellation-free formulation instead of the upstream median-shifted +// binomial expansion (see the comment above PrefixBackend). // // SPDX-License-Identifier: GPL-3.0-only + +// ---- NNS-CORE PROVENANCE (managed by scripts/sync_adapter.py; do not hand-edit) ---- +// NNS-R-SOURCE: src/partial_moments.cpp @ 82a92eb45c1a1892acce82e66175cf2e29434b65 +// NNS-R-SOURCE: src/partial_moments.h @ 82a92eb45c1a1892acce82e66175cf2e29434b65 +// ---- end NNS-CORE PROVENANCE ---- #include "nns/partial_moments.hpp" #include @@ -13,6 +20,7 @@ #include #include #include +#include #include #include "nns/parallel.hpp" @@ -32,6 +40,15 @@ inline double repeat_multiplication(double value, int n) { inline bool is_integer(double v) { return v == static_cast(static_cast(v)); } +// Rejects negative and NaN degrees (NNS 13.3 returned the degree-0 count for a +// negative degree). +inline void check_degree(double degree, const char* what = "degree") { + if (!(degree >= 0.0)) { + throw std::invalid_argument(std::string("`") + what + + "` must be a non-negative number"); + } +} + inline double lower_component(double diff, double degree, bool degree_is_int) { if (degree == 0) return diff >= 0.0 ? 1.0 : 0.0; if (diff < 0.0) return 0.0; @@ -46,7 +63,22 @@ inline double upper_component(double diff, double degree, bool degree_is_int) { : std::pow(diff, degree); } -// --- prefix-power backend (port of nns_pm_detail, NNS 13.0) ---------------- +// --- sorted-anchor prefix backend (vectorised LPM/UPM, > 32 targets) ------- +// +// NNS 13.0-13.3 expanded sum_{x_i <= t} (t - x_i)^d binomially around the +// sample median, which cancels catastrophically for d >= 3. This backend keeps +// the O((n + m) log n) cost but anchors every expansion at an adjacent order +// statistic, so all terms are non-negative: +// +// lower_j(k) = sum_{i < k} (x_(k-1) - x_(i))^j (anchor = largest point <= t) +// upper_j(k) = sum_{i >= k} (x_(i) - x_(k))^j (anchor = smallest point > t) +// +// Both tables are built with the exact recurrence +// lower_j(k+1) = sum_{l <= j} C(j,l) delta^(j-l) lower_l(k) + [j == 0], +// delta = x_(k) - x_(k-1) >= 0, +// and a target t with k = #{x_i <= t} is evaluated as +// LPM = sum_j C(d,j) (t - x_(k-1))^(d-j) lower_j(k) / n +// UPM = sum_j C(d,j) (x_(k) - t)^(d-j) upper_j(k) / n. constexpr int kPrefixMaxDegree = 32; constexpr std::size_t kDirectPathMaxTargets = 32; // NNS_DIRECT_PATH_MAX_TARGETS @@ -60,33 +92,16 @@ bool prefix_supported_degree(double degree, int& degree_int) { return true; } -std::vector binomial_coefficients(int degree) { - std::vector choose(static_cast(degree) + 1U, 1.0); - for (int j = 1; j < degree; ++j) { - choose[static_cast(j)] = - choose[static_cast(j - 1)] * - static_cast(degree - j + 1) / static_cast(j); - } - return choose; -} - struct PrefixBackend { std::vector sorted; - std::vector> prefix_power; - std::vector total_power; - std::vector choose; + std::vector lower; // (n + 1) x (degree + 1), row k = lower_j(k) + std::vector upper; // (n + 1) x (degree + 1), row k = upper_j(k) + std::vector pascal; // (degree + 1) x (degree + 1), C(j, l) std::size_t n; int degree; - double shift; PrefixBackend(const double* variable, std::size_t n_, int degree_) - : sorted(variable, variable + n_), - prefix_power(static_cast(degree_) + 1U), - total_power(static_cast(degree_) + 1U, 0.0), - choose(binomial_coefficients(degree_)), - n(n_), - degree(degree_), - shift(0.0) { + : sorted(variable, variable + n_), n(n_), degree(degree_) { for (std::size_t i = 0; i < n; ++i) { if (!std::isfinite(sorted[i])) { // defensive guard, as upstream sorted.clear(); @@ -95,22 +110,61 @@ struct PrefixBackend { } } std::sort(sorted.begin(), sorted.end()); - shift = sorted[n / 2U]; - for (int p = 0; p <= degree; ++p) - prefix_power[static_cast(p)].assign(n + 1U, 0.0); - for (std::size_t i = 0; i < n; ++i) { - const double x = sorted[i] - shift; - double x_power = 1.0; - for (int p = 0; p <= degree; ++p) { - const std::size_t ps = static_cast(p); - prefix_power[ps][i + 1U] = prefix_power[ps][i] + x_power; - x_power *= x; + const std::size_t w = static_cast(degree) + 1U; + + pascal.assign(w * w, 0.0); + for (std::size_t j = 0; j < w; ++j) { + pascal[j * w] = 1.0; + for (std::size_t l = 1; l <= j; ++l) + pascal[j * w + l] = pascal[(j - 1U) * w + l - 1U] + + (l < j ? pascal[(j - 1U) * w + l] : 0.0); + } + + lower.assign((n + 1U) * w, 0.0); + upper.assign((n + 1U) * w, 0.0); + if (n == 0U) return; + std::vector dpow(w); + + lower[1U * w] = 1.0; // k = 1: the single point equals its anchor + for (std::size_t k = 1; k < n; ++k) { + fill_powers(sorted[k] - sorted[k - 1U], dpow); + step(&lower[k * w], &lower[(k + 1U) * w], dpow); + } + upper[(n - 1U) * w] = 1.0; // k = n - 1: the single point equals its anchor + for (std::size_t k = n - 1U; k > 0; --k) { + fill_powers(sorted[k] - sorted[k - 1U], dpow); + step(&upper[k * w], &upper[(k - 1U) * w], dpow); + } + } + + void fill_powers(double base, std::vector& pw) const { + pw[0] = 1.0; + for (std::size_t p = 1; p < pw.size(); ++p) pw[p] = pw[p - 1U] * base; + } + + // next_j = sum_{l <= j} C(j,l) delta^(j-l) prev_l + [j == 0] + void step(const double* prev, double* next, const std::vector& dpow) const { + const std::size_t w = static_cast(degree) + 1U; + for (std::size_t j = 0; j < w; ++j) { + double acc = (j == 0U) ? 1.0 : 0.0; + for (std::size_t l = 0; l <= j; ++l) { + if (prev[l] != 0.0) acc += pascal[j * w + l] * dpow[j - l] * prev[l]; } + next[j] = acc; } - for (int p = 0; p <= degree; ++p) { - const std::size_t ps = static_cast(p); - total_power[ps] = prefix_power[ps][n]; + } + + // sum_j C(d,j) gap^(d-j) sums_j, all terms >= 0. + double combine(const double* sums, double gap) const { + const std::size_t d = static_cast(degree); + const std::size_t w = d + 1U; + double out = 0.0; + double gpow = 1.0; // gap^(d-j), built from j = d downwards + for (std::size_t jj = w; jj-- > 0U;) { + if (sums[jj] != 0.0) out += pascal[d * w + jj] * gpow * sums[jj]; + gpow *= gap; } + return out; } bool ok() const { return n > 0U; } @@ -124,51 +178,21 @@ struct PrefixBackend { double lpm(double target) const { if (!std::isfinite(target)) return kNaN; const std::size_t k = count_leq(target); - const double tc = target - shift; const double nd = static_cast(n); if (degree == 0) return static_cast(k) / nd; - if (degree == 1) - return (static_cast(k) * tc - prefix_power[1][k]) / nd; - if (degree == 2) { - const double t2 = tc * tc; - return (static_cast(k) * t2 - 2.0 * tc * prefix_power[1][k] + - prefix_power[2][k]) / - nd; - } - double out = 0.0; - for (int j = 0; j <= degree; ++j) { - const std::size_t js = static_cast(j); - const double sign = (j % 2 == 0) ? 1.0 : -1.0; - out += choose[js] * sign * std::pow(tc, static_cast(degree - j)) * - prefix_power[js][k]; - } - return out / nd; + if (k == 0U) return 0.0; + const std::size_t w = static_cast(degree) + 1U; + return combine(&lower[k * w], target - sorted[k - 1U]) / nd; } double upm(double target) const { if (!std::isfinite(target)) return kNaN; const std::size_t k = count_leq(target); - const double tc = target - shift; - const std::size_t above = n - k; const double nd = static_cast(n); - if (degree == 0) return static_cast(above) / nd; - const double suffix1 = total_power[1] - prefix_power[1][k]; - if (degree == 1) return (suffix1 - static_cast(above) * tc) / nd; - if (degree == 2) { - const double suffix2 = total_power[2] - prefix_power[2][k]; - const double t2 = tc * tc; - return (suffix2 - 2.0 * tc * suffix1 + static_cast(above) * t2) / - nd; - } - double out = 0.0; - for (int j = 0; j <= degree; ++j) { - const std::size_t js = static_cast(j); - const double suffix_j = total_power[js] - prefix_power[js][k]; - const double sign = ((degree - j) % 2 == 0) ? 1.0 : -1.0; - out += choose[js] * sign * std::pow(tc, static_cast(degree - j)) * - suffix_j; - } - return out / nd; + if (degree == 0) return static_cast(n - k) / nd; + if (k == n) return 0.0; + const std::size_t w = static_cast(degree) + 1U; + return combine(&upper[k * w], sorted[k] - target) / nd; } std::pair both(double target) const { @@ -192,6 +216,7 @@ std::shared_ptr make_prefix_backend(double degree, // --- univariate scalar kernels (ports of LPM_C / UPM_C) -------------------- double lpm(double degree, double target, const double* x, std::size_t n) { + check_degree(degree); double out = 0; const bool deg_is_int = is_integer(degree); for (std::size_t i = 0; i < n; ++i) { @@ -214,6 +239,7 @@ double lpm(double degree, double target, const double* x, std::size_t n) { } double upm(double degree, double target, const double* x, std::size_t n) { + check_degree(degree); double out = 0; const bool deg_is_int = is_integer(degree); for (std::size_t i = 0; i < n; ++i) { @@ -245,6 +271,7 @@ template void pm_vectorized(double degree, const double* target, std::size_t n_targets, const double* x, std::size_t n, double* out, int n_threads) { + check_degree(degree); // Direct path for few targets: bit-identical to pre-13.0 semantics and // avoids building the prefix backend (port of the <=32 guard). const bool few_targets = n_targets <= kDirectPathMaxTargets; @@ -286,7 +313,9 @@ void pm_vectorized(double degree, const double* target, std::size_t n_targets, if (few_targets) { body(0, n_targets); // serial direct path, as upstream } else { - parallel_for(0, n_targets, body, n_threads); + // Prefix lookups are O(log n + degree); the direct scan (fractional or + // > 32 degree) is O(n) per target. + parallel_for(0, n_targets, body, n_threads, prefix ? 64U : n); } } @@ -329,6 +358,8 @@ void upm_ratio_v(double degree, const double* target, std::size_t n_targets, double co_upm(double degree_x, double degree_y, const double* x, const double* y, std::size_t n_x, std::size_t n_y, double target_x, double target_y) { + check_degree(degree_x, "degree_x"); + check_degree(degree_y, "degree_y"); const std::size_t max_size = (n_x > n_y ? n_x : n_y); const std::size_t min_size = (n_x < n_y ? n_x : n_y); if (min_size == 0) return 0; @@ -360,6 +391,8 @@ double co_upm(double degree_x, double degree_y, const double* x, double co_lpm(double degree_x, double degree_y, const double* x, const double* y, std::size_t n_x, std::size_t n_y, double target_x, double target_y) { + check_degree(degree_x, "degree_x"); + check_degree(degree_y, "degree_y"); const std::size_t max_size = (n_x > n_y ? n_x : n_y); const std::size_t min_size = (n_x < n_y ? n_x : n_y); if (min_size == 0) return 0; @@ -391,6 +424,8 @@ double co_lpm(double degree_x, double degree_y, const double* x, double d_lpm(double degree_lpm, double degree_upm, const double* x, const double* y, std::size_t n_x, std::size_t n_y, double target_x, double target_y) { + check_degree(degree_lpm, "degree_lpm"); + check_degree(degree_upm, "degree_upm"); const std::size_t max_size = (n_x > n_y ? n_x : n_y); const std::size_t min_size = (n_x < n_y ? n_x : n_y); if (min_size == 0) return 0; @@ -429,6 +464,8 @@ double d_lpm(double degree_lpm, double degree_upm, const double* x, double d_upm(double degree_lpm, double degree_upm, const double* x, const double* y, std::size_t n_x, std::size_t n_y, double target_x, double target_y) { + check_degree(degree_lpm, "degree_lpm"); + check_degree(degree_upm, "degree_upm"); const std::size_t max_size = (n_x > n_y ? n_x : n_y); const std::size_t min_size = (n_x < n_y ? n_x : n_y); if (min_size == 0) return 0; @@ -471,7 +508,8 @@ namespace { template void two_var_vectorized(ScalarFn&& scalar, const double* target_x, std::size_t n_tx, const double* target_y, - std::size_t n_ty, double* out, int n_threads) { + std::size_t n_ty, double* out, int n_threads, + std::size_t work_per_target) { const std::size_t n_out = (n_tx > n_ty ? n_tx : n_ty); parallel_for( 0, n_out, @@ -479,7 +517,7 @@ void two_var_vectorized(ScalarFn&& scalar, const double* target_x, for (std::size_t i = begin; i < end; ++i) out[i] = scalar(target_x[i % n_tx], target_y[i % n_ty]); }, - n_threads); + n_threads, work_per_target); } } // namespace @@ -488,44 +526,56 @@ void co_lpm_v(double degree_x, double degree_y, const double* x, const double* y, std::size_t n_x, std::size_t n_y, const double* target_x, std::size_t n_tx, const double* target_y, std::size_t n_ty, double* out, int n_threads) { + // Validate before the parallel region: an exception must not escape a worker. + check_degree(degree_x, "degree_x"); + check_degree(degree_y, "degree_y"); two_var_vectorized( [&](double tx, double ty) { return co_lpm(degree_x, degree_y, x, y, n_x, n_y, tx, ty); }, - target_x, n_tx, target_y, n_ty, out, n_threads); + target_x, n_tx, target_y, n_ty, out, n_threads, std::min(n_x, n_y)); } void co_upm_v(double degree_x, double degree_y, const double* x, const double* y, std::size_t n_x, std::size_t n_y, const double* target_x, std::size_t n_tx, const double* target_y, std::size_t n_ty, double* out, int n_threads) { + // Validate before the parallel region: an exception must not escape a worker. + check_degree(degree_x, "degree_x"); + check_degree(degree_y, "degree_y"); two_var_vectorized( [&](double tx, double ty) { return co_upm(degree_x, degree_y, x, y, n_x, n_y, tx, ty); }, - target_x, n_tx, target_y, n_ty, out, n_threads); + target_x, n_tx, target_y, n_ty, out, n_threads, std::min(n_x, n_y)); } void d_lpm_v(double degree_lpm, double degree_upm, const double* x, const double* y, std::size_t n_x, std::size_t n_y, const double* target_x, std::size_t n_tx, const double* target_y, std::size_t n_ty, double* out, int n_threads) { + // Validate before the parallel region: an exception must not escape a worker. + check_degree(degree_lpm, "degree_lpm"); + check_degree(degree_upm, "degree_upm"); two_var_vectorized( [&](double tx, double ty) { return d_lpm(degree_lpm, degree_upm, x, y, n_x, n_y, tx, ty); }, - target_x, n_tx, target_y, n_ty, out, n_threads); + target_x, n_tx, target_y, n_ty, out, n_threads, std::min(n_x, n_y)); } void d_upm_v(double degree_lpm, double degree_upm, const double* x, const double* y, std::size_t n_x, std::size_t n_y, const double* target_x, std::size_t n_tx, const double* target_y, std::size_t n_ty, double* out, int n_threads) { + // Validate before the parallel region: an exception must not escape a worker. + check_degree(degree_lpm, "degree_lpm"); + check_degree(degree_upm, "degree_upm"); two_var_vectorized( [&](double tx, double ty) { return d_upm(degree_lpm, degree_upm, x, y, n_x, n_y, tx, ty); }, - target_x, n_tx, target_y, n_ty, out, n_threads); + target_x, n_tx, target_y, n_ty, out, n_threads, std::min(n_x, n_y)); } // --- n-dimensional co-partial moments (ports of *_nD_cpp) ------------------ @@ -547,14 +597,14 @@ void check_nd_args(std::size_t n, std::size_t d) { // Mirrors R's parallelFor-into-vector + sum() pattern (same summation order // as the upstream NumericVector accumulation). template -double reduce_rows(std::size_t n, int n_threads, RowFn&& row_value) { +double reduce_rows(std::size_t n, std::size_t d, int n_threads, RowFn&& row_value) { std::vector out(n); parallel_for( 0, n, [&](std::size_t begin, std::size_t end) { for (std::size_t i = begin; i < end; ++i) out[i] = row_value(i); }, - n_threads); + n_threads, d); double total = 0.0; for (std::size_t i = 0; i < n; ++i) total += out[i]; return total; @@ -564,14 +614,14 @@ double clpm_nd_impl(const double* data, std::size_t n, std::size_t d, const double* target, double degree, int n_threads) { const bool deg_is_int = is_integer(degree); if (degree == 0.0) { - const double count = reduce_rows(n, n_threads, [&](std::size_t i) { + const double count = reduce_rows(n, d, n_threads, [&](std::size_t i) { for (std::size_t j = 0; j < d; ++j) if (at(data, n, i, j) > target[j]) return 0.0; return 1.0; }); return count / static_cast(n); } - const double s = reduce_rows(n, n_threads, [&](std::size_t i) { + const double s = reduce_rows(n, d, n_threads, [&](std::size_t i) { double prod = 1.0; for (std::size_t j = 0; j < d; ++j) { const double diff = target[j] - at(data, n, i, j); @@ -588,14 +638,16 @@ double cupm_nd_impl(const double* data, std::size_t n, std::size_t d, const double* target, double degree, int n_threads) { const bool deg_is_int = is_integer(degree); if (degree == 0.0) { - const double count = reduce_rows(n, n_threads, [&](std::size_t i) { + // Strictly above, as Co.UPM does: ties go to the lower side so that at + // degree 0 CLPM + CUPM + DPM partition the sample. + const double count = reduce_rows(n, d, n_threads, [&](std::size_t i) { for (std::size_t j = 0; j < d; ++j) - if (at(data, n, i, j) < target[j]) return 0.0; + if (at(data, n, i, j) <= target[j]) return 0.0; return 1.0; }); return count / static_cast(n); } - const double s = reduce_rows(n, n_threads, [&](std::size_t i) { + const double s = reduce_rows(n, d, n_threads, [&](std::size_t i) { double prod = 1.0; for (std::size_t j = 0; j < d; ++j) { const double diff = at(data, n, i, j) - target[j]; @@ -612,11 +664,12 @@ double dpm_nd_impl(const double* data, std::size_t n, std::size_t d, const double* target, double degree, int n_threads) { const bool deg_is_int = is_integer(degree); if (degree == 0.0) { - const double count = reduce_rows(n, n_threads, [&](std::size_t i) { + // Discordant = neither all-lower (x <= target) nor all-upper (x > target). + const double count = reduce_rows(n, d, n_threads, [&](std::size_t i) { bool all_below = true, all_above = true; for (std::size_t j = 0; j < d; ++j) { const double diff = at(data, n, i, j) - target[j]; - if (diff >= 0.0) all_below = false; + if (diff > 0.0) all_below = false; if (diff <= 0.0) all_above = false; if (!all_below && !all_above) break; } @@ -624,7 +677,7 @@ double dpm_nd_impl(const double* data, std::size_t n, std::size_t d, }); return count / static_cast(n); } - const double s = reduce_rows(n, n_threads, [&](std::size_t i) { + const double s = reduce_rows(n, d, n_threads, [&](std::size_t i) { bool all_below = true, all_above = true; for (std::size_t j = 0; j < d; ++j) { const double diff = at(data, n, i, j) - target[j]; @@ -649,6 +702,7 @@ double dpm_nd_impl(const double* data, std::size_t n, std::size_t d, double clpm_nd(const double* data, std::size_t n, std::size_t d, const double* target, double degree, bool norm, int n_threads) { + check_degree(degree); check_nd_args(n, d); const double clpm_un = clpm_nd_impl(data, n, d, target, degree, n_threads); if (degree == 0.0 || !norm) return clpm_un; @@ -660,6 +714,7 @@ double clpm_nd(const double* data, std::size_t n, std::size_t d, double cupm_nd(const double* data, std::size_t n, std::size_t d, const double* target, double degree, bool norm, int n_threads) { + check_degree(degree); check_nd_args(n, d); const double cupm_un = cupm_nd_impl(data, n, d, target, degree, n_threads); if (degree == 0.0 || !norm) return cupm_un; @@ -671,6 +726,7 @@ double cupm_nd(const double* data, std::size_t n, std::size_t d, double dpm_nd(const double* data, std::size_t n, std::size_t d, const double* target, double degree, bool norm, int n_threads) { + check_degree(degree); check_nd_args(n, d); const double dpm_un = dpm_nd_impl(data, n, d, target, degree, n_threads); if (degree == 0.0 || !norm) return dpm_un; @@ -685,6 +741,7 @@ double dpm_nd(const double* data, std::size_t n, std::size_t d, void clpm_nd_batch(const double* data, std::size_t n, std::size_t d, const double* targets, std::size_t n_targets, double degree, bool norm, double* out, int n_threads) { + check_degree(degree); if (n == 0) throw std::invalid_argument("`data` must have at least one row"); const bool deg_is_int = is_integer(degree); @@ -740,7 +797,7 @@ void clpm_nd_batch(const double* data, std::size_t n, std::size_t d, } } }, - n_threads); + n_threads, n * d); } // --- PM matrix (port of the 13.0 tensorized PMMatrix_CPv) ------------------- @@ -749,6 +806,8 @@ PMMatrixResult pm_matrix(double degree_lpm, double degree_upm, const double* target, const double* variable, std::size_t n, std::size_t d, bool pop_adj, bool norm, int n_threads) { + check_degree(degree_lpm, "degree_lpm"); + check_degree(degree_upm, "degree_upm"); // Mirrors Rcpp::stop("variable matrix cols != target vector length") — // bindings pass target length separately, so enforce d > 0 here and let // the binding layer validate target length against d before calling. @@ -780,7 +839,7 @@ PMMatrixResult pm_matrix(double degree_lpm, double degree_upm, } } }, - n_threads); + n_threads, n); double adjust = 1.0; if (pop_adj && n > 1) @@ -836,7 +895,7 @@ PMMatrixResult pm_matrix(double degree_lpm, double degree_upm, } } }, - n_threads); + n_threads, (d + 1) * n / 2 + 1); // row i contracts d - i column pairs // Step 3: cellular normalization, preserving the crossed mirror. if (norm) { @@ -874,4 +933,4 @@ PMMatrixResult pm_matrix(double degree_lpm, double degree_upm, return res; } -} // namespace nns \ No newline at end of file +} // namespace nns diff --git a/extern/NNS-core/src/partition.cpp b/extern/NNS-core/src/partition.cpp index b0051e7a..0d46896f 100644 --- a/extern/NNS-core/src/partition.cpp +++ b/extern/NNS-core/src/partition.cpp @@ -1,6 +1,6 @@ // src/partition.cpp // -// Pure C++ port of NNS 13.0 NNS_part.cpp. Decoupled from Rcpp. +// Pure C++ port of NNS NNS_part.cpp (synced to R 13.3, 82a92eb). Decoupled from Rcpp. // // This file preserves the original NNS_part_cpp semantics: // - arguments: x, y, type, order_in, obs_req, min_obs_stop, @@ -33,11 +33,15 @@ namespace { constexpr double kNaN = std::numeric_limits::quiet_NaN(); +// R 13.2: NNS_part treats +-Inf as values and drops only NaN/NA +// (complete_case_value = !ISNAN), so -Inf lands below any finite centre. +inline bool complete_case_value(double x) { return !std::isnan(x); } + inline double mean_no_na(const std::vector& v) { long double s = 0.0L; std::size_t m = 0; for (double xi : v) { - if (std::isfinite(xi)) { + if (complete_case_value(xi)) { s += xi; ++m; } @@ -49,7 +53,7 @@ inline double median_no_na(const std::vector& v) { std::vector a; a.reserve(v.size()); for (double xi : v) { - if (std::isfinite(xi)) a.push_back(xi); + if (complete_case_value(xi)) a.push_back(xi); } if (a.empty()) return kNaN; @@ -174,11 +178,11 @@ PartitionResult partition(const double* x, xv[k] = xi; yv[k] = yi; - if (std::isfinite(xi)) { + if (complete_case_value(xi)) { if (xi < minx) minx = xi; if (xi > maxx) maxx = xi; } - if (std::isfinite(yi)) { + if (complete_case_value(yi)) { if (yi < miny) miny = yi; if (yi > maxy) maxy = yi; } @@ -188,12 +192,12 @@ PartitionResult partition(const double* x, centers[q] = c; if (!xonly) { - if (std::isfinite(c.y) && std::isfinite(minx) && std::isfinite(maxx)) { + if (complete_case_value(c.y) && complete_case_value(minx) && complete_case_value(maxx)) { H_x0.push_back(minx); H_x1.push_back(maxx); H_y.push_back(c.y); } - if (std::isfinite(c.x) && std::isfinite(miny) && std::isfinite(maxy)) { + if (complete_case_value(c.x) && complete_case_value(miny) && complete_case_value(maxy)) { V_x.push_back(c.x); V_y0.push_back(miny); V_y1.push_back(maxy); @@ -208,13 +212,13 @@ PartitionResult partition(const double* x, double maxx = -std::numeric_limits::infinity(); for (int i : idx) { const double xi = x[i]; - if (std::isfinite(xi)) { + if (complete_case_value(xi)) { if (xi < minx) minx = xi; if (xi > maxx) maxx = xi; } } - if (std::isfinite(minx)) V_lines.push_back(minx); - if (std::isfinite(maxx)) V_lines.push_back(maxx); + if (complete_case_value(minx)) V_lines.push_back(minx); + if (complete_case_value(maxx)) V_lines.push_back(maxx); } } @@ -225,11 +229,11 @@ PartitionResult partition(const double* x, int qn = 1; if (!xonly) { - const int lox = (std::isfinite(x[i]) && std::isfinite(c.x)) ? (x[i] <= c.x) : 0; - const int loy = (std::isfinite(y[i]) && std::isfinite(c.y)) ? (y[i] <= c.y) : 0; + const int lox = (complete_case_value(x[i]) && complete_case_value(c.x)) ? (x[i] <= c.x) : 0; + const int loy = (complete_case_value(y[i]) && complete_case_value(c.y)) ? (y[i] <= c.y) : 0; qn = 1 + lox + 2 * loy; } else { - const int lox = (std::isfinite(x[i]) && std::isfinite(c.x)) ? (x[i] > c.x) : 0; + const int lox = (complete_case_value(x[i]) && complete_case_value(c.x)) ? (x[i] > c.x) : 0; qn = 1 + lox; } diff --git a/extern/NNS-core/src/seasonality.cpp b/extern/NNS-core/src/seasonality.cpp index c38904ce..71e420e1 100644 --- a/extern/NNS-core/src/seasonality.cpp +++ b/extern/NNS-core/src/seasonality.cpp @@ -1,11 +1,13 @@ // src/seasonality.cpp // -// Implementation extracted from NNS 13.0 NNS_seas.cpp. Decoupled from Rcpp. +// Implementation extracted from NNS NNS_seas.cpp (R 13.3, 82a92eb), scoring every +// component series of each lag as later R does. Decoupled from Rcpp. // // SPDX-License-Identifier: GPL-3.0-only #include "nns/seasonality.hpp" #include +#include #include #include #include @@ -18,7 +20,6 @@ namespace nns { namespace { constexpr double kNaN = std::numeric_limits::quiet_NaN(); -constexpr double kInf = std::numeric_limits::infinity(); inline bool any_na_or_inf(const double* x, std::size_t n) { for (std::size_t i = 0; i < n; ++i) { @@ -46,58 +47,6 @@ inline double vec_sd(const std::vector& x) { return std::sqrt(ss / static_cast(n - 1)); } -// lag-1 Pearson autocorrelation -inline double acf1(const std::vector& x) { - std::size_t n = x.size(); - if (n < 2) return kNaN; - double m = vec_mean(x); - double num = 0.0; - double den = 0.0; - for (std::size_t t = 1; t < n; ++t) num += (x[t] - m) * (x[t - 1] - m); - for (std::size_t t = 0; t < n; ++t) { - double d = x[t] - m; - den += d * d; - } - if (den == 0.0) return kNaN; - return num / den; -} - -inline double cv_or_fallback(const std::vector& x, bool use_cv, double var_cov) { - std::size_t n = x.size(); - if (n < 2) return var_cov; - double z; - if (use_cv) { - double m = vec_mean(x); - double s = vec_sd(x); - z = std::fabs(s / m); - } else { - double a1 = acf1(x); - if (!std::isfinite(a1)) return var_cov; - z = std::pow(std::fabs(a1), -1.0); - } - if (!std::isfinite(z)) return var_cov; - return z; -} - -// 0-based indices stepping backwards -inline std::vector rev_step_indices(int n, int step) { - int len = (n - 1) / step + 1; - std::vector out(len); - int v = n - 1; // 0-based max index - for (int k = 0; k < len; ++k, v -= step) out[k] = v; - return out; -} - -inline std::vector take_by_index(const std::vector& x, const std::vector& idx) { - std::size_t m = idx.size(); - std::vector out(m); - for (std::size_t i = 0; i < m; ++i) { - int j = idx[i]; - out[i] = (j >= 0 && j < static_cast(x.size())) ? x[j] : kNaN; - } - return out; -} - } // namespace // ---------- Core Seasonality API ---------- @@ -118,43 +67,45 @@ SeasonalityResult seasonality(const double* x, std::size_t n, } std::vector variable(x, x + n); - std::vector variable_1(x, x + n - 1); - std::vector variable_2; - if (n - 1 >= 2) variable_2.assign(x, x + n - 2); - const int half_n = static_cast(n) / 2; + const int ni = static_cast(n); + const int half_n = ni / 2; const double mean_var = vec_mean(variable); - const bool use_cv = (mean_var != 0.0); - - double var_cov = use_cv ? std::fabs(vec_sd(variable) / mean_var) : std::pow(std::fabs(acf1(variable)), -1.0); - if (!std::isfinite(var_cov)) var_cov = kInf; - - std::vector out(half_n), out1(half_n), out2(half_n); - std::vector inst(half_n, 0), inst1(half_n, 0), inst2(half_n, 0); - - const int n1 = static_cast(n) - 1; - const int n2 = static_cast(variable_2.size()); - - for (int i = 1; i <= half_n; ++i) { - std::vector idx = rev_step_indices(static_cast(n), i); - std::vector idx1 = rev_step_indices(n1, i); - std::vector idx2 = (n2 > 0) ? rev_step_indices(n2, i) : std::vector(); - - double t = cv_or_fallback(take_by_index(variable, idx), use_cv, var_cov); - double t1 = cv_or_fallback(take_by_index(variable_1, idx1), use_cv, var_cov); - double t2 = cv_or_fallback(take_by_index(variable_2, idx2), use_cv, var_cov); - - if (t <= var_cov) { inst[i - 1] = i; out[i - 1] = t; } - if (t1 <= var_cov) { inst1[i - 1] = i; out1[i - 1] = t1; } - if (t2 <= var_cov) { inst2[i - 1] = i; out2[i - 1] = t2; } - } - + // Compare raw values: a repeated 0.1 need not centre to exactly 0. + const bool constant = std::all_of(variable.begin(), variable.end(), + [&](double v) { return v == variable[0]; }); + const double sd_var = constant ? 0.0 : vec_sd(variable); + // CV is meaningless for a mean that is zero up to rounding; compare sds instead + const double tol = std::sqrt(DBL_EPSILON); + const bool use_cv = std::fabs(mean_var) > tol * sd_var; + const double var_cov = use_cv ? std::fabs(sd_var / mean_var) : sd_var; + + // Lag m: length-weighted mean CV of its m component series x[j], x[j + m], ... std::vector periods_vec; std::vector cvmean_vec; - for (int i = 0; i < half_n; ++i) { - if (inst[i] > 0 && inst1[i] > 0 && inst2[i] > 0) { - periods_vec.push_back(inst[i]); - cvmean_vec.push_back((out[i] + out1[i] + out2[i]) / 3.0); + for (int m = 1; m <= half_n; ++m) { + double wsum = 0.0; + for (int j = 0; j < m; ++j) { + int len = (ni - 1 - j) / m + 1; + double mu = 0.0, ss = 0.0; + bool same = true; + for (int i = j; i < ni; i += m) { + mu += x[i]; + same = same && x[i] == x[j]; + } + mu /= static_cast(len); + for (int i = j; i < ni; i += m) { + double d = x[i] - mu; + ss += d * d; + } + double sd = same ? 0.0 : std::sqrt(ss / static_cast(len - 1)); + // a constant component has CV 0; a near-zero mean caps CV at 1/tol, where the series switches to sd + wsum += len * (use_cv ? (sd == 0.0 ? 0.0 : sd / std::max(std::fabs(mu), tol * sd)) : sd); + } + double cv_m = wsum / static_cast(ni); + if (cv_m < var_cov) { + periods_vec.push_back(m); + cvmean_vec.push_back(cv_m); } } @@ -170,7 +121,8 @@ SeasonalityResult seasonality(const double* x, std::size_t n, std::vector ord(m); std::iota(ord.begin(), ord.end(), 0); - std::sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); + // Equal CVs keep the shorter period first. + std::stable_sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); std::vector sortedP(m); std::vector sortedCV(m); @@ -230,7 +182,7 @@ SeasonalityResult seasonality(const double* x, std::size_t n, std::vector ord(m); std::iota(ord.begin(), ord.end(), 0); - std::sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); + std::stable_sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); std::vector sortedP(m); std::vector sortedCV(m); @@ -259,7 +211,7 @@ SeasonalityResult seasonality(const double* x, std::size_t n, int m = static_cast(Period.size()); std::vector ord(m); std::iota(ord.begin(), ord.end(), 0); - std::sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); + std::stable_sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); std::vector sortedP(m); std::vector sortedCV(m); @@ -294,7 +246,7 @@ SeasonalityResult seasonality(const double* x, std::size_t n, std::vector ord(m); std::iota(ord.begin(), ord.end(), 0); - std::sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); + std::stable_sort(ord.begin(), ord.end(), [&](int a, int b) { return CoefVar[a] < CoefVar[b]; }); std::vector sortedP(m); std::vector sortedCV(m); diff --git a/extern/NNS-core/src/stack_fast.cpp b/extern/NNS-core/src/stack_fast.cpp new file mode 100644 index 00000000..2d8f42d0 --- /dev/null +++ b/extern/NNS-core/src/stack_fast.cpp @@ -0,0 +1,90 @@ +// src/stack_fast.cpp +// +// Pure C++ port of R NNS 13.3 src/NNS_stack_fast.cpp (NNS_xstar_path_cpp, +// the NNS.stack X* projection path). Decoupled from Rcpp / RcppParallel. +// +// SPDX-License-Identifier: GPL-3.0-only +#include +#include + +#include "nns/mreg.hpp" +#include "nns/parallel.hpp" + +namespace nns { + +namespace { + +// Running weighted sum over the ordered columns, divided by the number of +// non-zero coefficients so far (port of FillProjectionWorker). +void fill_projection(const double* design, std::size_t rows, std::size_t p, + const std::vector& order, const double* coef, + const std::vector& denom, bool all_zero, double* out, + int n_threads) { + parallel_for( + 0, rows, + [&](std::size_t begin, std::size_t end) { + for (std::size_t i = begin; i < end; ++i) { + double numerator = 0.0; + for (std::size_t m = 0; m < p; ++m) { + const std::size_t j = order[m]; + const double c = all_zero ? 1.0 : coef[j]; + numerator += design[i + j * rows] * c; + out[i + m * rows] = numerator / denom[m]; + } + } + }, + n_threads, p); +} + +} // namespace + +XstarPathResult xstar_path(const double* train_design, std::size_t n_train, + const double* test_design, std::size_t n_test, std::size_t p, + const double* coefficients, const std::size_t* column_order, + int n_threads) { + XstarPathResult res; + res.n_train = n_train; + res.n_test = n_test; + res.p = p; + res.column_order.assign(column_order, column_order + p); + for (std::size_t m = 0; m < p; ++m) + if (res.column_order[m] >= p) + throw std::invalid_argument("xstar_path: column_order entries must be in [0, p)"); + + bool all_zero = true; + for (std::size_t j = 0; j < p; ++j) + if (coefficients[j] != 0.0) all_zero = false; + res.denominator.resize(p); + double den = 0.0; + for (std::size_t m = 0; m < p; ++m) { + const std::size_t j = res.column_order[m]; + if (all_zero || coefficients[j] != 0.0) den += 1.0; + res.denominator[m] = den; + } + + res.train.assign(n_train * p, 0.0); + res.test.assign(n_test * p, 0.0); + fill_projection(train_design, n_train, p, res.column_order, coefficients, res.denominator, + all_zero, res.train.data(), n_threads); + fill_projection(test_design, n_test, p, res.column_order, coefficients, res.denominator, + all_zero, res.test.data(), n_threads); + + res.representative.resize(p); + for (std::size_t m = 0; m < p; ++m) { + res.representative[m] = m; + for (std::size_t r = 0; r < m; ++r) { + bool same = true; + for (std::size_t i = 0; i < n_train && same; ++i) + same = (res.train[i + m * n_train] == res.train[i + r * n_train]); + for (std::size_t i = 0; i < n_test && same; ++i) + same = (res.test[i + m * n_test] == res.test[i + r * n_test]); + if (same) { + res.representative[m] = r; + break; + } + } + } + return res; +} + +} // namespace nns diff --git a/extern/NNS-core/src/stochastic_dominance.cpp b/extern/NNS-core/src/stochastic_dominance.cpp index 0ae032e0..1d60a432 100644 --- a/extern/NNS-core/src/stochastic_dominance.cpp +++ b/extern/NNS-core/src/stochastic_dominance.cpp @@ -1,18 +1,20 @@ // src/stochastic_dominance.cpp // -// Faithful port of original_src/SD.cpp and original_src/stoch_sup.cpp. -// Decoupled from Rcpp. +// Port of original_src/SD.cpp and original_src/stoch_sup.cpp, with the later +// R fixes to the pairwise test. Decoupled from Rcpp. // -// This file reproduces, exactly: +// This file reproduces: // - ColPre prefix-sum precompute (sorted values, P1, P2, min, mean) // - identical_samples() short-circuit (identical series never dominate) // - for_each_threshold(): the MERGED grid of both series' values // - sd_dom_pair(): // FSD gate X.mn >= Y.mn; discrete -> ECDF compare, -// continuous -> LPM1/(LPM1+UPM1) ratio compare; strict '>' fails -// SSD gates X.mn >= Y.mn and !(Y.mean > X.mean); LPM degree-1 compare -// TSD same gates; LPM degree-2 compare -// (no epsilon tolerances anywhere, matching upstream) +// continuous -> LPM1/(LPM1+UPM1) ratio compare, also between knots +// SSD gates X.mn >= Y.mn and the mean; LPM degree-1 compare +// TSD same gates; LPM degree-2 compare, also between knots +// (64 * DBL_EPSILON relative tie tolerance except for discrete FSD, +// on the pair centred on its joint midrange; coinciding curves never +// dominate) // - sd_dom_matrix (port of sd_dom_matrix_prefix_parallel) // - the efficient-set sweep (port of NNS_SD_efficient_set_parallel_cpp): // order columns by LPM(degree, tmax, .) ascending (stable tie-break by @@ -29,6 +31,7 @@ #include "nns/parallel.hpp" #include +#include #include #include #include @@ -75,30 +78,55 @@ struct ColPre { int m{0}; }; -ColPre precompute_ptr(const double* x, std::size_t n, std::size_t stride_rows, - std::size_t col) { - ColPre c; - c.m = static_cast(n); - c.vals.resize(n); - for (std::size_t i = 0; i < n; ++i) c.vals[i] = at(x, stride_rows, i, col); - std::sort(c.vals.begin(), c.vals.end()); +inline void neumaier_add(double& sum, double& err, double v) { + double t = sum + v; + err += (std::fabs(sum) >= std::fabs(v)) ? (sum - t) + v : (v - t) + sum; + sum = t; +} +// Prefix sums and stats of the already sorted c.vals. +void finish_col(ColPre& c) { + const std::size_t n = c.vals.size(); c.P1.assign(n + 1, 0.0); c.P2.assign(n + 1, 0.0); + // Compensated: plain running sums drift by ~n * eps * |x|, which for large + // samples exceeds the tie tolerance in sd_dom_pair. + double s1 = 0.0, e1 = 0.0, s2 = 0.0, e2 = 0.0; for (std::size_t k = 1; k <= n; ++k) { double v = c.vals[k - 1]; - c.P1[k] = c.P1[k - 1] + v; - c.P2[k] = c.P2[k - 1] + v * v; + neumaier_add(s1, e1, v); + neumaier_add(s2, e2, v * v); + c.P1[k] = s1 + e1; + c.P2[k] = s2 + e2; } c.S1 = c.P1[n]; c.S2 = c.P2[n]; - if (n > 0) { - c.mn = c.vals.front(); - c.mean = c.S1 / static_cast(n); - } + c.mn = c.vals.front(); + c.mean = c.S1 / static_cast(n); +} + +ColPre precompute_ptr(const double* x, std::size_t n, std::size_t stride_rows, + std::size_t col) { + if (n == 0) throw std::invalid_argument("Each sample needs at least one observation."); + ColPre c; + c.m = static_cast(n); + c.vals.resize(n); + for (std::size_t i = 0; i < n; ++i) c.vals[i] = at(x, stride_rows, i, col); + std::sort(c.vals.begin(), c.vals.end()); + finish_col(c); return c; } +// Same sample shifted by s; subtraction keeps the sort order. +ColPre shifted_col(const ColPre& c, double s) { + ColPre o; + o.m = c.m; + o.vals.resize(c.vals.size()); + for (std::size_t i = 0; i < c.vals.size(); ++i) o.vals[i] = c.vals[i] - s; + finish_col(o); + return o; +} + ColPre precompute_vec(const double* x, std::size_t n) { return precompute_ptr(x, n, n, 0); } @@ -129,35 +157,64 @@ inline double lpm_deg2(const ColPre& c, int k, double t) { } // Walk merged grid of both series' values and apply functor at each -// threshold t. Preserves the upstream quirk of bounding both walkers by -// a.m (columns of one matrix always share the same row count). +// threshold t, with the next threshold t_next: the counts (k_a, k_b) are +// constant on (t, t_next). template inline void for_each_threshold(const ColPre& a, const ColPre& b, F f) { - int ia = 0, ib = 0, m = a.m; - while (ia < m || ib < m) { - double next_a = (ia < m ? a.vals[ia] : kInf); - double next_b = (ib < m ? b.vals[ib] : kInf); + int ia = 0, ib = 0, ma = a.m, mb = b.m; + while (ia < ma || ib < mb) { + double next_a = (ia < ma ? a.vals[ia] : kInf); + double next_b = (ib < mb ? b.vals[ib] : kInf); double t = (next_a < next_b ? next_a : next_b); - while (ia < m && a.vals[ia] <= t) ++ia; // k_a = ia - while (ib < m && b.vals[ib] <= t) ++ib; // k_b = ib - f(t, ia, ib); + while (ia < ma && a.vals[ia] <= t) ++ia; // k_a = ia + while (ib < mb && b.vals[ib] <= t) ++ib; // k_b = ib + double t_next = std::min(ia < ma ? a.vals[ia] : kInf, ib < mb ? b.vals[ib] : kInf); + f(t, ia, ib, t_next); } } +// Continuous FSD ratio L1/(L1+U1) on an interval with fixed count k: +// L1 = alpha*t + beta, L1+U1 = gamma*t + delta. +inline void ratio_deg1_coefs(const ColPre& c, int k, double& alpha, double& beta, + double& gamma, double& delta) { + double m = static_cast(c.m); + alpha = k / m; + beta = -c.P1[k] / m; + gamma = (2.0 * k - c.m) / m; + delta = (c.S1 - 2.0 * c.P1[k]) / m; +} + // ===================================================================== // Pairwise dominance via prefix sums (O(m) per pair) // degree: 1=FSD, 2=SSD, 3=TSD. 'discrete' only matters for FSD. // Returns 1 iff X dominates Y, else 0. // ===================================================================== -inline int sd_dom_pair(const ColPre& X, const ColPre& Y, int degree, bool discrete) { +// level: |shift| the pair was centred by (0 if not), whose rounding the data still carry. +inline int sd_dom_pair(const ColPre& X, const ColPre& Y, int degree, bool discrete, + double level = 0.0) { + // Curves equal in exact arithmetic differ by rounding of order + // eps * S^degree (S = largest |value|); smaller differences are ties. + const double S = std::max(std::max(std::fabs(X.vals.front()), std::fabs(X.vals.back())), + std::max(std::fabs(Y.vals.front()), std::fabs(Y.vals.back()))); + const double eps64 = 64.0 * DBL_EPSILON; + // Bounds add the inputs' own rounding at |level|; for LPM2 at t it moves + // each term by about 2 (t - x) din, so that bound uses LPM1 at t. + const double din = 4.0 * DBL_EPSILON * level; + const double E1 = eps64 * S + din; + auto E2 = [&](int kx, int ky, double t) { + double Lx, Ux, Ly, Uy; + lpm_upm_deg1(X, kx, t, Lx, Ux); + lpm_upm_deg1(Y, ky, t, Ly, Uy); + return eps64 * S * S + 2.0 * din * (std::fabs(Lx) + std::fabs(Ly) + din); + }; if (degree == 1) { // FSD if (!(X.mn >= Y.mn)) return 0; // FSD gate if (identical_samples(X, Y)) return 0; // identical series -> 0 - bool x_gt_y = false; + bool x_gt_y = false, curves_differ = false; int deg = (discrete ? 0 : 1); // discrete->0, continuous->1 - for_each_threshold(X, Y, [&](double t, int kx, int ky) { - double Rx, Ry; + for_each_threshold(X, Y, [&](double t, int kx, int ky, double t_next) { + double Rx, Ry, tol = 0.0; if (deg == 0) { // L0/(L0+U0) == ECDF Rx = static_cast(kx) / static_cast(X.m); @@ -167,40 +224,116 @@ inline int sd_dom_pair(const ColPre& X, const ColPre& Y, int degree, bool discre lpm_upm_deg1(X, kx, t, Lx, Ux); lpm_upm_deg1(Y, ky, t, Ly, Uy); double Ax = Lx + Ux, Ay = Ly + Uy; - Rx = (Ax > 0.0 ? Lx / Ax : 0.0); - Ry = (Ay > 0.0 ? Ly / Ay : 0.0); + // 0/0 (every observation equals t): the CDF value k/m, as LPM.ratio. + Rx = (Ax > 0.0 ? Lx / Ax : static_cast(kx) / static_cast(X.m)); + Ry = (Ay > 0.0 ? Ly / Ay : static_cast(ky) / static_cast(Y.m)); + double Amin = std::min(Ax, Ay); + tol = (Amin > 0.0) ? std::max(eps64, E1 / Amin) : eps64; + } + if (Rx - Ry > tol) x_gt_y = true; + if (std::fabs(Rx - Ry) > tol) curves_differ = true; + + // The ratios are linear-fractional between knots, so their difference + // can peak inside (t, t_next), where Dy(t) = sqrt(Ay / Ax) * Dx(t). + if (deg == 1 && std::isfinite(t_next) && !x_gt_y) { + double ax, bx, gx, dx, ay, by, gy, dy; + ratio_deg1_coefs(X, kx, ax, bx, gx, dx); + ratio_deg1_coefs(Y, ky, ay, by, gy, dy); + double Ax = ax * dx - bx * gx, Ay = ay * dy - by * gy; + if (Ax != 0.0 && Ay != 0.0 && ((Ax > 0.0) == (Ay > 0.0))) { + double r = std::sqrt(Ay / Ax); + double den = gy - r * gx; + if (den != 0.0) { + double ts = (r * dx - dy) / den; + if (ts > t && ts < t_next) { + double Dx = gx * ts + dx, Dy = gy * ts + dy; + if (Dx > 0.0 && Dy > 0.0) { + double Rxs = (ax * ts + bx) / Dx, Rys = (ay * ts + by) / Dy; + double tol_r = std::max(eps64, E1 / std::min(Dx, Dy)); + if (Rxs - Rys > tol_r) { x_gt_y = true; curves_differ = true; } + } + } + } + } + } + // Ratios equal at both knots and the midpoint are equal on the interval. + if (deg == 1 && std::isfinite(t_next) && !curves_differ) { + double s = 0.5 * (t + t_next), Lx, Ux, Ly, Uy; + lpm_upm_deg1(X, kx, s, Lx, Ux); + lpm_upm_deg1(Y, ky, s, Ly, Uy); + double Dx = Lx + Ux, Dy = Ly + Uy; + double d = Lx / Dx - Ly / Dy; + double tol_m = std::max(eps64, E1 / std::min(Dx, Dy)); + if (std::fabs(d) > tol_m) curves_differ = true; + if (d > tol_m) x_gt_y = true; } - if (Rx > Ry) x_gt_y = true; }); + if (!curves_differ) return 0; return x_gt_y ? 0 : 1; // 1 iff "X FSD Y" } // SSD/TSD gates - if (!(X.mn >= Y.mn) || (Y.mean > X.mean)) return 0; + if (!(X.mn >= Y.mn) || (Y.mean - X.mean > E1)) return 0; if (identical_samples(X, Y)) return 0; // identical series -> 0 - if (degree == 2) { // SSD: compare LPM degree 1 - bool x_gt_y = false; - for_each_threshold(X, Y, [&](double t, int kx, int ky) { + if (degree == 2) { // SSD: compare LPM degree 1, exact at the knots + bool x_gt_y = false, curves_differ = false; + for_each_threshold(X, Y, [&](double t, int kx, int ky, double) { double Lx, Ux, Ly, Uy; (void)Ux; (void)Uy; // not used beyond calc lpm_upm_deg1(X, kx, t, Lx, Ux); lpm_upm_deg1(Y, ky, t, Ly, Uy); - if (Lx > Ly) x_gt_y = true; + if (Lx - Ly > E1) x_gt_y = true; + if (std::fabs(Lx - Ly) > E1) curves_differ = true; }); + if (!curves_differ) return 0; return x_gt_y ? 0 : 1; // 1 iff "X SSD Y" } // TSD: compare LPM degree 2 - bool x_gt_y = false; - for_each_threshold(X, Y, [&](double t, int kx, int ky) { + bool x_gt_y = false, curves_differ = false; + for_each_threshold(X, Y, [&](double t, int kx, int ky, double t_next) { double Lx2 = lpm_deg2(X, kx, t); double Ly2 = lpm_deg2(Y, ky, t); - if (Lx2 > Ly2) x_gt_y = true; + const double tol2 = E2(kx, ky, t); + if (Lx2 - Ly2 > tol2) x_gt_y = true; + if (std::fabs(Lx2 - Ly2) > tol2) curves_differ = true; + // On (t, t_next) the difference is a*s^2 - 2*b*s + c; for a < 0 its + // interior maximum at s = b / a is missed by the knots. + if (std::isfinite(t_next) && !x_gt_y) { + double a = static_cast(kx) / static_cast(X.m) - + static_cast(ky) / static_cast(Y.m); + if (a < 0.0) { + double b = X.P1[kx] / static_cast(X.m) - Y.P1[ky] / static_cast(Y.m); + double ts = b / a; + if (ts > t && ts < t_next) { + double Dts = lpm_deg2(X, kx, ts) - lpm_deg2(Y, ky, ts); + if (Dts > E2(kx, ky, ts)) { x_gt_y = true; curves_differ = true; } + } + } + } }); + if (!curves_differ) return 0; return x_gt_y ? 0 : 1; // 1 iff "X TSD Y" } +// Dominance is translation invariant but the tolerances scale with +// |value|^degree, so compare the pair centred on its joint midrange. +inline int sd_dom_pair_centred(const ColPre& X, const ColPre& Y, int degree, bool discrete) { + if (degree == 1 && discrete) return sd_dom_pair(X, Y, degree, discrete); + if (!(X.mn >= Y.mn) || identical_samples(X, Y)) return 0; + const double lo = std::min(X.vals.front(), Y.vals.front()); + const double hi = std::max(X.vals.back(), Y.vals.back()); + // A mean gap this large fails the centred gate too; skip the copies. + if (degree >= 2 && + Y.mean - X.mean > 2.0 * 64.0 * DBL_EPSILON * std::max(std::fabs(lo), std::fabs(hi))) + return 0; + const double c = 0.5 * lo + 0.5 * hi; + // An infinite value makes the shifted copies NaN, and the threshold walk never ends on NaN. + if (!std::isfinite(c)) return sd_dom_pair(X, Y, degree, discrete); + return sd_dom_pair(shifted_col(X, c), shifted_col(Y, c), degree, discrete, std::fabs(c)); +} + // Dominance matrix over an arbitrary column ordering (parallel over rows). // dom is p x p column-major: dom[j * p + i] = 1 iff column ord[i] dominates // column ord[j]. @@ -214,12 +347,12 @@ std::vector dom_matrix_for_order(const std::vector& cols, for (std::size_t j = 0; j < p; ++j) { dom[j * p + i] = (i == j) ? 0 - : sd_dom_pair(cols[static_cast(ord[i])], - cols[static_cast(ord[j])], - degree, discrete); + : sd_dom_pair_centred(cols[static_cast(ord[i])], + cols[static_cast(ord[j])], + degree, discrete); } } - }, nthreads); + }, nthreads, p * (cols.empty() ? 1U : static_cast(cols[0].m) + 1U)); return dom; } @@ -325,28 +458,29 @@ std::vector sd_dom_matrix(const double* X, std::size_t n, std::size_t p, // ---------- Univariate Wrappers ---------- -int fsd_uni(const double* x, const double* y, std::size_t n, bool discrete) { - check_no_nan(x, n); - check_no_nan(y, n); - ColPre X = precompute_vec(x, n); - ColPre Y = precompute_vec(y, n); - return sd_dom_pair(X, Y, 1, discrete); +int fsd_uni(const double* x, std::size_t n_x, const double* y, std::size_t n_y, + bool discrete) { + check_no_nan(x, n_x); + check_no_nan(y, n_y); + ColPre X = precompute_vec(x, n_x); + ColPre Y = precompute_vec(y, n_y); + return sd_dom_pair_centred(X, Y, 1, discrete); } -int ssd_uni(const double* x, const double* y, std::size_t n) { - check_no_nan(x, n); - check_no_nan(y, n); - ColPre X = precompute_vec(x, n); - ColPre Y = precompute_vec(y, n); - return sd_dom_pair(X, Y, 2, true); // discrete flag irrelevant past FSD +int ssd_uni(const double* x, std::size_t n_x, const double* y, std::size_t n_y) { + check_no_nan(x, n_x); + check_no_nan(y, n_y); + ColPre X = precompute_vec(x, n_x); + ColPre Y = precompute_vec(y, n_y); + return sd_dom_pair_centred(X, Y, 2, true); // discrete flag irrelevant past FSD } -int tsd_uni(const double* x, const double* y, std::size_t n) { - check_no_nan(x, n); - check_no_nan(y, n); - ColPre X = precompute_vec(x, n); - ColPre Y = precompute_vec(y, n); - return sd_dom_pair(X, Y, 3, true); // discrete flag irrelevant past FSD +int tsd_uni(const double* x, std::size_t n_x, const double* y, std::size_t n_y) { + check_no_nan(x, n_x); + check_no_nan(y, n_y); + ColPre X = precompute_vec(x, n_x); + ColPre Y = precompute_vec(y, n_y); + return sd_dom_pair_centred(X, Y, 3, true); // discrete flag irrelevant past FSD } // ---------- Multivariate Efficient-Set Wrappers ---------- diff --git a/gists/curve_fitting/curve_fitting.md b/gists/curve_fitting/curve_fitting.md index df9d4516..5ebc7a43 100644 --- a/gists/curve_fitting/curve_fitting.md +++ b/gists/curve_fitting/curve_fitting.md @@ -39,25 +39,12 @@ fig, axs = plt.subplots(3, N, figsize=(15, 10)) for i in range(1, N + 1): ax = axs[0, i - 1] - # Set the current axis in case nns.reg natively plots to the active plt.gca() - plt.sca(ax) - - # Run the NNS Regression - res = nns.reg(x, y, order=i) - + # NNS regression of the requested partition order (plot=False: we draw it ourselves) + res = nns.nns_reg(x, y, order=i, plot=False) + y_hat = res["Fitted.xy"]["y.hat"] + ax.plot(x, y, color='steelblue') - - # Extract predictions (Mimicking the parity port's return structure) - try: - # Note: If the Python API plots automatically, you may need to pass plot=False above. - if isinstance(res, dict) and 'Fitted.xy' in res: - y_hat = res['Fitted.xy']['y.hat'] if isinstance(res['Fitted.xy'], dict) else res['Fitted.xy'] - ax.plot(x, y_hat, color='red', linewidth=3) - elif hasattr(res, 'fitted'): - ax.plot(x, res.fitted, color='red', linewidth=3) - except Exception: - pass - + ax.plot(x, y_hat, color='red', linewidth=3) ax.set_title(f'NNS Order {i}') ax.set_ylim(-1.5, 1.5) diff --git a/pyproject.toml b/pyproject.toml index 7753876a..6bbba9a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dev = [ # Pinned to the exact versions CI gates on, so a local `ruff check .` or # `mypy` cannot disagree with the run that blocks the merge. "mypy==2.3.1", + "pandas", "pre-commit", "pytest", "pytest-benchmark", @@ -52,6 +53,7 @@ dev = [ "ruff==0.16.3", ] docs = [ + "mkdocs>=1.6,<2", "mkdocs-material>=9.5", "pymdown-extensions>=10", ] diff --git a/scripts/generate_api_reference.py b/scripts/generate_api_reference.py index 2f5e472c..50a50859 100644 --- a/scripts/generate_api_reference.py +++ b/scripts/generate_api_reference.py @@ -84,6 +84,8 @@ class ApiMeta: "dy_d": ApiMeta("Differentiation", "dy.d_", "partial", "Multivariate derivative helper."), "lpm_var": ApiMeta("VaR helpers", "LPM.VaR", "implemented", "Lower partial-moment VaR helper."), "upm_var": ApiMeta("VaR helpers", "UPM.VaR", "implemented", "Upper partial-moment VaR helper."), + "set_num_threads": ApiMeta("Native threads", "ncores", "Python-only", "Default native thread count."), + "get_num_threads": ApiMeta("Native threads", "", "Python-only", "Current default native thread count."), } AREA_ORDER = [ @@ -99,6 +101,7 @@ class ApiMeta: "Stochastic dominance, superiority, and simulation", "Differentiation", "VaR helpers", + "Native threads", "Other public exports", ] diff --git a/scripts/sync_nns_core_snapshot.py b/scripts/sync_nns_core_snapshot.py index 7f955b70..e2ac4214 100644 --- a/scripts/sync_nns_core_snapshot.py +++ b/scripts/sync_nns_core_snapshot.py @@ -13,6 +13,12 @@ "__pycache__", } +# Only the inputs of the nnscore CMake target are vendored. NNS-core's tests, +# fidelity harness, R reference sources, docs and CI stay upstream: the Python +# build forces NNSCORE_BUILD_TESTS off, and each vendored top-level entry must +# match the upstream git object verbatim (scripts/verify_release_provenance.py). +VENDORED_ENTRIES = ("CMakeLists.txt", "include", "src") + def ignore(_directory: str, names: list[str]) -> set[str]: ignored: set[str] = set() @@ -28,10 +34,12 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--core-checkout", type=Path, required=True) parser.add_argument("--core-commit", required=True) - parser.add_argument("--r-repo", required=True) - parser.add_argument("--r-commit", required=True) - parser.add_argument("--r-version", required=True) - parser.add_argument("--r-src-tree-hash", required=True) + # The R fields describe the R snapshot (scripts/sync_r_nns_snapshot.py); + # pass them only when the R snapshot moves together with the core. + parser.add_argument("--r-repo") + parser.add_argument("--r-commit") + parser.add_argument("--r-version") + parser.add_argument("--r-src-tree-hash") parser.add_argument("--manifest", type=Path, default=Path("sync/nns_source.json")) args = parser.parse_args() @@ -42,35 +50,52 @@ def main() -> None: dest = Path("extern/NNS-core") if dest.exists(): shutil.rmtree(dest) - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(core, dest, ignore=ignore) + dest.mkdir(parents=True) + for entry in VENDORED_ENTRIES: + source = core / entry + if source.is_dir(): + shutil.copytree(source, dest / entry, ignore=ignore) + elif source.is_file(): + shutil.copy2(source, dest / entry) + else: + raise SystemExit(f"{core} does not look like NNS-core: missing {entry}") - if not (dest / "CMakeLists.txt").is_file(): - raise SystemExit(f"copied NNS-core is invalid: missing {dest / 'CMakeLists.txt'}") + if args.manifest.exists(): + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + else: + manifest = {} - manifest = { + manifest.update( + { + "core_repo": "OVVO-Financial/NNS-core", + "core_commit": args.core_commit, + "vendored_core_path": "extern/NNS-core", + } + ) + r_fields = { "r_repo": args.r_repo, "r_commit": args.r_commit, "r_version": args.r_version, "r_src_tree_hash": args.r_src_tree_hash, - "core_repo": "OVVO-Financial/NNS-core", - "core_commit": args.core_commit, - "python_repo": "OVVO-Financial/NNS-python", - "python_commit": None, - "vendored_core_path": "extern/NNS-core", - "vendored_r_path": "tools/NNS", - "vendored_r_tarball": f"tools/NNS_{args.r_version}.tar.gz", - "r_cache_path": "tests/_r_cache", - "notes": ( + } + manifest.update({key: value for key, value in r_fields.items() if value is not None}) + manifest.setdefault("python_repo", "OVVO-Financial/NNS-python") + manifest.setdefault("python_commit", None) + manifest.setdefault("vendored_r_path", "tools/NNS") + manifest.setdefault("r_cache_path", "tests/_r_cache") + manifest.setdefault( + "notes", + ( "NNS-python consumes accepted NNS-core snapshots for native code and " "verifies public Python behavior against live or cached R NNS." ), - } + ) args.manifest.parent.mkdir(parents=True, exist_ok=True) args.manifest.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") print(f"Vendored NNS-core {args.core_commit} into {dest}") - print(f"Recorded R source {args.r_repo}@{args.r_commit} version {args.r_version}") + if args.r_commit is not None: + print(f"Recorded R source {args.r_repo}@{args.r_commit} version {args.r_version}") if __name__ == "__main__": diff --git a/src/nns/__init__.py b/src/nns/__init__.py index 302774cf..8eecdfa1 100644 --- a/src/nns/__init__.py +++ b/src/nns/__init__.py @@ -35,6 +35,7 @@ "factor_2_dummy_fr": ("nns.categorical", "factor_2_dummy_fr"), "fsd": ("nns.stochastic_dominance", "fsd"), "fsd_uni": ("nns.stochastic_dominance", "fsd_uni"), + "get_num_threads": ("nns._native", "get_num_threads"), "kurt_pm": ("nns.classical", "kurt_pm"), "lpm": ("nns.core", "lpm"), "lpm_ratio": ("nns.core", "lpm_ratio"), @@ -71,6 +72,7 @@ "nns_var": ("nns.var", "nns_var"), "prepare_factor_predictors": ("nns.regression", "prepare_factor_predictors"), "sd_efficient_set": ("nns.stochastic_dominance", "sd_efficient_set"), + "set_num_threads": ("nns._native", "set_num_threads"), "skew_pm": ("nns.classical", "skew_pm"), "ssd": ("nns.stochastic_dominance", "ssd"), "ssd_uni": ("nns.stochastic_dominance", "ssd_uni"), diff --git a/src/nns/_caus_r14.py b/src/nns/_caus_r14.py new file mode 100644 index 00000000..8f054a64 --- /dev/null +++ b/src/nns/_caus_r14.py @@ -0,0 +1,95 @@ +"""R 14.0 directional association (NNS.caus / Uni.caus). + +Ground truth: NNS@7b07a837 R/Uni_Causation.R, R/Causal_matrix.R. +Each pair is ranked to its own empirical lower-tail copula coordinate; +the raw scores combine variance explained within equal-frequency cells and +unfloored mean-of-partition-level dependence (not NNS.dep's maximum). +""" +from __future__ import annotations + +import math + +import numpy as np +from numpy.typing import NDArray + +from nns._dep_r14 import dep_levels_r14 + +Array = NDArray[np.float64] + + +def _ranks(v: Array) -> Array: + """R LPM.ratio(0, v, v): inclusive empirical CDF of every observation.""" + sorted_values = np.sort(v) + return np.searchsorted(sorted_values, v, side="right").astype(np.float64) / v.size + + +def _variance(v: Array) -> float: + """R (LPM(2,mean,v) + UPM(2,mean,v)) * m/(m-1), length >=2.""" + m = int(v.size) + if m < 2: + return float("nan") + # Population 2nd partial moments about the *same* sample mean. + mean = float(np.mean(v)) + centered = v - mean + return float(np.mean(centered * centered) * (m / (m - 1))) + + +def _explained(cause_u: Array, effect_u: Array) -> float: + """R .nns_caus_P: conditional share of effect variance resolved by cause.""" + n = int(cause_u.size) + if n < 3: + return 0.0 + groups = np.maximum(1, np.ceil(cause_u * max(2, math.floor(math.sqrt(n))))).astype(int) + total = _variance(effect_u) + if not math.isfinite(total) or total <= 0.0: + return 0.0 + numerator, denominator = 0.0, 0 + for group in np.unique(groups): + sample = effect_u[groups == group] + d = _variance(sample) + if math.isfinite(d): + numerator += sample.size * d + denominator += sample.size + if denominator == 0: + return 0.0 + within = numerator / denominator + return min(1.0, max(0.0, 1.0 - within / total)) + + +def _rho(cause_u: Array, effect_u: Array) -> float: + """R .nns_caus_rho: average, not maximum, of unfloored NNS.dep levels.""" + try: + value = float(np.mean(dep_levels_r14(cause_u, effect_u))) + return value if math.isfinite(value) else 0.0 + except (ValueError, ArithmeticError, OverflowError): + # Mirrors R's tryCatch(error=function(e) 0). + return 0.0 + + +def raw_scores(x: Array, y: Array, tau: int = 0) -> tuple[float, float]: + """R .nns_caus_raw: paired, equally lagged directional raw scores.""" + n = int(x.size) + k = n - tau + x_lag, y_lag = _ranks(x[:k]), _ranks(y[:k]) + x_lead, y_lead = _ranks(x[tau:]), _ranks(y[tau:]) + p_xy = _explained(x_lag, y_lead) + p_yx = _explained(y_lag, x_lead) + rho_xy = _rho(x_lag, y_lead) + rho_yx = _rho(y_lag, x_lead) + raw_xy = (p_xy + max(rho_xy - rho_yx, 0.0)) / 2.0 + raw_yx = (p_yx + max(rho_yx - rho_xy, 0.0)) / 2.0 + return raw_xy, raw_yx + + +def index(x: Array, y: Array, tau: int = 0) -> tuple[float, float]: + """R .nns_caus_index: complementary directional indices, neutral 0.5.""" + xy, yx = raw_scores(x, y, tau) + k = 1.0 / math.sqrt(float(x.size - tau)) + forward = (xy + k) / (xy + yx + 2.0 * k) + return forward, 1.0 - forward + + +def predictor_weight(response: Array, predictor: Array) -> float: + """R Uni.caus(response, predictor, tau): aligned raw predictor->response.""" + n = min(response.size, predictor.size) + return raw_scores(predictor[:n], response[:n], 0)[0] diff --git a/src/nns/_class_r14.py b/src/nns/_class_r14.py new file mode 100644 index 00000000..bf080994 --- /dev/null +++ b/src/nns/_class_r14.py @@ -0,0 +1,352 @@ +"""NNS 14.0 empirical-frequency nominal classification. + +Reference: OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae, +R/Classification.R. Class codes identify indicator columns only; numeric +interpolation is never performed on ordinalized class labels. +""" + +from __future__ import annotations + +import math +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +F = NDArray[np.float64] + + +def _normalize(probabilities: F, priors: F) -> F: + p = np.asarray(probabilities, dtype=np.float64).copy() + p[~np.isfinite(p) | (p < 0.0)] = 0.0 + p[p > 1.0] = 1.0 + totals = np.sum(p, axis=1) + valid = totals > 0.0 + if np.any(valid): + p[valid] /= totals[valid, None] + if np.any(~valid): + p[~valid] = priors + return p + + +def _select(probabilities: F | None, values: F, order: NDArray[np.intp]) -> F | None: + if probabilities is None: + return None + selected = np.argmax(probabilities[:, order], axis=1) + return np.asarray(values[order[selected]], dtype=np.float64) + + +def _class_base( + design_x: F, + indicators: F, + points: F | None, + order: Any, + n_best: Any, + dist: str, + categorical: bool, + point_only: bool, + labels: list[str], +) -> dict[str, Any]: + """R .nns_class_base: empirical class shares in deterministic NNS cells.""" + from nns._reg_engine import ( + _build_points, + _dependence, + _mreg_distances, + _mreg_ensemble_weights, + ) + + n, p = design_x.shape + mins = np.min(design_x, axis=0) + maxs = np.max(design_x, axis=0) + strength = np.array( + [ + max( + 0.0, + max( + _dependence(design_x[:, j], indicators[:, c]) + for c in range(indicators.shape[1]) + ), + ) + for j in range(p) + ], + dtype=np.float64, + ) + boundaries: list[F] = [] + exact: list[bool] = [] + memberships: list[NDArray[np.int64]] = [] + for j in range(p): + coord = design_x[:, j] + coord_exact = categorical or order == "max" or np.unique(coord).size <= 2 + exact.append(coord_exact) + if coord_exact: + b = np.unique(coord) + else: + if order is None: + depth = max(1, math.floor(float(strength[j]) * 10.0 + 0.5)) + if n < 100: + depth = max(1, depth // 2) + else: + depth = int(order) + b = np.asarray(_build_points(coord, coord, depth, "mean", False)["x"], dtype=np.float64) + boundaries.append(b) + if coord_exact: + memberships.append(np.searchsorted(b, coord, side="left").astype(np.int64) + 1) + else: + positions = np.searchsorted(b, coord, side="right").astype(np.int64) + # R findInterval(..., rightmost.closed=TRUE) maps the last + # breakpoint to the final interval, not to one past it. + memberships.append(np.minimum(positions, b.size - 1)) + + row_ids = [".".join(str(int(membership[i])) for membership in memberships) for i in range(n)] + groups: dict[str, list[int]] = {} + for i, ident in enumerate(row_ids): + groups.setdefault(ident, []).append(i) + + # split(seq_len(n), ids) groups names in lexicographic order in R. + keys = sorted(groups) + centres = np.array([np.mean(design_x[groups[key]], axis=0) for key in keys], dtype=np.float64) + shares = np.array([np.mean(indicators[groups[key]], axis=0) for key in keys], dtype=np.float64) + prior = np.mean(indicators, axis=0) + + if n_best == "all": + k = centres.shape[0] + elif n_best is not None: + k = int(n_best) + elif order == "max" or categorical: + k = 1 + else: + k = max(1, math.floor((1.0 - float(np.mean(strength))) * math.sqrt(n))) + k = max(1, min(k, centres.shape[0])) + + def predict(query: F | None) -> F | None: + if query is None: + return None + q = np.asarray(query, dtype=np.float64).reshape(-1, p) + if p == 1 and not categorical: + sorted_idx = np.argsort(centres[:, 0], kind="stable") + xx = centres[sorted_idx, 0] + if xx.size == 1: + out = np.tile(shares[sorted_idx[0]], (q.shape[0], 1)) + else: + # R stats::approx computes the fractional position before + # multiplying the ordinate difference. np.interp precomputes + # a slope instead, which can turn an exact probability tie + # into a one-ULP preference and change the selected class. + at = np.clip(q[:, 0], xx[0], xx[-1]) + lo = np.clip(np.searchsorted(xx, at, side="right") - 1, 0, xx.size - 2) + fraction = (at - xx[lo]) / (xx[lo + 1] - xx[lo]) + yy = shares[sorted_idx] + out = yy[lo] + fraction[:, None] * (yy[lo + 1] - yy[lo]) + return _normalize(out, prior) + ds = _mreg_distances(centres, q, dist, mins, maxs) + out = np.empty((q.shape[0], shares.shape[1]), dtype=np.float64) + for i, row in enumerate(ds): + matches = np.flatnonzero(row == 0.0) + if matches.size: + out[i] = np.mean(shares[matches], axis=0) + continue + ordered = np.argsort(row, kind="stable") + distance_sorted = row[ordered] + limit = distance_sorted[k - 1] + chosen = ordered[distance_sorted <= limit] + chosen_d = row[chosen] + if chosen.size == 1: + out[i] = shares[chosen[0]] + continue + w = _mreg_ensemble_weights(chosen_d[None, :])[0] + for value in np.unique(chosen_d): + same = chosen_d == value + if np.count_nonzero(same) > 1: + w[same] = np.mean(w[same]) + out[i] = np.dot(w, shares[chosen]) + return _normalize(out, prior) + + return { + "fitted": None if point_only else predict(design_x), + "predicted": predict(points), + "centres": centres, + "shares": shares, + "ids": row_ids, + "boundaries": boundaries, + "k": k, + "strength": strength, + } + + +def classify( + encoded: dict[str, Any], + task: dict[str, Any], + *, + order: Any, + n_best: Any, + dist: str, + dim_red_method: Any, + tau: Any, + threshold: Any, + point_only: bool, + confidence_interval: float | None, + dim_count: int | None = None, +) -> dict[str, Any]: + """R .nns_reg_classify: return nominal posterior probabilities and sets.""" + from nns._reg_engine import _dimred_coefficients, _dimreduce + + design_x = np.asarray(encoded["x"], dtype=np.float64) + query_x = encoded["point_est"] + y = np.asarray(task["y"], dtype=np.float64) + values = np.asarray(task["class_values"], dtype=np.float64) + levels = task["class_levels"] + labels = ( + list(levels) + if levels is not None and len(levels) == values.size + else [f"{v:.15g}" for v in values] + ) + indicators = np.asarray(y[:, None] == values[None, :], dtype=np.float64) + priors = np.mean(indicators, axis=0) + _, first = np.unique(y, return_index=True) + first_values = y[np.sort(first)] + tie_order = np.array([np.flatnonzero(values == z)[0] for z in first_values], dtype=np.intp) + metadata = encoded.get("metadata") + categorical = ( + metadata is not None and bool(metadata) and all(bool(m["categorical"]) for m in metadata) + ) + + equations = None + if dim_red_method is not None and design_x.shape[1] > 1: + if isinstance(threshold, (int, float)): + threshold_values = np.full(values.size, float(threshold)) + else: + threshold_values = np.asarray(threshold, dtype=np.float64) + if threshold_values.size not in (1, values.size): + raise ValueError("[threshold] must be one number, or one cutoff per class.") + if threshold_values.size == 1: + threshold_values = np.full(values.size, float(threshold_values[0])) + fitted_cols, point_cols, equations = [], [], {} + for c in range(values.size): + coefficients = _dimred_coefficients( + design_x, indicators[:, c], dim_red_method, tau, threshold_values[c] + ) + if dim_count is not None and dim_count < coefficients.size: + # R/Classification.R: retain the first dim_count columns + # in decreasing |coefficient| order (stable original order). + keep = np.argsort(-np.abs(coefficients), kind="stable")[:dim_count] + narrowed = np.zeros_like(coefficients) + narrowed[keep] = coefficients[keep] + coefficients = narrowed + if not np.any(np.abs(coefficients) > 0): + coefficients[:] = 1.0 + reduced = _dimreduce( + design_x, + query_x, + indicators[:, c], + coefficients, + tau, + 0, + variable_names=encoded.get("encoded_names"), + ) + y_indicator = np.column_stack((1.0 - indicators[:, c], indicators[:, c])) + reduced_x = np.asarray(reduced["x_star"], dtype=np.float64).reshape(-1, 1) + reduced_q = ( + None + if query_x is None + else np.asarray(reduced["point_star"], dtype=np.float64).reshape(-1, 1) + ) + fit = _class_base( + reduced_x, + y_indicator, + reduced_q, + order, + n_best, + dist, + False, + point_only, + ["not " + labels[c], labels[c]], + ) + if not point_only: + fitted_cols.append(np.asarray(fit["fitted"][:, 1], dtype=np.float64)) + if query_x is not None: + point_cols.append(np.asarray(fit["predicted"][:, 1], dtype=np.float64)) + equations[labels[c]] = reduced["equation"] + fit_prob = None if point_only else _normalize(np.column_stack(fitted_cols), priors) + point_prob = None if query_x is None else _normalize(np.column_stack(point_cols), priors) + centres, shares, boundaries, ids, k = ( + None, + None, + None, + [str(i + 1) for i in range(design_x.shape[0])], + None, + ) + else: + fit = _class_base( + design_x, + indicators, + query_x, + order, + n_best, + dist, + categorical, + point_only, + labels, + ) + fit_prob = fit["fitted"] + point_prob = fit["predicted"] + centres, shares = fit["centres"], fit["shares"] + boundaries, ids, k = fit["boundaries"], fit["ids"], fit["k"] + + point_pred = _select(point_prob, values, tie_order) + fitted_pred = _select(fit_prob, values, tie_order) + accuracy = None if point_only else float(np.mean(fitted_pred == y)) + fitted: dict[str, Any] | None = None + if not point_only and fitted_pred is not None: + names = encoded.get("encoded_names") or [f"X{i + 1}" for i in range(design_x.shape[1])] + fitted = {str(name): design_x[:, j] for j, name in enumerate(names)} + fitted.update( + { + "y": y, + "y.hat": fitted_pred, + "NNS.ID": ids, + "error": np.asarray(fitted_pred != y, dtype=np.float64), + } + ) + point_sets = None + if point_prob is not None and confidence_interval is not None: + point_sets = [] + for row in point_prob: + order_idx = tie_order[np.argsort(-row[tie_order], kind="stable")] + cumulative = np.cumsum(row[order_idx]) + index = min( + int(np.searchsorted(cumulative, confidence_interval - 1e-12)) + 1, order_idx.size + ) + point_sets.append(values[order_idx[:index]].tolist()) + rp = None + if centres is not None: + names = encoded.get("encoded_names") or [f"X{i + 1}" for i in range(design_x.shape[1])] + rp = {str(name): centres[:, j] for j, name in enumerate(names)} + rpm_probs = None if shares is None else {labels[c]: shares[:, c] for c in range(values.size)} + point_probs = ( + None if point_prob is None else {labels[c]: point_prob[:, c] for c in range(values.size)} + ) + fitted_probs = ( + None if fit_prob is None else {labels[c]: fit_prob[:, c] for c in range(values.size)} + ) + return { + "R2": accuracy, + "SE": None, + "Prediction.Accuracy": accuracy, + "equation": equations, + "x.star": None, + "derivative": None, + "Point.est": point_pred, + "pred.int": None, + "Prediction.set": point_sets, + "regression.points": rp, + "RPM": rp, + "RPM.probabilities": rpm_probs, + "rhs.partitions": boundaries, + "Fitted.xy": fitted, + "Fitted": fitted, + "Fitted.probabilities": fitted_probs, + "Point.probabilities": point_probs, + "n.best": k, + "class.levels": task["class_levels"], + "class.values": values, + } diff --git a/src/nns/_dep_r14.py b/src/nns/_dep_r14.py new file mode 100644 index 00000000..d9b9b300 --- /dev/null +++ b/src/nns/_dep_r14.py @@ -0,0 +1,402 @@ +"""Source-faithful R 14.0 multilevel dependence and causation levels. + +Port of OVVO-Financial/NNS@7b07a837 src/NNS_dep.cpp. This is deliberately +independent of the explicit R14 native build adapter for cross-checking. +No statistical redefinition, softened comparisons, or fixture substitutions. +""" + +from __future__ import annotations + +import math +import sys +from collections.abc import Sequence +from typing import TypeAlias + +import numpy as np +from numpy.typing import NDArray + +FloatArray: TypeAlias = NDArray[np.float64] +_EPS = sys.float_info.epsilon +_SQRT_2_PI = math.sqrt(2.0 / math.pi) + + +def _sum_ordered(values: FloatArray) -> float: + # C++ double accumulation in input order, rather than np.sum's pairwise reduction. + return float(np.cumsum(values, dtype=np.float64)[-1]) if values.size else 0.0 + + +def _has_ties(values: FloatArray) -> bool: + return bool(np.any(np.diff(np.sort(values)) == 0.0)) + + +def _null_t0(m: int, a: int, b: int) -> float: + """Hypergeometric E[|2*concordance/m-1|], R null_T0.""" + k_lo, k_hi = max(0, a + b - m), min(a, b) + if k_lo >= k_hi: + return 1.0 + md = float(m) + if m > 400: + ad, bd = float(a), float(b) + mean_k = ad * bd / md + var_k = bd * (ad / md) * ((md - ad) / md) * ((md - bd) / (md - 1.0)) + mu = (2.0 * (md - ad - bd + 2.0 * mean_k) - md) / md + sd = 4.0 * math.sqrt(max(var_k, 0.0)) / md + if sd <= 0.0: + return abs(mu) + z = mu / sd + return sd * _SQRT_2_PI * math.exp(-0.5 * z * z) + mu * math.erf(z / math.sqrt(2.0)) + + # Exact integer combinatorics to establish the first probability, followed + # by the same floating-point ratio recurrence as the R/C++ oracle. + pk = math.comb(a, k_lo) * math.comb(m - a, b - k_lo) / math.comb(m, b) + e = 0.0 + for k in range(k_lo, k_hi + 1): + conc = float(m - a - b + 2 * k) + e += pk * abs(2.0 * conc / md - 1.0) + if k < k_hi: + pk *= (float(a - k) * float(b - k)) / (float(k + 1) * float(m - a - b + k + 1)) + return e + + +def _log_arrangements(values: FloatArray) -> float: + _, multiplicity = np.unique(values, return_counts=True) + out = math.lgamma(float(values.size) + 1.0) + for count in multiplicity: + out -= math.lgamma(float(count) + 1.0) + return out + + +def _next_permutation(v: list[float]) -> bool: + """C++ next_permutation on sorted doubles; handles tied values exactly.""" + i = len(v) - 2 + while i >= 0 and v[i] >= v[i + 1]: + i -= 1 + if i < 0: + return False + j = len(v) - 1 + while v[j] <= v[i]: + j -= 1 + v[i], v[j] = v[j], v[i] + v[i + 1 :] = reversed(v[i + 1 :]) + return True + + +def _null_t1( + dx: FloatArray, dy: FloatArray, x_ties: bool, y_ties: bool, legacy: bool = False +) -> float: + """Permutation-null expectation of the absolute degree-1 orthant ratio.""" + m = int(dx.size) + lx, ly = 1e300, 1e300 + if m <= 6 or (x_ties and not legacy): + lx = _log_arrangements(dx) + if m <= 6 or (y_ties and not legacy): + ly = _log_arrangements(dy) + if min(lx, ly) <= math.log(720.0) + 1e-9: + v = sorted(dx.tolist() if lx <= ly else dy.tolist()) + w = dy if lx <= ly else dx + total = 0.0 + count = 0 + while True: + s = 0.0 + d = 0.0 + for i, vi in enumerate(v): + prod = float(w[i]) * vi + s += prod + d += abs(prod) + total += abs(s) / d if d > 0.0 else 0.0 + count += 1 + if not _next_permutation(v): + break + return total / count + + sx2, sy2, sax, say = 0.0, 0.0, 0.0, 0.0 + for xx, yy in zip(dx, dy, strict=True): + a, b = float(xx), float(yy) + sx2 += a * a + sy2 += b * b + sax += abs(a) + say += abs(b) + if sax <= 0.0 or say <= 0.0: + return 1.0 + md = float(m) + return min(1.0, _SQRT_2_PI * math.sqrt(sx2 * sy2 / (md - 1.0)) * md / (sax * say)) + + +def _debias(t: float, expectation: float) -> float: + return 0.0 if not expectation < 1.0 - 1e-12 else (t - expectation) / (1.0 - expectation) + + +# Cell kind: 0 varying, 1 constant in first variable, 2 both constant. +def _cell( + xv: FloatArray, + yv: FloatArray, + x_ties: bool, + y_ties: bool, + legacy: bool = False, +) -> tuple[float, float, int]: + m = int(xv.size) + if m < 2: + return 0.0, 0.0, 0 + rx = float(np.min(xv)) if not legacy else 0.0 + ry = float(np.min(yv)) if not legacy else 0.0 + shifted_x, shifted_y = xv - rx, yv - ry + tx, ty = _sum_ordered(shifted_x) / m, _sum_ordered(shifted_y) / m + ax, ay = float(np.max(shifted_x)), float(np.max(shifted_y)) + tie_x = 0.0 if legacy else (m + 1) * _EPS * ax + tie_y = 0.0 if legacy else (m + 1) * _EPS * ay + dx, dy = shifted_x - tx, shifted_y - ty + low_x, low_y = dx <= tie_x, dy <= tie_y + a, b = int(np.count_nonzero(low_x)), int(np.count_nonzero(low_y)) + conc = int(np.count_nonzero(low_x == low_y)) + product = dx * dy + s = _sum_ordered(product) + d = _sum_ordered(np.abs(product)) + + if bool(np.all(xv == xv[0])): + return 0.0, 0.0, 2 if bool(np.all(yv == yv[0])) else 1 + s_tol = 0.0 if legacy else 1e-12 * d + slope_sign = 1.0 if s > s_tol else (-1.0 if s < -s_tol else 0.0) + e0 = _null_t0(m, a, b) + if e0 < 1.0 - 1e-12: + if conc == m: + return 1.0, 1.0, 0 + if conc == 0: + return -1.0, 1.0, 0 + t0 = abs(2.0 * float(conc) / float(m) - 1.0) + t1 = abs(s) / d if d > 0.0 else 0.0 + e1 = _null_t1(dx, dy, x_ties, y_ties, legacy) + dep = 0.5 * (_debias(t0, e0) + _debias(t1, e1)) + if slope_sign == 0.0: + return 0.0, 0.0, 0 + return dep * slope_sign, dep, 0 + + +def _between_sign(x: FloatArray, y: FloatArray) -> float: + n = int(x.size) + x0, y0 = float(x[0]), float(y[0]) + mx = _sum_ordered(x - x0) / n + my = _sum_ordered(y - y0) / n + p = ((x - x0) - mx) * ((y - y0) - my) + cov = _sum_ordered(p) + tol = 1e-12 * _sum_ordered(np.abs(p)) + return 1.0 if cov > tol else (-1.0 if cov < -tol else 0.0) + + +def _p_constant(m: int, counts: list[int], n: int, cache: dict[int, float]) -> float: + if m not in cache: + denom = math.comb(n, m) + p = sum(math.comb(c, m) / denom for c in counts if c >= m) + cache[m] = min(1.0, p) + return cache[m] + + +def _best_level( + u: FloatArray, + v: FloatArray, + labels: Sequence[str], + between_sign: float, + conditional: bool, + u_ties: bool, + v_ties: bool, + levels: list[float] | None = None, +) -> tuple[float, float]: + """R best_level: all distinct prefix partitions, weighted cell copulas.""" + n = int(u.size) + order = sorted(range(n), key=lambda i: labels[i]) + longest = max((len(label) for label in labels), default=1) + v_counts = ( + sorted(np.unique(v, return_counts=True)[1].tolist(), reverse=True) + if conditional and v_ties + else [] + ) + p0_cache: dict[int, float] = {} + best_dep, best_corr = -1e300, 0.0 + previous_cell_count = -1 + + for length in range(1, longest + 1): + starts = [0] + for k in range(1, n): + if labels[order[k]][:length] != labels[order[k - 1]][:length]: + starts.append(k) + cell_count = len(starts) + if cell_count == previous_cell_count: + continue + previous_cell_count = cell_count + starts.append(n) + + dep, corr = 0.0, 0.0 + for i in range(cell_count): + ids = order[starts[i] : starts[i + 1]] + m = len(ids) + uq = u[ids] + vq = v[ids] + signed, magnitude, kind = _cell(uq, vq, u_ties, v_ties) + if kind: + mag = 0.0 + if conditional: + p0 = _p_constant(m, v_counts, n, p0_cache) if v_ties else 0.0 + score = 1.0 if kind == 2 else 0.0 + if p0 < 1.0 - 1e-12: + mag = (score - p0) / (1.0 - p0) + signed, magnitude = between_sign * mag, mag + if not (math.isfinite(signed) and math.isfinite(magnitude)): + signed, magnitude = 0.0, 0.0 + weight = float(m) / float(n) + dep += weight * magnitude + corr += weight * signed + if levels is not None: + levels.append(dep) + if dep >= best_dep: + best_dep, best_corr = dep, corr + + dep = max(0.0, best_dep) + return min(dep, max(-dep, best_corr)), dep + + +def _labels(x: FloatArray, y: FloatArray, *, xonly: bool, obs: int) -> list[str]: + # Import locally: nns.part uses the shared nns.dependence._gravity and + # must not be imported while dependence.py is still being initialized. + from nns.part import _nns_part + + part = _nns_part( + x, + y, + type="XONLY" if xonly else None, + order=None, + obs_req=obs, + min_obs_stop=False, + noise_reduction="off", + ) + return [str(q) for q in part["dt"]["quadrant"]] + + +def dep_pair_r14(x: FloatArray, y: FloatArray, asym: bool) -> tuple[float, float]: + """R 14.0 NNS.dep (bivariate): strongest of 1 or 3 partition families.""" + n = int(x.size) + if n < 2 or bool(np.all(x == x[0])) or bool(np.all(y == y[0])): + return 0.0, 0.0 + obs = max(8, n // 8) + quad_xy = _labels(x, y, xonly=True, obs=obs) + sign = _between_sign(x, y) + x_ties, y_ties = _has_ties(x), _has_ties(y) + best = _best_level(x, y, quad_xy, sign, True, x_ties, y_ties) + if asym: + return best + joint = _best_level( + x, + y, + _labels(x, y, xonly=False, obs=obs), + sign, + False, + x_ties, + y_ties, + ) + on_y = _best_level( + y, + x, + _labels(y, x, xonly=True, obs=obs), + sign, + True, + y_ties, + x_ties, + ) + out = joint + if best[1] > out[1]: + out = best + if on_y[1] > out[1]: + out = on_y + return out + + +def dep_levels_r14(x: FloatArray, y: FloatArray) -> list[float]: + """R NNS_dep_levels_cpp: all distinct unfloored directional level scores.""" + n = int(x.size) + if n < 2 or bool(np.all(x == x[0])) or bool(np.all(y == y[0])): + return [0.0] + scores: list[float] = [] + _best_level( + x, + y, + _labels(x, y, xonly=True, obs=max(8, n // 8)), + 0.0, + True, + _has_ties(x), + _has_ties(y), + scores, + ) + return scores + + +def _local_dir( + u: FloatArray, + v: FloatArray, + labels: Sequence[str], + u_ties: bool, + v_ties: bool, +) -> float: + """R local_dep_dir: signed-square-root finest-cell dependence.""" + n = int(u.size) + + def score(uu: FloatArray, vv: FloatArray) -> float: + _, magnitude, kind = _cell(uu, vv, u_ties, v_ties, legacy=True) + if kind == 2: + return 1.0 + if kind == 1: + return 0.0 + return math.copysign(math.sqrt(abs(magnitude)), magnitude) + + global_score = score(u, v) + if not math.isfinite(global_score): + global_score = 0.0 + indices = sorted(range(n), key=lambda i: labels[i]) + dep = 0.0 + lo = 0 + while lo < n: + hi = lo + 1 + while hi < n and labels[indices[hi]] == labels[indices[lo]]: + hi += 1 + cell_ids = indices[lo:hi] + value = score(u[cell_ids], v[cell_ids]) + if not math.isfinite(value): + value = global_score + dep += value * (float(hi - lo) / float(n)) + lo = hi + return max(0.0, dep) + + +def _copula_degree0_unsigned(x: FloatArray, y: FloatArray) -> float: + n = int(x.size) + if n < 2: + return 0.0 + mx = _sum_ordered(x) / n + my = _sum_ordered(y) / n + lo_x, lo_y = (x - mx) <= 0.0, (y - my) <= 0.0 + concordant = int(np.count_nonzero(lo_x == lo_y)) + p = float(concordant) / n + q = float(n - concordant) / n + disc_dep = min(1.0, max(0.0, abs(p - 0.5) / 0.5)) + nd_disc = abs(q - 0.5) / 0.5 + return math.sqrt((disc_dep + nd_disc) / 2.0) + + +def dep_local_r14(x: FloatArray, y: FloatArray, asym: bool = True) -> float: + """R .nns_dep_local / NNS_dep_local_cpp for default regression depth. + + Unlike the R14 public NNS.dep, this is intentionally the legacy + *finest-cell* signed-square-root statistic; R14 retains it solely for + the regression-order / differentiation heuristics. + """ + n = int(x.size) + if n < 2 or bool(np.all(x == x[0])) or bool(np.all(y == y[0])): + return 0.0 + obs = max(8, n // 8) + xtie, ytie = _has_ties(x), _has_ties(y) + dep = _local_dir(x, y, _labels(x, y, xonly=True, obs=obs), xtie, ytie) + if not asym: + dep = max(dep, _local_dir(y, x, _labels(y, x, xonly=True, obs=obs), ytie, xtie)) + if np.unique(x).size < math.sqrt(n) and np.unique(y).size < math.sqrt(n): + disc = _copula_degree0_unsigned(x, y) + if not math.isfinite(disc): + disc = dep + dep = 0.5 * (dep + disc) + return dep diff --git a/src/nns/_helpers.py b/src/nns/_helpers.py index 3ddf4675..3e43979b 100644 --- a/src/nns/_helpers.py +++ b/src/nns/_helpers.py @@ -5,7 +5,7 @@ import numpy as np from numpy.typing import NDArray -from nns._native import native_fn +from nns._native import external_stacklevel, native_fn def _warn_unsupported(**was_set: bool) -> None: @@ -20,7 +20,7 @@ def _warn_unsupported(**was_set: bool) -> None: f"{', '.join(ignored)}: accepted for R NNS API compatibility " "but not implemented in NNS Python; ignored.", UserWarning, - stacklevel=3, + stacklevel=external_stacklevel(), ) @@ -36,43 +36,6 @@ def _as_matrix(x: NDArray[np.float64], name: str) -> NDArray[np.float64]: return values -def _as_vector_or_matrix(x: NDArray[np.float64], name: str) -> NDArray[np.float64]: - """Return x as a non-empty, all-finite 2D float64 matrix, promoting 1D to a column.""" - values = np.asarray(x, dtype=np.float64) - if values.ndim == 1: - values = values.reshape(-1, 1) - if values.ndim != 2 or values.shape[0] == 0 or values.shape[1] == 0: - raise ValueError(f"{name} must be a non-empty numeric vector or matrix.") - if not np.all(np.isfinite(values)): - raise ValueError(f"{name} must contain only finite values.") - return values - - -def _as_point_matrix(x: NDArray[np.float64], n_cols: int) -> NDArray[np.float64]: - """Return evaluation points as an all-finite matrix with the training column count.""" - values = np.asarray(x, dtype=np.float64) - if values.ndim == 1: - if n_cols == 1: - values = values.reshape(-1, 1) - else: - values = values.reshape(1, -1) - if values.ndim != 2 or values.shape[1] != n_cols: - raise ValueError("ivs_test must have the same column count as ivs_train.") - if not np.all(np.isfinite(values)): - raise ValueError("ivs_test must contain only finite values.") - return values - - -def _as_flat_vector(x: NDArray[np.float64], name: str) -> NDArray[np.float64]: - """Return x flattened to a non-empty, all-finite 1D float64 vector.""" - values = np.asarray(x, dtype=np.float64).reshape(-1) - if values.size == 0: - raise ValueError(f"{name} must be non-empty.") - if not np.all(np.isfinite(values)): - raise ValueError(f"{name} must contain only finite values.") - return values - - def _as_pair( x: NDArray[np.float64], y: NDArray[np.float64], @@ -108,14 +71,20 @@ def _fast_lm(x: NDArray[np.float64], y: NDArray[np.float64]) -> tuple[float, flo coef = result["coef"] return float(coef[0]), float(coef[1]) - mean_x = float(np.mean(x_values)) - mean_y = float(np.mean(y_values)) + # Same arithmetic as NNS-core's fast_lm (R's Rcpp kernel): long-double two-pass + # means and sequential double sums. np.mean/np.sum sum pairwise and can differ in + # the last bit, which reorders tied residuals (e.g. nns_meboot's ordxx). + from nns._rmean import r_mean + + mean_x = r_mean(x_values) + mean_y = r_mean(y_values) dx = x_values - mean_x - var_x = float(np.sum(dx * dx)) + dy = y_values - mean_y + var_x = float(np.cumsum(dx * dx)[-1]) if var_x == 0.0: return mean_y, 0.0 - slope = float(np.sum(dx * (y_values - mean_y)) / var_x) + slope = float(np.cumsum(dx * dy)[-1]) / var_x intercept = mean_y - slope * mean_x return intercept, slope diff --git a/src/nns/_indices.py b/src/nns/_indices.py index fe30eff7..b6102c66 100644 --- a/src/nns/_indices.py +++ b/src/nns/_indices.py @@ -21,4 +21,4 @@ def ordered_complement( mask_size = int(max(np.max(all_index), np.max(excluded))) + 1 excluded_mask = np.zeros(mask_size, dtype=bool) excluded_mask[excluded] = True - return all_index[~excluded_mask[all_index]] + return np.asarray(all_index[~excluded_mask[all_index]], dtype=np.int64) diff --git a/src/nns/_labels.py b/src/nns/_labels.py new file mode 100644 index 00000000..33d31dc3 --- /dev/null +++ b/src/nns/_labels.py @@ -0,0 +1,440 @@ +"""Column, name and level handling shared by the regression/stack/boost ports. + +R's ``NNS.reg``, ``NNS.M.reg``, ``NNS.stack`` and ``NNS.boost`` work on data +frames: columns keep their names and their type (numeric, character, +factor), and prediction columns are matched to the training columns by name. +``np.asarray`` on a pandas frame loses all of that, so the ports split their +inputs column by column here first. + +pandas is optional: frames, series and categoricals are recognised by duck +typing and never imported. +""" + +from __future__ import annotations + +import math +import numbers +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from decimal import Decimal +from typing import Any + +import numpy as np +from numpy.typing import NDArray + + +@dataclass(frozen=True) +class Columns: + """An input split into 1-D columns. + + ``labels`` holds the column labels of a frame (``None`` for arrays), + ``row_labels`` the index labels of a 1-D series (R's named vector), and + ``categories`` the declared categories of pandas categorical columns. + """ + + values: list[NDArray[Any]] + categories: list[list[Any] | None] + labels: list[str] | None + row_labels: list[str] | None + vector: bool + ordered: list[bool] | None = None + + +def _categories_of(obj: Any) -> list[Any] | None: + dtype = getattr(obj, "dtype", None) + if getattr(dtype, "name", None) != "category": + return None + categories = getattr(dtype, "categories", None) + if categories is None: + return None + return list(categories) + + +def _label(value: Any) -> str: + if value is None: + return "" + if isinstance(value, float) and math.isnan(value): + return "" + return str(value) + + +def split_columns(x: Any) -> Columns: + """Split ``x`` (frame, series, categorical, array or nested list) into columns.""" + columns = getattr(x, "columns", None) + if columns is not None and hasattr(x, "iloc") and getattr(x, "ndim", 0) == 2: + values = [np.asarray(x.iloc[:, j]) for j in range(x.shape[1])] + categories = [_categories_of(x.iloc[:, j]) for j in range(x.shape[1])] + return Columns( + values=values, + categories=categories, + labels=[_label(c) for c in list(columns)], + row_labels=None, + vector=False, + ordered=[ + bool(getattr(x.iloc[:, j].dtype, "ordered", False)) for j in range(x.shape[1]) + ], + ) + + categories_1d = _categories_of(x) + index = getattr(x, "index", None) + if categories_1d is not None or (index is not None and getattr(x, "ndim", 0) == 1): + arr = np.asarray(x) + if arr.ndim == 1: + row_labels = None if index is None else [_label(i) for i in list(index)] + return Columns( + values=[arr], + categories=[categories_1d], + labels=None, + row_labels=row_labels, + vector=True, + ordered=[bool(getattr(getattr(x, "dtype", None), "ordered", False))], + ) + + arr = np.asarray(x) + if arr.ndim == 0: + return Columns( + values=[arr.reshape(1)], + categories=[None], + labels=None, + row_labels=None, + vector=True, + ) + if arr.ndim == 1: + return Columns(values=[arr], categories=[None], labels=None, row_labels=None, vector=True) + if arr.ndim != 2: + raise ValueError("Expected a vector, matrix, or data frame.") + return Columns( + values=[arr[:, j] for j in range(arr.shape[1])], + categories=[None] * arr.shape[1], + labels=None, + row_labels=None, + vector=False, + ) + + +def response_values( + y: Any, message: str = "[y] must be a vector or one-column object." +) -> tuple[NDArray[Any], list[Any] | None]: + """A response as a 1-D array plus its declared categories (R ``y[[1]]``).""" + if np.ndim(y) > 2: + raise ValueError(message) + cols = split_columns(y) + if len(cols.values) != 1: + raise ValueError(message) + return cols.values[0], cols.categories[0] + + +# -------------------------------------------------------------------------- +# Value typing +# -------------------------------------------------------------------------- + + +def is_missing(value: Any) -> bool: + """R ``is.na`` for one element: None, NaN, pandas NA / NaT.""" + if value is None: + return True + if isinstance(value, (float, np.floating)): + return bool(math.isnan(value)) + return type(value).__name__ in {"NAType", "NaTType"} + + +def _is_number(value: Any) -> bool: + # Decimal is registered only as numbers.Number, not numbers.Real. + return isinstance(value, (numbers.Real, Decimal, np.bool_)) and not isinstance(value, str) + + +def is_text(values: NDArray[Any]) -> bool: + """Whether a column holds labels (R character) rather than numbers. + + String dtypes are labels even when they look numeric ("1", "10"), as in R. + An object column is numeric only when every non-missing element is a real + number (as from ``df.to_numpy()`` of a mixed frame); any string or other + object makes it a label column. + """ + kind = values.dtype.kind + if kind in {"U", "S"}: + return True + if kind != "O": + return False + return any(not is_missing(v) and not _is_number(v) for v in values.tolist()) + + +def is_logical(values: NDArray[Any]) -> bool: + """Whether a column is logical (R ``is.logical``): bool dtype, or an object + column whose non-missing elements are all booleans.""" + kind = values.dtype.kind + if kind == "b": + return True + if kind != "O": + return False + present = [v for v in values.tolist() if not is_missing(v)] + return bool(present) and all(isinstance(v, (bool, np.bool_)) for v in present) + + +def as_float(values: NDArray[Any]) -> NDArray[np.float64]: + """Numeric conversion where missing or unparsable elements become NaN.""" + try: + return np.asarray(values, dtype=np.float64) + except (TypeError, ValueError): + pass + out = np.empty(values.shape[0], dtype=np.float64) + for i, v in enumerate(values.tolist()): + if is_missing(v): + out[i] = np.nan + continue + try: + out[i] = float(v) + except (TypeError, ValueError): + out[i] = np.nan + return out + + +def level_key(value: Any) -> str: + """The label of one value, as R's ``as.character`` would print it. + + Integral numbers print without a decimal part, so the values ``1``, ``1.0`` + and the declared level ``"1"`` are the same level. + """ + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, bool): + return str(value).upper() + if isinstance(value, numbers.Integral): + return str(int(value)) + if isinstance(value, numbers.Real): + v = float(value) + if math.isfinite(v) and v.is_integer(): + return str(int(v)) + return repr(v) + if isinstance(value, bytes): + return value.decode() + return str(value) + + +def level_keys(values: NDArray[Any], missing_message: str) -> list[str]: + raw = values.tolist() + if any(is_missing(v) for v in raw): + raise ValueError(missing_message) + if values.dtype.kind == "U": + return [str(v) for v in raw] + return [level_key(v) for v in raw] + + +def declared_levels(levels: Sequence[Any], what: str) -> list[str]: + """Normalise a user / pandas level list to unique label keys.""" + if isinstance(levels, (str, bytes)): + raise ValueError(f"[{what}] must be a sequence of levels, not a single string.") + keys = [level_key(level) for level in levels] + if len(set(keys)) != len(keys): + raise ValueError(f"[{what}] contains duplicate levels.") + if not keys: + raise ValueError(f"[{what}] must contain at least one level.") + return keys + + +def codes_for(keys: list[str], levels: list[str]) -> NDArray[np.float64]: + position = {level: i + 1.0 for i, level in enumerate(levels)} + return np.asarray([position.get(k, np.nan) for k in keys], dtype=np.float64) + + +def _as_codes(values: NDArray[Any], k: int) -> NDArray[np.float64] | None: + """``values`` as 1-based codes into ``k`` levels, or None if they are not.""" + if values.dtype.kind == "b" or is_text(values): + return None + num = as_float(values) + if num.size == 0 or not np.all(np.isfinite(num)): + return None + if np.any(num != np.round(num)) or num.min() < 1 or num.max() > k: + return None + return num + + +def declared_codes( + values: NDArray[Any], levels: list[str], missing_message: str +) -> tuple[NDArray[np.float64], bool, list[str]]: + """Code ``values`` against declared ``levels`` (R ``factor(values, levels)``). + + Values are matched to the levels as labels. When they are not labels but + numeric codes ``1..K`` (an already integer-coded factor, e.g. ``y`` coded + 1, 2, 3 with ``class_levels=["setosa", "versicolor", "virginica"]``), the + codes are kept and the levels label them. Returns the codes, whether the + values were taken as codes, and the values matching neither (sorted). + """ + keys = level_keys(values, missing_message) + codes = codes_for(keys, levels) + if np.all(np.isfinite(codes)): + return codes, False, [] + as_codes = _as_codes(values, len(levels)) + if as_codes is not None: + return as_codes, True, [] + absent = sorted({k for k, c in zip(keys, codes.tolist(), strict=True) if not c == c}) + return codes, False, absent + + +def point_codes( + values: NDArray[Any], levels: list[str], by_code: bool, missing_message: str +) -> tuple[NDArray[np.float64], list[str]]: + """Code prediction values the way the training column was coded.""" + keys = level_keys(values, missing_message) + if by_code: + as_codes = _as_codes(values, len(levels)) + if as_codes is not None: + return as_codes, [] + codes = np.full(len(keys), np.nan) + else: + codes = codes_for(keys, levels) + unseen = sorted({k for k, c in zip(keys, codes.tolist(), strict=True) if not c == c}) + return codes, unseen + + +def canonical_predictor_codes( + levels: list[str], + train: NDArray[np.float64], + point: NDArray[np.float64] | None, + *, + ordered: bool = False, +) -> tuple[list[str], NDArray[np.float64], NDArray[np.float64] | None]: + """R14 sorts unordered predictor levels, but preserves ordered factors. + + Recode already-integerized factors against their original levels before + changing column order. Response-class level mappings are not canonicalized. + """ + if ordered or levels == sorted(levels): + return levels, train, point + canonical = sorted(levels) + lookup = np.asarray([canonical.index(level) + 1.0 for level in levels]) + return ( + canonical, + lookup[train.astype(np.intp) - 1], + None if point is None else lookup[point.astype(np.intp) - 1], + ) + + +# -------------------------------------------------------------------------- +# R name helpers +# -------------------------------------------------------------------------- + +_RESERVED = { + "if", + "else", + "repeat", + "while", + "function", + "for", + "next", + "break", + "TRUE", + "FALSE", + "NULL", + "Inf", + "NaN", + "NA", + "NA_integer_", + "NA_real_", + "NA_character_", + "NA_complex_", + "in", +} + + +def r_make_unique(names: Sequence[str], sep: str = ".") -> list[str]: + """R ``make.unique``: later duplicates get the smallest free ``k``.""" + out = list(names) + used = set(out) + counters: dict[str, int] = {} + seen: set[str] = set() + for i, name in enumerate(names): + if name not in seen: + seen.add(name) + continue + k = counters.get(name, 0) + while True: + k += 1 + candidate = f"{name}{sep}{k}" + if candidate not in used: + break + counters[name] = k + used.add(candidate) + out[i] = candidate + return out + + +def r_make_names(names: Sequence[str], unique: bool = True) -> list[str]: + """R ``make.names`` (ASCII letters; invalid characters become ".").""" + out: list[str] = [] + for raw in names: + s = re.sub(r"[^0-9A-Za-z._]", ".", str(raw)) + if s == "" or not re.match(r"^[A-Za-z.]", s) or re.match(r"^\.[0-9]", s): + s = "X" + s + if s in _RESERVED: + s = s + "." + out.append(s) + return r_make_unique(out, ".") if unique else out + + +def match_columns(supplied: list[str], train: list[str], label: str) -> list[int] | None: + """R ``.nns_match_predictor_names``. + + The same set of names gives the permutation into training order; no names + in common means positional matching (``None``); a partial overlap is an + error. + """ + if set(supplied) == set(train) and len(set(supplied)) == len(supplied): + position = {name: i for i, name in enumerate(supplied)} + return [position[name] for name in train] + if not any(name in train for name in supplied): + return None + missing = [name for name in train if name not in supplied] + extra = [name for name in supplied if name not in train] + raise ValueError( + f"[{label}] columns must exactly match the training predictors, or share no " + "names with them to be matched positionally. " + f"Missing: {', '.join(missing) if missing else 'none'}; " + f"extra: {', '.join(extra) if extra else 'none'}." + ) + + +def resolve_factor_levels( + factor_levels: Any, names: list[str], labels: list[str] | None +) -> list[list[Any] | None]: + """Per-column declared levels from a ``factor_levels`` argument. + + Accepted forms: a mapping from column name to levels; a sequence with one + entry per predictor column (``None`` for columns without declared levels); + or, when there is a single predictor, a flat sequence of its levels. + """ + p = len(names) + if factor_levels is None: + return [None] * p + if isinstance(factor_levels, Mapping): + out: list[list[Any] | None] = [None] * p + lookup: dict[str, int] = {} + for j, name in enumerate(names): + lookup.setdefault(name, j) + if labels is not None: + for j, name in enumerate(labels): + lookup.setdefault(name, j) + for key, levels in factor_levels.items(): + idx = lookup.get(str(key)) + if idx is None: + raise ValueError(f"[factor_levels] names unknown predictor '{key}'.") + out[idx] = None if levels is None else list(levels) + return out + if isinstance(factor_levels, (str, bytes)): + raise ValueError("[factor_levels] must be a sequence of levels, not a single string.") + entries = list(factor_levels) + + def is_level_list(v: Any) -> bool: + return v is None or ( + not isinstance(v, (str, bytes)) and isinstance(v, (Sequence, np.ndarray)) + ) + + if p == 1 and not all(is_level_list(v) for v in entries): + return [entries] + if len(entries) != p or not all(is_level_list(v) for v in entries): + raise ValueError( + f"[factor_levels] must give one level list (or None) per predictor: " + f"expected {p} entries." + ) + return [None if v is None else list(v) for v in entries] diff --git a/src/nns/_native.py b/src/nns/_native.py index 1f716f75..04d9cbde 100644 --- a/src/nns/_native.py +++ b/src/nns/_native.py @@ -1,10 +1,28 @@ from __future__ import annotations +import functools import importlib import importlib.util -from typing import Any +import inspect +import numbers +import os +import sys +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any, ParamSpec, TypeVar -_NNSCORE_SPEC = importlib.util.find_spec("nns._nnscore") +_P = ParamSpec("_P") +_R = TypeVar("_R") + +# NNS_DISABLE_NATIVE=1 (or true/yes/on) forces the pure-Python/NumPy fallbacks, +# exactly as when the extension is not built (for testing them). +_NATIVE_DISABLED = os.environ.get("NNS_DISABLE_NATIVE", "").strip().lower() in { + "1", + "true", + "yes", + "on", +} +_NNSCORE_SPEC = None if _NATIVE_DISABLED else importlib.util.find_spec("nns._nnscore") try: _nnscore = importlib.import_module("nns._nnscore") if _NNSCORE_SPEC is not None else None @@ -17,3 +35,107 @@ def native_fn(name: str) -> Any | None: if _nnscore is None: return None return getattr(_nnscore, name, None) + + +def set_num_threads(n: int | None) -> None: + """Set the default number of threads used by the native NNS-core kernels. + + ``n=None`` (or ``n <= 0``) restores the automatic choice: the + ``NNS_NUM_THREADS`` environment variable when it is a positive integer, else + the number of hardware threads. Small inputs always run serially, and results + never depend on the thread count. A function's ``ncores`` argument overrides + this setting for that call. Without the native extension this does nothing. + """ + value = 0 if n is None else max(_integer(n, "n"), 0) + setter = native_fn("set_num_threads") + if setter is not None: + setter(value) + + +def get_num_threads() -> int: + """Return the default number of native threads (1 without the extension).""" + getter = native_fn("get_num_threads") + return int(getter()) if getter is not None else 1 + + +@contextmanager +def native_threads(ncores: Any) -> Iterator[None]: + """Use at most ``ncores`` native threads for the calls made inside the block. + + This is R's ``ncores`` argument: ``None`` keeps the default + (:func:`set_num_threads`), and ``1`` runs every native kernel serially. The + setting is per Python thread, so concurrent calls with different ``ncores`` + do not interfere. + """ + if ncores is None: + yield + return + value = _integer(ncores, "ncores") + if value < 1: + # Earlier releases ignored ncores, and R's univariate paths accept any value, + # so treat 0 / negative (e.g. scikit-style -1) as "default" with a warning. + import warnings + + warnings.warn( + "ncores < 1 is treated as the default thread count; pass None instead. " + "This will raise in a future release.", + DeprecationWarning, + stacklevel=external_stacklevel(), + ) + yield + return + setter = native_fn("_set_call_threads") + if setter is None: + yield + return + previous = setter(value) + try: + yield + finally: + setter(previous) + + +def honours_ncores(func: Callable[_P, _R]) -> Callable[_P, _R]: + """Run ``func`` under :func:`native_threads` with its ``ncores`` argument.""" + signature = inspect.signature(func) + default = signature.parameters["ncores"].default + + @functools.wraps(func) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + if "ncores" in kwargs: + ncores = kwargs["ncores"] + else: + ncores = signature.bind_partial(*args, **kwargs).arguments.get("ncores", default) + with native_threads(ncores): + return func(*args, **kwargs) + + return wrapper + + +def external_stacklevel() -> int: + """``warnings.warn`` stacklevel of the first caller outside the nns package. + + Called from the function that warns. Public entry points are wrapped (for + example by :func:`honours_ncores`), so a fixed stacklevel would point at the + wrapper rather than at the user's call. + """ + frame = sys._getframe(1) + level = 1 + while frame.f_back is not None and _is_internal(frame.f_globals.get("__name__", "")): + frame = frame.f_back + level += 1 + return level + + +def _is_internal(module: str) -> bool: + # native_threads warns from inside a @contextmanager, under contextlib's __enter__. + return module.startswith("nns.") or module == "contextlib" + + +def _integer(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, numbers.Real): + raise ValueError(f"[{name}] must be NULL or a positive integer.") + number = float(value) + if not number.is_integer(): + raise ValueError(f"[{name}] must be NULL or a positive integer.") + return int(number) diff --git a/src/nns/_nnscore.pyi b/src/nns/_nnscore.pyi index add7887d..8f7d997a 100644 --- a/src/nns/_nnscore.pyi +++ b/src/nns/_nnscore.pyi @@ -202,3 +202,32 @@ def mode(x: NDArray[np.float64], discrete: bool, multi: bool) -> Sequence[float] def stochastic_superiority( x: NDArray[np.float64], y: NDArray[np.float64] ) -> StochSupResult: ... +def set_num_threads(n: int) -> None: ... +def get_num_threads() -> int: ... +def _set_call_threads(n: int) -> int: ... +def mreg_predict( + rpm_x: NDArray[np.float64], + n: int, + p: int, + yhat: NDArray[np.float64], + xtest: NDArray[np.float64], + m: int, + k: int, + dist_code: int, + mins: NDArray[np.float64], + maxs: NDArray[np.float64], + is_class: bool, +) -> Sequence[float]: ... +def mreg_predict_path( + rpm_x: NDArray[np.float64], + n: int, + p: int, + yhat: NDArray[np.float64], + xtest: NDArray[np.float64], + m: int, + kmax: int, + dist_code: int, + mins: NDArray[np.float64], + maxs: NDArray[np.float64], + is_class: bool, +) -> NDArray[np.float64]: ... diff --git a/src/nns/_nnscore_bindings.cpp b/src/nns/_nnscore_bindings.cpp index ab502d47..4826ea09 100644 --- a/src/nns/_nnscore_bindings.cpp +++ b/src/nns/_nnscore_bindings.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,11 +17,22 @@ #include #include "nns/nns.hpp" +#include "_r14_dependence.hpp" namespace nb = nanobind; namespace { +// Worker threads for the NNS-core kernels called from this OS thread: 0 means +// the library default (nns::get_num_threads(): set_num_threads(), else the +// NNS_NUM_THREADS environment variable, else the hardware thread count). +// nns._native.native_threads(ncores) sets it for the duration of one public +// call, so Python threads with different ncores do not interfere. Results never +// depend on it: every parallel kernel writes disjoint output slots. +thread_local int t_call_threads = 0; + +int call_threads() { return t_call_threads > 0 ? t_call_threads : -1; } + using Vector = nb::ndarray, nb::c_contig>; using IntVector = nb::ndarray, nb::c_contig>; using Matrix = nb::ndarray, nb::f_contig>; @@ -41,6 +53,15 @@ Matrix matrix_from_column_major_vector(std::vector&& values, std::size_t return Matrix(storage->data(), {dim, dim}, owner, {1, static_cast(dim)}); } +Matrix matrix_from_column_major(std::vector&& values, std::size_t rows, + std::size_t cols) { + auto* storage = new std::vector(std::move(values)); + nb::capsule owner(storage, [](void* p) noexcept { + delete static_cast*>(p); + }); + return Matrix(storage->data(), {rows, cols}, owner, {1, static_cast(rows)}); +} + void check_same_size(const Vector& x, const Vector& y, const char* x_name, const char* y_name) { if (checked_size(x, x_name) != checked_size(y, y_name)) { throw std::invalid_argument(std::string(x_name) + " and " + y_name + " must have the same length."); @@ -80,10 +101,11 @@ std::vector moment_ratio_vector(bool lower, double degree, const Vector& if (n_targets == 0U) { return out; } + nb::gil_scoped_release release; if (lower) { - nns::lpm_ratio_v(degree, targets.data(), n_targets, x.data(), n, out.data()); + nns::lpm_ratio_v(degree, targets.data(), n_targets, x.data(), n, out.data(), call_threads()); } else { - nns::upm_ratio_v(degree, targets.data(), n_targets, x.data(), n, out.data()); + nns::upm_ratio_v(degree, targets.data(), n_targets, x.data(), n, out.data(), call_threads()); } return out; } @@ -95,10 +117,11 @@ std::vector moment_vector(bool lower, double degree, const Vector& targe if (n_targets == 0U) { return out; } + nb::gil_scoped_release release; if (lower) { - nns::lpm_v(degree, targets.data(), n_targets, x.data(), n, out.data()); + nns::lpm_v(degree, targets.data(), n_targets, x.data(), n, out.data(), call_threads()); } else { - nns::upm_v(degree, targets.data(), n_targets, x.data(), n, out.data()); + nns::upm_v(degree, targets.data(), n_targets, x.data(), n, out.data(), call_threads()); } return out; } @@ -122,18 +145,19 @@ std::vector co_moment_vector(const std::string& kind, if (n_out == 0U) { return out; } + nb::gil_scoped_release release; if (kind == "co_lpm") { nns::co_lpm_v(degree_x, degree_y, x.data(), y.data(), n_x, n_y, target_x.data(), n_target_x, - target_y.data(), n_target_y, out.data()); + target_y.data(), n_target_y, out.data(), call_threads()); } else if (kind == "co_upm") { nns::co_upm_v(degree_x, degree_y, x.data(), y.data(), n_x, n_y, target_x.data(), n_target_x, - target_y.data(), n_target_y, out.data()); + target_y.data(), n_target_y, out.data(), call_threads()); } else if (kind == "d_lpm") { nns::d_lpm_v(degree_x, degree_y, x.data(), y.data(), n_x, n_y, target_x.data(), n_target_x, - target_y.data(), n_target_y, out.data()); + target_y.data(), n_target_y, out.data(), call_threads()); } else if (kind == "d_upm") { nns::d_upm_v(degree_x, degree_y, x.data(), y.data(), n_x, n_y, target_x.data(), n_target_x, - target_y.data(), n_target_y, out.data()); + target_y.data(), n_target_y, out.data(), call_threads()); } else { throw std::invalid_argument("unknown co-moment kind."); } @@ -152,8 +176,12 @@ nb::dict pm_matrix_dict(double degree_lpm, throw std::invalid_argument("target length must equal d."); } checked_flat_matrix_size(variable, n, d, "variable"); - nns::PMMatrixResult result = nns::pm_matrix(degree_lpm, degree_upm, target.data(), - variable.data(), n, d, pop_adj, norm); + nns::PMMatrixResult result; + { + nb::gil_scoped_release release; + result = nns::pm_matrix(degree_lpm, degree_upm, target.data(), variable.data(), n, d, pop_adj, + norm, call_threads()); + } nb::dict out; out["cupm"] = matrix_from_column_major_vector(std::move(result.cupm), result.dim); out["dupm"] = matrix_from_column_major_vector(std::move(result.dupm), result.dim); @@ -232,143 +260,15 @@ nb::dict stochastic_superiority_dict(const Vector& x, const Vector& y) { return out; } -// Bit-faithful port of the pure-Python NNS gravity central tendency -// (`nns.dependence._gravity` and its helpers `_quartiles_like_r_code`, -// `_interpolate_position`, `_simple_bin_counts`). This is intentionally a -// separate symbol from the vendored `nns::gravity`, which derives from a -// different revision and diverges from the R-faithful Python the parity suite -// enforces. Non-finite values are filtered and the remainder is sorted, exactly -// as the Python does, so the caller may pass a raw array. -// Forward declarations: gravity reuses numpy-faithful pairwise summation so its -// internal mean/weighted sums are bit-identical to np.mean/np.sum (a ~1e-11 -// difference is enough to flip a discrete partition split downstream). -double np_pairwise_sum(const double* a, std::size_t n); -double np_mean(const double* a, std::size_t n); - -double gravity_at_position(const std::vector& sorted, double position) { - const std::size_t n = sorted.size(); - const long floor_pos = std::min(static_cast(n), - std::max(1L, static_cast(std::floor(position)))); - const long ceil_pos = std::min(static_cast(n), - std::max(1L, static_cast(std::ceil(position)))); - const double weight = position - std::floor(position); - const double lo = sorted[static_cast(floor_pos) - 1U]; - const double hi = sorted[static_cast(ceil_pos) - 1U]; - return lo + weight * (hi - lo); -} - +// NNS.gravity as R's NNS_gravity_cpp (non-discrete), used by the partitioners +// below and by nns.dependence._gravity. This is NNS-core's nns::gravity: R's +// long-double mean and weighted bin average, and an O(n) histogram that only +// shortens runs of empty bins. The previous port here clipped the +// histogram at 1e6 bins, which put the mode term on the outlier, and +// used pairwise double sums that differed from R in the last ulp. +// Non-finite values are dropped, so the caller may pass a raw array. double gravity_exact_impl(const double* data, std::size_t raw_n) { - std::vector values; - values.reserve(raw_n); - for (std::size_t i = 0; i < raw_n; ++i) { - if (std::isfinite(data[i])) { - values.push_back(data[i]); - } - } - const std::size_t n = values.size(); - if (n == 0U) { - return std::numeric_limits::quiet_NaN(); - } - std::sort(values.begin(), values.end()); - if (n <= 3U) { - if (n == 1U) return values[0]; - if (n == 2U) return 0.5 * (values[0] + values[1]); - return values[1]; // numpy median of 3 sorted values - } - if (values.front() == values.back()) { - return values[0]; - } - const double value_range = values.back() - values.front(); - if (value_range == 0.0) { - return values[0]; - } - - // _quartiles_like_r_code - const double dn = static_cast(n); - const double p25 = dn * 0.25, p50 = dn * 0.50, p75 = dn * 0.75; - double q1, q2, q3; - if (n % 2U == 0U) { - q1 = values[static_cast(std::max(1L, static_cast(std::floor(p25)))) - 1U]; - q2 = values[static_cast(std::max(1L, static_cast(std::floor(p50)))) - 1U]; - q3 = values[static_cast(std::max(1L, static_cast(std::floor(p75)))) - 1U]; - } else { - q1 = gravity_at_position(values, p25); - const long f50 = std::min(static_cast(n), std::max(1L, static_cast(std::floor(p50)))); - const long c50 = std::min(static_cast(n), std::max(1L, static_cast(std::ceil(p50)))); - q2 = 0.5 * (values[static_cast(f50) - 1U] + values[static_cast(c50) - 1U]); - q3 = gravity_at_position(values, p75); - } - - double width = (q3 - q1) * std::pow(dn, -0.5); - if (!(width > 0.0) || !std::isfinite(width)) { - width = value_range / 128.0; - } - - // _simple_bin_counts - const double origin = values.front(); - const double int_max = static_cast(std::numeric_limits::max()); - long bin_count; - if (!(width > 0.0) || !std::isfinite(width)) { - bin_count = 1; - } else { - const double bin_ratio = (values.back() - origin) / width + 1e-12; - if (!std::isfinite(bin_ratio) || bin_ratio > int_max) { - bin_count = 1; - } else { - bin_count = static_cast(std::floor(bin_ratio)) + 1; - } - } - // R (NNS 13.x) does not cap the bin count; the guard only bounds memory for - // pathologically narrow widths (mirrors _MAX_DENSE_BINS in nns.dependence). - bin_count = std::min(std::max(1L, bin_count), 1000000L); - const std::size_t nb = static_cast(bin_count); - - std::vector counts(nb, 0); - if (nb == 1U) { - counts[0] = static_cast(n); - } else { - for (const double v : values) { - long idx = static_cast(std::floor((v - origin) / width)); - idx = std::min(std::max(0L, idx), bin_count - 1); - counts[static_cast(idx)] += 1; - } - } - - int64_t max_count = counts[0]; - for (const int64_t c : counts) { - if (c > max_count) max_count = c; - } - std::size_t n_max = 0, last_max = 0; - for (std::size_t i = 0; i < nb; ++i) { - if (counts[i] == max_count) { - ++n_max; - last_max = i; - } - } - std::size_t lo, hi; - if (n_max == 1U) { - const long center = static_cast(last_max); - lo = static_cast(std::max(0L, center - 1)); - hi = static_cast(std::min(bin_count - 1, center + 1)); - } else { - lo = 0; - hi = nb - 1U; - } - - std::vector products; - products.reserve(hi - lo + 1U); - double denom = 0.0; - for (std::size_t i = lo; i <= hi; ++i) { - denom += static_cast(counts[i]); - products.push_back((origin + static_cast(i) * width) * static_cast(counts[i])); - } - const double weighted = np_pairwise_sum(products.data(), products.size()); - const double mode_gravity = - denom > 0.0 ? weighted / denom : origin + static_cast((lo + hi) / 2U) * width; - - const double mean = np_mean(values.data(), values.size()); - - return 0.25 * (q2 + mode_gravity + mean + 0.5 * (q1 + q3)); + return nns::gravity(data, raw_n, false); } // Bit-faithful port of the pure-Python NNS.part recursive partitioner @@ -476,10 +376,11 @@ PartitionLabels partition_labels(const double* x, const double* y, std::size_t n return {std::move(quad), std::move(prior), depth}; } -// regression points grouped by prior.quadrant (sorted labels via std::map), -// with the discrete half-up rounding of x that _is_discrete_like_r triggers. +// regression points grouped by prior.quadrant (sorted labels via std::map): the +// (x, y) gravity centroid of each cell. x is not rounded for integer-valued x +// (R fix "NNS.part: do not round regression-point x for integer-valued x"). void regression_points_from_prior(const double* x, const double* y, std::size_t n, - const std::vector& prior, bool x_discrete, + const std::vector& prior, std::vector* rp_quadrant, std::vector* rp_x, std::vector* rp_y) { std::map> prior_groups; @@ -499,23 +400,6 @@ void regression_points_from_prior(const double* x, const double* y, std::size_t rp_x->push_back(gravity_exact_impl(gx.data(), gx.size())); rp_y->push_back(gravity_exact_impl(gy.data(), gy.size())); } - if (x_discrete) { - for (double& v : *rp_x) { - const double f = std::floor(v); - v = (v - f < 0.5) ? f : std::ceil(v); - } - } -} - -bool x_is_discrete_like_r(const double* x, std::size_t n) { - bool any_finite = false; - for (std::size_t i = 0; i < n; ++i) { - if (std::isfinite(x[i])) { - any_finite = true; - if (x[i] != std::floor(x[i])) return false; - } - } - return any_finite; } nb::dict nns_part_off(const Vector& xa, const Vector& ya, bool xonly, int max_order, int obs_req, @@ -527,13 +411,15 @@ nb::dict nns_part_off(const Vector& xa, const Vector& ya, bool xonly, int max_or const double* x = xa.data(); const double* y = ya.data(); - PartitionLabels pl = partition_labels(x, y, n, xonly, max_order, obs_req, min_obs_stop); - + PartitionLabels pl; std::vector rp_quadrant; std::vector rp_x; std::vector rp_y; - regression_points_from_prior(x, y, n, pl.prior, x_is_discrete_like_r(x, n), &rp_quadrant, &rp_x, - &rp_y); + { + nb::gil_scoped_release release; + pl = partition_labels(x, y, n, xonly, max_order, obs_req, min_obs_stop); + regression_points_from_prior(x, y, n, pl.prior, &rp_quadrant, &rp_x, &rp_y); + } std::vector dt_x(x, x + n); std::vector dt_y(y, y + n); @@ -555,11 +441,10 @@ nb::dict nns_part_off(const Vector& xa, const Vector& ya, bool xonly, int max_or return out; } -// --- NNS.dep faithful port (nns.dependence.nns_dep) ------------------------- -// Matches numpy's pairwise summation so means/sums used as comparison -// thresholds are bit-identical to np.mean/np.sum. Co-moments reuse the -// verified nns:: kernels; the n-d partial moments and copula logic mirror the -// pure-Python formulas exactly. +// --- NumPy-faithful sums and the NNS.dep partition ------------------------- +// np_pairwise_sum matches numpy's pairwise summation, so means/sums used as +// comparison thresholds by the copula port below are bit-identical to +// np.mean/np.sum. double np_pairwise_sum(const double* a, std::size_t n) { if (n < 8U) { @@ -589,116 +474,6 @@ double np_mean(const double* a, std::size_t n) { return np_pairwise_sum(a, n) / double clamp01(double v) { return std::min(std::max(v, 0.0), 1.0); } -std::size_t unique_count(const double* a, std::size_t n) { - std::vector v(a, a + n); - std::sort(v.begin(), v.end()); - return static_cast(std::distance(v.begin(), std::unique(v.begin(), v.end()))); -} - -double dep_ols_sign(const double* x, const double* y, std::size_t n) { - if (n < 2U) return 0.0; - const double mx = np_mean(x, n); - std::vector dx(n), dxy(n); - for (std::size_t i = 0; i < n; ++i) dx[i] = (x[i] - mx) * (x[i] - mx); - const double denom = np_pairwise_sum(dx.data(), n); - if (denom == 0.0) return 0.0; - const double my = np_mean(y, n); - for (std::size_t i = 0; i < n; ++i) dxy[i] = (x[i] - mx) * (y[i] - my); - const double slope = np_pairwise_sum(dxy.data(), n) / denom; - if (slope > 0.0) return 1.0; - if (slope < 0.0) return -1.0; - return 0.0; -} - -// _dpm_nd(degree=0, norm=True) == mean(discordant) -double nd_dpm_deg0(const double* x, const double* y, std::size_t n, double tx, double ty) { - std::vector disc(n); - for (std::size_t i = 0; i < n; ++i) { - const double dxi = x[i] - tx, dyi = y[i] - ty; - const bool below = (dxi < 0.0) && (dyi < 0.0); - const bool above = (dxi > 0.0) && (dyi > 0.0); - disc[i] = (below || above) ? 0.0 : 1.0; - } - return np_mean(disc.data(), n); -} - -double nd_clpm_deg1(const double* x, const double* y, std::size_t n, double tx, double ty) { - std::vector v(n); - for (std::size_t i = 0; i < n; ++i) { - const double cdx = tx - x[i], cdy = ty - y[i]; - v[i] = (cdx >= 0.0 && cdy >= 0.0) ? cdx * cdy : 0.0; - } - return np_mean(v.data(), n); -} - -double nd_cupm_deg1(const double* x, const double* y, std::size_t n, double tx, double ty) { - std::vector v(n); - for (std::size_t i = 0; i < n; ++i) { - const double ddx = x[i] - tx, ddy = y[i] - ty; - v[i] = (ddx >= 0.0 && ddy >= 0.0) ? ddx * ddy : 0.0; - } - return np_mean(v.data(), n); -} - -// _dpm_nd(degree=1, norm=True) -double nd_dpm_deg1_norm(const double* x, const double* y, std::size_t n, double tx, double ty) { - std::vector v(n); - for (std::size_t i = 0; i < n; ++i) { - const double dxi = x[i] - tx, dyi = y[i] - ty; - const bool below = (dxi < 0.0) && (dyi < 0.0); - const bool above = (dxi > 0.0) && (dyi > 0.0); - const bool discordant = !(below || above); - v[i] = discordant ? std::abs(dxi) * std::abs(dyi) : 0.0; - } - const double dpm = np_mean(v.data(), n); - const double clpm = nd_clpm_deg1(x, y, n, tx, ty); - const double cupm = nd_cupm_deg1(x, y, n, tx, ty); - const double total = clpm + cupm + dpm; - return total > 0.0 ? dpm / total : 0.0; -} - -double dep_copula_signed(const double* x, const double* y, std::size_t n) { - if (n < 2U) return 0.0; - const double tx = np_mean(x, n), ty = np_mean(y, n); - const double d0_cupm = nns::co_upm(0.0, 0.0, x, y, n, n, tx, ty); - const double d0_clpm = nns::co_lpm(0.0, 0.0, x, y, n, n, tx, ty); - const double d0_co = d0_cupm + d0_clpm; - if (d0_co == 1.0 || d0_co == 0.0) return 1.0; - - double c1_cupm = nns::co_upm(1.0, 1.0, x, y, n, n, tx, ty); - double c1_clpm = nns::co_lpm(1.0, 1.0, x, y, n, n, tx, ty); - double c1_dlpm = nns::d_lpm(1.0, 1.0, x, y, n, n, tx, ty); - double c1_dupm = nns::d_upm(1.0, 1.0, x, y, n, n, tx, ty); - const double adjust = static_cast(n) / (static_cast(n) - 1.0); - c1_cupm *= adjust; c1_clpm *= adjust; c1_dlpm *= adjust; c1_dupm *= adjust; - const double total = c1_cupm + c1_dupm + c1_dlpm + c1_clpm; - if (total > 0.0) { c1_cupm /= total; c1_clpm /= total; } - - const double dpm_d0 = nd_dpm_deg0(x, y, n, tx, ty); - const double dpm_d1 = nd_dpm_deg1_norm(x, y, n, tx, ty); - const double discrete_dep = clamp01(std::abs(d0_co - 0.5) / 0.5); - const double continuous_dep = clamp01(std::abs(c1_cupm + c1_clpm - 0.5) / 0.5); - // Bivariate discordant independence null is 1 - 2*0.5^2 = 0.5 (DPM counts a - // point concordant when all-below OR all-above the target), not 1 - 0.5^2 = - // 0.75. The old 0.75 anchor left a fixed 1/3 residual per discordant term and - // a ~0.41 dependence floor that never vanished for independent data. - const double nd_disc = std::abs(dpm_d0 - 0.5) / 0.5; - const double nd_cont = std::abs(dpm_d1 - 0.5) / 0.5; - const double copula = std::sqrt((discrete_dep + continuous_dep + nd_disc + nd_cont) / 4.0); - return copula * dep_ols_sign(x, y, n); -} - -double dep_copula_degree0_unsigned(const double* x, const double* y, std::size_t n) { - const double tx = np_mean(x, n), ty = np_mean(y, n); - const double d0_co = - nns::co_upm(0.0, 0.0, x, y, n, n, tx, ty) + nns::co_lpm(0.0, 0.0, x, y, n, n, tx, ty); - const double dpm_d0 = nd_dpm_deg0(x, y, n, tx, ty); - const double disc_dep = clamp01(std::abs(d0_co - 0.5) / 0.5); - // Bivariate discordant independence null is 1 - 2*0.5^2 = 0.5, not 0.75. - const double nd_disc = std::abs(dpm_d0 - 0.5) / 0.5; - return std::sqrt((disc_dep + nd_disc) / 2.0); -} - std::vector dep_xonly_partition(const double* x, std::size_t n, int obs_req) { int max_order = 0; while ((static_cast(1) << max_order) < n) ++max_order; // ceil(log2(max(1,n))) @@ -728,91 +503,31 @@ std::vector dep_xonly_partition(const double* x, std::size_t n, int return quad; } -void dep_directional(const double* x, const double* y, const std::vector& quad, - double fallback, double& corr_out, double& dep_out) { - // group by first-seen quadrant order (matches Python defaultdict iteration). - std::map seen; - std::vector> groups; - for (std::size_t i = 0; i < quad.size(); ++i) { - auto it = seen.find(quad[i]); - if (it == seen.end()) { - seen.emplace(quad[i], groups.size()); - groups.emplace_back(); - groups.back().push_back(i); - } else { - groups[it->second].push_back(i); - } - } - const double n = static_cast(quad.size()); - double corr = 0.0, dep = 0.0; - std::vector gx, gy; - for (const auto& idx : groups) { - gx.clear(); gy.clear(); - gx.reserve(idx.size()); gy.reserve(idx.size()); - for (const std::size_t i : idx) { gx.push_back(x[i]); gy.push_back(y[i]); } - double cop = dep_copula_signed(gx.data(), gy.data(), gx.size()); - if (!std::isfinite(cop)) cop = fallback; - const double weight = static_cast(idx.size()) / n; - corr += cop * weight; - dep += std::abs(cop) * weight; - } - corr_out = corr; - dep_out = dep; -} - +// R14 dependence consumes the full prefix labels of all three partition +// families. Hashing only the leaves loses the multilevel maximum entirely. nb::dict nns_dep_pair(const Vector& xa, const Vector& ya, bool asym) { const std::size_t n = checked_size(xa, "x"); if (ya.shape(0) != n) throw std::invalid_argument("x and y must have the same length."); const double* x = xa.data(); const double* y = ya.data(); - nb::dict out; - bool x_const = true, y_const = true; - for (std::size_t i = 1; i < n; ++i) { - if (x[i] != x[0]) x_const = false; - if (y[i] != y[0]) y_const = false; - } - if (x_const || y_const) { - out["Correlation"] = 0.0; - out["Dependence"] = 0.0; - return out; - } - - const int obs_req = std::max(8, static_cast(n / 8U)); - const std::vector quad_xy = dep_xonly_partition(x, n, obs_req); - const std::vector quad_yx = dep_xonly_partition(y, n, obs_req); - - double global_cop = dep_copula_signed(x, y, n); - if (!std::isfinite(global_cop)) global_cop = 0.0; - - double corr_xy, dep_xy, corr_yx, dep_yx; - dep_directional(x, y, quad_xy, global_cop, corr_xy, dep_xy); - dep_directional(y, x, quad_yx, global_cop, corr_yx, dep_yx); - - const bool discrete_case = - static_cast(unique_count(x, n)) < std::sqrt(static_cast(n)) && - static_cast(unique_count(y, n)) < std::sqrt(static_cast(n)); - if (discrete_case) { - double disc_cop = dep_copula_degree0_unsigned(x, y, n); - if (!std::isfinite(disc_cop)) disc_cop = std::max(dep_xy, dep_yx); - if (asym) { - const double pair[2] = {dep_xy, disc_cop}; - dep_xy = gravity_exact_impl(pair, 2); - } else { - const double pair[2] = {std::max(dep_xy, dep_yx), disc_cop}; - const double dep_sym = gravity_exact_impl(pair, 2); - dep_xy = dep_sym; - dep_yx = dep_sym; - } + nns_r14::DepResult res; + { + nb::gil_scoped_release release; + const int obs_req = std::max(8, static_cast(n / 8U)); + const auto quad_xy = dep_xonly_partition(x, n, obs_req); + const auto quad_yx = asym ? std::vector{} : dep_xonly_partition(y, n, obs_req); + int depth = 1; + while (static_cast(1) << depth < n) ++depth; + const auto joint = asym ? std::vector{} + : partition_labels(x, y, n, false, depth, obs_req, false).quadrant; + res = nns_r14::dep_pair(std::vector(x, x + n), std::vector(y, y + n), + quad_xy, quad_yx, joint, asym); } - if (asym) { - out["Correlation"] = corr_xy; - out["Dependence"] = dep_xy; - } else { - out["Correlation"] = std::max(corr_xy, corr_yx); - out["Dependence"] = std::max(dep_xy, dep_yx); - } + nb::dict out; + out["Correlation"] = res.correlation; + out["Dependence"] = res.dependence; return out; } @@ -863,7 +578,10 @@ double nd_dpm_general(const double* data, std::size_t n, std::size_t d, const do double prod_abs = 1.0; for (std::size_t c = 0; c < d; ++c) { const double diff = data[i + c * n] - target[c]; - if (!(diff < 0.0)) below = false; + // Degree 0 partitions the sample like CLPM (all <= target) and CUPM (all > + // target), as R's DPM_nD; at degree 1 a tied coordinate adds a + // zero factor either way. + if (!(diff <= 0.0)) below = false; if (!(diff > 0.0)) above = false; prod_abs *= std::abs(diff); } @@ -889,14 +607,43 @@ double upper_tri_pairwise(const std::vector& m, std::size_t d) { double copula_nd(const double* data, std::size_t n, std::size_t d, const double* target, bool continuous) { - const nns::PMMatrixResult disc = nns::pm_matrix(0.0, 0.0, target, data, n, d, false, false); + const nns::PMMatrixResult disc = nns::pm_matrix(0.0, 0.0, target, data, n, d, false, false, call_threads()); const double disc_co = upper_tri_pairwise(disc.cupm, d) + upper_tri_pairwise(disc.clpm, d); - if (disc_co == 1.0 || disc_co == 0.0) return 1.0; + // Each margin's lower-tail share at its target, also the independence anchors. + std::vector p0(d), p1(d); + for (std::size_t c = 0; c < d; ++c) { + const double* col = data + c * n; + double below = 0.0, lower = 0.0, upper = 0.0; + for (std::size_t i = 0; i < n; ++i) { + const double diff = target[c] - col[i]; + if (diff >= 0.0) below += 1.0; + if (diff > 0.0) lower += diff; else upper -= diff; + } + p0[c] = below / static_cast(n); + p1[c] = lower / (lower + upper); + } + // With every margin on one side of its target each co-partial moment equals its + // independence product, and every pair is trivially concordant or discordant. + std::size_t one_sided = 0; + for (double p : p0) one_sided += (p == 0.0 || p == 1.0) ? 1U : 0U; + if (one_sided == d) return 0.0; + // Perfect dependence: every pair fully concordant (1) or fully discordant (0). + bool perfect = one_sided == 0; + for (std::size_t i = 0; i < d && perfect; ++i) { + for (std::size_t j = i + 1U; j < d; ++j) { + const double pair = disc.cupm[i + j * d] + disc.clpm[i + j * d]; + if (!(std::abs(pair - 1.0) < 1e-9 || std::abs(pair) < 1e-9)) { + perfect = false; + break; + } + } + } + if (perfect) return 1.0; const double disc_d = nd_dpm_general(data, n, d, target, 0.0, true); double cont_co, cont_d; if (continuous) { - const nns::PMMatrixResult cont = nns::pm_matrix(1.0, 1.0, target, data, n, d, true, true); + const nns::PMMatrixResult cont = nns::pm_matrix(1.0, 1.0, target, data, n, d, true, true, call_threads()); cont_co = upper_tri_pairwise(cont.cupm, d) + upper_tri_pairwise(cont.clpm, d); cont_d = nd_dpm_general(data, n, d, target, 1.0, true); } else { @@ -904,193 +651,69 @@ double copula_nd(const double* data, std::size_t n, std::size_t d, const double* cont_d = disc_d; } - const double dd = static_cast(d); - const double indep_co = 0.25 * (dd * dd - dd); - const double discrete_dep = clamp01(std::abs(disc_co - indep_co) / indep_co); - const double continuous_dep = clamp01(std::abs(cont_co - indep_co) / indep_co); - // DPM_nD counts a point concordant when all-below OR all-above the target - // (both fully-aligned orthants), so under independence - // P(discordant) = 1 - 2*0.5^d (0.5 when d == 2), not 1 - 0.5^d. - const double indep_d = 1.0 - 2.0 * std::pow(0.5, dd); - const double nd_disc = std::abs(disc_d - indep_d) / indep_d; - const double nd_cont = std::abs(cont_d - indep_d) / indep_d; + const auto co_anchor = [d](const std::vector& p) { + std::vector terms; + for (std::size_t j = 1; j < d; ++j) + for (std::size_t i = 0; i < j; ++i) terms.push_back(p[i] * p[j] + (1.0 - p[i]) * (1.0 - p[j])); + return np_pairwise_sum(terms.data(), terms.size()); + }; + const auto d_anchor = [](const std::vector& p) { + double lo = 1.0, hi = 1.0; + for (double v : p) { lo *= v; hi *= 1.0 - v; } + return 1.0 - lo - hi; + }; + const auto rel_dev = [](double obs, double indep) { + return obs == indep ? 0.0 : std::abs(obs - indep) / indep; + }; + const std::vector& pc = continuous ? p1 : p0; + const double discrete_dep = clamp01(rel_dev(disc_co, co_anchor(p0))); + const double continuous_dep = clamp01(rel_dev(cont_co, co_anchor(pc))); + const double nd_disc = std::min(rel_dev(disc_d, d_anchor(p0)), 1.0); + const double nd_cont = std::min(rel_dev(cont_d, d_anchor(pc)), 1.0); return std::sqrt((discrete_dep + continuous_dep + nd_disc + nd_cont) / 4.0); } -// --- NNS.reg multivariate-call regression points ---------------------------- -// Faithful port of nns.regression._nns_reg_univariate_core's multivariate_call -// path for the common regime: noise="off", not class_mode, not smooth, the -// standard XONLY partition branch (dependence != 1, integer dep_order). The -// Python wrapper computes `dependence`/`dep_order` and gates on this regime, -// falling back to pure Python otherwise. - -// _consolidate_points: finite filter, lexsort by (x, y), unique x, gravity y. -void consolidate_points(const std::vector& xin, const std::vector& yin, - std::vector* ux, std::vector* uy) { - std::vector> pts; - pts.reserve(xin.size()); - for (std::size_t i = 0; i < xin.size(); ++i) { - if (std::isfinite(xin[i]) && std::isfinite(yin[i])) pts.emplace_back(xin[i], yin[i]); - } - std::sort(pts.begin(), pts.end(), [](const std::pair& a, - const std::pair& b) { - if (a.first != b.first) return a.first < b.first; - return a.second < b.second; - }); - std::size_t i = 0; - while (i < pts.size()) { - const double cx = pts[i].first; - std::vector gy; - std::size_t j = i; - while (j < pts.size() && pts[j].first == cx) { - gy.push_back(pts[j].second); - ++j; - } - ux->push_back(cx); - uy->push_back(gravity_exact_impl(gy.data(), gy.size())); - i = j; - } -} +// --- NNS.M.reg prediction kernels (NNS-core ports of R 13.3) ---------------- +// Matrices are passed flattened column-major (element (i, j) at i + j * rows). +// NNS-core's mreg_setup and xstar_path are not bound: the NumPy partition, RPM +// reduction and X* path already take a few ms, and mreg_setup's "mode" reducer +// is the exact-frequency mode rather than NNS.mode. -double mean_of_unique(std::vector v) { - std::sort(v.begin(), v.end()); - v.erase(std::unique(v.begin(), v.end()), v.end()); - return np_mean(v.data(), v.size()); -} - -// _edge_lm_fit: fast_lm then evaluate at the low/high edge of x. -double edge_lm_fit(const std::vector& xx, const std::vector& yy, bool low) { - const nns::FastLmResult fit = nns::fast_lm(xx.data(), yy.data(), xx.size()); - const double intercept = fit.coef[0]; - const double slope = fit.coef[1]; - double edge = xx[0]; - for (double v : xx) edge = low ? std::min(edge, v) : std::max(edge, v); - return intercept + slope * edge; -} - -// _endpoint_y for the non-class path. -double endpoint_y(const double* x, const double* y, std::size_t n, const std::vector& rp_x, - bool low, double dependence) { - double boundary = low ? *std::min_element(x, x + n) : *std::max_element(x, x + n); - double reg_range = low ? *std::min_element(rp_x.begin(), rp_x.end()) - : *std::max_element(rp_x.begin(), rp_x.end()); - const double mid_range = 0.5 * (boundary + reg_range); - - std::vector y_boundary, y_mid, x_mid, x_boundary; - for (std::size_t i = 0; i < n; ++i) { - const bool in_boundary = low ? (x[i] <= reg_range) : (x[i] >= reg_range); - const bool in_mid = low ? (x[i] <= mid_range) : (x[i] >= mid_range); - if (in_boundary) { - y_boundary.push_back(y[i]); - x_boundary.push_back(x[i]); - } - if (in_mid) { - y_mid.push_back(y[i]); - x_mid.push_back(x[i]); - } - } - std::size_t unique_x_mid; - { - std::vector xm = x_mid; - std::sort(xm.begin(), xm.end()); - unique_x_mid = static_cast(std::distance(xm.begin(), std::unique(xm.begin(), xm.end()))); +void check_mreg_predict_args(const Vector& rpm_x, std::size_t n, std::size_t p, + const Vector& yhat, const Vector& xtest, std::size_t m, + const Vector& mins, const Vector& maxs) { + checked_flat_matrix_size(rpm_x, n, p, "rpm_x"); + if (yhat.shape(0) != n) throw std::invalid_argument("yhat length must equal n."); + if (xtest.shape(0) != m * p) throw std::invalid_argument("xtest length must equal m * p."); + if (mins.shape(0) != p || maxs.shape(0) != p) { + throw std::invalid_argument("mins and maxs must have one value per column."); } - - if (unique_x_mid > 1U && y_boundary.size() > 5U) { - if (dependence < 0.95 && y_boundary.size() > 1U && y_mid.size() > 1U) { - const double fit_boundary = edge_lm_fit(x_boundary, y_boundary, low); - const double fit_mid = edge_lm_fit(x_mid, y_mid, low); - return (fit_boundary * static_cast(y_boundary.size()) + - fit_mid * static_cast(y_mid.size())) / - static_cast(y_boundary.size() + y_mid.size()); - } - std::vector boundary_values; - for (std::size_t i = 0; i < n; ++i) - if (x[i] == boundary) boundary_values.push_back(y[i]); - return mean_of_unique(std::move(boundary_values)); - } - std::vector at_boundary; - for (std::size_t i = 0; i < n; ++i) - if (x[i] == boundary) at_boundary.push_back(y[i]); - return gravity_exact_impl(at_boundary.data(), at_boundary.size()); } -nb::dict nns_reg_mv(const Vector& xa, const Vector& ya, double dependence, int dep_order) { - const std::size_t n = checked_size(xa, "x"); - if (ya.shape(0) != n) { - throw std::invalid_argument("x and y must have the same length."); - } - const double* x = xa.data(); - const double* y = ya.data(); - const double minx = *std::min_element(x, x + n); - const double maxx = *std::max_element(x, x + n); - const double miny = *std::min_element(y, y + n); - const double maxy = *std::max_element(y, y + n); - - // XONLY, noise="off" partition -> regression points. - PartitionLabels pl = partition_labels(x, y, n, /*xonly=*/true, dep_order, /*obs_req=*/0, - /*min_obs_stop=*/true); - std::vector prp_x, prp_y; - regression_points_from_prior(x, y, n, pl.prior, x_is_discrete_like_r(x, n), nullptr, &prp_x, - &prp_y); - - // _initial_regression_points: clamp x to [minx, maxx], then consolidate. - for (double& v : prp_x) v = std::min(maxx, std::max(v, minx)); - std::vector rp_x, rp_y; - consolidate_points(prp_x, prp_y, &rp_x, &rp_y); - - // _central_point (non-class): median row positions of 1..m -> (m+1)/2. - const double med = (static_cast(rp_x.size()) + 1.0) / 2.0; - const long r0 = static_cast(std::floor(med)); - const long r1 = static_cast(std::ceil(med)); - const double cxv0 = rp_x[static_cast(r0) - 1U]; - const double cxv1 = rp_x[static_cast(r1) - 1U]; - double central_y; - if (r0 != r1) { - std::vector gy; - for (std::size_t i = 0; i < n; ++i) - if (x[i] >= cxv0 && x[i] <= cxv1) gy.push_back(y[i]); - central_y = gravity_exact_impl(gy.data(), gy.size()); - } else { - central_y = rp_y[static_cast(r0) - 1U]; - } - const double cxvals[2] = {cxv0, cxv1}; - const double central_x = gravity_exact_impl(cxvals, 2); - - // _append_and_consolidate_point(central). - { - std::vector ax = rp_x, ay = rp_y; - ax.push_back(central_x); - ay.push_back(central_y); - rp_x.clear(); - rp_y.clear(); - consolidate_points(ax, ay, &rp_x, &rp_y); - } +std::vector mreg_predict_vector(const Vector& rpm_x, std::size_t n, std::size_t p, + const Vector& yhat, const Vector& xtest, std::size_t m, + int k, int dist_code, const Vector& mins, + const Vector& maxs, bool is_class) { + check_mreg_predict_args(rpm_x, n, p, yhat, xtest, m, mins, maxs); + if (m == 0U) return {}; + nb::gil_scoped_release release; + return nns::mreg_predict(rpm_x.data(), n, p, yhat.data(), xtest.data(), m, k, dist_code, + mins.data(), maxs.data(), is_class, call_threads()); +} - // _endpoint_y_values (non-class, dependence < 1 so the else branch). - const double min_y = endpoint_y(x, y, n, rp_x, /*low=*/true, dependence); - const double max_y = endpoint_y(x, y, n, rp_x, /*low=*/false, dependence); - { - std::vector ax = rp_x, ay = rp_y; - ax.push_back(minx); - ax.push_back(maxx); - ax.push_back(central_x); - ay.push_back(min_y); - ay.push_back(max_y); - ay.push_back(central_y); - rp_x.clear(); - rp_y.clear(); - consolidate_points(ax, ay, &rp_x, &rp_y); +Matrix mreg_predict_path_matrix(const Vector& rpm_x, std::size_t n, std::size_t p, + const Vector& yhat, const Vector& xtest, std::size_t m, int kmax, + int dist_code, const Vector& mins, const Vector& maxs, + bool is_class) { + check_mreg_predict_args(rpm_x, n, p, yhat, xtest, m, mins, maxs); + const std::size_t cols = std::min(static_cast(std::max(kmax, 1)), n); + std::vector out; + if (m > 0U) { + nb::gil_scoped_release release; + out = nns::mreg_predict_path(rpm_x.data(), n, p, yhat.data(), xtest.data(), m, kmax, + dist_code, mins.data(), maxs.data(), is_class, call_threads()); } - - // Final clamps. - for (double& v : rp_x) v = std::min(maxx, std::max(v, minx)); - for (double& v : rp_y) v = std::min(maxy, std::max(v, miny)); - - nb::dict out; - out["x"] = std::move(rp_x); - out["y"] = std::move(rp_y); - return out; + return matrix_from_column_major(std::move(out), m, cols); } } // namespace @@ -1171,19 +794,22 @@ NB_MODULE(_nnscore, m) { double degree, bool norm) { checked_flat_matrix_size(data, n, d, "data"); if (target.shape(0) != d) throw std::invalid_argument("target length must equal d."); - return nns::clpm_nd(data.data(), n, d, target.data(), degree, norm); + nb::gil_scoped_release release; + return nns::clpm_nd(data.data(), n, d, target.data(), degree, norm, call_threads()); }); m.def("cupm_nd", [](const Vector& data, std::size_t n, std::size_t d, const Vector& target, double degree, bool norm) { checked_flat_matrix_size(data, n, d, "data"); if (target.shape(0) != d) throw std::invalid_argument("target length must equal d."); - return nns::cupm_nd(data.data(), n, d, target.data(), degree, norm); + nb::gil_scoped_release release; + return nns::cupm_nd(data.data(), n, d, target.data(), degree, norm, call_threads()); }); m.def("dpm_nd", [](const Vector& data, std::size_t n, std::size_t d, const Vector& target, double degree, bool norm) { checked_flat_matrix_size(data, n, d, "data"); if (target.shape(0) != d) throw std::invalid_argument("target length must equal d."); - return nns::dpm_nd(data.data(), n, d, target.data(), degree, norm); + nb::gil_scoped_release release; + return nns::dpm_nd(data.data(), n, d, target.data(), degree, norm, call_threads()); }); m.def("clpm_nd_batch", [](const Vector& data, std::size_t n, std::size_t d, const Vector& targets, std::size_t n_targets, double degree, bool norm) { @@ -1192,11 +818,26 @@ NB_MODULE(_nnscore, m) { throw std::invalid_argument("targets length must equal n_targets * d."); } std::vector out(n_targets, 0.0); - nns::clpm_nd_batch(data.data(), n, d, targets.data(), n_targets, degree, norm, out.data()); + nb::gil_scoped_release release; + nns::clpm_nd_batch(data.data(), n, d, targets.data(), n_targets, degree, norm, out.data(), + call_threads()); return out; }); m.def("pm_matrix", &pm_matrix_dict); + m.def( + "set_num_threads", [](int n) { nns::set_num_threads(n); }, nb::arg("n"), + "Default NNS-core worker threads (n <= 0: NNS_NUM_THREADS or the hardware count)."); + m.def("get_num_threads", []() { return nns::get_num_threads(); }); + m.def( + "_set_call_threads", + [](int n) { + const int previous = t_call_threads; + t_call_threads = n > 0 ? n : 0; + return previous; + }, + nb::arg("n")); + m.def("fast_lm", &fast_lm_dict); m.def("fast_lm_mult", &fast_lm_mult_dict); @@ -1216,20 +857,39 @@ NB_MODULE(_nnscore, m) { m.def("generate_lin_vectors", &forecast_vectors_dict); m.def("gravity", [](const Vector& x, bool discrete) { - return nns::gravity(x.data(), checked_size(x, "x"), discrete); + const std::size_t n = checked_size(x, "x"); + nb::gil_scoped_release release; + return nns::gravity(x.data(), n, discrete); }); m.def("mode", [](const Vector& x, bool discrete, bool multi) { - return nns::mode(x.data(), checked_size(x, "x"), discrete, multi); + const std::size_t n = checked_size(x, "x"); + nb::gil_scoped_release release; + return nns::mode(x.data(), n, discrete, multi); }); m.def("stochastic_superiority", &stochastic_superiority_dict); m.def("gravity_exact", [](const Vector& x) { + nb::gil_scoped_release release; return gravity_exact_impl(x.data(), x.shape(0)); }); m.def("nns_part_off", &nns_part_off, nb::arg("x"), nb::arg("y"), nb::arg("xonly"), nb::arg("max_order"), nb::arg("obs_req"), nb::arg("min_obs_stop")); + m.def("mreg_predict", &mreg_predict_vector, nb::arg("rpm_x"), nb::arg("n"), nb::arg("p"), + nb::arg("yhat"), nb::arg("xtest"), nb::arg("m"), nb::arg("k"), nb::arg("dist_code"), + nb::arg("mins"), nb::arg("maxs"), nb::arg("is_class")); + m.def("mreg_predict_path", &mreg_predict_path_matrix, nb::arg("rpm_x"), nb::arg("n"), + nb::arg("p"), nb::arg("yhat"), nb::arg("xtest"), nb::arg("m"), nb::arg("kmax"), + nb::arg("dist_code"), nb::arg("mins"), nb::arg("maxs"), nb::arg("is_class")); m.def("nns_dep_pair", &nns_dep_pair, nb::arg("x"), nb::arg("y"), nb::arg("asym")); + // Versioned aliases prevent new Python code from loading the old R13 + // statistic when an installation accidentally retains a stale extension. + m.attr("gravity_r14") = m.attr("gravity"); + m.attr("mode_r14") = m.attr("mode"); + m.attr("gravity_exact_r14") = m.attr("gravity_exact"); + m.attr("nns_dep_pair_r14") = m.attr("nns_dep_pair"); + m.attr("r14_source_commit") = "7b07a837c130ba7220dbe16c584221508757d5ae"; + m.def( "copula_nd", [](const Vector& data, std::size_t n, std::size_t d, const Vector& target, bool continuous) { @@ -1239,9 +899,8 @@ NB_MODULE(_nnscore, m) { if (target.shape(0) != d) { throw std::invalid_argument("target length must equal d."); } + nb::gil_scoped_release release; return copula_nd(data.data(), n, d, target.data(), continuous); }, nb::arg("data"), nb::arg("n"), nb::arg("d"), nb::arg("target"), nb::arg("continuous")); - m.def("nns_reg_mv", &nns_reg_mv, nb::arg("x"), nb::arg("y"), nb::arg("dependence"), - nb::arg("dep_order")); } diff --git a/src/nns/_public_reg.py b/src/nns/_public_reg.py index dc924282..6e4c6ae3 100644 --- a/src/nns/_public_reg.py +++ b/src/nns/_public_reg.py @@ -4,10 +4,12 @@ from typing import Any +from nns._native import honours_ncores from nns.regression import _maybe_render_reg from nns.regression import nns_reg as _nns_reg +@honours_ncores def nns_reg( x: Any, y: Any, @@ -34,10 +36,72 @@ def nns_reg( class_levels: list[object] | None = None, factor_levels: Any = None, ) -> Any: - """Run the repaired regression engine and render only when requested. + """Nonlinear nonparametric regression and classification (R ``NNS.reg``). - Plot flags are handled strictly as side effects and therefore cannot alter - the statistical return value used by the parity suite. + Parameters + ---------- + x + Predictor(s): a vector, a matrix, or a pandas frame. Frame column + names are kept. String and categorical columns are factors (declared + category order, else sorted levels); ``bool`` and numeric columns are + numeric. More than one encoded predictor runs the multivariate + ``NNS.M.reg`` branch unless ``dim_red_method`` is given. + y + Response. Strings, ``bool`` and pandas categoricals are classification + targets (as is an exact 0/1 numeric response); everything else, + including an object array of numbers, is regression. + factor_2_dummy + One-hot encode factor predictors (default, as R) instead of using + integer level codes. + order, n_best, noise_reduction, dist, threshold, tau + Partition order (``None``, an integer or ``"max"``), number of nearest + regression points in the multivariate branch (``None``, an integer or + ``"all"``), within-partition reducer (``"off"``, ``"mean"``, + ``"median"``, ``"mode"``), distance (``None``/``"NNS"``, ``"L1"``, + ``"L2"``, ``"FACTOR"``), dimension-reduction coefficient threshold and + causation lag type (``"cs"`` / ``"ts"``). + dim_red_method + ``None``, ``"cor"``, ``"NNS.dep"``, ``"NNS.caus"``, ``"all"``, + ``"equal"`` or a numeric coefficient vector: collapse the predictors + to one synthetic regressor ``x.star``. + type + ``None``, ``"CLASS"`` (force classification) or ``"XONLY"``. + point_est + Points to predict. Frame (or named-row) columns are matched to the + training columns by name: the same names are reordered, names + disjoint from the training names are positional, a partial overlap + raises ``ValueError``. + confidence_interval + Residual-quantile band level in (0, 1) for ``Fitted.xy`` and + ``pred.int``. + smooth + Smoothing-spline fit through the regression points (univariate). + point_only + Skip the fit statistics (``R2``, ``SE``, ``Fitted.xy`` are ``None``). + multivariate_call + Return only the regression points ``{"x", "y"}``. + class_levels + Response levels in code order (codes ``1..K``); implies + classification. + factor_levels + Predictor levels: a mapping from column name to levels, one entry per + column (``None`` for numeric columns), or a flat list for a single + predictor. Forces those columns to be factors. + plot, plot_regions, residual_plot + Render plots as a side effect; the returned values never change. + return_values, ncores + Accepted for R compatibility and ignored. + + Returns + ------- + dict + R's list names: ``R2``, ``SE``, ``Prediction.Accuracy``, ``equation``, + ``x.star``, ``derivative``, ``Point.est``, ``pred.int``, + ``regression.points``, ``Fitted.xy``, ``class.levels`` (univariate), + or ``R2``, ``rhs.partitions``, ``RPM``, ``Point.est``, ``pred.int``, + ``Fitted.xy``, ``n.best``, ``dist``, ``class.levels`` (multivariate). + Classification returns numeric class codes; ``class.levels`` maps + them to labels. """ result = _nns_reg( # type: ignore[misc] x, diff --git a/src/nns/_r14_central_tendencies.cpp b/src/nns/_r14_central_tendencies.cpp new file mode 100644 index 00000000..90160023 --- /dev/null +++ b/src/nns/_r14_central_tendencies.cpp @@ -0,0 +1,586 @@ +// R14 overlay for NNS-core's central-tendency implementation. +// Source: OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae +// src/central_tendencies.cpp (gravity_value and NNS_mode_cpp). +// Rcpp marshalling is replaced by std::vector; statistical arithmetic is retained. +// The vendored NNS-core snapshot remains unmodified. CMake selects this unit +// instead of its R13.3 central_tendencies.cpp, so multivariate kernels use +// the same R14 gravity for nearest-distance ties as the Python fallback. +// SPDX-License-Identifier: GPL-3.0-only +#include "nns/central_tendencies.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace nns { +namespace { +constexpr double kNaN = std::numeric_limits::quiet_NaN(); +constexpr double kInf = std::numeric_limits::infinity(); +static inline double frac_part(double x) { + return x - std::floor(x); +} + +static inline double mean_vec(const std::vector& v) { + if (v.empty()) return kNaN; + long double s = 0.0L; + for (double x : v) s += x; + return static_cast(s / v.size()); +} + +static inline double nearest_int_half_up(double x) { + double f = std::floor(x); + return ( (x - f) < 0.5 ) ? f : std::ceil(x); +} + +// Given a sorted vector xs, reproduce the q1, q2, q3 *exactly* as in the R code. +// - If length is even: q1 = xs[l*.25], q2 = xs[l*.5], q3 = xs[l*.75] (1-based indexing) +// - If length is odd : q1 and q3 via linear interpolation at l*p, q2 = average of floor/ceil +static void quartiles_like_R_code(const std::vector& xs, double& q1, double& q2, double& q3) { + const int l = static_cast(xs.size()); + const double l25 = l * 0.25; + const double l50 = l * 0.50; + const double l75 = l * 0.75; + + if (l % 2 == 0) { + // 1-based positions in R -> convert to 0-based for C++ + int i25 = std::max(1, (int)std::floor(l25)) - 1; + int i50 = std::max(1, (int)std::floor(l50)) - 1; + int i75 = std::max(1, (int)std::floor(l75)) - 1; + q1 = xs[i25]; + q2 = xs[i50]; + q3 = xs[i75]; + } else { + // q1 interpolated + int f25 = (int)std::floor(l25); + int c25 = (int)std::ceil(l25); + f25 = std::max(1, f25); // guard + c25 = std::max(1, c25); + f25 = std::min(f25, l); + c25 = std::min(c25, l); + double w25 = frac_part(l25); + q1 = xs[f25 - 1] + w25 * (xs[c25 - 1] - xs[f25 - 1]); + + // q2 average of floor and ceil positions + int f50 = (int)std::floor(l50); + int c50 = (int)std::ceil(l50); + f50 = std::max(1, f50); + c50 = std::max(1, c50); + f50 = std::min(f50, l); + c50 = std::min(c50, l); + q2 = 0.5 * (xs[f50 - 1] + xs[c50 - 1]); + + // q3 interpolated + int f75 = (int)std::floor(l75); + int c75 = (int)std::ceil(l75); + f75 = std::max(1, f75); + c75 = std::max(1, c75); + f75 = std::min(f75, l); + c75 = std::min(c75, l); + double w75 = frac_part(l75); + q3 = xs[f75 - 1] + w75 * (xs[c75 - 1] - xs[f75 - 1]); + } +} + +// Minimal replacement for NNS_bin used by mode/gravity. Bins are +// [origin + k*width, origin + (k+1)*width), k = 0 .. nbins-1, with +// nbins = floor((max - origin) / width + 1e-12) + 1. +// +// Gravity and mode only read counts within 6 bins of a non-empty bin, so runs +// of more than kMaxEmptyRun empty bins keep only their first and last +// kMaxEmptyRun / 2 bins: results are bit-identical and memory is O(n). A width +// so small that the bin count exceeds 2^52 is widened to (max - origin) / 2^52. +static constexpr std::int64_t kMaxEmptyRun = 16; +static constexpr double kMaxBinIndex = 4503599627370496.0; // 2^52 + +static void simple_bin_counts(const std::vector& xs, double width, double origin, + std::vector& bin_names, std::vector& counts) { + bin_names.clear(); + counts.clear(); + if (xs.empty()) return; + + const double xmax = xs.back(); + double last = std::floor((xmax - origin) / width + 1e-12); + if (!(last < kMaxBinIndex)) { + width = (xmax - origin) / kMaxBinIndex; + last = std::floor((xmax - origin) / width + 1e-12); + if (!(last < kMaxBinIndex)) last = kMaxBinIndex; + } + const std::int64_t nbins = (last >= 0.0) ? static_cast(last) + 1 : 1; + const std::int64_t cap = static_cast(xs.size()) * (kMaxEmptyRun + 1) + kMaxEmptyRun; + bin_names.reserve(static_cast(std::min(nbins, cap))); + counts.reserve(static_cast(std::min(nbins, cap))); + + auto push_bin = [&](std::int64_t k, int c) { + bin_names.push_back(origin + static_cast(k) * width); + counts.push_back(c); + }; + auto push_empty_run = [&](std::int64_t first, std::int64_t last_bin) { + const std::int64_t len = last_bin - first + 1; + if (len <= 0) return; + if (len <= kMaxEmptyRun) { + for (std::int64_t k = first; k <= last_bin; ++k) push_bin(k, 0); + return; + } + const std::int64_t half = kMaxEmptyRun / 2; + for (std::int64_t k = first; k < first + half; ++k) push_bin(k, 0); + for (std::int64_t k = last_bin - half + 1; k <= last_bin; ++k) push_bin(k, 0); + }; + + // xs is sorted, so bin indices are non-decreasing. + const double top = static_cast(nbins - 1); + std::int64_t prev = -1; + for (double v : xs) { + const double q = std::floor((v - origin) / width); + const std::int64_t k = (q < 0.0) ? 0 : (q >= top ? nbins - 1 : static_cast(q)); + if (k == prev) { + ++counts.back(); + continue; + } + push_empty_run(prev + 1, k - 1); + push_bin(k, 1); + prev = k; + } + push_empty_run(prev + 1, nbins - 1); +} + +// ---------- NNS.gravity ---------- + +// R's default quantile (type 7) of sorted xs: position 1 + (n - 1) p. +static inline double quantile_type7(const std::vector& xs, double p) { + const int n = (int)xs.size(); + const double h = (n - 1) * p; + const int lo = (int)std::floor(h); + const int hi = std::min(lo + 1, n - 1); + return xs[lo] + (h - lo) * (xs[hi] - xs[lo]); +} + +// Core gravity computation operating on a vector of finite values. +// Shared by NNS_gravity_cpp and the NNS.reg regression-point builder. The +// result depends only on the set of values (x is sorted internally), so input +// order is irrelevant. +double gravity_value(std::vector x, bool discrete) { + const int l = (int)x.size(); + if (l == 0) return kNaN; + if (l <= 3) { + // median(x) + std::vector t = x; + std::sort(t.begin(), t.end()); + double med = (l % 2 ? t[l/2] : 0.5*(t[l/2 - 1] + t[l/2])); + if (discrete) return nearest_int_half_up(med); + return med; + } + + bool all_eq = true; + for (int i = 1; i < l; ++i) if (x[i] != x[0]) { all_eq = false; break; } + if (all_eq) return x[0]; + + std::sort(x.begin(), x.end()); + double range = std::fabs(x.back() - x.front()); + if (range == 0.0) return x.front(); + + // The usual median and quartiles (#117): the floor-position order + // statistics used before took the lower middle value for even n, so + // gravity sat below the centre of symmetric data. + const double q1 = quantile_type7(x, 0.25); + const double q2 = quantile_type7(x, 0.50); + const double q3 = quantile_type7(x, 0.75); + + double width = (q3 - q1) * std::pow((double)l, -0.5); + if (!(width > 0.0) || !std::isfinite(width)) width = range / 128.0; + + std::vector z_names; + std::vector counts; + simple_bin_counts(x, width, x.front(), z_names, counts); + const int lz = (int)counts.size(); + + // If unique max, use neighborhood; else use all bins + int maxc = 0; + for (int c : counts) if (c > maxc) maxc = c; + int ties = 0; + for (int c : counts) if (c == maxc) ++ties; + + int lo = 0, hi = lz - 1; + if (ties == 1) { + int zc = 0; for (int i = 0; i < lz; ++i) if (counts[i] == maxc) { zc = i; break; } + lo = std::max(0, zc - 1); + hi = std::min(lz - 1, zc + 1); + } + + // The mode term is the mean of the observations in those bins (#117): the + // bins' left edges, used before, sat half a bin below the data. x is + // sorted, so the observations of bins lo..hi are a contiguous slice. + int a = 0; + for (int i = 0; i < lo; ++i) a += counts[i]; + int e = a; + for (int i = lo; i <= hi; ++i) e += counts[i]; + long double num = 0.0L; + for (int t = a; t < e; ++t) num += (long double)x[t]; + double m = (e > a) ? (double)(num / (long double)(e - a)) : z_names[ (lo+hi)/2 ]; + + double mu = mean_vec(x); + double mid = 0.25 * ( q2 + m + mu + 0.5*(q1 + q3) ); + + double out = std::isfinite(mid) ? mid : q2; + if (discrete) out = nearest_int_half_up(out); + return out; +} + +static void smooth_counts_tri7(const std::vector& counts, std::vector& smooth) { + static const int w[7] = {1,2,3,4,3,2,1}; + static const int Wsum = 16; // 1+2+3+4+3+2+1 + const int n = (int)counts.size(); + smooth.assign(n, 0.0); + if (n == 0) return; + + // Mirror at edges (symmetric extension); the clamp only binds for n <= 3 bins. + auto at = [&](int idx)->int{ + if (idx < 0) idx = -idx; // reflect: -1 -> 1, -2 -> 2, ... + if (idx >= n) idx = 2*n - 2 - idx; // reflect: n -> n-2, n+1 -> n-3, ... + idx = std::min(std::max(idx, 0), n - 1); + return counts[idx]; + }; + + for (int i = 0; i < n; ++i) { + int acc = 0; + acc += w[0]*at(i-3); acc += w[1]*at(i-2); acc += w[2]*at(i-1); + acc += w[3]*at(i ); + acc += w[4]*at(i+1); acc += w[5]*at(i+2); acc += w[6]*at(i+3); + smooth[i] = (double)acc / (double)Wsum; + } +} + + +} // namespace + +double gravity(const double* data, std::size_t n, bool discrete) { + std::vector finite; + finite.reserve(n); + for (std::size_t i = 0; i < n; ++i) + if (std::isfinite(data[i])) finite.push_back(data[i]); + return gravity_value(std::move(finite), discrete); +} + +std::vector rescale(const double* x_in, std::size_t n, double a, double b, + const std::string& method, + std::optional T, + const std::string& type) { + std::vector out(n, kNaN); + + std::string method_lower = method; + std::transform(method_lower.begin(), method_lower.end(), method_lower.begin(), + [](unsigned char c){ return std::tolower(c); }); + + std::string type_lower = type; + std::transform(type_lower.begin(), type_lower.end(), type_lower.begin(), + [](unsigned char c){ return std::tolower(c); }); + + if (method_lower == "minmax") { + double xmin = std::numeric_limits::infinity(); + double xmax = -std::numeric_limits::infinity(); + + for (std::size_t i = 0; i < n; ++i) { + if (std::isfinite(x_in[i])) { + if (x_in[i] < xmin) xmin = x_in[i]; + if (x_in[i] > xmax) xmax = x_in[i]; + } + } + + // Fallback if all values identical or no valid values + if (!std::isfinite(xmin) || !std::isfinite(xmax) || xmax == xmin) { + for (std::size_t i = 0; i < n; ++i) out[i] = (a + b) / 2.0; + return out; + } + + for (std::size_t i = 0; i < n; ++i) { + out[i] = a + (b - a) * ((x_in[i] - xmin) / (xmax - xmin)); + } + return out; + } + + if (method_lower == "riskneutral") { + if (!T.has_value()) { + throw std::invalid_argument("T (time to maturity) must be provided for riskneutral method"); + } + double T_val = T.value(); + + if (!(a > 0.0)) { + throw std::invalid_argument("S_0 (a) must be positive for riskneutral method"); + } + + double S0 = a; + double r = b; + + long double s = 0.0L; + int cnt = 0; + for (std::size_t i = 0; i < n; ++i) { + if (std::isfinite(x_in[i])) { s += x_in[i]; ++cnt; } + } + double mx = (cnt > 0) ? static_cast(s / cnt) : kNaN; + + if (!std::isfinite(mx) || mx <= 0.0) { + throw std::invalid_argument("Mean(x) must be positive/finite for riskneutral scaling"); + } + + double target = (type_lower == "discounted") ? S0 : (S0 * std::exp(r * T_val)); + double theta = std::log(target / mx); + + for (std::size_t i = 0; i < n; ++i) { + out[i] = x_in[i] * std::exp(theta); + } + return out; + } + + throw std::invalid_argument("Invalid method: use 'minmax' or 'riskneutral'"); +} + +std::vector mode(const double* data, std::size_t n, bool discrete, bool multi) { + std::vector x(data, data + n); + + // Coerce to numeric & drop non-finite + std::vector xnum; xnum.reserve(x.size()); + for (double v : x) if (std::isfinite(v)) xnum.push_back((double)v); + + const int l = (int)xnum.size(); + if (l == 0) return std::vector{kNaN}; + + // ====================== DISCRETE PATH ====================== + if (discrete) { + // Integerize and count exact frequencies. The key is a double so values + // beyond the int range stay exact; "+ 0.0" folds -0.0 into the 0 key. + std::unordered_map freq; freq.reserve(l * 2u); + for (double v : xnum) ++freq[ nearest_int_half_up(v) + 0.0 ]; + + int maxf = 0; for (auto &kv : freq) if (kv.second > maxf) maxf = kv.second; + + std::vector modes_int; + for (auto &kv : freq) if (kv.second == maxf) modes_int.push_back(kv.first); + std::sort(modes_int.begin(), modes_int.end()); + + if (multi) { + std::vector out((int)modes_int.size()); + for (int i = 0; i < (int)modes_int.size(); ++i) out[i] = modes_int[i]; + return out; // e.g., 2 3 4 for c(1,2,2,3,3,4,4,5) + } else { + // Tied modes: the one nearest the gravity of the data, then the lower + // value, as in the continuous path; a mean of tied classes is no class. + if (modes_int.size() == 1) return std::vector{modes_int[0]}; + const double g = gravity_value(xnum, false); + double best_c = kNaN, best_d = kInf; + for (double m : modes_int) { + const double dist = std::fabs(m - g); + if (dist < best_d) { best_d = dist; best_c = m; } + } + return std::vector{best_c}; + } + } + + // ====================== CONTINUOUS PATH ====================== + if (l <= 3) { + std::vector tmp = xnum; std::sort(tmp.begin(), tmp.end()); + double med = (l % 2 == 1) ? tmp[l/2] : 0.5*(tmp[l/2 - 1] + tmp[l/2]); + return std::vector{med}; + } + + // All-equal? + bool all_eq = true; + for (int i = 1; i < l; ++i) if (xnum[i] != xnum[0]) { all_eq = false; break; } + if (all_eq) return std::vector{xnum[0]}; + + // Sort & basic stats + std::sort(xnum.begin(), xnum.end()); + double range = std::fabs(xnum.back() - xnum.front()); + if (range == 0.0) return std::vector{xnum.front()}; + + // Quartiles & default bin width + double q1, q2, q3; + quartiles_like_R_code(xnum, q1, q2, q3); + double width = (q3 - q1) * std::pow((double)l, -0.5); + if (!(width > 0.0) || !std::isfinite(width)) width = range / 128.0; + + // ----- Discrete-valued data (#114) ----- + // When every gap between consecutive distinct values exceeds the bin width, + // the histogram only spaces the distinct values out with empty bins, and + // smoothing it moves the peaks off the values. The modes are then read from + // the values' own frequencies: a run of consecutive distinct values with + // equal counts is a mode when the runs beside it have lower counts (or it + // reaches an end), so every mode is a data value and the end values count. + { + std::vector vals; + std::vector cnt; + for (int i = 0; i < l;) { + int j = i + 1; + while (j < l && xnum[j] == xnum[i]) ++j; + vals.push_back(xnum[i]); + cnt.push_back(j - i); + i = j; + } + const int k = (int)vals.size(); + bool coarse = k >= 2; + for (int i = 1; coarse && i < k; ++i) if (!(vals[i] - vals[i - 1] > width)) coarse = false; + if (coarse) { + std::vector modes; + for (int i = 0; i < k;) { + int j = i + 1; + while (j < k && cnt[j] == cnt[i]) ++j; + const bool left_ok = (i == 0) || cnt[i - 1] < cnt[i]; + const bool right_ok = (j == k) || cnt[j] < cnt[i]; + if (left_ok && right_ok) for (int t = i; t < j; ++t) modes.push_back(t); + i = j; + } + if (multi) { + std::vector out((int)modes.size()); + for (int t = 0; t < (int)modes.size(); ++t) out[t] = vals[modes[t]]; + return out; + } + // The most frequent mode; equally frequent ones go to the value nearest + // the gravity of the data, then the lower value (modes ascend). + int top = 0; + for (int t : modes) top = std::max(top, cnt[t]); + int n_top = 0, only = -1; + for (int t : modes) if (cnt[t] == top) { ++n_top; only = t; } + if (n_top == 1) return std::vector{vals[only]}; + const double g = gravity_value(xnum, false); + double best_c = kNaN, best_d = kInf; + for (int t : modes) { + if (cnt[t] != top) continue; + const double dist = std::fabs(vals[t] - g); + if (dist < best_d) { best_d = dist; best_c = vals[t]; } + } + return std::vector{best_c}; + } + } + + // Histogram + std::vector z_names; // left edge of each bin + std::vector counts; // histogram counts + simple_bin_counts(xnum, width, xnum.front(), z_names, counts); + const int lz = (int)counts.size(); + if (lz == 0) return std::vector{kNaN}; + + // xnum is sorted, so the observations of bins lo..hi are the contiguous + // slice [start[lo], start[hi + 1]). A peak is centred on the mean of the + // observations in its window rather than on the bin names, which sit up to + // a bin below the data (#114); a window holding one value returns it. + std::vector start(lz + 1, 0); + for (int b = 0; b < lz; ++b) start[b + 1] = start[b] + counts[b]; + auto obs_centre = [&](int lo, int hi) -> double { + const int a = start[lo], e = start[hi + 1]; + if (e <= a) return z_names[lo]; + if (xnum[a] == xnum[e - 1]) return xnum[a]; + long double s = 0.0L; + for (int t = a; t < e; ++t) s += (long double)xnum[t]; + return (double)(s / (long double)(e - a)); + }; + + // For fallback paths + int maxc = 0; for (int c : counts) if (c > maxc) maxc = c; + + // ----- Peak detection on SMOOTHED counts (concavity) ----- + std::vector cs; smooth_counts_tri7(counts, cs); + + // Optional margin above side maxima (in smoothed counts) + const double MARGIN = 0.0; + + std::vector peak_idx; peak_idx.reserve(lz); + // A peak beats the neighbours at offsets 1..3 that exist. Within three bins + // of either end, where that neighbourhood is cut short, a peak must also be + // the tallest bin of the smoothed histogram: a density highest at its + // minimum (an exponential, say) has its mode there, while an isolated + // extreme observation does not. + const double cs_max = (lz > 0) ? *std::max_element(cs.begin(), cs.end()) : 0.0; + for (int i = 0; i < lz && lz >= 7; ++i) { + const double ci = cs[i]; + if (ci <= 0.0) continue; + if ((i < 3 || i > lz - 4) && ci < cs_max) continue; + + double Ls = -kInf, Rs = -kInf; + for (int d = 1; d <= 3; ++d) { + if (i - d >= 0) Ls = std::max(Ls, cs[i - d]); + if (i + d < lz) Rs = std::max(Rs, cs[i + d]); + } + if (!(ci > Ls + MARGIN && ci > Rs + MARGIN)) continue; + + // Negative curvature gate (concavity), reflected at the ends + const double left = (i > 0) ? cs[i - 1] : ((i + 1 < lz) ? cs[i + 1] : ci); + const double right = (i + 1 < lz) ? cs[i + 1] : ((i > 0) ? cs[i - 1] : ci); + if (!(left - 2.0 * ci + right < 0.0)) continue; + + peak_idx.push_back(i); + } + + // Non-maximum suppression on smoothed heights: keep peaks >= 4 bins apart + if (!peak_idx.empty()) { + // Equal heights keep bin order, so the kept peaks don't depend on the sort implementation. + std::stable_sort(peak_idx.begin(), peak_idx.end(), + [&](int a, int b){ return cs[a] > cs[b]; }); + std::vector kept; + for (int idx : peak_idx) { + bool too_close = false; + for (int jdx : kept) if (std::abs(idx - jdx) <= 3) { too_close = true; break; } + if (!too_close) kept.push_back(idx); + } + + if (!kept.empty()) { + // Per-peak centre: the observations within +-3 bins + std::vector centers((int)kept.size()); + for (int t = 0; t < (int)kept.size(); ++t) { + const int zc = kept[t]; + centers[t] = obs_centre(std::max(0, zc - 3), std::min(lz - 1, zc + 3)); + } + + if (multi) { + std::vector out = centers; + std::sort(out.begin(), out.end()); + return out; + } else { + // GLOBAL-HEIGHT RULE: the kept peak with the largest smoothed height. + // Equally high peaks are broken by distance to the gravity of the + // data (then the lower value), so the result is always a real peak: + // an average of tied peaks can land in a gap with no data. + double best_h = cs[kept[0]]; + for (int t = 1; t < (int)kept.size(); ++t) best_h = std::max(best_h, cs[kept[t]]); + const double g = gravity_value(xnum, false); + double best_c = kNaN, best_d = kInf; + for (int t = 0; t < (int)kept.size(); ++t) { + if (cs[kept[t]] != best_h) continue; + const double dist = std::fabs(centers[t] - g); + if (dist < best_d || (dist == best_d && centers[t] < best_c)) { best_d = dist; best_c = centers[t]; } + } + return std::vector{best_c}; + } + } + } + + // Fallback: if multiple global-max bins exist + int ties = 0; for (int c : counts) if (c == maxc) ++ties; + if (ties > 1) { + if (multi) { + std::vector out(ties); + int pos = 0; + for (int i = 0; i < lz; ++i) if (counts[i] == maxc) out[pos++] = obs_centre(i, i); + std::sort(out.begin(), out.end()); + return out; + } else { + // Tied highest bins: the one nearest the gravity of the data (then the + // lower value), as for tied peaks above. + const double g = gravity_value(xnum, false); + double best_c = kNaN, best_d = kInf; + for (int i = 0; i < lz; ++i) { + if (counts[i] != maxc) continue; + const double c = obs_centre(i, i); + const double dist = std::fabs(c - g); + if (dist < best_d || (dist == best_d && c < best_c)) { best_d = dist; best_c = c; } + } + return std::vector{best_c}; + } + } + + // Final fallback: single winning bin -> centre of the observations within +-1 bin + int zc = 0; for (int i = 0; i < lz; ++i) if (counts[i] == maxc) { zc = i; break; } + return std::vector{obs_centre(std::max(0, zc - 1), std::min(lz - 1, zc + 1))}; +} + +} // namespace nns diff --git a/src/nns/_r14_dependence.hpp b/src/nns/_r14_dependence.hpp new file mode 100644 index 00000000..e4f5ab64 --- /dev/null +++ b/src/nns/_r14_dependence.hpp @@ -0,0 +1,472 @@ +// R14 multilevel dependence with explicit non-R numerical adapters. +// Source: OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae +// src/NNS_dep.cpp: null corrections, constant-cell rules and level selection. +// Rcpp marshalling uses vectors, normal probabilities use erfc, and the +// initial hypergeometric mass uses long-double binomial products. The +// remaining selection/null-correction arithmetic follows the pinned source. +// SPDX-License-Identifier: GPL-3.0-only +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nns_r14 { +struct DepResult { + double correlation; + double dependence; +}; + +// ------------------------------------------------------------ +// Per-cell dependence with the independence floor removed (#70). +// +// Inside a partition cell of m points, around the cell means, two concordance +// terms are measured: +// T0 = |2 p0 - 1|, p0 = share of points in the co-lower or co-upper orthant +// (degree 0: frequency) +// T1 = |sum dx dy| / sum |dx dy| (degree 1: area) +// (the discordant terms are exactly 1 - p0 and 1 - p1, so they add nothing). +// Each term is measured against its own expectation under the within-cell +// permutation null, +// T* = (T - E0[T]) / (1 - E0[T]), +// which keeps perfect dependence at 1 and centres independence near 0. The +// cell's dependence is (T0* + T1*) / 2: no square root, which would turn the +// sampling noise of an unbiased estimate of 0 back into a floor of order +// n^(-1/4). +// E0[T0] uses the hypergeometric distribution (a normal approximation for +// cells larger than 400). E0[T1] is exact whenever the distinct +// arrangements of the cell can be enumerated cheaply (every cell of up to 6 +// points, and larger cells with heavily tied values) and uses the permutation +// variance of sum dx dy otherwise. +// ------------------------------------------------------------ + +// A plain aggregate, so `return {a, b};` compiles as C++11 too: the package +// sets no C++ standard, and older R (4.2 on Windows, for one) builds as C++11. +struct CellCopula { + double signed_value; // debiased dependence times the sign of the cell covariance + double magnitude; // debiased dependence; may be negative below chance +}; + +// E0|2 p0 - 1| for a cell of m points with a points at or below the x mean and +// b points at or below the y mean, under random pairing: the co-lower count K +// is hypergeometric and the concordant count is m - a - b + 2K. +static double null_T0(int m, int a, int b) { + const int k_lo = std::max(0, a + b - m); + const int k_hi = std::min(a, b); + if (k_lo >= k_hi) { // a single possible count: the term carries no information + return 1.0; + } + const double md = static_cast(m); + if (m > 400) { + // Normal approximation with the exact hypergeometric mean and variance. + const double ad = a, bd = b; + const double mean_k = ad * bd / md; + const double var_k = bd * (ad / md) * ((md - ad) / md) * ((md - bd) / (md - 1.0)); + const double mu = (2.0 * (md - ad - bd + 2.0 * mean_k) - md) / md; + const double sd = 4.0 * std::sqrt(std::max(var_k, 0.0)) / md; + if (sd <= 0.0) return std::abs(mu); + const double z = mu / sd; + return sd * std::sqrt(2.0 / 3.14159265358979323846264338327950288) * std::exp(-0.5 * z * z) + + mu * (1.0 - 2.0 * (0.5 * std::erfc(z / std::sqrt(2.0)))); + } + // Hypergeometric probabilities by the ratio recurrence from the lowest count. + // The first hypergeometric mass, evaluated as long-double binomial + // products; the following double recurrence is the pinned R14 one. + const auto choose = [](int n, int k) { + k = std::min(k, n - k); + long double out = 1.0L; + for (int j = 1; j <= k; ++j) out = out * (n - k + j) / j; + return out; + }; + double pk = static_cast(choose(a, k_lo) * choose(m - a, b - k_lo) / choose(m, b)); + double e = 0.0; + for (int k = k_lo; k <= k_hi; ++k) { + const double conc = static_cast(m - a - b + 2 * k); + e += pk * std::abs(2.0 * conc / md - 1.0); + if (k < k_hi) { + pk *= (static_cast(a - k) * static_cast(b - k)) / + (static_cast(k + 1) * static_cast(m - a - b + k + 1)); + } + } + return e; +} + +// log of the number of distinct arrangements of v: m! / prod(multiplicity!). +static double log_arrangements(std::vector v) { + std::sort(v.begin(), v.end()); + const int m = static_cast(v.size()); + double out = std::lgamma(static_cast(m) + 1.0); + for (int i = 0; i < m;) { + int j = i; + while (j < m && v[j] == v[i]) ++j; + out -= std::lgamma(static_cast(j - i) + 1.0); + i = j; + } + return out; +} + +// Enumerate up to this many distinct arrangements exactly (6! = 720, so every +// cell of up to 6 points is exact, and so are larger cells with heavy ties, +// such as 7 points of which 6 are tied). +static const double kLogExactArrangements = std::log(720.0) + 1e-9; +static const int kExactCellSize = 6; + +// E0[ |sum dx dy| / sum |dx dy| ] under random pairing of dx and dy. The +// arrangement counts are only worth computing when a variable has ties. +// legacy = true keeps the rule NNS.dep used before #95 (exact only for cells of +// at most 6 points), for the local measure behind NNS.reg's default order. +static double null_T1(const std::vector& dx, const std::vector& dy, + bool x_ties, bool y_ties, bool legacy = false) { + const int m = static_cast(dx.size()); + // Pairing dy with a random permutation of dx is the same null as the reverse, + // so permute whichever has fewer distinct arrangements. Every distinct + // arrangement of a multiset is equally likely under a uniform permutation. + double lx = 1e300, ly = 1e300; + if (m <= kExactCellSize || (x_ties && !legacy)) lx = log_arrangements(dx); + if (m <= kExactCellSize || (y_ties && !legacy)) ly = log_arrangements(dy); + if (std::min(lx, ly) <= kLogExactArrangements) { + std::vector v = (lx <= ly) ? dx : dy; + const std::vector& w = (lx <= ly) ? dy : dx; + std::sort(v.begin(), v.end()); + double total = 0.0; + double count = 0.0; + do { + double s = 0.0, d = 0.0; + for (int i = 0; i < m; ++i) { + const double p = w[i] * v[i]; + s += p; + d += std::abs(p); + } + total += d > 0.0 ? std::abs(s) / d : 0.0; + count += 1.0; + } while (std::next_permutation(v.begin(), v.end())); + return total / count; + } + double sx2 = 0.0, sy2 = 0.0, sax = 0.0, say = 0.0; + for (int i = 0; i < m; ++i) { + sx2 += dx[i] * dx[i]; + sy2 += dy[i] * dy[i]; + sax += std::abs(dx[i]); + say += std::abs(dy[i]); + } + if (sax <= 0.0 || say <= 0.0) return 1.0; + // sum dx dy has mean 0 and variance sx2 * sy2 / (m - 1) under permutation; + // sum |dx dy| has mean sax * say / m. + const double md = static_cast(m); + const double e = std::sqrt(2.0 / 3.14159265358979323846264338327950288) * std::sqrt(sx2 * sy2 / (md - 1.0)) * md / (sax * say); + return std::min(1.0, e); +} + +static inline double debias_term(double t, double e0) { + if (!(e0 < 1.0 - 1e-12)) return 0.0; + return (t - e0) / (1.0 - e0); +} + +// Constancy of a cell, for the rules applied by the partition (best_level). +enum CellKind { CELL_VARYING = 0, CELL_X_CONST = 1, CELL_BOTH_CONST = 2 }; + +static CellCopula cell_dependence(const std::vector& xv, + const std::vector& yv, + bool x_ties, bool y_ties, CellKind& kind, + bool legacy_null = false) { + const int n = static_cast(xv.size()); + kind = CELL_VARYING; + if (n < 2) return {0.0, 0.0}; + + // Measure from the cell minimum, so deviations round at the cell's spread rather than its level. + double rx = 0.0, ry = 0.0; + if (!legacy_null) { + rx = *std::min_element(xv.begin(), xv.end()); + ry = *std::min_element(yv.begin(), yv.end()); + } + double tx = 0.0, ty = 0.0, ax = 0.0, ay = 0.0; + for (int i = 0; i < n; ++i) { + tx += xv[i] - rx; ty += yv[i] - ry; + ax = std::max(ax, xv[i] - rx); ay = std::max(ay, yv[i] - ry); + } + tx /= n; ty /= n; + // Bound on the mean's rounding, which depends on row order: a value equal to the mean can't fall either side. + const double tie_x = legacy_null ? 0.0 : (n + 1) * DBL_EPSILON * ax; + const double tie_y = legacy_null ? 0.0 : (n + 1) * DBL_EPSILON * ay; + + std::vector dx(n), dy(n); + int a = 0, b = 0; + double conc = 0.0, s = 0.0, d = 0.0; + for (int i = 0; i < n; ++i) { + dx[i] = (xv[i] - rx) - tx; + dy[i] = (yv[i] - ry) - ty; + const bool lo_x = dx[i] <= tie_x, lo_y = dy[i] <= tie_y; + a += lo_x; + b += lo_y; + // Concordant = co-lower (<= target) or co-upper (> target); the degree-0 + // counts partition the cell. + if (lo_x == lo_y) conc += 1.0; + const double p = dx[i] * dy[i]; + s += p; + d += std::abs(p); + } + + // x constant in the cell (discrete x): there is no within-cell variation to + // test, only whether y is determined there. The partition scores that (see + // best_level). Compare the raw values: a repeated 0.1 need not centre to + // exactly 0. + bool x_const = true, y_const = true; + for (int i = 1; i < n; ++i) { + if (xv[i] != xv[0]) x_const = false; + if (yv[i] != yv[0]) y_const = false; + } + if (x_const) { + kind = y_const ? CELL_BOTH_CONST : CELL_X_CONST; + return {0.0, 0.0}; + } + + // s is summed in partition order, so a zero covariance can round either way. + const double s_tol = legacy_null ? 0.0 : 1e-12 * d; + const double slope_sign = (s > s_tol) ? 1.0 : (s < -s_tol) ? -1.0 : 0.0; + const double e0 = null_T0(n, a, b); + + // All concordant or all discordant is perfect dependence, unless the cell is + // too small for that to be informative (a single attainable count). + // Compare counts, not count * (1/n): 49 * (1/49) != 1. + if (e0 < 1.0 - 1e-12) { + if (conc == static_cast(n)) return {1.0, 1.0}; + if (conc == 0.0) return {-1.0, 1.0}; + } + + const double md = static_cast(n); + const double t0 = std::abs(2.0 * conc / md - 1.0); + const double t1 = d > 0.0 ? std::abs(s) / d : 0.0; + const double e1 = null_T1(dx, dy, x_ties, y_ties, legacy_null); + + const double dep = 0.5 * (debias_term(t0, e0) + debias_term(t1, e1)); + if (slope_sign == 0.0) return {0.0, 0.0}; + return {dep * slope_sign, dep}; +} + +struct LevelBest { + double dependence; + double correlation; +}; + +enum PartitionKind { CONDITIONAL, JOINT }; + +// Probability that m values drawn without replacement from v are all equal, +// i.e. that y is constant in a cell of m points under the permutation null. +// counts holds the multiplicities of v's distinct values, largest first. +static double p_constant(int m, const std::vector& counts, int n, + std::vector& cache) { + if (cache[m] >= 0.0) return cache[m]; + const double lchoose_n = std::lgamma(n + 1.0) - std::lgamma(m + 1.0) - std::lgamma(n - m + 1.0); + double p = 0.0; + for (int c : counts) { + if (c < m) break; + p += std::exp(std::lgamma(c + 1.0) - std::lgamma(m + 1.0) - std::lgamma(c - m + 1.0) - lchoose_n); + } + cache[m] = std::min(1.0, p); + return cache[m]; +} + +static std::vector value_counts(std::vector v) { + std::sort(v.begin(), v.end()); + std::vector counts; + for (size_t i = 0; i < v.size();) { + size_t j = i; + while (j < v.size() && v[j] == v[i]) ++j; + counts.push_back(static_cast(j - i)); + i = j; + } + std::sort(counts.begin(), counts.end(), std::greater()); + return counts; +} + +// A partition label: a view of the R string, which stays alive with the +// CharacterVector it was read from. (std::string_view needs C++17.) +// Comparisons are bytewise, as std::string_view's are. +struct Label { + const char* p; + std::size_t n; + std::size_t size() const { return n; } +}; + +static inline bool operator<(const Label& a, const Label& b) { + const int c = std::memcmp(a.p, b.p, std::min(a.n, b.n)); + return c < 0 || (c == 0 && a.n < b.n); +} + +static inline bool operator==(const Label& a, const Label& b) { + return a.n == b.n && std::memcmp(a.p, b.p, a.n) == 0; +} + +// The first len characters of a and b agree (a shorter label counts whole). +static inline bool same_prefix(const Label& a, const Label& b, std::size_t len) { + const std::size_t la = std::min(a.n, len), lb = std::min(b.n, len); + return la == lb && std::memcmp(a.p, b.p, la) == 0; +} + +static std::vector