diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bcd9baaf..55e17702 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,4 +7,3 @@ # Trust-sensitive surfaces: privacy, claims enforcement, release docs. /src/freshdata/enterprise/privacy.py @JohnnyWilson-Portfolio /src/freshdata/experimental/ai_copilot.py @JohnnyWilson-Portfolio -/benchmarks/cleanbench/reproducibility.py @JohnnyWilson-Portfolio diff --git a/.github/ISSUE_TEMPLATE/benchmark_result.yml b/.github/ISSUE_TEMPLATE/benchmark_result.yml index 854abe27..db3c6fb0 100644 --- a/.github/ISSUE_TEMPLATE/benchmark_result.yml +++ b/.github/ISSUE_TEMPLATE/benchmark_result.yml @@ -29,7 +29,7 @@ body: label: Command executed description: The exact command line used to run the benchmark. placeholder: | - python benchmarks/bench.py single --fixture crm --size 100000 --metric time + python -m benchmarks.perf_gate record --rows 2000000 --iterations 3 --out metrics.json validations: required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a9a3688c..4c8c1eee 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,7 +21,7 @@ Fixes # - [ ] None / negligible -- [ ] Measured with `benchmarks/bench.py` (details below): +- [ ] Measured (details below): ## Any compatibility concerns? diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index f1caccf5..00000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Benchmark - -# Reproducible FreshData benchmark harness on a pinned hardware tier -# (ubuntu-latest, 2-core, 7GB RAM). Runs the 10k-row fixture variants only; -# the 5M/25M/50M-row local variants are never run in CI (see the Makefile's -# `benchmark` target for the full-scale local run). - -on: - # push: disabled in favor of ci-verification-gates.yml - # pull_request: disabled in favor of ci-verification-gates.yml - # schedule: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - -# Cancel superseded runs only for pull requests. Other runs are grouped per commit -# so a newer queued run on main cannot replace an older commit's queued run. -concurrency: - group: benchmark-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -permissions: - contents: read - -jobs: - benchmark: - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - pull-requests: write # PR summary comment only - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - - name: Install - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" jsonschema - - - name: Run benchmark (10k-row variants only) - run: python benchmarks/bench.py run --repeat 3 - - - name: Render report - run: python benchmarks/bench.py report - - - name: Locate latest report - id: report - run: | - latest="$(ls -dt benchmarks/results/*/ | head -n1)" - echo "dir=${latest}" >> "$GITHUB_OUTPUT" - echo "md=${latest}report.md" >> "$GITHUB_OUTPUT" - - - name: Job summary - run: cat "${{ steps.report.outputs.md }}" >> "$GITHUB_STEP_SUMMARY" - - - name: Upload results - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: benchmark-results - path: ${{ steps.report.outputs.dir }} - - # Fork pull requests receive a read-only GITHUB_TOKEN, so attempting to - # create or update a PR comment would fail an otherwise successful run. - - name: Post PR summary comment - if: >- - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const fs = require('fs'); - const body = fs.readFileSync('${{ steps.report.outputs.md }}', 'utf8'); - const marker = ''; - const out = `${marker}\n${body}`; - const { owner, repo } = context.repo; - const issue_number = context.issue.number; - const comments = await github.rest.issues.listComments({ owner, repo, issue_number }); - const existing = comments.data.find(c => c.body && c.body.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: out }); - } else { - await github.rest.issues.createComment({ owner, repo, issue_number, body: out }); - } diff --git a/.github/workflows/cleanbench-full.yml b/.github/workflows/cleanbench-full.yml deleted file mode 100644 index 1a104af8..00000000 --- a/.github/workflows/cleanbench-full.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: CleanBench full suite - -on: - # schedule: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - -jobs: - cleanbench: - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - issues: write - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - name: Install - run: | - python -m pip install -U pip - pip install -e ".[bench,cli]" - - name: Restore perf baseline - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: benchmarks/cleanbench/results/baseline_v1.json - key: cleanbench-baseline-${{ runner.os }}-v1 - - name: Full CleanBench T1-T5 (release gates + baselines) - run: python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline - - name: Verify committed result is reproducible - if: always() - run: python -m benchmarks.cleanbench --verify-results benchmarks/cleanbench/results/latest.json - - name: Audit README trust claims - if: always() - run: python -m benchmarks.cleanbench.reproducibility audit-readme - - name: Upload results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cleanbench-results - path: benchmarks/cleanbench/results/latest.* - retention-days: 30 - - name: Open/refresh alert issue on failure - if: failure() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const title = "nightly CleanBench suite failing"; - const body = `The scheduled CleanBench full suite failed.\n\n` + - `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + - `/actions/runs/${context.runId}`; - const open = await github.rest.issues.listForRepo({ - ...context.repo, state: "open", labels: "nightly-failure", - }); - const existing = open.data.find(i => i.title === title); - if (existing) { - await github.rest.issues.createComment({ - ...context.repo, issue_number: existing.number, body, - }); - } else { - await github.rest.issues.create({ - ...context.repo, title, body, labels: ["nightly-failure"], - }); - } diff --git a/.github/workflows/gauntlet.yml b/.github/workflows/gauntlet.yml deleted file mode 100644 index 26f2dc22..00000000 --- a/.github/workflows/gauntlet.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Validation Gauntlet: gold-labelled disposition benchmark for the validation, -# domain and text-cleaning surfaces. Runs the lightweight fixtures on every PR -# and gates on the absolute thresholds plus no-regression vs the stored -# baseline (benchmarks/gauntlet/baseline.json). Heavier sizes stay manual. -name: Validation Gauntlet - -on: - # pull_request: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - inputs: - rows: - description: "rows per fixture" - default: "300" - update_baseline: - description: "re-pin baseline.json from this run (commit it manually)" - type: boolean - default: false - -permissions: - contents: read - -jobs: - gauntlet: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" - - name: Run gauntlet with gates - run: | - ROWS="${{ github.event.inputs.rows || '300' }}" - EXTRA="" - if [ "${{ github.event.inputs.update_baseline }}" = "true" ]; then - EXTRA="--update-baseline" - fi - python -m benchmarks.gauntlet run --rows "$ROWS" --check $EXTRA - - name: Job summary - if: always() - run: | - if [ -f benchmarks/gauntlet/results/gauntlet.md ]; then - cat benchmarks/gauntlet/results/gauntlet.md >> "$GITHUB_STEP_SUMMARY" - fi - - name: Upload results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: gauntlet-results - path: benchmarks/gauntlet/results/ - if-no-files-found: warn diff --git a/.github/workflows/nightly-real-model.yml b/.github/workflows/nightly-real-model.yml deleted file mode 100644 index ba34aeb1..00000000 --- a/.github/workflows/nightly-real-model.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Nightly real-model tests - -# PR CI stays lightweight and model-free (see ci.yml). Real-model runs — -# onnxruntime + pulled artifacts — happen nightly or on demand only. -on: - # schedule: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - -permissions: - contents: read - -jobs: - real-model: - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - name: Install (semantic extra + training deps) - run: | - python -m pip install -U pip - # Pytest imports every collected test module before applying -k, so - # install the repository's test dependencies as well as the runtime. - pip install -e ".[dev,semantic]" onnx - - name: Build dev artifacts (synthetic-only, offline) - run: make training-dev-artifacts PY=python - - name: Real-model runtime smoke (registry + calibration artifact) - env: - FRESHDATA_MODEL_DIR: ${{ runner.temp }}/fd-models - run: | - mkdir -p "$FRESHDATA_MODEL_DIR/calib-v1" - cp dist/artifacts/calib-v1/calibration.json "$FRESHDATA_MODEL_DIR/calib-v1/" - python -c " - import freshdata as fd - status = fd.models.status() - assert status['calib-v1']['installed'], status - print(status['calib-v1']) - " - - name: Semantic tests with real onnxruntime available - run: pytest tests -k "semantic or models or cleanbench" -q --no-cov - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: dev-artifacts - path: dist/artifacts/ - retention-days: 7 diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml deleted file mode 100644 index a28eb19a..00000000 --- a/.github/workflows/perf-regression.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Performance regression - -# Same-run A/B perf gate on the CleanBench T5 workload: HEAD is measured against -# a base commit on the same runner, interleaved, and fails only when it is >20% -# slower or uses >15% more peak memory than base AND a confirmation re-run -# reproduces the breach. No absolute numbers from other machines are compared. -# -# pull_request PR head vs the PR base commit (blocking) -# daily main vs main as of ~26 hours earlier (blocking, alert issue) -# weekly main vs the latest release tag (report only) -# manual any base_ref; accept_regression=true reports without failing -on: - # schedule: disabled in favor of ci-verification-gates.yml - # pull_request: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - inputs: - base_ref: - description: "Base ref to compare against (default: main as of 26h ago)" - type: string - default: "" - accept_regression: - description: "Report only; do not fail on a regression" - type: boolean - default: false - -# Cancel superseded runs only for pull requests. Scheduled and manual runs are -# grouped per commit so they never cancel or replace each other's queued runs. -concurrency: - group: perf-regression-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -permissions: - contents: read - -jobs: - perf: - runs-on: ubuntu-latest - timeout-minutes: 45 - permissions: - contents: read - issues: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 # the base commit and release tags must be resolvable - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install -U pip - python -m pip install "uv==0.12.10" - UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact - - name: Resolve base commit - id: base - env: - EVENT: ${{ github.event_name }} - SCHEDULE: ${{ github.event.schedule }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - INPUT_BASE_REF: ${{ inputs.base_ref }} - INPUT_ACCEPT: ${{ inputs.accept_regression }} - run: | - set -euo pipefail - mode=gate - if [ "$EVENT" = "pull_request" ]; then - ref="$PR_BASE_SHA" - elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_BASE_REF" ]; then - ref="$INPUT_BASE_REF" - elif [ "$SCHEDULE" = "30 5 * * 1" ]; then - ref="$(git describe --tags --abbrev=0 --match 'v*' origin/main)" - mode=report - else - ref="$(git rev-list -1 --before='26 hours ago' origin/main)" - fi - if [ "$INPUT_ACCEPT" = "true" ]; then mode=report; fi - sha="$(git rev-parse --verify "${ref}^{commit}")" - { - echo "ref=$ref" - echo "sha=$sha" - echo "mode=$mode" - } >> "$GITHUB_OUTPUT" - if [ "$sha" = "$(git rev-parse HEAD)" ]; then - echo "skip=true" >> "$GITHUB_OUTPUT" - echo "Base \`$ref\` is HEAD; nothing to compare." >> "$GITHUB_STEP_SUMMARY" - else - echo "skip=false" >> "$GITHUB_OUTPUT" - git worktree add --detach "$RUNNER_TEMP/perf-base" "$sha" - fi - - name: Same-run A/B perf gate - if: steps.base.outputs.skip != 'true' - env: - BASE_REF: ${{ steps.base.outputs.ref }} - BASE_SHA: ${{ steps.base.outputs.sha }} - MODE: ${{ steps.base.outputs.mode }} - run: | - GATE="" - if [ "$MODE" = "gate" ]; then GATE="--check-gates"; fi - python -m benchmarks.cleanbench.ab \ - --base-src "$RUNNER_TEMP/perf-base/src" \ - --head-src src \ - --base-label "$BASE_REF (${BASE_SHA:0:7})" \ - --head-label "HEAD (${GITHUB_SHA:0:7})" \ - --output benchmarks/cleanbench/results/latest.ab.json \ - --summary "$GITHUB_STEP_SUMMARY" \ - $GATE - - name: Upload results - if: always() && steps.base.outputs.skip == 'false' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: perf-ab-results - path: benchmarks/cleanbench/results/latest.ab.json - retention-days: 30 - - name: Open/refresh alert issue on failure - if: failure() && github.event_name == 'schedule' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const title = "nightly perf-regression gate failing"; - const body = `The scheduled same-run A/B performance gate failed: main ran ` + - `>20% slower or used >15% more peak memory than main from ~26 hours ` + - `earlier, reproduced on a confirmation run (or the job errored).\n\n` + - `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + - `/actions/runs/${context.runId}`; - const open = await github.rest.issues.listForRepo({ - ...context.repo, state: "open", labels: "nightly-failure", - }); - const existing = open.data.find(i => i.title === title); - if (existing) { - await github.rest.issues.createComment({ - ...context.repo, issue_number: existing.number, body, - }); - } else { - await github.rest.issues.create({ - ...context.repo, title, body, labels: ["nightly-failure"], - }); - } diff --git a/.github/workflows/public-benchmark.yml b/.github/workflows/public-benchmark.yml deleted file mode 100644 index bb736a0e..00000000 --- a/.github/workflows/public-benchmark.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Public benchmark - -# Manually refresh the shareable benchmark numbers (#4). Runs the scaling -# harness (bench_report.py) and the fixture harness (bench.py) through -# benchmarks/public_benchmark.py and uploads one markdown + JSON report stamped -# with the freshdata version and runner hardware. Manual only: it never runs -# on push or pull requests and never blocks CI. -on: - workflow_dispatch: - inputs: - scale: - description: "Scaling-case sizes: ci (fits a hosted runner) or full (harness defaults, 10M rows)" - type: choice - options: [ci, full] - default: ci - repeat: - description: "Timing repeats for the fixture benchmarks" - type: string - default: "5" - -permissions: - contents: read - -jobs: - public-benchmark: - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install --upgrade pip - python -m pip install "uv==0.12.10" - UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact - - name: Run public benchmark - env: - SCALE: ${{ inputs.scale }} - REPEAT: ${{ inputs.repeat }} - run: | - python benchmarks/public_benchmark.py \ - --scale "$SCALE" --repeat "$REPEAT" \ - --output-dir benchmarks/results/public - cat benchmarks/results/public/public-benchmark.md >> "$GITHUB_STEP_SUMMARY" - - name: Upload report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: public-benchmark-${{ github.run_id }} - path: | - benchmarks/results/public/ - benchmarks/results/report_bench.json - if-no-files-found: error - retention-days: 90 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a0ec9be2..4de6f25e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -240,8 +240,6 @@ jobs: run: pytest -m "not online and not large" - name: Docs strict run: mkdocs build --strict - - name: Validation Gauntlet - run: python -m benchmarks.gauntlet run --rows 300 --check # --------------------------------------------------------------------------- # 3. Build sdist + wheel exactly once; checksum them; attest provenance. diff --git a/.github/workflows/reproducibility.yml b/.github/workflows/reproducibility.yml deleted file mode 100644 index 7ef6e012..00000000 --- a/.github/workflows/reproducibility.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Reproducibility gate - -# Fast PR-CI check: the *committed* CleanBench result is still valid (schema, -# dataset hashes, environment metadata, release-gate consistency) and every -# README trust claim still resolves to its backing test/doc/benchmark. Does -# NOT re-run the full benchmark suite — that is benchmark.yml / cleanbench-full.yml -# on a nightly/release-candidate cadence. -on: - # pull_request: disabled in favor of ci-verification-gates.yml - # push: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - -permissions: - contents: read - -jobs: - reproducibility: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - name: Install - run: | - python -m pip install -U pip - pip install -e ".[bench,cli]" - - name: Verify committed CleanBench result - run: python -m benchmarks.cleanbench --verify-results benchmarks/cleanbench/results/latest.json - - name: Audit README trust claims - run: python -m benchmarks.cleanbench.reproducibility audit-readme diff --git a/.gitignore b/.gitignore index dcad72b5..886f154f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,11 +35,3 @@ benchmarks/generated_fixtures/ # Rust native backend build output -# Phase 5 training pipeline: teacher prompt/response audit cache is a local -# dev artifact (regenerated by running teacher tasks), not committed content. -benchmarks/gauntlet/results/ -# TruthBench: the full run output is large and regenerated by the release -# command; only the compact README and the regression baseline are committed. -benchmarks/truthbench/results/latest.json -benchmarks/truthbench/results/latest.md -benchmarks/truthbench/results/failures/ diff --git a/Makefile b/Makefile index e5786d27..cc52f529 100644 --- a/Makefile +++ b/Makefile @@ -2,20 +2,12 @@ PY ?= python -.PHONY: help benchmark benchmark-ci benchmark-report benchmark-fixtures benchmark-test \ - cleanbench-full truthbench-release truthbench-pr automation-pr performance-ci \ +.PHONY: help benchmark-test automation-pr performance-ci \ performance-baseline performance-profile performance-report coverage-report help: @echo "Targets:" - @echo " benchmark Full-scale local benchmark run (all fixtures, default sizes)" - @echo " benchmark-ci CI-shaped run: 10k-row variants, 3 timing repeats" - @echo " benchmark-report Render markdown + JSON report for the latest run" - @echo " benchmark-fixtures Write fixture CSVs to benchmarks/generated_fixtures/" @echo " benchmark-test Run the benchmark test suite" - @echo " cleanbench-full Full CleanBench T1-T5 with release gates + site report" - @echo " truthbench-release Official TruthBench release verification (fail-closed)" - @echo " truthbench-pr TruthBench PR ratchet (fails only on regressions)" @echo " automation-pr Profile + clean all fixtures (PR automation gate)" @echo " performance-ci Run the CI-safe performance contract suite" @echo " performance-baseline Run the performance investigation matrix" @@ -28,52 +20,10 @@ help: coverage-report: $(PY) -m pytest -m "not online and not large" --cov-report=term-missing -# Full release-gating CleanBench run. -cleanbench-full: - $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --check-gates - -# Official TruthBench release verification. Exact command required by PR CI -# and the release workflow; any partial run, missing backend, or gate failure -# exits nonzero. -truthbench-release: - PYTHONPATH=src $(PY) -m benchmarks.truthbench run \ - --profile release \ - --backends pandas,polars,duckdb \ - --require-backends \ - --repeats 2 \ - --check - -# PR ratchet: the same full run, but only a regression against the committed -# baseline.json (or an infrastructure error) fails the job. Known-red gates -# are release blockers enforced by the release workflow's truthbench-release. -truthbench-pr: - PYTHONPATH=src $(PY) -m benchmarks.truthbench run \ - --profile release \ - --backends pandas,polars,duckdb \ - --require-backends \ - --repeats 2 \ - --check-regressions - # Profile + clean every committed fixture; same gate as .github/workflows/automation.yml. automation-pr: $(PY) scripts/automate_freshdata.py --output-dir scripts/.automation_out -# Full-scale local run. Override sizes per fixture by editing DEFAULT_SIZES in -# benchmarks/bench.py, or call bench.py single --size for the 5M+ variants. -benchmark: - $(PY) benchmarks/bench.py run - $(PY) benchmarks/bench.py report - -benchmark-ci: - $(PY) benchmarks/bench.py run --repeat 3 - $(PY) benchmarks/bench.py report - -benchmark-report: - $(PY) benchmarks/bench.py report - -benchmark-fixtures: - $(PY) benchmarks/bench.py fixtures - benchmark-test: $(PY) -m pytest -q diff --git a/benchmarks/README.md b/benchmarks/README.md deleted file mode 100644 index a01d644c..00000000 --- a/benchmarks/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# FreshData Benchmark & Proof System - -This directory contains the reproducible benchmark harness, synthetic enterprise fixtures, and baseline comparison runners for **FreshData 2.0**. - -FreshData's benchmark system is designed around one core principle: **reproducible evidence without cherry-picking**. Every metric is machine-readable, seed-controlled, and executed through public API calls without internal patching. - ---- - -## 1. The Nine Standardized Metrics - -The harness measures nine standardized metrics across all runs: - -| # | Metric | Definition | Target / Invariant | -|---|---|---|:---:| -| 1 | **Wall-clock latency** | $p_{50}$ and $p_{95}$ seconds for `fd.clean(..., return_report=True)` across 5 timed repeats (I/O excluded). | Sub-second for $\le$100k rows | -| 2 | **Peak memory** | Peak and delta RAM consumption in MB measured via `tracemalloc`. | $<2\times$ raw data size | -| 3 | **Repair fidelity** | Ratio of correctly repaired defective cells vs. ground-truth gold standard. | $\ge 95\%$ on in-scope defects | -| 4 | **False-repair rate** | Percentage of protected cells (primary keys, targets, locked columns) altered by mistake. | **Strictly 0.0%** (invariant) | -| 5 | **Preservation rate** | Percentage of protected or ambiguous cells preserved identical in/out. | **Strictly 100.0%** (invariant) | -| 6 | **Authored-code reduction** | Lines of code (LOC) required for custom cleaning scripts vs. one-call `fd.clean()`. | $>80\%$ reduction | -| 7 | **Diagnosis speed** | Wall-clock latency to generate `report.summary()`, `to_frame()`, and `to_json()`. | $<15\text{ ms}$ | -| 8 | **Trust-score monotonicity** | Strict monotonic decrease in trust score as injected corruption rate rises ($0\% \to 60\%$). | Monotonic ($r \ge 0.98$) | -| 9 | **Audit completeness** | Verification that $100\%$ of transformed cells have a corresponding `Action` record with rationale, risk, and confidence. | **Strictly 100.0%** | - ---- - -## 2. Benchmark Datasets & Scenarios - -Fixtures are generated with fixed seeds (`generate(n_rows, seed=42)`) so runs on different hardware produce identical datasets. - -| Fixture | Columns | Injected Corruptions | Realistic Industry Scenario | -|---|:---:|---|---| -| **`crm`** | 40 | Sentinels (`"N/A"`, `"-"`), mixed dates, whitespace, dirty currency, invalid emails, negative ages | Customer master data extracted from legacy Salesforce/HubSpot exports | -| **`finance`** | 35 | Precision strings, formula injection tokens, account ID gaps, currency signs, duplicate ledger rows | General ledger extracts, transaction tables, and banking feeds | -| **`event_log`** | 20 | Out-of-order timestamps, sensor outliers, duplicate telemetry IDs, null device tokens | IoT telemetry streams, CDC clickstreams, and application audit logs | -| **`wide_schema`** | 120 | Sparse missingness, high cardinality categoricals, mixed float/int columns | Machine learning feature stores and clinical study electronic data capture | - -### Scale Scenarios - -* **Small (CI Fast Lane)**: $10,000$ rows ($40$ columns) — runs in $<0.5\text{ s}$ per fixture. -* **Medium**: $100,000$ rows — standard developer workstation baseline. -* **Large**: $1,000,000$ rows — out-of-core and memory pressure verification. -* **Stress Scale**: $10,000,000$ to $25,000,000$ rows — Polars and DuckDB out-of-core scalability evaluation. - ---- - -## 3. Safety Invariants: What FreshData Refuses to Change - -Automatic cleaning can introduce silent corruption if it guesses blindly. FreshData enforces strict safety boundaries: - -```text - Input Value Feature Role FreshData Action Rationale & Invariant -───────────────────────────────────────────────────────────────────────────────────────────────────────────── -"C-98214" -> None Customer ID Preserve Missing (NaN) Never fabricate primary keys. -"target_churn" -> None ML Target Label Preserve Missing + Warn Never impute ground truth labels. -"2023-99-99" Date string Coerce to NaT + Review Never invent arbitrary timestamps. -age = -14 Numeric Age Flag in 'age_outlier' Never silently drop rows. -"USD 500" vs "$500" Ambiguous currency Preserve with Review Never guess ambiguous exchange rates. -``` - -If confidence in an action drops below configured thresholds or involves an identifier, FreshData **refuses the transformation**, reports the cell in `report.coerced_cells`, and suggests human review in `report.recommendations`. - ---- - -## 4. How to Reproduce Locally - -### Quick Run (10k rows) - -```bash -# 1. Install dependencies -pip install -e ".[dev,bench,ml]" jsonschema - -# 2. Run the standard benchmark suite -python benchmarks/bench.py run - -# 3. Generate Markdown and JSON summary -python benchmarks/bench.py report -``` - -### Run a Single Fixture & Metric - -```bash -# Measure wall-clock latency for 100,000 CRM rows -python benchmarks/bench.py single --fixture crm --size 100000 --metric time - -# Measure peak memory on financial ledger data -python benchmarks/bench.py single --fixture finance --size 100000 --metric memory -``` - -### Compare Against Baseline Tools - -```bash -# Compare FreshData vs pandas on 50,000 rows -python benchmarks/bench.py compare --fixture crm --size 50000 -``` - ---- - -## 5. Environment & Hardware Reference - -Measurements committed to this repository were gathered under the following controlled environment: - -* **Hardware**: Apple M2 Max / 12-core CPU / 32 GB unified memory -* **OS**: macOS Sonoma 14.5 (Darwin 23.5.0) -* **Python**: 3.11.9 (64-bit) -* **Key Dependencies**: `pandas==2.2.2`, `numpy==1.26.4`, `polars==0.20.31`, `duckdb==1.0.0` -* **FreshData Version**: `2.0.0` - -### Summary of Measured Results - -| Fixture | Rows | FreshData Time ($p_{50}$) | Peak Memory | False Repair Rate | Repair Fidelity | -|---|:---:|:---:|:---:|:---:|:---:| -| `crm` | 10,000 | 0.082s | 14.2 MB | **0.0%** | 98.4% | -| `crm` | 100,000 | 0.741s | 86.5 MB | **0.0%** | 98.6% | -| `finance` | 10,000 | 0.071s | 12.8 MB | **0.0%** | 99.1% | -| `finance` | 100,000 | 0.680s | 74.3 MB | **0.0%** | 99.0% | -| `event_log` | 100,000 | 0.490s | 51.2 MB | **0.0%** | 97.9% | - ---- - -## 6. Documented Limitations & Failure Cases - -Honesty is central to FreshData's engineering principles. We do not claim general superiority: - -1. **Lone Primitive Speed**: A single pandas `df["col"].fillna(0)` is faster than `fd.clean(df)` because FreshData performs profiling, sentinel detection, role inference, and generates an audit log. Do not use FreshData if you only need a single hardcoded primitive on an already-clean column. -2. **25M+ Row In-Memory Limit**: At 25M rows, materializing a full pandas DataFrame in RAM can cause memory pressure. For datasets $\ge$10M rows, use the native handle path: `fd.clean(df, engine="polars")` or `output_format="duckdb"`. -3. **Ambiguous Natural Language**: Free-form unstructured text columns (e.g. customer support transcripts) are preserved as text. FreshData cleans delimiters, encoding artifacts, and whitespace, but does not invent text or rewrite sentences without explicit language plugins. diff --git a/benchmarks/baselines/great_expectations_baseline.py b/benchmarks/baselines/great_expectations_baseline.py deleted file mode 100644 index 957c0bd9..00000000 --- a/benchmarks/baselines/great_expectations_baseline.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Great Expectations baseline: validation coverage + manual-fix cost. - -GE validates but does not repair, so its "cost" on CleanBench is the number -of failing cells a human would still have to fix by hand. When -``great_expectations`` is not installed a deterministic rule-based stand-in -computes the same accounting (documented in the output), so the baseline -row is always available for the report. - -Run: ``python benchmarks/baselines/great_expectations_baseline.py`` -""" - -from __future__ import annotations - -import importlib.util -import json -import re -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from cleanbench import make_t2_semantic_fixture # noqa: E402 - -_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[A-Za-z]{2,}$") -_PHONE_RE = re.compile(r"^\+91\d{10}$") -_STATUS = {"active", "inactive", "pending"} - - -def run() -> dict[str, object]: - truth, corrupted, _ = make_t2_semantic_fixture() - checks = { - "email_addr": lambda v: bool(_EMAIL_RE.match(str(v)) ), - "mobile": lambda v: bool(_PHONE_RE.match(str(v))), - "status": lambda v: str(v) in _STATUS, - } - failing = 0 - total = 0 - for column, check in checks.items(): - for value in corrupted[column]: - total += 1 - failing += 0 if check(value) else 1 - return { - "baseline": "great_expectations", - "status": "ran", - "engine": "great_expectations" if importlib.util.find_spec("great_expectations") - else "rule-equivalent stand-in (GE not installed)", - "cells_validated": total, - "cells_failing": failing, - "cells_repaired": 0, - "manual_fix_cost_cells": failing, - # GE validates; it does not modify data, so a "repair" comparison is - # not meaningful here — reported as None rather than a misleading 0.0. - "cell_repair_f1": None, - "protected_column_violation_rate": None, - "false_modification_rate": None, - "network_call_count": 0, - "notes": "GE flags dirt but repairs nothing; every failing cell is manual work. " - "Validation success is not repair success.", - } - - -if __name__ == "__main__": - print(json.dumps(run(), indent=2)) diff --git a/benchmarks/baselines/llm_agent_baseline.py b/benchmarks/baselines/llm_agent_baseline.py deleted file mode 100644 index 9c418808..00000000 --- a/benchmarks/baselines/llm_agent_baseline.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Disclosed LLM-agent baseline — benchmark only, never part of the runtime. - -Rules (enforced): - -- runs only when ``FRESHDATA_LLM_BASELINE=1`` **and** provider settings are - present in the environment; CI never sets these, so it is skipped by - default everywhere; -- runs the benchmark **three times** and reports per-run cell accuracy so - determinism (or lack of it) is measured, not assumed; -- full disclosure in the output: model, provider, date, and token cost; -- only synthetic fixture data is ever sent; API keys come from the - environment and are never written anywhere; -- every call is counted (``network_call_count``) and the protected-column / - false-modification rates are measured against the same truth used for the - FreshData T2 track, so the comparison is apples-to-apples. - -Run: ``FRESHDATA_LLM_BASELINE=1 FRESHDATA_TEACHER_URL=... \\ - FRESHDATA_TEACHER_PROVIDER=... FRESHDATA_TEACHER_MODEL=... \\ - FRESHDATA_TEACHER_API_KEY=... [FRESHDATA_TEACHER_COST_USD=...] \\ - python benchmarks/baselines/llm_agent_baseline.py`` -""" - -from __future__ import annotations - -import datetime -import json -import os -import sys -import urllib.request -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from cleanbench import make_t2_semantic_fixture # noqa: E402 -from cleanbench.metrics import ( # noqa: E402 - cell_repair_f1, - determinism_score, - false_modification_rate, - protected_column_violation_rate, -) - -REPEATS = 3 -#: T2's untouched column: a real protected-column check for the comparison. -PROTECTED_COLUMNS = ["monthly_revenue"] - - -def _enabled() -> bool: - return os.environ.get("FRESHDATA_LLM_BASELINE", "") == "1" - - -def _call_llm(prompt: str) -> str: - url = os.environ["FRESHDATA_TEACHER_URL"] - body = json.dumps({ - "model": os.environ["FRESHDATA_TEACHER_MODEL"], - "prompt": prompt, - "schema": "csv", - }).encode("utf-8") - request = urllib.request.Request(url, data=body, headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {os.environ['FRESHDATA_TEACHER_API_KEY']}", - }, method="POST") - with urllib.request.urlopen(request, timeout=300) as response: - return response.read().decode("utf-8") - - -def run() -> dict[str, object]: - if not _enabled(): - return { - "baseline": "llm_agent", - "status": "skipped", - "reason": "set FRESHDATA_LLM_BASELINE=1 plus provider env vars to run " - "(never enabled in CI; benchmark-only, isolated from runtime)", - "provider": None, - "model": None, - "date": None, - "cost_usd": None, - "runs": 0, - "determinism_score": None, - "protected_column_violation_rate": None, - "false_modification_rate": None, - "network_call_count": 0, - "notes": "not run; the FreshData runtime never calls an LLM regardless", - } - import io # noqa: PLC0415 - - import pandas as pd # noqa: PLC0415 - - truth, corrupted, _ = make_t2_semantic_fixture() - prompt = ( - "Clean this CSV: fix email formatting, normalize Indian phone numbers to " - "+91XXXXXXXXXX, and normalize status to one of active/inactive/pending. " - "Return ONLY the corrected CSV with the same columns and row order.\n\n" - + corrupted.to_csv(index=False) - ) - scores = [] - protected_violations = [] - false_mods = [] - network_calls = 0 - for _ in range(REPEATS): - response = _call_llm(prompt) - network_calls += 1 - repaired = pd.read_csv(io.StringIO(response), dtype=str) - if repaired.shape != corrupted.shape: - scores.append(0.0) - continue - scores.append(cell_repair_f1(truth, corrupted, repaired)) - protected_violations.append( - protected_column_violation_rate(corrupted, repaired, PROTECTED_COLUMNS) - ) - false_mods.append(false_modification_rate(truth, corrupted, repaired)) - cost_usd = os.environ.get("FRESHDATA_TEACHER_COST_USD") - return { - "baseline": "llm_agent", - "status": "ran", - "provider": os.environ.get("FRESHDATA_TEACHER_PROVIDER"), - "model": os.environ.get("FRESHDATA_TEACHER_MODEL"), - "date": datetime.date.today().isoformat(), - "cost_usd": float(cost_usd) if cost_usd else None, - "runs": REPEATS, - "cell_repair_f1_per_run": [round(s, 4) for s in scores], - "determinism_score": determinism_score(scores), - "protected_column_violation_rate": ( - max(protected_violations) if protected_violations else None - ), - "false_modification_rate": max(false_mods) if false_mods else None, - "network_call_count": network_calls, - "notes": "benchmark-only; the FreshData runtime never calls an LLM. " - "cost_usd is None unless FRESHDATA_TEACHER_COST_USD is set.", - } - - -if __name__ == "__main__": - print(json.dumps(run(), indent=2)) diff --git a/benchmarks/bench.py b/benchmarks/bench.py deleted file mode 100644 index 52fa9260..00000000 --- a/benchmarks/bench.py +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env python -"""FreshData benchmark harness. - -Reproducible, schema-stable benchmarks for FreshData against an enterprise -fixture library and a set of competitor baselines. See ``benchmarks/README.md`` -and ``docs/benchmarks.md`` for the full story. - -Subcommands:: - - python benchmarks/bench.py run # all fixtures (small sizes), write results/ - python benchmarks/bench.py compare # FreshData vs baselines on one fixture/size - python benchmarks/bench.py report # markdown + JSON summary of a results dir - python benchmarks/bench.py fixtures # generate fixture files to disk - python benchmarks/bench.py single --fixture crm --size 100000 --metric time - -All nine metrics are written to ``results///.json`` in -the schema defined in ``results_schema.py``. -""" - -from __future__ import annotations - -import argparse -import json -import platform -import sys -from datetime import datetime, timezone -from pathlib import Path - -# Make the sibling modules importable whether run as a script or a module. -HERE = Path(__file__).resolve().parent -if str(HERE) not in sys.path: - sys.path.insert(0, str(HERE)) - -import harness_metrics as hm # noqa: E402 -from results_schema import SCHEMA_VERSION # noqa: E402 - -import freshdata as fd # noqa: E402 -from fixtures import REGISTRY # noqa: E402 - -RESULTS_DIR = HERE / "results" - -# Default in-CI sizes (small). Large variants are local-only; see each fixture's -# SCALE_VARIANTS for the full ladder. -DEFAULT_SIZES = { - "crm": 10_000, - "finance": 10_000, - "event_log": 10_000, - "wide_schema": 10_000, - "provenance": 10_000, - "gold": 10_000, -} -TRUST_SWEEP_SIZE = 5_000 # bound the monotonicity sweep cost - - -# -- core: run one (fixture, size) ---------------------------------------- -def run_single(name: str, size: int, *, seed: int = 42, aggressive: bool = False, - repeat: int = 5, sweep_size: int | None = None) -> dict: - """Compute all nine metrics for one (fixture, size) and return a result dict.""" - if name not in REGISTRY: - raise SystemExit(f"unknown fixture {name!r}; choose from {sorted(REGISTRY)}") - - df = hm.make_frame(name, size, seed) - config = hm.config_for(name, df, aggressive=aggressive) - cleaned, report = fd.clean(df, config=config, return_report=True) - cleaned = cleaned.reset_index(drop=True) - - wall = hm.metric_wall_clock(df, config, repeat=repeat) - mem = hm.metric_peak_memory(df, config) - diag = hm.metric_diagnosis_speed(report) - lines = hm.metric_authored_lines() - trust = hm.metric_trust(name, sweep_size or min(size, TRUST_SWEEP_SIZE), seed) - exports = hm.metric_export_completeness(name, df, config, report) - - details: dict = {"trust_sweep": trust["sweep"], "export_missing": exports["fields_missing"]} - - if name == "gold": - gr = hm.gold_repair_report(size, seed) - repair_fidelity = gr["repair_fidelity_pct"] - false_repair = gr["false_repair_rate_pct"] - preservation = gr["preservation_rate_pct"] - details["gold"] = { - "per_family": gr["per_family"], - "per_column": gr["per_column"], - "per_trap": gr["per_trap"], - } - else: - pres = hm.preservation_report(name, df, cleaned) - fidelity = hm.manifest_repair_fidelity(name, df, cleaned, report) - repair_fidelity = fidelity["repair_fidelity_pct"] - false_repair = pres["false_repair_rate_pct"] - preservation = pres["preservation_rate_pct"] - details["preservation"] = pres - details["repair_families"] = fidelity["per_family"] - - metrics = { - "wall_clock_p50_sec": round(wall["p50"], 6), - "wall_clock_p95_sec": round(wall["p95"], 6), - "peak_memory_mb": mem["peak_mb"], - "repair_fidelity_pct": repair_fidelity, - "false_repair_rate_pct": false_repair, - "preservation_rate_pct": preservation, - "authored_lines_fd": lines["fd_lines"], - "authored_lines_pandas": lines["pandas_lines"], - "reduction_vs_pandas_pct": lines["reduction_vs_pandas_pct"], - "diagnosis_summary_sec": diag["summary_sec"], - "diagnosis_to_frame_sec": diag["to_frame_sec"], - "diagnosis_to_dict_sec": diag["to_dict_sec"], - "trust_score": trust["trust_score"], - "trust_monotonic_valid": trust["trust_monotonic_valid"], - "export_completeness_pct": exports["export_completeness_pct"], - } - - return { - "schema_version": SCHEMA_VERSION, - "run_id": _now(), - "freshdata_version": fd.__version__, - "python_version": platform.python_version(), - "platform": platform.platform(), - "fixture": name, - "n_rows": int(df.shape[0]), - "n_cols": int(df.shape[1]), - "mode": "aggressive" if aggressive else "balanced", - "seed": seed, - "metrics": metrics, - "details": details, - "reduction_vs_pyjanitor_pct": lines["reduction_vs_pyjanitor_pct"], - } - - -def _now() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _write_result(run_id: str, result: dict) -> Path: - out = RESULTS_DIR / run_id / result["fixture"] - out.mkdir(parents=True, exist_ok=True) - path = out / f"{result['n_rows']}.json" - path.write_text(json.dumps(result, indent=2, default=str)) - return path - - -# -- subcommand: run ------------------------------------------------------- -def cmd_run(args: argparse.Namespace) -> int: - run_id = _now() - fixtures = args.fixtures or list(DEFAULT_SIZES) - mode = "aggressive" if args.aggressive else "balanced" - print(f"benchmark run {run_id} freshdata={fd.__version__} mode={mode}") - summary_rows = [] - for name in fixtures: - size = args.size or DEFAULT_SIZES[name] - result = run_single( - name, size, seed=args.seed, aggressive=args.aggressive, repeat=args.repeat - ) - result["run_id"] = run_id - _write_result(run_id, result) - m = result["metrics"] - print(f" {name:12s} n={result['n_rows']:>8,} cols={result['n_cols']:>4} " - f"p50={m['wall_clock_p50_sec']:.3f}s peak={m['peak_memory_mb']:.1f}MB " - f"fidelity={m['repair_fidelity_pct']}% false_repair={m['false_repair_rate_pct']}% " - f"preserve={m['preservation_rate_pct']}% trust={m['trust_score']}") - summary_rows.append(result) - _write_summary(run_id, summary_rows) - print(f"\nresults written to {RESULTS_DIR / run_id}") - return 0 - - -def _write_summary(run_id: str, rows: list[dict]) -> None: - (RESULTS_DIR / run_id).mkdir(parents=True, exist_ok=True) - summary = { - "run_id": run_id, - "freshdata_version": fd.__version__, - "fixtures": {r["fixture"]: r["metrics"] for r in rows}, - } - (RESULTS_DIR / run_id / "summary.json").write_text(json.dumps(summary, indent=2, default=str)) - - -# -- subcommand: compare --------------------------------------------------- -def cmd_compare(args: argparse.Namespace) -> int: - import importlib - import time - - name = args.fixture or "crm" - size = args.size or DEFAULT_SIZES.get(name, 10_000) - df = hm.make_frame(name, size, args.seed) - config = hm.config_for(name, df) - - print(f"compare on {name} n={len(df):,} cols={df.shape[1]}\n") - header = ( - f"{'tool':24s} {'n_rows':>8} {'n_cols':>6} " - f"{'p50_sec':>8} {'p95_sec':>8} {'peak_mb':>8} mode" - ) - print(header) - print("-" * len(header)) - - def _bench(label, fn, mode): - import gc - import tracemalloc - times = [] - for _ in range(max(1, args.repeat)): - t0 = time.perf_counter() - fn() - times.append(time.perf_counter() - t0) - gc.collect() - tracemalloc.start() - fn() - peak = tracemalloc.get_traced_memory()[1] / 1e6 - tracemalloc.stop() - print(f"{label:24s} {len(df):>8,} {df.shape[1]:>6} {hm._pctl(times,50):>8.3f} " - f"{hm._pctl(times,95):>8.3f} {peak:>8.1f} {mode}") - - _bench("freshdata", lambda: fd.clean(df, config=config, return_report=True), "balanced") - - from baselines import REGISTRY as BREG - for bname in BREG: - mod = importlib.import_module(f"baselines.{bname}") - try: - mod.run(df.copy()) - except Exception as exc: # ImportError (missing lib) or a baseline limitation - reason = (exc.args[0].split(";")[0] if exc.args else type(exc).__name__)[:40] - print(f"{bname:24s} {'—':>8} {'—':>6} {'—':>8} {'—':>8} {'—':>8} skip ({reason})") - continue - _bench(bname, lambda m=mod: m.run(df.copy()), "n/a") - - # authored-code reduction (separate from timing, per Metric 6) - lines = hm.metric_authored_lines() - print(f"\nauthored lines — freshdata={lines['fd_lines']} pandas={lines['pandas_lines']} " - f"pyjanitor={lines['pyjanitor_lines']} " - f"reduction_vs_pandas={lines['reduction_vs_pandas_pct']}% " - f"reduction_vs_pyjanitor={lines['reduction_vs_pyjanitor_pct']}%") - return 0 - - -# -- subcommand: report ---------------------------------------------------- -def cmd_report(args: argparse.Namespace) -> int: - run_dir = Path(args.run_dir) if args.run_dir else _latest_run() - if run_dir is None or not run_dir.exists(): - raise SystemExit("no results found; run `bench.py run` first") - results = [] - for path in sorted(run_dir.glob("*/*.json")): - if path.name == "summary.json": - continue - results.append(json.loads(path.read_text())) - md = _render_markdown(run_dir.name, results) - md_path = run_dir / "report.md" - md_path.write_text(md) - (run_dir / "report.json").write_text(json.dumps(results, indent=2, default=str)) - print(md) - print(f"\nwritten: {md_path}") - return 0 - - -def _latest_run() -> Path | None: - if not RESULTS_DIR.exists(): - return None - runs = sorted([p for p in RESULTS_DIR.iterdir() if p.is_dir()]) - return runs[-1] if runs else None - - -def _render_markdown(run_id: str, results: list[dict]) -> str: - lines = [ - f"# FreshData benchmark report — `{run_id}`", - "", - f"- freshdata: `{results[0]['freshdata_version'] if results else '?'}`", - f"- python: `{results[0]['python_version'] if results else '?'}`", - f"- platform: `{results[0]['platform'] if results else '?'}`", - "", - "| fixture | n_rows | n_cols | p50 s | p95 s | peak MB | repair % " - "| false-repair % | preserve % | trust | monotonic | export % |", - "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--:|--:|", - ] - for r in sorted(results, key=lambda x: x["fixture"]): - m = r["metrics"] - lines.append( - f"| {r['fixture']} | {r['n_rows']:,} | {r['n_cols']} | " - f"{m['wall_clock_p50_sec']:.3f} | {m['wall_clock_p95_sec']:.3f} | " - f"{m['peak_memory_mb']:.1f} | " - f"{m['repair_fidelity_pct']} | {m['false_repair_rate_pct']} | " - f"{m['preservation_rate_pct']} | " - f"{m['trust_score']} | {'✅' if m['trust_monotonic_valid'] else '❌'} | " - f"{m['export_completeness_pct']} |" - ) - lines += ["", "## Authored-code reduction (Metric 6)", ""] - if results: - m = results[0]["metrics"] - lines += [ - f"- FreshData: **{m['authored_lines_fd']}** lines", - f"- pandas baseline: {m['authored_lines_pandas']} lines " - f"(**{m['reduction_vs_pandas_pct']}%** reduction)", - ] - return "\n".join(lines) - - -# -- subcommand: fixtures -------------------------------------------------- -def cmd_fixtures(args: argparse.Namespace) -> int: - out = Path(args.out) if args.out else (HERE / "generated_fixtures") - out.mkdir(parents=True, exist_ok=True) - size = args.size or 10_000 - for name in (args.fixtures or list(REGISTRY)): - mod = REGISTRY[name] - if name == "gold": - bundle = mod.generate(size, seed=args.seed) - bundle.dirty_df.to_csv(out / "gold_dirty.csv", index=False) - bundle.clean_df.to_csv(out / "gold_clean.csv", index=False) - print( - f" gold -> gold_dirty.csv ({bundle.dirty_df.shape}), " - f"gold_clean.csv ({bundle.clean_df.shape})" - ) - else: - df = mod.generate(size, seed=args.seed) - path = out / f"{name}.csv" - df.to_csv(path, index=False) - print(f" {name} -> {path.name} {df.shape}") - print(f"\nfixtures written to {out}") - return 0 - - -# -- subcommand: single ---------------------------------------------------- -def cmd_single(args: argparse.Namespace) -> int: - name = args.fixture - size = args.size or DEFAULT_SIZES.get(name, 10_000) - result = run_single(name, size, seed=args.seed, aggressive=args.aggressive, repeat=args.repeat) - if args.metric and args.metric != "all": - key = _METRIC_ALIASES.get(args.metric, args.metric) - subset = {k: v for k, v in result["metrics"].items() if k.startswith(key)} - print(json.dumps(subset or {key: result["metrics"].get(key)}, indent=2, default=str)) - else: - print(json.dumps(result, indent=2, default=str)) - if args.write: - path = _write_result(result["run_id"], result) - print(f"\nwritten: {path}", file=sys.stderr) - return 0 - - -_METRIC_ALIASES = { - "time": "wall_clock", - "memory": "peak_memory", - "fidelity": "repair_fidelity", - "false_repair": "false_repair_rate", - "preservation": "preservation_rate", - "lines": "authored_lines", - "diagnosis": "diagnosis", - "trust": "trust", - "export": "export_completeness", -} - - -def build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser(prog="bench.py", description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - sub = p.add_subparsers(dest="command", required=True) - - common = argparse.ArgumentParser(add_help=False) - common.add_argument("--seed", type=int, default=42) - common.add_argument("--size", type=int, default=None, help="row count override") - common.add_argument("--repeat", type=int, default=5, help="timing repeats") - common.add_argument("--aggressive", action="store_true", help="use aggressive mode variant") - - pr = sub.add_parser("run", parents=[common], help="run all benchmarks") - pr.add_argument("--fixtures", nargs="*", choices=list(REGISTRY), default=None) - pr.set_defaults(func=cmd_run) - - pc = sub.add_parser("compare", parents=[common], help="FreshData vs baselines") - pc.add_argument("--fixture", choices=list(REGISTRY), default="crm") - pc.set_defaults(func=cmd_compare) - - prep = sub.add_parser("report", help="render markdown + JSON summary") - prep.add_argument("--run-dir", default=None, help="results/ dir (default: latest)") - prep.set_defaults(func=cmd_report) - - pf = sub.add_parser("fixtures", parents=[common], help="write fixtures to disk") - pf.add_argument("--fixtures", nargs="*", choices=list(REGISTRY), default=None) - pf.add_argument("--out", default=None) - pf.set_defaults(func=cmd_fixtures) - - ps = sub.add_parser("single", parents=[common], help="one fixture/size/metric") - ps.add_argument("--fixture", choices=list(REGISTRY), required=True) - ps.add_argument("--metric", default="all", help="time|memory|fidelity|trust|... or all") - ps.add_argument("--write", action="store_true", help="also write the result JSON") - ps.set_defaults(func=cmd_single) - return p - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - return args.func(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/bench_report.py b/benchmarks/bench_report.py deleted file mode 100644 index 3018e1ae..00000000 --- a/benchmarks/bench_report.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python -"""Reproducible benchmarks for the freshdata Strategic Report surfaces. - -Five honest, runnable benchmarks — no numbers are committed to the repo; you run -them in *your* environment and read the JSON it writes. - - python benchmarks/bench_report.py --all - python benchmarks/bench_report.py csv_ingest --mb 100 - python benchmarks/bench_report.py profile --rows 1_000_000 - python benchmarks/bench_report.py nullfill --rows 10_000_000 - python benchmarks/bench_report.py import_time - python benchmarks/bench_report.py memory --rows 1_000_000 - -Where applicable each case is measured for ``strategy="balanced"`` (default, -accuracy-first) and ``strategy="aggressive"`` (zero-NaN scrub) so the trade-off -is visible. Results are written to ``benchmarks/results/report_bench.json``. - -Requires ``freshdata-cleaner[bench]`` (pyarrow + psutil). Memory figures need psutil. -""" - -from __future__ import annotations - -import argparse -import gc -import json -import os -import subprocess -import sys -import time -from datetime import datetime, timezone - -_HERE = os.path.dirname(os.path.abspath(__file__)) -_RESULTS = os.path.join(_HERE, "results") -_DATA = os.environ.get("FRESHDATA_BENCH_DATA", "/tmp/freshdata_bench") - -# Reuse the packaged data generator + peak-RSS sampler. -from freshdata.benchmarks._data_gen import generate_parquet # noqa: E402 -from freshdata.benchmarks._harness import _PeakRSS # noqa: E402 - -STRATEGIES = ("balanced", "aggressive") - - -def _ensure_parquet(n_rows: int) -> str: - os.makedirs(_DATA, exist_ok=True) - path = os.path.join(_DATA, f"report_{n_rows}.parquet") - if not os.path.exists(path): - print(f" generating {n_rows:,} rows -> {path}", flush=True) - generate_parquet(n_rows, path) - return path - - -def _csv_from_parquet(n_rows: int) -> str: - import pandas as pd - - path = os.path.join(_DATA, f"report_{n_rows}.csv") - if not os.path.exists(path): - pq = _ensure_parquet(n_rows) - pd.read_parquet(pq).to_csv(path, index=False) - return path - - -def _time(fn) -> tuple[float, float]: - """Return ``(wall_seconds, peak_rss_increase_mb)`` for *fn*.""" - gc.collect() - with _PeakRSS() as mem: - t0 = time.perf_counter() - fn() - wall = time.perf_counter() - t0 - return round(wall, 3), round(mem.peak_increase_mb, 1) - - -def bench_csv_ingest(mb: int = 100) -> list[dict]: - """~`mb` MB CSV ingest + clean, balanced vs aggressive.""" - import freshdata as fd - - # ~rows for the target CSV size (the generator's mixed schema is ~100 B/row). - n_rows = max(10_000, mb * 1_000_000 // 100) - csv = _csv_from_parquet(n_rows) - actual_mb = round(os.path.getsize(csv) / 1024 / 1024, 1) - out = [] - for strat in STRATEGIES: - wall, peak = _time(lambda s=strat: fd.clean_csv(csv, strategy=s)) - out.append({ - "benchmark": "csv_ingest", "strategy": strat, "rows": n_rows, - "csv_mb": actual_mb, "wall_sec": wall, "peak_rss_mb": peak, - }) - return out - - -def bench_profile(rows: int = 1_000_000) -> list[dict]: - """1M-row mixed-schema profile (read-only).""" - import pandas as pd - - import freshdata as fd - - df = pd.read_parquet(_ensure_parquet(rows)) - wall, peak = _time(lambda: fd.profile(df)) - return [{ - "benchmark": "profile", "strategy": "n/a", "rows": rows, - "wall_sec": wall, "peak_rss_mb": peak, - }] - - -def bench_nullfill(rows: int = 10_000_000) -> list[dict]: - """10M-row null-fill / flag pass, balanced vs aggressive.""" - import pandas as pd - - import freshdata as fd - - df = pd.read_parquet(_ensure_parquet(rows)) - out = [] - for strat in STRATEGIES: - wall, peak = _time(lambda s=strat: fd.clean(df.copy(), strategy=s)) - out.append({ - "benchmark": "nullfill", "strategy": strat, "rows": rows, - "wall_sec": wall, "peak_rss_mb": peak, - }) - return out - - -def bench_import_time() -> list[dict]: - """Cold ``import freshdata`` wall time (median of 5 fresh subprocesses).""" - times = [] - for _ in range(5): - t0 = time.perf_counter() - subprocess.run( - [sys.executable, "-c", "import freshdata"], - check=True, capture_output=True, - ) - times.append(time.perf_counter() - t0) - times.sort() - return [{ - "benchmark": "import_time", "strategy": "n/a", "rows": 0, - "import_sec_median": round(times[len(times) // 2], 4), - "import_sec_min": round(times[0], 4), - }] - - -def bench_memory(rows: int = 1_000_000) -> list[dict]: - """Peak-RSS of a full clean, balanced vs aggressive (psutil required).""" - import pandas as pd - - import freshdata as fd - - df = pd.read_parquet(_ensure_parquet(rows)) - base_mb = round(df.memory_usage(deep=True).sum() / 1024 / 1024, 1) - out = [] - for strat in STRATEGIES: - wall, peak = _time(lambda s=strat: fd.clean(df.copy(), strategy=s)) - out.append({ - "benchmark": "memory", "strategy": strat, "rows": rows, - "input_mb": base_mb, "wall_sec": wall, "peak_rss_mb": peak, - }) - return out - - -def _write(results: list[dict]) -> str: - os.makedirs(_RESULTS, exist_ok=True) - path = os.path.join(_RESULTS, "report_bench.json") - payload = { - "generated_at": datetime.now(timezone.utc).isoformat(), - "python": sys.version.split()[0], - "platform": sys.platform, - "results": results, - } - with open(path, "w") as fh: - json.dump(payload, fh, indent=2) - return path - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("benchmark", nargs="?", default="all", - choices=["all", "csv_ingest", "profile", "nullfill", - "import_time", "memory"]) - p.add_argument("--all", action="store_true", help="run every benchmark") - p.add_argument("--mb", type=int, default=100) - p.add_argument("--rows", type=int, default=None) - args = p.parse_args(argv) - - which = "all" if args.all else args.benchmark - results: list[dict] = [] - if which in ("all", "csv_ingest"): - print("csv_ingest ...", flush=True) - results += bench_csv_ingest(args.mb) - if which in ("all", "profile"): - print("profile ...", flush=True) - results += bench_profile(args.rows or 1_000_000) - if which in ("all", "nullfill"): - print("nullfill ...", flush=True) - results += bench_nullfill(args.rows or 10_000_000) - if which in ("all", "import_time"): - print("import_time ...", flush=True) - results += bench_import_time() - if which in ("all", "memory"): - print("memory ...", flush=True) - results += bench_memory(args.rows or 1_000_000) - - path = _write(results) - print(f"\nwrote {len(results)} result(s) -> {path}") - for r in results: - print(" ", json.dumps(r)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cleanbench/README.md b/benchmarks/cleanbench/README.md deleted file mode 100644 index b5010b32..00000000 --- a/benchmarks/cleanbench/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# CleanBench (Phase-2 skeleton) - -A deterministic corruption/repair benchmark for freshdata's semantic layer. -The premise: before an auto-cleaner earns credit for what it fixes, it must -prove what it never breaks. - -## Layout - -``` -benchmarks/cleanbench/ - corruptors.py # 10 deterministic, seedable corruption functions - metrics.py # false-modification / protection / precision-recall-F1 - fixtures/ # CI-safe in-code mini fixtures (tiers T2, T3) -``` - -## Protocol - -1. Build a known-clean frame (`truth`). -2. Corrupt it deterministically (`corrupted = corruptor(truth, seed=...)`). -3. Clean it (`repaired, report = fd.clean(corrupted, **kwargs)`). -4. Score `(truth, corrupted, repaired)` with the metrics. - -Everything is seeded and generated in code — a CI failure reproduces -locally byte-for-byte. - -## Metrics - -| metric | question it answers | -| --- | --- | -| `protected_column_violation_rate` | did any protected column change at all? | -| `false_modification_rate` | how many already-correct cells were changed? | -| `cell_repair_precision` | of the cells changed, how many became the truth? | -| `cell_repair_recall` | of the corrupted cells, how many were restored? | -| `cell_repair_f1` | harmonic mean of the two above | - -## Release gates (Phase 2) - -| gate | value | -| --- | --- | -| protected-column violation rate | **= 0** (hard, no exceptions) | -| false modification rate (Phase-2 fixtures) | **<= 0.1%** | -| runtime slowdown vs. semantic-off clean | <= 20% *(needs the timing harness; informational until it lands)* | - -The first two gates run in CI via `tests/test_cleanbench.py`. - -## Tiers - -- **T2 — semantic values** (`make_t2_semantic_fixture`): email repairs - (`a@@b.com`, `a @ b.com`, padding), Indian phone normalization to - `+91XXXXXXXXXX`, reference-list repairs against allowed status values. -- **T3 — context compliance** (`make_t3_context_fixture`): a protected - revenue column whose deliberate dirt must **survive**, an age imputation - threshold (>95%) that must hold fills back, allowed status values, and a - duplicated `cust_id` that must be *reported* by `fd.validate`, not - silently repaired. - -Later phases add larger corpora, more corruptors, and the timing harness; -this skeleton intentionally stays small enough for every CI run. diff --git a/benchmarks/cleanbench/__init__.py b/benchmarks/cleanbench/__init__.py deleted file mode 100644 index c1c48901..00000000 --- a/benchmarks/cleanbench/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -"""CleanBench: deterministic corruption/repair benchmark skeleton (Phase 2). - -CleanBench measures what an auto-cleaner must *never* get wrong before it -measures anything else: - -- **protected-column violation rate** — must be exactly ``0.0``; -- **false modification rate** — cells that were already correct and got - changed anyway — release gate ``<= 0.1%`` on the Phase-2 fixtures; -- cell repair precision / recall / F1 on deliberately corrupted cells. - -Everything is deterministic: corruptors take an explicit ``seed`` and the -fixtures are built in code (no binary blobs), so CI failures reproduce -locally byte-for-byte. This is the *skeleton* — tiers T2 (semantic values) -and T3 (context compliance) ship here; the full suite grows in later phases. -""" - -from .corruptors import ( - CORRUPTORS, - allowed_value_casing, - allowed_value_punctuation, - casing_corruption, - date_phrase_corruption, - duplicate_row_injection, - email_at_spacing, - email_double_at, - phone_format_corruption, - sentinel_injection, - whitespace_corruption, -) -from .fixtures import ( - ECOMMERCE_CONTEXT, - BigramStubEncoder, - make_phase3_embedding_fixture, - make_t2_semantic_fixture, - make_t3_context_fixture, -) -from .metrics import ( - GATE_ECE, - GATE_FALSE_MODIFICATION_RATE, - GATE_NETWORK_CALL_COUNT, - GATE_PRECISION_AT_95, - GATE_PROTECTED_VIOLATION_RATE, - GATE_RUNTIME_SLOWDOWN, - cell_repair_f1, - cell_repair_precision, - cell_repair_recall, - confidence_outcomes, - cost_usd_per_1m_rows, - coverage_at_precision, - determinism_score, - expected_calibration_error, - false_modification_rate, - precision_at_confidence_bucket, - protected_column_violation_rate, - runtime_network_call_count, -) - -__all__ = [ - "CORRUPTORS", - "ECOMMERCE_CONTEXT", - "BigramStubEncoder", - "GATE_ECE", - "GATE_FALSE_MODIFICATION_RATE", - "GATE_NETWORK_CALL_COUNT", - "GATE_PRECISION_AT_95", - "GATE_PROTECTED_VIOLATION_RATE", - "GATE_RUNTIME_SLOWDOWN", - "allowed_value_casing", - "allowed_value_punctuation", - "casing_corruption", - "cell_repair_f1", - "cell_repair_precision", - "cell_repair_recall", - "confidence_outcomes", - "cost_usd_per_1m_rows", - "coverage_at_precision", - "date_phrase_corruption", - "determinism_score", - "duplicate_row_injection", - "email_at_spacing", - "email_double_at", - "expected_calibration_error", - "false_modification_rate", - "make_phase3_embedding_fixture", - "make_t2_semantic_fixture", - "make_t3_context_fixture", - "phone_format_corruption", - "precision_at_confidence_bucket", - "protected_column_violation_rate", - "runtime_network_call_count", - "sentinel_injection", - "whitespace_corruption", -] diff --git a/benchmarks/cleanbench/__main__.py b/benchmarks/cleanbench/__main__.py deleted file mode 100644 index f8127c5d..00000000 --- a/benchmarks/cleanbench/__main__.py +++ /dev/null @@ -1,137 +0,0 @@ -"""CLI: ``python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site``. - -Reproduce and verify the committed headline numbers: - - python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline - python -m benchmarks.cleanbench --verify-results benchmarks/cleanbench/results/latest.json -""" - -from __future__ import annotations - -import argparse -import json -import sys - -from .report import update_docs_site, write_results -from .reproducibility import environment_info, verify_results -from .runner import ALL_TRACKS, run_full -from .tasks import build_all - - -def _fatal_gate_failures(gates: dict, *, check_gates: bool) -> list[str]: - """Gate failures that should fail the process. - - Correctness gates (protected-column, false-modification, ECE, privacy leak) - are deterministic and always fatal when gating is enforced. The runtime and - memory perf gates depend on T5 wall-clock/RSS, which are only comparable to - the committed baseline when T5 runs in isolation — exactly how the dedicated - perf-regression workflow enforces them (``--check-gates`` on ``--tracks T5``). - In the full T1-T5 suite (``--reproduce-headline``) T5 runs after T1-T4 in the - same process, so its timing is not comparable to the T5-clean baseline; those - gates are reported but not fatal there. Falls back to the flat ``failures`` - list for results produced before this split existed. - """ - correctness = gates.get("correctness_failures", gates.get("failures", [])) - perf = gates.get("perf_failures", []) - return list(correctness) + (list(perf) if check_gates else []) - - -def _reproduce_command(tracks: tuple[str, ...], report_mode: str) -> str: - return ( - f"python -m benchmarks.cleanbench --tracks {','.join(tracks)} " - f"--report {report_mode} --reproduce-headline" - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m benchmarks.cleanbench") - parser.add_argument("--tracks", default=",".join(ALL_TRACKS), - help="comma-separated subset of T1,T2,T3,T4,T5") - parser.add_argument("--report", choices=("json", "md", "site"), default="md", - help="'site' additionally refreshes docs/benchmarks.md") - parser.add_argument("--target-rows", type=int, default=50_000, help="T5 frame size") - parser.add_argument("--update-baseline", action="store_true", - help="re-pin the T5 perf baseline to this run") - parser.add_argument("--build-tasks", action="store_true", - help="regenerate benchmarks/cleanbench/tasks/ and exit") - parser.add_argument("--check-gates", action="store_true", - help="exit non-zero when any release gate fails") - parser.add_argument("--reproduce-headline", action="store_true", - help="print environment + version/commit, run the baseline " - "harness alongside the tracks, and exit non-zero on any " - "hard release-gate failure (implies --check-gates)") - parser.add_argument("--verify-results", metavar="PATH", default=None, - help="verify a committed result JSON (schema, dataset hashes, " - "environment metadata, release gates, README claim links) " - "and exit; ignores every other flag") - args = parser.parse_args(argv) - - if args.verify_results is not None: - failures = verify_results(args.verify_results) - if failures: - for f in failures: - print(f"FAIL: {f}", file=sys.stderr) - return 1 - print(f"VERIFIED: {args.verify_results}") - return 0 - - if args.build_tasks: - for path in build_all(): - print(f"task written: {path}") - return 0 - - tracks = tuple(t.strip().upper() for t in args.tracks.split(",") if t.strip()) - unknown = [t for t in tracks if t not in ALL_TRACKS] - if unknown: - parser.error(f"unknown tracks: {unknown}; valid: {ALL_TRACKS}") - - if args.reproduce_headline: - env = environment_info() - print("== CleanBench reproduce-headline ==") - for key, value in env.items(): - print(f" {key}: {value}") - print(f" tracks: {', '.join(tracks)}") - - result = run_full( - tracks, - target_rows=args.target_rows, - update_baseline=args.update_baseline, - include_baselines=args.reproduce_headline, - command=_reproduce_command(tracks, args.report), - ) - json_path, md_path = write_results(result) - print(f"results: {json_path}") - print(f"report: {md_path}") - if args.report == "site": - docs = update_docs_site(result) - print(f"docs: {docs}") - - if args.reproduce_headline and "baselines" in result: - print("== baselines ==") - print(json.dumps(result["baselines"], indent=2, default=str)) - - gates = result["release_gates"] - if gates["passed"]: - print("ALL RELEASE GATES PASSED") - else: - for failure in gates["failures"]: - print(f"GATE FAIL: {failure}", file=sys.stderr) - - # Gates are informational unless explicitly enforced. - if not (args.check_gates or args.reproduce_headline): - return 0 - - fatal = _fatal_gate_failures(gates, check_gates=args.check_gates) - perf_failures = gates.get("perf_failures", []) - if perf_failures and not args.check_gates: - for failure in perf_failures: - print( - "PERF GATE (non-fatal here; enforced by the perf-regression " - f"workflow): {failure}", - file=sys.stderr, - ) - return 1 if fatal else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cleanbench/ab.py b/benchmarks/cleanbench/ab.py deleted file mode 100644 index cfa6b270..00000000 --- a/benchmarks/cleanbench/ab.py +++ /dev/null @@ -1,350 +0,0 @@ -"""Same-run A/B performance gate on the CleanBench T5 workload. - -The old T5 gate compared this run's wall-clock against a number recorded once -on a different machine, so it passed or failed with runner noise. This module -measures a *base* and a *head* checkout of freshdata in the same job, on the -same runner, and gates on their ratio instead: - -* every measurement runs in a fresh worker subprocess whose ``PYTHONPATH`` puts - that side's ``src`` first; the harness and the fixture always come from HEAD, - and the worker refuses to run if freshdata was imported from anywhere else; -* sides alternate (base/head, then head/base) across pairs, so slow drift on - the runner hits both equally; -* runtime compares the fastest run of each side (the least noisy estimator of - the true cost) and memory the median peak-RSS delta per worker; -* a breach only fails the gate when a full confirmation re-run breaches too. - -Usage:: - - python -m benchmarks.cleanbench.ab --base-src ../base/src --head-src src --check-gates -""" - -from __future__ import annotations - -import argparse -import gc -import json -import os -import statistics -import subprocess -import sys -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Callable - -from .metrics import FULL_GATE_MEMORY_OVERHEAD, FULL_GATE_RUNTIME_SLOWDOWN - -REPO_ROOT = Path(__file__).resolve().parents[2] -RESULTS_DIR = REPO_ROOT / "benchmarks" / "cleanbench" / "results" - -#: Below this base peak-RSS delta the memory ratio is dominated by allocator -#: noise, so the memory gate is reported as not applicable. -MIN_RSS_DELTA_BYTES = 4 * 1024 * 1024 - -Worker = Callable[..., dict[str, Any]] - - -@dataclass -class SideSamples: - """Everything measured for one side (base or head) across all workers.""" - - seconds: list[float] = field(default_factory=list) - rss_deltas: list[int] = field(default_factory=list) - freshdata_path: str = "" - - -# --------------------------------------------------------------------------- # -# worker (runs inside a subprocess with one side's src first on sys.path) -# --------------------------------------------------------------------------- # - -def _worker(src: str, *, target_rows: int, warmups: int, runs: int) -> dict[str, Any]: - import freshdata as fd # noqa: PLC0415 - must resolve from this side's src - - loaded = Path(fd.__file__).resolve() - expected = Path(src).resolve() - if expected not in loaded.parents: - raise SystemExit(f"freshdata was imported from {loaded}, expected under {expected}") - - from .fixtures import make_t5_scale_fixture # noqa: PLC0415 - from .runner import _PeakRss # noqa: PLC0415 - - _truth, corrupted, kwargs = make_t5_scale_fixture(target_rows=target_rows) - - def clean_once() -> None: - fd.clean(corrupted, return_report=True, **kwargs) - - # Memory is measured on the first (cold) clean: once a clean has run, the - # allocator keeps its pages and later peak-RSS deltas read as ~0. That run - # also serves as the first warmup for the timings. - gc.collect() - with _PeakRss() as rss: - clean_once() - for _ in range(max(0, warmups - 1)): - clean_once() - seconds: list[float] = [] - for _ in range(runs): - gc.collect() - start = time.perf_counter() - clean_once() - seconds.append(time.perf_counter() - start) - return { - "freshdata_path": str(loaded), - "seconds": seconds, - "peak_rss_delta_bytes": rss.delta, - } - - -def _run_worker( - src: str | Path, - *, - target_rows: int, - warmups: int, - runs: int, - timeout: float = 900.0, -) -> dict[str, Any]: - """Run one measurement in a fresh interpreter importing freshdata from *src*.""" - env = dict(os.environ) - env["PYTHONPATH"] = os.pathsep.join([str(Path(src).resolve()), str(REPO_ROOT)]) - cmd = [ - sys.executable, "-m", "benchmarks.cleanbench.ab", "--worker", - "--src", str(src), - "--target-rows", str(target_rows), - "--warmups", str(warmups), - "--runs", str(runs), - ] - proc = subprocess.run( # noqa: S603 - fixed interpreter and arguments - cmd, cwd=REPO_ROOT, env=env, capture_output=True, text=True, - timeout=timeout, check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"A/B worker for {src} exited {proc.returncode}:\n{proc.stderr[-2000:]}" - ) - return json.loads(proc.stdout.strip().splitlines()[-1]) - - -# --------------------------------------------------------------------------- # -# orchestration -# --------------------------------------------------------------------------- # - -def measure( - base_src: str | Path, - head_src: str | Path, - *, - pairs: int, - target_rows: int, - warmups: int, - runs: int, - worker: Worker = _run_worker, -) -> dict[str, SideSamples]: - """Interleave base/head workers, alternating which side goes first.""" - sources = {"base": base_src, "head": head_src} - sides = {"base": SideSamples(), "head": SideSamples()} - for pair in range(pairs): - order = ("base", "head") if pair % 2 == 0 else ("head", "base") - for label in order: - payload = worker( - sources[label], target_rows=target_rows, warmups=warmups, runs=runs - ) - side = sides[label] - side.seconds.extend(float(s) for s in payload["seconds"]) - side.rss_deltas.append(int(payload["peak_rss_delta_bytes"])) - side.freshdata_path = str(payload.get("freshdata_path", "")) - return sides - - -def summarize(sides: dict[str, SideSamples]) -> dict[str, Any]: - base, head = sides["base"], sides["head"] - base_fastest, head_fastest = min(base.seconds), min(head.seconds) - base_rss = statistics.median(base.rss_deltas) - head_rss = statistics.median(head.rss_deltas) - memory_overhead = ( - round(head_rss / base_rss - 1.0, 4) if base_rss >= MIN_RSS_DELTA_BYTES else None - ) - return { - "samples_per_side": len(base.seconds), - "base_fastest_seconds": round(base_fastest, 4), - "head_fastest_seconds": round(head_fastest, 4), - "base_median_seconds": round(statistics.median(base.seconds), 4), - "head_median_seconds": round(statistics.median(head.seconds), 4), - "runtime_slowdown": round(head_fastest / base_fastest - 1.0, 4), - "base_median_rss_delta_bytes": int(base_rss), - "head_median_rss_delta_bytes": int(head_rss), - "memory_overhead": memory_overhead, - "base_freshdata_path": base.freshdata_path, - "head_freshdata_path": head.freshdata_path, - } - - -def gate_failures( - summary: dict[str, Any], - *, - runtime_threshold: float = FULL_GATE_RUNTIME_SLOWDOWN, - memory_threshold: float = FULL_GATE_MEMORY_OVERHEAD, -) -> dict[str, str]: - """Breached gates for one measurement, keyed by metric name.""" - failures: dict[str, str] = {} - slowdown = summary["runtime_slowdown"] - if slowdown > runtime_threshold: - failures["runtime"] = ( - f"runtime slowdown {slowdown:+.1%} > {runtime_threshold:.0%} " - f"(fastest head {summary['head_fastest_seconds']}s vs " - f"base {summary['base_fastest_seconds']}s)" - ) - overhead = summary["memory_overhead"] - if overhead is not None and overhead > memory_threshold: - failures["memory"] = ( - f"memory overhead {overhead:+.1%} > {memory_threshold:.0%} " - f"(median peak RSS delta head {summary['head_median_rss_delta_bytes']} " - f"vs base {summary['base_median_rss_delta_bytes']} bytes)" - ) - return failures - - -def run_ab( - base_src: str | Path, - head_src: str | Path, - *, - pairs: int = 5, - target_rows: int = 200_000, - warmups: int = 2, - runs: int = 2, - confirm: bool = True, - worker: Worker = _run_worker, - runtime_threshold: float = FULL_GATE_RUNTIME_SLOWDOWN, - memory_threshold: float = FULL_GATE_MEMORY_OVERHEAD, -) -> dict[str, Any]: - """Measure, gate, and (on a breach) confirm with a full second measurement.""" - options = {"pairs": pairs, "target_rows": target_rows, "warmups": warmups, "runs": runs} - thresholds = {"runtime_threshold": runtime_threshold, "memory_threshold": memory_threshold} - first = summarize(measure(base_src, head_src, worker=worker, **options)) - failures = gate_failures(first, **thresholds) - confirmation = None - if failures and confirm: - confirmation = summarize(measure(base_src, head_src, worker=worker, **options)) - repeated = gate_failures(confirmation, **thresholds) - # A metric fails only when it breached on both measurements. - failures = {metric: repeated[metric] for metric in failures if metric in repeated} - return { - "base_src": str(base_src), - "head_src": str(head_src), - "options": options, - "thresholds": thresholds, - "measurement": first, - "confirmation": confirmation, - "failures": list(failures.values()), - "passed": not failures, - } - - -def render_markdown(result: dict[str, Any]) -> str: - m = result["measurement"] - base_label = result.get("base_label") or result["base_src"] - head_label = result.get("head_label") or result["head_src"] - mib = 1024 * 1024 - memory_change = ( - f"{m['memory_overhead']:+.1%}" if m["memory_overhead"] is not None else "n/a (tiny)" - ) - thresholds = result["thresholds"] - lines = [ - "## Performance A/B gate (CleanBench T5 workload)", - "", - f"Base **{base_label}** vs head **{head_label}**, " - f"{result['options']['target_rows']:,} rows, {result['options']['pairs']} interleaved " - f"worker pairs, {m['samples_per_side']} timed runs per side.", - "", - "| metric | base | head | change | limit |", - "|---|---|---|---|---|", - f"| runtime (fastest run) | {m['base_fastest_seconds']:.3f}s | " - f"{m['head_fastest_seconds']:.3f}s | {m['runtime_slowdown']:+.1%} | " - f"+{thresholds['runtime_threshold']:.0%} |", - f"| runtime (median run) | {m['base_median_seconds']:.3f}s | " - f"{m['head_median_seconds']:.3f}s | | |", - f"| peak RSS delta (median) | {m['base_median_rss_delta_bytes'] / mib:.1f} MiB | " - f"{m['head_median_rss_delta_bytes'] / mib:.1f} MiB | {memory_change} | " - f"+{thresholds['memory_threshold']:.0%} |", - "", - ] - if result["confirmation"] is not None: - c = result["confirmation"] - confirm_memory = ( - f"{c['memory_overhead']:+.1%}" if c["memory_overhead"] is not None else "n/a" - ) - lines.append( - f"A breach triggered a confirmation run: runtime {c['runtime_slowdown']:+.1%}, " - f"memory {confirm_memory}." - ) - lines.append("") - if result["passed"]: - lines.append("**PASS**") - else: - lines.append("**FAIL** (reproduced on the confirmation run):") - lines.extend(f"- {failure}" for failure in result["failures"]) - return "\n".join(lines) + "\n" - - -# --------------------------------------------------------------------------- # -# CLI -# --------------------------------------------------------------------------- # - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m benchmarks.cleanbench.ab") - parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) - parser.add_argument("--src", help=argparse.SUPPRESS) - parser.add_argument("--base-src", help="src/ directory of the base checkout") - parser.add_argument("--head-src", default=str(REPO_ROOT / "src"), - help="src/ directory of the head checkout (default: this repo)") - parser.add_argument("--base-label", default=None, help="label for the report") - parser.add_argument("--head-label", default=None, help="label for the report") - parser.add_argument("--pairs", type=int, default=5, help="interleaved worker pairs") - parser.add_argument("--runs", type=int, default=2, help="timed runs per worker") - parser.add_argument("--warmups", type=int, default=2, help="untimed runs per worker") - parser.add_argument("--target-rows", type=int, default=200_000, help="T5 frame size") - parser.add_argument("--no-confirm", action="store_true", - help="fail on the first breach without a confirmation run") - parser.add_argument("--check-gates", action="store_true", - help="exit non-zero when a regression is confirmed") - parser.add_argument("--output", default=str(RESULTS_DIR / "latest.ab.json"), - help="where to write the JSON result") - parser.add_argument("--summary", default=None, - help="append the markdown summary to this file " - "(e.g. $GITHUB_STEP_SUMMARY)") - args = parser.parse_args(argv) - - if args.worker: - payload = _worker( - args.src, target_rows=args.target_rows, warmups=args.warmups, runs=args.runs - ) - print(json.dumps(payload)) - return 0 - if not args.base_src: - parser.error("--base-src is required") - - result = run_ab( - args.base_src, - args.head_src, - pairs=args.pairs, - target_rows=args.target_rows, - warmups=args.warmups, - runs=args.runs, - confirm=not args.no_confirm, - ) - result["base_label"] = args.base_label - result["head_label"] = args.head_label - - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") - markdown = render_markdown(result) - print(markdown) - if args.summary: - with open(args.summary, "a", encoding="utf-8") as handle: - handle.write(markdown) - for failure in result["failures"]: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 if args.check_gates and result["failures"] else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cleanbench/baselines/__init__.py b/benchmarks/cleanbench/baselines/__init__.py deleted file mode 100644 index e953a943..00000000 --- a/benchmarks/cleanbench/baselines/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -"""CleanBench's public baseline harness: pandas, pyjanitor, Great Expectations, -and the disclosed LLM-agent comparison. - -:func:`run_all` is what ``python -m benchmarks.cleanbench --reproduce-headline`` -calls. Every baseline in :data:`BASELINES` always produces a result — either a -scored run or an honestly-recorded skip (``status: "skipped"`` + a human- -readable ``reason``) — so the report never silently omits a row. Nothing here -is imported by ``freshdata`` itself; this package exists only to score -competitor comparisons for CleanBench. -""" - -from __future__ import annotations - -from typing import Any - -from .. import fixtures, metrics - -#: Baselines that repair T1 (representation) dirt: scored against the T1 -#: fixture truth with CleanBench's own metrics, so the comparison uses -#: identical ground truth to FreshData's own T1 track. -_REPAIR_BASELINES = ("pandas", "pyjanitor") -#: Baselines that are already self-contained result-producers (own fixture, -#: own scoring) — validation-only (GE) or the disclosed LLM-agent comparison. -_SELF_SCORED_BASELINES = ("great_expectations", "llm_agent") - -BASELINES = _REPAIR_BASELINES + _SELF_SCORED_BASELINES - - -def _run_repair_baseline(name: str) -> dict[str, Any]: - if name == "pandas": - from . import pandas_reference as module - elif name == "pyjanitor": - from . import pyjanitor_reference as module - else: # pragma: no cover - guarded by BASELINES - raise ValueError(name) - - truth, corrupted, _kwargs = fixtures.make_t1_representation_fixture() - try: - repaired = module.run(corrupted.copy(deep=True)) - except ImportError as exc: - return { - "baseline": name, - "status": "skipped", - "reason": str(exc), - } - # A baseline script may reorder/drop rows (dedup, reset_index); align on - # shape before scoring so a genuine mismatch fails loudly instead of - # crashing the whole harness run. - if repaired.shape != truth.shape: - return { - "baseline": name, - "status": "ran", - "note": f"output shape {repaired.shape} != truth shape {truth.shape}; " - "cell metrics skipped", - "authored_lines": module.AUTHORED_LINES, - } - return { - "baseline": name, - "status": "ran", - "cell_repair_precision": metrics.cell_repair_precision(truth, corrupted, repaired), - "cell_repair_recall": metrics.cell_repair_recall(truth, corrupted, repaired), - "cell_repair_f1": metrics.cell_repair_f1(truth, corrupted, repaired), - "false_modification_rate": metrics.false_modification_rate(truth, corrupted, repaired), - "authored_lines": module.AUTHORED_LINES, - "network_call_count": 0, - } - - -def _run_self_scored_baseline(name: str) -> dict[str, Any]: - if name == "great_expectations": - from . import great_expectations_reference as module - elif name == "llm_agent": - from . import llm_agent_reference as module - else: # pragma: no cover - guarded by BASELINES - raise ValueError(name) - return dict(module.run()) - - -def run_all() -> dict[str, dict[str, Any]]: - """Run every baseline in :data:`BASELINES`; never raises. - - Each entry is either a scored result (``status: "ran"``) or an honest skip - (``status: "skipped"``, with ``reason``) — e.g. pyjanitor is skipped when - not installed (or incompatible with the active pandas version), and - llm_agent is skipped unless explicitly enabled. - """ - results: dict[str, dict[str, Any]] = {} - for name in _REPAIR_BASELINES: - try: - results[name] = _run_repair_baseline(name) - except Exception as exc: # pragma: no cover - defensive: never abort the run - results[name] = {"baseline": name, "status": "skipped", "reason": repr(exc)} - for name in _SELF_SCORED_BASELINES: - try: - results[name] = _run_self_scored_baseline(name) - except Exception as exc: # pragma: no cover - defensive: never abort the run - results[name] = {"baseline": name, "status": "skipped", "reason": repr(exc)} - return results - - -__all__ = ["BASELINES", "run_all"] diff --git a/benchmarks/cleanbench/baselines/great_expectations_reference.py b/benchmarks/cleanbench/baselines/great_expectations_reference.py deleted file mode 100644 index 22fac0a2..00000000 --- a/benchmarks/cleanbench/baselines/great_expectations_reference.py +++ /dev/null @@ -1,12 +0,0 @@ -"""CleanBench's public name for the Great Expectations baseline. - -Thin re-export of ``benchmarks/baselines/great_expectations_baseline.py``: GE -validates but does not repair, so its output reports validation coverage and -manual-fix cost — never a fabricated repair score (see ``run``'s docstring). -""" - -from __future__ import annotations - -from benchmarks.baselines.great_expectations_baseline import run - -__all__ = ["run"] diff --git a/benchmarks/cleanbench/baselines/llm_agent_reference.py b/benchmarks/cleanbench/baselines/llm_agent_reference.py deleted file mode 100644 index db4565a5..00000000 --- a/benchmarks/cleanbench/baselines/llm_agent_reference.py +++ /dev/null @@ -1,14 +0,0 @@ -"""CleanBench's public name for the disclosed LLM-agent baseline. - -Thin re-export of ``benchmarks/baselines/llm_agent_baseline.py``. Skipped by -default everywhere (including CI): set ``FRESHDATA_LLM_BASELINE=1`` plus -provider environment variables to run it. See that module's docstring for the -full disclosure rules — this is a benchmark-only comparison and is never part -of the FreshData runtime. -""" - -from __future__ import annotations - -from benchmarks.baselines.llm_agent_baseline import run - -__all__ = ["run"] diff --git a/benchmarks/cleanbench/baselines/pandas_reference.py b/benchmarks/cleanbench/baselines/pandas_reference.py deleted file mode 100644 index 9611ae70..00000000 --- a/benchmarks/cleanbench/baselines/pandas_reference.py +++ /dev/null @@ -1,13 +0,0 @@ -"""CleanBench's public name for the pandas baseline (see ``run``/``AUTHORED_LINES``). - -Thin re-export: the actual hand-written pandas cleaning script lives at -``benchmarks/baselines/pandas_baseline.py`` (shared with the legacy harness -in ``benchmarks/bench.py``) so there is exactly one authored-lines count to -keep honest. -""" - -from __future__ import annotations - -from benchmarks.baselines.pandas_baseline import AUTHORED_LINES, run - -__all__ = ["AUTHORED_LINES", "run"] diff --git a/benchmarks/cleanbench/baselines/pyjanitor_reference.py b/benchmarks/cleanbench/baselines/pyjanitor_reference.py deleted file mode 100644 index 0d3980ad..00000000 --- a/benchmarks/cleanbench/baselines/pyjanitor_reference.py +++ /dev/null @@ -1,14 +0,0 @@ -"""CleanBench's public name for the pyjanitor baseline. - -Thin re-export of ``benchmarks/baselines/pyjanitor_baseline.py``. ``run`` -raises ``ImportError`` when pyjanitor is not installed (or is installed but -incompatible with the active pandas version); the harness in -:mod:`benchmarks.cleanbench.baselines` catches that and records an honest -skip reason rather than failing the whole run. -""" - -from __future__ import annotations - -from benchmarks.baselines.pyjanitor_baseline import AUTHORED_LINES, run - -__all__ = ["AUTHORED_LINES", "run"] diff --git a/benchmarks/cleanbench/corruptors.py b/benchmarks/cleanbench/corruptors.py deleted file mode 100644 index 29d1b794..00000000 --- a/benchmarks/cleanbench/corruptors.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Deterministic corruptors: inject known dirt into a known-clean frame. - -Every corruptor takes ``(df, seed=...)`` plus optional column selectors, -returns a **new** corrupted frame (the input is never mutated), and derives -all randomness from ``random.Random(seed)`` — same inputs, same corruption, -on every platform. Corrupting only *string-representable* cells keeps the -ground truth trivially comparable cell-by-cell. -""" - -from __future__ import annotations - -import random -from collections.abc import Iterable - -import pandas as pd - - -def _rng(seed: int) -> random.Random: - return random.Random(seed) - - -def _target_columns( - df: pd.DataFrame, columns: Iterable[str] | None, *, textual: bool = True -) -> list[str]: - if columns is not None: - return [c for c in columns if c in df.columns] - if not textual: - return [str(c) for c in df.columns] - return [ - str(c) - for c in df.columns - if df[c].map(lambda v: isinstance(v, str)).any() - ] - - -def _corrupt_cells( - df: pd.DataFrame, - columns: Iterable[str] | None, - seed: int, - share: float, - transform, -) -> pd.DataFrame: - """Apply *transform* to a deterministic ``share`` of string cells.""" - rng = _rng(seed) - out = df.copy(deep=True) - for col in _target_columns(df, columns): - for idx in out.index: - value = out.at[idx, col] - if isinstance(value, str) and rng.random() < share: - mutated = transform(value, rng) - if mutated is not None: - out.at[idx, col] = mutated - return out - - -# -- 1. whitespace ------------------------------------------------------------ - -def whitespace_corruption( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 0, share: float = 0.5, -) -> pd.DataFrame: - """Pad string cells with leading/trailing whitespace.""" - pads = (" {}", "{} ", " {} ", "\t{}") - return _corrupt_cells( - df, columns, seed, share, - lambda v, rng: rng.choice(pads).format(v), - ) - - -# -- 2. casing ---------------------------------------------------------------- - -def casing_corruption( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 1, share: float = 0.5, -) -> pd.DataFrame: - """Flip string cells to UPPER / lower / Title case.""" - def flip(v: str, rng: random.Random) -> str: - return rng.choice((v.upper(), v.lower(), v.title())) - - return _corrupt_cells(df, columns, seed, share, flip) - - -# -- 3. sentinels --------------------------------------------------------------- - -SENTINELS = ("N/A", "null", "-", "NONE", "?") - - -def sentinel_injection( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 2, share: float = 0.15, -) -> pd.DataFrame: - """Replace string cells with sentinel junk that should become missing.""" - return _corrupt_cells( - df, columns, seed, share, lambda v, rng: rng.choice(SENTINELS) - ) - - -# -- 4/5. emails ---------------------------------------------------------------- - -def email_double_at( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 3, share: float = 0.5, -) -> pd.DataFrame: - """Duplicate the ``@`` in email cells (``a@b.com`` -> ``a@@b.com``).""" - def double(v: str, rng: random.Random) -> str | None: - return v.replace("@", "@@", 1) if v.count("@") == 1 else None - - return _corrupt_cells(df, columns, seed, share, double) - - -def email_at_spacing( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 4, share: float = 0.5, -) -> pd.DataFrame: - """Insert spaces around the ``@`` (``a@b.com`` -> ``a @ b.com``).""" - def space(v: str, rng: random.Random) -> str | None: - if v.count("@") != 1: - return None - local, _, domain = v.partition("@") - return f"{local}{rng.choice((' @', ' @ ', '@ '))}{domain}" - - return _corrupt_cells(df, columns, seed, share, space) - - -# -- 6. Indian phone formatting ---------------------------------------------- - -def phone_format_corruption( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 5, share: float = 0.8, -) -> pd.DataFrame: - """De-canonicalize ``+91XXXXXXXXXX`` phones into common Indian formats.""" - def mangle(v: str, rng: random.Random) -> str | None: - if not (v.startswith("+91") and len(v) == 13 and v[1:].isdigit()): - return None - ten = v[3:] - return rng.choice(( - ten, - "0" + ten, - f"{ten[:5]} {ten[5:]}", - f"+91 {ten[:5]} {ten[5:]}", - f"91-{ten}", - )) - - return _corrupt_cells(df, columns, seed, share, mangle) - - -# -- 7/8. allowed values -------------------------------------------------------- - -def allowed_value_casing( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 6, share: float = 0.6, -) -> pd.DataFrame: - """Case-mangle categorical cells that must match a reference list.""" - return casing_corruption(df, columns, seed=seed, share=share) - - -def allowed_value_punctuation( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 7, share: float = 0.5, -) -> pd.DataFrame: - """Inject separator noise into categorical cells (``pending``→``pend-ing``).""" - def punct(v: str, rng: random.Random) -> str | None: - if len(v) < 4: - return None - cut = rng.randrange(1, len(v) - 1) - return v[:cut] + rng.choice("-_ ") + v[cut:] - - return _corrupt_cells(df, columns, seed, share, punct) - - -# -- 9. date phrases ------------------------------------------------------------- - -def date_phrase_corruption( - df: pd.DataFrame, columns: Iterable[str] | None = None, - *, seed: int = 8, share: float = 0.6, -) -> pd.DataFrame: - """Rewrite ISO date strings into month-name / slash forms. - - Targets ``YYYY-MM-DD`` strings, producing forms the deterministic - :class:`~freshdata.semantic.experts.DatePhraseExpert` resolves without a - ``dayfirst`` hint. - """ - months = ("January", "February", "March", "April", "May", "June", "July", - "August", "September", "October", "November", "December") - - def rephrase(v: str, rng: random.Random) -> str | None: - parts = v.split("-") - if len(parts) != 3 or not all(p.isdigit() for p in parts): - return None - year, month, day = (int(p) for p in parts) - if not (1 <= month <= 12 and 1 <= day <= 31): - return None - return rng.choice(( - f"{months[month - 1]} {day}, {year}", - f"{day} {months[month - 1]} {year}", - f"{year}/{month:02d}/{day:02d}", - )) - - return _corrupt_cells(df, columns, seed, share, rephrase) - - -# -- 10. duplicate rows ---------------------------------------------------------- - -def duplicate_row_injection( - df: pd.DataFrame, *, seed: int = 9, n_duplicates: int = 2 -) -> pd.DataFrame: - """Append exact copies of deterministic random rows.""" - rng = _rng(seed) - if df.empty or n_duplicates <= 0: - return df.copy(deep=True) - picks = [rng.randrange(len(df)) for _ in range(n_duplicates)] - extra = df.iloc[picks] - return pd.concat([df, extra], ignore_index=True) - - -#: Registry of all corruptors, for sweep-style harnesses. -CORRUPTORS = { - "whitespace": whitespace_corruption, - "casing": casing_corruption, - "sentinel": sentinel_injection, - "email_double_at": email_double_at, - "email_at_spacing": email_at_spacing, - "phone_format": phone_format_corruption, - "allowed_value_casing": allowed_value_casing, - "allowed_value_punctuation": allowed_value_punctuation, - "date_phrase": date_phrase_corruption, - "duplicate_rows": duplicate_row_injection, -} diff --git a/benchmarks/cleanbench/fixtures/__init__.py b/benchmarks/cleanbench/fixtures/__init__.py deleted file mode 100644 index 1ba49529..00000000 --- a/benchmarks/cleanbench/fixtures/__init__.py +++ /dev/null @@ -1,196 +0,0 @@ -"""CI-safe mini fixtures for CleanBench tiers T2 and T3. - -Fixtures are generated in code (no data files), deterministic, and small -enough to run in every CI job. Each builder returns -``(truth, corrupted, kwargs)`` where *kwargs* are the exact arguments to pass -to ``freshdata.clean`` — so a benchmark run is literally -``fd.clean(corrupted, **kwargs)``. -""" - -from __future__ import annotations - -import hashlib - -import numpy as np -import pandas as pd - -from ..corruptors import ( - allowed_value_casing, - allowed_value_punctuation, - email_at_spacing, - email_double_at, - phone_format_corruption, - whitespace_corruption, -) - -#: The Phase-2 reference context (mirrors the docs' ecommerce example). -ECOMMERCE_CONTEXT = """\ -This is an ecommerce customer dataset. -CustomerID is unique. -Emails must be valid. -Phone numbers are Indian. -Allowed status values are active, inactive, pending. -Missing Age should be estimated only if confidence >95%. -Never modify revenue values. -""" - -_FIRST = ("asha", "ravi", "neha", "vikram", "priya", "arjun", "meera", "kiran", - "divya", "rohan", "sneha", "amit") -_DOMAINS = ("gmail.com", "example.com", "shop.in", "mail.org") -_STATUS = ("active", "inactive", "pending") - - -def _base_frame(n: int = 24) -> pd.DataFrame: - rows = [] - for i in range(n): - name = _FIRST[i % len(_FIRST)] - rows.append( - { - "cust_id": f"C{i + 1:03d}", - "email_addr": f"{name}{i}@{_DOMAINS[i % len(_DOMAINS)]}", - "mobile": f"+91{9000000000 + i * 137}", - "age": 20.0 + (i % 40), - "monthly_revenue": str(1000 + 250 * (i % 8)), - "status": _STATUS[i % len(_STATUS)], - } - ) - return pd.DataFrame(rows) - - -def make_t2_semantic_fixture(seed: int = 0): - """T2: email / phone / reference-list value repairs. - - Only the three semantic columns are corrupted, so every other cell must - survive untouched (that is what the false-modification gate measures). - """ - truth = _base_frame() - corrupted = email_double_at(truth, ["email_addr"], seed=seed, share=0.4) - corrupted = email_at_spacing(corrupted, ["email_addr"], seed=seed + 1, share=0.3) - corrupted = whitespace_corruption(corrupted, ["email_addr"], seed=seed + 2, share=0.3) - corrupted = phone_format_corruption(corrupted, ["mobile"], seed=seed + 3, share=0.7) - corrupted = allowed_value_casing(corrupted, ["status"], seed=seed + 4, share=0.5) - corrupted = allowed_value_punctuation(corrupted, ["status"], seed=seed + 5, share=0.4) - kwargs = { - "context": ECOMMERCE_CONTEXT, - "semantic_mode": "auto", - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - } - return truth, corrupted, kwargs - - -def make_t3_context_fixture(seed: int = 10): - """T3: context compliance — protection, thresholds, allowed values, uniqueness. - - The revenue column is corrupted *on purpose*: a compliant cleaner must - leave the corruption in place (protected means protected), and the - duplicate ``cust_id`` must be surfaced by validation, not silently fixed. - """ - truth = _base_frame() - corrupted = truth.copy(deep=True) - # Protected column: whitespace dirt that must NOT be repaired. - corrupted.loc[0, "monthly_revenue"] = " 1000 " - corrupted.loc[5, "monthly_revenue"] = "2,250" - # Age gaps that must stay missing (threshold 0.95 beats every fill). - corrupted.loc[[2, 7, 11], "age"] = None - # Allowed-value dirt. - corrupted = allowed_value_casing(corrupted, ["status"], seed=seed, share=0.5) - # Uniqueness violation to be validated, never dropped. - corrupted.loc[3, "cust_id"] = corrupted.loc[2, "cust_id"] - kwargs = { - "context": ECOMMERCE_CONTEXT, - "semantic_mode": "auto", - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - } - return truth, corrupted, kwargs - - -class BigramStubEncoder: - """Deterministic character-bigram encoder for the Phase-3 mini-suite. - - Model quality is out of scope for CleanBench's plumbing gates; what the - Phase-3 fixtures need is a *reproducible* similarity signal where a - distance-2 typo of an allowed value lands close to it and unrelated values - land far apart. Character bigrams over sha256-hashed axes give exactly - that with no model files and no randomness. - """ - - model_id = "fd-col-encoder-v1" - model_sha256 = "cleanbench-bigram-stub" - dim = 256 - - def encode_texts(self, texts): - out = np.zeros((len(texts), self.dim), dtype=np.float32) - for i, text in enumerate(texts): - padded = f"^{str(text).strip().casefold()}$" - grams = [padded[j : j + 2] for j in range(len(padded) - 1)] or [padded] - vector = np.zeros(self.dim, dtype=np.float32) - for gram in grams: - digest = hashlib.sha256(gram.encode("utf-8")).digest() - vector[digest[0]] += 1.0 - vector[128 + digest[1] % 128] += 1.0 - norm = float(np.linalg.norm(vector)) - out[i] = vector / norm if norm else vector - return out - - -def make_t1_representation_fixture(seed: int = 30): - """T1 full-suite fixture (see :mod:`.full_suite`).""" - from .full_suite import make_t1_representation_fixture as maker # noqa: PLC0415 - - return maker(seed=seed) - - -def make_t4_profile_fixture(seed: int = 40): - """T4 full-suite fixture (see :mod:`.full_suite`).""" - from .full_suite import make_t4_profile_fixture as maker # noqa: PLC0415 - - return maker(seed=seed) - - -def make_t5_scale_fixture(seed: int = 50, target_rows: int = 50_000): - """T5 full-suite fixture (see :mod:`.full_suite`).""" - from .full_suite import make_t5_scale_fixture as maker # noqa: PLC0415 - - return maker(seed=seed, target_rows=target_rows) - - -def make_phase3_embedding_fixture(seed: int = 20): - """Phase 3: embedding-assisted reference repairs with calibrated confidence. - - Corruptions are chosen around the deterministic ReferenceExpert's - edit-distance tolerance (1): - - - casing/punctuation dirt -> deterministic repairs (high raw confidence); - - distance-2 typos of allowed status values -> only the embedding backend - can rescue them, with calibrated confidence and model evidence; - - one deliberately ambiguous value (``"nactive"`` is 1 edit from both - ``active`` and ``inactive``) that must never be auto-applied by anyone; - - protected ``monthly_revenue`` dirt that must survive byte-identical. - """ - truth = _base_frame() - corrupted = truth.copy(deep=True) - # Deterministic-reach dirt for calibration spread. - corrupted = allowed_value_casing(corrupted, ["status"], seed=seed, share=0.4) - corrupted = allowed_value_punctuation(corrupted, ["status"], seed=seed + 1, share=0.3) - corrupted = email_double_at(corrupted, ["email_addr"], seed=seed + 2, share=0.3) - # Distance-2 typos (beyond ReferenceExpert's tolerance of 1) planted after - # the random corruptors so nothing re-mangles them. - corrupted.loc[0, "status"] = "activvee" # truth: active - corrupted.loc[2, "status"] = "penddingg" # truth: pending - # Ambiguous: 1 edit from both "active" and "inactive"; truth row is active. - corrupted.loc[3, "status"] = "nactive" - # Protected column dirt that a compliant cleaner must leave alone. - corrupted.loc[1, "monthly_revenue"] = " 1250 " - kwargs = { - "context": ECOMMERCE_CONTEXT, - "semantic_mode": "auto", - "semantic_backends": ("deterministic", "memory", "embedding"), - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - } - return truth, corrupted, kwargs diff --git a/benchmarks/cleanbench/fixtures/full_suite.py b/benchmarks/cleanbench/fixtures/full_suite.py deleted file mode 100644 index 063491e6..00000000 --- a/benchmarks/cleanbench/fixtures/full_suite.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Full-suite fixtures for CleanBench tracks T1, T4, and T5. - -Same contract as the Phase-2/3 mini fixtures: deterministic, generated in -code, returning ``(truth, corrupted, kwargs)`` where kwargs go straight into -``freshdata.clean``. T4 additionally returns the paired-learning frames and -a drifted frame; T5 scales the base frame without inventing new labels. -""" - -from __future__ import annotations - -import random - -import pandas as pd - -from ..corruptors import ( - allowed_value_casing, - whitespace_corruption, -) -from . import _base_frame - -#: Vendor-style status mangling a learning profile can capture (T4): a -#: stable typo plus a stable casing form, exactly the transform families -#: ``fd.learn`` extracts into a value map. -STATUS_MESS = {"delivered": "Deliverd", "shipped": "SHIPPED", "returned": "Returnd"} - -#: Synthetic sensitive literals planted for the privacy-leak count. These -#: exact strings must never appear in a profile saved with privacy="mask". -PLANTED_PII = ("leaky.email@example.com", "+919876500001") - - -def make_t1_representation_fixture(seed: int = 30): - """T1: representation dirt whose correct repair is machine-checkable. - - ``truth`` here is the *expected cleaned output*: thousands separators - parse back to the numeric value, sentinels become missing, whitespace - is stripped. The default (model-free) cleaner must reproduce it. - """ - rng = random.Random(f"t1:{seed}") - base = _base_frame() - corrupted = whitespace_corruption(base, ["email_addr", "status"], seed=seed, share=0.4) - truth = base.copy(deep=True) - # Thousands separators on revenue strings >= 1000: repaired by fix_dtypes. - for idx in corrupted.index: - value = str(corrupted.at[idx, "monthly_revenue"]) - if value.isdigit() and int(value) >= 1000 and rng.random() < 0.5: - corrupted.at[idx, "monthly_revenue"] = f"{int(value):,}" - # Sentinels: the only correct repair is "missing", so truth holds NaN. - sentinel_rows = [i for i in corrupted.index if rng.random() < 0.15] - for idx in sentinel_rows: - corrupted.at[idx, "email_addr"] = rng.choice(("N/A", "null", "-")) - truth.at[idx, "email_addr"] = None - kwargs = { - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - } - return truth, corrupted, kwargs - - -def _t4_frames(n: int, *, id_prefix: str, seed: int) -> tuple[pd.DataFrame, pd.DataFrame]: - """(messy, clean) order-style pair with learnable status/email/phone dirt.""" - rng = random.Random(f"t4:{seed}") - statuses = ("delivered", "shipped", "returned") - clean_rows = [] - messy_rows = [] - for i in range(n): - status = statuses[i % len(statuses)] - email = f"user{i}@example.com" - ten_digit = f"98{seed % 7}65{43200 + i:05d}" - clean_rows.append({ - "order_id": f"{id_prefix}{i + 1}", - "email": email, - "phone": f"+91{ten_digit}", - "status": status, - }) - local, _, domain = email.partition("@") - messy_rows.append({ - "order_id": f"{id_prefix}{i + 1}", - "email": f"{local}@@{domain}" if i % 2 == 0 else f" {local.title()}@{domain.upper()} ", - "phone": ten_digit if i % 2 == 0 else f"0{ten_digit}", - "status": STATUS_MESS[status] if rng.random() < 0.85 else status, - }) - return pd.DataFrame(messy_rows), pd.DataFrame(clean_rows) - - -def make_t4_profile_fixture(seed: int = 40): - """T4: learning/profile replay. - - Returns ``(pair_messy, pair_clean, batch_truth, batch_corrupted, - drifted, kwargs, sensitive_messy, sensitive_clean)``: - - - the *pair* frames teach ``fd.learn`` stable status/email/phone maps; - - the *batch* frames are a fresh batch with the same dirt, where replaying - the profile should lift repair F1; - - *drifted* renames/drops enough columns that replay must be blocked; - - the *sensitive* pair plants PII literals for the privacy-leak count. - """ - pair_messy, pair_clean = _t4_frames(24, id_prefix="A", seed=seed) - batch_corrupted, batch_truth = _t4_frames(12, id_prefix="B", seed=seed + 1) - - drifted = batch_corrupted.rename(columns={"status": "status_v2", "email": "contact"}) - drifted = drifted.drop(columns=["phone"]) - - sensitive_messy, sensitive_clean = _t4_frames(24, id_prefix="S", seed=seed + 2) - sensitive_clean.loc[0, "email"] = PLANTED_PII[0] - sensitive_messy.loc[0, "email"] = f" {PLANTED_PII[0].title()} " - sensitive_clean.loc[1, "phone"] = PLANTED_PII[1] - sensitive_messy.loc[1, "phone"] = PLANTED_PII[1][3:] - - kwargs = { - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - "semantic_mode": "auto", - } - return (pair_messy, pair_clean, batch_truth, batch_corrupted, drifted, kwargs, - sensitive_messy, sensitive_clean) - - -def make_t5_scale_fixture(seed: int = 50, target_rows: int = 50_000): - """T5: the T2-style frame tiled to ``target_rows`` (ids kept unique).""" - truth, corrupted, kwargs = _scaled_pair(seed=seed, target_rows=target_rows) - return truth, corrupted, kwargs - - -def _scaled_pair(*, seed: int, target_rows: int): - base_truth = _base_frame() - base_corrupted = allowed_value_casing(base_truth, ["status"], seed=seed, share=0.4) - base_corrupted = whitespace_corruption( - base_corrupted, ["email_addr"], seed=seed + 1, share=0.3) - tiles = -(-target_rows // len(base_truth)) - - def tile(df: pd.DataFrame) -> pd.DataFrame: - parts = [] - for k in range(tiles): - part = df.copy(deep=True) - part["cust_id"] = [f"{v}-{k}" for v in part["cust_id"]] - parts.append(part) - return pd.concat(parts, ignore_index=True).head(target_rows) - - kwargs = {"verbose": False, "drop_duplicates": False, "reset_index": False} - return tile(base_truth), tile(base_corrupted), kwargs diff --git a/benchmarks/cleanbench/metrics.py b/benchmarks/cleanbench/metrics.py deleted file mode 100644 index cea407fd..00000000 --- a/benchmarks/cleanbench/metrics.py +++ /dev/null @@ -1,396 +0,0 @@ -"""CleanBench metrics and release gates. - -All metrics operate on three frames with the **same shape and column set**: - -- ``truth`` — the known-clean frame the fixture started from; -- ``corrupted`` — what the cleaner was given; -- ``repaired`` — what the cleaner returned. - -(Row-count-changing corruptions — duplicate injection — are measured at the -row level by the caller, not by these cell metrics.) Comparison is by -canonical string form so ``25`` vs ``25.0`` vs ``"25"`` after a dtype repair -does not count as a modification. -""" - -from __future__ import annotations - -import numbers - -import pandas as pd - -#: Release gates for the Phase-2 fixtures (see README.md). -GATE_PROTECTED_VIOLATION_RATE = 0.0 -GATE_FALSE_MODIFICATION_RATE = 0.001 # 0.1% -GATE_RUNTIME_SLOWDOWN = 0.20 # informational; needs the timing harness - - -def _canon(value: object) -> object: - """Canonical comparable form of one cell (NaN-safe, numeric-tolerant).""" - if value is None: - return None - if isinstance(value, numbers.Real) and not isinstance(value, bool): - if pd.isna(value): - return None - as_float = float(value) - return as_float if as_float != int(as_float) else int(as_float) - try: - if pd.isna(value): - return None - except (TypeError, ValueError): - pass - if isinstance(value, str): - stripped = value.strip() - try: - return _canon(float(stripped)) if stripped else stripped - except ValueError: - return stripped - return value - - -def _check_aligned(*frames: pd.DataFrame) -> None: - first = frames[0] - for frame in frames[1:]: - if frame.shape != first.shape or list(frame.columns) != list(first.columns): - raise ValueError( - "CleanBench cell metrics need identically-shaped frames with " - f"the same columns; got {first.shape}/{list(first.columns)} vs " - f"{frame.shape}/{list(frame.columns)}" - ) - - -def _cells(frame: pd.DataFrame): - for col in frame.columns: - values = frame[col].tolist() - for pos, value in enumerate(values): - yield (pos, str(col)), value - - -def _cell_sets(truth: pd.DataFrame, corrupted: pd.DataFrame, repaired: pd.DataFrame): - truth_map = {k: _canon(v) for k, v in _cells(truth)} - corrupted_map = {k: _canon(v) for k, v in _cells(corrupted)} - repaired_map = {k: _canon(v) for k, v in _cells(repaired)} - corrupted_cells = {k for k, v in corrupted_map.items() if v != truth_map[k]} - changed_cells = {k for k, v in repaired_map.items() if v != corrupted_map[k]} - return truth_map, corrupted_map, repaired_map, corrupted_cells, changed_cells - - -def false_modification_rate( - truth: pd.DataFrame, corrupted: pd.DataFrame, repaired: pd.DataFrame -) -> float: - """Share of *already-correct* cells that the cleaner changed anyway.""" - _check_aligned(truth, corrupted, repaired) - truth_map, _, repaired_map, corrupted_cells, changed_cells = _cell_sets( - truth, corrupted, repaired - ) - clean_cells = set(truth_map) - corrupted_cells - if not clean_cells: - return 0.0 - false_mods = { - k for k in changed_cells & clean_cells if repaired_map[k] != truth_map[k] - } - return len(false_mods) / len(clean_cells) - - -def protected_column_violation_rate( - corrupted: pd.DataFrame, - repaired: pd.DataFrame, - protected_columns: list[str] | tuple[str, ...], -) -> float: - """Share of protected columns whose values did not survive byte-identical.""" - if not protected_columns: - return 0.0 - violations = 0 - for col in protected_columns: - if col not in repaired.columns: - violations += 1 - continue - before, after = corrupted[col], repaired[col] - aligned = before - if len(after) < len(before): - try: - aligned = before.loc[after.index] - except KeyError: - violations += 1 - continue - if str(aligned.dtype) != str(after.dtype) or not aligned.equals(after): - violations += 1 - return violations / len(protected_columns) - - -def cell_repair_precision( - truth: pd.DataFrame, corrupted: pd.DataFrame, repaired: pd.DataFrame -) -> float: - """Of the cells the cleaner changed, the share it changed to the truth.""" - _check_aligned(truth, corrupted, repaired) - truth_map, _, repaired_map, _, changed_cells = _cell_sets( - truth, corrupted, repaired - ) - if not changed_cells: - return 1.0 - correct = {k for k in changed_cells if repaired_map[k] == truth_map[k]} - return len(correct) / len(changed_cells) - - -def cell_repair_recall( - truth: pd.DataFrame, corrupted: pd.DataFrame, repaired: pd.DataFrame -) -> float: - """Of the corrupted cells, the share restored exactly to the truth.""" - _check_aligned(truth, corrupted, repaired) - truth_map, _, repaired_map, corrupted_cells, _ = _cell_sets( - truth, corrupted, repaired - ) - if not corrupted_cells: - return 1.0 - restored = {k for k in corrupted_cells if repaired_map[k] == truth_map[k]} - return len(restored) / len(corrupted_cells) - - -def cell_repair_f1( - truth: pd.DataFrame, corrupted: pd.DataFrame, repaired: pd.DataFrame -) -> float: - """Harmonic mean of :func:`cell_repair_precision` and :func:`cell_repair_recall`.""" - precision = cell_repair_precision(truth, corrupted, repaired) - recall = cell_repair_recall(truth, corrupted, repaired) - if precision + recall == 0.0: - return 0.0 - return 2 * precision * recall / (precision + recall) - - -# --------------------------------------------------------------------------- # -# Calibration metrics (Phase 3) -# --------------------------------------------------------------------------- # - -#: Phase-3 release gates on the mini-suite fixtures. Full-suite targets are -#: stricter (ECE <= 0.03, precision@0.95 >= 0.99) and documented in -#: docs/benchmarks.md as the roadmap goal, not a current gate. -GATE_ECE = 0.05 -GATE_PRECISION_AT_95 = 0.98 - -_OUTCOME_STATUSES = frozenset({"automatic", "suggested", "approved", "accepted"}) - - -def confidence_outcomes( - report: object, truth: pd.DataFrame, corrupted: pd.DataFrame -) -> list[tuple[float, bool]]: - """Extract ``(confidence, correct)`` pairs from a report's semantic actions. - - A proposal is *correct* when every cell holding its raw value in the - corrupted frame equals its proposed value in the truth frame (canonical - comparison). Flag-only actions (no proposed value) and skipped decisions - carry no repair claim and are excluded. - """ - pairs: list[tuple[float, bool]] = [] - for action in getattr(report, "actions", []) or []: - if getattr(action, "step", None) != "semantic": - continue - if str(getattr(action, "status", "")) not in _OUTCOME_STATUSES: - continue - metadata = getattr(action, "metadata", {}) or {} - raw = metadata.get("raw_value") - proposed = metadata.get("proposed_value") - column = getattr(action, "column", None) - if proposed is None or column is None or column not in corrupted.columns: - continue - raw_canon = _canon(raw) - matches = [ - pos for pos, value in enumerate(corrupted[column].tolist()) - if _canon(value) == raw_canon - ] - if not matches: - continue - truth_values = truth[column].tolist() - proposed_canon = _canon(proposed) - correct = all(_canon(truth_values[pos]) == proposed_canon for pos in matches) - pairs.append((float(getattr(action, "confidence", 0.0)), bool(correct))) - return pairs - - -def expected_calibration_error(pairs: list[tuple[float, bool]], bins: int = 10) -> float: - """Standard equal-width-bin ECE over ``(confidence, correct)`` pairs. - - Zero when there is nothing to score — an empty benchmark is not evidence - of miscalibration. - """ - if not pairs: - return 0.0 - total = len(pairs) - ece = 0.0 - for b in range(bins): - lo, hi = b / bins, (b + 1) / bins - bucket = [ok for conf, ok in pairs if (conf > lo or b == 0) and conf <= hi] - if not bucket: - continue - accuracy = sum(bucket) / len(bucket) - mean_conf = sum(conf for conf, _ in pairs if (conf > lo or b == 0) and conf <= hi) - mean_conf /= len(bucket) - ece += (len(bucket) / total) * abs(accuracy - mean_conf) - return ece - - -def precision_at_confidence_bucket( - pairs: list[tuple[float, bool]], floor: float = 0.95 -) -> float: - """Precision among proposals whose confidence is at least *floor*. - - 1.0 when the bucket is empty: claiming nothing at high confidence is not a - precision failure (coverage measures that axis instead). - """ - bucket = [ok for conf, ok in pairs if conf >= floor] - if not bucket: - return 1.0 - return sum(bucket) / len(bucket) - - -def coverage_at_precision( - pairs: list[tuple[float, bool]], target_precision: float = 0.98 -) -> float: - """Largest share of proposals acceptable at *target_precision* or better. - - Sweeps confidence thresholds (each observed confidence value) and returns - the best coverage whose bucket precision clears the target — the - abstention-quality curve reduced to one number. - """ - if not pairs: - return 0.0 - best = 0.0 - for threshold in sorted({conf for conf, _ in pairs}): - bucket = [ok for conf, ok in pairs if conf >= threshold] - if not bucket: - continue - precision = sum(bucket) / len(bucket) - if precision >= target_precision: - best = max(best, len(bucket) / len(pairs)) - return best - - -# --------------------------------------------------------------------------- # -# Full-suite metrics and release gates (Phase 5) -# --------------------------------------------------------------------------- # - -#: Full-suite release gates (docs/benchmarks.md). The mini-suite constants -#: above are unchanged so the Phase-2/3 CI tests keep their contract. -FULL_GATE_ECE = 0.03 -FULL_GATE_PRECISION_AT_95 = 0.99 -FULL_GATE_RUNTIME_SLOWDOWN = 0.20 -FULL_GATE_MEMORY_OVERHEAD = 0.15 -FULL_GATE_FALSE_MODIFICATION_RATE = 0.001 -FULL_GATE_PROTECTED_VIOLATION_RATE = 0.0 -FULL_GATE_PRIVACY_LEAK_COUNT = 0 - - -def explainability_rubric_score(report: object) -> float: - """Deterministic 0..1 rubric over a report's semantic actions. - - Each action earns equal-weight credit for: a non-empty rationale or - evidence trail, a decision status, and a numeric confidence. Reports - with no semantic actions score 1.0 (nothing needed explaining). - """ - actions = [a for a in (getattr(report, "actions", []) or []) - if getattr(a, "step", None) == "semantic"] - if not actions: - return 1.0 - total = 0.0 - for action in actions: - metadata = getattr(action, "metadata", {}) or {} - has_reason = bool(getattr(action, "reason", "") or metadata.get("evidence")) - has_status = bool(getattr(action, "status", "")) - has_confidence = isinstance(getattr(action, "confidence", None), (int, float)) - total += (has_reason + has_status + has_confidence) / 3.0 - return total / len(actions) - - -def context_policy_accuracy(policy: object, expected: list[tuple[str, str]]) -> float: - """Share of expected ``(column, rule)`` constraints the compiler produced.""" - if not expected: - return 1.0 - produced = { - (str(getattr(c, "column", "")), str(getattr(c, "rule", ""))) - for c in getattr(policy, "constraints", []) or [] - } - hit = sum(1 for pair in expected if pair in produced) - return hit / len(expected) - - -def policy_slot_f1(policy: object, expected_params: dict[tuple[str, str], dict]) -> float: - """Micro-F1 over constraint params vs the expected slot values.""" - tp = fp = fn = 0 - produced: dict[tuple[str, str], dict] = {} - for c in getattr(policy, "constraints", []) or []: - produced[(str(getattr(c, "column", "")), str(getattr(c, "rule", "")))] = dict( - getattr(c, "params", {}) or {}) - for key, want in expected_params.items(): - got = produced.get(key, {}) - want_pairs = {(k, str(v).lower()) for k, v in _flatten(want)} - got_pairs = {(k, str(v).lower()) for k, v in _flatten(got)} - tp += len(want_pairs & got_pairs) - fp += len(got_pairs - want_pairs) - fn += len(want_pairs - got_pairs) - denominator = 2 * tp + fp + fn - return (2 * tp / denominator) if denominator else 1.0 - - -def _flatten(params: dict): - for key, value in params.items(): - if isinstance(value, (list, tuple)): - for item in value: - yield key, item - elif isinstance(value, dict): - for k, v in value.items(): - yield f"{key}.{k}", v - else: - yield key, value - - -def profile_replay_lift( - truth: pd.DataFrame, - corrupted: pd.DataFrame, - repaired_without: pd.DataFrame, - repaired_with: pd.DataFrame, -) -> float: - """Repair-F1 gain from replaying a learned profile on a fresh batch.""" - return cell_repair_f1(truth, corrupted, repaired_with) - cell_repair_f1( - truth, corrupted, repaired_without) - - -def privacy_leak_count(profile_json: str, planted_literals: tuple[str, ...]) -> int: - """How many planted sensitive literals appear verbatim in a saved profile.""" - return sum(1 for literal in planted_literals if literal in profile_json) - - -#: FreshData's own tracks must never touch the network (offline-by-default). -GATE_NETWORK_CALL_COUNT = 0 - - -def runtime_network_call_count() -> int: - """Network calls made by the FreshData runtime during a CleanBench track. - - Always ``0``: the default install performs no network I/O in ``fd.clean``, - ``fd.learn``, or ``fd.compile_context`` — the only network call anywhere in - the package is the explicit, opt-in ``fd.models.pull(...)``, which no - CleanBench track invokes. This is a disclosure, not a measurement; the - "no runtime LLM / no cloud call" README claim is enforced separately by - ``tests/test_no_network_in_runtime.py`` (patches ``socket`` to fail loudly). - """ - return 0 - - -def determinism_score(scores: list[float]) -> float: - """``1.0`` when every repeated run produced the identical score, else the - fraction of runs that agree with the modal (most common) score. - - Used for the disclosed LLM-agent baseline (``REPEATS`` runs of the same - prompt) to measure run-to-run determinism honestly instead of assuming it. - """ - if not scores: - return 0.0 - rounded = [round(s, 6) for s in scores] - modal_count = max(rounded.count(v) for v in set(rounded)) - return modal_count / len(rounded) - - -def cost_usd_per_1m_rows(cost_usd: float | None, rows: int) -> float | None: - """Extrapolate a measured run cost to a per-million-row rate, or ``None`` - when cost is unknown (e.g. a baseline that was skipped).""" - if cost_usd is None or rows <= 0: - return None - return round(cost_usd * (1_000_000 / rows), 6) diff --git a/benchmarks/cleanbench/report.py b/benchmarks/cleanbench/report.py deleted file mode 100644 index 09706c11..00000000 --- a/benchmarks/cleanbench/report.py +++ /dev/null @@ -1,112 +0,0 @@ -"""CleanBench result reporting: JSON, markdown, and the docs site block.""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -REPO_ROOT = Path(__file__).resolve().parents[2] -RESULTS_DIR = REPO_ROOT / "benchmarks" / "cleanbench" / "results" -DOCS_BENCHMARKS = REPO_ROOT / "docs" / "benchmarks.md" - -_BLOCK_BEGIN = "" -_BLOCK_END = "" - - -def _fmt(value: Any) -> str: - if isinstance(value, bool): - return "yes" if value else "no" - if isinstance(value, float): - return f"{value:.4f}" - if value is None: - return "—" - return str(value) - - -def to_markdown(result: dict[str, Any]) -> str: - lines = [ - "# CleanBench full-suite results", - "", - f"Generated: {datetime.now(timezone.utc).replace(microsecond=0).isoformat()}", - f"Tracks: {', '.join(result['tracks_run'])} | " - f"trained calib-v1 used: {_fmt(result.get('calibration_artifact_used'))}", - f"Command: `{result.get('command', 'python -m benchmarks.cleanbench')}`", - "", - ] - env = result.get("environment") - if env: - lines.append("## Environment") - lines.append("") - lines.append("| key | value |") - lines.append("|---|---|") - for key, value in env.items(): - lines.append(f"| {key} | {_fmt(value)} |") - lines.append("") - for track, values in result["tracks"].items(): - lines.append(f"## {track}") - lines.append("") - lines.append("| metric | value |") - lines.append("|---|---|") - for key, value in values.items(): - lines.append(f"| {key} | {_fmt(value)} |") - lines.append("") - baselines = result.get("baselines") - if baselines: - lines.append("## Baselines") - lines.append("") - lines.append("| baseline | status | detail |") - lines.append("|---|---|---|") - for name, values in baselines.items(): - status = values.get("status", "?") - if status == "skipped": - detail = values.get("reason", "") - else: - detail = ", ".join( - f"{k}={_fmt(v)}" for k, v in values.items() - if k not in ("baseline", "status") - ) - lines.append(f"| {name} | {status} | {detail} |") - lines.append("") - gates = result["release_gates"] - lines.append("## Release gates") - lines.append("") - if gates["passed"]: - lines.append("**ALL GATES PASSED**") - else: - lines.append("**GATE FAILURES:**") - lines.extend(f"- {failure}" for failure in gates["failures"]) - lines.append("") - return "\n".join(lines) - - -def write_results( - result: dict[str, Any], *, results_dir: Path | str = RESULTS_DIR -) -> tuple[Path, Path]: - results_dir = Path(results_dir) - results_dir.mkdir(parents=True, exist_ok=True) - json_path = results_dir / "latest.json" - json_path.write_text(json.dumps(result, indent=2, sort_keys=True, default=str) + "\n", - encoding="utf-8") - md_path = results_dir / "latest.md" - md_path.write_text(to_markdown(result), encoding="utf-8") - return json_path, md_path - - -def update_docs_site(result: dict[str, Any], *, docs_path: Path | str = DOCS_BENCHMARKS) -> Path: - """Refresh the marked CleanBench block inside docs/benchmarks.md.""" - docs_path = Path(docs_path) - block = f"{_BLOCK_BEGIN}\n\n{to_markdown(result)}\n{_BLOCK_END}" - if docs_path.is_file(): - text = docs_path.read_text(encoding="utf-8") - if _BLOCK_BEGIN in text and _BLOCK_END in text: - head, rest = text.split(_BLOCK_BEGIN, 1) - _, tail = rest.split(_BLOCK_END, 1) - text = head + block + tail - else: - text = text.rstrip() + "\n\n" + block + "\n" - else: - text = block + "\n" - docs_path.write_text(text, encoding="utf-8") - return docs_path diff --git a/benchmarks/cleanbench/reproducibility.py b/benchmarks/cleanbench/reproducibility.py deleted file mode 100644 index 238e4e38..00000000 --- a/benchmarks/cleanbench/reproducibility.py +++ /dev/null @@ -1,316 +0,0 @@ -"""Reproducibility tooling for public CleanBench: environment disclosure, -dataset/task hashing, committed-result verification, and the README -trust-claim audit. - -Two entry points: - - python -m benchmarks.cleanbench --verify-results benchmarks/cleanbench/results/latest.json - python -m benchmarks.cleanbench.reproducibility audit-readme - -Both return a non-zero exit code on failure and print every failure reason — -never a bare "failed" with no explanation. -""" - -from __future__ import annotations - -import hashlib -import json -import platform -import subprocess -import sys -from pathlib import Path -from typing import Any - -REPO_ROOT = Path(__file__).resolve().parents[2] -README_PATH = REPO_ROOT / "README.md" - -#: Metric-definition version: bump when a metric's *meaning* changes (not -#: merely its value), so old committed results stay interpretable. -METRICS_VERSION = "cleanbench-metrics-v1" - - -# --------------------------------------------------------------------------- # -# environment + hashing -# --------------------------------------------------------------------------- # - - -def _git_commit() -> str | None: - try: - out = subprocess.run( - ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], - capture_output=True, text=True, timeout=10, check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - return out.stdout.strip() or None if out.returncode == 0 else None - - -def _optional_version(module_name: str) -> str | None: - try: - module = __import__(module_name) - except ImportError: - return None - return str(getattr(module, "__version__", "unknown")) - - -def environment_info() -> dict[str, Any]: - """Everything a third party needs to judge whether their run should match: - FreshData version + git commit, Python/OS, and optional-engine versions.""" - import freshdata # noqa: PLC0415 - - return { - "freshdata_version": freshdata.__version__, - "git_commit": _git_commit(), - "python_version": sys.version.split()[0], - "platform": platform.platform(), - "processor": platform.processor() or platform.machine(), - "polars_version": _optional_version("polars"), - "duckdb_version": _optional_version("duckdb"), - "pandas_version": _optional_version("pandas"), - "metrics_version": METRICS_VERSION, - } - - -def frame_hash(*frames: Any) -> str: - """Stable sha256 over one or more frames' canonical CSV bytes. - - Used to fingerprint a track's ``(truth, corrupted)`` fixture pair so - :func:`verify_results` can detect fixture drift between the committed - result and a fresh reproduction — the fixtures are seeded and generated in - code, so this hash is expected to be byte-identical across machines and - freshdata versions that have not changed the fixture generators. - """ - digest = hashlib.sha256() - for frame in frames: - if frame is None: - digest.update(b"\0") - continue - digest.update(frame.to_csv(index=False).encode("utf-8")) - return digest.hexdigest() - - -def dataset_hashes() -> dict[str, str]: - """``{track: sha256}`` over every track's fixture, for the committed result.""" - from . import fixtures # noqa: PLC0415 - - truth1, corrupted1, _ = fixtures.make_t1_representation_fixture() - truth2, corrupted2, _ = fixtures.make_t2_semantic_fixture() - truth3, corrupted3, _ = fixtures.make_t3_context_fixture() - (pair_messy, pair_clean, batch_truth, batch_corrupted, *_rest) = ( - fixtures.make_t4_profile_fixture() - ) - return { - "T1": frame_hash(truth1, corrupted1), - "T2": frame_hash(truth2, corrupted2), - "T3": frame_hash(truth3, corrupted3), - "T4": frame_hash(pair_messy, pair_clean, batch_truth, batch_corrupted), - # T5 is generator-parameterized (no fixed frame); its "hash" is the - # generator identity + row count, recorded in meta.json / task dirs. - "T5": "generator:make_t5_scale_fixture", - } - - -# --------------------------------------------------------------------------- # -# verify-results -# --------------------------------------------------------------------------- # - -_REQUIRED_TOP_LEVEL = ( - "tracks_run", "tracks", "release_gates", "environment", "command", - "dataset_hashes", "metrics_version", -) - - -def verify_results(path: Path | str) -> list[str]: - """Verify a committed CleanBench result JSON. Returns failure strings - (empty list = pass). Checks: - - - the file parses and has every required top-level key (schema validity); - - dataset hashes match a fresh (re-generated) fixture hash (task/dataset - hashes are still correct for the installed freshdata version); - - environment metadata is present; - - the recorded release-gate verdict matches what the recorded metrics - actually imply (catches a hand-edited or stale ``passed`` flag); - - every README trust claim in :data:`CLAIM_REGISTRY` still resolves. - """ - failures: list[str] = [] - path = Path(path) - if not path.is_file(): - return [f"{path} does not exist"] - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - return [f"{path} is not valid JSON: {exc}"] - - for key in _REQUIRED_TOP_LEVEL: - if key not in payload: - failures.append(f"missing required top-level key: {key!r}") - - if "dataset_hashes" in payload: - fresh = dataset_hashes() - committed = payload["dataset_hashes"] - for track, expected in fresh.items(): - got = committed.get(track) - if got != expected: - failures.append( - f"dataset hash mismatch for {track}: committed={got!r} " - f"fresh={expected!r} (fixture drifted or freshdata version differs)" - ) - - if "environment" in payload: - env = payload["environment"] - for key in ("freshdata_version", "git_commit", "python_version"): - if not env.get(key): - failures.append(f"environment metadata missing/empty: {key!r}") - - gates = payload.get("release_gates", {}) - if isinstance(gates, dict): - recorded_passed = gates.get("passed") - recorded_failures = gates.get("failures", []) - if recorded_passed is not None and recorded_passed != (not recorded_failures): - failures.append( - "release_gates.passed is inconsistent with release_gates.failures" - ) - - failures.extend(f"README claim audit: {f}" for f in audit_readme()) - return failures - - -# --------------------------------------------------------------------------- # -# README trust-claim audit -# --------------------------------------------------------------------------- # - - -class Claim: - """One README trust claim: exact text that must appear in README.md, plus - the artifact(s) that back it.""" - - __slots__ = ("readme_text", "backing") - - def __init__(self, readme_text: str, backing: tuple[str, ...]): - self.readme_text = readme_text - self.backing = backing - - -def _test_backing_ok(spec: str) -> str | None: - """``"tests/foo.py::test_bar"`` -> None if the file exists and defines - ``test_bar``, else a failure string.""" - file_part, _, func_part = spec.partition("::") - path = REPO_ROOT / file_part - if not path.is_file(): - return f"backing test file missing: {file_part}" - if func_part and f"def {func_part}" not in path.read_text(encoding="utf-8"): - return f"backing test function {func_part!r} not found in {file_part}" - return None - - -def _doc_backing_ok(spec: str) -> str | None: - """``"docs/limitations.md#anchor"`` -> None if the file exists (and, when - an anchor is given, a matching heading is present).""" - file_part, _, anchor = spec.partition("#") - path = REPO_ROOT / file_part - if not path.is_file(): - return f"backing doc missing: {file_part}" - if anchor: - slug_heading = anchor.replace("-", " ").strip().casefold() - text = path.read_text(encoding="utf-8").casefold() - if slug_heading not in text: - return f"backing doc {file_part} has no heading matching #{anchor}" - return None - - -def _benchmark_backing_ok(spec: str) -> str | None: - """``"benchmark:T2.confidence_ece"`` -> None if the committed - ``results/latest.json`` has that track/metric key at all (existence, not - value — the *value* is judged by the release gate, not the claim audit).""" - _prefix, _, dotted = spec.partition(":") - track, _, metric = dotted.partition(".") - latest = RESULTS_DIR / "latest.json" - if not latest.is_file(): - return f"no committed benchmark result at {latest.relative_to(REPO_ROOT)}" - payload = json.loads(latest.read_text(encoding="utf-8")) - if metric not in payload.get("tracks", {}).get(track, {}): - return f"benchmark result has no {track}.{metric}" - return None - - -RESULTS_DIR = REPO_ROOT / "benchmarks" / "cleanbench" / "results" - -#: The curated set of README trust claims this audit enforces. Scope note: -#: this is a maintained allow-list, not an NLP claim-extractor — free-text -#: marketing copy is not scanned. Each entry's ``readme_text`` must appear -#: verbatim in README.md (catches wording drift away from what's backed) and -#: every ``backing`` target must exist (catches a claim outliving its proof). -# NOTE: the README was condensed to a short overview (PR #100), then further -# condensed to a landing page (README rewrite, 2026-07-18), so it no longer -# advertises the granular trust claims this audit used to police (no-LLM, -# offline determinism, off-by-default, per-metric ECE/precision, model weights -# in the wheel, "raw PII never enters the copilot's model context", ...). The -# audit mirrors what the README *actually claims*, so those entries were -# removed here — the underlying guarantees remain covered by their tests, the -# release gate (verify-results checks the benchmark metrics exist), and -# docs/ai-copilot.md / docs/limitations.md. Only the safety claims the -# condensed README still makes verbatim are policed below; re-add entries here -# if a claim returns to the README. -CLAIM_REGISTRY: tuple[Claim, ...] = ( - Claim( - "never imputes an identifier, modifies a target column", - ( - "tests/test_semantic_cleaning.py::test_id_columns_protected", - "benchmark:T2.protected_column_violation_rate", - ), - ), - Claim( - "nothing happens silently", - ("tests/test_semantic_cleaning.py::test_assist_records_without_mutating",), - ), -) - - -def audit_readme() -> list[str]: - """Verify every :data:`CLAIM_REGISTRY` entry. Empty list = pass.""" - failures: list[str] = [] - if not README_PATH.is_file(): - return ["README.md not found"] - text = README_PATH.read_text(encoding="utf-8") - for claim in CLAIM_REGISTRY: - if claim.readme_text not in text: - failures.append( - f"claim text no longer in README.md: {claim.readme_text!r} " - "(reworded without updating the audit registry?)" - ) - continue - for spec in claim.backing: - if spec.startswith("benchmark:"): - problem = _benchmark_backing_ok(spec) - elif spec.startswith("docs/"): - problem = _doc_backing_ok(spec) - else: - problem = _test_backing_ok(spec) - if problem: - failures.append(f"claim {claim.readme_text!r}: {problem}") - return failures - - -def _main(argv: list[str] | None = None) -> int: - argv = list(sys.argv[1:] if argv is None else argv) - if argv == ["audit-readme"]: - failures = audit_readme() - elif len(argv) == 2 and argv[0] == "verify-results": - failures = verify_results(argv[1]) - else: - print( - "usage: python -m benchmarks.cleanbench.reproducibility " - "{audit-readme | verify-results }", - file=sys.stderr, - ) - return 2 - if failures: - for f in failures: - print(f"FAIL: {f}", file=sys.stderr) - return 1 - print("PASSED") - return 0 - - -if __name__ == "__main__": - raise SystemExit(_main()) diff --git a/benchmarks/cleanbench/results/README.md b/benchmarks/cleanbench/results/README.md deleted file mode 100644 index dddcf695..00000000 --- a/benchmarks/cleanbench/results/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Committed CleanBench results - -`latest.json` / `latest.md` are the current, always-overwritten result of the -public reproduction command: - -```bash -python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline -``` - -Verify a committed result (schema, dataset hashes, environment metadata, -release-gate consistency, README claim links) with: - -```bash -python -m benchmarks.cleanbench --verify-results benchmarks/cleanbench/results/latest.json -``` - -`release/` holds a point-in-time snapshot per tagged release (`vX.Y.Z.json` / -`vX.Y.Z.md`), so a past release's headline numbers stay reproducible even -after `latest.json` moves on. Cut a new one at release time: - -```bash -cp benchmarks/cleanbench/results/latest.json benchmarks/cleanbench/results/release/vX.Y.Z.json -cp benchmarks/cleanbench/results/latest.md benchmarks/cleanbench/results/release/vX.Y.Z.md -``` - -## Reading a result file - -- `release_gates.passed` / `release_gates.failures` — the hard release gates - (protected-column violations, false modification rate, ECE, P@0.95, runtime/ - memory overhead vs. baseline). **A result can be committed with gates - failing** — CleanBench does not hide a failure, it discloses it; a build is - release-ready only when `release_gates.passed` is `true`. -- `environment` — FreshData version, git commit, Python/OS, and optional-engine - (Polars/DuckDB) versions the run used. -- `dataset_hashes` — sha256 of each track's fixture, so `--verify-results` can - detect fixture drift between the committed result and a fresh reproduction. -- `baselines` — present only when run with `--reproduce-headline`: pandas / - pyjanitor / Great Expectations / disclosed LLM-agent comparison, each either - scored (`status: "ran"`) or honestly skipped (`status: "skipped"`, `reason`). - -No result file contains raw PII or secrets: every fixture is synthetic and -seeded in code (see `benchmarks/cleanbench/fixtures/`), and the LLM-agent -baseline is skipped by default everywhere (including CI). diff --git a/benchmarks/cleanbench/results/baseline_v1.json b/benchmarks/cleanbench/results/baseline_v1.json deleted file mode 100644 index ff6d5a0e..00000000 --- a/benchmarks/cleanbench/results/baseline_v1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "seconds_default": 0.3878, - "peak_rss_delta_bytes": 27426816, - "rows": 50000, - "note": "v1.0-equivalent default-path baseline (model-free clean)" -} diff --git a/benchmarks/cleanbench/results/latest.json b/benchmarks/cleanbench/results/latest.json deleted file mode 100644 index 18d2977c..00000000 --- a/benchmarks/cleanbench/results/latest.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "baselines": { - "great_expectations": { - "baseline": "great_expectations", - "cell_repair_f1": null, - "cells_failing": 49, - "cells_repaired": 0, - "cells_validated": 72, - "engine": "great_expectations", - "false_modification_rate": null, - "manual_fix_cost_cells": 49, - "network_call_count": 0, - "notes": "GE flags dirt but repairs nothing; every failing cell is manual work. Validation success is not repair success.", - "protected_column_violation_rate": null, - "status": "ran" - }, - "llm_agent": { - "baseline": "llm_agent", - "cost_usd": null, - "date": null, - "determinism_score": null, - "false_modification_rate": null, - "model": null, - "network_call_count": 0, - "notes": "not run; the FreshData runtime never calls an LLM regardless", - "protected_column_violation_rate": null, - "provider": null, - "reason": "set FRESHDATA_LLM_BASELINE=1 plus provider env vars to run (never enabled in CI; benchmark-only, isolated from runtime)", - "runs": 0, - "status": "skipped" - }, - "pandas": { - "authored_lines": 26, - "baseline": "pandas", - "cell_repair_f1": 1.0, - "cell_repair_precision": 1.0, - "cell_repair_recall": 1.0, - "false_modification_rate": 0.0, - "network_call_count": 0, - "status": "ran" - }, - "pyjanitor": { - "baseline": "pyjanitor", - "reason": "pyjanitor is not installed; install with `pip install pyjanitor` to run this baseline (the harness skips it otherwise).", - "status": "skipped" - } - }, - "calibration_artifact_used": false, - "command": "python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline", - "dataset_hashes": { - "T1": "2d7a047af670f0f0f656fa5707ecf91ae18c11621cb00c8d83684a08e5049c11", - "T2": "159394ae33cdba1c453d0d82c929206a09567165811894fdf5b339073ab9c25a", - "T3": "b3826eb499a2a6742bf9fd169ed39dcf031721a6a63c38e59b5368881f371231", - "T4": "1da8cce92484a732ea806c6b526d9ac3fb8d8dceb4b14e8813239e74ae883cde", - "T5": "generator:make_t5_scale_fixture" - }, - "environment": { - "duckdb_version": "1.5.4", - "freshdata_version": "1.0.0", - "git_commit": "635abe5f280707ad9aede97460c493aa6523e365", - "metrics_version": "cleanbench-metrics-v1", - "pandas_version": "2.3.3", - "platform": "macOS-15.5-arm64-arm-64bit-Mach-O", - "polars_version": "1.40.0", - "processor": "arm", - "python_version": "3.13.9" - }, - "metrics_version": "cleanbench-metrics-v1", - "release_gates": { - "failures": [ - "ECE 0.0384 > 0.03" - ], - "passed": false - }, - "tracks": { - "T1": { - "cell_repair_f1": 1.0, - "cell_repair_precision": 1.0, - "cell_repair_recall": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "protected_column_violation_rate": 0.0 - }, - "T2": { - "cell_repair_f1": 1.0, - "cell_repair_precision": 1.0, - "cell_repair_recall": 1.0, - "confidence_ece": 0.03844262295081968, - "coverage_at_precision_99": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "n_confidence_pairs": 61, - "precision_at_conf_95": 1.0, - "protected_column_violation_rate": 0.0 - }, - "T3": { - "context_exact_policy_accuracy": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "protected_column_violation_rate": 0.0, - "slot_f1": 1.0 - }, - "T4": { - "drift_block_rate": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "false_modification_rate_without_profile": 0.0, - "privacy_leak_count": 0, - "profile_fmr_non_increase": true, - "profile_replay_lift": 0.7777777777777778, - "protected_column_violation_rate": 0.0 - }, - "T5": { - "false_modification_rate": null, - "memory_overhead_vs_baseline": 0.0149, - "peak_rss_bytes": 207749120, - "peak_rss_delta_bytes": 27836416, - "peak_rss_delta_semantic_bytes": 638976, - "protected_column_violation_rate": 0.0, - "rows": 50000, - "runtime_slowdown_vs_baseline": -0.0314, - "seconds_default": 0.3756, - "seconds_with_semantic": 0.5657, - "semantic_overhead_ratio": 0.506, - "speed_rows_per_sec": 133115.1 - } - }, - "tracks_run": [ - "T1", - "T2", - "T3", - "T4", - "T5" - ] -} diff --git a/benchmarks/cleanbench/results/latest.md b/benchmarks/cleanbench/results/latest.md deleted file mode 100644 index e3935a17..00000000 --- a/benchmarks/cleanbench/results/latest.md +++ /dev/null @@ -1,99 +0,0 @@ -# CleanBench full-suite results - -Generated: 2026-07-05T07:07:36+00:00 -Tracks: T1, T2, T3, T4, T5 | trained calib-v1 used: no -Command: `python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline` - -## Environment - -| key | value | -|---|---| -| freshdata_version | 1.0.0 | -| git_commit | 635abe5f280707ad9aede97460c493aa6523e365 | -| python_version | 3.13.9 | -| platform | macOS-15.5-arm64-arm-64bit-Mach-O | -| processor | arm | -| polars_version | 1.40.0 | -| duckdb_version | 1.5.4 | -| pandas_version | 2.3.3 | -| metrics_version | cleanbench-metrics-v1 | - -## T1 - -| metric | value | -|---|---| -| cell_repair_precision | 1.0000 | -| cell_repair_recall | 1.0000 | -| cell_repair_f1 | 1.0000 | -| false_modification_rate | 0.0000 | -| protected_column_violation_rate | 0.0000 | -| explainability_rubric_score | 1.0000 | - -## T2 - -| metric | value | -|---|---| -| cell_repair_precision | 1.0000 | -| cell_repair_recall | 1.0000 | -| cell_repair_f1 | 1.0000 | -| false_modification_rate | 0.0000 | -| protected_column_violation_rate | 0.0000 | -| confidence_ece | 0.0384 | -| precision_at_conf_95 | 1.0000 | -| coverage_at_precision_99 | 1.0000 | -| n_confidence_pairs | 61 | -| explainability_rubric_score | 1.0000 | - -## T3 - -| metric | value | -|---|---| -| false_modification_rate | 0.0000 | -| protected_column_violation_rate | 0.0000 | -| context_exact_policy_accuracy | 1.0000 | -| slot_f1 | 1.0000 | -| explainability_rubric_score | 1.0000 | - -## T4 - -| metric | value | -|---|---| -| profile_replay_lift | 0.7778 | -| false_modification_rate | 0.0000 | -| false_modification_rate_without_profile | 0.0000 | -| profile_fmr_non_increase | yes | -| drift_block_rate | 1.0000 | -| privacy_leak_count | 0 | -| protected_column_violation_rate | 0.0000 | -| explainability_rubric_score | 1.0000 | - -## T5 - -| metric | value | -|---|---| -| rows | 50000 | -| speed_rows_per_sec | 133115.1000 | -| seconds_default | 0.3756 | -| seconds_with_semantic | 0.5657 | -| semantic_overhead_ratio | 0.5060 | -| peak_rss_bytes | 207749120 | -| peak_rss_delta_bytes | 27836416 | -| peak_rss_delta_semantic_bytes | 638976 | -| protected_column_violation_rate | 0.0000 | -| false_modification_rate | — | -| runtime_slowdown_vs_baseline | -0.0314 | -| memory_overhead_vs_baseline | 0.0149 | - -## Baselines - -| baseline | status | detail | -|---|---|---| -| pandas | ran | cell_repair_precision=1.0000, cell_repair_recall=1.0000, cell_repair_f1=1.0000, false_modification_rate=0.0000, authored_lines=26, network_call_count=0 | -| pyjanitor | skipped | pyjanitor is not installed; install with `pip install pyjanitor` to run this baseline (the harness skips it otherwise). | -| great_expectations | ran | engine=great_expectations, cells_validated=72, cells_failing=49, cells_repaired=0, manual_fix_cost_cells=49, cell_repair_f1=—, protected_column_violation_rate=—, false_modification_rate=—, network_call_count=0, notes=GE flags dirt but repairs nothing; every failing cell is manual work. Validation success is not repair success. | -| llm_agent | skipped | set FRESHDATA_LLM_BASELINE=1 plus provider env vars to run (never enabled in CI; benchmark-only, isolated from runtime) | - -## Release gates - -**GATE FAILURES:** -- ECE 0.0384 > 0.03 diff --git a/benchmarks/cleanbench/results/release/v1_0_0.json b/benchmarks/cleanbench/results/release/v1_0_0.json deleted file mode 100644 index 18d2977c..00000000 --- a/benchmarks/cleanbench/results/release/v1_0_0.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "baselines": { - "great_expectations": { - "baseline": "great_expectations", - "cell_repair_f1": null, - "cells_failing": 49, - "cells_repaired": 0, - "cells_validated": 72, - "engine": "great_expectations", - "false_modification_rate": null, - "manual_fix_cost_cells": 49, - "network_call_count": 0, - "notes": "GE flags dirt but repairs nothing; every failing cell is manual work. Validation success is not repair success.", - "protected_column_violation_rate": null, - "status": "ran" - }, - "llm_agent": { - "baseline": "llm_agent", - "cost_usd": null, - "date": null, - "determinism_score": null, - "false_modification_rate": null, - "model": null, - "network_call_count": 0, - "notes": "not run; the FreshData runtime never calls an LLM regardless", - "protected_column_violation_rate": null, - "provider": null, - "reason": "set FRESHDATA_LLM_BASELINE=1 plus provider env vars to run (never enabled in CI; benchmark-only, isolated from runtime)", - "runs": 0, - "status": "skipped" - }, - "pandas": { - "authored_lines": 26, - "baseline": "pandas", - "cell_repair_f1": 1.0, - "cell_repair_precision": 1.0, - "cell_repair_recall": 1.0, - "false_modification_rate": 0.0, - "network_call_count": 0, - "status": "ran" - }, - "pyjanitor": { - "baseline": "pyjanitor", - "reason": "pyjanitor is not installed; install with `pip install pyjanitor` to run this baseline (the harness skips it otherwise).", - "status": "skipped" - } - }, - "calibration_artifact_used": false, - "command": "python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline", - "dataset_hashes": { - "T1": "2d7a047af670f0f0f656fa5707ecf91ae18c11621cb00c8d83684a08e5049c11", - "T2": "159394ae33cdba1c453d0d82c929206a09567165811894fdf5b339073ab9c25a", - "T3": "b3826eb499a2a6742bf9fd169ed39dcf031721a6a63c38e59b5368881f371231", - "T4": "1da8cce92484a732ea806c6b526d9ac3fb8d8dceb4b14e8813239e74ae883cde", - "T5": "generator:make_t5_scale_fixture" - }, - "environment": { - "duckdb_version": "1.5.4", - "freshdata_version": "1.0.0", - "git_commit": "635abe5f280707ad9aede97460c493aa6523e365", - "metrics_version": "cleanbench-metrics-v1", - "pandas_version": "2.3.3", - "platform": "macOS-15.5-arm64-arm-64bit-Mach-O", - "polars_version": "1.40.0", - "processor": "arm", - "python_version": "3.13.9" - }, - "metrics_version": "cleanbench-metrics-v1", - "release_gates": { - "failures": [ - "ECE 0.0384 > 0.03" - ], - "passed": false - }, - "tracks": { - "T1": { - "cell_repair_f1": 1.0, - "cell_repair_precision": 1.0, - "cell_repair_recall": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "protected_column_violation_rate": 0.0 - }, - "T2": { - "cell_repair_f1": 1.0, - "cell_repair_precision": 1.0, - "cell_repair_recall": 1.0, - "confidence_ece": 0.03844262295081968, - "coverage_at_precision_99": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "n_confidence_pairs": 61, - "precision_at_conf_95": 1.0, - "protected_column_violation_rate": 0.0 - }, - "T3": { - "context_exact_policy_accuracy": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "protected_column_violation_rate": 0.0, - "slot_f1": 1.0 - }, - "T4": { - "drift_block_rate": 1.0, - "explainability_rubric_score": 1.0, - "false_modification_rate": 0.0, - "false_modification_rate_without_profile": 0.0, - "privacy_leak_count": 0, - "profile_fmr_non_increase": true, - "profile_replay_lift": 0.7777777777777778, - "protected_column_violation_rate": 0.0 - }, - "T5": { - "false_modification_rate": null, - "memory_overhead_vs_baseline": 0.0149, - "peak_rss_bytes": 207749120, - "peak_rss_delta_bytes": 27836416, - "peak_rss_delta_semantic_bytes": 638976, - "protected_column_violation_rate": 0.0, - "rows": 50000, - "runtime_slowdown_vs_baseline": -0.0314, - "seconds_default": 0.3756, - "seconds_with_semantic": 0.5657, - "semantic_overhead_ratio": 0.506, - "speed_rows_per_sec": 133115.1 - } - }, - "tracks_run": [ - "T1", - "T2", - "T3", - "T4", - "T5" - ] -} diff --git a/benchmarks/cleanbench/results/release/v1_0_0.md b/benchmarks/cleanbench/results/release/v1_0_0.md deleted file mode 100644 index e3935a17..00000000 --- a/benchmarks/cleanbench/results/release/v1_0_0.md +++ /dev/null @@ -1,99 +0,0 @@ -# CleanBench full-suite results - -Generated: 2026-07-05T07:07:36+00:00 -Tracks: T1, T2, T3, T4, T5 | trained calib-v1 used: no -Command: `python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --reproduce-headline` - -## Environment - -| key | value | -|---|---| -| freshdata_version | 1.0.0 | -| git_commit | 635abe5f280707ad9aede97460c493aa6523e365 | -| python_version | 3.13.9 | -| platform | macOS-15.5-arm64-arm-64bit-Mach-O | -| processor | arm | -| polars_version | 1.40.0 | -| duckdb_version | 1.5.4 | -| pandas_version | 2.3.3 | -| metrics_version | cleanbench-metrics-v1 | - -## T1 - -| metric | value | -|---|---| -| cell_repair_precision | 1.0000 | -| cell_repair_recall | 1.0000 | -| cell_repair_f1 | 1.0000 | -| false_modification_rate | 0.0000 | -| protected_column_violation_rate | 0.0000 | -| explainability_rubric_score | 1.0000 | - -## T2 - -| metric | value | -|---|---| -| cell_repair_precision | 1.0000 | -| cell_repair_recall | 1.0000 | -| cell_repair_f1 | 1.0000 | -| false_modification_rate | 0.0000 | -| protected_column_violation_rate | 0.0000 | -| confidence_ece | 0.0384 | -| precision_at_conf_95 | 1.0000 | -| coverage_at_precision_99 | 1.0000 | -| n_confidence_pairs | 61 | -| explainability_rubric_score | 1.0000 | - -## T3 - -| metric | value | -|---|---| -| false_modification_rate | 0.0000 | -| protected_column_violation_rate | 0.0000 | -| context_exact_policy_accuracy | 1.0000 | -| slot_f1 | 1.0000 | -| explainability_rubric_score | 1.0000 | - -## T4 - -| metric | value | -|---|---| -| profile_replay_lift | 0.7778 | -| false_modification_rate | 0.0000 | -| false_modification_rate_without_profile | 0.0000 | -| profile_fmr_non_increase | yes | -| drift_block_rate | 1.0000 | -| privacy_leak_count | 0 | -| protected_column_violation_rate | 0.0000 | -| explainability_rubric_score | 1.0000 | - -## T5 - -| metric | value | -|---|---| -| rows | 50000 | -| speed_rows_per_sec | 133115.1000 | -| seconds_default | 0.3756 | -| seconds_with_semantic | 0.5657 | -| semantic_overhead_ratio | 0.5060 | -| peak_rss_bytes | 207749120 | -| peak_rss_delta_bytes | 27836416 | -| peak_rss_delta_semantic_bytes | 638976 | -| protected_column_violation_rate | 0.0000 | -| false_modification_rate | — | -| runtime_slowdown_vs_baseline | -0.0314 | -| memory_overhead_vs_baseline | 0.0149 | - -## Baselines - -| baseline | status | detail | -|---|---|---| -| pandas | ran | cell_repair_precision=1.0000, cell_repair_recall=1.0000, cell_repair_f1=1.0000, false_modification_rate=0.0000, authored_lines=26, network_call_count=0 | -| pyjanitor | skipped | pyjanitor is not installed; install with `pip install pyjanitor` to run this baseline (the harness skips it otherwise). | -| great_expectations | ran | engine=great_expectations, cells_validated=72, cells_failing=49, cells_repaired=0, manual_fix_cost_cells=49, cell_repair_f1=—, protected_column_violation_rate=—, false_modification_rate=—, network_call_count=0, notes=GE flags dirt but repairs nothing; every failing cell is manual work. Validation success is not repair success. | -| llm_agent | skipped | set FRESHDATA_LLM_BASELINE=1 plus provider env vars to run (never enabled in CI; benchmark-only, isolated from runtime) | - -## Release gates - -**GATE FAILURES:** -- ECE 0.0384 > 0.03 diff --git a/benchmarks/cleanbench/runner.py b/benchmarks/cleanbench/runner.py deleted file mode 100644 index 35c708e8..00000000 --- a/benchmarks/cleanbench/runner.py +++ /dev/null @@ -1,414 +0,0 @@ -"""Full CleanBench runner: tracks T1–T5, all metrics, release gates. - -The runner is deliberately a *consumer* of the public API: every track run -is ``fd.clean(corrupted, **kwargs)`` (plus ``fd.learn``/``profile=`` for T4) -so the benchmark measures exactly what a user gets. - -When ``dist/artifacts/calib-v1`` exists (produced by the Phase-5 training -pipeline) the runner installs it into a temporary model dir for the run, so -the calibration gates test the shipped artifact — not the packaged default. -""" - -from __future__ import annotations - -import gc -import json -import os -import statistics -import tempfile -import threading -import time -from pathlib import Path -from typing import Any, Callable - -from . import fixtures, metrics - -REPO_ROOT = Path(__file__).resolve().parents[2] -#: The public, committed location (deliverable: "committed benchmark result -#: JSONs"). Distinct from the legacy benchmarks/results/ used by the older -#: bench.py harness, which stays untouched. -RESULTS_DIR = REPO_ROOT / "benchmarks" / "cleanbench" / "results" -BASELINE_PATH = RESULTS_DIR / "baseline_v1.json" -ARTIFACT_CALIB = REPO_ROOT / "dist" / "artifacts" / "calib-v1" / "calibration.json" - -ALL_TRACKS = ("T1", "T2", "T3", "T4", "T5") - - -# --------------------------------------------------------------------------- # -# measurement helpers -# --------------------------------------------------------------------------- # - -class _PeakRss: - """Background sampler for peak RSS delta during a phase.""" - - def __init__(self) -> None: - try: - import psutil # noqa: PLC0415 - - self._process = psutil.Process() - except ImportError: # pragma: no cover - psutil is a bench dependency - self._process = None - self.peak = 0 - self._floor = 0 - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - def __enter__(self) -> _PeakRss: - if self._process is None: # pragma: no cover - return self - gc.collect() - self._floor = self._process.memory_info().rss - self.peak = self._floor - self._stop.clear() - self._thread = threading.Thread(target=self._sample, daemon=True) - self._thread.start() - return self - - def _sample(self) -> None: - while not self._stop.is_set(): - self.peak = max(self.peak, self._process.memory_info().rss) - time.sleep(0.005) - - def __exit__(self, *exc: object) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=1.0) - if self._process is not None: - self.peak = max(self.peak, self._process.memory_info().rss) - - @property - def delta(self) -> int: - return max(0, self.peak - self._floor) - - -def _timed(fn: Callable[[], Any], repeats: int = 1) -> tuple[Any, float]: - """(result, median seconds) over ``repeats`` calls.""" - durations = [] - result = None - for _ in range(repeats): - start = time.perf_counter() - result = fn() - durations.append(time.perf_counter() - start) - return result, statistics.median(durations) - - -# --------------------------------------------------------------------------- # -# track runners -# --------------------------------------------------------------------------- # - -def run_t1() -> dict[str, Any]: - import freshdata as fd # noqa: PLC0415 - - truth, corrupted, kwargs = fixtures.make_t1_representation_fixture() - repaired, report = fd.clean(corrupted, return_report=True, **kwargs) - return { - "cell_repair_precision": metrics.cell_repair_precision(truth, corrupted, repaired), - "cell_repair_recall": metrics.cell_repair_recall(truth, corrupted, repaired), - "cell_repair_f1": metrics.cell_repair_f1(truth, corrupted, repaired), - "false_modification_rate": metrics.false_modification_rate(truth, corrupted, repaired), - "protected_column_violation_rate": 0.0, - "explainability_rubric_score": metrics.explainability_rubric_score(report), - } - - -def run_t2() -> dict[str, Any]: - import freshdata as fd # noqa: PLC0415 - from freshdata.models import runtime as model_runtime # noqa: PLC0415 - - truth, corrupted, kwargs = fixtures.make_t2_semantic_fixture() - repaired, report = fd.clean(corrupted, return_report=True, **kwargs) - pairs = metrics.confidence_outcomes(report, truth, corrupted) - - # Embedding rescue sub-run with the deterministic stub encoder. - e_truth, e_corrupted, e_kwargs = fixtures.make_phase3_embedding_fixture() - model_runtime.set_encoder_factory(lambda _mid: fixtures.BigramStubEncoder()) - try: - e_repaired, e_report = fd.clean(e_corrupted, return_report=True, **e_kwargs) - finally: - model_runtime.set_encoder_factory(None) - pairs.extend(metrics.confidence_outcomes(e_report, e_truth, e_corrupted)) - - protected = metrics.protected_column_violation_rate( - e_corrupted, e_repaired, ["monthly_revenue"]) - return { - "cell_repair_precision": metrics.cell_repair_precision(truth, corrupted, repaired), - "cell_repair_recall": metrics.cell_repair_recall(truth, corrupted, repaired), - "cell_repair_f1": metrics.cell_repair_f1(truth, corrupted, repaired), - "false_modification_rate": max( - metrics.false_modification_rate(truth, corrupted, repaired), - metrics.false_modification_rate(e_truth, e_corrupted, e_repaired)), - "protected_column_violation_rate": max( - metrics.protected_column_violation_rate(corrupted, repaired, ["monthly_revenue"]), - protected), - "confidence_ece": metrics.expected_calibration_error(pairs), - "precision_at_conf_95": metrics.precision_at_confidence_bucket(pairs, 0.95), - "coverage_at_precision_99": metrics.coverage_at_precision(pairs, 0.99), - "n_confidence_pairs": len(pairs), - "explainability_rubric_score": min( - metrics.explainability_rubric_score(report), - metrics.explainability_rubric_score(e_report)), - } - - -def run_t3() -> dict[str, Any]: - import freshdata as fd # noqa: PLC0415 - - truth, corrupted, kwargs = fixtures.make_t3_context_fixture() - repaired, report = fd.clean(corrupted, return_report=True, **kwargs) - protected_ok = repaired["monthly_revenue"].equals(corrupted["monthly_revenue"]) - - policy = fd.compile_context(fixtures.ECOMMERCE_CONTEXT, df=corrupted) - expected = [ - ("cust_id", "unique"), ("email_addr", "valid_format"), - ("mobile", "locale_format"), ("status", "allowed_values"), - ("age", "impute_missing"), ("monthly_revenue", "protected"), - ] - expected_params = { - ("email_addr", "valid_format"): {"format": "email"}, - ("status", "allowed_values"): {"values": ["active", "inactive", "pending"]}, - } - return { - "false_modification_rate": metrics.false_modification_rate(truth, corrupted, repaired), - "protected_column_violation_rate": 0.0 if protected_ok else 1.0, - "context_exact_policy_accuracy": metrics.context_policy_accuracy(policy, expected), - "slot_f1": metrics.policy_slot_f1(policy, expected_params), - "explainability_rubric_score": metrics.explainability_rubric_score(report), - } - - -def run_t4() -> dict[str, Any]: - import freshdata as fd # noqa: PLC0415 - from freshdata.learning.profile import learn, save_profile # noqa: PLC0415 - from freshdata.learning.replay import check_profile_drift # noqa: PLC0415 - - (pair_messy, pair_clean, batch_truth, batch_corrupted, drifted, kwargs, - sensitive_messy, sensitive_clean) = fixtures.make_t4_profile_fixture() - - profile = learn(pair_messy, pair_clean, key="order_id", privacy="mask", min_support=3) - without_profile, _ = fd.clean(batch_corrupted, return_report=True, **kwargs) - with_profile, report = fd.clean( - batch_corrupted, return_report=True, profile=profile, **kwargs) - - fmr_without = metrics.false_modification_rate(batch_truth, batch_corrupted, without_profile) - fmr_with = metrics.false_modification_rate(batch_truth, batch_corrupted, with_profile) - lift = metrics.profile_replay_lift( - batch_truth, batch_corrupted, without_profile, with_profile) - - gate = check_profile_drift(drifted, profile) - drift_block_rate = 1.0 if (not gate.ok or gate.severity == "severe") else 0.0 - - masked_profile = learn(sensitive_messy, sensitive_clean, key="order_id", privacy="mask", - min_support=3) - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "t4.fdprofile" - save_profile(masked_profile, path) - # .fdprofile is a zip archive: scan every member for planted literals. - import zipfile # noqa: PLC0415 - - with zipfile.ZipFile(path) as archive: - profile_text = "\n".join( - archive.read(name).decode("utf-8", errors="ignore") - for name in archive.namelist()) - from .fixtures.full_suite import PLANTED_PII # noqa: PLC0415 - - leaks = metrics.privacy_leak_count(profile_text, PLANTED_PII) - return { - "profile_replay_lift": lift, - "false_modification_rate": fmr_with, - "false_modification_rate_without_profile": fmr_without, - "profile_fmr_non_increase": fmr_with <= fmr_without + 1e-12, - "drift_block_rate": drift_block_rate, - "privacy_leak_count": leaks, - "protected_column_violation_rate": 0.0, - "explainability_rubric_score": metrics.explainability_rubric_score(report), - } - - -def run_t5(*, target_rows: int = 50_000, repeats: int = 3, - baseline: dict[str, Any] | None = None) -> dict[str, Any]: - import freshdata as fd # noqa: PLC0415 - - truth, corrupted, kwargs = fixtures.make_t5_scale_fixture(target_rows=target_rows) - - with _PeakRss() as rss_default: - (_repaired, _), seconds_default = _timed( - lambda: fd.clean(corrupted, return_report=True, **kwargs), repeats=repeats) - rows_per_sec = len(corrupted) / seconds_default if seconds_default else 0.0 - - semantic_kwargs = dict(kwargs, context=fixtures.ECOMMERCE_CONTEXT, semantic_mode="auto") - with _PeakRss() as rss_semantic: - _, seconds_semantic = _timed( - lambda: fd.clean(corrupted, return_report=True, **semantic_kwargs), repeats=1) - - result: dict[str, Any] = { - "rows": len(corrupted), - "speed_rows_per_sec": round(rows_per_sec, 1), - "seconds_default": round(seconds_default, 4), - "seconds_with_semantic": round(seconds_semantic, 4), - "semantic_overhead_ratio": round( - seconds_semantic / seconds_default - 1.0, 4) if seconds_default else None, - "peak_rss_bytes": rss_default.peak, - "peak_rss_delta_bytes": rss_default.delta, - "peak_rss_delta_semantic_bytes": rss_semantic.delta, - "protected_column_violation_rate": 0.0, - "false_modification_rate": None, # measured on T1-T4; T5 is perf-only - } - if baseline: - base_seconds = float(baseline.get("seconds_default", 0.0)) - base_rss = float(baseline.get("peak_rss_delta_bytes", 0.0)) - if base_seconds: - result["runtime_slowdown_vs_baseline"] = round( - seconds_default / base_seconds - 1.0, 4) - if base_rss: - result["memory_overhead_vs_baseline"] = round( - rss_default.delta / base_rss - 1.0, 4) - return result - - -# --------------------------------------------------------------------------- # -# orchestration -# --------------------------------------------------------------------------- # - -_TRACK_RUNNERS: dict[str, Callable[..., dict[str, Any]]] = { - "T1": run_t1, "T2": run_t2, "T3": run_t3, "T4": run_t4, -} - - -def _install_artifact_calibration() -> tempfile.TemporaryDirectory | None: - """Point the model registry at the trained calib-v1, when built.""" - if not ARTIFACT_CALIB.is_file(): - return None - holder = tempfile.TemporaryDirectory() - target = Path(holder.name) / "calib-v1" - target.mkdir() - target.joinpath("calibration.json").write_text( - ARTIFACT_CALIB.read_text(encoding="utf-8"), encoding="utf-8") - os.environ["FRESHDATA_MODEL_DIR"] = holder.name - from freshdata.semantic import scoring # noqa: PLC0415 - - scoring.reset_calibration_cache() - return holder - - -def _release_artifact_calibration( - holder: tempfile.TemporaryDirectory | None, previous: str | None -) -> None: - if holder is None: - return - if previous is None: - os.environ.pop("FRESHDATA_MODEL_DIR", None) - else: - os.environ["FRESHDATA_MODEL_DIR"] = previous - from freshdata.semantic import scoring # noqa: PLC0415 - - scoring.reset_calibration_cache() - holder.cleanup() - - -def check_release_gates(tracks: dict[str, dict[str, Any]]) -> list[str]: - failures: list[str] = [] - - def all_values(key: str) -> list[float]: - return [t[key] for t in tracks.values() if t.get(key) is not None] - - for value in all_values("protected_column_violation_rate"): - if value > metrics.FULL_GATE_PROTECTED_VIOLATION_RATE: - failures.append(f"protected-column violation rate {value} > 0") - break - for value in all_values("false_modification_rate"): - if value > metrics.FULL_GATE_FALSE_MODIFICATION_RATE: - failures.append(f"false modification rate {value} > 0.1%") - break - t2 = tracks.get("T2", {}) - if t2.get("confidence_ece") is not None and t2["confidence_ece"] > metrics.FULL_GATE_ECE: - failures.append(f"ECE {round(t2['confidence_ece'], 4)} > {metrics.FULL_GATE_ECE}") - if (t2.get("precision_at_conf_95") is not None - and t2["precision_at_conf_95"] < metrics.FULL_GATE_PRECISION_AT_95): - failures.append( - f"P@0.95 {round(t2['precision_at_conf_95'], 4)} < {metrics.FULL_GATE_PRECISION_AT_95}") - t4 = tracks.get("T4", {}) - if t4 and not t4.get("profile_fmr_non_increase", True): - failures.append("profile replay increased the false modification rate") - if t4.get("privacy_leak_count", 0) > metrics.FULL_GATE_PRIVACY_LEAK_COUNT: - failures.append(f"privacy leak count {t4['privacy_leak_count']} > 0") - t5 = tracks.get("T5", {}) - slowdown = t5.get("runtime_slowdown_vs_baseline") - if slowdown is not None and slowdown > metrics.FULL_GATE_RUNTIME_SLOWDOWN: - failures.append(f"runtime slowdown {slowdown} > 20% vs baseline") - overhead = t5.get("memory_overhead_vs_baseline") - if overhead is not None and overhead > metrics.FULL_GATE_MEMORY_OVERHEAD: - failures.append(f"memory overhead {overhead} > 15% vs baseline") - return failures - - -# Runtime/memory gate messages. These depend on T5 wall-clock/RSS, which are only -# comparable to baseline_v1.json when T5 runs in isolation (as the perf-regression -# workflow does). The remaining gates are deterministic correctness checks. -_PERF_GATE_PREFIXES = ("runtime slowdown", "memory overhead") - - -def is_perf_gate_failure(failure: str) -> bool: - """True for the timing-dependent runtime/memory gates (as opposed to the - deterministic correctness gates).""" - return failure.startswith(_PERF_GATE_PREFIXES) - - -def run_full( - tracks: tuple[str, ...] = ALL_TRACKS, - *, - target_rows: int = 50_000, - baseline_path: Path | str = BASELINE_PATH, - update_baseline: bool = False, - include_baselines: bool = False, - command: str | None = None, -) -> dict[str, Any]: - from . import reproducibility # noqa: PLC0415 - avoid import cycle at module load - - previous_model_dir = os.environ.get("FRESHDATA_MODEL_DIR") - holder = _install_artifact_calibration() - baseline_path = Path(baseline_path) - baseline = None - if baseline_path.is_file(): - baseline = json.loads(baseline_path.read_text(encoding="utf-8")) - results: dict[str, dict[str, Any]] = {} - try: - for track in tracks: - if track == "T5": - results["T5"] = run_t5(target_rows=target_rows, baseline=baseline) - else: - results[track] = _TRACK_RUNNERS[track]() - finally: - _release_artifact_calibration(holder, previous_model_dir) - - if "T5" in results and (update_baseline or baseline is None): - baseline_path.parent.mkdir(parents=True, exist_ok=True) - baseline_path.write_text(json.dumps({ - "seconds_default": results["T5"]["seconds_default"], - "peak_rss_delta_bytes": results["T5"]["peak_rss_delta_bytes"], - "rows": results["T5"]["rows"], - "note": "v1.0-equivalent default-path baseline (model-free clean)", - }, indent=2) + "\n", encoding="utf-8") - - failures = check_release_gates(results) - perf_failures = [f for f in failures if is_perf_gate_failure(f)] - correctness_failures = [f for f in failures if not is_perf_gate_failure(f)] - payload: dict[str, Any] = { - "tracks_run": list(tracks), - "calibration_artifact_used": holder is not None, - "tracks": results, - "release_gates": { - "failures": failures, - "passed": not failures, - "correctness_failures": correctness_failures, - "perf_failures": perf_failures, - }, - "environment": reproducibility.environment_info(), - "dataset_hashes": reproducibility.dataset_hashes(), - "metrics_version": reproducibility.METRICS_VERSION, - "command": command or "python -m benchmarks.cleanbench", - } - if include_baselines: - from .baselines import run_all as run_all_baselines # noqa: PLC0415 - - payload["baselines"] = run_all_baselines() - return payload diff --git a/benchmarks/cleanbench/tasks.py b/benchmarks/cleanbench/tasks.py deleted file mode 100644 index 654ab499..00000000 --- a/benchmarks/cleanbench/tasks.py +++ /dev/null @@ -1,177 +0,0 @@ -"""CleanBench task directories: the on-disk exchange format. - -Every track task materializes as:: - - benchmarks/cleanbench/tasks// - messy.csv (or messy.parquet) - clean.csv (or clean.parquet) - context.txt (may be empty) - policy.json (compiled policy snapshot, may be null) - meta.json (track, seed, protected columns, clean kwargs, expectations) - -Tasks are generated from the in-code fixtures — deterministic, reviewable, -and small. T5 stores only its generator parameters in ``meta.json``; its -frames are regenerated at load time so no 50k-row CSV lives in git. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import pandas as pd - -TASKS_ROOT = Path(__file__).resolve().parent / "tasks" - - -@dataclass -class Task: - task_id: str - track: str - truth: pd.DataFrame | None - corrupted: pd.DataFrame | None - context: str - clean_kwargs: dict[str, Any] - meta: dict[str, Any] - - -def _write_frame(path: Path, frame: pd.DataFrame) -> str: - try: - target = path.with_suffix(".parquet") - frame.to_parquet(target, index=False) - except (ImportError, ValueError): - target = path.with_suffix(".csv") - frame.to_csv(target, index=False) - return target.name - - -def _read_frame(directory: Path, stem: str) -> pd.DataFrame | None: - parquet = directory / f"{stem}.parquet" - if parquet.is_file(): - return pd.read_parquet(parquet) - csv = directory / f"{stem}.csv" - if csv.is_file(): - return pd.read_csv(csv, keep_default_na=False, na_values=[""], dtype=str) - return None - - -def write_task( - root: Path, - task_id: str, - *, - track: str, - truth: pd.DataFrame | None, - corrupted: pd.DataFrame | None, - context: str, - clean_kwargs: dict[str, Any], - meta: dict[str, Any], -) -> Path: - directory = root / task_id - directory.mkdir(parents=True, exist_ok=True) - files: dict[str, str] = {} - if truth is not None: - files["clean"] = _write_frame(directory / "clean", truth) - if corrupted is not None: - files["messy"] = _write_frame(directory / "messy", corrupted) - (directory / "context.txt").write_text(context, encoding="utf-8") - - policy_snapshot = None - if context.strip() and corrupted is not None: - try: - import freshdata as fd # noqa: PLC0415 - - policy_snapshot = fd.compile_context(context, df=corrupted).to_dict() - except Exception: # pragma: no cover - snapshot is best-effort metadata - policy_snapshot = None - (directory / "policy.json").write_text( - json.dumps(policy_snapshot, indent=2, sort_keys=True, default=str) + "\n", - encoding="utf-8") - - payload = dict(meta, track=track, task_id=task_id, - clean_kwargs=clean_kwargs, files=files) - (directory / "meta.json").write_text( - json.dumps(payload, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8") - return directory - - -def load_task(directory: Path | str) -> Task: - directory = Path(directory) - meta = json.loads((directory / "meta.json").read_text(encoding="utf-8")) - context = (directory / "context.txt").read_text(encoding="utf-8") - truth = _read_frame(directory, "clean") - corrupted = _read_frame(directory, "messy") - return Task( - task_id=str(meta["task_id"]), - track=str(meta["track"]), - truth=truth, - corrupted=corrupted, - context=context, - clean_kwargs=dict(meta.get("clean_kwargs", {})), - meta=meta, - ) - - -def build_all(root: Path | str = TASKS_ROOT) -> list[Path]: - """(Re)generate every track's task directory from the in-code fixtures.""" - from . import fixtures # noqa: PLC0415 - - root = Path(root) - written: list[Path] = [] - - truth, corrupted, kwargs = fixtures.make_t1_representation_fixture() - written.append(write_task( - root, "t1_representation", track="T1", truth=truth, corrupted=corrupted, - context="", clean_kwargs=kwargs, - meta={"seed": 30, "protected_columns": []}, - )) - - truth, corrupted, kwargs = fixtures.make_t2_semantic_fixture() - context = kwargs.pop("context") - written.append(write_task( - root, "t2_semantic_values", track="T2", truth=truth, corrupted=corrupted, - context=context, clean_kwargs=kwargs, - meta={"seed": 0, "protected_columns": ["monthly_revenue"]}, - )) - - truth, corrupted, kwargs = fixtures.make_t3_context_fixture() - context = kwargs.pop("context") - written.append(write_task( - root, "t3_context_compliance", track="T3", truth=truth, corrupted=corrupted, - context=context, clean_kwargs=kwargs, - meta={ - "seed": 10, - "protected_columns": ["monthly_revenue"], - "expected_constraints": [ - ["cust_id", "unique"], ["email_addr", "valid_format"], - ["mobile", "locale_format"], ["status", "allowed_values"], - ["age", "impute_missing"], ["monthly_revenue", "protected"], - ], - "expected_params": { - "email_addr|valid_format": {"format": "email"}, - "status|allowed_values": {"values": ["active", "inactive", "pending"]}, - }, - }, - )) - - (pair_messy, pair_clean, batch_truth, batch_corrupted, _drifted, kwargs, - _sm, _sc) = fixtures.make_t4_profile_fixture() - directory = write_task( - root, "t4_profile_replay", track="T4", truth=batch_truth, - corrupted=batch_corrupted, context="", clean_kwargs=kwargs, - meta={"seed": 40, "protected_columns": [], - "note": "pair frames stored alongside as pair_messy/pair_clean"}, - ) - _write_frame(directory / "pair_messy", pair_messy) - _write_frame(directory / "pair_clean", pair_clean) - written.append(directory) - - written.append(write_task( - root, "t5_scale", track="T5", truth=None, corrupted=None, - context="", clean_kwargs={"verbose": False, "drop_duplicates": False, - "reset_index": False}, - meta={"seed": 50, "protected_columns": [], - "generator": {"name": "make_t5_scale_fixture", "target_rows": 50_000}}, - )) - return written diff --git a/benchmarks/cleanbench/tasks/t1_representation/clean.parquet b/benchmarks/cleanbench/tasks/t1_representation/clean.parquet deleted file mode 100644 index 60b5134c..00000000 Binary files a/benchmarks/cleanbench/tasks/t1_representation/clean.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t1_representation/context.txt b/benchmarks/cleanbench/tasks/t1_representation/context.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/benchmarks/cleanbench/tasks/t1_representation/messy.parquet b/benchmarks/cleanbench/tasks/t1_representation/messy.parquet deleted file mode 100644 index 1b6c5879..00000000 Binary files a/benchmarks/cleanbench/tasks/t1_representation/messy.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t1_representation/meta.json b/benchmarks/cleanbench/tasks/t1_representation/meta.json deleted file mode 100644 index ffb78f59..00000000 --- a/benchmarks/cleanbench/tasks/t1_representation/meta.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "clean_kwargs": { - "drop_duplicates": false, - "reset_index": false, - "verbose": false - }, - "files": { - "clean": "clean.parquet", - "messy": "messy.parquet" - }, - "protected_columns": [], - "seed": 30, - "task_id": "t1_representation", - "track": "T1" -} diff --git a/benchmarks/cleanbench/tasks/t1_representation/policy.json b/benchmarks/cleanbench/tasks/t1_representation/policy.json deleted file mode 100644 index 19765bd5..00000000 --- a/benchmarks/cleanbench/tasks/t1_representation/policy.json +++ /dev/null @@ -1 +0,0 @@ -null diff --git a/benchmarks/cleanbench/tasks/t2_semantic_values/clean.parquet b/benchmarks/cleanbench/tasks/t2_semantic_values/clean.parquet deleted file mode 100644 index b25f01f8..00000000 Binary files a/benchmarks/cleanbench/tasks/t2_semantic_values/clean.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t2_semantic_values/context.txt b/benchmarks/cleanbench/tasks/t2_semantic_values/context.txt deleted file mode 100644 index 14539a1d..00000000 --- a/benchmarks/cleanbench/tasks/t2_semantic_values/context.txt +++ /dev/null @@ -1,7 +0,0 @@ -This is an ecommerce customer dataset. -CustomerID is unique. -Emails must be valid. -Phone numbers are Indian. -Allowed status values are active, inactive, pending. -Missing Age should be estimated only if confidence >95%. -Never modify revenue values. diff --git a/benchmarks/cleanbench/tasks/t2_semantic_values/messy.parquet b/benchmarks/cleanbench/tasks/t2_semantic_values/messy.parquet deleted file mode 100644 index 4a1d7a4d..00000000 Binary files a/benchmarks/cleanbench/tasks/t2_semantic_values/messy.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t2_semantic_values/meta.json b/benchmarks/cleanbench/tasks/t2_semantic_values/meta.json deleted file mode 100644 index f1b8ea0c..00000000 --- a/benchmarks/cleanbench/tasks/t2_semantic_values/meta.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clean_kwargs": { - "drop_duplicates": false, - "reset_index": false, - "semantic_mode": "auto", - "verbose": false - }, - "files": { - "clean": "clean.parquet", - "messy": "messy.parquet" - }, - "protected_columns": [ - "monthly_revenue" - ], - "seed": 0, - "task_id": "t2_semantic_values", - "track": "T2" -} diff --git a/benchmarks/cleanbench/tasks/t2_semantic_values/policy.json b/benchmarks/cleanbench/tasks/t2_semantic_values/policy.json deleted file mode 100644 index 43437109..00000000 --- a/benchmarks/cleanbench/tasks/t2_semantic_values/policy.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "constraints": [ - { - "action": "validate_only", - "column": "cust_id", - "enforcement": "soft", - "id": "c1", - "params": {}, - "provenance": { - "parse_confidence": 1.0, - "sentence": "CustomerID is unique", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "CustomerID", - "rule": "unique" - }, - { - "action": "repair_or_flag", - "column": "email_addr", - "enforcement": "soft", - "id": "c2", - "params": { - "format": "email" - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Emails must be valid", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "Emails", - "rule": "valid_format" - }, - { - "action": "normalize_or_flag", - "column": "mobile", - "enforcement": "soft", - "id": "c3", - "params": { - "format": "phone", - "region": "IN" - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Phone numbers are Indian", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "Phone numbers", - "rule": "locale_format" - }, - { - "action": "validate_only", - "column": "status", - "enforcement": "soft", - "id": "c4", - "params": { - "values": [ - "active", - "inactive", - "pending" - ] - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Allowed status values are active, inactive, pending", - "tier": 0 - }, - "resolution_confidence": 1.0, - "resolved_from": "status", - "rule": "allowed_values" - }, - { - "action": "impute_if_confident_else_preserve", - "column": "age", - "enforcement": "soft", - "id": "c5", - "params": { - "min_confidence": 0.95 - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Missing Age should be estimated only if confidence >95%", - "tier": 0 - }, - "resolution_confidence": 1.0, - "resolved_from": "Age", - "rule": "impute_missing" - }, - { - "action": "never_modify", - "column": "monthly_revenue", - "enforcement": "hard", - "id": "c6", - "params": {}, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Never modify revenue values", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "revenue", - "rule": "protected" - } - ], - "dataset_domain": "ecommerce_customer", - "issues": [], - "policy_version": "1", - "source_text_sha256": "8a4c97c997c198069bc41d1d252a23b3c63bdbb7dfb28b14ab5bd3cab8758ca5", - "strict": false, - "unresolved": [] -} diff --git a/benchmarks/cleanbench/tasks/t3_context_compliance/clean.parquet b/benchmarks/cleanbench/tasks/t3_context_compliance/clean.parquet deleted file mode 100644 index b25f01f8..00000000 Binary files a/benchmarks/cleanbench/tasks/t3_context_compliance/clean.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t3_context_compliance/context.txt b/benchmarks/cleanbench/tasks/t3_context_compliance/context.txt deleted file mode 100644 index 14539a1d..00000000 --- a/benchmarks/cleanbench/tasks/t3_context_compliance/context.txt +++ /dev/null @@ -1,7 +0,0 @@ -This is an ecommerce customer dataset. -CustomerID is unique. -Emails must be valid. -Phone numbers are Indian. -Allowed status values are active, inactive, pending. -Missing Age should be estimated only if confidence >95%. -Never modify revenue values. diff --git a/benchmarks/cleanbench/tasks/t3_context_compliance/messy.parquet b/benchmarks/cleanbench/tasks/t3_context_compliance/messy.parquet deleted file mode 100644 index 2585ecb5..00000000 Binary files a/benchmarks/cleanbench/tasks/t3_context_compliance/messy.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t3_context_compliance/meta.json b/benchmarks/cleanbench/tasks/t3_context_compliance/meta.json deleted file mode 100644 index 0c42afcc..00000000 --- a/benchmarks/cleanbench/tasks/t3_context_compliance/meta.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "clean_kwargs": { - "drop_duplicates": false, - "reset_index": false, - "semantic_mode": "auto", - "verbose": false - }, - "expected_constraints": [ - [ - "cust_id", - "unique" - ], - [ - "email_addr", - "valid_format" - ], - [ - "mobile", - "locale_format" - ], - [ - "status", - "allowed_values" - ], - [ - "age", - "impute_missing" - ], - [ - "monthly_revenue", - "protected" - ] - ], - "expected_params": { - "email_addr|valid_format": { - "format": "email" - }, - "status|allowed_values": { - "values": [ - "active", - "inactive", - "pending" - ] - } - }, - "files": { - "clean": "clean.parquet", - "messy": "messy.parquet" - }, - "protected_columns": [ - "monthly_revenue" - ], - "seed": 10, - "task_id": "t3_context_compliance", - "track": "T3" -} diff --git a/benchmarks/cleanbench/tasks/t3_context_compliance/policy.json b/benchmarks/cleanbench/tasks/t3_context_compliance/policy.json deleted file mode 100644 index 43437109..00000000 --- a/benchmarks/cleanbench/tasks/t3_context_compliance/policy.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "constraints": [ - { - "action": "validate_only", - "column": "cust_id", - "enforcement": "soft", - "id": "c1", - "params": {}, - "provenance": { - "parse_confidence": 1.0, - "sentence": "CustomerID is unique", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "CustomerID", - "rule": "unique" - }, - { - "action": "repair_or_flag", - "column": "email_addr", - "enforcement": "soft", - "id": "c2", - "params": { - "format": "email" - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Emails must be valid", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "Emails", - "rule": "valid_format" - }, - { - "action": "normalize_or_flag", - "column": "mobile", - "enforcement": "soft", - "id": "c3", - "params": { - "format": "phone", - "region": "IN" - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Phone numbers are Indian", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "Phone numbers", - "rule": "locale_format" - }, - { - "action": "validate_only", - "column": "status", - "enforcement": "soft", - "id": "c4", - "params": { - "values": [ - "active", - "inactive", - "pending" - ] - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Allowed status values are active, inactive, pending", - "tier": 0 - }, - "resolution_confidence": 1.0, - "resolved_from": "status", - "rule": "allowed_values" - }, - { - "action": "impute_if_confident_else_preserve", - "column": "age", - "enforcement": "soft", - "id": "c5", - "params": { - "min_confidence": 0.95 - }, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Missing Age should be estimated only if confidence >95%", - "tier": 0 - }, - "resolution_confidence": 1.0, - "resolved_from": "Age", - "rule": "impute_missing" - }, - { - "action": "never_modify", - "column": "monthly_revenue", - "enforcement": "hard", - "id": "c6", - "params": {}, - "provenance": { - "parse_confidence": 1.0, - "sentence": "Never modify revenue values", - "tier": 0 - }, - "resolution_confidence": 0.9, - "resolved_from": "revenue", - "rule": "protected" - } - ], - "dataset_domain": "ecommerce_customer", - "issues": [], - "policy_version": "1", - "source_text_sha256": "8a4c97c997c198069bc41d1d252a23b3c63bdbb7dfb28b14ab5bd3cab8758ca5", - "strict": false, - "unresolved": [] -} diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/clean.parquet b/benchmarks/cleanbench/tasks/t4_profile_replay/clean.parquet deleted file mode 100644 index 74e0ed23..00000000 Binary files a/benchmarks/cleanbench/tasks/t4_profile_replay/clean.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/context.txt b/benchmarks/cleanbench/tasks/t4_profile_replay/context.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/messy.parquet b/benchmarks/cleanbench/tasks/t4_profile_replay/messy.parquet deleted file mode 100644 index fcd4c6fe..00000000 Binary files a/benchmarks/cleanbench/tasks/t4_profile_replay/messy.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/meta.json b/benchmarks/cleanbench/tasks/t4_profile_replay/meta.json deleted file mode 100644 index 6bf154f0..00000000 --- a/benchmarks/cleanbench/tasks/t4_profile_replay/meta.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "clean_kwargs": { - "drop_duplicates": false, - "reset_index": false, - "semantic_mode": "auto", - "verbose": false - }, - "files": { - "clean": "clean.parquet", - "messy": "messy.parquet" - }, - "note": "pair frames stored alongside as pair_messy/pair_clean", - "protected_columns": [], - "seed": 40, - "task_id": "t4_profile_replay", - "track": "T4" -} diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/pair_clean.parquet b/benchmarks/cleanbench/tasks/t4_profile_replay/pair_clean.parquet deleted file mode 100644 index 2e05fe62..00000000 Binary files a/benchmarks/cleanbench/tasks/t4_profile_replay/pair_clean.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/pair_messy.parquet b/benchmarks/cleanbench/tasks/t4_profile_replay/pair_messy.parquet deleted file mode 100644 index ea926546..00000000 Binary files a/benchmarks/cleanbench/tasks/t4_profile_replay/pair_messy.parquet and /dev/null differ diff --git a/benchmarks/cleanbench/tasks/t4_profile_replay/policy.json b/benchmarks/cleanbench/tasks/t4_profile_replay/policy.json deleted file mode 100644 index 19765bd5..00000000 --- a/benchmarks/cleanbench/tasks/t4_profile_replay/policy.json +++ /dev/null @@ -1 +0,0 @@ -null diff --git a/benchmarks/cleanbench/tasks/t5_scale/context.txt b/benchmarks/cleanbench/tasks/t5_scale/context.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/benchmarks/cleanbench/tasks/t5_scale/meta.json b/benchmarks/cleanbench/tasks/t5_scale/meta.json deleted file mode 100644 index 092bad9c..00000000 --- a/benchmarks/cleanbench/tasks/t5_scale/meta.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "clean_kwargs": { - "drop_duplicates": false, - "reset_index": false, - "verbose": false - }, - "files": {}, - "generator": { - "name": "make_t5_scale_fixture", - "target_rows": 50000 - }, - "protected_columns": [], - "seed": 50, - "task_id": "t5_scale", - "track": "T5" -} diff --git a/benchmarks/cleanbench/tasks/t5_scale/policy.json b/benchmarks/cleanbench/tasks/t5_scale/policy.json deleted file mode 100644 index 19765bd5..00000000 --- a/benchmarks/cleanbench/tasks/t5_scale/policy.json +++ /dev/null @@ -1 +0,0 @@ -null diff --git a/benchmarks/corpus/__init__.py b/benchmarks/corpus/__init__.py deleted file mode 100644 index b40d8958..00000000 --- a/benchmarks/corpus/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -"""The adversarial trap corpus shared by tests and benchmark harnesses. - -``pythonpath = ["."]`` in ``pyproject.toml`` already makes ``benchmarks.*`` -importable from ``tests/``, so any test can do:: - - from benchmarks.corpus import TRAPS, Disposition, by_token - -See :mod:`benchmarks.corpus.traps` for the corpus itself and -:mod:`benchmarks.corpus.dispositions` for the vocabulary it is scored against. -""" - -from .adapters import all_cases, from_gauntlet, from_truthbench -from .dispositions import ( - MUTATING, - REVIEW_FAMILY, - Disposition, - from_field_action, - satisfies, -) -from .traps import ( - TRAPS, - UNSET, - TrapCase, - by_family, - by_role, - by_token, - families, - roles, -) - -__all__ = [ - "Disposition", - "REVIEW_FAMILY", - "MUTATING", - "from_field_action", - "satisfies", - "TrapCase", - "TRAPS", - "UNSET", - "by_family", - "by_role", - "by_token", - "families", - "roles", - "all_cases", - "from_gauntlet", - "from_truthbench", -] diff --git a/benchmarks/corpus/adapters.py b/benchmarks/corpus/adapters.py deleted file mode 100644 index b7effe43..00000000 --- a/benchmarks/corpus/adapters.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Re-export the existing gold corpora through :class:`TrapCase`. - -Gauntlet and TruthBench each own a corpus that is authoritative for its own -harness. Nothing here replaces them: these adapters read their cases and -present them in one shape, so a test can ask "every leading-zero trap this -repository knows about" without caring which harness defined it. - -Both source harnesses use the same four-value vocabulary, so the mapping onto -:class:`~.dispositions.Disposition` is exact and lossless. The two additional -values (``QUARANTINE``, ``REJECT``) only ever come from the hand-written corpus -in :mod:`.traps` -- which is precisely why they were added. - -Building a fixture is not free (each generates a frame), so both readers are -cached and both degrade to an empty tuple when their optional dependencies are -missing, rather than breaking collection of every test that imports the corpus. -""" - -from __future__ import annotations - -from functools import lru_cache -from typing import Any - -from .dispositions import Disposition -from .traps import UNSET, TrapCase - -__all__ = ["from_gauntlet", "from_truthbench", "all_cases"] - -#: Both harnesses spell the four shared dispositions identically. -_SHARED = { - "preserve": Disposition.PRESERVE, - "repair": Disposition.REPAIR, - "flag": Disposition.FLAG, - "review": Disposition.REVIEW, -} - - -def _disposition(value: Any) -> Disposition: - key = value.value if hasattr(value, "value") else str(value) - try: - return _SHARED[key] - except KeyError: - raise KeyError( - f"unmapped source disposition {value!r}; if a harness gained a new " - "disposition, add it to benchmarks/corpus/adapters._SHARED rather " - "than letting it score as something it is not" - ) from None - - -@lru_cache(maxsize=1) -def from_gauntlet() -> tuple[TrapCase, ...]: - """Gauntlet's labelled cells as TrapCases. - - Gauntlet labels a cell by ``(row, column)`` in a generated frame, so - ``role`` becomes the column name; the frame itself is not carried over. - """ - try: - from benchmarks.gauntlet.fixtures import FIXTURES, build_fixture - except Exception: # pragma: no cover - optional dependency - return () - - cases: list[TrapCase] = [] - for name in sorted(FIXTURES): - try: - fixture = build_fixture(name) - except Exception: # pragma: no cover - fixture needs an optional extra - continue - for cell in fixture.cells: - expected = _disposition(cell.expect) - cases.append( - TrapCase( - token=cell.dirty, - family=cell.kind, - role=cell.column, - semantic_type=fixture.field_types.get(cell.column), - expected=expected, - rationale=f"Gauntlet {name} gold cell ({cell.kind}).", - repaired=cell.repaired if expected is Disposition.REPAIR else UNSET, - source="gauntlet", - ) - ) - return tuple(cases) - - -@lru_cache(maxsize=1) -def from_truthbench() -> tuple[TrapCase, ...]: - """TruthBench's gold cells as TrapCases. - - TruthBench already carries a ``family`` per cell, which maps straight onto - ``TrapCase.family``. Its ``expected_output`` is a ``TypedValue``; only the - plain value is carried across, because the dtype fidelity that type encodes - is TruthBench's own contract to enforce, not this corpus's. - - The dirty token is read out of the adversarial frame by ``(row_id, column)`` - so the case is usable standalone. - """ - try: - from benchmarks.truthbench.fixtures import DOMAINS, build_fixture - except Exception: # pragma: no cover - optional dependency - return () - - cases: list[TrapCase] = [] - for domain in DOMAINS: - try: - fixture = build_fixture(domain) - except Exception: # pragma: no cover - continue - frame = fixture.adversarial - for cell in fixture.cells: - expected = _disposition(cell.disposition) - repaired = UNSET - if expected is Disposition.REPAIR: - # expected_output of None is a real gold value here: "repair - # this cell to missing". UNSET would fail TrapCase validation. - repaired = getattr(cell.expected_output, "value", cell.expected_output) - token = None - try: - if cell.column in frame.columns: - token = frame.at[cell.row_id, cell.column] - except Exception: # pragma: no cover - non-scalar or odd label - token = None - cases.append( - TrapCase( - token=token, - family=cell.family or "unfamilied", - role=cell.column, - semantic_type=None, - expected=expected, - rationale=f"TruthBench {cell.domain} gold cell.", - repaired=repaired, - source="truthbench", - ) - ) - return tuple(cases) - - -def all_cases(include_harnesses: bool = True) -> tuple[TrapCase, ...]: - """The hand-written corpus, optionally plus every harness corpus.""" - from .traps import TRAPS - - if not include_harnesses: - return TRAPS - return TRAPS + from_gauntlet() + from_truthbench() diff --git a/benchmarks/corpus/dispositions.py b/benchmarks/corpus/dispositions.py deleted file mode 100644 index fd423556..00000000 --- a/benchmarks/corpus/dispositions.py +++ /dev/null @@ -1,106 +0,0 @@ -"""The disposition vocabulary shared by every FreshData evaluation harness. - -FreshData deliberately has no single ``disposition`` enum in ``src/``: the -library speaks several narrower vocabularies, each correct for its own layer -(``fieldcheck.ACTIONS``, the semantic gate's ``apply/suggest/skip``, the plan's -``auto/suggest/skip/blocked``, ``findings`` severities). This module does not -add a new public contract to the library. It defines the vocabulary the -*evaluation* harnesses score against, plus explicit mappings from the -library's own words onto it, so that a gold label means exactly one thing. - -Why six values and not the four TruthBench and Gauntlet already use: -``benchmarks/gauntlet/metrics.REVIEW_ACTIONS`` folds ``quarantine``, -``manual_review`` and ``reject`` together, so no gold corpus can currently say -"this row must be *rejected*, not merely queued for a human". Those are -materially different outcomes for a caller -- a rejected row is not in the -accepted frame at all, a quarantined one is recoverable from the quarantine -sink, and a reviewed one is still in the accepted frame pending a decision. - -Back-compatibility rule: a corpus case labelled :data:`Disposition.REVIEW` -stays satisfied by any of the three review-family outcomes, which is exactly -what ``REVIEW_ACTIONS`` means today. Only a case that *specifically* demands -``QUARANTINE`` or ``REJECT`` is scored strictly. Existing four-value fixtures -therefore keep their current meaning and keep passing. -""" - -from __future__ import annotations - -from enum import Enum - -__all__ = [ - "Disposition", - "REVIEW_FAMILY", - "MUTATING", - "from_field_action", - "satisfies", -] - - -class Disposition(str, Enum): - """What a knowledgeable human says should happen to one cell or row.""" - - #: Valid data, possibly unusual. Must survive byte-identical. - PRESERVE = "preserve" - #: A safe deterministic repair exists and should be applied. - REPAIR = "repair" - #: Must be surfaced to the user but never auto-changed. - FLAG = "flag" - #: Ambiguous. Must reach a human; must not be auto-repaired or dropped. - REVIEW = "review" - #: Must be removed from the accepted frame but stay recoverable. - QUARANTINE = "quarantine" - #: Must be refused outright; not recoverable from the accepted output. - REJECT = "reject" - - -#: The three outcomes that all satisfy a plain ``REVIEW`` label. -REVIEW_FAMILY = frozenset({Disposition.REVIEW, Disposition.QUARANTINE, Disposition.REJECT}) - -#: Dispositions under which the cell's value is allowed to change. -MUTATING = frozenset({Disposition.REPAIR}) - - -#: ``fieldcheck.ACTIONS`` -> disposition. ``normalize`` is a repair; -#: ``accept_with_warning`` is a flag; ``replace_with_null`` destroys the value -#: without preserving it, so it scores as a quarantine only when the report -#: also records the original (checked by the harness, not by this table). -_FIELD_ACTION_TO_DISPOSITION = { - "accept": Disposition.PRESERVE, - "accept_with_warning": Disposition.FLAG, - "normalize": Disposition.REPAIR, - "replace_with_null": Disposition.QUARANTINE, - "quarantine": Disposition.QUARANTINE, - "manual_review": Disposition.REVIEW, - "reject": Disposition.REJECT, -} - - -def from_field_action(action: str) -> Disposition: - """Map a ``fieldcheck`` remediation action onto a disposition. - - Raises ``KeyError`` with the offending action rather than guessing, so a - new action added to ``fieldcheck.ACTIONS`` fails loudly here instead of - being silently scored as something it is not. - """ - try: - return _FIELD_ACTION_TO_DISPOSITION[action] - except KeyError: - raise KeyError( - f"no disposition mapping for fieldcheck action {action!r}; " - f"known actions: {sorted(_FIELD_ACTION_TO_DISPOSITION)}" - ) from None - - -def satisfies(expected: Disposition, observed: Disposition) -> bool: - """Does ``observed`` satisfy a gold label of ``expected``? - - Exact match always satisfies. A plain ``REVIEW`` expectation is satisfied - by any review-family outcome, preserving today's ``REVIEW_ACTIONS`` - semantics. Nothing else widens: a ``PRESERVE`` label is never satisfied by - a repair, and a ``REJECT`` label is never satisfied by a mere review. - """ - if expected is observed: - return True - if expected is Disposition.REVIEW: - return observed in REVIEW_FAMILY - return False diff --git a/benchmarks/corpus/frames.py b/benchmarks/corpus/frames.py deleted file mode 100644 index b7a5d5a2..00000000 --- a/benchmarks/corpus/frames.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Build a realistic frame for one :class:`TrapCase`. - -A trap token on its own teaches the library nothing: role inference and the -semantic layer both need evidence, and ``semantic_types.MIN_DISTINCT_SUPPORT`` -is 5, so a one-row column is deliberately ignored. Every case therefore runs -inside a column of plausible values for its role, with the trap appended as the -last row. - -The filler is also what makes a *preservation* result meaningful: if the whole -column were traps, "nothing changed" could just mean "the library gave up". -""" - -from __future__ import annotations - -from typing import Any - -import pandas as pd - -from .traps import TrapCase - -__all__ = ["FILLER", "frame_for", "trap_position", "supported_roles"] - -#: Eight plausible values per role. Eight is above MIN_DISTINCT_SUPPORT (5) so -#: the semantic layer engages, and small enough to keep the corpus fast. -FILLER: dict[str, list[Any]] = { - "customer_id": ["1001", "1002", "1003", "1004", "1005", "1006", "1007", "1008"], - "invoice_no": ["9001", "9002", "9003", "9004", "9005", "9006", "9007", "9008"], - "account_id": ["5001", "5002", "5003", "5004", "5005", "5006", "5007", "5008"], - "unit_number": ["101A", "102B", "103C", "104D", "105E", "106F", "107G", "108H"], - "postal_code": ["02134", "10001", "94105", "60601", "30301", "98101", "19103", "33101"], - "product_code": ["A100", "A101", "B200", "B201", "C300", "C301", "D400", "D401"], - "part_number": ["P-100", "P-101", "P-200", "P-201", "P-300", "P-301", "P-400", "P-401"], - "quantity": [1, 2, 3, 4, 5, 6, 7, 8], - "login_count": [1, 2, 3, 4, 5, 6, 7, 8], - "price": [10.5, 20.25, 30.0, 40.75, 50.5, 60.25, 70.0, 80.5], - "amount": [10.5, 20.25, 30.0, 40.75, 50.5, 60.25, 70.0, 80.5], - "rate": [0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80], - "weight_kg": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], - "weight": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], - "gender": ["Male", "Female", "Male", "Female", "Male", "Female", "Male", "Female"], - "shirt_size": ["Small", "Large", "Small", "Large", "Small", "Large", "Small", "Large"], - "marital_status": [ - "Single", - "Married", - "Single", - "Married", - "Single", - "Married", - "Single", - "Single", - ], - "initial": ["A", "B", "C", "D", "E", "F", "G", "H"], - "status": [ - "active", - "inactive", - "pending", - "active", - "inactive", - "pending", - "active", - "active", - ], - "is_active": [True, False, True, False, True, False, True, False], - "country": ["US", "GB", "FR", "DE", "JP", "CA", "AU", "BR"], - "last_name": ["Smith", "Jones", "Brown", "Davis", "Miller", "Wilson", "Moore", "Taylor"], - "first_name": ["Ann", "Bob", "Cara", "Dan", "Eve", "Finn", "Gail", "Hugo"], - "middle_name": ["Lee", "Kay", "Ray", "Jo", "Max", "Sam", "Tom", "Wes"], - "company_name": [ - "Acme", - "Globex", - "Initech", - "Umbrella", - "Stark", - "Wayne", - "Hooli", - "Vandelay", - ], - "product_name": ["Widget", "Gadget", "Doohickey", "Thing", "Gizmo", "Device", "Tool", "Part"], - "ticker": ["AAPL", "MSFT", "GOOG", "AMZN", "META", "NFLX", "TSLA", "NVDA"], - "email": [f"{c}@example.com" for c in "abcdefgh"], - "notes": ["ok", "fine", "good", "great", "nice", "sure", "fine too", "all good"], - "comment": ["yes", "no", "maybe", "later", "soon", "done", "not yet", "ready"], - "description": ["red", "blue", "green", "black", "white", "grey", "pink", "teal"], - "order_date": [ - "2026-01-01", - "2026-01-02", - "2026-01-03", - "2026-01-04", - "2026-01-05", - "2026-01-06", - "2026-01-07", - "2026-01-08", - ], -} - -#: A second column so the frame is never single-column. Without it a row whose -#: only cell becomes missing is removed by ``drop_empty_rows``, which would -#: confound "the value was nulled" with "the row was dropped". -_ANCHOR = "row_key" - - -def supported_roles() -> tuple[str, ...]: - """Roles the corpus can build a frame for.""" - return tuple(sorted(FILLER)) - - -def trap_position() -> int: - """Positional index of the trap row in every built frame.""" - return len(next(iter(FILLER.values()))) - - -def frame_for(case: TrapCase) -> pd.DataFrame | None: - """A frame whose last row holds ``case.token`` in ``case.role``. - - Returns ``None`` when the corpus has no filler for the role, so a caller - can report the case as unmeasured rather than inventing a column. - """ - filler = FILLER.get(case.role) - if filler is None: - return None - column = [*filler, case.token] - return pd.DataFrame( - { - _ANCHOR: [f"r{i}" for i in range(len(column))], - case.role: column, - } - ) diff --git a/benchmarks/corpus/traps.py b/benchmarks/corpus/traps.py deleted file mode 100644 index ba2085c6..00000000 --- a/benchmarks/corpus/traps.py +++ /dev/null @@ -1,1018 +0,0 @@ -"""The shared adversarial trap corpus. - -Four gold corpora already exist under ``benchmarks/`` (Gauntlet, TruthBench, -CleanBench and the ``benchmarks/fixtures`` generator). They are authoritative -and this module does not replace them -- ``adapters.py`` re-exports their cases -through :class:`TrapCase` so they can be scored with one vocabulary. - -What this module adds is the part none of them provide: a corpus that the ~272 -ordinary files under ``tests/`` can import. Today each test re-invents its own -trap literals, which is why ``'1,000'``, ``'1.000'``, ``'0001'`` and ``'None'`` -appear as literals in zero test files. - -The organising idea is the one the library's own design turns on: **the same -token means different things in different fields**. Cases are therefore keyed -by ``(token, role)``, and the corpus deliberately contains the *same* token -under several roles with different expected dispositions. A cleaner that -applies one rule per token -- rather than per token-in-context -- fails here. - -Honesty rule ------------- -An expected disposition is recorded only where the repository actually defines -one (documentation, an existing gold fixture, or an explicit code path). Where -behaviour is genuinely undefined, ``expected`` is ``None`` and ``spec_gap`` -says what decision is missing. A ``None`` expectation is *measured and -reported*, never asserted -- inventing a contract here would be indistinguishable -from inventing one in the library. -""" - -from __future__ import annotations - -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Any - -from .dispositions import Disposition - -__all__ = [ - "TrapCase", - "TRAPS", - "UNSET", - "by_family", - "by_role", - "by_token", - "families", - "roles", -] - -D = Disposition - - -class _Unset: - """Sentinel: no gold repair value was supplied. - - Distinct from ``None``, which is a legitimate gold value meaning "the - correct repair is to make this cell missing" (an ``NA`` sentinel in a free - text column, say). Without this distinction those two cases collide. - """ - - _instance = None - - def __new__(cls): - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - - def __repr__(self) -> str: - return "UNSET" - - def __bool__(self) -> bool: - return False - - -UNSET = _Unset() - - -@dataclass(frozen=True) -class TrapCase: - """One token, in one semantic role, with the disposition a human expects.""" - - token: Any - #: Trap family, e.g. ``"leading-zero-id"``. Shared with TruthBench naming. - family: str - #: The field the token sits in, e.g. ``"customer_id"``. Drives the context. - role: str - #: The semantic type hint a caller would declare for ``role``, if any. - semantic_type: str | None - #: What should happen. ``None`` means the repository does not define it. - expected: Disposition | None - #: Why -- in terms a reviewer can check, not a restatement of ``expected``. - rationale: str - #: The gold value when ``expected`` is ``REPAIR``. ``None`` means "repair - #: to missing"; :data:`UNSET` means no value was supplied. - repaired: Any = UNSET - #: Where this case came from. - source: str = "corpus" - #: Set when ``expected`` is ``None``: the decision the repo has not made. - spec_gap: str | None = None - - def __post_init__(self) -> None: - if self.expected is None and not self.spec_gap: - raise ValueError( - f"{self.family}/{self.role}: an undefined expectation must carry " - "a spec_gap explaining which decision is missing" - ) - if self.expected is D.REPAIR and self.repaired is UNSET: - raise ValueError( - f"{self.family}/{self.role}: a REPAIR case must supply the gold " - "'repaired' value, otherwise the repair cannot be scored" - ) - if self.expected is not D.REPAIR and self.repaired is not UNSET: - raise ValueError( - f"{self.family}/{self.role}: 'repaired' is only meaningful for a " - f"REPAIR case, got expected={self.expected}" - ) - - -def _c(*args: Any, **kw: Any) -> TrapCase: - return TrapCase(*args, **kw) - - -# -------------------------------------------------------------------------- -# 1. Identifier vs quantity -- the canonical same-token-different-role trap. -# src/freshdata/semantic/experts.py:353 treats a >1-char all-digit string -# starting with '0' as an identifier value; five independent layers protect -# it. In a quantity column the numeric reading is the correct one. -# -------------------------------------------------------------------------- -_IDENTITY = [ - _c( - "007", - "leading-zero-id", - "customer_id", - "identifier", - D.PRESERVE, - "Zero padding is payload in an identifier; dropping it merges 007 with 7.", - ), - _c( - "007", - "leading-zero-id", - "postal_code", - "postal_code", - D.PRESERVE, - "Postal codes are codes, not magnitudes; 007 and 7 are different places.", - ), - _c( - "007", - "leading-zero-id", - "product_code", - "category_code", - D.PRESERVE, - "Product codes are opaque tokens.", - ), - _c( - "007", - "leading-zero-numeric", - "quantity", - "quantity", - D.REPAIR, - "In a count column the value is the number seven; the padding is noise.", - repaired=7, - ), - _c( - "0001", - "leading-zero-id", - "invoice_no", - "identifier", - D.PRESERVE, - "Same rule at a different width; guards against a 3-char special case.", - ), - _c( - "0001", - "leading-zero-numeric", - "quantity", - "quantity", - D.REPAIR, - "Numeric role: one.", - repaired=1, - ), - _c( - "0", - "zero-is-valid", - "quantity", - "quantity", - D.PRESERVE, - "Zero is a legitimate count, not a missing value. The classic " - "false-positive: treating 0 as a sentinel destroys real data.", - ), - _c( - "0", - "zero-is-valid", - "account_id", - "identifier", - D.PRESERVE, - "A single '0' is a valid identifier and must not become missing.", - ), - _c( - "0.00", - "free-item", - "price", - "currency_amount", - D.PRESERVE, - "A free item has a real price of zero; TruthBench family 'free-item'.", - source="truthbench", - ), - _c( - "A-100", - "punctuated-code", - "part_number", - "category_code", - D.PRESERVE, - "Hyphenated codes are identifiers, not ranges or arithmetic.", - ), - _c( - "BRK.B", - "ticker-with-dot", - "ticker", - "category_code", - D.PRESERVE, - "A real share-class ticker; the dot is not a decimal point.", - ), - _c( - "105A", - "alphanumeric-id", - "unit_number", - "identifier", - D.PRESERVE, - "Mixed alphanumerics are identifiers (semantic/experts.py:353).", - ), -] - -# -------------------------------------------------------------------------- -# 2. Sentinel-looking tokens that are sometimes real data. -# fieldcheck.py:466-469 already documents that explicit vocabulary outranks -# null markers -- 'NA' may be Namibia. -# -------------------------------------------------------------------------- -_SENTINELS = [ - _c( - "NA", - "sentinel-vs-country", - "country", - "country", - None, - "ISO 3166-1 alpha-2 for Namibia, and also a default sentinel token.", - spec_gap="The repository contradicts itself. fieldcheck.py:466 states " - "'explicit vocabulary outranks generic null markers -- NA may be " - "Namibia' and validate_fields honours a declared allowed_values; " - "fd.clean's normalize_sentinels ignores allowed_values and nulls it " - "(FD2-001). Gauntlet calls the collision contract behaviour. Until the " - "two layers agree, the expectation is undefined.", - ), - _c( - "NA", - "sentinel-vs-surname", - "last_name", - "person_name", - None, - "'Na' is a real surname, and also a default sentinel token.", - spec_gap="Same cross-layer conflict as country (FD2-001): a declared " - "allowed_values protects the value in validate_fields but not in " - "fd.clean. Undefined until the layers agree.", - ), - _c( - "NA", - "sentinel", - "notes", - "free_text", - D.REPAIR, - "In free text with no competing vocabulary, NA is a missing marker.", - repaired=None, - ), - _c( - "None", - "sentinel-vs-brand", - "product_name", - "free_text", - D.REPAIR, - "'None' is a real brand/typeface name, but the repository has decided: " - "Gauntlet labels this exact trap repair-to-missing and calls nulling it " - "'contract behaviour' (gauntlet/fixtures.py:178). Encoded as the repo " - "decided it, not as one might prefer -- the residual risk is recorded " - "in FD2-001 instead.", - repaired=None, - source="gauntlet", - ), - _c( - "None", - "sentinel", - "middle_name", - "person_name", - None, - "Python's None repr and a deliberate 'no middle name' are indistinguishable " - "from the token alone.", - spec_gap="Undecided whether the literal string 'None' in a name column is " - "a missing marker or asserted absence. Needs a documented rule.", - ), - _c( - "null", - "sentinel", - "notes", - "free_text", - D.REPAIR, - "Lowercase 'null' in free text is a missing marker.", - repaired=None, - ), - _c( - "", - "empty-string", - "notes", - "free_text", - D.REPAIR, - "An empty string in free text is missing.", - repaired=None, - ), - _c( - "", - "empty-string", - "comment", - "free_text", - None, - "An empty comment may mean 'deliberately blank'.", - spec_gap="Undecided whether empty string is distinguishable from NULL for " - "columns where blank is a meaningful user choice.", - ), -] - -# -------------------------------------------------------------------------- -# 3. One-character category collisions. The same 'M' across four fields. -# -------------------------------------------------------------------------- -_COLLISIONS = [ - _c( - "M", - "category-collision", - "gender", - "category_code", - D.REPAIR, - "In a gender field with a declared vocabulary, M expands to Male.", - repaired="Male", - ), - _c( - "M", - "category-collision", - "shirt_size", - "category_code", - D.REPAIR, - "In a size field the same token expands to Medium -- the opposite of the " - "gender expansion. This pair is the memory-replay trap.", - repaired="Medium", - ), - _c( - "M", - "category-collision", - "marital_status", - "category_code", - D.REPAIR, - "And again to Married. One token, three correct answers.", - repaired="Married", - ), - _c( - "M", - "category-collision", - "initial", - "person_name", - D.PRESERVE, - "A middle initial is already complete; expanding it fabricates data.", - ), - _c( - "Male", - "already-canonical", - "gender", - "category_code", - D.PRESERVE, - "Already the canonical form; a second pass must be a no-op (idempotency).", - ), -] - -# -------------------------------------------------------------------------- -# 4. Spelled numbers. semantic/experts.py:71 invalidates the whole phrase on -# any unknown token, which is what makes 'twenty apples' safe. -# -------------------------------------------------------------------------- -_SPELLED = [ - _c( - "twenty", - "spelled-number", - "quantity", - "quantity", - D.REPAIR, - "Unambiguous spelled integer.", - repaired=20, - ), - _c( - "Twenty", - "spelled-number-case", - "quantity", - "quantity", - D.REPAIR, - "Capitalisation is not semantic here.", - repaired=20, - ), - _c( - "twenty one", - "spelled-number-compound", - "quantity", - "quantity", - D.REPAIR, - "Space-separated compound.", - repaired=21, - ), - _c( - "twenty-one", - "spelled-number-hyphen", - "quantity", - "quantity", - D.REPAIR, - "Hyphenated compound must match the spaced form.", - repaired=21, - ), - _c( - "one hundred and twenty", - "spelled-number-long", - "quantity", - "quantity", - D.REPAIR, - "British 'and' form.", - repaired=120, - ), - _c( - "twenty apples", - "spelled-number-with-noun", - "quantity", - "quantity", - D.REVIEW, - "MUST NOT become 20. The cell carries a unit/noun the schema does not " - "model, so the value is not a bare count. Silently dropping 'apples' " - "is exactly the false-positive corruption this corpus exists to catch.", - ), - _c( - "twenty", - "spelled-number", - "product_name", - "free_text", - D.PRESERVE, - "In free text the word is the content.", - ), -] - -# -------------------------------------------------------------------------- -# 5. Currency and decimal separators. semantic/experts.py:110 requires an -# explicit marker, so a bare '1,200' is left to ordinary dtype repair. -# -------------------------------------------------------------------------- -_CURRENCY = [ - _c( - "$1,200.50", - "currency-marked", - "amount", - "currency_amount", - D.REPAIR, - "Explicit symbol and unambiguous en-US grouping.", - repaired=1200.50, - ), - _c( - "1,200", - "grouping-ambiguous", - "amount", - "currency_amount", - None, - "'1,200' is 1200 under en-US grouping and 1.2 under a decimal comma.", - spec_gap="Undecided without a declared locale. The library must either " - "require an explicit locale or route to review -- it must not " - "silently assume en-US.", - ), - _c( - "1.200", - "grouping-ambiguous", - "amount", - "currency_amount", - None, - "'1.200' is 1.2 under en-US and 1200 under a decimal point as separator.", - spec_gap="Undecided without a declared locale, and the mirror of " - "'1,200': if the pair resolves under different conventions the " - "same column yields 1200 and 1.2 for equivalent inputs.", - ), - _c( - "EUR 1.200,50", - "currency-declared-locale", - "amount", - "currency_amount", - D.REPAIR, - "Currency code plus a decimal comma is unambiguous.", - repaired=1200.50, - ), - _c( - "12.5%", - "percent", - "rate", - "rate", - D.REPAIR, - "Percent literal; Gauntlet carries this trap.", - repaired=0.125, - source="gauntlet", - ), - _c( - "$1,200.50", - "currency-in-text", - "description", - "free_text", - D.PRESERVE, - "A price quoted inside prose is content, not a number to extract.", - ), -] - -# -------------------------------------------------------------------------- -# 6. Units. -# -------------------------------------------------------------------------- -_UNITS = [ - _c( - "10 kg", - "unit-matches-column", - "weight_kg", - "quantity_with_unit", - D.REPAIR, - "Unit agrees with the declared column unit.", - repaired=10.0, - ), - _c( - "10 kilograms", - "unit-spelled", - "weight_kg", - "quantity_with_unit", - D.REPAIR, - "Spelled unit, same meaning.", - repaired=10.0, - ), - _c( - "10kg", - "unit-no-space", - "weight_kg", - "quantity_with_unit", - D.REPAIR, - "Missing space is formatting, not meaning.", - repaired=10.0, - ), - _c( - "10 lbs", - "unit-conflicts-column", - "weight_kg", - "quantity_with_unit", - D.REVIEW, - "The cell's unit contradicts the column's declared unit. Converting " - "silently and stripping the unit both lose the conflict.", - ), - _c( - "10", - "unit-absent", - "weight_kg", - "quantity_with_unit", - D.REPAIR, - "Bare number in a column with a declared unit takes that unit.", - repaired=10.0, - ), - _c( - "10", - "unit-unknown", - "weight", - None, - None, - "No declared column unit and no unit in the cell.", - spec_gap="Undecided whether a bare number in an undeclared-unit column is " - "accepted as-is or flagged. Must never be silently equated with " - "values carrying a known unit.", - ), - _c( - "10 zorkmids", - "unit-unrecognised", - "weight_kg", - "quantity_with_unit", - D.REVIEW, - "An unknown unit must not be discarded to salvage the number.", - ), -] - -# -------------------------------------------------------------------------- -# 7. Booleans. _TRUE_TOKENS/_FALSE_TOKENS at semantic/experts.py:159-160 are -# deliberately narrow; 'enabled'/'active' are NOT in them. -# -------------------------------------------------------------------------- -_BOOLEANS = [ - _c( - "yes", - "boolean-synonym", - "is_active", - "boolean_like", - D.REPAIR, - "In the documented true-token set.", - repaired=True, - ), - _c( - "Y", - "boolean-synonym", - "is_active", - "boolean_like", - D.REPAIR, - "In the documented true-token set.", - repaired=True, - ), - _c( - "TRUE", - "boolean-synonym", - "is_active", - "boolean_like", - D.REPAIR, - "In the documented true-token set.", - repaired=True, - ), - _c( - "1", - "boolean-numeric", - "is_active", - "boolean_like", - D.REPAIR, - "Declared boolean column: 1 is true.", - repaired=True, - ), - _c( - "1", - "boolean-numeric", - "login_count", - "quantity", - D.PRESERVE, - "Same token in a count column is the number one, not True. " - "semantic/context.py:166 never sets boolean_like for a numeric role.", - ), - _c( - "enabled", - "boolean-correlated-word", - "status", - None, - None, - "'enabled' correlates with true but is not in the documented token set, " - "and a status column may have more than two states.", - spec_gap="Undecided whether status vocabularies collapse to boolean. " - "Must not assume every truthy-sounding word is True.", - ), - _c( - "active", - "boolean-correlated-word", - "status", - None, - None, - "Same: 'active' may sit beside 'suspended' and 'pending'.", - spec_gap="Undecided whether a status vocabulary collapses to boolean. " - "'active' commonly sits beside 'suspended', 'pending' and " - "'cancelled', so a two-valued reading silently discards states.", - ), - _c( - "unknown", - "boolean-third-state", - "is_active", - "boolean_like", - D.REVIEW, - "'unknown' is a third state, not False. Mapping it to False fabricates a negative answer.", - ), - _c( - "N/A", - "boolean-missing", - "is_active", - "boolean_like", - D.REPAIR, - "Not-applicable in a boolean column is missing, not False.", - repaired=None, - ), -] - -# -------------------------------------------------------------------------- -# 8. Dates. -# -------------------------------------------------------------------------- -_DATES = [ - _c( - "01/02/2026", - "date-ambiguous", - "order_date", - "date_like", - None, - "1 Feb or 2 Jan depending on convention; both are real dates, so no " - "parse error reveals the mistake.", - spec_gap="Undecided without dayfirst. Must route to review rather than " - "silently pick a convention (experts.py:291 marks these " - "high-risk; the corpus verifies that end to end).", - ), - _c( - "02/01/2026", - "date-ambiguous", - "order_date", - "date_like", - None, - "The mirror case. If the pair resolves inconsistently, ordering breaks.", - spec_gap="Undecided without dayfirst, and critically the mirror of " - "'01/02/2026': both must resolve under the SAME convention or " - "date ordering between the two rows silently inverts.", - ), - _c( - "2026-02-01", - "date-iso", - "order_date", - "date_like", - D.REPAIR, - "ISO 8601 is unambiguous by construction.", - repaired="2026-02-01", - ), - _c( - "2026-13-01", - "date-impossible", - "order_date", - "date_like", - D.QUARANTINE, - "Month 13 cannot be repaired without guessing which field is wrong.", - ), - _c( - "2023-02-30", - "date-impossible", - "order_date", - "date_like", - D.QUARANTINE, - "February 30 never existed; Gauntlet carries this trap.", - source="gauntlet", - ), - _c( - "today", - "date-relative", - "order_date", - "date_like", - None, - "Resolvable only against a reference date.", - spec_gap="With an explicit reference_date the result must be deterministic " - "and independent of the real clock; without one the library must " - "not silently use today's date.", - ), -] - -# -------------------------------------------------------------------------- -# 9. Valid-but-weird entities. These exist to measure false-positive -# corruption -- every one is real data that a naive cleaner damages. -# -------------------------------------------------------------------------- -_ENTITIES = [ - _c( - "O'Connor", - "apostrophe-name", - "last_name", - "person_name", - D.PRESERVE, - "The apostrophe is part of the name. Stripping punctuation, or escaping " - "it as if it were SQL, corrupts a very common surname.", - ), - _c( - "X Æ A-12", - "unusual-name", - "first_name", - "person_name", - D.PRESERVE, - "A real registered given name mixing a ligature, spaces and a hyphen-digit.", - ), - _c( - "José", - "accented-name", - "first_name", - "person_name", - D.PRESERVE, - "Accents are content. NFC normalisation may change the byte sequence but " - "must not change the rendered name.", - ), - _c( - "john@example.com", - "email-valid", - "email", - "email", - D.PRESERVE, - "Already valid; must not be 'repaired'.", - ), - _c( - "John & Sons", - "html-entity", - "company_name", - "free_text", - D.REPAIR, - "A real ampersand that survived an HTML round-trip.", - repaired="John & Sons", - ), - _c( - "ABC", - "html-markup", - "company_name", - "free_text", - D.REPAIR, - "Presentational markup around the real value.", - repaired="ABC", - ), - _c( - "a < b and c > d", - "html-lookalike", - "notes", - "free_text", - D.PRESERVE, - "Angle brackets used as comparison operators are not markup.", - ), - _c( - "茶", - "multilingual", - "product_name", - "free_text", - D.PRESERVE, - "Non-Latin scripts are content; TruthBench 'multilingual-product'.", - source="truthbench", - ), -] - -# -------------------------------------------------------------------------- -# 10. Near misses -- values one character away from valid. These exist to -# measure false negatives: a regex-only validator accepts them. -# -------------------------------------------------------------------------- -_NEAR_MISSES = [ - _c( - "100O", - "ocr-digit-letter", - "quantity", - "quantity", - D.REVIEW, - "Trailing capital O for a zero. Coercing to 100 guesses; rejecting " - "outright loses a recoverable value. A human must choose.", - ), - _c( - "l0", - "ocr-letter-digit", - "quantity", - "quantity", - D.REVIEW, - "Lowercase L for a one. Same argument.", - ), - _c( - "1,2O0.50", - "ocr-in-currency", - "amount", - "currency_amount", - D.REVIEW, - "A letter inside an otherwise well-formed money string.", - ), - _c( - "AAPL ", - "trailing-space-ticker", - "ticker", - "category_code", - D.REPAIR, - "Trailing whitespace is never payload in a ticker.", - repaired="AAPL", - ), - _c( - "Malee", - "near-miss-name", - "first_name", - "person_name", - D.PRESERVE, - "One letter from 'Malee'/'Marlee' but itself a real name. Fuzzy-matching " - "it to a reference list is false-positive corruption.", - ), - _c( - "not-an-email@", - "email-malformed", - "email", - "email", - D.QUARANTINE, - "Structurally invalid; recoverable for review but not usable.", - ), - _c( - "test@test.test", - "email-plausible-fake", - "email", - "email", - D.FLAG, - "Syntactically valid and semantically worthless. Regex validation alone " - "cannot catch it, which is the point.", - ), -] - -# -------------------------------------------------------------------------- -# 11. Unicode traps. textclean strips zero-width by default; there is no -# homoglyph detection in src/ (gap B5), so those cases carry a spec_gap. -# -------------------------------------------------------------------------- -_UNICODE = [ - _c( - "A\u200bpple", - "zero-width", - "company_name", - "free_text", - D.REPAIR, - "A zero-width space inside a word is invisible and breaks equality " - "against the clean form.", - repaired="Apple", - ), - _c( - "Аpple", - "homoglyph-cyrillic", - "company_name", - "free_text", - None, - "Leading character is Cyrillic A, not Latin A. Renders identically.", - spec_gap="No homoglyph/confusable detection exists in src/ (nearest is " - "textlint mixed_script, diagnostic only). Undecided whether this " - "is flagged, repaired or preserved.", - ), - _c( - "APPL", - "fullwidth", - "ticker", - "category_code", - None, - "Full-width Latin letters spelling AAPL.", - spec_gap="NFKC would fold these to ASCII but the semantic repair path uses " - "NFC only (semantic/canonical.py:71). Undecided for identifier-like " - "columns where folding changes the payload.", - ), - _c( - "José", - "combining-mark", - "first_name", - "person_name", - D.REPAIR, - "Decomposed e-acute; NFC composes it without changing the rendered name.", - repaired="José", - ), - _c( - "Café", - "mojibake", - "company_name", - "free_text", - D.REPAIR, - "UTF-8 bytes read as Latin-1; TruthBench 'html-entity-mojibake'.", - repaired="Café", - source="truthbench", - ), -] - -# -------------------------------------------------------------------------- -# 12. Values that are valid in one column and invalid in another -- the -# field-validation matrix from Phase 7. -# -------------------------------------------------------------------------- -_FIELD_MATRIX = [ - _c( - "apple", - "text-in-numeric", - "amount", - "currency_amount", - D.QUARANTINE, - "Unparseable as money; fieldcheck defaults parse_failure to quarantine " - "so the original stays recoverable.", - ), - _c( - "apple", - "text-in-text", - "company_name", - "free_text", - D.PRESERVE, - "A perfectly good company name.", - ), - _c( - "apple", - "text-in-ticker", - "ticker", - "category_code", - D.QUARANTINE, - "Not a valid ticker; the real one is AAPL.", - ), - _c( - "apple", - "text-in-freetext", - "notes", - "free_text", - D.PRESERVE, - "Free text accepts any string.", - ), -] - -TRAPS: tuple[TrapCase, ...] = tuple( - _IDENTITY - + _SENTINELS - + _COLLISIONS - + _SPELLED - + _CURRENCY - + _UNITS - + _BOOLEANS - + _DATES - + _ENTITIES - + _NEAR_MISSES - + _UNICODE - + _FIELD_MATRIX -) - - -def by_family(family: str) -> tuple[TrapCase, ...]: - """Every case in one trap family.""" - return tuple(c for c in TRAPS if c.family == family) - - -def by_role(role: str) -> tuple[TrapCase, ...]: - """Every case that sits in one field.""" - return tuple(c for c in TRAPS if c.role == role) - - -def by_token(token: Any) -> tuple[TrapCase, ...]: - """Every role a given token appears in -- the same-token-different-role view.""" - return tuple(c for c in TRAPS if c.token == token) - - -def families() -> tuple[str, ...]: - return tuple(sorted({c.family for c in TRAPS})) - - -def roles() -> tuple[str, ...]: - return tuple(sorted({c.role for c in TRAPS})) - - -def __iter__() -> Iterator[TrapCase]: # pragma: no cover - convenience only - return iter(TRAPS) diff --git a/benchmarks/fixtures/_common.py b/benchmarks/fixtures/_common.py index 0a016415..0ba19d62 100644 --- a/benchmarks/fixtures/_common.py +++ b/benchmarks/fixtures/_common.py @@ -2,8 +2,7 @@ Every fixture in this package is *deterministic*: the same ``(n_rows, seed, defect_rate)`` triple always yields byte-for-byte identical output. That is what -makes the benchmark harness reproducible across machines (see -``benchmarks/README.md``). +makes benchmark runs reproducible across machines. Two conventions hold across all fixtures: diff --git a/benchmarks/gauntlet/__init__.py b/benchmarks/gauntlet/__init__.py deleted file mode 100644 index a8457936..00000000 --- a/benchmarks/gauntlet/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -"""FreshData Validation Gauntlet. - -Gold-labelled adversarial fixtures plus a harness that measures how -FreshData's validation surfaces (``fd.clean``, ``fd.validate_fields``, -``fd.clean_text``, domain packs, the semantic layer, PII detection) treat -each labelled cell: preserve, repair, flag, or route to review. - -Unlike CleanBench (which scores whole-frame repair fidelity against a clean -oracle), the gauntlet scores *dispositions*: every injected defect carries the -disposition FreshData should choose, and every adversarial trap is a valid -value that must survive cleaning untouched. - -Run ``python -m benchmarks.gauntlet run`` from the repo root. -""" - -from .fixtures import FIXTURES, GauntletFixture, GoldCell, build_fixture -from .metrics import compute_metrics -from .runner import run_fixture, run_gauntlet - -__all__ = [ - "FIXTURES", - "GauntletFixture", - "GoldCell", - "build_fixture", - "compute_metrics", - "run_fixture", - "run_gauntlet", -] diff --git a/benchmarks/gauntlet/__main__.py b/benchmarks/gauntlet/__main__.py deleted file mode 100644 index 0ee2dd1c..00000000 --- a/benchmarks/gauntlet/__main__.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Validation Gauntlet CLI. - -Run from the repo root:: - - python -m benchmarks.gauntlet run # run + write results/ - python -m benchmarks.gauntlet run --check # also gate (CI mode) - python -m benchmarks.gauntlet run --update-baseline -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -from .fixtures import DEFAULT_ROWS, DEFAULT_SEED, FIXTURES -from .metrics import compute_metrics -from .report import check_gates, render_markdown, results_payload, write_json -from .runner import run_gauntlet - -RESULTS_DIR = Path(__file__).parent / "results" -BASELINE_PATH = Path(__file__).parent / "baseline.json" - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m benchmarks.gauntlet") - sub = parser.add_subparsers(dest="command", required=True) - run_p = sub.add_parser("run", help="run the gauntlet and write JSON + Markdown") - run_p.add_argument("--rows", type=int, default=DEFAULT_ROWS) - run_p.add_argument("--seed", type=int, default=DEFAULT_SEED) - run_p.add_argument("--fixtures", nargs="*", choices=sorted(FIXTURES)) - run_p.add_argument("--check", action="store_true", - help="exit 1 when a gate fails or the baseline regresses") - run_p.add_argument("--update-baseline", action="store_true", - help="write this run as the stored baseline") - args = parser.parse_args(argv) - - runs = run_gauntlet(n_rows=args.rows, seed=args.seed, fixtures=args.fixtures) - metrics = {name: compute_metrics(r) for name, r in runs.items()} - payload = results_payload(metrics, n_rows=args.rows, seed=args.seed) - - write_json(payload, RESULTS_DIR / "gauntlet.json") - markdown = render_markdown(payload) - (RESULTS_DIR / "gauntlet.md").write_text(markdown) - print(markdown) - print(f"results: {RESULTS_DIR / 'gauntlet.json'}") - - if args.update_baseline: - write_json(payload, BASELINE_PATH) - print(f"baseline updated: {BASELINE_PATH}") - - if args.check: - baseline = (json.loads(BASELINE_PATH.read_text()) - if BASELINE_PATH.exists() else None) - problems = check_gates(payload, baseline) - if problems: - print("\nGATE FAILURES:", file=sys.stderr) - for p in problems: - print(f" - {p}", file=sys.stderr) - return 1 - print("all gates passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/gauntlet/baseline.json b/benchmarks/gauntlet/baseline.json deleted file mode 100644 index f22eed59..00000000 --- a/benchmarks/gauntlet/baseline.json +++ /dev/null @@ -1,228 +0,0 @@ -{ - "schema_version": 1, - "generated_at": "2026-07-13T04:14:15+00:00", - "n_rows": 300, - "seed": 42, - "fixtures": { - "crm": { - "fixture": "crm", - "n_rows": 304, - "labelled_cells": 19, - "detection": { - "precision": 1.0, - "recall": 0.9167, - "f1": 0.9565, - "true_positives": 11, - "false_negatives": 1, - "false_positives": 0 - }, - "repair_accuracy": 1.0, - "repair_sources": { - "clean": 3, - "clean_text_opt_in": 1 - }, - "review_routing": 1.0, - "preservation_rate": 1.0, - "corruption_count": 0, - "escape_rate": 0.0833, - "false_positive_rate": 0.0, - "duplicates": { - "expected": 4, - "removed": 4, - "ok": true - }, - "audit_completeness": 1.0, - "deterministic": true, - "trust": { - "pristine_frame": 100.0, - "dirty_frame": 99.688, - "monotonic": true - }, - "performance": { - "clean_seconds": 0.2098, - "validate_seconds": 0.0133, - "peak_memory_mb": 0.269 - }, - "failures": [ - { - "row": 24, - "column": "full_name", - "kind": "injection_text", - "expect": "flag", - "detected": false, - "dirty": "\"ROBERT'); DROP TABLE users;--\"", - "verdict": "escaped" - } - ], - "detected_only": [] - }, - "ecommerce": { - "fixture": "ecommerce", - "n_rows": 303, - "labelled_cells": 15, - "detection": { - "precision": 1.0, - "recall": 1.0, - "f1": 1.0, - "true_positives": 11, - "false_negatives": 0, - "false_positives": 0 - }, - "repair_accuracy": 1.0, - "repair_sources": { - "clean": 2, - "clean_text_opt_in": 1 - }, - "review_routing": 1.0, - "preservation_rate": 1.0, - "corruption_count": 0, - "escape_rate": 0.0, - "false_positive_rate": 0.0, - "duplicates": { - "expected": 3, - "removed": 3, - "ok": true - }, - "audit_completeness": 1.0, - "deterministic": true, - "trust": { - "pristine_frame": 100.0, - "dirty_frame": 94.74, - "monotonic": true - }, - "performance": { - "clean_seconds": 0.1883, - "validate_seconds": 0.0189, - "peak_memory_mb": 0.157 - }, - "failures": [], - "detected_only": [] - }, - "finance": { - "fixture": "finance", - "n_rows": 303, - "labelled_cells": 25, - "detection": { - "precision": 1.0, - "recall": 1.0, - "f1": 1.0, - "true_positives": 21, - "false_negatives": 0, - "false_positives": 0 - }, - "repair_accuracy": 1.0, - "repair_sources": { - "clean": 10 - }, - "review_routing": 1.0, - "preservation_rate": 1.0, - "corruption_count": 0, - "escape_rate": 0.0, - "false_positive_rate": 0.0, - "duplicates": { - "expected": 3, - "removed": 3, - "ok": true - }, - "audit_completeness": 1.0, - "deterministic": true, - "trust": { - "pristine_frame": 100.0, - "dirty_frame": 92.178, - "monotonic": true - }, - "performance": { - "clean_seconds": 0.223, - "validate_seconds": 0.021, - "peak_memory_mb": 0.181 - }, - "failures": [], - "detected_only": [] - }, - "healthcare": { - "fixture": "healthcare", - "n_rows": 303, - "labelled_cells": 16, - "detection": { - "precision": 1.0, - "recall": 1.0, - "f1": 1.0, - "true_positives": 15, - "false_negatives": 0, - "false_positives": 0 - }, - "repair_accuracy": 1.0, - "repair_sources": { - "clean": 1, - "validate_fields": 1 - }, - "review_routing": 1.0, - "preservation_rate": 1.0, - "corruption_count": 0, - "escape_rate": 0.0, - "false_positive_rate": 0.0, - "duplicates": { - "expected": 3, - "removed": 3, - "ok": true - }, - "audit_completeness": 1.0, - "deterministic": true, - "trust": { - "pristine_frame": 100.0, - "dirty_frame": 94.74, - "monotonic": true - }, - "performance": { - "clean_seconds": 0.2001, - "validate_seconds": 0.0217, - "peak_memory_mb": 0.16 - }, - "failures": [], - "detected_only": [] - }, - "text": { - "fixture": "text", - "n_rows": 302, - "labelled_cells": 16, - "detection": { - "precision": 1.0, - "recall": 1.0, - "f1": 1.0, - "true_positives": 10, - "false_negatives": 0, - "false_positives": 0 - }, - "repair_accuracy": 1.0, - "repair_sources": { - "clean": 1, - "clean_text_opt_in": 2, - "validate_fields": 4 - }, - "review_routing": null, - "preservation_rate": 1.0, - "corruption_count": 0, - "escape_rate": 0.0, - "false_positive_rate": 0.0, - "duplicates": { - "expected": 2, - "removed": 2, - "ok": true - }, - "audit_completeness": 1.0, - "deterministic": true, - "trust": { - "pristine_frame": 100.0, - "dirty_frame": 99.801, - "monotonic": true - }, - "performance": { - "clean_seconds": 0.0689, - "validate_seconds": 0.0054, - "peak_memory_mb": 0.103 - }, - "failures": [], - "detected_only": [] - } - } -} diff --git a/benchmarks/gauntlet/fixtures.py b/benchmarks/gauntlet/fixtures.py deleted file mode 100644 index b1c87021..00000000 --- a/benchmarks/gauntlet/fixtures.py +++ /dev/null @@ -1,520 +0,0 @@ -"""Gold-labelled gauntlet fixtures. - -Every fixture is deterministic for a given seed: a base frame of valid -records, plus injected problem cells whose *expected disposition* is known. - -Dispositions (``GoldCell.expect``): - -- ``preserve`` — valid (often unusual) data; must survive every cleaning - surface byte-identical and must not draw an error-severity issue. -- ``repair`` — a safe deterministic repair exists; ``repaired`` is the gold - value. Silently leaving it is an escape; changing it to anything else is a - corruption. -- ``flag`` — must be *detected* (issue / warning / finding) but never - auto-changed by the default pipeline. -- ``review`` — ambiguous; must be routed to a quarantine / manual-review - action, never auto-deleted or auto-accepted. - -Whole-row duplicate injections are tracked separately in ``dup_row_count`` -because row removal is a row-level (not cell-level) disposition. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Callable - -import numpy as np -import pandas as pd - -from freshdata import FieldSpec - -DEFAULT_ROWS = 300 -DEFAULT_SEED = 42 - - -@dataclass(frozen=True) -class GoldCell: - row: int - column: str - kind: str #: defect / trap family, e.g. "text_in_numeric" - dirty: Any #: the value placed in the frame - expect: str #: preserve | repair | flag | review - repaired: Any = None - #: the cell may legitimately end up imputed by the audited missing-value - #: engine after its repair-to-missing (sentinels, empty strings): both - #: "left missing" and "audited fill" count as the gold repair. - accept_impute: bool = False - pii: str | None = None #: expected PII entity_type, when the cell is PII - note: str = "" - replaced: Any = None #: the valid value the injection overwrote - - -@dataclass -class GauntletFixture: - name: str - df: pd.DataFrame - cells: list[GoldCell] - schema: dict[str, FieldSpec | str] - field_types: dict[str, str] = field(default_factory=dict) - domain: str | None = None - dup_row_count: int = 0 - n_rows: int = 0 - - def labelled(self, *expects: str) -> list[GoldCell]: - return [c for c in self.cells if not expects or c.expect in expects] - - def pristine(self) -> pd.DataFrame: - """The base frame before injection: labelled cells restored, dups gone.""" - base = self.df.iloc[: self.n_rows].copy() - for c in self.cells: - base.iloc[c.row, base.columns.get_loc(c.column)] = c.replaced - return base - - -class _Injector: - """Places labelled values on distinct (row, column) slots, deterministically.""" - - def __init__(self, df: pd.DataFrame, seed: int) -> None: - self.df = df - self.cells: list[GoldCell] = [] - self._order = { - col: list(np.random.default_rng(seed + i).permutation(len(df))) - for i, col in enumerate(df.columns) - } - - def place(self, column: str, dirty: Any, kind: str, expect: str, *, - repaired: Any = None, accept_impute: bool = False, - pii: str | None = None, note: str = "") -> int: - row = self._order[column].pop() - loc = self.df.columns.get_loc(column) - replaced = self.df.iloc[row, loc] - self.df.iloc[row, loc] = dirty - self.cells.append(GoldCell(row=row, column=column, kind=kind, dirty=dirty, - expect=expect, repaired=repaired, - accept_impute=accept_impute, pii=pii, note=note, - replaced=replaced)) - return row - - -def _rng(seed: int) -> np.random.Generator: - return np.random.default_rng(seed) - - -# --------------------------------------------------------------------------- -# finance -# --------------------------------------------------------------------------- - -_TICKERS = ("AAPL", "MSFT", "TSLA", "GOOG", "AMZN", "NVDA", "JPM", "V") -_COMPANIES = ("Apple", "Microsoft", "Tesla", "Alphabet", "Amazon", - "Nvidia", "JPMorgan", "Visa") -_CURRENCIES = frozenset({"USD", "EUR", "GBP", "JPY"}) - - -def _finance(n: int, seed: int) -> GauntletFixture: - r = _rng(seed) - idx = r.integers(0, len(_TICKERS), n) - df = pd.DataFrame({ - "txn_id": [f"TXN{100000 + i}" for i in range(n)], - "company": [_COMPANIES[i] for i in idx], - "ticker": [_TICKERS[i] for i in idx], - "price": np.round(r.uniform(10, 900, n), 2).astype(object), - "revenue": np.round(r.uniform(1e4, 5e6, n), 2).astype(object), - "currency": r.choice(sorted(_CURRENCIES), n), - "trade_date": pd.to_datetime("2025-01-01") - + pd.to_timedelta(r.integers(0, 365, n), unit="D"), - "pct_change": np.round(r.uniform(-9, 9, n), 3).astype(object), - }) - df["trade_date"] = df["trade_date"].dt.strftime("%Y-%m-%d") - inj = _Injector(df, seed) - - # -- the flagship case: 'apple' across financial columns ------------------- - inj.place("price", "apple", "text_in_numeric", "flag", - note="company name in a price column; never auto-delete") - inj.place("revenue", "apple", "text_in_numeric", "flag") - inj.place("company", "Apple", "valid_company_name", "preserve") - inj.place("ticker", "AAPL", "valid_ticker", "preserve") - inj.place("ticker", "apple", "lowercase_ticker", "review", - note="structurally invalid ticker; plausible fix exists -> review") - - # numeric problems - inj.place("price", -12.5, "impossible_range", "flag", note="negative price") - inj.place("price", 9_999_999.0, "extreme_outlier", "flag") - inj.place("price", "402.10", "numeric_as_text", "repair", repaired=402.1) - inj.place("revenue", "1,200,500.00", "thousands_separator", "repair", - repaired=1200500.0) - inj.place("revenue", "$3,400.50", "currency_symbol", "repair", repaired=3400.5) - inj.place("pct_change", "12.5%", "percent_as_text", "review", - note="'12.5%' could mean 12.5 or 0.125 in a change column — " - "quarantine for review, never guess") - inj.place("price", " 212.0 ", "whitespace_numeric", "repair", repaired=212.0) - inj.place("revenue", "N/A", "sentinel", "repair", repaired=None, - accept_impute=True) - inj.place("price", "null", "sentinel", "repair", repaired=None, - accept_impute=True) - - # dates: coercing garbage to missing is an acceptable audited repair; - # inventing a date is not. - inj.place("trade_date", "2023-02-30", "impossible_date", "repair", - repaired=None, accept_impute=True) - inj.place("trade_date", "31/45/2020", "malformed_date", "repair", - repaired=None, accept_impute=True) - inj.place("trade_date", "not a date", "text_in_date", "repair", - repaired=None, accept_impute=True) - - # identifiers / vocabulary - inj.place("txn_id", "TXN 12@34!", "malformed_id", "flag") - inj.place("currency", "usd", "case_variant_code", "flag", - note="unambiguous case variant: surfaced with the canonical " - "suggestion 'USD', accepted with a warning") - inj.place("currency", "US Dollar", "verbose_code", "review") - inj.place("ticker", "XXXX", "unknown_ticker", "flag", - note="structurally valid but not in the reference universe") - inj.place("ticker", "BRK.B", "rare_valid_ticker", "flag", - note="valid class-B share; outside this fund's trading universe, " - "so the reference lookup flags it — but it must NOT be changed") - - # adversarial traps - inj.place("company", "None", "brandlike_sentinel", "repair", repaired=None, - accept_impute=True, - note="'None' is a documented sentinel; nulling it is contract " - "behaviour, but the fill must be audited") - inj.place("company", "Ünïcode Holdings ÅB", "unicode_company", "preserve") - inj.place("pct_change", 0.0, "zero_is_valid", "preserve") - - schema: dict[str, FieldSpec | str] = { - "txn_id": FieldSpec(semantic_type="identifier", required=True), - "company": FieldSpec(semantic_type="company_name"), - "ticker": FieldSpec(semantic_type="stock_ticker", reference=set(_TICKERS), - suggest={"apple": "AAPL"}), - "price": FieldSpec(semantic_type="numeric", min_value=0.0), - "revenue": FieldSpec(semantic_type="currency_amount"), - "currency": FieldSpec(semantic_type="category_code", - allowed_values=_CURRENCIES, - suggest={"usd": "USD", "US Dollar": "USD"}), - "trade_date": FieldSpec(semantic_type="date"), - "pct_change": FieldSpec(semantic_type="percentage", - min_value=-100.0, max_value=100.0), - } - fx = GauntletFixture( - name="finance", df=df, cells=inj.cells, schema=schema, - field_types={"company": "company_name", "ticker": "stock_ticker"}, - domain="finance", n_rows=n, - ) - return _append_duplicates(fx, rows=3, seed=seed) - - -# --------------------------------------------------------------------------- -# healthcare -# --------------------------------------------------------------------------- - -_BLOOD = frozenset({"A+", "A-", "B+", "B-", "AB+", "AB-", "O+", "O-"}) -_ICD10 = ("E11.9", "I10", "J45.909", "M54.5", "F41.1", "K21.9") - - -def _healthcare(n: int, seed: int) -> GauntletFixture: - r = _rng(seed + 1) - df = pd.DataFrame({ - "mrn": [f"MRN{500000 + i}" for i in range(n)], - "icd10": r.choice(_ICD10, n), - "dob": pd.to_datetime("1950-01-01") - + pd.to_timedelta(r.integers(0, 20000, n), unit="D"), - "age": r.integers(18, 90, n).astype(object), - "temp_c": np.round(r.uniform(36.0, 39.5, n), 1).astype(object), - "heart_rate": r.integers(48, 130, n).astype(object), - "blood_type": r.choice(sorted(_BLOOD), n), - "notes": [f"Follow-up scheduled for visit {i}; vitals stable." for i in range(n)], - }) - df["dob"] = df["dob"].dt.strftime("%Y-%m-%d") - inj = _Injector(df, seed + 1) - - inj.place("age", 400, "impossible_range", "flag") - inj.place("age", "forty", "spelled_number", "flag", - note="detectable; a semantic layer may suggest 40 but must not force it") - inj.place("temp_c", 98.6, "unit_confusion", "flag", - note="Fahrenheit value in a Celsius column: syntactically valid, " - "semantically impossible") - inj.place("temp_c", "37,2", "decimal_comma", "review", - note="European decimal comma; plausible repair 37.2 needs review") - inj.place("heart_rate", 0, "impossible_range", "flag") - inj.place("dob", "2031-05-01", "future_dob", "flag") - inj.place("dob", "1875-01-01", "implausible_dob", "flag") - inj.place("icd10", "ZZZ.99.9X", "invalid_code", "flag") - inj.place("icd10", "e11.9", "case_variant_code", "review") - inj.place("blood_type", "AB-", "rare_valid_category", "preserve", - note="rarest blood type is still valid — never 'correct' it") - inj.place("blood_type", "abplus", "invalid_category", "flag") - inj.place("mrn", "", "missing_required", "flag") - inj.place("notes", "Patient SSN 123-45-6789 noted.", "pii_ssn", "flag", - pii="SSN") - inj.place("notes", "Call 555-867-5309 to reschedule.", "pii_phone", "flag", - pii="PHONE") - inj.place("notes", " Routine visit.\u200b ", "whitespace_noise", "repair", - repaired="Routine visit.") - inj.place("age", " 45 ", "whitespace_numeric", "repair", repaired=45) - - schema: dict[str, FieldSpec | str] = { - "mrn": FieldSpec(semantic_type="identifier", required=True, nullable=False), - "icd10": FieldSpec(semantic_type="category_code", - pattern=r"[A-Z]\d{2}(?:\.\d{1,4})?"), - "dob": FieldSpec(semantic_type="date", - min_value="1900-01-01", max_value="2026-07-13"), - "age": FieldSpec(semantic_type="numeric", min_value=0, max_value=120), - "temp_c": FieldSpec(semantic_type="numeric", min_value=30.0, max_value=45.0), - "heart_rate": FieldSpec(semantic_type="numeric", min_value=20, max_value=250), - "blood_type": FieldSpec(allowed_values=_BLOOD), - "notes": FieldSpec(semantic_type="free_text"), - } - fx = GauntletFixture( - name="healthcare", df=df, cells=inj.cells, schema=schema, - field_types={"notes": "free_text"}, domain="healthcare", n_rows=n, - ) - return _append_duplicates(fx, rows=3, seed=seed + 1) - - -# --------------------------------------------------------------------------- -# crm -# --------------------------------------------------------------------------- - -_COUNTRIES = frozenset({ - "United States", "Germany", "France", "Japan", "Brazil", "Namibia", "India", -}) - - -def _crm(n: int, seed: int) -> GauntletFixture: - r = _rng(seed + 2) - first = ("Ana", "Bob", "Chloé", "Dmitri", "Emeka", "Fatima", "Göran", "Hana") - last = ("Ivanov", "Jones", "Kowalski", "López", "Müller", "Ncube", "O'Brien") - df = pd.DataFrame({ - "customer_id": [f"C{20000 + i}" for i in range(n)], - "full_name": [f"{first[i % 8]} {last[i % 7]}" for i in range(n)], - "email": [f"user{i}@example.com" for i in range(n)], - "phone": [f"+1-555-{1000 + i:04d}" for i in range(n)], - "country": r.choice(sorted(_COUNTRIES), n), - "signup_date": (pd.to_datetime("2024-01-01") - + pd.to_timedelta(r.integers(0, 500, n), unit="D") - ).strftime("%Y-%m-%d"), - "status": r.choice(["active", "churned", "trial"], n), - "notes": [f"Imported from CSV batch {i}; verified by ops." for i in range(n)], - }) - inj = _Injector(df, seed + 2) - - inj.place("email", "not-an-email", "invalid_email", "flag") - inj.place("email", "jane@@corp..com", "invalid_email", "flag") - inj.place("email", "o'brien+crm@sub.domain.co.uk", "unusual_valid_email", - "preserve", note="RFC-valid oddball address") - inj.place("phone", "12", "invalid_phone", "flag") - inj.place("phone", " +1-555-0199 ", "whitespace_phone", "repair", - repaired="+1-555-0199") - inj.place("country", "Untied States", "misspelling", "review", - note="obvious typo but a guess must go through review",) - inj.place("country", "NA", "sentinel_collision", "repair", repaired=None, - accept_impute=True, - note="without a vocabulary containing 'NA', the null-marker " - "reading wins; with allowed_values that includes 'NA' the " - "value survives (see TestAllowedValuesBeatNullMarkers)") - inj.place("country", "Namibia", "rare_valid_country", "preserve") - inj.place("full_name", "Ýrsa Þorsteinsdóttir", "unicode_name", "preserve") - inj.place("full_name", "X Æ A-12", "adversarial_name", "preserve", - note="legally real name; aggressive cleaners mangle it") - inj.place("full_name", "ROBERT'); DROP TABLE users;--", "injection_text", - "flag", note="hostile payload in a name field") - inj.place("status", "ACTIVE", "case_variant", "flag", - note="surfaced with canonical suggestion 'active'; never forced") - inj.place("status", "cancelled", "unknown_category", "flag") - inj.place("signup_date", "03/04/2025", "ambiguous_date", "review", - note="US vs EU day/month ambiguity") - inj.place("customer_id", "C20001 ", "trailing_space_id", "repair", - repaired="C20001") - inj.place("notes", "Great customer 👍🎉", "emoji_text", "preserve") - inj.place("notes", "
copied from web
", "html_fragment", "repair", - repaired="copied from web") - inj.place("notes", "très bien — merci béaucoup", "mixed_language", "preserve") - inj.place("email", "USER42@EXAMPLE.COM", "case_variant_email", "preserve", - note="uppercase emails are deliverable; do not force-lower silently") - - schema: dict[str, FieldSpec | str] = { - "customer_id": FieldSpec(semantic_type="identifier", required=True), - "full_name": FieldSpec(semantic_type="person_name"), - "email": FieldSpec(semantic_type="email"), - "phone": FieldSpec(semantic_type="phone"), - "country": FieldSpec(allowed_values=_COUNTRIES, - suggest={"Untied States": "United States"}), - "signup_date": FieldSpec(semantic_type="date"), - "status": FieldSpec(allowed_values=frozenset({"active", "churned", "trial"})), - "notes": FieldSpec(semantic_type="free_text"), - } - fx = GauntletFixture( - name="crm", df=df, cells=inj.cells, schema=schema, - field_types={"notes": "free_text", "full_name": "person_name", - "email": "email", "phone": "phone"}, - n_rows=n, - ) - return _append_duplicates(fx, rows=4, seed=seed + 2) - - -# --------------------------------------------------------------------------- -# ecommerce -# --------------------------------------------------------------------------- - - -def _ecommerce(n: int, seed: int) -> GauntletFixture: - r = _rng(seed + 3) - products = ("USB-C Cable", "Desk Lamp", "Notebook", "Water Bottle", - "Backpack", "Monitor Stand") - df = pd.DataFrame({ - "order_id": [f"ORD-{700000 + i}" for i in range(n)], - "sku": [f"SKU-{r.integers(10000, 99999)}" for _ in range(n)], - "product_name": [products[i % 6] for i in range(n)], - "qty": r.integers(1, 12, n).astype(object), - "unit_price": np.round(r.uniform(3, 250, n), 2).astype(object), - "discount_pct": np.round(r.uniform(0, 40, n), 1).astype(object), - "order_date": (pd.to_datetime("2025-06-01") - + pd.to_timedelta(r.integers(0, 200, n), unit="D") - ).strftime("%Y-%m-%d"), - "review": [f"Arrived on time; order {i} matched the listing." for i in range(n)], - }) - inj = _Injector(df, seed + 3) - - inj.place("qty", "two", "spelled_number", "flag", - note="semantic layer may *suggest* 2; auto-writing it needs review") - inj.place("qty", -3, "impossible_range", "flag") - inj.place("qty", "3 pcs", "unit_suffix", "review", note="repairable to 3 via review") - inj.place("unit_price", "€49.99", "currency_symbol", "repair", repaired=49.99) - inj.place("unit_price", "12,99", "decimal_comma", "review") - inj.place("unit_price", 0.0, "suspicious_zero", "flag", - note="free items exist but deserve a flag in a price audit") - inj.place("discount_pct", 150.0, "impossible_range", "flag") - inj.place("order_date", "2025-13-01", "impossible_date", "repair", - repaired=None, accept_impute=True) - inj.place("sku", "sku-1234", "case_variant_id", "review") - inj.place("product_name", "Café Press — 12″ (limited)", "unicode_product", - "preserve") - inj.place("product_name", - "Deluxe " * 40 + "Bundle", "very_long_text", "flag", - note="overlong name; flag, never truncate silently") - inj.place("review", "GREAT!!!!!!! 🔥🔥🔥🔥🔥", "shouting_review", "preserve", - note="free text keeps its voice under the safe default config") - inj.place("review", "b’uy n’ow", "html_entities", "repair", - repaired="b’uy n’ow") - inj.place("review", "Visit http://spam.example.com now", "url_in_text", - "preserve", - note="URLs in reviews are content; spam policy is not the " - "cleaner's contract") - inj.place("review", "", "empty_string", "preserve", - note="empty free text: canonically missing either way; text-role " - "columns are never filled with fabricated content") - - schema: dict[str, FieldSpec | str] = { - "order_id": FieldSpec(semantic_type="identifier", required=True), - "sku": FieldSpec(semantic_type="identifier", - pattern=r"SKU-\d{5}"), - "product_name": FieldSpec(semantic_type="entity_name", max_length=120), - "qty": FieldSpec(semantic_type="numeric", min_value=0), - "unit_price": FieldSpec(semantic_type="currency_amount", min_value=0.01), - "discount_pct": FieldSpec(semantic_type="percentage", - min_value=0.0, max_value=100.0), - "order_date": FieldSpec(semantic_type="date"), - "review": FieldSpec(semantic_type="free_text"), - } - fx = GauntletFixture( - name="ecommerce", df=df, cells=inj.cells, schema=schema, - field_types={"review": "free_text", "product_name": "entity_name"}, - n_rows=n, - ) - return _append_duplicates(fx, rows=3, seed=seed + 3) - - -# --------------------------------------------------------------------------- -# text (adversarial free text) -# --------------------------------------------------------------------------- - - -def _text(n: int, seed: int) -> GauntletFixture: - r = _rng(seed + 4) - df = pd.DataFrame({ - "doc_id": [f"{i:03d}" for i in range(n)], - "category": r.choice(["news", "support", "sales"], n), - "comment": [f"Everything works as expected in run {i}." for i in range(n)], - }) - inj = _Injector(df, seed + 4) - - inj.place("comment", "zero\u200bwidth‌joined", "zero_width", "repair", - repaired="zerowidthjoined") - inj.place("comment", "bell\x07and\x00null", "control_chars", "repair", - repaired="bellandnull") - inj.place("comment", "line\r\nbreak\ttab", "crlf_tab", "repair", - repaired="line break tab") - inj.place("comment", " spaced out ", "whitespace", "repair", - repaired="spaced out") - inj.place("comment", "curly “quotes” and — dash", "typographic", - "preserve", note="typographic punctuation is legitimate content") - inj.place("comment", "FULLWIDTH text", "fullwidth", "repair", - repaired="FULLWIDTH text", - note="safe default (NFC) keeps fullwidth forms; the opt-in NFKC " - "pass folds them") - inj.place("comment", "café au lait", "mojibake", "flag", - note="classic UTF-8-as-Latin-1; detection wanted, silent guess not") - inj.place("comment", "sooooo cooool!!!!!!!!", "char_flood", "preserve", - note="enthusiasm is not a defect under the safe defaults") - inj.place("comment", "नमस्ते + hello + שלום", "mixed_scripts", "preserve") - inj.place("comment", "🙂🙂🙂", "emoji_only", "preserve") - inj.place("comment", "ok", "script_tag", "repair", - repaired="ok", - note="hostile HTML; kept under the safe default, stripped (with " - "script content dropped) only under the opt-in HTML pass") - inj.place("comment", "N/A", "sentinel_freetext", "repair", repaired=None, - accept_impute=True, - note="default contract: sentinels normalize to missing in every " - "column; opt out with preserve_columns=('comment',) when " - "'N/A' is a real answer") - inj.place("doc_id", "007", "leading_zero_id", "preserve", - note="dtype coercion to int would destroy the identifier") - inj.place("doc_id", "1e5", "scientific_lookalike", "preserve") - inj.place("category", "Support", "case_variant", "flag", - note="surfaced with canonical suggestion 'support'; never forced") - inj.place("category", "spam", "unknown_category", "flag") - - schema: dict[str, FieldSpec | str] = { - "doc_id": FieldSpec(semantic_type="identifier", required=True), - "category": FieldSpec(allowed_values=frozenset({"news", "support", "sales"})), - "comment": FieldSpec(semantic_type="free_text"), - } - fx = GauntletFixture( - name="text", df=df, cells=inj.cells, schema=schema, - field_types={"comment": "free_text"}, n_rows=n, - ) - return _append_duplicates(fx, rows=2, seed=seed + 4) - - -# --------------------------------------------------------------------------- - - -def _append_duplicates(fx: GauntletFixture, *, rows: int, seed: int) -> GauntletFixture: - """Duplicate ``rows`` untouched records at the end of the frame.""" - labelled_rows = {c.row for c in fx.cells} - clean_rows = [i for i in range(len(fx.df)) if i not in labelled_rows] - picks = list(np.random.default_rng(seed + 99).choice(clean_rows, rows, replace=False)) - dup = fx.df.iloc[picks].copy() - fx.df = pd.concat([fx.df, dup], ignore_index=True) - fx.dup_row_count = rows - return fx - - -FIXTURES: dict[str, Callable[[int, int], GauntletFixture]] = { - "finance": _finance, - "healthcare": _healthcare, - "crm": _crm, - "ecommerce": _ecommerce, - "text": _text, -} - - -def build_fixture(name: str, n_rows: int = DEFAULT_ROWS, - seed: int = DEFAULT_SEED) -> GauntletFixture: - try: - builder = FIXTURES[name] - except KeyError: - raise KeyError(f"unknown gauntlet fixture {name!r}; " - f"available: {sorted(FIXTURES)}") from None - return builder(n_rows, seed) diff --git a/benchmarks/gauntlet/metrics.py b/benchmarks/gauntlet/metrics.py deleted file mode 100644 index 2e8396e8..00000000 --- a/benchmarks/gauntlet/metrics.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Score gauntlet observations against gold dispositions.""" - -from __future__ import annotations - -from typing import Any - -from .runner import CellObservation, FixtureRun, _values_equal - -#: validate_fields actions that satisfy an ``expect="review"`` label. -REVIEW_ACTIONS = frozenset({"quarantine", "manual_review", "reject"}) - - -def _detected(o: CellObservation) -> bool: - """The cell was surfaced somewhere a user would see it.""" - return bool( - o.issue is not None - or o.semantic is not None - or o.pii_types - or o.lint_hit - or o.quarantined - or (o.cell.expect == "repair" and (o.changed_by_clean or o.textclean_changed - or o.normalized is not None)) - ) - - -def _repair_source(o: CellObservation) -> str | None: - """Which surface produced the gold value, or ``None``.""" - gold = o.cell.repaired - if _values_equal(o.cell.dirty, gold): - # representational repair ('402.10' -> 402.1): the canonical value is - # already right; success = the final cell equals the gold value - return "clean" if _values_equal(o.clean_value, gold) else None - if _values_equal(o.clean_value, gold): - return "clean" - if o.cell.accept_impute and o.changed_by_clean and o.audit_covered: - return "clean" # repair-to-missing followed by an audited fill - if o.normalized is not None and _values_equal(o.normalized, gold): - return "validate_fields" - if o.textclean_changed and _values_equal(o.textclean_value, gold): - return "clean_text" - if o.semantic is not None and _values_equal(o.semantic_value, gold): - return "semantic_auto" - if o.aggressive_value is not None and _values_equal(o.aggressive_value, gold) \ - and not _values_equal(o.cell.dirty, gold): - return "clean_text_opt_in" - return None - - -def score_cell(o: CellObservation) -> dict[str, Any]: - """One labelled cell -> outcome dict with ``verdict`` and failure detail.""" - expect = o.cell.expect - detected = _detected(o) - outcome: dict[str, Any] = { - "row": int(o.cell.row), "column": o.cell.column, "kind": o.cell.kind, - "expect": expect, "detected": detected, - "dirty": repr(o.cell.dirty), - } - - if expect == "preserve": - corrupted = o.changed_by_clean or o.textclean_changed - false_alarm = o.issue is not None and o.issue["severity"] == "error" - outcome["verdict"] = ( - "corrupted" if corrupted else "false_positive" if false_alarm else "ok" - ) - if corrupted: - outcome["became"] = repr(o.textclean_value if o.textclean_changed - else o.clean_value) - return outcome - - if expect == "repair": - source = _repair_source(o) - if source is not None: - outcome["verdict"] = "repaired" - outcome["source"] = source - elif o.quarantined: - outcome["verdict"] = "detected_only" # safe, reviewable, not the gold - elif o.changed_by_clean: - outcome["verdict"] = "misrepaired" - outcome["became"] = repr(o.clean_value) - elif detected: - outcome["verdict"] = "detected_only" - else: - outcome["verdict"] = "escaped" - return outcome - - if expect == "flag": - if o.changed_by_clean and not o.quarantined: - outcome["verdict"] = "corrupted" # flag-only cells must not mutate - outcome["became"] = repr(o.clean_value) - elif detected: - outcome["verdict"] = "flagged" - else: - outcome["verdict"] = "escaped" - return outcome - - # review: the cell must land in a human-review pathway - routed = (o.issue is not None and o.issue["action"] in REVIEW_ACTIONS) \ - or o.quarantined - if o.changed_by_clean and not o.quarantined: - outcome["verdict"] = "corrupted" - outcome["became"] = repr(o.clean_value) - elif routed: - outcome["verdict"] = "reviewed" - elif detected: - outcome["verdict"] = "detected_only" - else: - outcome["verdict"] = "escaped" - return outcome - - -def compute_metrics(run: FixtureRun) -> dict[str, Any]: - outcomes = [score_cell(o) for o in run.observations] - by_expect: dict[str, list[dict[str, Any]]] = {} - for oc in outcomes: - by_expect.setdefault(oc["expect"], []).append(oc) - - problems = [oc for oc in outcomes if oc["expect"] != "preserve"] - preserves = by_expect.get("preserve", []) - repairs = by_expect.get("repair", []) - reviews = by_expect.get("review", []) - - tp = sum(1 for oc in problems - if oc["detected"] or oc["verdict"] in ("repaired", "flagged", "reviewed")) - fn = len(problems) - tp - fp = len(run.false_positive_cells) + sum( - 1 for oc in preserves if oc["verdict"] == "false_positive") - precision = tp / (tp + fp) if tp + fp else 1.0 - recall = tp / (tp + fn) if tp + fn else 1.0 - f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 - - corrupted = [oc for oc in outcomes if oc["verdict"] == "corrupted"] - misrepaired = [oc for oc in outcomes if oc["verdict"] == "misrepaired"] - escaped = [oc for oc in outcomes if oc["verdict"] == "escaped"] - repaired = [oc for oc in repairs if oc["verdict"] == "repaired"] - reviewed = [oc for oc in reviews if oc["verdict"] == "reviewed"] - - dup_ok = run.duplicates_removed == run.fixture.dup_row_count - - return { - "fixture": run.fixture.name, - "n_rows": len(run.fixture.df), - "labelled_cells": len(outcomes), - "detection": { - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "true_positives": tp, - "false_negatives": fn, - "false_positives": fp, - }, - "repair_accuracy": round(len(repaired) / len(repairs), 4) if repairs else None, - "repair_sources": { - src: sum(1 for oc in repaired if oc.get("source") == src) - for src in sorted({oc.get("source") for oc in repaired} - {None}) - }, - "review_routing": round(len(reviewed) / len(reviews), 4) if reviews else None, - "preservation_rate": round( - sum(1 for oc in preserves if oc["verdict"] == "ok") / len(preserves), 4) - if preserves else None, - "corruption_count": len(corrupted) + len(misrepaired), - "escape_rate": round(len(escaped) / len(problems), 4) if problems else 0.0, - "false_positive_rate": round( - fp / (run.n_clean_cells_checked + len(preserves)), 6), - "duplicates": {"expected": run.fixture.dup_row_count, - "removed": run.duplicates_removed, "ok": dup_ok}, - "audit_completeness": round(run.audit_recorded / run.audit_mutations, 4) - if run.audit_mutations else 1.0, - "deterministic": run.deterministic, - "trust": { - "pristine_frame": round(run.trust_pristine, 3), - "dirty_frame": round(run.trust_dirty, 3), - "monotonic": run.trust_dirty <= run.trust_pristine, - }, - "performance": { - "clean_seconds": run.clean_seconds, - "validate_seconds": run.validate_seconds, - "peak_memory_mb": run.peak_memory_mb, - }, - "failures": [oc for oc in outcomes - if oc["verdict"] in ("corrupted", "misrepaired", "escaped", - "false_positive")], - "detected_only": [oc for oc in outcomes if oc["verdict"] == "detected_only"], - } diff --git a/benchmarks/gauntlet/report.py b/benchmarks/gauntlet/report.py deleted file mode 100644 index a429375c..00000000 --- a/benchmarks/gauntlet/report.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Render gauntlet results as JSON and Markdown, and gate against a baseline.""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -SCHEMA_VERSION = 1 - -#: Regression gates: a PR fails when any fixture drops below these. -GATES = { - "preservation_rate": 1.0, # valid unusual data is never corrupted - "corruption_count": 0, # zero mutations of flag/review/preserve cells - "repair_accuracy": 0.95, - "detection_f1": 0.85, - "escape_rate_max": 0.10, - "audit_completeness": 1.0, -} - - -def results_payload(metrics: dict[str, dict[str, Any]], *, n_rows: int, - seed: int) -> dict[str, Any]: - return { - "schema_version": SCHEMA_VERSION, - "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), - "n_rows": n_rows, - "seed": seed, - "fixtures": metrics, - } - - -def write_json(payload: dict[str, Any], path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, default=str) + "\n") - - -def render_markdown(payload: dict[str, Any]) -> str: - lines = [ - "# FreshData Validation Gauntlet", - "", - f"Generated {payload['generated_at']} · {payload['n_rows']} rows per " - f"fixture · seed {payload['seed']}", - "", - "Gold-labelled dispositions: every injected defect carries the outcome " - "FreshData should choose (preserve / repair / flag / review). " - "`corrupt` counts labelled cells the pipeline mutated when it should " - "not have; `escape` counts defects no surface caught.", - "", - "| fixture | cells | P | R | F1 | repair | review | preserve " - "| corrupt | escape | FPR | audit | determinism | trust mono | clean s |", - "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--:|:--:|--:|", - ] - for name, m in sorted(payload["fixtures"].items()): - d = m["detection"] - fmt = lambda v: "—" if v is None else f"{v:g}" # noqa: E731 - lines.append( - f"| {name} | {m['labelled_cells']} | {d['precision']:g} | " - f"{d['recall']:g} | {d['f1']:g} | {fmt(m['repair_accuracy'])} | " - f"{fmt(m['review_routing'])} | {fmt(m['preservation_rate'])} | " - f"{m['corruption_count']} | {m['escape_rate']:g} | " - f"{m['false_positive_rate']:g} | {m['audit_completeness']:g} | " - f"{'✅' if m['deterministic'] else '❌'} | " - f"{'✅' if m['trust']['monotonic'] else '❌'} | " - f"{m['performance']['clean_seconds']:g} |" - ) - - lines += ["", "## Failure catalogue", ""] - any_fail = False - for name, m in sorted(payload["fixtures"].items()): - for f in m["failures"]: - any_fail = True - lines.append(f"- **{name}** `{f['column']}` row {f['row']} " - f"({f['kind']}): {f['verdict']} — value {f['dirty']}" - + (f" became {f['became']}" if "became" in f else "")) - if not any_fail: - lines.append("No corruption, misrepair, escape or false positive on any " - "labelled cell.") - - partial = [(name, f) for name, m in sorted(payload["fixtures"].items()) - for f in m["detected_only"]] - if partial: - lines += ["", "## Detected but not auto-resolved (safe partial credit)", ""] - lines += [f"- **{name}** `{f['column']}` row {f['row']} ({f['kind']}): " - f"value {f['dirty']} surfaced for review" - for name, f in partial] - return "\n".join(lines) + "\n" - - -def check_gates(payload: dict[str, Any], - baseline: dict[str, Any] | None) -> list[str]: - """Absolute gates plus no-regression against the stored baseline.""" - problems: list[str] = [] - for name, m in sorted(payload["fixtures"].items()): - if m["preservation_rate"] is not None \ - and m["preservation_rate"] < GATES["preservation_rate"]: - problems.append(f"{name}: preservation_rate {m['preservation_rate']} " - f"< {GATES['preservation_rate']}") - if m["corruption_count"] > GATES["corruption_count"]: - problems.append(f"{name}: corruption_count {m['corruption_count']}") - if m["repair_accuracy"] is not None \ - and m["repair_accuracy"] < GATES["repair_accuracy"]: - problems.append(f"{name}: repair_accuracy {m['repair_accuracy']} " - f"< {GATES['repair_accuracy']}") - if m["detection"]["f1"] < GATES["detection_f1"]: - problems.append(f"{name}: F1 {m['detection']['f1']} " - f"< {GATES['detection_f1']}") - if m["escape_rate"] > GATES["escape_rate_max"]: - problems.append(f"{name}: escape_rate {m['escape_rate']} " - f"> {GATES['escape_rate_max']}") - if m["audit_completeness"] < GATES["audit_completeness"]: - problems.append(f"{name}: audit_completeness {m['audit_completeness']}") - if not m["deterministic"]: - problems.append(f"{name}: non-deterministic run") - if not m["trust"]["monotonic"]: - problems.append(f"{name}: trust score not monotonic") - - if baseline: - for name, m in sorted(payload["fixtures"].items()): - base = baseline.get("fixtures", {}).get(name) - if base is None: - continue - for key in ("recall", "f1"): - now, was = m["detection"][key], base["detection"][key] - if now < was: - problems.append( - f"{name}: detection {key} regressed {was} -> {now}") - return problems diff --git a/benchmarks/gauntlet/runner.py b/benchmarks/gauntlet/runner.py deleted file mode 100644 index 589971f1..00000000 --- a/benchmarks/gauntlet/runner.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Drive FreshData's validation surfaces over one gauntlet fixture. - -The runner calls the library exactly as a user would — public API only — -and reduces every surface's output to per-labelled-cell observations that -:mod:`benchmarks.gauntlet.metrics` scores against the gold dispositions. - -Surfaces exercised: - -1. ``fd.clean`` with defaults (the safety contract: preservation, dtype - repair, sentinel handling, dedupe, quarantine of unparseable cells). -2. ``fd.validate_fields`` with the fixture schema (per-cell detection). -3. ``fd.clean_text`` under the safe default config (lossless repairs) *and* - an explicit opt-in config (HTML stripping, NFKC folding) — opt-in repairs - are credited separately so defaults are never graded on lossy behaviour. -4. The semantic layer in ``auto`` mode (high-confidence applied repairs). -5. ``fd.lint_text_encoding`` (mojibake / mixed-script / control detection). -6. ``detect_pii`` (labelled PII cells). -7. The domain pack, when the fixture declares one. -""" - -from __future__ import annotations - -import numbers -import time -import tracemalloc -from dataclasses import dataclass, field -from typing import Any - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise.privacy import detect_pii -from freshdata.textclean import TextCleanConfig - -from .fixtures import DEFAULT_ROWS, DEFAULT_SEED, FIXTURES, GauntletFixture, GoldCell - -#: Opt-in text config used for the second clean_text pass: lossy-but-audited -#: operations a caller must ask for. Punctuation/case stay untouched. -AGGRESSIVE_TEXT = TextCleanConfig(unicode_form="NFKC", strip_html=True) - - -@dataclass -class CellObservation: - """Everything the surfaces said about one labelled cell.""" - - cell: GoldCell - changed_by_clean: bool = False - clean_value: Any = None - quarantined: bool = False #: nulled + recorded in coerced_cells - issue: dict[str, Any] | None = None #: first validate_fields issue - normalized: Any = None #: validate_fields text normalization - textclean_value: Any = None - textclean_changed: bool = False - aggressive_value: Any = None #: opt-in text pass result - semantic: dict[str, Any] | None = None #: semantic-layer action (auto mode) - semantic_value: Any = None #: cell value after semantic auto clean - lint_hit: bool = False #: textlint flagged this value - pii_types: tuple[str, ...] = () - audit_covered: bool = False #: mutation has an audit record - - -@dataclass -class FixtureRun: - fixture: GauntletFixture - observations: list[CellObservation] - false_positive_cells: list[dict[str, Any]] #: error issues on unlabelled cells - n_clean_cells_checked: int - duplicates_removed: int - deterministic: bool - trust_pristine: float - trust_dirty: float - clean_seconds: float - validate_seconds: float - peak_memory_mb: float - audit_mutations: int = 0 - audit_recorded: int = 0 - domain_findings: int = 0 - warnings: list[str] = field(default_factory=list) - - -def _canon(value: Any) -> Any: - """Loose canonical form so 45 == 45.0 == '45' == Timestamp('45')…""" - if value is None: - return None - if isinstance(value, pd.Timestamp): - return value.isoformat() - if isinstance(value, numbers.Real) and not isinstance(value, bool): - if pd.isna(value): - return None - f = float(value) - return int(f) if f == int(f) else f - try: - if pd.isna(value): - return None - except (TypeError, ValueError): - pass - if isinstance(value, str): - s = value.strip() - if not s: - return None - try: - return _canon(float(s)) - except ValueError: - ts = pd.to_datetime(s, errors="coerce") if _dateish(s) else None - return ts.isoformat() if ts is not None and not pd.isna(ts) else s - return value - - -def _dateish(s: str) -> bool: - return len(s) >= 8 and s[:4].isdigit() and s.count("-") >= 2 - - -def _values_equal(a: Any, b: Any) -> bool: - return _canon(a) == _canon(b) - - -def _timed_clean(df: pd.DataFrame) -> tuple[pd.DataFrame, Any, float, float]: - tracemalloc.start() - t0 = time.perf_counter() - out, report = fd.clean(df, return_report=True) - seconds = time.perf_counter() - t0 - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - return out, report, seconds, peak / 1e6 - - -def _observe_clean(obs: dict, cleaned: pd.DataFrame, report: Any) -> tuple[int, int]: - """Fold the default fd.clean results into the observations.""" - audit_columns = {a.column for a in report.actions if a.column} | { - w.split("'")[1] for w in report.warnings if "'" in w - } - mutations = recorded = 0 - for (row, col), o in obs.items(): - if col not in cleaned.columns or row not in cleaned.index: - o.changed_by_clean = True # cell no longer addressable (dropped) - continue - o.clean_value = cleaned.at[row, col] - o.changed_by_clean = not _values_equal(o.clean_value, o.cell.dirty) - o.quarantined = (pd.isna(o.clean_value) - and row in report.coerced_cells.get(col, {})) - if o.changed_by_clean: - mutations += 1 - if col in audit_columns: - recorded += 1 - o.audit_covered = True - return mutations, recorded - - -def _observe_validate(obs: dict, vf: Any) -> list[dict[str, Any]]: - fp_cells: list[dict[str, Any]] = [] - for issue in vf.issues: - entry = { - "row": issue.row, "column": issue.column, - "classification": issue.classification, "severity": issue.severity, - "action": issue.action, "reason": issue.reason, - "confidence": issue.confidence, "suggestion": issue.suggestion, - } - key = (issue.row, issue.column) - if key in obs: - if obs[key].issue is None: - obs[key].issue = entry - elif issue.severity == "error": - fp_cells.append(entry) - for norm in vf.normalized_cells: - key = (norm["row"], norm["column"]) - if key in obs: - obs[key].normalized = norm["cleaned"] - return fp_cells - - -def _observe_text(obs: dict, fx: GauntletFixture, df: pd.DataFrame) -> None: - text_cols = [c for c in fx.field_types if c in df.columns] - if not text_cols: - return - safe, _ = fd.clean_text(df, columns=text_cols, field_types=fx.field_types) - hard, _ = fd.clean_text(df, columns=text_cols, field_types=fx.field_types, - config=AGGRESSIVE_TEXT) - for (row, col), o in obs.items(): - if col in text_cols: - o.textclean_value = safe.at[row, col] - o.textclean_changed = not _values_equal(o.textclean_value, o.cell.dirty) - o.aggressive_value = hard.at[row, col] - - lint = fd.lint_text_encoding(df, columns=text_cols) - flagged: dict[str, set] = {} - for issue in lint.issues: - flagged.setdefault(issue.column, set()).update(issue.examples) - for (_row, col), o in obs.items(): - examples = flagged.get(col, ()) - if isinstance(o.cell.dirty, str) and any( - str(o.cell.dirty) in ex or ex in str(o.cell.dirty) - for ex in examples): - o.lint_hit = True - - -def _observe_semantic(obs: dict, df: pd.DataFrame) -> None: - out, report = fd.clean(df, semantic_mode="auto", return_report=True) - by_key: dict[tuple, dict[str, Any]] = {} - for action in report.actions: - if not action.step.startswith("semantic"): - continue - meta = action.metadata or {} - row = meta.get("row") - if row is None: - continue - by_key.setdefault((row, action.column), { - "status": action.status, - "confidence": action.confidence, - "description": action.description, - "rationale": action.rationale, - }) - for (row, col), o in obs.items(): - if (row, col) in by_key: - o.semantic = by_key[(row, col)] - if col in out.columns and row in out.index: - o.semantic_value = out.at[row, col] - - -def _observe_pii(obs: dict, df: pd.DataFrame) -> None: - for entity in detect_pii(df).entities: - meta = entity.metadata or {} - key = (meta.get("row"), meta.get("column")) - if key in obs: - o = obs[key] - o.pii_types = (*o.pii_types, entity.entity_type) - - -def _check_determinism(fx: GauntletFixture, cleaned: pd.DataFrame, vf: Any) -> bool: - cleaned2, report2, _, _ = _timed_clean(fx.df) - vf2 = fd.validate_fields(fx.df, schema=fx.schema) - return bool( - cleaned.equals(cleaned2) - and len(vf.issues) == len(vf2.issues) - and all(a.reason == b.reason and a.row == b.row and a.column == b.column - for a, b in zip(vf.issues, vf2.issues)) - ) - - -def run_fixture(fx: GauntletFixture) -> FixtureRun: - df = fx.df - obs = {(c.row, c.column): CellObservation(cell=c) for c in fx.cells} - - cleaned, report, clean_s, peak_mb = _timed_clean(df) - mutations, recorded = _observe_clean(obs, cleaned, report) - - t0 = time.perf_counter() - vf = fd.validate_fields(df, schema=fx.schema) - validate_s = time.perf_counter() - t0 - fp_cells = _observe_validate(obs, vf) - - _observe_text(obs, fx, df) - _observe_semantic(obs, df) - _observe_pii(obs, df) - - domain_findings = 0 - if fx.domain is not None: - _, dom_report = fd.clean(df, domain=fx.domain, return_report=True) - domain_findings = len(dom_report.domain_findings or []) - - deterministic = _check_determinism(fx, cleaned, vf) - - trust_pristine = float(fd.compute_trust_score(fx.pristine()).overall) - trust_dirty = float(fd.compute_trust_score(df).overall) - - return FixtureRun( - fixture=fx, - observations=list(obs.values()), - false_positive_cells=fp_cells, - n_clean_cells_checked=int(df.shape[0] * df.shape[1]) - len(fx.cells), - duplicates_removed=int(report.duplicates_removed), - deterministic=deterministic, - trust_pristine=trust_pristine, - trust_dirty=trust_dirty, - clean_seconds=round(clean_s, 4), - validate_seconds=round(validate_s, 4), - peak_memory_mb=round(peak_mb, 3), - audit_mutations=mutations, - audit_recorded=recorded, - domain_findings=domain_findings, - warnings=list(report.warnings), - ) - - -def run_gauntlet(n_rows: int = DEFAULT_ROWS, seed: int = DEFAULT_SEED, - fixtures: list[str] | None = None) -> dict[str, FixtureRun]: - from .fixtures import build_fixture - - names = fixtures or sorted(FIXTURES) - return {name: run_fixture(build_fixture(name, n_rows, seed)) for name in names} diff --git a/benchmarks/harness_metrics.py b/benchmarks/harness_metrics.py deleted file mode 100644 index 6520d975..00000000 --- a/benchmarks/harness_metrics.py +++ /dev/null @@ -1,605 +0,0 @@ -"""Metric computation for the FreshData benchmark harness. - -Nine standardized metrics (see ``docs/benchmarks.md``). Every metric is a pure -function of a fixture and the FreshData API; none of them reach into library -internals. The harness (``bench.py``) orchestrates these and serialises the -results to the stable JSON schema in ``results_schema.py``. - -The two safety metrics — false-repair rate and preservation rate — are the load -bearing ones: FreshData's contract is that id / target / free-text columns are -never mutated, and these functions verify it on every fixture, not just gold. -""" - -from __future__ import annotations - -import gc -import json -import math -import os -import tempfile -import tracemalloc -from time import perf_counter -from typing import Any - -import numpy as np -import pandas as pd - -import freshdata as fd -from fixtures import REGISTRY -from fixtures import gold as gold_mod -from freshdata import CleanConfig -from freshdata._sentinels import DEFAULT_SENTINELS - -# Canonical FreshData invocation for the authored-code metric: a config line and -# the clean call. Three logical lines covers id/target hints + clean + report. -FD_AUTHORED_LINES = 3 -_RISK_LEVELS = {"low", "medium", "high"} -TRUST_DEFECT_RATES = (0.0, 0.05, 0.15, 0.30, 0.60) - - -# -- fixture plumbing ------------------------------------------------------ -def config_for(name: str, df: pd.DataFrame, *, aggressive: bool = False) -> CleanConfig: - """Build the balanced (default) CleanConfig for a fixture. - - id/target hints come from the fixture module so the engine's role detection - is anchored, exactly as a real user would configure it. - """ - mod = REGISTRY[name] - strategy = "aggressive" if aggressive else "balanced" - return CleanConfig( - strategy=strategy, - id_columns=tuple(getattr(mod, "ID_COLUMNS", ())), - target_column=getattr(mod, "TARGET_COLUMN", None), - ) - - -def protected_columns(name: str, df: pd.DataFrame) -> dict[str, list[str]]: - """Map role -> protected column names present in ``df`` for this fixture.""" - mod = REGISTRY[name] - ids = [c for c in getattr(mod, "ID_COLUMNS", ()) if c in df.columns] - target = getattr(mod, "TARGET_COLUMN", None) - targets = [target] if target and target in df.columns else [] - if name == "wide_schema": - texts = [c for c in df.columns if c.startswith("text_")] - else: - texts = [c for c in getattr(mod, "TEXT_COLUMNS", ()) if c in df.columns] - return {"id": ids, "target": targets, "text": texts} - - -def make_frame(name: str, size: int, seed: int, defect_rate: float | None = None): - """Return the input frame for a fixture (gold returns its dirty frame).""" - mod = REGISTRY[name] - if name == "gold": - return mod.generate(size, seed=seed, defect_rate=defect_rate).dirty_df - return mod.generate(size, seed=seed, defect_rate=defect_rate) - - -# -- metric 1: wall-clock -------------------------------------------------- -def metric_wall_clock( - df: pd.DataFrame, config: CleanConfig, *, repeat: int = 5 -) -> dict[str, float]: - """p50 / p95 wall-clock seconds for clean + full report materialisation. - - Times only ``fd.clean(..., return_report=True)``; FreshData does not mutate - its input under the default ``preserve_original=True``, so no per-iteration - copy is needed and none is timed (I/O excluded, per the metric definition). - """ - times: list[float] = [] - for _ in range(max(1, repeat)): - t0 = perf_counter() - _cleaned, _report = fd.clean(df, config=config, return_report=True) - times.append(perf_counter() - t0) - return {"p50": _pctl(times, 50), "p95": _pctl(times, 95)} - - -def _pctl(values: list[float], pct: float) -> float: - if not values: - return 0.0 - if len(values) == 1: - return float(values[0]) - s = sorted(values) - k = (len(s) - 1) * (pct / 100.0) - lo = int(k) - hi = min(lo + 1, len(s) - 1) - return float(s[lo] + (s[hi] - s[lo]) * (k - lo)) - - -# -- metric 2: peak memory ------------------------------------------------- -def metric_peak_memory(df: pd.DataFrame, config: CleanConfig) -> dict[str, float]: - """Peak / delta MB of Python allocations during one clean+report run. - - Uses :mod:`tracemalloc` (stdlib, dependency-free, reproducible across - machines). ``peak_mb`` is the high-water mark; ``delta_mb`` is the increase - over the pre-call baseline. - """ - gc.collect() - tracemalloc.start() - base = tracemalloc.get_traced_memory()[0] - _cleaned, _report = fd.clean(df, config=config, return_report=True) - _cur, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - return {"peak_mb": round(peak / 1e6, 3), "delta_mb": round((peak - base) / 1e6, 3)} - - -# -- metrics 4 & 5: false-repair & preservation (all fixtures) ------------- -def _as_array(values: Any) -> np.ndarray: - if hasattr(values, "to_numpy"): - return values.to_numpy(copy=True) - return np.asarray(values) - - -def _changed_mask(a: Any, b: Any) -> np.ndarray: - """NaN-aware inequality between two aligned series.""" - av = _as_array(a) - bv = _as_array(b) - eq = np.zeros(len(av), dtype=bool) - # compare element-wise without raising on mixed dtypes - for i in range(len(av)): - a_na = _is_missing_scalar(av[i]) - b_na = _is_missing_scalar(bv[i]) - if a_na and b_na: - eq[i] = True - elif a_na or b_na: - eq[i] = False - else: - try: - eq[i] = bool(av[i] == bv[i]) - except Exception: - eq[i] = False - return ~eq - - -def _first_rows_by_key(df: pd.DataFrame, key: str) -> dict[Any, int]: - rows: dict[Any, int] = {} - for idx, value in enumerate(df[key].to_numpy(copy=True)): - if _is_missing_scalar(value) or value in rows: - continue - rows[value] = idx - return rows - - -def _take_column(df: pd.DataFrame, column: str, rows: list[int]) -> np.ndarray: - values = df[column].to_numpy(copy=True) - return values[np.asarray(rows, dtype=int)] if rows else values[:0] - - -def preservation_report(name: str, dirty: pd.DataFrame, cleaned: pd.DataFrame) -> dict[str, Any]: - """Per-role preservation + false-repair for a non-gold fixture. - - Aligns ``dirty`` and ``cleaned`` on the fixture's primary (unique) id key, - then compares protected columns cell-by-cell. The primary key itself is - validated by set membership (no non-null id may become a value absent from - the input — that would be a mutation or a fabricated fill). - """ - roles = protected_columns(name, dirty) - if not roles["id"]: - return {"per_role": {}, "preservation_rate_pct": 100.0, "false_repair_rate_pct": 0.0} - key = roles["id"][0] - - # primary key: membership check (covers both mutation and null-fill) - dirty_ids = dirty[key].to_numpy(copy=True) - cleaned_ids = cleaned[key].to_numpy(copy=True) - in_ids = {value for value in dirty_ids if not _is_missing_scalar(value)} - out_ids = [value for value in cleaned_ids if not _is_missing_scalar(value)] - key_total = len(out_ids) - key_preserved = sum(1 for value in out_ids if value in in_ids) - nulls_in = sum(1 for value in dirty_ids if _is_missing_scalar(value)) - nulls_out = sum(1 for value in cleaned_ids if _is_missing_scalar(value)) - - # align other protected columns on the key - dirty_rows = _first_rows_by_key(dirty, key) - cleaned_rows = _first_rows_by_key(cleaned, key) - common = [value for value in dirty_rows if value in cleaned_rows] - dirty_common_rows = [dirty_rows[value] for value in common] - cleaned_common_rows = [cleaned_rows[value] for value in common] - - per_role: dict[str, dict[str, float]] = {} - agg_changed = 0 - agg_total = 0 - for role in ("id", "target", "text"): - changed = 0 - total = 0 - cols = [c for c in roles[role] if c != key] - if role == "id": - # fold the primary-key membership result into the id role - changed += key_total - key_preserved - total += key_total - for c in cols: - if c not in dirty.columns or c not in cleaned.columns: - continue - cm = _changed_mask( - _take_column(dirty, c, dirty_common_rows), - _take_column(cleaned, c, cleaned_common_rows), - ) - changed += int(cm.sum()) - total += int(len(cm)) - if total: - per_role[role] = { - "preservation_rate_pct": round(100.0 * (total - changed) / total, 4), - "false_repair_rate_pct": round(100.0 * changed / total, 4), - "n_cells": total, - } - agg_changed += changed - agg_total += total - - return { - "per_role": per_role, - "preservation_rate_pct": ( - round(100.0 * (agg_total - agg_changed) / agg_total, 4) if agg_total else 100.0 - ), - "false_repair_rate_pct": round(100.0 * agg_changed / agg_total, 4) if agg_total else 0.0, - "ids_never_filled": nulls_out >= max(0, nulls_in - (len(dirty) - len(cleaned))), - } - - -# -- metrics 3/4/5 on the gold fixture (cell-level ground truth) ----------- -def gold_repair_report(size: int, seed: int) -> dict[str, Any]: - """Repair fidelity, false-repair and preservation against the gold oracle.""" - bundle = gold_mod.generate(size, seed=seed) - cfg = CleanConfig( - strategy="balanced", - # The gold oracle grades duplicate *repair* (200 appended duplicate - # rows must go away), so removal is requested explicitly now that - # detection-only is the library default. - drop_duplicates=True, - id_columns=tuple(gold_mod.ID_COLUMNS), - target_column=gold_mod.TARGET_COLUMN, - ) - cleaned = fd.clean(bundle.dirty_df, config=cfg).reset_index(drop=True) - clean = bundle.clean_df - dirty_head = bundle.dirty_df.iloc[: len(clean)].reset_index(drop=True) - n = len(clean) - - # defect family per column from the manifest - fam_by_col: dict[str, str] = {} - for d in gold_mod.DEFECT_MANIFEST: - fam_by_col.setdefault(d["column"], d["defect_type"]) - - # repair fidelity, per column + per family - per_col: dict[str, dict[str, Any]] = {} - fam_correct: dict[str, int] = {} - fam_total: dict[str, int] = {} - rep_correct = rep_total = 0 - for col in clean.columns: - if col not in cleaned.columns: - continue - rm = bundle.repair_mask[col].to_numpy() - if rm.sum() == 0: - continue - exp = clean[col] - act = cleaned[col] - ok = _values_match(exp[rm], act[rm]) - nok = int(ok.sum()) - ntot = int(rm.sum()) - per_col[col] = { - "defect_type": fam_by_col.get(col, "unknown"), - "expected_action": gold_mod.GOLD_LABELS[col]["fill_action"], - "correct_cells": nok, - "total_cells": ntot, - "fidelity_pct": round(100.0 * nok / ntot, 3), - } - fam = per_col[col]["defect_type"] - fam_correct[fam] = fam_correct.get(fam, 0) + nok - fam_total[fam] = fam_total.get(fam, 0) + ntot - rep_correct += nok - rep_total += ntot - - # duplicate removal is a row-level repair family - dup_expected = bundle.n_duplicates - dup_removed = len(bundle.dirty_df) - len(cleaned) - fam_correct["exact_duplicate_row"] = min(dup_removed, dup_expected) - fam_total["exact_duplicate_row"] = dup_expected - if dup_expected: - rep_correct += min(dup_removed, dup_expected) - rep_total += dup_expected - - # false-repair on trap cells - trap_changed = trap_total = 0 - per_trap: dict[str, dict[str, float]] = {} - for col in clean.columns: - tm = bundle.false_repair_traps[col].to_numpy() - if tm.sum() == 0: - continue - cm = _changed_mask(dirty_head[col], cleaned[col]) - changed = int((cm & tm).sum()) - tot = int(tm.sum()) - per_trap[col] = { - "false_repair_rate_pct": round(100.0 * changed / tot, 4), - "n_traps": tot, - } - trap_changed += changed - trap_total += tot - - # preservation on protected columns - pres_unchanged = pres_total = 0 - for col in gold_mod.PROTECTED: - pm = bundle.preservation_mask[col].to_numpy() - cm = _changed_mask(dirty_head[col], cleaned[col]) - unchanged = int((~cm & pm).sum()) - tot = int(pm.sum()) - pres_unchanged += unchanged - pres_total += tot - - return { - "repair_fidelity_pct": round(100.0 * rep_correct / rep_total, 3) if rep_total else 100.0, - "per_column": per_col, - "per_family": { - fam: round(100.0 * fam_correct[fam] / fam_total[fam], 3) - for fam in fam_total - if fam_total[fam] - }, - "false_repair_rate_pct": ( - round(100.0 * trap_changed / trap_total, 4) if trap_total else 0.0 - ), - "per_trap": per_trap, - "preservation_rate_pct": ( - round(100.0 * pres_unchanged / pres_total, 4) if pres_total else 100.0 - ), - "n_rows": n, - } - - -def _values_match(exp: pd.Series, act: pd.Series) -> np.ndarray: - """Element-wise match with float tolerance and NaN/NaT awareness.""" - ev = exp.to_numpy() - av = act.reindex(exp.index).to_numpy() if hasattr(act, "reindex") else np.asarray(act) - out = np.zeros(len(ev), dtype=bool) - for i in range(len(ev)): - e, a = ev[i], av[i] - e_na = _is_missing_scalar(e) - a_na = _is_missing_scalar(a) - if e_na and a_na: - out[i] = True - elif e_na or a_na: - out[i] = False - else: - try: - if ( - isinstance(e, (int, float, np.floating, np.integer)) - and not isinstance(e, bool) - ): - out[i] = bool(np.isclose(float(e), float(a), rtol=1e-6, atol=1e-9)) - elif _is_dateish(e) or _is_dateish(a): - out[i] = _date_values_equal(e, a) - else: - out[i] = bool(e == a) - except Exception: - try: - out[i] = bool(e == a) - except Exception: - out[i] = False - return out - - -def _is_missing_scalar(value: Any) -> bool: - """Scalar-only missing check that avoids pandas scalar NA dispatch.""" - if value is None or value is pd.NA or value is pd.NaT: - return True - if isinstance(value, (float, np.floating)): - return math.isnan(float(value)) - if isinstance(value, (np.datetime64, np.timedelta64)): - return bool(np.isnat(value)) - return False - - -def _is_dateish(v: Any) -> bool: - return isinstance(v, (pd.Timestamp, np.datetime64)) - - -def _date_values_equal(expected: Any, actual: Any) -> bool: - try: - return bool(np.datetime64(expected) == np.datetime64(actual)) - except Exception: - try: - return bool(expected == actual) - except Exception: - return False - - -# -- metric 3 (named fixtures): manifest-driven repair fidelity ------------- -def _expand_pattern(pattern: str, cols: list[str]) -> list[str] | None: - """Resolve a DEFECT_MANIFEST column pattern to concrete columns. - - Returns ``None`` for the row-level wildcard ``"*"`` (whole-row families like - duplicate removal). - """ - if pattern == "*": - return None - if "|" in pattern: - wanted = set(pattern.split("|")) - return [c for c in cols if c in wanted] - if pattern.endswith("*"): - prefix = pattern[:-1] - return [c for c in cols if c.startswith(prefix)] - return [c for c in cols if c == pattern] - - -# The engine deliberately *normalises* a sentinel to one of these canonical -# missing markers (e.g. for medium-missingness categoricals). Their presence is -# the repair succeeding, not a residual raw sentinel, so they are excluded from -# the "did any injected sentinel survive?" check. -_NORMALIZED_MARKERS = {"missing", "unknown"} -_RAW_SENTINELS = DEFAULT_SENTINELS - _NORMALIZED_MARKERS - - -def _has_sentinel(series: pd.Series) -> bool: - """True if any *raw* injected sentinel token survived cleaning.""" - if series.dtype != object: - return False - vals = series.dropna() - if vals.empty: - return False - lowered = vals.map(lambda v: v.strip().lower() if isinstance(v, str) else v) - return bool(lowered.isin(_RAW_SENTINELS).any()) - - -def _has_surrounding_ws(series: pd.Series) -> bool: - if series.dtype != object: - return False - vals = series.dropna() - return bool(vals.map(lambda v: isinstance(v, str) and v != v.strip()).any()) - - -def manifest_repair_fidelity( - name: str, dirty: pd.DataFrame, cleaned: pd.DataFrame, report -) -> dict[str, Any]: - """Family-level repair fidelity for a named fixture, from DEFECT_MANIFEST. - - Each documented defect family contributes a single post-condition (did - FreshData do the correct thing?), scored 1.0 / 0.0. This is coarser than the - gold fixture's cell-level fidelity but is anchored entirely in the declared - ground truth, so it is reported for every fixture. "Correct" for a flagging - family (reference / missing-flag / CDC / provenance) means non-destructive - preservation, not silent rewriting. - """ - mod = REGISTRY[name] - cols = list(cleaned.columns) - per_family: dict[str, bool] = {} - - for d in getattr(mod, "DEFECT_MANIFEST", []): - fam = d["id"] - repair = d["in_scope_repair"] - targets = _expand_pattern(d["column"], cols) - - if repair == "drop_duplicate" or targets is None: - satisfied = not cleaned.duplicated().any() - elif repair == "sentinel_normalize": - satisfied = all(not _has_sentinel(cleaned[c]) for c in targets if c in cleaned) - elif repair == "median_fill": - satisfied = all( - c in cleaned and cleaned[c].isna().sum() == 0 and cleaned[c].dtype.kind in "fi" - for c in targets - ) - elif repair == "dtype_coerce": - satisfied = all( - c in cleaned and cleaned[c].dtype.kind in ("f", "i", "M") for c in targets - ) - elif repair == "normalize_whitespace": - satisfied = all(not _has_surrounding_ws(cleaned[c]) for c in targets if c in cleaned) - elif repair in ("reference_flag",): - # correct = bad values preserved (not silently rewritten to a valid one) - satisfied = all(c in cleaned for c in targets) - elif repair in ( - "flag_missing", "preserve", "cdc_flag", "provenance_flag", "dtype_coerce_or_flag" - ): - # non-destructive: the column still exists and missing stays missing - satisfied = all(c in cleaned for c in targets) - else: - satisfied = all(c in cleaned for c in targets) - per_family[fam] = bool(satisfied) - - n = len(per_family) - pct = round(100.0 * sum(per_family.values()) / n, 3) if n else 100.0 - return {"repair_fidelity_pct": pct, "per_family": per_family} - - -# -- metric 6: authored-code reduction ------------------------------------- -def metric_authored_lines() -> dict[str, Any]: - """Authored-line counts for FreshData vs pandas / pyjanitor baselines.""" - from baselines import pandas_baseline, pyjanitor_baseline - - pandas_lines = pandas_baseline.AUTHORED_LINES - pj_lines = pyjanitor_baseline.AUTHORED_LINES - return { - "fd_lines": FD_AUTHORED_LINES, - "pandas_lines": pandas_lines, - "pyjanitor_lines": pj_lines, - "reduction_vs_pandas_pct": round( - 100.0 * (pandas_lines - FD_AUTHORED_LINES) / pandas_lines, 2 - ), - "reduction_vs_pyjanitor_pct": round(100.0 * (pj_lines - FD_AUTHORED_LINES) / pj_lines, 2), - } - - -# -- metric 7: diagnosis speed --------------------------------------------- -def metric_diagnosis_speed(report) -> dict[str, float]: - """Report-materialisation latency: summary / to_frame / to_dict.""" - def _timed(fn) -> float: - t0 = perf_counter() - fn() - return perf_counter() - t0 - - return { - "summary_sec": round(_timed(report.summary), 6), - "to_frame_sec": round(_timed(report.to_frame), 6), - "to_dict_sec": round(_timed(report.to_dict), 6), - } - - -# -- metric 8: trust score + monotonicity ---------------------------------- -def metric_trust(name: str, sweep_size: int, seed: int) -> dict[str, Any]: - """Trust score at base defects + strict-monotonic decrease over a sweep. - - The sweep is run at a bounded size so the metric stays cheap on large - fixtures; monotonicity is a property of the defect rate, not the row count. - """ - base_df = make_frame(name, sweep_size, seed, defect_rate=None) - base_trust = round(float(fd.compute_trust_score(base_df).overall), 3) - - scores = [] - for r in TRUST_DEFECT_RATES: - dfr = make_frame(name, sweep_size, seed, defect_rate=r) - scores.append(round(float(fd.compute_trust_score(dfr).overall), 3)) - monotonic = all(scores[i] > scores[i + 1] for i in range(len(scores) - 1)) - return { - "trust_score": base_trust, - "trust_monotonic_valid": bool(monotonic), - "sweep": {str(r): s for r, s in zip(TRUST_DEFECT_RATES, scores)}, - } - - -# -- metric 9: export completeness ----------------------------------------- -def metric_export_completeness( - name: str, df: pd.DataFrame, config: CleanConfig, report -) -> dict[str, Any]: - """All required report fields populated + export methods non-empty/valid.""" - checks: list[bool] = [] - missing: list[str] = [] - - def check(label: str, cond: bool) -> None: - checks.append(bool(cond)) - if not cond: - missing.append(label) - - summary = report.summary() - frame = report.to_frame() - asdict = report.to_dict() - check("summary_nonempty", isinstance(summary, str) and len(summary) > 0) - check("to_frame_nonempty", frame is not None and len(frame) > 0) - check("to_dict_dict", isinstance(asdict, dict) and len(asdict) > 0) - - # Every repaired action must carry a before/after summary (description), a - # valid risk level and a numeric confidence. ``rationale`` is the engine's - # *statistical* justification — populated for decision-engine actions - # (impute / outliers, identifiable by a model_id or a non-low risk); the - # deterministic Layer-1 representation steps explain themselves through the - # description instead, which is the before/after summary the metric checks. - for action in report: - if not getattr(action, "count", 0): - continue - col = getattr(action, "column", None) or action.step - check(f"before_after:{col}", bool(getattr(action, "description", ""))) - check(f"risk:{col}", getattr(action, "risk", None) in _RISK_LEVELS) - check(f"confidence:{col}", isinstance(getattr(action, "confidence", None), float)) - is_engine = ( - bool(getattr(action, "model_id", "")) or getattr(action, "risk", "low") != "low" - ) - if is_engine: - check(f"rationale:{col}", bool(getattr(action, "rationale", ""))) - - # enterprise exports: quality markdown + lineage json - try: - res = fd.clean_enterprise(df, clean_config=config) - check("quality_markdown", len(res.quality.to_markdown()) > 0) - path = os.path.join(tempfile.gettempdir(), f"fd_lineage_{name}.json") - res.lineage.emit(path) - with open(path) as fh: - json.load(fh) - check("lineage_json_valid", os.path.exists(path)) - os.remove(path) - except Exception as exc: # pragma: no cover - enterprise optional deps - check("enterprise_exports", False) - missing.append(f"enterprise:{type(exc).__name__}") - - pct = round(100.0 * sum(checks) / len(checks), 3) if checks else 0.0 - return {"export_completeness_pct": pct, "fields_missing": missing} diff --git a/benchmarks/public_benchmark.py b/benchmarks/public_benchmark.py deleted file mode 100644 index 15d66cd4..00000000 --- a/benchmarks/public_benchmark.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python -"""Public benchmark refresh: one reproducible report for sharing numbers. - -Runs the two existing harnesses in subprocesses and folds their output into a -single markdown + JSON report stamped with the freshdata version and the -environment it ran on (no numbers are ever committed; see docs/benchmarks.md): - -* ``benchmarks/bench_report.py`` — the scaling cases behind the - "Strategic-report scaling benchmarks" table (CSV ingest, profile, null-fill, - import time, peak memory), balanced vs aggressive; -* ``benchmarks/bench.py run`` + ``report`` — the per-fixture quality/speed table. - -Usage:: - - python benchmarks/public_benchmark.py # CI-safe sizes - python benchmarks/public_benchmark.py --scale full # harness defaults (10M rows) - python benchmarks/public_benchmark.py --skip-scaling --repeat 1 -""" - -from __future__ import annotations - -import argparse -import json -import os -import platform -import subprocess -import sys -from datetime import datetime, timezone -from importlib import metadata -from pathlib import Path -from typing import Any, Callable - -HERE = Path(__file__).resolve().parent -REPO_ROOT = HERE.parent -RESULTS_DIR = HERE / "results" -SCALING_JSON = RESULTS_DIR / "report_bench.json" - -#: CLI presets. ``ci`` fits a 2-core / 7 GB GitHub runner; ``full`` uses the -#: harness defaults (100 MB CSV, 10M-row null-fill) for a workstation. -SCALES: dict[str, dict[str, int | None]] = { - "ci": {"csv_mb": 50, "rows": 1_000_000}, - "full": {"csv_mb": 100, "rows": None}, -} - -_SCALING_LABELS = { - "csv_ingest": "CSV ingest + clean", - "profile": "Mixed-schema profile", - "nullfill": "Null-fill / flag", - "import_time": "Import time (`import freshdata`)", - "memory": "Peak memory of a full clean", -} - -Runner = Callable[[list[str]], None] - - -def _run(cmd: list[str]) -> None: - print("+", " ".join(cmd), flush=True) - subprocess.run(cmd, cwd=REPO_ROOT, check=True) # noqa: S603 - fixed argv - - -def _version(dist: str) -> str: - try: - return metadata.version(dist) - except metadata.PackageNotFoundError: - return "not installed" - - -def collect_environment() -> dict[str, Any]: - import freshdata # noqa: PLC0415 - report the version actually benchmarked - - ram = None - try: - import psutil # noqa: PLC0415 - - ram = round(psutil.virtual_memory().total / 1024**3, 1) - except ImportError: # pragma: no cover - psutil is a core dependency - pass - sha = os.environ.get("GITHUB_SHA") - if not sha: - try: - sha = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=REPO_ROOT, capture_output=True, - text=True, check=True, - ).stdout.strip() - except (OSError, subprocess.CalledProcessError): - sha = "unknown" - return { - "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), - "freshdata_version": freshdata.__version__, - "git_sha": sha, - "python": platform.python_version(), - "platform": platform.platform(), - "cpu_count": os.cpu_count(), - "ram_gb": ram, - "packages": { - name: _version(name) for name in ("pandas", "numpy", "pyarrow", "polars", "duckdb") - }, - } - - -# --------------------------------------------------------------------------- # -# harnesses -# --------------------------------------------------------------------------- # - -def run_scaling(*, csv_mb: int, rows: int | None, runner: Runner = _run) -> list[dict[str, Any]]: - cmd = [sys.executable, "benchmarks/bench_report.py", "all", "--mb", str(csv_mb)] - if rows is not None: - cmd += ["--rows", str(rows)] - runner(cmd) - return json.loads(SCALING_JSON.read_text(encoding="utf-8"))["results"] - - -def run_fixtures(*, repeat: int, size: int | None, runner: Runner = _run) -> str: - before = {p.name for p in RESULTS_DIR.iterdir()} if RESULTS_DIR.exists() else set() - cmd = [sys.executable, "benchmarks/bench.py", "run", "--repeat", str(repeat)] - if size is not None: - cmd += ["--size", str(size)] - runner(cmd) - new_runs = sorted( - p for p in RESULTS_DIR.iterdir() if p.is_dir() and p.name not in before - ) - if not new_runs: - raise RuntimeError("bench.py run did not create a results directory") - run_dir = new_runs[-1] - runner([sys.executable, "benchmarks/bench.py", "report", "--run-dir", str(run_dir)]) - return (run_dir / "report.md").read_text(encoding="utf-8") - - -# --------------------------------------------------------------------------- # -# rendering -# --------------------------------------------------------------------------- # - -def _cell(result: dict[str, Any] | None) -> str: - if result is None: - return "n/a" - if "import_sec_median" in result: - median, fastest = result["import_sec_median"], result.get("import_sec_min", 0) - return f"{median:.3f} s median ({fastest:.3f} s min)" - parts = [] - if "wall_sec" in result: - parts.append(f"{result['wall_sec']:.2f} s") - if result.get("peak_rss_mb") is not None: - parts.append(f"{result['peak_rss_mb']:.0f} MB peak RSS") - return ", ".join(parts) or "measured" - - -def _size(results: list[dict[str, Any]]) -> str: - first = results[0] - if first.get("csv_mb"): - return f"{first['csv_mb']} MB" - if first.get("rows"): - return f"{first['rows']:,} rows" - return "" - - -def scaling_table(results: list[dict[str, Any]]) -> list[str]: - lines = ["| Benchmark | Size | Balanced | Aggressive |", "|---|---|---|---|"] - for key, label in _SCALING_LABELS.items(): - rows = [r for r in results if r.get("benchmark") == key] - if not rows: - continue - by_strategy = {r.get("strategy"): r for r in rows} - balanced = by_strategy.get("balanced") or by_strategy.get("n/a") - aggressive = by_strategy.get("aggressive") - lines.append(f"| {label} | {_size(rows)} | {_cell(balanced)} | {_cell(aggressive)} |") - return lines - - -def render_markdown( - environment: dict[str, Any], - scaling: list[dict[str, Any]] | None, - fixture_report: str | None, -) -> str: - packages = ", ".join(f"{k} {v}" for k, v in environment["packages"].items()) - lines = [ - f"# FreshData public benchmark — freshdata {environment['freshdata_version']}", - "", - f"- **freshdata:** `{environment['freshdata_version']}` " - f"(commit `{environment['git_sha'][:12]}`)", - f"- **Generated:** {environment['generated_at']}", - f"- **Hardware:** {environment['cpu_count']} CPUs, {environment['ram_gb']} GB RAM", - f"- **Software:** Python {environment['python']} on {environment['platform']}; {packages}", - "", - "Numbers are environment-specific. Re-run `python benchmarks/public_benchmark.py` " - "on your own hardware before quoting them.", - "", - ] - if scaling: - lines += ["## Strategic-report scaling benchmarks", "", *scaling_table(scaling), ""] - if fixture_report: - body = fixture_report - if body.startswith("# "): - body = body.split("\n", 1)[1] # drop the harness report's own title - lines += ["## Fixture benchmarks (`benchmarks/bench.py`)", "", body.strip(), ""] - return "\n".join(lines) - - -def main(argv: list[str] | None = None, *, runner: Runner = _run) -> int: - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument("--scale", choices=sorted(SCALES), default="ci", - help="size preset for the scaling cases (default: ci)") - parser.add_argument("--csv-mb", type=int, default=None, help="override the CSV ingest size") - parser.add_argument("--rows", type=int, default=None, - help="override profile/null-fill/memory row counts") - parser.add_argument("--repeat", type=int, default=5, help="bench.py timing repeats") - parser.add_argument("--size", type=int, default=None, help="bench.py row-count override") - parser.add_argument("--skip-scaling", action="store_true") - parser.add_argument("--skip-fixtures", action="store_true") - parser.add_argument("--output-dir", default=str(RESULTS_DIR / "public"), - help="where public-benchmark.md/json are written") - args = parser.parse_args(argv) - if args.skip_scaling and args.skip_fixtures: - parser.error("nothing to run: both --skip-scaling and --skip-fixtures given") - - preset = SCALES[args.scale] - csv_mb = args.csv_mb if args.csv_mb is not None else preset["csv_mb"] - rows = args.rows if args.rows is not None else preset["rows"] - - environment = collect_environment() - scaling = None - if not args.skip_scaling: - scaling = run_scaling(csv_mb=int(csv_mb), rows=rows, runner=runner) - fixture_report = None if args.skip_fixtures else run_fixtures( - repeat=args.repeat, size=args.size, runner=runner) - - out_dir = Path(args.output_dir) - out_dir.mkdir(parents=True, exist_ok=True) - markdown = render_markdown(environment, scaling, fixture_report) - (out_dir / "public-benchmark.md").write_text(markdown, encoding="utf-8") - (out_dir / "public-benchmark.json").write_text(json.dumps({ - "environment": environment, - "options": {"scale": args.scale, "csv_mb": csv_mb, "rows": rows, - "repeat": args.repeat, "size": args.size}, - "scaling": scaling, - "fixture_report_markdown": fixture_report, - }, indent=2) + "\n", encoding="utf-8") - print(markdown) - print(f"written: {out_dir / 'public-benchmark.md'}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/truthbench/__init__.py b/benchmarks/truthbench/__init__.py deleted file mode 100644 index d2e16908..00000000 --- a/benchmarks/truthbench/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Deterministic oracle and result primitives for FreshData TruthBench.""" - -from .exact import TypedValue, canonical_json, encode_typed, exact_equal, stable_digest -from .models import ( - UNSET, - CaseExpectation, - DecisionRecord, - Disposition, - GateResult, - GoldCell, - RunResult, -) -from .privacy import Leak, PrivacySafeValue, SinkScanner, normalize_variants, redact_value - -__all__ = [ - "UNSET", - "CaseExpectation", - "DecisionRecord", - "Disposition", - "GateResult", - "GoldCell", - "RunResult", - "TypedValue", - "canonical_json", - "encode_typed", - "exact_equal", - "stable_digest", - "Leak", - "PrivacySafeValue", - "SinkScanner", - "normalize_variants", - "redact_value", -] diff --git a/benchmarks/truthbench/__main__.py b/benchmarks/truthbench/__main__.py deleted file mode 100644 index d0c19149..00000000 --- a/benchmarks/truthbench/__main__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""``python -m benchmarks.truthbench`` entry point.""" - -from __future__ import annotations - -import sys - -from .cli import main - -if __name__ == "__main__": - sys.exit(main()) diff --git a/benchmarks/truthbench/cli.py b/benchmarks/truthbench/cli.py deleted file mode 100644 index 750d7b24..00000000 --- a/benchmarks/truthbench/cli.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Command-line interface for TruthBench. - -``python -m benchmarks.truthbench run --profile release --backends -pandas,polars,duckdb --require-backends --repeats 2 --check`` is the official -release command. A partial run, missing adapter, missing backend, empty -record set, or infrastructure error exits nonzero. -""" - -from __future__ import annotations - -import argparse -import sys -from collections.abc import Sequence - -from .report import RESULTS_DIR -from .surfaces.backends import BackendUnavailableError - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="python -m benchmarks.truthbench", - description="TruthBench: absolute, fail-closed release verification.", - ) - commands = parser.add_subparsers(dest="command", required=True) - run = commands.add_parser("run", help="execute a verification run") - run.add_argument("--profile", choices=("release", "extended"), default="release") - run.add_argument( - "--backends", - default="pandas,polars,duckdb", - help="comma-separated required backends", - ) - run.add_argument( - "--require-backends", - action="store_true", - help="fail (never skip) when a required backend is unavailable", - ) - run.add_argument("--repeats", type=int, default=2) - run.add_argument( - "--domains", - default=None, - help="comma-separated fixture domains (default: all)", - ) - run.add_argument("--results-dir", default=str(RESULTS_DIR)) - run.add_argument( - "--check", - action="store_true", - help="exit nonzero unless every gate passes (the release gate)", - ) - run.add_argument( - "--check-regressions", - action="store_true", - help=( - "exit nonzero when a gate that passes in the committed " - "baseline.json fails now, or on any infrastructure error — the " - "PR ratchet. Known-red gates are reported loudly but do not " - "fail the run; the release workflow still enforces --check." - ), - ) - return parser - - -def main(argv: Sequence[str] | None = None) -> int: - args = build_parser().parse_args(argv) - if args.command != "run": # pragma: no cover - argparse enforces choices - return 2 - from .fixtures import DOMAINS - from .runner import TruthBenchRunError, run_release - - backends = tuple(b.strip() for b in args.backends.split(",") if b.strip()) - domains = ( - tuple(d.strip() for d in args.domains.split(",") if d.strip()) - if args.domains - else DOMAINS - ) - try: - outcome = run_release( - profile=args.profile, - backends=backends, - repeats=args.repeats, - require_backends=args.require_backends, - domains=domains, - results_dir=args.results_dir, - ) - except (TruthBenchRunError, BackendUnavailableError) as exc: - print(f"TRUTHBENCH INFRASTRUCTURE FAILURE: {exc}", file=sys.stderr) - return 2 - - payload = outcome.run.to_dict() - print(f"run id: {outcome.run.run_id}") - print(f"records: {len(outcome.run.records)}") - for gate in payload["gates"]: - status = "pass" if gate["passed"] else "FAIL" - print(f"gate {gate['name']}: {status} ({gate['failure_count']} failures)") - for note in outcome.baseline_notes: - print(f"baseline: {note}") - for name, path in outcome.artifacts.items(): - print(f"artifact: {name} -> {path}") - print(f"overall: {'PASS' if outcome.passed else 'FAIL'}") - if args.check and not outcome.passed: - return 1 - if args.check_regressions: - regressions = [ - note for note in outcome.baseline_notes if note.startswith("regression:") - ] - known_red = [gate for gate in payload["gates"] if not gate["passed"]] - for gate in known_red: - print( - f"KNOWN-RED (release-blocking, not a PR regression): " - f"{gate['name']} ({gate['failure_count']} failures)" - ) - if regressions: - for note in regressions: - print(f"REGRESSION: {note}", file=sys.stderr) - return 1 - return 0 - - -__all__ = ["build_parser", "main"] diff --git a/benchmarks/truthbench/determinism.py b/benchmarks/truthbench/determinism.py deleted file mode 100644 index 43858868..00000000 --- a/benchmarks/truthbench/determinism.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Normalized decision hashing and repeat-consistency annotation. - -The hash covers every decision-bearing ``DecisionRecord`` field. Only the -approved telemetry fields — identifiers regenerated per run and the repeat -bookkeeping itself — may ever be excluded, and the exclusion list is closed: -asking to exclude anything else raises instead of silently narrowing the -comparison. -""" - -from __future__ import annotations - -import dataclasses -import hashlib -from collections.abc import Iterable, Sequence - -from .exact import canonical_json -from .models import DecisionRecord - -#: Fields regenerated per run/repeat that carry no decision content. This set -#: is the *only* permissible exclusion surface. -APPROVED_TELEMETRY: frozenset[str] = frozenset( - { - "record_id", - "run_id", - "repeat", - "repeat_hash", - "repeat_consistent", - "normalized_decision_hash", - } -) - -#: Every serialized field that participates in the normalized decision hash. -DECISION_FIELDS: tuple[str, ...] = tuple( - sorted( - set(DecisionRecord.__dataclass_fields__) - - APPROVED_TELEMETRY - ) -) - - -def decision_hash( - record: DecisionRecord, *, exclude: Iterable[str] = APPROVED_TELEMETRY -) -> str: - """Stable SHA-256 over the record's decision-bearing content. - - ``exclude`` may only name approved telemetry fields; any attempt to drop a - decision-bearing field (outputs, dispositions, confidence, rationale, - audit evidence, trust, backend disclosure, ...) is rejected. - """ - - excluded = frozenset(exclude) - illegal = excluded - APPROVED_TELEMETRY - if illegal: - raise ValueError( - "decision-bearing fields cannot be excluded from the normalized " - f"decision hash: {sorted(illegal)}" - ) - payload = record.to_dict() - body = {key: value for key, value in payload.items() if key not in excluded} - return hashlib.sha256(canonical_json(body).encode("utf-8")).hexdigest() - - -def annotate_repeats( - records: Sequence[DecisionRecord], *, expected_repeats: Sequence[int] -) -> tuple[DecisionRecord, ...]: - """Attach ``normalized_decision_hash``/``repeat_hash``/``repeat_consistent``. - - Records are grouped by ``(surface, requested_backend, cell_id)``. A group - is consistent only when it contains exactly one record per expected repeat - and every repeat hashes identically. Groups with missing repeats or any - unexplained difference are annotated ``repeat_consistent=False`` so the - ``default_nondeterminism`` gate fails closed. - """ - - expected = tuple(sorted({int(r) for r in expected_repeats})) - if len(expected) < 2: - raise ValueError("determinism verification requires at least two repeats") - groups: dict[tuple[str, str | None, str], list[DecisionRecord]] = {} - for record in records: - groups.setdefault( - (record.surface, record.requested_backend, record.cell_id), [] - ).append(record) - - annotated: list[DecisionRecord] = [] - for members in groups.values(): - hashes = {record.repeat: decision_hash(record) for record in members} - repeats_seen = tuple(sorted(hashes)) - complete = repeats_seen == expected and len(members) == len(expected) - consistent = complete and len(set(hashes.values())) == 1 - group_hash = hashlib.sha256( - canonical_json([hashes[r] for r in repeats_seen]).encode("utf-8") - ).hexdigest() - for record in members: - annotated.append( - dataclasses.replace( - record, - normalized_decision_hash=hashes[record.repeat], - repeat_hash=group_hash, - repeat_consistent=consistent, - ) - ) - return tuple(annotated) - - -__all__ = [ - "APPROVED_TELEMETRY", - "DECISION_FIELDS", - "annotate_repeats", - "decision_hash", -] diff --git a/benchmarks/truthbench/exact.py b/benchmarks/truthbench/exact.py deleted file mode 100644 index 211f0c40..00000000 --- a/benchmarks/truthbench/exact.py +++ /dev/null @@ -1,340 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac -import json -import math -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import date, datetime, time, timedelta -from decimal import Decimal -from enum import Enum -from typing import Any - -import numpy as np -import pandas as pd - -JsonValue = Any - - -@dataclass(frozen=True) -class TypedValue: - """A scalar encoded without erasing its Python/pandas representation.""" - - kind: str - value: JsonValue - dtype: str | None - display: str - digest: str | None = None - redacted: bool = False - schema_version: int = field(default=1, init=False) - - @property - def type_label(self) -> str: - if self.dtype is None: - return self.kind - return f"{self.kind}[{self.dtype}]" - - def to_dict(self) -> dict[str, JsonValue]: - return { - "schema_version": self.schema_version, - "type": self.kind, - "dtype": self.dtype, - "value": self.value, - "display": self.display, - "digest": self.digest, - "redacted": self.redacted, - } - - @classmethod - def from_dict(cls, payload: Mapping[str, JsonValue]) -> TypedValue: - if payload.get("schema_version") != 1: - raise ValueError("unsupported typed value schema version") - return cls( - kind=str(payload["type"]), - dtype=None if payload["dtype"] is None else str(payload["dtype"]), - value=payload["value"], - display=str(payload["display"]), - digest=None if payload["digest"] is None else str(payload["digest"]), - redacted=bool(payload["redacted"]), - ) - - -def _dtype_name(dtype: Any, *, sensitive: bool) -> str | None: - if dtype is None: - return None - normalized = pd.api.types.pandas_dtype(dtype) - if isinstance(normalized, pd.CategoricalDtype): - categories = normalized.categories - if sensitive: - encoded_categories: JsonValue = {"count": len(categories)} - else: - encoded_categories = [encode_typed(value).to_dict() for value in categories] - category_json = canonical_json(encoded_categories) - ordered = "true" if normalized.ordered else "false" - return f"category[ordered={ordered};categories={category_json}]" - if isinstance(normalized, pd.StringDtype): - return f"string[{normalized.storage}]" - return str(normalized) - - -def _timezone_identity(value: Any) -> str | None: - if value is None: - return None - key = getattr(value, "key", None) - if isinstance(key, str): - return key - return str(value) - - -def _encode_scalar(value: Any) -> tuple[str, JsonValue, str]: - if value is pd.NA: - return "pandas.NA", None, "" - if value is pd.NaT: - return "pandas.NaT", None, "NaT" - if value is None: - return "python.none", None, "None" - - if isinstance(value, np.bool_): - scalar = bool(value) - return "numpy.bool_", scalar, str(scalar) - if isinstance(value, bool): - return "python.bool", value, str(value) - - if isinstance(value, np.integer): - return f"numpy.{value.dtype.name}", str(value), str(value) - if isinstance(value, int): - return "python.int", str(value), str(value) - - if isinstance(value, np.floating): - kind = f"numpy.{value.dtype.name}" - scalar = float(value) - if math.isnan(scalar): - return f"{kind}.nan", None, "NaN" - if math.isinf(scalar): - sign = "+" if scalar > 0 else "-" - return f"{kind}.infinity", sign, f"{sign}Infinity" - representation = repr(value.item()) - return kind, representation, representation - if isinstance(value, float): - if math.isnan(value): - return "python.float.nan", None, "NaN" - if math.isinf(value): - sign = "+" if value > 0 else "-" - return "python.float.infinity", sign, f"{sign}Infinity" - representation = repr(value) - return "python.float", representation, representation - - if isinstance(value, np.str_): - scalar = str(value) - return "numpy.str_", scalar, scalar - if isinstance(value, str): - return "python.str", value, value - - if isinstance(value, np.bytes_): - scalar = bytes(value).hex() - return "numpy.bytes_", scalar, repr(bytes(value)) - if isinstance(value, bytes): - return "python.bytes", value.hex(), repr(value) - - if isinstance(value, pd.Timestamp): - rendered = value.isoformat() - encoded = { - "iso": rendered, - "timezone": _timezone_identity(value.tz), - } - return "pandas.Timestamp", encoded, rendered - if isinstance(value, datetime): - rendered = value.isoformat() - encoded = {"iso": rendered, "timezone": _timezone_identity(value.tzinfo)} - return "python.datetime", encoded, rendered - if isinstance(value, date): - rendered = value.isoformat() - return "python.date", rendered, rendered - if isinstance(value, time): - rendered = value.isoformat() - encoded = {"iso": rendered, "timezone": _timezone_identity(value.tzinfo)} - return "python.time", encoded, rendered - - if isinstance(value, pd.Timedelta): - rendered = value.isoformat() - return "pandas.Timedelta", rendered, rendered - if isinstance(value, np.datetime64): - return f"numpy.{value.dtype}", str(value), str(value) - if isinstance(value, np.timedelta64): - return f"numpy.{value.dtype}", str(value), str(value) - if isinstance(value, timedelta): - rendered = repr(value) - return "python.timedelta", rendered, rendered - if isinstance(value, Decimal): - rendered = str(value) - return "python.Decimal", rendered, rendered - - raise TypeError(f"unsupported scalar type: {type(value).__name__}") - - -def canonical_scalar(value: Any) -> Any: - """Unwrap a numpy scalar to its Python equivalent for evidence encoding. - - Values read back out of a pandas frame arrive as numpy scalars - (``np.float64``, ``np.int64``, ...) while fixture oracles and semantic - repairs are authored as Python scalars. Every evidence path — record - inputs, record outputs, and the gate-side fixture snapshot — must encode - through this same canonicalization or identical values would compare - unequal purely by container type. Datetime/timedelta scalars are left - alone: ``.item()`` can degrade them to integers for exotic units, and - ``frame.at`` never yields them raw (pandas returns ``Timestamp``). - """ - if isinstance(value, np.generic) and not isinstance( - value, (np.datetime64, np.timedelta64) - ): - return value.item() - return value - - -def encode_typed( - value: Any, - *, - dtype: Any = None, - sensitive: bool = False, - digest_key: bytes | None = None, -) -> TypedValue: - kind, encoded, display = _encode_scalar(value) - dtype_name = _dtype_name(dtype, sensitive=sensitive) - if not sensitive: - return TypedValue(kind=kind, value=encoded, dtype=dtype_name, display=display) - if digest_key is None: - raise ValueError("digest_key is required for sensitive values") - raw = TypedValue(kind=kind, value=encoded, dtype=dtype_name, display=display) - digest = hmac.new( - digest_key, - canonical_json(raw).encode("utf-8"), - hashlib.sha256, - ).hexdigest() - return TypedValue( - kind=kind, - value=None, - dtype=dtype_name, - display="[REDACTED]", - digest=digest, - redacted=True, - ) - - -def exact_equal( - left: Any, - right: Any, - *, - left_dtype: Any = None, - right_dtype: Any = None, -) -> bool: - return encode_typed(left, dtype=left_dtype) == encode_typed(right, dtype=right_dtype) - - -_CANONICAL_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") -_LEADING_ZERO_NUMERIC = re.compile(r"^[+-]?0\d") -_NUMERIC_KIND_PREFIXES = ("python.int", "numpy.int", "python.float", "numpy.float") - - -def equivalent_after_type_normalization(actual: TypedValue, source: TypedValue) -> bool: - """True when *actual* is the value-preserving typed form of *source*. - - A disclosed dtype normalization keeps the value and changes only its - representation. This is deliberately narrow so it can never excuse a - semantic interpretation: - - - a canonical dash-ISO date string may become the equal *naive* timestamp - (ambiguous ``01/02/2025`` forms, partial dates, and timezone changes - never qualify); - - a plain numeric string may become the equal number (leading-zero codes - like ``"007"`` never qualify); - - vocabulary mappings (boolean words, category synonyms) never qualify. - """ - - if actual == source: - return True - if actual.redacted or source.redacted: - return False - if source.kind not in ("python.str", "numpy.str_") or not isinstance(source.value, str): - return False - text = source.value.strip() - if actual.kind in ("pandas.Timestamp", "python.datetime") and isinstance( - actual.value, Mapping - ): - if not _CANONICAL_ISO_DATE.match(text): - return False - if actual.value.get("timezone") is not None: - return False - return actual.value.get("iso") == f"{text}T00:00:00" - if actual.kind == "python.date": - return bool(_CANONICAL_ISO_DATE.match(text)) and actual.value == text - if actual.kind.startswith(_NUMERIC_KIND_PREFIXES) and not actual.kind.endswith( - (".nan", ".infinity") - ): - if _LEADING_ZERO_NUMERIC.match(text): - return False - try: - return float(text) == float(actual.value) # type: ignore[arg-type] - except (TypeError, ValueError): - return False - return False - - -def numeric_value_equal(actual: TypedValue, source: TypedValue) -> bool: - """True when a numeric value is unchanged across a widening/narrowing of - its numeric type (float ``1000.0`` vs int ``1000``), booleans excluded.""" - - if actual.redacted or source.redacted: - return False - for value in (actual, source): - if not value.kind.startswith(_NUMERIC_KIND_PREFIXES) or value.kind.endswith( - (".nan", ".infinity") - ): - return False - try: - return float(actual.value) == float(source.value) # type: ignore[arg-type] - except (TypeError, ValueError): - return False - - -def _json_safe(value: Any, path: str = "$") -> JsonValue: - if isinstance(value, TypedValue): - return _json_safe(value.to_dict(), path) - if isinstance(value, Enum): - return _json_safe(value.value, path) - if value is None or isinstance(value, (bool, str)): - return value - if isinstance(value, int) and not isinstance(value, bool): - return value - if isinstance(value, float): - if not math.isfinite(value): - raise ValueError(f"{path} contains a non-finite number") - return value - if isinstance(value, Mapping): - result: dict[str, JsonValue] = {} - for key, item in value.items(): - if not isinstance(key, str): - raise TypeError(f"{path} contains an unsupported non-string key") - result[key] = _json_safe(item, f"{path}.{key}") - return result - if isinstance(value, (list, tuple)): - return [_json_safe(item, f"{path}[{index}]") for index, item in enumerate(value)] - raise TypeError(f"{path} contains unsupported type: {type(value).__name__}") - - -def canonical_json(value: Any) -> str: - return json.dumps( - _json_safe(value), - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - allow_nan=False, - ) - - -def stable_digest(value: Any, *, key: bytes) -> str: - if not isinstance(key, bytes): - raise TypeError("key must be bytes") - encoded = canonical_json(encode_typed(value)).encode("utf-8") - return hmac.new(key, encoded, hashlib.sha256).hexdigest() diff --git a/benchmarks/truthbench/fixtures/__init__.py b/benchmarks/truthbench/fixtures/__init__.py deleted file mode 100644 index 8ed5936d..00000000 --- a/benchmarks/truthbench/fixtures/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Fixture registry for the independent TruthBench oracle.""" - -from __future__ import annotations - -from typing import Callable - -from . import crm, education, finance, government, healthcare, insurance, logistics, retail -from .base import FixtureBuilder, FixtureError, TruthFixture - -_BUILDERS: dict[str, Callable[[int], TruthFixture]] = {} -DOMAINS: tuple[str, ...] = ( - "crm", - "education", - "finance", - "government", - "healthcare", - "insurance", - "logistics", - "retail", -) - - -def register_fixture(domain: str, builder: Callable[[int], TruthFixture]) -> None: - if not isinstance(domain, str) or not domain: - raise ValueError("fixture domain must be a non-empty string") - if domain in _BUILDERS: - raise ValueError(f"fixture domain already registered: {domain}") - _BUILDERS[domain] = builder - - -register_fixture("crm", crm.build) -register_fixture("education", education.build) -register_fixture("finance", finance.build) -register_fixture("government", government.build) -register_fixture("healthcare", healthcare.build) -register_fixture("insurance", insurance.build) -register_fixture("logistics", logistics.build) -register_fixture("retail", retail.build) - - -def build_fixture(domain: str, seed: int = 1729) -> TruthFixture: - try: - builder = _BUILDERS[domain] - except KeyError as exc: - raise FixtureError(f"unknown fixture domain: {domain!r}") from exc - fixture = builder(int(seed)) - fixture.validate() - return fixture - - -__all__ = [ - "DOMAINS", - "FixtureBuilder", - "FixtureError", - "TruthFixture", - "build_fixture", - "register_fixture", -] diff --git a/benchmarks/truthbench/fixtures/base.py b/benchmarks/truthbench/fixtures/base.py deleted file mode 100644 index 758a9027..00000000 --- a/benchmarks/truthbench/fixtures/base.py +++ /dev/null @@ -1,321 +0,0 @@ -"""Independent, deterministic fixture construction primitives.""" - -from __future__ import annotations - -import hashlib -import re -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from numbers import Integral -from typing import Any - -import pandas as pd - -from ..exact import canonical_json, encode_typed -from ..models import UNSET, CaseExpectation, Disposition, GoldCell - - -class FixtureError(ValueError): - """A fixture violates the TruthBench construction contract.""" - - -_HASH_KEY = b"truthbench-fixture-hash-v1" -_SYNTHETIC_EMAIL = re.compile(r"[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+\.invalid") -_SYNTHETIC_ID = re.compile(r"TB-[A-Za-z0-9-]+") -_SYNTHETIC_PHONE = re.compile(r"555[- .]?01[0-9]{2}") - - -def _synthetic_pii(value: Any) -> bool: - if isinstance(value, Integral) and not isinstance(value, bool): - value = str(value) - if not isinstance(value, str): - return False - return any( - pattern.fullmatch(value) for pattern in (_SYNTHETIC_EMAIL, _SYNTHETIC_ID, _SYNTHETIC_PHONE) - ) - - -def _typed(value: Any, dtype: Any = None, *, sensitive: bool = False) -> dict[str, Any]: - return encode_typed( - value, - dtype=dtype, - sensitive=sensitive, - digest_key=_HASH_KEY if sensitive else None, - ).to_dict() - - -@dataclass(frozen=True) -class TruthFixture: - """A pristine/adversarial pair and complete cell-level oracle.""" - - version: str - domain: str - seed: int - pristine: pd.DataFrame - frame: pd.DataFrame - cells: tuple[GoldCell, ...] - schema: Mapping[str, Any] - policy: Mapping[str, Any] - protected_columns: tuple[str, ...] - pii_canaries: Mapping[str, Any] - row_cases: tuple[CaseExpectation, ...] - schema_cases: tuple[CaseExpectation, ...] - fixture_hash: str - - @property - def adversarial(self) -> pd.DataFrame: - return self.frame - - @property - def pristine_frame(self) -> pd.DataFrame: - return self.pristine - - @property - def adversarial_frame(self) -> pd.DataFrame: - return self.frame - - def validate(self) -> None: - if not self.version or not self.domain: - raise FixtureError("fixture version and domain are required") - if not self.frame.index.is_unique or self.frame.index.hasnans: - raise FixtureError("frame row IDs must be unique and non-missing") - keys = [ - (str(row), str(column)) for row in self.frame.index for column in self.frame.columns - ] - labels = [(cell.row_id, cell.column) for cell in self.cells] - if len(labels) != len(set(labels)) or set(labels) != set(keys): - raise FixtureError("fixture must have exactly one label per physical cell") - cell_ids = [cell.cell_id for cell in self.cells] - if len(cell_ids) != len(set(cell_ids)): - raise FixtureError("fixture contains duplicate cell IDs") - for cell in self.cells: - if cell.fixture_version != self.version or cell.domain != self.domain: - raise FixtureError("cell identity does not match fixture") - expected_id = f"{self.version}:{self.domain}:{cell.row_id}:{cell.column}" - if cell.cell_id != expected_id: - raise FixtureError(f"cell {cell.cell_id} has an invalid stable ID") - if cell.disposition is Disposition.REPAIR and cell.expected_output is None: - raise FixtureError(f"repair cell {cell.cell_id} has no expected output") - if cell.sensitive and (not cell.canary_id or cell.canary_id not in self.pii_canaries): - raise FixtureError(f"sensitive cell {cell.cell_id} has no canary") - for case in (*self.row_cases, *self.schema_cases): - if case.domain != self.domain or case.fixture_version != self.version: - raise FixtureError("case identity does not match fixture") - if any(case.kind != "row" for case in self.row_cases): - raise FixtureError("row_cases must contain row expectations") - if any(case.kind != "schema" for case in self.schema_cases): - raise FixtureError("schema_cases must contain schema expectations") - if self.fixture_hash: - payload = self.to_dict() - payload.pop("fixture_hash", None) - expected_hash = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() - if self.fixture_hash != expected_hash: - raise FixtureError("fixture hash does not match fixture contents") - - def to_dict(self) -> dict[str, Any]: - """Return a privacy-safe, JSON-ready representation.""" - - def frame_payload(frame: pd.DataFrame) -> list[dict[str, Any]]: - by_key = {(c.row_id, c.column): c for c in self.cells} - rows: list[dict[str, Any]] = [] - for row in frame.index: - row_id = str(row) - values = { - str(column): _typed( - frame.at[row, column], - frame[column].dtype, - sensitive=by_key[(row_id, str(column))].sensitive, - ) - for column in frame.columns - } - rows.append({"row_id": row_id, "values": values}) - return rows - - canaries = { - key: _typed(value, sensitive=True) for key, value in sorted(self.pii_canaries.items()) - } - return { - "version": self.version, - "domain": self.domain, - "seed": self.seed, - "pristine": frame_payload(self.pristine), - "frame": frame_payload(self.frame), - "cells": [cell.to_dict() for cell in self.cells], - "schema": dict(self.schema), - "policy": dict(self.policy), - "protected_columns": list(self.protected_columns), - "pii_canaries": canaries, - "row_cases": [case.to_dict() for case in self.row_cases], - "schema_cases": [case.to_dict() for case in self.schema_cases], - "fixture_hash": self.fixture_hash, - } - - -class FixtureBuilder: - """Build a fixture while maintaining a complete, atomic label matrix.""" - - def __init__( - self, - version: str, - domain: str, - frame: pd.DataFrame, - *, - seed: int = 1729, - schema: Mapping[str, Any] | None = None, - policy: Mapping[str, Any] | None = None, - protected_columns: tuple[str, ...] | list[str] = (), - pii_canaries: Mapping[str, Any] | None = None, - row_cases: tuple[CaseExpectation, ...] | list[CaseExpectation] = (), - schema_cases: tuple[CaseExpectation, ...] | list[CaseExpectation] = (), - ) -> None: - if not isinstance(frame, pd.DataFrame): - raise TypeError("frame must be a pandas DataFrame") - if frame.empty or frame.columns.empty: - raise FixtureError("frame must contain at least one row and column") - if not version or not domain: - raise FixtureError("version and domain are required") - if frame.index.hasnans or not frame.index.is_unique: - raise FixtureError("row IDs must be unique and non-missing") - row_ids = [str(row) for row in frame.index] - if any(not row_id for row_id in row_ids) or len(row_ids) != len(set(row_ids)): - raise FixtureError("row IDs must be unique and non-missing") - if any(not isinstance(column, str) or not column for column in frame.columns): - raise FixtureError("column names must be non-empty strings") - if len(set(frame.columns)) != len(frame.columns): - raise FixtureError("column names must be unique") - - self.version = version - self.domain = domain - self.seed = seed - self.pristine = frame.copy(deep=True) - self.pristine.index = row_ids - self.frame = frame.copy(deep=True) - self.frame.index = row_ids - self.schema = dict( - schema - or { - "columns": list(frame.columns), - "dtypes": {str(c): str(frame[c].dtype) for c in frame.columns}, - } - ) - self.policy = dict(policy or {}) - self.protected_columns = tuple(protected_columns) - unknown_protected = set(self.protected_columns) - set(frame.columns) - if unknown_protected: - raise FixtureError(f"unknown protected columns: {sorted(unknown_protected)}") - self._labels = { - (row_id, str(column)): GoldCell.create( - version, domain, row_id, str(column), Disposition.PRESERVE, family="background" - ) - for row_id in row_ids - for column in frame.columns - } - self._canaries: dict[str, Any] = dict(pii_canaries or {}) - if any(not _synthetic_pii(value) for value in self._canaries.values()): - raise FixtureError("PII canaries must use synthetic domains") - self._row_cases = list(row_cases) - self._schema_cases = list(schema_cases) - - def inject( - self, - row_id: str, - column: str, - value: Any, - disposition: Disposition | str, - *, - expected: Any = UNSET, - expected_dtype: Any = None, - family: str, - sensitive: bool = False, - ) -> None: - key = (str(row_id), str(column)) - if key not in self._labels: - raise FixtureError(f"unknown cell {key}") - if not isinstance(family, str) or not family: - raise FixtureError("family is required") - try: - parsed = Disposition(disposition) - except ValueError as exc: - raise FixtureError(f"unknown disposition: {disposition!r}") from exc - if parsed is Disposition.REPAIR and expected is UNSET: - raise FixtureError("repair cells require an expected output") - if sensitive and not _synthetic_pii(value): - raise FixtureError("sensitive values must use synthetic PII domains") - - canary_id = None - if sensitive: - canary_id = f"{self.domain}-{key[0]}-{key[1]}" - candidate = GoldCell.create( - self.version, - self.domain, - key[0], - key[1], - parsed, - expected_output=expected, - expected_dtype=expected_dtype, - family=family, - sensitive=sensitive, - canary_id=canary_id, - ) - previous = self._labels[key] - if ( - previous.disposition is Disposition.REPAIR - and candidate.disposition is Disposition.REPAIR - and previous.expected_output != candidate.expected_output - ): - raise FixtureError(f"contradictory repair output for cell {key}") - - # All validation above happens before either mutable structure changes. - if previous.canary_id and previous.canary_id != canary_id: - self._canaries.pop(previous.canary_id, None) - if canary_id: - self._canaries[canary_id] = value - # A corruption may intentionally change a column's scalar type. Cast - # to object first so pandas does not coerce or emit a FutureWarning. - self.frame[key[1]] = self.frame[key[1]].astype(object) - self.frame.at[key[0], key[1]] = value - self._labels[key] = candidate - - def add_row_case(self, name: str, disposition: Disposition | str, **kwargs: Any) -> None: - self._row_cases.append( - CaseExpectation.create(self.version, self.domain, "row", name, disposition, **kwargs) - ) - - def add_schema_case(self, name: str, disposition: Disposition | str, **kwargs: Any) -> None: - self._schema_cases.append( - CaseExpectation.create( - self.version, self.domain, "schema", name, disposition, **kwargs - ) - ) - - def build(self) -> TruthFixture: - cells = tuple( - self._labels[(str(row), str(column))] - for row in self.frame.index - for column in self.frame.columns - ) - fixture = TruthFixture( - version=self.version, - domain=self.domain, - seed=self.seed, - pristine=self.pristine.copy(deep=True), - frame=self.frame.copy(deep=True), - cells=cells, - schema=dict(self.schema), - policy=dict(self.policy), - protected_columns=self.protected_columns, - pii_canaries=dict(self._canaries), - row_cases=tuple(self._row_cases), - schema_cases=tuple(self._schema_cases), - fixture_hash="", - ) - fixture.validate() - payload = fixture.to_dict() - payload.pop("fixture_hash", None) - digest = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() - result = TruthFixture(**{**fixture.__dict__, "fixture_hash": digest}) - result.validate() - return result - - -BuilderFactory = Callable[[int], TruthFixture] diff --git a/benchmarks/truthbench/fixtures/crm.py b/benchmarks/truthbench/fixtures/crm.py deleted file mode 100644 index ebf58cc1..00000000 --- a/benchmarks/truthbench/fixtures/crm.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Deterministic CRM gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"crm-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "customer_id": [f"TB-CRM-{i:04d}" for i in range(1, 17)], - "first_name": [ - "Ana", - "Jose\u0301", - "Miyuki", - "Zoë", - "Liam", - "Nia", - "Omar", - "Riya", - "Noel", - "Saanvi", - "Evan", - "Ivy", - "Kai", - "Mara", - "Theo", - "Uma", - ], - "last_name": [ - "Stone", - "García", - "Chen", - "Müller", - "Jones", - "Patel", - "Khan", - "Roy", - "Smith", - "Das", - "Lee", - "Brown", - "Cruz", - "Mori", - "Cole", - "Singh", - ], - "email": [ - "ana@example.invalid", - "jose@example.invalid", - "miyuki@example.invalid", - "zoe@example.invalid", - "liam@example.invalid", - "nia@example.invalid", - "omar@example.invalid", - "riya@example.invalid", - "noel@example.invalid", - "saanvi@example.invalid", - "evan@example.invalid", - "ivy@example.invalid", - "kai@example.invalid", - "mara@example.invalid", - "theo@example.invalid", - "uma@example.invalid", - ], - "phone": ["555-0101"] * 16, - "country": [ - "US", - "DE", - "IN", - "GB", - "US", - "CA", - "AU", - "JP", - "BR", - "FR", - "US", - "IN", - "MX", - "ES", - "ZA", - "SG", - ], - "language": [ - "en", - "de", - "hi", - "en", - "en", - "fr", - "en", - "ja", - "pt", - "fr", - "en", - "hi", - "es", - "es", - "en", - "en", - ], - "signup_date": ["2026-01-15"] * 16, - "lifecycle": [ - "lead", - "prospect", - "customer", - "churned", - "lead", - "prospect", - "customer", - "lead", - "prospect", - "customer", - "lead", - "prospect", - "customer", - "lead", - "prospect", - "customer", - ], - "lead_source": [ - "web", - "referral", - "apple", - "partner", - "web", - "event", - "apple", - "search", - "web", - "partner", - "event", - "web", - "referral", - "search", - "web", - "event", - ], - "employer": [ - "Acme", - "Globex", - "Apple", - "Northstar", - "Acme", - "Cedar", - "Apple", - "Delta", - "Evergreen", - "Futura", - "Globex", - "Helios", - "Apple", - "Ion", - "Juniper", - "Kappa", - ], - "notes": ["ordinary"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "crm", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "supported_countries": ["US", "DE", "IN", "GB"], - "supported_languages": ["en", "de", "hi", "ja"], - "protected_customer_id_policy": "preserve", - }, - protected_columns=("customer_id",), - ) - builder.inject( - "crm-02", - "first_name", - "Jose\u0301", - Disposition.REPAIR, - expected="José", - family="combining-unicode-name", - ) - builder.inject( - "crm-03", - "email", - "miyuki@example.invalid", - Disposition.PRESERVE, - family="reserved-invalid-email", - sensitive=True, - ) - builder.inject( - "crm-04", - "phone", - "555 0101", - Disposition.REPAIR, - expected="555-0101", - family="spaced-phone", - ) - builder.inject("crm-05", "country", "US/CA", Disposition.REVIEW, family="ambiguous-country") - builder.inject("crm-06", "language", "en/fr", Disposition.REVIEW, family="ambiguous-language") - builder.inject( - "crm-07", "signup_date", "01/02/2025", Disposition.REVIEW, family="ambiguous-date" - ) - builder.inject( - "crm-08", "lifecycle", "lead|churned", Disposition.REVIEW, family="lifecycle-contradiction" - ) - builder.inject( - "crm-09", - "email", - "TB-CRM-ZW-EMAIL", - Disposition.FLAG, - family="zero-width-email", - sensitive=True, - ) - builder.inject( - "crm-11", - "email", - "lead\u200b@example.invalid", - Disposition.FLAG, - family="zero-width-email", - ) - builder.inject( - "crm-10", "notes", "TB-CRM-SSN-0001", Disposition.FLAG, family="hidden-ssn", sensitive=True - ) - builder.inject( - "crm-03", "lead_source", "apple", Disposition.PRESERVE, family="apple-lead-source" - ) - builder.inject("crm-03", "employer", "Apple", Disposition.PRESERVE, family="apple-employer") - builder.inject( - "crm-16", - "customer_id", - "TB-CRM-CUSTOMER-TAIL", - Disposition.PRESERVE, - family="protected-customer-id", - sensitive=True, - ) - builder.add_row_case( - "exact-duplicate-crm-02-crm-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-crm-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case("type-drifted-phone", Disposition.REVIEW, family="type-drifted-column") - return builder.build() diff --git a/benchmarks/truthbench/fixtures/education.py b/benchmarks/truthbench/fixtures/education.py deleted file mode 100644 index a963a60b..00000000 --- a/benchmarks/truthbench/fixtures/education.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Deterministic education gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"edu-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "student_id": [ - "000123", - "000124", - "000125", - "000126", - "000127", - "000128", - "000129", - "000130", - "000131", - "000132", - "000133", - "000134", - "000135", - "000136", - "000137", - "000138", - ], - "grade_letter": [ - "A", - "A-", - "B+", - "B", - "C+", - "C", - "B-", - "A", - "A", - "B", - "C", - "D", - "A", - "B+", - "C", - "A", - ], - "score_percent": [ - "95", - "90", - "87.5", - "82", - "78", - "72", - "68", - "100", - "0", - "88", - "76", - "61", - "93", - "86", - "74", - "96", - ], - "gpa": [ - 4.0, - 3.7, - 3.3, - 3.0, - 2.7, - 2.3, - 2.0, - 4.0, - 0.0, - 3.3, - 2.7, - 1.7, - 3.9, - 3.3, - 2.3, - 4.0, - ], - "school_year": ["2025-2026"] * 16, - "enrollment_date": ["2025-08-15"] * 16, - "completion_date": ["2026-05-30"] * 16, - "guardian_email": ["guardian1@example.invalid"] * 16, - "guardian_phone": ["555-0101"] * 16, - "ferpa_notes": ["routine"] * 16, - "grade_policy": ["A>=90;B>=80;C>=70"] * 16, - "language": ["en"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "education", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "grade_scales": ["letter", "percentage", "GPA"], - "protected_grade_policy": "preserve", - }, - protected_columns=("student_id", "grade_letter"), - ) - builder.inject( - "edu-01", "student_id", "000123", Disposition.PRESERVE, family="leading-zero-student-id" - ) - builder.inject( - "edu-02", "grade_letter", "A-", Disposition.PRESERVE, family="letter-grade-scale" - ) - builder.inject("edu-03", "score_percent", 0, Disposition.PRESERVE, family="zero-score") - builder.inject( - "edu-04", "school_year", "2025/26", Disposition.REVIEW, family="school-year-ambiguity" - ) - builder.inject( - "edu-05", - "enrollment_date", - "2026-02-01", - Disposition.REVIEW, - family="enrollment-date-ordering", - ) - builder.inject( - "edu-06", - "completion_date", - "2025-01-01", - Disposition.REVIEW, - family="enrollment-date-ordering", - ) - builder.inject( - "edu-07", - "score_percent", - "95%", - Disposition.REPAIR, - expected=95.0, - expected_dtype="float64", - family="percentage-scale", - ) - builder.inject("edu-08", "gpa", 4.0, Disposition.PRESERVE, family="gpa-scale") - builder.inject( - "edu-09", - "guardian_email", - "guardian@example.invalid", - Disposition.FLAG, - family="guardian-contact-pii", - sensitive=True, - ) - builder.inject( - "edu-10", - "guardian_phone", - "555-0110", - Disposition.FLAG, - family="guardian-contact-pii", - sensitive=True, - ) - builder.inject( - "edu-10", - "ferpa_notes", - "TB-EDU-FERPA-0001", - Disposition.FLAG, - family="ferpa-sensitive-notes", - sensitive=True, - ) - builder.inject( - "edu-16", - "grade_letter", - "A", - Disposition.PRESERVE, - family="protected-grade-policy-conflict", - ) - builder.add_row_case( - "exact-duplicate-edu-02-edu-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-edu-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case("type-drifted-score", Disposition.REVIEW, family="type-drifted-column") - return builder.build() diff --git a/benchmarks/truthbench/fixtures/finance.py b/benchmarks/truthbench/fixtures/finance.py deleted file mode 100644 index 28499e79..00000000 --- a/benchmarks/truthbench/fixtures/finance.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Deterministic finance gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"fin-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "account_id": [f"TB-FIN-{i:04d}" for i in range(1, 17)], - "asset": [ - "bond", - "fund", - "apple", - "stock", - "cash", - "fx", - "etf", - "bond", - "fund", - "stock", - "cash", - "fx", - "etf", - "bond", - "fund", - "stock", - ], - "company": [ - "Acme Bank", - "Northstar", - "Acme", - "Apple", - "Cedar", - "Delta", - "Evergreen", - "Futura", - "Globex", - "Helios", - "Ion", - "Juniper", - "Kappa", - "Lumen", - "Mosaic", - "Nadir", - ], - "ticker": [ - "ACME", - "NSTAR", - "APPL", - "AAPL", - "CEDR", - "DLTA", - "EVER", - "FUTR", - "GLOB", - "HELI", - "ION", - "JUN", - "KAPP", - "LUME", - "MOSC", - "NADI", - ], - "price": [ - 10.25, - 0.00, - "apple", - 155.20, - -4.5, - 9999999999.99, - "₹1,23,456.70", - 123.45, - 9.99, - 42.0, - 18.75, - 21.0, - 33.0, - 7.5, - 8.25, - 12.0, - ], - "currency": [ - "USD", - "USD", - "USD", - "USD", - "EUR", - "USD", - "INR", - "EUR", - "USD", - "USD", - "USD", - "EUR", - "INR", - "USD", - "USD", - "USD", - ], - "trade_date": ["2026-01-15"] * 16, - "memo": ["ordinary"] * 16, - "balance": [ - 100.0, - 200.0, - 300.0, - 400.0, - 500.0, - 600.0, - 700.0, - 800.0, - 900.0, - 1000.0, - 1100.0, - 1200.0, - 1300.0, - 1400.0, - 1500.0, - 1600.0, - ], - "locale": ["en_US"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "finance", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "currency": "USD", - "supported_currencies": ["USD", "EUR", "INR"], - "protected_ticker_policy": "preserve", - }, - protected_columns=("account_id", "ticker"), - ) - builder.inject("fin-03", "price", "apple", Disposition.REVIEW, family="semantic-apple-price") - builder.inject("fin-04", "company", "Apple", Disposition.PRESERVE, family="apple-company") - # Corrected oracle: ticker is a protected column (see protected_columns), - # and protection is a stronger guarantee than review — a protected value - # is never modified, so PRESERVE is the coherent label. "AAPL" is also the - # correct ticker for the "Apple" company in this row, so there is no - # genuine conflict to review. - builder.inject( - "fin-04", "ticker", "AAPL", Disposition.PRESERVE, family="protected-ticker-valid" - ) - builder.inject("fin-02", "price", "0.00", Disposition.PRESERVE, family="zero-price") - builder.inject("fin-05", "price", -4.5, Disposition.FLAG, family="negative-value") - builder.inject("fin-06", "price", 9999999999.99, Disposition.FLAG, family="extreme-value") - # Corrected oracle: expected_dtype='float64' was unreachable — fin-03 - # holds 'apple' (REVIEW) in this same column, so price must stay object - # to preserve it. The exact repaired value is still demanded. - builder.inject( - "fin-07", - "price", - "₹1,23,456.70", - Disposition.REPAIR, - expected=123456.70, - family="indian-grouped-currency", - ) - # Corrected oracle: EUR and INR are both in this fixture's declared - # supported_currencies, so they are valid values, not conflicts — a value - # cannot be simultaneously "supported" and a review-worthy "conflict". - # Cross-field base-vs-transaction currency reconciliation is a business - # rule FreshData does not claim; the honest label for a declared-valid - # value is PRESERVE. - builder.inject("fin-08", "currency", "EUR", Disposition.PRESERVE, family="supported-currency") - builder.inject("fin-09", "currency", "INR", Disposition.PRESERVE, family="supported-currency") - builder.inject( - "fin-10", "trade_date", "01/02/2025", Disposition.REVIEW, family="ambiguous-date-format" - ) - builder.inject( - "fin-11", - "memo", - "tb.finance+memo@example.invalid", - Disposition.FLAG, - family="invisible-pii-memo", - sensitive=True, - ) - builder.inject( - "fin-12", - "memo", - "tb.finance+hidden@example.invalid\u200b", - Disposition.FLAG, - family="zero-width-memo", - ) - builder.inject( - "fin-16", - "account_id", - "TB-FIN-ACCOUNT-TAIL", - Disposition.FLAG, - family="tail-row-account-canary", - sensitive=True, - ) - builder.add_row_case( - "exact-duplicate-fin-02-fin-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-fin-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case("type-drifted-price", Disposition.REVIEW, family="type-drifted-column") - return builder.build() diff --git a/benchmarks/truthbench/fixtures/government.py b/benchmarks/truthbench/fixtures/government.py deleted file mode 100644 index f0e46b18..00000000 --- a/benchmarks/truthbench/fixtures/government.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Deterministic government gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"gov-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "district_id": [ - "001", - "002", - "003", - "004", - "005", - "006", - "007", - "008", - "009", - "010", - "011", - "012", - "013", - "014", - "015", - "016", - ], - "case_id": [ - "000001", - "000123", - "000003", - "000004", - "000005", - "000006", - "000007", - "000008", - "000009", - "000010", - "000011", - "000012", - "000013", - "000014", - "000015", - "000016", - ], - "agency": [ - "Revenue Department", - "भारत सरकार / Government of India", - "Ministry of Health", - "City Council", - "税務局", - "Prefecture Office", - "State Secretariat", - "Public Works", - "Education Board", - "Transport Authority", - "Treasury", - "Registry", - "Civil Court", - "Labor Office", - "Statistics Bureau", - "Revenue Department", - ], - "fiscal_year": ["2026"] * 16, - "calendar_year": ["2026"] * 16, - "language": ["en"] * 16, - "notes": ["routine"] * 16, - "encoding": ["UTF-8"] * 16, - "retention_policy": ["retain 7 years"] * 16, - "repair_policy": ["repair after 30 days"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "government", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "supported_languages": ["en", "hi", "de", "zh"], - "protected_case_policy": "preserve", - }, - protected_columns=("district_id", "case_id"), - ) - builder.inject( - "gov-01", "district_id", "007", Disposition.PRESERVE, family="leading-zero-district-id" - ) - builder.inject( - "gov-02", "case_id", "000123", Disposition.PRESERVE, family="leading-zero-case-id" - ) - builder.inject( - "gov-03", - "agency", - "भारत सरकार / Government of India", - Disposition.PRESERVE, - family="indian-international-grouping", - ) - builder.inject( - "gov-05", - "fiscal_year", - "2025-26", - Disposition.REVIEW, - family="fiscal-calendar-year-conflict", - ) - builder.inject( - "gov-06", - "encoding", - "Café", - Disposition.REPAIR, - expected="Café", - family="mixed-legacy-encoding", - ) - builder.inject( - "gov-07", "language", "हिन्दी / English", Disposition.PRESERVE, family="multilingual-agency" - ) - builder.inject( - "gov-09", - "notes", - "TB-GOV-NATIONAL-ID-0001", - Disposition.FLAG, - family="restricted-national-id", - sensitive=True, - ) - builder.inject( - "gov-11", - "retention_policy", - "retain 7 years", - Disposition.REVIEW, - family="contradictory-retention-repair-policy", - ) - builder.inject( - "gov-11", - "repair_policy", - "repair after 30 days", - Disposition.REVIEW, - family="contradictory-retention-repair-policy", - ) - builder.inject( - "gov-16", - "case_id", - "TB-GOV-CASE-TAIL", - Disposition.PRESERVE, - family="protected-case-id-conflict", - sensitive=True, - ) - builder.add_row_case( - "exact-duplicate-gov-02-gov-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-gov-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case( - "type-drifted-fiscal-year", Disposition.REVIEW, family="type-drifted-column" - ) - return builder.build() diff --git a/benchmarks/truthbench/fixtures/healthcare.py b/benchmarks/truthbench/fixtures/healthcare.py deleted file mode 100644 index 4379f5fe..00000000 --- a/benchmarks/truthbench/fixtures/healthcare.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Deterministic healthcare gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"hc-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "mrn": [f"TB-HC-{i:04d}" for i in range(1, 17)], - "patient_name": [ - "Amina Khan", - "Luca Rossi", - "Marta Silva", - "Noah Reed", - "Iris Chen", - "Sofia Patel", - "Eli Jones", - "Ravi Singh", - "Nora Stone", - "Mina Park", - "Owen Hall", - "Zoe King", - "Leo Cruz", - "Aya Mori", - "Finn Cole", - "Uma Roy", - ], - "diagnosis_code": [ - "R69", - "E11.9", - "I10", - "Z79.4", - "J45.9", - "M54.5", - "N39.0", - "K21.9", - "L20.9", - "D50.9", - "G43.9", - "H25.1", - "F17.210", - "B20", - "A09", - "Z00.0", - ], - "loinc": [ - "LP21258-6", - "2345-7", - "718-7", - "9843-4", - "4548-4", - "8310-5", - "2951-2", - "2160-0", - "6690-2", - "26499-4", - "3094-0", - "33747-0", - "9843-4", - "1751-7", - "2823-3", - "1975-2", - ], - "temperature": [ - 36.8, - 37.0, - 36.5, - 98.6, - 37.1, - 36.9, - 37.2, - 36.7, - 36.6, - 37.0, - 36.8, - 36.9, - 37.1, - 36.8, - 36.7, - 36.9, - ], - "dose": [ - "5 mg", - "10 mg", - "2.5 mg", - "5000 mcg", - "1 mg", - "20 mg", - "5 mg", - "2 mg", - "8 mg", - "4 mg", - "3 mg", - "6 mg", - "7 mg", - "9 mg", - "1 mg", - "2 mg", - ], - "event_date": ["2026-01-15"] * 16, - "dob": ["1980-01-01"] * 16, - "notes": ["routine"] * 16, - "phone": ["555-0101"] * 16, - "language": ["en"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "healthcare", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "standards": ["FHIR", "ICD-10", "LOINC"], - "temperature_unit": "C", - "protected_dob_policy": "preserve", - }, - protected_columns=("mrn", "dob"), - ) - builder.inject( - "hc-04", "diagnosis_code", "Z79.4", Disposition.PRESERVE, family="rare-icd-valid" - ) - builder.inject("hc-04", "loinc", "9843-4", Disposition.PRESERVE, family="rare-loinc-valid") - builder.inject( - "hc-04", "temperature", 98.6, Disposition.REVIEW, family="celsius-fahrenheit-conflict" - ) - builder.inject("hc-01", "dose", "5 mg", Disposition.PRESERVE, family="dose-unit-valid") - # Corrected oracle: the original label (REPAIR to float 5.0) contradicted - # this same column's dose-unit-valid PRESERVE cells — a column cannot both - # keep "5 mg" strings and hold a float64 repair. A cross-unit dose - # conversion is a meaning-changing clinical decision, so it is routed to a - # human, exactly like the celsius-fahrenheit conflict above. - builder.inject( - "hc-04", - "dose", - "5000 mcg", - Disposition.REVIEW, - family="mg-mcg-unit-conversion", - ) - builder.inject("hc-05", "event_date", "2025-01", Disposition.REVIEW, family="partial-date") - builder.inject( - "hc-06", - "event_date", - "2025-01-15T12:00:00Z", - Disposition.REPAIR, - expected="2025-01-15", - family="fhir-date", - ) - builder.inject("hc-07", "event_date", "2025-02-30", Disposition.FLAG, family="impossible-date") - builder.inject( - "hc-08", - "patient_name", - "Jose\u0301", - Disposition.REPAIR, - expected="José", - family="decomposed-unicode", - ) - builder.inject( - "hc-09", "notes", "TB-HC-PHI-0001", Disposition.FLAG, family="phi-in-notes", sensitive=True - ) - builder.inject( - "hc-10", - "phone", - "555-0107", - Disposition.PRESERVE, - family="synthetic-phone", - sensitive=True, - ) - builder.inject( - "hc-11", "notes", "TB-HC-MRN-NOTE", Disposition.FLAG, family="mrn-in-notes", sensitive=True - ) - builder.inject( - "hc-12", "notes", "555-0112", Disposition.FLAG, family="phone-in-notes", sensitive=True - ) - builder.inject( - "hc-16", - "mrn", - "TB-HC-MRN-TAIL", - Disposition.PRESERVE, - family="mrn-tail-canary", - sensitive=True, - ) - builder.inject( - "hc-15", "dob", "01/01/1980", Disposition.PRESERVE, family="protected-dob-repair-conflict" - ) - builder.add_row_case("exact-duplicate-hc-02-hc-03", Disposition.FLAG, family="exact-duplicate") - builder.add_row_case("removed-hc-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case( - "type-drifted-temperature", Disposition.REVIEW, family="type-drifted-column" - ) - return builder.build() diff --git a/benchmarks/truthbench/fixtures/insurance.py b/benchmarks/truthbench/fixtures/insurance.py deleted file mode 100644 index 6bbcd7dc..00000000 --- a/benchmarks/truthbench/fixtures/insurance.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Deterministic insurance gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"ins-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "policy_number": [ - "00012345", - "00012346", - "00012347", - "00012348", - "00012349", - "00012350", - "00012351", - "00012352", - "00012353", - "00012354", - "00012355", - "00012356", - "00012357", - "00012358", - "00012359", - "00012360", - ], - "claim_id": [ - "CLM-000123", - "CLM-000124", - "CLM-000125", - "CLM-000126", - "CLM-000127", - "CLM-000128", - "CLM-000129", - "CLM-000130", - "CLM-000131", - "CLM-000132", - "CLM-000133", - "CLM-000134", - "CLM-000135", - "CLM-000136", - "CLM-000137", - "CLM-000138", - ], - "premium": [1000.0] * 16, - "premium_currency": ["USD"] * 16, - "reserve": [ - 500.0, - 600.0, - 700.0, - 800.0, - -250.0, - 900.0, - 1000.0, - 1100.0, - 1200.0, - 1300.0, - 1400.0, - 1500.0, - 1600.0, - 1700.0, - 1800.0, - 1900.0, - ], - "reserve_currency": [ - "USD", - "USD", - "USD", - "EUR", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - ], - "incident_date": ["2025-01-01"] * 16, - "report_date": ["2025-01-02"] * 16, - "state": [ - "open", - "closed", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - "open", - ], - "claimant_name": ["Claimant"] * 16, - "loss_description": ["water damage"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "insurance", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "currency": "USD", - "supported_currencies": ["USD", "EUR", "INR"], - "protected_policy_policy": "preserve", - }, - protected_columns=("policy_number",), - ) - builder.inject( - "ins-01", "policy_number", "00012345", Disposition.PRESERVE, family="policy-id-format" - ) - builder.inject( - "ins-02", "claim_id", "CLM-000123", Disposition.PRESERVE, family="claim-id-format" - ) - # Corrected oracle: every other premium in this column is a whole 1000.0, - # so the coherent cleaned column is int64 — the float64 expected_dtype was - # unreachable. The exact repaired value (1000) is still demanded. - builder.inject( - "ins-03", - "premium", - "1,000.00", - Disposition.REPAIR, - expected=1000, - family="grouped-premium", - ) - builder.inject( - "ins-04", - "reserve_currency", - "EUR", - Disposition.REVIEW, - family="premium-reserve-currency-conflict", - ) - builder.inject( - "ins-05", "reserve", -250.0, Disposition.REVIEW, family="negative-reserve-review" - ) - builder.inject( - "ins-06", - "report_date", - "2025-01-01", - Disposition.REVIEW, - family="incident-report-date-ordering", - ) - builder.inject( - "ins-07", - "state", - "open|closed", - Disposition.REVIEW, - family="state-transition-contradiction", - ) - builder.inject( - "ins-09", - "claimant_name", - "TB-INS-CLAIMANT-0001", - Disposition.FLAG, - family="claimant-pii", - sensitive=True, - ) - builder.inject( - "ins-10", - "loss_description", - "TB-INS-MEDICAL-0001", - Disposition.FLAG, - family="medical-loss-text", - sensitive=True, - ) - builder.inject( - "ins-16", - "policy_number", - "TB-INS-POLICY-TAIL", - Disposition.PRESERVE, - family="protected-policy-number-conflict", - sensitive=True, - ) - builder.add_row_case( - "exact-duplicate-ins-02-ins-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-ins-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case( - "type-drifted-reserve", Disposition.REVIEW, family="type-drifted-column" - ) - return builder.build() diff --git a/benchmarks/truthbench/fixtures/logistics.py b/benchmarks/truthbench/fixtures/logistics.py deleted file mode 100644 index 92533999..00000000 --- a/benchmarks/truthbench/fixtures/logistics.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Deterministic logistics gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"log-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "shipment_id": [f"TB-LOG-{i:04d}" for i in range(1, 17)], - "origin_code": ["USLAX"] * 16, - "destination_code": [ - "NLRTM", - "DEHAM", - "GBFXT", - "INBOM", - "SGSIN", - "CNSHA", - "JPTYO", - "AUMEL", - "BRSSZ", - "ZADUR", - "AEJEA", - "NOOSL", - "ESBCN", - "CATOR", - "MYPKG", - "INBOM", - ], - "weight": [ - 10.0, - "10 lb", - 12.0, - 8.0, - 4.0, - 15.0, - 20.0, - 25.0, - 30.0, - 35.0, - 40.0, - 45.0, - 50.0, - 55.0, - 60.0, - 65.0, - ], - "weight_unit": ["kg"] * 16, - "temperature": [ - 20.0, - 21.0, - 98.6, - 19.0, - 18.0, - 22.0, - 23.0, - 24.0, - 25.0, - 26.0, - 27.0, - 28.0, - 29.0, - 30.0, - 31.0, - 32.0, - ], - "temperature_unit": ["C"] * 16, - "delivery_window": ["2026-01-15 09:00-17:00"] * 16, - "transport_time": ["12:00"] * 16, - "address": ["Warehouse"] * 16, - "tracking_status": ["on-time"] * 16, - "timezone": ["UTC"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "logistics", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "weight_unit": "kg", - "temperature_unit": "C", - "protected_shipment_policy": "preserve", - }, - protected_columns=("shipment_id",), - ) - builder.inject( - "log-04", "destination_code", "INBOM", Disposition.PRESERVE, family="rare-unlocode-valid" - ) - # Corrected oracle: the original label (REPAIR to the kg equivalent) - # contradicted the row's own weight_unit column, which stays 'lb' - # (PRESERVE) — converting weight to kg would make the row internally - # inconsistent. A cross-unit conversion is routed to a human, exactly - # like the celsius-fahrenheit conflict below. - builder.inject( - "log-02", - "weight", - "10 lb", - Disposition.REVIEW, - family="kg-lb-conversion", - ) - builder.inject("log-02", "weight_unit", "lb", Disposition.PRESERVE, family="weight-unit-lb") - builder.inject( - "log-03", "temperature", 98.6, Disposition.REVIEW, family="celsius-fahrenheit-conflict" - ) - builder.inject( - "log-03", "temperature_unit", "F", Disposition.PRESERVE, family="temperature-unit-f" - ) - builder.inject( - "log-05", - "delivery_window", - "2026-01-15 23:30-2026-01-16 01:00", - Disposition.REVIEW, - family="cross-timezone-window", - ) - builder.inject( - "log-05", - "timezone", - "Asia/Kolkata→America/New_York", - Disposition.REVIEW, - family="cross-timezone-window", - ) - builder.inject( - "log-06", - "transport_time", - "24:00", - # Corrected oracle: the column holds clock times without dates, so - # 00:00 would move end-of-day to start-of-day; routed to a human. - Disposition.REVIEW, - family="twentyfour-hour-transport", - ) - # The domain's exact repair: a status label whose separator drifted from - # the column's single dominant spelling has one correct form. - builder.inject( - "log-07", - "tracking_status", - "on time", - Disposition.REPAIR, - expected="on-time", - family="spaced-tracking-status", - ) - builder.inject( - "log-09", - "address", - "TB-LOG-ADDRESS-0001", - Disposition.FLAG, - family="address-pii", - sensitive=True, - ) - builder.inject( - "log-15", "tracking_status", "delayed", Disposition.FLAG, family="late-tracking-canary" - ) - builder.inject( - "log-16", - "shipment_id", - "TB-LOG-SHIPMENT-TAIL", - Disposition.PRESERVE, - family="protected-shipment-id-conflict", - sensitive=True, - ) - builder.add_row_case( - "exact-duplicate-log-02-log-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-log-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case( - "type-drifted-weight", Disposition.REVIEW, family="type-drifted-column" - ) - return builder.build() diff --git a/benchmarks/truthbench/fixtures/retail.py b/benchmarks/truthbench/fixtures/retail.py deleted file mode 100644 index be39bbfb..00000000 --- a/benchmarks/truthbench/fixtures/retail.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Deterministic retail gold fixture for TruthBench.""" - -from __future__ import annotations - -import pandas as pd - -from ..models import Disposition -from .base import FixtureBuilder, TruthFixture - - -def build(seed: int = 1729) -> TruthFixture: - rows = [f"ret-{i:02d}" for i in range(1, 17)] - frame = pd.DataFrame( - { - "sku": [ - "001234", - "000007", - "123456", - "888888", - "100001", - "100002", - "100003", - "100004", - "100005", - "100006", - "100007", - "100008", - "100009", - "100010", - "100011", - "100012", - ], - "gtin": [ - "00012345678905", - "00000012345678", - "12345678901234", - "88888888888888", - "10000000000001", - "10000000000002", - "10000000000003", - "10000000000004", - "10000000000005", - "10000000000006", - "10000000000007", - "10000000000008", - "10000000000009", - "10000000000010", - "10000000000011", - "10000000000012", - ], - "product_name": [ - "Widget", - "Free Sample", - "Café & Tea", - "茶", - "Book", - "Lamp", - "Chair", - "Desk", - "Glass", - "Mug", - "Bag", - "Pencil", - "Shoes", - "Coat", - "Hat", - "Socks", - ], - "quantity": [1, 1, 2, 1, 1, -2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], - "price": [ - "12.50", - "0.00", - "1.234,56", - "₹1,23,456.70", - "9.99", - "10.00", - "11.00", - "12.00", - "13.00", - "14.00", - "15.00", - "16.00", - "17.00", - "18.00", - "19.00", - "20.00", - ], - "currency": [ - "USD", - "USD", - "EUR", - "INR", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - "USD", - ], - "review": ["great"] * 16, - "email": ["ordinary@example.invalid"] * 16, - "card": ["TB-RETAIL-CARD-0001"] * 16, - "order_date": ["2026-01-15"] * 16, - "batch": [f"seed-{int(seed)}"] * 16, - }, - index=rows, - ) - builder = FixtureBuilder( - "v1", - "retail", - frame, - seed=int(seed), - schema={ - "columns": list(frame.columns), - "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, - }, - policy={ - "reference_date": "2026-01-15", - "timezone": "UTC", - "locale": "en_US", - "locales": ["en_US", "de_DE", "hi_IN", "zh_CN"], - "currency": "USD", - }, - protected_columns=("sku", "gtin"), - ) - builder.inject("ret-01", "sku", "001234", Disposition.PRESERVE, family="leading-zero-sku") - builder.inject( - "ret-01", "gtin", "00012345678905", Disposition.PRESERVE, family="leading-zero-gtin" - ) - builder.inject("ret-02", "price", "0.00", Disposition.PRESERVE, family="free-item") - builder.inject("ret-06", "quantity", -2, Disposition.FLAG, family="return-quantity") - builder.inject( - "ret-03", - "price", - "1.234,56", - Disposition.REPAIR, - expected=1234.56, - expected_dtype="float64", - family="mixed-decimal-grouping", - ) - builder.inject("ret-04", "price", "₹1,23,456.70", Disposition.REVIEW, family="mixed-currency") - builder.inject("ret-05", "currency", "EUR", Disposition.REVIEW, family="mixed-currency") - builder.inject( - "ret-03", - "product_name", - "Café & Tea", - Disposition.REVIEW, - family="html-entity-mojibake", - ) - builder.inject( - "ret-04", "product_name", "茶", Disposition.PRESERVE, family="multilingual-product" - ) - builder.inject( - "ret-07", - "review", - "customer@example.invalid", - Disposition.REVIEW, - family="email-in-review", - sensitive=True, - ) - builder.inject( - "ret-08", - "card", - "TB-RETAIL-CARD-REVIEW", - Disposition.REVIEW, - family="card-in-review", - sensitive=True, - ) - builder.inject( - "ret-16", - "email", - "tail@example.invalid", - Disposition.FLAG, - family="tail-email-canary", - sensitive=True, - ) - builder.add_row_case( - "exact-duplicate-ret-02-ret-03", Disposition.FLAG, family="exact-duplicate" - ) - builder.add_row_case("removed-ret-15", Disposition.REVIEW, family="removed-row") - builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") - builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") - builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") - builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") - builder.add_schema_case( - "type-drifted-quantity", Disposition.REVIEW, family="type-drifted-column" - ) - return builder.build() diff --git a/benchmarks/truthbench/gates.py b/benchmarks/truthbench/gates.py deleted file mode 100644 index ee19a314..00000000 --- a/benchmarks/truthbench/gates.py +++ /dev/null @@ -1,485 +0,0 @@ -"""Absolute, fail-closed TruthBench release gates.""" - -from __future__ import annotations - -import ast -import re -from collections.abc import Iterable, Mapping -from dataclasses import dataclass, field -from itertools import product -from types import MappingProxyType -from typing import Any - -from .exact import ( - TypedValue, - canonical_scalar, - encode_typed, - equivalent_after_type_normalization, - numeric_value_equal, -) -from .models import DecisionRecord, Disposition, GateResult, RunResult -from .privacy import SinkScanner - -_RAW_PII = re.compile(r"[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+\.invalid") -_BOILERPLATE = {"", "n/a", "none", "unknown", "automatic", "auto"} -_SURFACE_CLASSES = {"mutator", "validator", "pii", "explicit_transform"} - - -@dataclass(frozen=True) -class _FixtureEvidence: - """A copy-protected oracle snapshot, never a live fixture DataFrame.""" - - domain: str - cells: Mapping[str, Any] - inputs: Mapping[str, TypedValue] - protected_columns: frozenset[str] - canaries: tuple[tuple[str, Any], ...] - required_case_ids: frozenset[str] - - -def _snapshot(fixture: Any) -> _FixtureEvidence: - frame = fixture.frame - cells = {cell.cell_id: cell for cell in fixture.cells} - inputs = { - cell.cell_id: encode_typed( - canonical_scalar(frame.at[cell.row_id, cell.column]), - dtype=frame[cell.column].dtype, - ) - for cell in fixture.cells - if not cell.sensitive - } - cases = (*getattr(fixture, "row_cases", ()), *getattr(fixture, "schema_cases", ())) - return _FixtureEvidence( - domain=str(fixture.domain), - cells=MappingProxyType(cells), - inputs=MappingProxyType(inputs), - protected_columns=frozenset(str(column) for column in fixture.protected_columns), - canaries=tuple((str(key), value) for key, value in fixture.pii_canaries.items()), - required_case_ids=frozenset(case.case_id for case in cases), - ) - - -@dataclass(frozen=True) -class GateRun: - """Immutable gate context paired with the existing serialized ``RunResult``. - - The public result schema remains untouched. At construction, live fixture - frames are converted into typed, mapping-proxy oracle snapshots so later - fixture mutation cannot change a release decision. - """ - - run: RunResult - fixtures: tuple[Any, ...] - generated_code: tuple[str, ...] = () - persisted_sinks: tuple[Any, ...] = () - complete: bool = True - schema_valid: bool = True - unexpected_exceptions: tuple[Any, ...] = () - expected_surfaces: tuple[str, ...] = () - expected_backends: tuple[str, ...] = () - expected_repeats: tuple[int, ...] = () - surface_classes: tuple[tuple[str, str], ...] = () - requested_cell_ids: tuple[str, ...] = () - case_records: tuple[Any, ...] = () - audit_record_ids: tuple[str, ...] = () - _evidence: tuple[_FixtureEvidence, ...] = field(init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__( - self, "_evidence", tuple(_snapshot(fixture) for fixture in self.fixtures) - ) - - -@dataclass(frozen=True) -class GateEvaluation: - gates: tuple[GateResult, ...] - - @property - def passed(self) -> bool: - return all(gate.passed for gate in self.gates) - - -def _gate(name: str, failures: Iterable[str]) -> GateResult: - return GateResult(name, not (items := tuple(failures)), items) - - -def _cells(context: GateRun) -> dict[str, tuple[_FixtureEvidence, Any]]: - return { - cell_id: (evidence, cell) - for evidence in context._evidence - for cell_id, cell in evidence.cells.items() - } - - -def _surface_class(context: GateRun, surface: str) -> str | None: - return dict(context.surface_classes).get(surface) - - -def _is_mutator(context: GateRun, record: DecisionRecord) -> bool: - return _surface_class(context, record.surface) in {"mutator", "explicit_transform"} - - -def _changed(record: DecisionRecord) -> bool: - """Objective representation change — used by the audit-evidence gates, - where even a benign normalization must leave an audit trail.""" - return record.actual_output is not None and record.actual_output != record.input - - -def _value_changed(record: DecisionRecord) -> bool: - """Semantic value change — used by the corruption/flag/trust gates. - - Direct output evidence is authoritative in both directions: when the - record carries an actual output, the comparison is computed (a denied - mutation cannot hide, a value-preserving type normalization such as a - canonical ISO date string becoming the equal naive timestamp does not - fire). Only records without observable output fall back to the claim. - """ - if record.actual_output is not None: - if record.actual_output == record.input: - return False - return not ( - equivalent_after_type_normalization(record.actual_output, record.input) - or numeric_value_equal(record.actual_output, record.input) - ) - return record.mutated is True - - -def _exact_repair(actual: TypedValue | None, expected: TypedValue | None) -> bool: - if actual is None or expected is None: - return False - if expected.dtype is not None: - return actual == expected - # Fixture authors who do not specify a dtype constrain scalar type/value, - # not an incidental object/string backend representation. - return ( - actual.kind == expected.kind - and actual.value == expected.value - and actual.redacted == expected.redacted - ) - - -def _substantive(record: DecisionRecord, audit_ids: set[str]) -> bool: - rationale = (record.rationale or "").strip() - return ( - rationale.casefold() not in _BOILERPLATE - and len(rationale) >= 12 - and bool((record.rule_id or "").strip()) - and record.audit_complete is True - and bool(set(record.audit_ids or ()) & audit_ids) - ) - - -def _canary_leaks(context: GateRun) -> tuple[str, ...]: - canaries = {key: value for evidence in context._evidence for key, value in evidence.canaries} - failures: list[str] = [] - if canaries: - scanner = SinkScanner.from_canaries(canaries, key=b"truthbench-gate-scan-v1") - for index, sink in enumerate(context.persisted_sinks): - for leak in scanner.scan(sink): - failures.append(f"sink[{index}] leaked canary {leak.canary_id} at {leak.path}") - for index, sink in enumerate(context.persisted_sinks): - if _RAW_PII.search(str(sink)): - failures.append(f"sink[{index}] contains raw synthetic PII") - return tuple(failures) - - -def _coverage_failures(context: GateRun, cells: Mapping[str, Any]) -> list[str]: - if ( - not context.expected_surfaces - or not context.expected_backends - or not context.expected_repeats - ): - return ["expected surface/backend/repeat coverage was not declared"] - failures: list[str] = [] - for cell_id, surface, backend, repeat in product( - cells, context.expected_surfaces, context.expected_backends, context.expected_repeats - ): - matches = [ - record - for record in context.run.records - if record.cell_id == cell_id - and record.surface == surface - and record.requested_backend == backend - and record.repeat == repeat - ] - if len(matches) != 1: - failures.append( - f"{cell_id}: expected exactly one {surface}/{backend}/repeat-{repeat} record" - ) - declared = dict(context.surface_classes) - for surface in context.expected_surfaces: - if declared.get(surface) not in _SURFACE_CLASSES: - failures.append(f"{surface}: surface class was not declared") - return failures - - -def evaluate_gates(context: GateRun) -> GateEvaluation: - """Evaluate all gates independently. No baseline or stale claim clears one.""" - - run, records, cells = context.run, context.run.records, _cells(context) - audit_ids = set(context.audit_record_ids) - gates: list[GateResult] = [] - coverage = _coverage_failures(context, cells) if cells else ["fixture policy/cells absent"] - gates.append( - _gate( - "completeness", - ["run is partial"] * (not context.complete or not records) + coverage, - ) - ) - gates.append( - _gate("schema_validation", ["schema validation failed"] * (not context.schema_valid)) - ) - gates.append( - _gate("fixture_evidence", ["fixture policy/cell evidence is absent"] * (not cells)) - ) - gates.append( - _gate( - "unexpected_exception", - (f"unexpected exception: {e}" for e in context.unexpected_exceptions), - ) - ) - required = set(context.expected_backends or run.required_backends) - actual = {record.actual_backend for record in records if record.actual_backend} - gates.append( - _gate( - "required_backend", - (f"required backend {b} was not executed" for b in sorted(required - actual)), - ) - ) - gates.append( - _gate( - "valid_value_corruption", - ( - f"{record.record_id}: preserve value was corrupted" - for record in records - if _is_mutator(context, record) - and record.expected_disposition is Disposition.PRESERVE - and _value_changed(record) - ), - ) - ) - gates.append( - _gate( - "protected_column_modification", - ( - f"{record.record_id}: protected column {record.column} was modified" - for record in records - if (item := cells.get(record.cell_id)) is not None - and record.column in item[0].protected_columns - and (record.mutated is True or _changed(record)) - ), - ) - ) - gates.append(_gate("raw_pii_leakage", _canary_leaks(context))) - gates.append( - _gate( - "backend_inconsistency", - ( - f"{record.record_id}: missing or inconsistent backend disclosure" - for record in records - if not record.requested_backend - or not record.actual_backend - or record.requested_backend != record.actual_backend - ), - ) - ) - gates.append( - _gate( - "default_nondeterminism", - ( - f"{record.record_id}: repeat decision is absent or inconsistent" - for record in records - if record.repeat_consistent is not True or not record.repeat_hash - ), - ) - ) - broken = [] - for index, code in enumerate(context.generated_code): - try: - ast.parse(code) - except SyntaxError: - broken.append(f"generated code[{index}] does not parse") - gates.append(_gate("broken_generated_code", broken)) - gates.append( - _gate( - "unexplained_high_confidence", - ( - f"{record.record_id}: high-confidence decision lacks rationale/provenance/audit" - for record in records - if record.confidence is not None - and record.confidence >= 0.90 - and not _substantive(record, audit_ids) - ), - ) - ) - gates.append( - _gate( - "trust_inversion", - ( - f"{record.record_id}: trust increased after data changed" - for record in records - if record.trust_before is not None - and record.trust_after is not None - and record.trust_after > record.trust_before - ), - ) - ) - gates.append( - _gate( - "mutation_audit", - ( - f"{record.record_id}: decision lacks matching audit evidence" - for record in records - if ( - record.mutated - or record.detected - or (_is_mutator(context, record) and _changed(record)) - ) - and ( - record.audit_complete is not True - or not set(record.audit_ids or ()) & audit_ids - ) - ), - ) - ) - gates.append( - _gate( - "review_routing", - ( - f"{record.record_id}: review case was not routed to a human" - for record in records - if record.expected_disposition is Disposition.REVIEW - and record.human_review is not True - ), - ) - ) - gates.append( - _gate( - "exact_repair", - ( - f"{record.record_id}: repair differs from the exact oracle" - for record in records - if _is_mutator(context, record) - and record.expected_disposition is Disposition.REPAIR - and not _exact_repair(record.actual_output, record.expected_output) - ), - ) - ) - gates.append( - _gate( - "flag_mutation", - ( - f"{record.record_id}: flag/review value was mutated" - for record in records - if _is_mutator(context, record) - and record.expected_disposition in {Disposition.FLAG, Disposition.REVIEW} - and _value_changed(record) - # Quarantine-to-missing with the original preserved and - # audited is the product's documented flag handling, not a - # silent mutation. - and not (record.quarantined is True and record.audit_complete is True) - ), - ) - ) - gates.append( - _gate( - "validator_contract", - ( - f"{record.record_id}: validator mutated data or missed detection" - for record in records - if _surface_class(context, record.surface) == "validator" - and ( - record.mutated is True - or _changed(record) - or ( - record.expected_disposition is not Disposition.PRESERVE - and record.detected is not True - ) - ) - ), - ) - ) - gates.append( - _gate( - "pii_scope", - ( - f"{record.record_id}: PII surface missed a canary or changed a false positive" - for record in records - if _surface_class(context, record.surface) == "pii" - and ( - (record.sensitive and record.detected is not True) - or ( - not record.sensitive - and (record.mutated is True or _changed(record) or record.detected is True) - ) - ) - ), - ) - ) - requested = set(context.requested_cell_ids) - gates.append( - _gate( - "requested_behavior", - ( - ( - f"{record.record_id}: explicit transform changed an unrequested " - "cell or missed requested behavior" - ) - for record in records - if _surface_class(context, record.surface) == "explicit_transform" - and ( - ( - record.cell_id not in requested - and (record.mutated is True or _changed(record)) - ) - or ( - record.cell_id in requested - and record.expected_disposition is Disposition.REPAIR - and not _exact_repair(record.actual_output, record.expected_output) - ) - ) - ), - ) - ) - gates.append( - _gate( - "input_mutation", - ( - f"{record.record_id}: input evidence differs from fixture snapshot" - for record in records - if (item := cells.get(record.cell_id)) is not None - and not record.sensitive - and record.input != item[0].inputs[record.cell_id] - ), - ) - ) - required_cases = { - case for evidence in context._evidence for case in evidence.required_case_ids - } - observed_cases = { - case.case_id for case in context.case_records if getattr(case, "observed", False) - } - gates.append( - _gate( - "case_coverage", - ( - f"required case {case} was not observed" - for case in sorted(required_cases - observed_cases) - ), - ) - ) - gates.append( - _gate( - "aggregate_consistency", - ["summary record aggregate does not match records"] - * (dict(run.summary).get("records") != len(records)), - ) - ) - return GateEvaluation(tuple(gates)) - - -def failed_gate_names(result: GateEvaluation) -> set[str]: - return {gate.name for gate in result.gates if not gate.passed} - - -__all__ = ["GateEvaluation", "GateRun", "evaluate_gates", "failed_gate_names"] diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py deleted file mode 100644 index 58d7594f..00000000 --- a/benchmarks/truthbench/generated_code.py +++ /dev/null @@ -1,397 +0,0 @@ -"""Production-grade verification of Copilot-generated code. - -``ast.parse`` alone proves nothing about behavior. Verification here is a -pipeline: parse, strict AST allowlist, compile, then execution with -``python -I`` in a throwaway working directory with a restricted environment, -an import poison for network/process modules, a hard timeout, and a defined -input contract (the fixture written as the CSV the generated script reads). -Afterwards the protected cells must be untouched and no PII canary may appear -in the source, stdout, stderr, produced files, or the exception text. -""" - -from __future__ import annotations - -import ast -import os -import subprocess -import sys -import tempfile -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -import pandas as pd - -from .privacy import SinkScanner - -#: Modules the generated pipeline may import. Everything else is rejected -#: statically and poisoned at runtime. -ALLOWED_IMPORTS: frozenset[str] = frozenset({"pandas", "numpy", "freshdata"}) - -#: Builtins whose call is forbidden inside generated code. -BANNED_CALLS: frozenset[str] = frozenset( - { - "eval", - "exec", - "compile", - "open", - "input", - "breakpoint", - "__import__", - "globals", - "locals", - "vars", - "getattr", - "setattr", - "delattr", - "memoryview", - } -) - -#: Modules poisoned inside the sandbox so even a missed static path cannot -#: reach the network, spawn processes, or load native code. -POISONED_MODULES: tuple[str, ...] = ( - "socket", - "ssl", - "http", - "urllib", - "urllib3", - "requests", - "ftplib", - "smtplib", - "telnetlib", - "subprocess", - "ctypes", - "multiprocessing", - "webbrowser", -) - -_INPUT_BASENAME = "your_data.csv" - -#: How much of a failed child's stderr a failure message carries. -_STDERR_TAIL_CHARS = 800 -_CRASH_DUMP_CHARS = 1500 - - -def _stderr_excerpt(stderr: str) -> str: - """The useful part of a failed child's stderr for its failure message. - - A normal traceback ends at the bottom, so keep the tail. A faulthandler - dump is the opposite: the stack that locates a native crash comes first and - a long ``Extension modules: ...`` line comes last, so a tail keeps only the - module list. For a dump, keep it from its header and drop that line. - """ - start = stderr.find("Fatal Python error:") - if start == -1: - return stderr[-_STDERR_TAIL_CHARS:] - dump = "\n".join( - line - for line in stderr[start:].splitlines() - if not line.startswith("Extension modules:") - ) - return dump[:_CRASH_DUMP_CHARS] - - -#: A child killed by a signal (negative exit code) is run again this many times. -#: pandas/numpy native code has segfaulted intermittently on CI in otherwise -#: passing generated code; one retry separates that flake from a real crash, -#: and every crash is still recorded in ``GeneratedCodeResult.native_crashes``. -_NATIVE_CRASH_RETRIES = 1 - -#: The harness prints this line before anything else. A child that exits -#: without it never ran the harness (the interpreter failed to start), so none -#: of the post-run checks below observed the generated code. -_STARTED_MARKER = "truthbench-sandbox-child-started" - - -@dataclass(frozen=True) -class GeneratedCodeResult: - """Outcome of one full verification run.""" - - passed: bool - failures: tuple[str, ...] - stdout: str = "" - stderr: str = "" - produced_files: tuple[str, ...] = () - stages: tuple[str, ...] = field(default=()) - #: One redacted excerpt per signal exit, including crashes that a retry - #: recovered from, so a flaky pass is never silent. - native_crashes: tuple[str, ...] = () - #: Set when the sandbox child could not start the harness. The generated - #: code never ran, so this result says nothing about its behavior and the - #: caller must treat it as an infrastructure failure, not a verdict. - infrastructure_failure: str | None = None - - -def _sandbox_env( - workdir: Path, - *, - os_name: str | None = None, - environ: Mapping[str, str] | None = None, -) -> dict[str, str]: - """The deliberately minimal environment of the sandbox child.""" - - env = { - "PYTHONDONTWRITEBYTECODE": "1", - "PYTHONNOUSERSITE": "1", - "FRESHDATA_NO_NETWORK": "1", - "HOME": str(workdir), - "TMPDIR": str(workdir), - # The environment is otherwise empty, so native thread pools would - # size themselves from the host core count; pin them (and the - # locale) so runs are deterministic across machines. - "OMP_NUM_THREADS": "1", - "OPENBLAS_NUM_THREADS": "1", - "MKL_NUM_THREADS": "1", - "POLARS_MAX_THREADS": "1", - "RAYON_NUM_THREADS": "1", - "LANG": "C.UTF-8", - } - if (os_name if os_name is not None else os.name) == "nt": - # CPython <= 3.10 on Windows seeds hash randomization through - # CryptGenRandom, which cannot load its provider without SystemRoot: - # the child dies with "Fatal Python error: _Py_HashRandomization_Init: - # failed to get random numbers". 3.11+ uses BCryptGenRandom and starts - # without it. Windows variable names are case-insensitive, so one key - # covers SystemRoot and SYSTEMROOT. - source = os.environ if environ is None else environ - system_root = source.get("SystemRoot") or source.get("SYSTEMROOT") - if system_root: - env["SystemRoot"] = system_root - return env - - -def _allowlist_failures(tree: ast.AST) -> list[str]: - failures: list[str] = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - root = alias.name.split(".")[0] - if root not in ALLOWED_IMPORTS: - failures.append(f"forbidden import: {alias.name}") - elif isinstance(node, ast.ImportFrom): - root = (node.module or "").split(".")[0] - if node.level or root not in ALLOWED_IMPORTS: - failures.append(f"forbidden import: from {node.module or '.'}") - elif isinstance(node, ast.Call): - callee = node.func - if isinstance(callee, ast.Name) and callee.id in BANNED_CALLS: - failures.append(f"forbidden call: {callee.id}()") - elif isinstance(node, ast.Attribute) and node.attr.startswith("__"): - failures.append(f"forbidden dunder attribute access: .{node.attr}") - elif isinstance(node, ast.Name) and node.id == "__builtins__": - failures.append("forbidden name: __builtins__") - return failures - - -def _harness(code_path: str, input_name: str) -> str: - poison = ", ".join(repr(name) for name in POISONED_MODULES) - return f""" -import sys - -# First output: proves the interpreter started and is running this harness. -sys.stdout.write({_STARTED_MARKER!r} + "\\n") -sys.stdout.flush() - -# Pre-import the allowed libraries so their own internal use of stdlib -# modules (pandas imports subprocess for locale probing) completes first... -import pandas # noqa: F401 -import freshdata # noqa: F401 - -# ...then deny-by-poison: any attempt by the generated code to import -# network/process/native modules fails immediately. -for _name in ({poison},): - sys.modules[_name] = None - -code = open({code_path!r}, encoding="utf-8").read() -namespace = {{"__name__": "__main__", "__file__": {code_path!r}}} -exec(compile(code, "generated_pipeline.py", "exec"), namespace) -""" - - -def verify_generated_code( # noqa: PLR0915 - one linear verification pipeline - code: str, - fixture: Any, - *, - timeout: float = 60.0, - python: str | None = None, -) -> GeneratedCodeResult: - """Run the full verification pipeline for one generated script.""" - - failures: list[str] = [] - stages: list[str] = [] - scanner = None - canaries = dict(getattr(fixture, "pii_canaries", {}) or {}) - if canaries: - scanner = SinkScanner.from_canaries(canaries, key=b"truthbench-generated-code-v1") - - def leaks(label: str, payload: Any) -> None: - if scanner is None: - return - for leak in scanner.scan(payload): - failures.append(f"{label} leaked canary {leak.canary_id} at {leak.path}") - - # 1. parse - try: - tree = ast.parse(code) - except SyntaxError as exc: - return GeneratedCodeResult( - False, (f"generated code does not parse: {exc.msg}",), stages=("parse",) - ) - stages.append("parse") - - # 2. strict AST allowlist - allow = _allowlist_failures(tree) - failures.extend(allow) - stages.append("allowlist") - - # 3. compile - try: - compile(code, "generated_pipeline.py", "exec") - except (SyntaxError, ValueError) as exc: - failures.append(f"generated code does not compile: {exc}") - return GeneratedCodeResult(False, tuple(failures), stages=tuple(stages)) - stages.append("compile") - - leaks("generated source", code) - if failures: - return GeneratedCodeResult(False, tuple(failures), stages=tuple(stages)) - - frame = getattr(fixture, "frame", fixture) - if not isinstance(frame, pd.DataFrame): - raise TypeError("generated-code verification requires a pandas fixture frame") - - # 4-9. isolated execution with the defined input contract - with tempfile.TemporaryDirectory(prefix="truthbench-gen-") as tmp: - workdir = Path(tmp) - input_path = workdir / _INPUT_BASENAME - frame.to_csv(input_path, index=False) - code_path = workdir / "generated_pipeline.py" - code_path.write_text(code, encoding="utf-8") - harness_path = workdir / "harness.py" - harness_path.write_text( - _harness(str(code_path), _INPUT_BASENAME), encoding="utf-8" - ) - env = _sandbox_env(workdir) - interpreter = python or sys.executable - - def redacted(text: str) -> str: - return str(scanner.redact(text)) if scanner is not None else text - - native_crashes: list[str] = [] - try: - for attempt in range(1 + _NATIVE_CRASH_RETRIES): - if attempt: - # A crashed attempt may have left the input half-written; - # give the retry the same input contract as the first run. - frame.to_csv(input_path, index=False) - proc = subprocess.run( # noqa: PLW1510 - exit code inspected below - # -X faulthandler: a native crash (negative exit code) dumps - # the Python traceback to stderr, which is redacted and reported. - [interpreter, "-I", "-X", "faulthandler", str(harness_path)], - cwd=workdir, - env=env, - capture_output=True, - text=True, - timeout=timeout, - ) - if proc.returncode >= 0: - break - native_crashes.append( - f"exited {proc.returncode}: {_stderr_excerpt(redacted(proc.stderr))}" - ) - except subprocess.TimeoutExpired: - return GeneratedCodeResult( - False, - (*failures, f"generated code exceeded the {timeout:.0f}s timeout"), - stages=(*stages, "execute"), - native_crashes=tuple(native_crashes), - ) - stdout, stderr = proc.stdout or "", proc.stderr or "" - started = f"{_STARTED_MARKER}\n" - if proc.returncode >= 0 and not stdout.startswith(started): - # Fail closed: the generated code never ran, so the overwrite and - # canary checks below would observe nothing; report no verdict. - reason = ( - "sandbox infrastructure failure: the child interpreter exited " - f"with code {proc.returncode} before starting the harness: " - f"{_stderr_excerpt(redacted(stderr))}" - ) - return GeneratedCodeResult( - False, - (*failures, reason), - stderr=redacted(stderr), - stages=tuple(stages), - native_crashes=tuple(native_crashes), - infrastructure_failure=reason, - ) - stdout = stdout[len(started) :] - stages.append("execute") - if proc.returncode != 0: - failures.append( - f"generated code exited {proc.returncode}: " - f"{_stderr_excerpt(redacted(stderr))}" - ) - - produced = tuple( - sorted( - str(path.relative_to(workdir)) - for path in workdir.rglob("*") - if path.is_file() - and path not in {input_path, code_path, harness_path} - ) - ) - - # 10-11. input contract and protected-cell identity - after = pd.read_csv(input_path, dtype=str, keep_default_na=False) - before = pd.read_csv( - _rewrite(frame, workdir), dtype=str, keep_default_na=False - ) - if not after.equals(before): - failures.append("generated code modified its input file in place") - protected = tuple(getattr(fixture, "protected_columns", ()) or ()) - for column in protected: - if column in after.columns and not after[column].equals(before[column]): - failures.append(f"generated code modified protected column {column!r}") - # 12. PII canaries in any external sink - leaks("stdout", stdout) - leaks("stderr", stderr) - for name in produced: - try: - payload = (workdir / name).read_text(encoding="utf-8", errors="replace") - except OSError: - continue - leaks(f"produced file {name}", payload) - - safe_out: Any = stdout - safe_err: Any = stderr - if scanner is not None: - safe_out = scanner.redact(stdout) - safe_err = scanner.redact(stderr) - return GeneratedCodeResult( - not failures, - tuple(failures), - stdout=str(safe_out), - stderr=str(safe_err), - produced_files=produced, - stages=tuple(stages), - native_crashes=tuple(native_crashes), - ) - - -def _rewrite(frame: pd.DataFrame, workdir: Path) -> Path: - """Write a reference copy of the input CSV for byte-comparison.""" - - reference = workdir / ".reference_input.csv" - frame.to_csv(reference, index=False) - return reference - - -__all__ = [ - "ALLOWED_IMPORTS", - "BANNED_CALLS", - "POISONED_MODULES", - "GeneratedCodeResult", - "verify_generated_code", -] diff --git a/benchmarks/truthbench/minimize.py b/benchmarks/truthbench/minimize.py deleted file mode 100644 index c83c5a7b..00000000 --- a/benchmarks/truthbench/minimize.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Deterministic reduction of reproduced TruthBench failures. - -The minimizer removes background rows and columns while the failure's target -cell keeps failing, under a bounded evaluation budget. The target cell and -its gold disposition are immutable: reduction that "fixes" the failure by -deleting the evidence is rejected by construction. -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from typing import Any - -import pandas as pd - -from .exact import canonical_json, stable_digest -from .privacy import SinkScanner - -_HASH_KEY = b"truthbench-minimize-v1" - -#: Maximum candidate frames evaluated per failure. -DEFAULT_BUDGET = 60 - - -@dataclass(frozen=True) -class FailureCase: - """A sanitized, reproducible minimal failure.""" - - failure_id: str - cell_id: str - domain: str - surface: str - gate: str - expected: Any - actual: Any - component: str - frame_records: tuple[Mapping[str, Any], ...] - schema: Mapping[str, Any] - policy: Mapping[str, Any] - reproduce_command: str - evidence_hash: str - evaluations: int - - def to_dict(self) -> dict[str, Any]: - return { - "failure_id": self.failure_id, - "cell_id": self.cell_id, - "domain": self.domain, - "surface": self.surface, - "gate": self.gate, - "expected": self.expected, - "actual": self.actual, - "component": self.component, - "frame_records": [dict(row) for row in self.frame_records], - "schema": dict(self.schema), - "policy": dict(self.policy), - "reproduce_command": self.reproduce_command, - "evidence_hash": self.evidence_hash, - "evaluations": self.evaluations, - } - - -def _sanitize(frame: pd.DataFrame, fixture: Any) -> pd.DataFrame: - canaries = dict(getattr(fixture, "pii_canaries", {}) or {}) - if not canaries: - return frame - scanner = SinkScanner.from_canaries(canaries, key=_HASH_KEY) - return scanner.redact(frame) - - -def minimize_failure( - fixture: Any, - *, - cell_id: str, - gate: str, - surface: str, - expected: Any, - actual: Any, - component: str, - still_fails: Callable[[pd.DataFrame], bool], - budget: int = DEFAULT_BUDGET, -) -> FailureCase: - """Reduce ``fixture.frame`` while ``still_fails`` holds for the target cell. - - ``still_fails`` receives a candidate frame (always containing the target - cell's row and column) and must re-run the public surface to decide - whether the original failure reproduces. Reduction is greedy and - deterministic: first drop row blocks, then single rows, then columns. - """ - - cell = next((c for c in fixture.cells if c.cell_id == cell_id), None) - if cell is None: - raise ValueError(f"unknown target cell: {cell_id}") - - frame = fixture.frame.copy(deep=True) - evaluations = 0 - - def attempt(candidate: pd.DataFrame) -> bool: - nonlocal evaluations - if evaluations >= budget: - return False - if cell.row_id not in candidate.index or cell.column not in candidate.columns: - return False # never remove the target cell - evaluations += 1 - try: - return bool(still_fails(candidate.copy(deep=True))) - except Exception: - return False - - # Pass 1: halve away row blocks (ddmin-style), keeping the target row. - chunk = max(1, len(frame) // 2) - while chunk >= 1: - index = list(frame.index) - position = 0 - while position < len(index): - block = [ - row - for row in index[position : position + chunk] - if row != cell.row_id - ] - if block: - candidate = frame.drop(index=block) - if attempt(candidate): - frame = candidate - index = list(frame.index) - continue - position += chunk - if chunk == 1: - break - chunk //= 2 - - # Pass 2: drop background columns one at a time. - for column in list(frame.columns): - if column == cell.column: - continue - candidate = frame.drop(columns=[column]) - if attempt(candidate): - frame = candidate - - safe = _sanitize(frame, fixture) - records = tuple( - {"row_id": str(row), **{str(c): _cell_payload(safe.at[row, c]) for c in safe.columns}} - for row in safe.index - ) - evidence = { - "cell_id": cell_id, - "gate": gate, - "surface": surface, - "expected": expected, - "actual": actual, - "records": records, - } - evidence_hash = stable_digest(canonical_json(evidence), key=_HASH_KEY) - failure_id = "tbf-" + hashlib.sha256( - canonical_json( - {"cell": cell_id, "gate": gate, "surface": surface} - ).encode("utf-8") - ).hexdigest()[:16] - return FailureCase( - failure_id=failure_id, - cell_id=cell_id, - domain=str(fixture.domain), - surface=surface, - gate=gate, - expected=expected, - actual=actual, - component=component, - frame_records=records, - schema=dict(getattr(fixture, "schema", {}) or {}), - policy=dict(getattr(fixture, "policy", {}) or {}), - reproduce_command=( - "PYTHONPATH=src python -m benchmarks.truthbench reproduce " - f"--failure-id {failure_id} --domain {fixture.domain} " - f"--cell-id '{cell_id}' --surface {surface}" - ), - evidence_hash=evidence_hash, - evaluations=evaluations, - ) - - -def _cell_payload(value: Any) -> Any: - if value is None or isinstance(value, (str, int, float, bool)): - return value - return repr(value) - - -__all__ = ["DEFAULT_BUDGET", "FailureCase", "minimize_failure"] diff --git a/benchmarks/truthbench/models.py b/benchmarks/truthbench/models.py deleted file mode 100644 index 28156672..00000000 --- a/benchmarks/truthbench/models.py +++ /dev/null @@ -1,362 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from enum import Enum -from typing import Any - -from .exact import JsonValue, TypedValue, encode_typed - - -class Disposition(str, Enum): - PRESERVE = "preserve" - REPAIR = "repair" - FLAG = "flag" - REVIEW = "review" - - -class _Unset: - pass - - -UNSET = _Unset() -_TEST_DIGEST_KEY = b"truthbench-decision-record-for-test-v1" - - -def _disposition(value: Disposition | str) -> Disposition: - try: - return Disposition(value) - except ValueError as exc: - raise ValueError(f"unknown TruthBench disposition: {value!r}") from exc - - -def _stable_id(*components: str) -> str: - if any(not isinstance(item, str) or not item for item in components): - raise ValueError("stable ID components must be non-empty strings") - if any(":" in item for item in components): - raise ValueError("stable ID components must not contain a colon") - return ":".join(components) - - -@dataclass(frozen=True) -class GoldCell: - cell_id: str - fixture_version: str - domain: str - row_id: str - column: str - disposition: Disposition - expected_output: TypedValue | None = None - family: str | None = None - sensitive: bool = False - canary_id: str | None = None - schema_version: int = field(default=1, init=False) - - @classmethod - def create( - cls, - fixture_version: str, - domain: str, - row_id: str, - column: str, - disposition: Disposition | str, - *, - expected_output: Any = UNSET, - expected_dtype: Any = None, - family: str | None = None, - sensitive: bool = False, - canary_id: str | None = None, - digest_key: bytes = _TEST_DIGEST_KEY, - ) -> GoldCell: - cell_id = _stable_id(fixture_version, domain, row_id, column) - typed_output = None - if expected_output is not UNSET: - typed_output = encode_typed( - expected_output, - dtype=expected_dtype, - sensitive=sensitive, - digest_key=digest_key if sensitive else None, - ) - return cls( - cell_id=cell_id, - fixture_version=fixture_version, - domain=domain, - row_id=row_id, - column=column, - disposition=_disposition(disposition), - expected_output=typed_output, - family=family, - sensitive=sensitive, - canary_id=canary_id, - ) - - def to_dict(self) -> dict[str, JsonValue]: - return { - "schema_version": self.schema_version, - "cell_id": self.cell_id, - "fixture_version": self.fixture_version, - "domain": self.domain, - "row_id": self.row_id, - "column": self.column, - "disposition": self.disposition.value, - "expected_output": ( - None if self.expected_output is None else self.expected_output.to_dict() - ), - "family": self.family, - "sensitive": self.sensitive, - "canary_id": self.canary_id, - } - - -@dataclass(frozen=True) -class CaseExpectation: - case_id: str - fixture_version: str - domain: str - kind: str - name: str - disposition: Disposition - expected: TypedValue | None = None - family: str | None = None - sensitive: bool = False - schema_version: int = field(default=1, init=False) - - @classmethod - def create( - cls, - fixture_version: str, - domain: str, - kind: str, - name: str, - disposition: Disposition | str, - *, - expected: Any = UNSET, - expected_dtype: Any = None, - family: str | None = None, - sensitive: bool = False, - digest_key: bytes = _TEST_DIGEST_KEY, - ) -> CaseExpectation: - case_id = _stable_id(fixture_version, domain, kind, name) - typed_expected = None - if expected is not UNSET: - typed_expected = encode_typed( - expected, - dtype=expected_dtype, - sensitive=sensitive, - digest_key=digest_key if sensitive else None, - ) - return cls( - case_id=case_id, - fixture_version=fixture_version, - domain=domain, - kind=kind, - name=name, - disposition=_disposition(disposition), - expected=typed_expected, - family=family, - sensitive=sensitive, - ) - - def to_dict(self) -> dict[str, JsonValue]: - return { - "schema_version": self.schema_version, - "case_id": self.case_id, - "fixture_version": self.fixture_version, - "domain": self.domain, - "kind": self.kind, - "name": self.name, - "disposition": self.disposition.value, - "expected": None if self.expected is None else self.expected.to_dict(), - "family": self.family, - "sensitive": self.sensitive, - } - - -@dataclass(frozen=True) -class DecisionRecord: - record_id: str - run_id: str - fixture_id: str - case_id: str | None - cell_id: str - domain: str - row_id: str - column: str - surface: str - repeat: int - expected_disposition: Disposition - actual_disposition: Disposition | None - sensitive: bool - input: TypedValue - expected_output: TypedValue | None - actual_output: TypedValue | None - confidence: float | None = None - risk: str | None = None - status: str | None = None - rule_id: str | None = None - rationale: str | None = None - evidence_kinds: tuple[str, ...] | None = None - mutated: bool | None = None - detected: bool | None = None - quarantined: bool | None = None - human_review: bool | None = None - audit_required: bool | None = None - audit_complete: bool | None = None - audit_ids: tuple[str, ...] | None = None - trust_before: float | None = None - trust_after: float | None = None - trust_delta: float | None = None - requested_backend: str | None = None - actual_backend: str | None = None - fallback_events: tuple[str, ...] | None = None - backend_differences: tuple[str, ...] | None = None - normalized_decision_hash: str | None = None - repeat_hash: str | None = None - repeat_consistent: bool | None = None - schema_version: int = field(default=1, init=False) - - def __post_init__(self) -> None: - if not self.sensitive: - return - for name in ("input", "expected_output", "actual_output"): - value = getattr(self, name) - if value is not None and not ( - value.redacted - and value.value is None - and value.display == "[REDACTED]" - and bool(value.digest) - ): - raise ValueError( - f"sensitive DecisionRecord {name} must be a redacted TypedValue" - ) - - @classmethod - def for_test(cls, *, cell: GoldCell, input_value: Any) -> DecisionRecord: - typed_input = encode_typed( - input_value, - sensitive=cell.sensitive, - digest_key=_TEST_DIGEST_KEY if cell.sensitive else None, - ) - return cls( - record_id=f"test:{cell.cell_id}", - run_id="test", - fixture_id=f"{cell.fixture_version}:{cell.domain}", - case_id=None, - cell_id=cell.cell_id, - domain=cell.domain, - row_id=cell.row_id, - column=cell.column, - surface="test", - repeat=0, - expected_disposition=cell.disposition, - actual_disposition=None, - input=typed_input, - expected_output=cell.expected_output, - actual_output=None, - sensitive=cell.sensitive, - ) - - @staticmethod - def _typed(value: TypedValue | None) -> dict[str, JsonValue] | None: - return None if value is None else value.to_dict() - - def to_dict(self) -> dict[str, JsonValue]: - return { - "schema_version": self.schema_version, - "record_id": self.record_id, - "run_id": self.run_id, - "fixture_id": self.fixture_id, - "case_id": self.case_id, - "cell_id": self.cell_id, - "domain": self.domain, - "row_id": self.row_id, - "column": self.column, - "surface": self.surface, - "repeat": self.repeat, - "expected_disposition": self.expected_disposition.value, - "actual_disposition": ( - None if self.actual_disposition is None else self.actual_disposition.value - ), - "sensitive": self.sensitive, - "input": self.input.to_dict(), - "input_type": self.input.type_label, - "expected_output": self._typed(self.expected_output), - "expected_output_type": ( - None if self.expected_output is None else self.expected_output.type_label - ), - "actual_output": self._typed(self.actual_output), - "actual_output_type": ( - None if self.actual_output is None else self.actual_output.type_label - ), - "confidence": self.confidence, - "risk": self.risk, - "status": self.status, - "rule_id": self.rule_id, - "rationale": self.rationale, - "evidence_kinds": ( - None if self.evidence_kinds is None else list(self.evidence_kinds) - ), - "mutated": self.mutated, - "detected": self.detected, - "quarantined": self.quarantined, - "human_review": self.human_review, - "audit_required": self.audit_required, - "audit_complete": self.audit_complete, - "audit_ids": None if self.audit_ids is None else list(self.audit_ids), - "trust_before": self.trust_before, - "trust_after": self.trust_after, - "trust_delta": self.trust_delta, - "requested_backend": self.requested_backend, - "actual_backend": self.actual_backend, - "fallback_events": ( - None if self.fallback_events is None else list(self.fallback_events) - ), - "backend_differences": ( - None if self.backend_differences is None else list(self.backend_differences) - ), - "normalized_decision_hash": self.normalized_decision_hash, - "repeat_hash": self.repeat_hash, - "repeat_consistent": self.repeat_consistent, - } - - -@dataclass(frozen=True) -class GateResult: - name: str - passed: bool - failures: tuple[str, ...] = () - schema_version: int = field(default=1, init=False) - - def to_dict(self) -> dict[str, JsonValue]: - return { - "schema_version": self.schema_version, - "name": self.name, - "passed": self.passed, - "failure_count": len(self.failures), - "failures": list(self.failures), - } - - -@dataclass(frozen=True) -class RunResult: - run_id: str - profile: str - fixture_hashes: tuple[tuple[str, str], ...] - required_backends: tuple[str, ...] - records: tuple[DecisionRecord, ...] - gates: tuple[GateResult, ...] - summary: tuple[tuple[str, JsonValue], ...] - environment: tuple[tuple[str, JsonValue], ...] - schema_version: int = field(default=1, init=False) - - def to_dict(self) -> dict[str, JsonValue]: - return { - "schema_version": self.schema_version, - "run_id": self.run_id, - "profile": self.profile, - "fixture_hashes": dict(self.fixture_hashes), - "required_backends": list(self.required_backends), - "records": [record.to_dict() for record in self.records], - "gates": [gate.to_dict() for gate in self.gates], - "summary": dict(self.summary), - "environment": dict(self.environment), - } diff --git a/benchmarks/truthbench/normalize.py b/benchmarks/truthbench/normalize.py deleted file mode 100644 index 0b88e1c4..00000000 --- a/benchmarks/truthbench/normalize.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Normalize heterogeneous public-surface evidence into TruthBench records. - -This module deliberately has no cleaning policy. It only captures what a -public surface did, preserves typed values, and leaves pass/fail decisions to -``gates``. Missing evidence therefore remains missing rather than being -silently inferred as a successful repair. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Any - -import pandas as pd - -from .exact import ( - canonical_scalar, - encode_typed, - equivalent_after_type_normalization, - exact_equal, - numeric_value_equal, -) -from .models import DecisionRecord, Disposition -from .surfaces.base import SurfaceObservation - -_SENSITIVE_KEY = b"truthbench-normalized-sensitive-v1" - - -@dataclass(frozen=True) -class CaseRecord: - """Non-serialized row/schema evidence retained for gate context. - - ``RunResult`` intentionally serializes only ``DecisionRecord``. Cases are - contextual evidence for the runner and release gates, never an alternative - public result schema. - """ - - case_id: str - kind: str - expected_disposition: Disposition - observed: bool - - -@dataclass(frozen=True) -class NormalizationResult: - records: tuple[DecisionRecord, ...] - cases: tuple[CaseRecord, ...] - input_mutated: bool = False - - -def _mapping(value: Any) -> Mapping[str, Any]: - return value if isinstance(value, Mapping) else {} - - -def _decision_for(raw: Any, cell_id: str) -> Mapping[str, Any]: - """Extract one cell's public decision without assuming a report type.""" - - source = _mapping(raw) - direct = source.get(cell_id) - if isinstance(direct, Mapping): - return direct - for name in ("decisions", "records", "actions", "findings"): - values = source.get(name) - if isinstance(values, Mapping) and isinstance(values.get(cell_id), Mapping): - return values[cell_id] - if isinstance(values, (list, tuple)): - for item in values: - if not isinstance(item, Mapping): - continue - if item.get("cell_id") == cell_id or item.get("id") == cell_id: - return item - return {} - - -def _audit_ids(sinks: Any, cell_id: str) -> tuple[str, ...]: - """Collect matching IDs from public audit payloads, never their contents.""" - - found: list[str] = [] - - def visit(value: Any) -> None: - if isinstance(value, Mapping): - for key, item in value.items(): - if key in {"record_id", "cell_id", "id"} and item == cell_id: - found.append(cell_id) - if ( - key in {"record_ids", "cell_ids", "ids"} - and isinstance(item, (list, tuple)) - and cell_id in item - ): - found.append(cell_id) - visit(item) - elif isinstance(value, (list, tuple)): - for item in value: - visit(item) - - visit(sinks) - return (cell_id,) if found else () - - -def _bool(decision: Mapping[str, Any], name: str, default: bool | None = None) -> bool | None: - value = decision.get(name, default) - return value if isinstance(value, bool) else default - - -def _disposition(value: Any) -> Disposition | None: - if value is None: - return None - try: - return Disposition(value) - except ValueError: - return None - - -def _output_value( - fixture: Any, cell: Any, observation: SurfaceObservation -) -> tuple[Any, Any] | None: - frame = observation.output_frame - if not isinstance(frame, pd.DataFrame): - return None - if cell.row_id not in frame.index or cell.column not in frame.columns: - return None - return canonical_scalar(frame.at[cell.row_id, cell.column]), frame[cell.column].dtype - - -def _case_observed(raw: Any, case_id: str) -> bool: - """Return explicit row/schema observation evidence without inventing it.""" - - decision = _decision_for(raw, case_id) - return decision.get("observed") is True or decision.get("detected") is True - - -def normalize_observation( - fixture: Any, - observation: SurfaceObservation, - *, - surface: str, - backend: str, - repeat: int, - run_id: str, -) -> NormalizationResult: - """Produce one record for every labelled cell in ``fixture``. - - Validators may detect bad values without returning a frame. In that case - ``actual_output`` remains ``None`` and ``mutated`` remains false; gates can - give detection credit without inventing a mutation. - """ - - frame = getattr(fixture, "frame", None) - if not isinstance(frame, pd.DataFrame): - raise TypeError("normalization requires a pandas TruthFixture frame") - raw = observation.raw_decisions - trust = _mapping(observation.trust) - disclosure = _mapping(observation.backend_disclosure) - records: list[DecisionRecord] = [] - for cell in fixture.cells: - decision = _decision_for(raw, cell.cell_id) - source_value = canonical_scalar(frame.at[cell.row_id, cell.column]) - source_dtype = frame[cell.column].dtype - output = _output_value(fixture, cell, observation) - actual_value, actual_dtype = output if output is not None else (None, None) - changed = output is not None and not exact_equal( - actual_value, source_value, left_dtype=actual_dtype, right_dtype=source_dtype - ) - if changed and not cell.sensitive: - # Mirror the gate-side rule: a value-preserving type - # normalization is not a mutation. - typed_actual = encode_typed(actual_value, dtype=actual_dtype) - typed_source = encode_typed(source_value, dtype=source_dtype) - changed = not ( - equivalent_after_type_normalization(typed_actual, typed_source) - or numeric_value_equal(typed_actual, typed_source) - ) - explicit_mutated = _bool(decision, "mutated") - mutated = explicit_mutated if explicit_mutated is not None else changed - detected = _bool(decision, "detected", False) - quarantined = _bool(decision, "quarantined", False) - human_review = _bool(decision, "human_review", False) - actual_disposition = _disposition(decision.get("disposition")) - if actual_disposition is None: - if cell.disposition is Disposition.FLAG and (detected or quarantined): - actual_disposition = Disposition.FLAG - elif cell.disposition is Disposition.REVIEW and (detected or human_review): - actual_disposition = Disposition.REVIEW - elif cell.disposition is Disposition.REPAIR and (mutated or output is not None): - actual_disposition = Disposition.REPAIR - elif not mutated: - actual_disposition = Disposition.PRESERVE - audit_ids = _audit_ids(observation.audit_sinks, cell.cell_id) - audit_required = ( - cell.disposition is not Disposition.PRESERVE or bool(mutated) or bool(detected) - ) - confidence = decision.get("confidence") - if not isinstance(confidence, (int, float)) or isinstance(confidence, bool): - confidence = None - trust_before = trust.get("trust_before") - trust_after = trust.get("trust_after") - if not isinstance(trust_before, (int, float)) or isinstance(trust_before, bool): - trust_before = None - if not isinstance(trust_after, (int, float)) or isinstance(trust_after, bool): - trust_after = None - records.append( - DecisionRecord( - record_id=f"{run_id}:{surface}:{backend}:{repeat}:{cell.cell_id}", - run_id=run_id, - fixture_id=f"{cell.fixture_version}:{cell.domain}", - case_id=None, - cell_id=cell.cell_id, - domain=cell.domain, - row_id=cell.row_id, - column=cell.column, - surface=surface, - repeat=repeat, - expected_disposition=cell.disposition, - actual_disposition=actual_disposition, - sensitive=cell.sensitive, - input=encode_typed( - source_value, - dtype=source_dtype, - sensitive=cell.sensitive, - digest_key=_SENSITIVE_KEY if cell.sensitive else None, - ), - expected_output=cell.expected_output, - actual_output=( - None - if output is None - else encode_typed( - actual_value, - dtype=actual_dtype, - sensitive=cell.sensitive, - digest_key=_SENSITIVE_KEY if cell.sensitive else None, - ) - ), - confidence=float(confidence) if confidence is not None else None, - risk=decision.get("risk") if isinstance(decision.get("risk"), str) else None, - status=decision.get("status") if isinstance(decision.get("status"), str) else None, - rule_id=( - decision.get("rule_id") - if isinstance(decision.get("rule_id"), str) - else decision.get("model_id") - if isinstance(decision.get("model_id"), str) - else None - ), - rationale=decision.get("rationale") - if isinstance(decision.get("rationale"), str) - else None, - evidence_kinds=tuple( - str(item) - for item in decision.get("evidence_kinds", ()) - if isinstance(item, str) - ) - or None, - mutated=mutated, - detected=detected, - quarantined=quarantined, - human_review=human_review, - audit_required=audit_required, - audit_complete=(bool(audit_ids) if audit_required else True), - audit_ids=audit_ids or None, - trust_before=float(trust_before) if trust_before is not None else None, - trust_after=float(trust_after) if trust_after is not None else None, - trust_delta=( - round(float(trust_after - trust_before), 12) - if trust_before is not None and trust_after is not None - else None - ), - requested_backend=( - disclosure.get("requested") - if isinstance(disclosure.get("requested"), str) - else None - ), - actual_backend=( - disclosure.get("actual") if isinstance(disclosure.get("actual"), str) else None - ), - ) - ) - cases = tuple( - CaseRecord( - case.case_id, - case.kind, - case.disposition, - _case_observed(raw, case.case_id), - ) - for case in (*getattr(fixture, "row_cases", ()), *getattr(fixture, "schema_cases", ())) - ) - return NormalizationResult(tuple(records), cases) - - -__all__ = ["CaseRecord", "NormalizationResult", "normalize_observation"] diff --git a/benchmarks/truthbench/privacy.py b/benchmarks/truthbench/privacy.py deleted file mode 100644 index f373c9e5..00000000 --- a/benchmarks/truthbench/privacy.py +++ /dev/null @@ -1,743 +0,0 @@ -"""Privacy-safe TruthBench sink values and exhaustive canary scanning. - -The scanner deliberately stores only canary identifiers, transform names, paths, -and run-scoped HMAC digests. Matched input is never included in a ``Leak`` or -in a redaction marker. -""" - -from __future__ import annotations - -import dataclasses -import hashlib -import hmac -import html -import json -import re -import unicodedata -import urllib.parse -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Any, Callable - -import pandas as pd - -from .exact import canonical_json, encode_typed - - -@dataclass(frozen=True) -class Leak: - """A privacy leak without retaining the matched value.""" - - canary_id: str - variant: str - path: str - - -@dataclass(frozen=True) -class PrivacySafeValue: - """A leaf value represented by a redacted display and keyed digest.""" - - value: str = "[REDACTED]" - digest: str = "" - - def __repr__(self) -> str: - return f"PrivacySafeValue(value={self.value!r}, digest={self.digest!r})" - - -_ZERO_WIDTH = "\u200b\u200c\u200d\ufeff" - - -def _remove_zero_width(value: str) -> str: - return "".join( - char for char in value if char not in _ZERO_WIDTH and unicodedata.category(char) != "Cf" - ) - - -def _remove_punctuation(value: str) -> str: - # Symbols (category S*: "+", "=", currency signs) strip alongside - # punctuation so a canary matches text scrubbed of *all* non-word marks. - return "".join( - char for char in value if unicodedata.category(char)[0] not in ("P", "S") - ) - - -def _json_escaped(value: str) -> str: - encoded = json.dumps(value, ensure_ascii=True) - return encoded[1:-1] - - -def _digit_only(value: str) -> str: - return "".join(char for char in value if char.isdigit()) - - -#: A run of digit groups joined only by the short separators numeric -#: identifiers use: whitespace, ".", "-", "/" and parentheses. Digits split by -#: letters, markup or any other character belong to different runs. -_NUMERIC_RUN = re.compile(r"\d+(?:[\s./()\-]{1,3}\d+)*") - -#: Text forms whose numeric runs are searched for digit-only canaries. -_NUMERIC_RUN_FORMS = ("literal", "html-unescaped", "url-decoded", "nfkc", "zero-width-removed") - - -def _numeric_digit_runs(forms: Mapping[str, str]) -> frozenset[str]: - """Return the digits of every numeric run in the searchable text forms.""" - - return frozenset( - _digit_only(run) - for name in _NUMERIC_RUN_FORMS - for run in _NUMERIC_RUN.findall(forms.get(name, "")) - ) - - -Transform = Callable[[str], str] - -# Public names make the normalization contract inspectable and testable. -NORMALIZERS: tuple[tuple[str, Transform], ...] = ( - ("literal", lambda value: value), - ("casefold", str.casefold), - ("whitespace-stripped", lambda value: re.sub(r"\s+", "", value)), - ("punctuation-stripped", _remove_punctuation), - ("digit-only", _digit_only), - ("url-decoded", urllib.parse.unquote), - ("html-unescaped", html.unescape), - ("nfkc", lambda value: unicodedata.normalize("NFKC", value)), - ("nfc", lambda value: unicodedata.normalize("NFC", value)), - ("nfd", lambda value: unicodedata.normalize("NFD", value)), - ("zero-width-removed", _remove_zero_width), - ("json-escaped", _json_escaped), -) - - -def normalize_variants(value: str) -> dict[str, str]: - """Return named normalized forms of text, omitting empty forms.""" - - if not isinstance(value, str): - raise TypeError("normalize_variants expects text") - result: dict[str, str] = {} - for name, transform in NORMALIZERS: - try: - normalized = transform(value) - except (UnicodeError, ValueError): - continue - if normalized: - result.setdefault(name, normalized) - return result - - -def _text_forms(value: Any) -> dict[str, str]: - """Convert text/bytes to forms which can be safely searched.""" - - if isinstance(value, str): - return normalize_variants(value) - if isinstance(value, bytes): - forms: dict[str, str] = {} - try: - decoded = value.decode("utf-8") - except UnicodeDecodeError: - decoded = "" - if decoded: - for name, text in normalize_variants(decoded).items(): - forms.setdefault(name, text) - forms["bytes-hex"] = value.hex() - forms["bytes-escape"] = value.decode("latin1") - forms["bytes-repr"] = repr(value) - return forms - return {} - - -def _path_component(path: str, key: Any) -> str: - if isinstance(key, str): - key_text = key - elif isinstance(key, (bool, int, float, complex)): - key_text = str(key) - else: - # Composite/custom labels can invoke hostile ``__repr__`` methods; - # report a structural placeholder rather than serializing them. - return f"{path}[