diff --git a/.github/workflows/benchmark-gate.yml b/.github/workflows/benchmark-gate.yml index 75d90d4c..2263300e 100644 --- a/.github/workflows/benchmark-gate.yml +++ b/.github/workflows/benchmark-gate.yml @@ -33,10 +33,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e . polars pandas pyarrow pytest + pip install -e . polars pandas pyarrow pytest pytest-cov - name: Run gate & engine unit tests - run: pytest tests/test_polars_engine.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q + # pyproject addopts enforce 90% repo-wide coverage; this 3-file subset + # keeps coverage on with the lower floor that other subset lanes use. + run: pytest tests/test_backend_honesty.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q --cov-fail-under=40 - name: Record Candidate (PR Head) Benchmark run: | @@ -61,7 +63,9 @@ jobs: - name: Evaluate Regression Gate run: | - git checkout "${{ github.event.pull_request.head.sha }}" + # The base step copied the PR's benchmarks/ over the base tree; --force + # discards that overlay so PRs that change benchmarks/ can switch back. + git checkout --force "${{ github.event.pull_request.head.sha }}" python -m benchmarks.perf_gate compare \ --baseline /tmp/baseline_metrics.json \ --candidate /tmp/candidate_metrics.json \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6c4b47c..0ecb6d95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ concurrency: permissions: contents: read -# Dependency pinning: the gating jobs install with `-c constraints/ci.txt` -# (exported from uv.lock), so an upstream release cannot turn CI red overnight. +# Dependency pinning: the gating jobs install with `uv sync --frozen` from +# uv.lock, so an upstream release cannot turn CI red overnight. # test-matrix deliberately floats to the latest compatible releases: it is the -# early warning for upstream breakage. Regenerate with `make constraints`. +# early warning for upstream breakage. Regenerate with `uv lock`. jobs: lockfile: @@ -34,15 +34,10 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - name: uv.lock and constraints/ci.txt are up to date + - name: uv.lock is up to date run: | python -m pip install "uv==0.12.10" uv lock --check - uv export --frozen --no-hashes --all-extras --no-emit-project --quiet -o constraints/ci.txt - if ! git diff --exit-code -- constraints/ci.txt; then - echo "::error::constraints/ci.txt is stale; run 'make constraints' and commit the result" - exit 1 - fi quality-fast: runs-on: ubuntu-latest @@ -56,7 +51,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Lint run: ruff check . - name: Test (required fast lane) @@ -68,33 +64,6 @@ jobs: pip install "numpy<2.5" mypy src/freshdata - truthbench: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" - - name: TruthBench PR ratchet - # PRs must not regress any gate that passes in the committed - # baseline.json; the known-red gates are release blockers and are - # enforced absolutely by the release workflow (make truthbench-release), - # so they are reported here as KNOWN-RED without failing every - # unrelated PR. Infrastructure errors and partial runs still fail. - run: make truthbench-pr - - name: Upload run artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: truthbench-results - path: benchmarks/truthbench/results/ - test-matrix: runs-on: ubuntu-latest timeout-minutes: 30 @@ -144,7 +113,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Nightly online/large drift checks # --no-cov: this lane deliberately runs ~20 tests, so the repo-wide # --cov-fail-under=93 inherited from addopts can never be met here. @@ -190,7 +160,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Assert the large/online lane still collects tests run: | N=$(pytest -m "online or large or tier1" --collect-only -q -o addopts="" \ @@ -223,41 +194,6 @@ jobs: python -m build twine check dist/* - freshcore-native: - # Builds the optional FreshCore Rust extension (crates/freshcore) and runs - # the FreshCore tests against it. No other job installs the extension, so - # the native parity tests skip everywhere else. - runs-on: ubuntu-latest - timeout-minutes: 10 - env: - RUSTUP_TOOLCHAIN: "1.98.1" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install Rust toolchain - run: rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal - - name: Install - # maturin develop installs into a virtualenv, so create one in the - # workspace and put it first on PATH for the remaining steps. - run: | - python -m venv .venv - echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV" - echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - .venv/bin/python -m pip install --upgrade pip - .venv/bin/pip install -c constraints/ci.txt -e ".[dev,freshcore]" - - name: Rust unit tests - run: cargo test --locked --manifest-path crates/freshcore/Cargo.toml - - name: Build the extension - run: maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module - - name: FreshCore tests - # The coverage gate in addopts applies to the full suite, not this subset. - run: | - python -c "import freshdata_freshcore" - pytest tests/test_execution -k freshcore -o addopts="--strict-markers" -q - coverage-badge: # Publish a self-hosted shields endpoint badge to the `badges` branch # (no third-party account needed). Runs only on main. @@ -276,7 +212,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Compute coverage and write badge JSON run: | pytest -m "not online and not large and not tier1" \ diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml index d95ec659..a28eb19a 100644 --- a/.github/workflows/perf-regression.yml +++ b/.github/workflows/perf-regression.yml @@ -50,7 +50,8 @@ jobs: - name: Install run: | python -m pip install -U pip - pip install -c constraints/ci.txt -e ".[bench]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact - name: Resolve base commit id: base env: diff --git a/.github/workflows/performance-large.yml b/.github/workflows/performance-large.yml index f87be8b4..35077b5b 100644 --- a/.github/workflows/performance-large.yml +++ b/.github/workflows/performance-large.yml @@ -23,7 +23,8 @@ jobs: - name: Install benchmark dependencies run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,bench,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Run large performance matrix id: benchmark diff --git a/.github/workflows/plugin-contract.yml b/.github/workflows/plugin-contract.yml deleted file mode 100644 index 2dba406d..00000000 --- a/.github/workflows/plugin-contract.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Plugin contract tests - -# PR-CI-weight: the plugin registry, its safety guarantees (protected-column -# safety, risk ceiling, network-off-by-default, failure isolation), the -# fd.testing contract helpers, and the shipped examples/plugins/* stay valid. -on: - # pull_request: disabled in favor of ci-verification-gates.yml - # push: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - -permissions: - contents: read - -jobs: - plugin-contract: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install -U pip - pip install -e ".[dev]" - - name: Plugin system + contract tests - run: pytest tests/test_plugins.py -q --no-cov - - name: Example plugins are importable and contract-valid - run: | - python - <<'EOF' - import sys - import freshdata as fd - - sys.path[:0] = [ - "examples/plugins/custom_expert", - "examples/plugins/custom_backend", - "examples/plugins/custom_validator", - ] - from acronym_expert import AcronymExpert - from keyword_backend import KeywordBackend - from min_rows_validator import MinRowsValidator - - fd.testing.expert_contract(AcronymExpert()) - fd.testing.semantic_backend_contract(KeywordBackend()) - fd.testing.validator_contract(MinRowsValidator()) - print("all example plugins pass their contracts") - EOF diff --git a/.github/workflows/public-benchmark.yml b/.github/workflows/public-benchmark.yml index fdbc1cf5..bb736a0e 100644 --- a/.github/workflows/public-benchmark.yml +++ b/.github/workflows/public-benchmark.yml @@ -34,7 +34,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[bench]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact - name: Run public benchmark env: SCALE: ${{ inputs.scale }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7592a07d..a0ec9be2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -242,12 +242,6 @@ jobs: run: mkdocs build --strict - name: Validation Gauntlet run: python -m benchmarks.gauntlet run --rows 300 --check - - name: CleanBench release gates - run: python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --check-gates - - name: TruthBench release verification - # Fail-closed: a partial run, missing backend, or any absolute gate - # failure blocks the release before an artifact is built. - run: make truthbench-release # --------------------------------------------------------------------------- # 3. Build sdist + wheel exactly once; checksum them; attest provenance. diff --git a/.gitignore b/.gitignore index a37d06cc..dcad72b5 100644 --- a/.gitignore +++ b/.gitignore @@ -34,12 +34,9 @@ benchmarks/results/performance/* benchmarks/generated_fixtures/ # Rust native backend build output -crates/freshcore/target/ # Phase 5 training pipeline: teacher prompt/response audit cache is a local # dev artifact (regenerated by running teacher tasks), not committed content. -training/cache/* -!training/cache/.gitkeep benchmarks/gauntlet/results/ # TruthBench: the full run output is large and regenerated by the release # command; only the compact README and the regression baseline are committed. diff --git a/MANIFEST.in b/MANIFEST.in index 3384efb1..a07d895a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,6 @@ include README.md include LICENSE include pyproject.toml recursive-include src/freshdata *.py py.typed -recursive-include crates/freshcore Cargo.toml Cargo.lock *.rs prune tests prune benchmarks prune scripts diff --git a/Makefile b/Makefile index eb686466..e5786d27 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,9 @@ PY ?= python -# training-* targets are matched by the pattern rule below (pattern rules -# cannot be .PHONY; the delegated targets are .PHONY inside training/Makefile). .PHONY: help benchmark benchmark-ci benchmark-report benchmark-fixtures benchmark-test \ cleanbench-full truthbench-release truthbench-pr automation-pr performance-ci \ - performance-baseline performance-profile performance-report coverage-report \ - constraints + performance-baseline performance-profile performance-report coverage-report help: @echo "Targets:" @@ -25,14 +22,6 @@ help: @echo " performance-profile Profile one 100k-row performance case" @echo " performance-report Analyze and render compact performance evidence" @echo " coverage-report Show per-module coverage for the fast test lane" - @echo " training-* Phase-5 training pipeline (see training/Makefile)" - @echo " constraints Refresh uv.lock and the pinned constraints/ci.txt CI installs with" - -# Refresh the lockfile and the pip constraints the gating CI jobs install with. -# The CI `lockfile` job fails when either is stale; commit both after running. -constraints: - uv lock - uv export --frozen --no-hashes --all-extras --no-emit-project --quiet -o constraints/ci.txt # Run the same test scope as the required PR lane and print uncovered lines for # every measured module so contributors can identify useful testing targets. @@ -69,10 +58,6 @@ truthbench-pr: automation-pr: $(PY) scripts/automate_freshdata.py --output-dir scripts/.automation_out -# Phase-5 training pipeline targets delegate to training/Makefile. -training-%: - $(MAKE) -C training PY=$(PY) $@ - # Full-scale local run. Override sizes per fixture by editing DEFAULT_SIZES in # benchmarks/bench.py, or call bench.py single --size for the 5M+ variants. benchmark: diff --git a/README.md b/README.md index 4671c5be..6bb5ae93 100644 --- a/README.md +++ b/README.md @@ -45,24 +45,26 @@ lf = pl.scan_parquet("s3://bucket/raw_customer_events.parquet") config = fd.CleanConfig( id_columns=("customer_id",), target_column="churned", - default_impute="auto", # median for numeric, mode for string - default_outliers="clip", # precomputed IQR clipping + default_impute="auto", # median for numeric columns + default_outliers="preserve", drop_duplicates=True, # streaming-safe unique(maintain_order=False) rules={ "signup_age": fd.ColumnRule(impute="median", outliers="clip"), - "invoice_total": fd.ColumnRule(outliers="flag"), }, ) -# 3. Generate a dry-run repair plan (single 1-row streaming summary query) +# 3. Generate a dry-run repair plan (IQR bounds come from one 1-row summary query) plan = fd.suggest_plan(lf, config=config) for action in plan.actions: - print(f"[{action.stage}] {action.column}: {action.operation} ({action.rationale})") -# [strings] segment: strip_whitespace (Strip leading and trailing whitespace) -# [missing] signup_age: fill_null (Impute 412 nulls via median=34.0) -# [outliers] signup_age: clip_iqr (IQR bounds [18, 72] (factor=1.5)) -# [missing] churned: preserve (Protected role='target' or conservative strategy) + print(f"[{action.stage}] {action.column}: {action.action} {action.parameters}") +# Example output (abridged) for raw columns "Customer ID", "Signup Age", "Segment", ...: +# [column_names] Signup Age: normalize_name {'to': 'signup_age'} +# [missing] signup_age: impute_median {} +# [strings] segment: strip_strings {} +# [outliers] signup_age: clip_iqr {'lower': 27.125, 'upper': 48.125} +# [duplicates] *: drop_duplicates {} +# customer_id (id) and churned (target) get no repair actions. # 4. Compile the approved plan onto the LazyFrame (zero materialization) cleaned_lf, report = fd.apply_plan(lf, plan, return_report=True) @@ -81,17 +83,18 @@ df = cleaned_lf.collect(engine="streaming") Never guess how a configuration change alters your dataset. Compare two `CleanPlan` artifacts directly in code or CI: ```python -conservative = fd.suggest_plan( - lf, fd.CleanConfig(strategy="conservative", default_outliers="preserve") -) -aggressive = fd.suggest_plan( - lf, fd.CleanConfig(strategy="aggressive", default_outliers="clip") -) - -for (col, stage), (before, after) in conservative.diff(aggressive).items(): - before_op = before.operation if before else "none" - after_op = after.operation if after else "none" - print(f"{col} ({stage}): {before_op} -> {after_op}") +import dataclasses + +clipped = fd.suggest_plan(lf, dataclasses.replace(config, default_outliers="clip")) +diff = plan.diff(clipped) + +for action in diff["added"]: + print(f"+ [{action.stage}] {action.column}: {action.action} {action.parameters}") +for action in diff["removed"]: + print(f"- [{action.stage}] {action.column}: {action.action}") +for change in diff["modified"]: + print(f"~ [{change['stage']}] {change['column']}: {change['before']['parameters']} -> {change['after']['parameters']}") +# + [outliers] invoice_total: clip_iqr {'lower': 7.125, 'upper': 15.125} ``` --- @@ -157,7 +160,7 @@ freshdata-migrate src/ tests/ Bug reports, benchmark reproductions, and pull requests are welcome. 1. Open an issue with a minimal reproducible Polars or pandas snippet. 2. Run the test suite locally via `pytest -q`. -3. Ensure any changes to `polars_engine.py` pass the lazy materialization poison test (`tests/test_polars_engine.py`). +3. Ensure any changes to `_core/backends.py` pass the lazy materialization poison test (`tests/test_backend_honesty.py`). ## License diff --git a/admin-actions.md b/admin-actions.md deleted file mode 100644 index 2a549a36..00000000 --- a/admin-actions.md +++ /dev/null @@ -1,71 +0,0 @@ -# FreshData GitHub Repository Administrative Actions - -The following administrative actions require GitHub repository administrator privileges on [`FreshCode-Org/freshdata`](https://github.com/FreshCode-Org/freshdata). These settings cannot be altered directly via Git commits. - ---- - -## 1. Repository Details & Description - -Navigate to the repository homepage `https://github.com/FreshCode-Org/freshdata` and click the **βš™οΈ (Edit repository details)** button on the top-right of the "About" box: - -* **Description**: - ```text - Automated, explainable data cleaning for pandas and Polars β€” repair messy tabular data safely and see exactly what changed. - ``` -* **Website**: - ```text - https://freshcode-org.github.io/freshdata/ - ``` - ---- - -## 2. GitHub Repository Topics (Discoverability & Search Intent) - -In the same "About" settings dialog, set the following **15 developer-search-intent topics** (replace any stale or generic tags): - -```text -data-cleaning -data-quality -data-preprocessing -dataframe -pandas -polars -python -tabular-data -data-validation -etl -data-engineering -data-science -machine-learning -missing-values -outlier-detection -``` - -> **Why these tags**: Developers actively search GitHub for `data-cleaning`, `missing-values`, and `pandas` when looking for libraries to replace fragile manual preprocessing scripts. - ---- - -## 3. GitHub Discussions Setup - -Navigate to **Settings** $\rightarrow$ **General** $\rightarrow$ **Features** $\rightarrow$ Check **Discussions**. - -Once enabled, navigate to the **Discussions** tab $\rightarrow$ **Categories** (click edit pencil) and configure these 6 official categories: - -| Category | Format | Description | -|---|---|---| -| πŸ“’ **Announcements** | Announcement (maintainers only) | Project releases, benchmarks, and community roadmap updates | -| πŸ’¬ **Q&A** | Q&A (mark answers) | Ask how to clean specific messy datasets or configure rules | -| πŸ’‘ **Ideas** | Open discussion | Share ideas for new cleaning heuristics, formats, or algorithms | -| πŸ™Œ **Show and Tell** | Open discussion | Share datasets, pipelines, and projects you cleaned using FreshData | -| ⚑ **Benchmarks** | Open discussion | Share benchmark runs across different machines and dataset sizes | -| πŸ”Œ **Integrations** | Open discussion | Discuss integrations with Polars, DuckDB, Airflow, dbt, and GX | - ---- - -## 4. Public Issues & Pull Requests Settings - -Ensure the following are enabled under **Settings** $\rightarrow$ **General**: - -* **Issues**: Checked (Enabled) -* **Automatically delete head branches**: Checked (keeps repository clean after PR merges) -* **Squash and merge**: Allowed (recommended with PR title and commit description) diff --git a/benchmarks/bench_entity_resolution.py b/benchmarks/bench_entity_resolution.py deleted file mode 100644 index 5cdb8ef8..00000000 --- a/benchmarks/bench_entity_resolution.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Benchmark for freshdata probabilistic entity resolution. - -Generates a synthetic dataset with *known* duplicate labels, runs the -rule-weighted linkage pipeline, and reports the metrics that matter for a -record-linkage system: - -* **candidate-pair reduction ratio** β€” how much blocking shrinks the O(n^2) - comparison space (1.0 means everything was pruned, 0.0 means full cartesian). -* **runtime** β€” wall-clock for the resolve call. -* **precision / recall / F1** β€” measured against the ground-truth duplicate - pairs (available because the synthetic generator tags each row's true entity). - -It runs both the ``pandas`` and ``duckdb`` backends so they can be compared -side by side (DuckDB is skipped automatically if not installed). - -> **Splink-class linkage target; current implementation is rule-weighted, not -> EM-trained unless calibration is enabled.** - - python benchmarks/bench_entity_resolution.py --rows 5000 --dup-rate 0.3 --json -""" - -from __future__ import annotations - -import argparse -import importlib.util -import json -import sys -import time -from itertools import combinations - -import numpy as np -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import BlockingRule, ComparisonLevel, EntityResolutionConfig - -_FIRST = ["alice", "bob", "carol", "dan", "erin", "frank", "grace", "heidi"] -_LAST = ["smith", "jones", "lee", "patel", "kim", "garcia", "khan", "obrien"] - - -def synth(rows: int, dup_rate: float, *, seed: int = 0) -> pd.DataFrame: - """Build *rows* records; a ``dup_rate`` fraction are noisy copies of earlier - records. ``entity_label`` is the ground-truth identity. - """ - rng = np.random.default_rng(seed) - records: list[dict] = [] - labels: list[int] = [] - next_label = 0 - for i in range(rows): - make_dup = records and rng.random() < dup_rate - if make_dup: - src_idx = int(rng.integers(0, len(records))) - base = records[src_idx] - label = labels[src_idx] - email = base["email"] - name = base["name"] - if rng.random() < 0.4: # perturb the name slightly - name = name + "e" if not name.endswith("e") else name[:-1] - if rng.random() < 0.3: # perturb the email casing - email = email.upper() - rec = { - "name": name, - "email": email, - "dob": base["dob"], - "phone": base["phone"], - } - else: - label = next_label - next_label += 1 - fn = rng.choice(_FIRST) - ln = rng.choice(_LAST) - rec = { - "name": f"{fn} {ln}", - "email": f"{fn}.{ln}{label}@mail.test", - "dob": f"19{rng.integers(60, 99):02d}-{rng.integers(1, 13):02d}-" - f"{rng.integers(1, 28):02d}", - "phone": f"555{rng.integers(1000000, 9999999)}", - } - rec["id"] = i - rec["entity_label"] = label - records.append(rec) - labels.append(label) - df = pd.DataFrame(records) - return df.sample(frac=1.0, random_state=seed).reset_index(drop=True) - - -def _config(backend: str) -> EntityResolutionConfig: - return EntityResolutionConfig( - enabled=True, - backend=backend, # type: ignore[arg-type] - unique_id_column="id", - blocking_rules=( - BlockingRule("lower(l.email) = lower(r.email)", "same email"), - BlockingRule( - "l.dob = r.dob and substr(lower(l.name), 1, 3) = " - "substr(lower(r.name), 1, 3)", - "same dob + name prefix", - ), - ), - comparisons=( - ComparisonLevel("name", "jaro_winkler", threshold=0.85, weight=2.0), - ComparisonLevel("email", "jaro_winkler", threshold=0.90, weight=3.0), - ComparisonLevel("dob", "exact", weight=2.0), - ComparisonLevel("phone", "levenshtein", threshold=0.85, weight=1.0), - ), - match_threshold=0.82, - clerical_review_threshold=0.6, - ) - - -def truth_pairs(df: pd.DataFrame) -> set[tuple]: - """All ground-truth duplicate id-pairs (order-independent).""" - out: set[tuple] = set() - for _, group in df.groupby("entity_label"): - ids = sorted(group["id"].tolist()) - for a, b in combinations(ids, 2): - out.add((a, b)) - return out - - -def _norm(a, b) -> tuple: - return (a, b) if a <= b else (b, a) - - -def evaluate(df: pd.DataFrame, backend: str) -> dict: - config = _config(backend) - t0 = time.perf_counter() - _, report = fd.resolve_entities(df, config=config) - runtime = time.perf_counter() - t0 - - n = len(df) - total_pairs = n * (n - 1) // 2 - reduction = 1.0 - (report.n_candidate_pairs / total_pairs) if total_pairs else 0.0 - - predicted = {_norm(p.left_id, p.right_id) for p in report.pairs if p.decision == "match"} - truth = truth_pairs(df) - tp = len(predicted & truth) - fp = len(predicted - truth) - fn = len(truth - predicted) - precision = tp / (tp + fp) if (tp + fp) else 0.0 - recall = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 - - return { - "backend": backend, - "n_records": n, - "total_pairs": total_pairs, - "candidate_pairs": report.n_candidate_pairs, - "reduction_ratio": round(reduction, 6), - "runtime_s": round(runtime, 4), - "matches": report.n_matches, - "clusters": report.n_clusters, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - } - - -# ===================================================================== -# Labelled-dataset mode (benchmarks/data/er_labelled_*.csv) -# ===================================================================== - - -def _labelled_config(null_policy: str, mode: str) -> EntityResolutionConfig: - """Linkage config for the labelled dataset schema (see gen_er_labelled.py).""" - return EntityResolutionConfig( - enabled=True, - backend="pandas", - unique_id_column="id", - blocking_rules=( - BlockingRule("lower(l.email) = lower(r.email)", "same email"), - BlockingRule( - "l.dob = r.dob and substr(lower(l.last_name), 1, 2) = " - "substr(lower(r.last_name), 1, 2)", - "same dob + surname prefix", - ), - BlockingRule("l.phone = r.phone", "same phone"), - ), - comparisons=( - ComparisonLevel("full_name", "token_set", weight=2.0), - ComparisonLevel("full_name", "jaro_winkler", threshold=0.85, weight=1.5), - ComparisonLevel("last_name", "metaphone", weight=1.0), - ComparisonLevel("email", "jaro_winkler", threshold=0.90, weight=2.5), - ComparisonLevel("dob", "exact", weight=2.0), - ComparisonLevel("first_name", "jaro_winkler", threshold=0.85, weight=1.0), - ), - null_policy=null_policy, # type: ignore[arg-type] - mode=mode, # type: ignore[arg-type] - ) - - -def _false_merge_count(report, df: pd.DataFrame) -> int: - """Clusters that merged more than one true entity (the costly failure).""" - label_of = dict(zip(df["id"], df["entity_id"])) - return sum( - 1 - for c in report.clusters - if len({label_of[rid] for rid in c.record_ids if rid in label_of}) > 1 - ) - - -def _peak_rss_mb() -> float | None: - try: - import resource - - peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return round(peak / (1 << 20) if sys.platform == "darwin" else peak / 1024, 1) - except ImportError: # pragma: no cover - return None - - -def evaluate_labelled(df: pd.DataFrame, null_policy: str, mode: str) -> dict: - config = _labelled_config(null_policy, mode) - t0 = time.perf_counter() - _, report = fd.resolve_entities(df, config=config) - runtime = time.perf_counter() - t0 - - n = len(df) - total_pairs = n * (n - 1) // 2 - predicted = {_norm(p.left_id, p.right_id) for p in report.pairs if p.decision == "match"} - truth = truth_pairs(df.rename(columns={"entity_id": "entity_label"})) - tp, fp, fn = ( - len(predicted & truth), - len(predicted - truth), - len(truth - predicted), - ) - precision = tp / (tp + fp) if (tp + fp) else 0.0 - recall = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 - return { - "config": f"null_policy={null_policy}, mode={mode}", - "null_policy": null_policy, - "mode": mode, - "n_records": n, - "candidate_pairs": report.n_candidate_pairs, - "reduction_ratio": round(1.0 - report.n_candidate_pairs / total_pairs, 6) - if total_pairs - else 0.0, - "true_pairs": len(truth), - "tp": tp, - "fp": fp, - "fn": fn, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "false_merged_clusters": _false_merge_count(report, df), - "n_clusters": report.n_clusters, - "runtime_s": round(runtime, 4), - "peak_rss_mb": _peak_rss_mb(), - } - - -def evaluate_exact_baseline(df: pd.DataFrame) -> dict: - """pandas drop_duplicates on identity fields β€” the honesty baseline.""" - fields = ["full_name", "email", "phone", "dob"] - t0 = time.perf_counter() - dup_mask = df.duplicated(subset=fields, keep=False) - runtime = time.perf_counter() - t0 - predicted: set[tuple] = set() - for _, group in df[dup_mask].groupby(fields, dropna=False): - ids = sorted(group["id"].tolist()) - for a, b in combinations(ids, 2): - predicted.add((a, b)) - truth = truth_pairs(df.rename(columns={"entity_id": "entity_label"})) - tp, fp, fn = ( - len(predicted & truth), - len(predicted - truth), - len(truth - predicted), - ) - precision = tp / (tp + fp) if (tp + fp) else 1.0 - recall = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 - return { - "config": "pandas exact drop_duplicates (baseline)", - "n_records": len(df), - "true_pairs": len(truth), - "tp": tp, - "fp": fp, - "fn": fn, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "false_merged_clusters": 0, - "runtime_s": round(runtime, 4), - } - - -_SWEEP = [ - ("penalize", "balanced"), - ("neutral", "balanced"), - ("neutral", "precision"), - ("neutral", "recall"), -] - - -def run_labelled(path: str, write_results: str | None) -> list[dict]: - df = pd.read_csv(path) - results = [evaluate_exact_baseline(df)] - results += [evaluate_labelled(df, np_, m) for np_, m in _SWEEP] - if write_results: - import platform - - payload = { - "dataset": path, - "generator": "benchmarks/gen_er_labelled.py", - "machine": f"{platform.system()} {platform.machine()}", - "python": platform.python_version(), - "pandas": pd.__version__, - "freshdata": fd.__version__, - "method": "rule-weighted linkage (normalized weighted average; " - "no EM / Fellegi-Sunter)", - "results": results, - } - with open(write_results, "w", encoding="utf-8") as fh: - json.dump(payload, fh, indent=2) - return results - - -def main(argv: list[str] | None = None) -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--rows", type=int, default=2000) - ap.add_argument("--dup-rate", type=float, default=0.3) - ap.add_argument("--seed", type=int, default=0) - ap.add_argument("--backend", choices=["pandas", "duckdb", "both"], default="both") - ap.add_argument("--json", action="store_true", help="emit machine-readable JSON") - ap.add_argument( - "--labelled", - metavar="CSV", - help="run the labelled-dataset accuracy sweep (see gen_er_labelled.py)", - ) - ap.add_argument( - "--write-results", metavar="JSON", help="with --labelled: write results here" - ) - args = ap.parse_args(argv) - - if args.labelled: - results = run_labelled(args.labelled, args.write_results) - if args.json: - print(json.dumps(results, indent=2)) - else: - for r in results: - print( - f"[{r['config']:<44}] P={r['precision']:.4f} " - f"R={r['recall']:.4f} F1={r['f1']:.4f} " - f"FP={r['fp']} FN={r['fn']} " - f"false-merges={r['false_merged_clusters']}" - ) - return 0 - - df = synth(args.rows, args.dup_rate, seed=args.seed) - backends = ["pandas", "duckdb"] if args.backend == "both" else [args.backend] - - results = [] - for backend in backends: - if backend == "duckdb" and importlib.util.find_spec("duckdb") is None: - print("duckdb not installed β€” skipping duckdb backend", file=sys.stderr) - continue - results.append(evaluate(df, backend)) - - if args.json: - print(json.dumps(results, indent=2)) - else: - for r in results: - print( - f"[{r['backend']:>6}] n={r['n_records']} " - f"pairs {r['candidate_pairs']}/{r['total_pairs']} " - f"(reduction {r['reduction_ratio']:.4f}) " - f"runtime {r['runtime_s']}s | " - f"P={r['precision']} R={r['recall']} F1={r['f1']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/bench_fieldcheck.py b/benchmarks/bench_fieldcheck.py deleted file mode 100644 index 86ea32b1..00000000 --- a/benchmarks/bench_fieldcheck.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Benchmark fd.validate_fields against regex and pandas-coercion baselines. - -Reproducible (seeded) invalid-cell detection benchmark on a synthetic -financial feed with labeled corruptions:: - - python benchmarks/bench_fieldcheck.py # default 20k rows - python benchmarks/bench_fieldcheck.py 100000 # custom row count - -For each validator we report precision / recall / F1 over the labeled -invalid cells, plus wall time and throughput. Baselines: - -* ``regex`` β€” per-column regular expressions only (no context, no ranges); -* ``pandas`` β€” ``pd.to_numeric`` / ``pd.to_datetime`` coercion NaN-diffing - (types only: no vocabulary, format or semantic checks); -* ``fieldcheck`` β€” ``fd.validate_fields`` with a declared schema. -""" - -from __future__ import annotations - -import json -import random -import re -import sys -import time - -import pandas as pd - -from freshdata.fieldcheck import FieldSpec, validate_fields - -SEED = 20260711 -CORRUPTION_RATE = 0.02 - -SCHEMA = { - "transaction_id": FieldSpec(semantic_type="identifier", required=True, nullable=False), - "transaction_amount": FieldSpec(semantic_type="currency_amount"), - "currency": FieldSpec(allowed_values=frozenset({"USD", "EUR", "GBP", "INR"})), - "stock_ticker": FieldSpec(semantic_type="ticker"), - "interest_rate": FieldSpec(semantic_type="rate", min_value=0, max_value=1), - "transaction_date": FieldSpec(semantic_type="date"), -} - -#: (column, corrupt value) pool β€” all are invalid for their column. -CORRUPTIONS = [ - ("transaction_amount", "apple"), - ("transaction_amount", "approved"), - ("transaction_amount", "12O.50"), - ("currency", "BTC"), - ("currency", "dollars"), - ("stock_ticker", "apple"), - ("stock_ticker", "AА PL"), - ("interest_rate", "high"), - ("interest_rate", "5"), # out of [0, 1] - ("transaction_date", "2026-02-30"), - ("transaction_date", "not a date"), -] - - -def make_dataset(n: int) -> tuple[pd.DataFrame, set]: - rng = random.Random(SEED) - tickers = ["AAPL", "MSFT", "GOOG", "TSLA", "BRK.B", "AMZN"] - rows = [] - for i in range(n): - rows.append({ - "transaction_id": f"T{i:07d}", - "transaction_amount": f"{rng.uniform(1, 5000):.2f}", - "currency": rng.choice(["USD", "EUR", "GBP", "INR"]), - "stock_ticker": rng.choice(tickers), - "interest_rate": f"{rng.uniform(0.001, 0.2):.4f}", - "transaction_date": f"2026-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}", - }) - df = pd.DataFrame(rows) - truth: set = set() - n_bad = max(1, int(n * CORRUPTION_RATE)) - for row in rng.sample(range(n), n_bad): - col, bad = rng.choice(CORRUPTIONS) - df.loc[row, col] = bad - truth.add((row, col)) - return df, truth - - -# --- baselines --------------------------------------------------------------- - -REGEXES = { - "transaction_id": re.compile(r"^T\d{7}$"), - "transaction_amount": re.compile(r"^-?\d+(\.\d+)?$"), - "currency": re.compile(r"^[A-Z]{3}$"), - "stock_ticker": re.compile(r"^[A-Z]{1,6}([.\-][A-Z0-9]{1,4})?$"), - "interest_rate": re.compile(r"^-?\d+(\.\d+)?$"), - "transaction_date": re.compile(r"^\d{4}-\d{2}-\d{2}$"), -} - - -def regex_validator(df: pd.DataFrame) -> set: - flagged = set() - for col, rx in REGEXES.items(): - bad = ~df[col].astype(str).str.fullmatch(rx) - flagged.update((row, col) for row in df.index[bad]) - return flagged - - -def pandas_validator(df: pd.DataFrame) -> set: - flagged = set() - for col in ("transaction_amount", "interest_rate"): - bad = pd.to_numeric(df[col], errors="coerce").isna() & df[col].notna() - flagged.update((row, col) for row in df.index[bad]) - dates = pd.to_datetime(df["transaction_date"], errors="coerce", format="%Y-%m-%d") - bad = dates.isna() & df["transaction_date"].notna() - flagged.update((row, "transaction_date") for row in df.index[bad]) - return flagged - - -def fieldcheck_validator(df: pd.DataFrame) -> set: - report = validate_fields(df, SCHEMA) - return {(i.row, i.column) for i in report.issues if i.severity == "error"} - - -# --- scoring ----------------------------------------------------------------- - - -def score(name: str, fn, df: pd.DataFrame, truth: set) -> dict: - start = time.perf_counter() - flagged = fn(df) - elapsed = time.perf_counter() - start - tp = len(flagged & truth) - fp = len(flagged - truth) - fn_ = len(truth - flagged) - precision = tp / (tp + fp) if tp + fp else 0.0 - recall = tp / (tp + fn_) if tp + fn_ else 0.0 - f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 - return { - "validator": name, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "false_accepts": fn_, - "false_rejects": fp, - "seconds": round(elapsed, 3), - "rows_per_s": int(len(df) / elapsed) if elapsed else None, - } - - -def main() -> None: - n = int(sys.argv[1]) if len(sys.argv) > 1 else 20_000 - df, truth = make_dataset(n) - print(f"rows={n} corrupted_cells={len(truth)} seed={SEED}\n") - results = [ - score("regex", regex_validator, df, truth), - score("pandas_coercion", pandas_validator, df, truth), - score("fd.validate_fields", fieldcheck_validator, df, truth), - ] - print(json.dumps(results, indent=2)) - print() - header = f"{'validator':<20}{'precision':>10}{'recall':>8}{'f1':>8}{'sec':>8}{'rows/s':>10}" - print(header) - for r in results: - print(f"{r['validator']:<20}{r['precision']:>10}{r['recall']:>8}" - f"{r['f1']:>8}{r['seconds']:>8}{r['rows_per_s']:>10}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/bench_native_semantic.py b/benchmarks/bench_native_semantic.py deleted file mode 100644 index 3c5a4605..00000000 --- a/benchmarks/bench_native_semantic.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Scale benchmark for the native distinct-value semantic path (Phase 6). - -The semantic stage only reasons about a column's *distinct values*. On a native -engine we extract that bounded distinct table natively (Polars group-by / DuckDB -GROUP BY), score it through the pandas gate, and map repairs back natively β€” so -peak memory tracks the *distinct* cardinality, not the row count. This benchmark -generates a wide, mostly-low-cardinality frame and compares: - - * pandas β€” reference path; whole frame in memory, semantics in pandas - * polars-lazy β€” native distinct path; frame scanned lazily from parquet - * duckdb β€” native distinct path; frame scanned from parquet - -For each it records rows/sec, peak RSS, and confirms the same repairs landed -(boolean synonyms -> Boolean, spelled numbers -> Int). The headline property: -polars-lazy / duckdb peak RSS stays far below the pandas path as ``--rows`` grows. - - python benchmarks/bench_native_semantic.py --rows 1000000 - python benchmarks/bench_native_semantic.py --rows 10000000 --engines polars-lazy,duckdb -""" - -from __future__ import annotations - -import argparse -import json -import sys -import time - -import numpy as np -import pandas as pd - -from freshdata.config import CleanConfig -from freshdata.execution import run_with_engine - -# A native-compatible config: conservative engine + no sampled dtype fixes, so -# the deterministic stages run natively and only the semantic stage is measured. -CONFIG = CleanConfig( - strategy="conservative", fix_dtypes=False, semantic_mode="auto", verbose=False -) - - -def _rss_mb() -> float: - try: - import psutil - - return psutil.Process().memory_info().rss / 1e6 - except ImportError: - import resource - - maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return maxrss / 1e6 if sys.platform == "darwin" else maxrss / 1e3 - - -def write_parquet(path: str, rows: int, batch: int, seed: int = 0) -> None: - """Write ``rows`` rows to parquet in batches β€” never all in memory at once.""" - import pyarrow as pa - import pyarrow.parquet as pq - - rng = np.random.default_rng(seed) - active = np.array(["yes", "y", "YES", "no", "n", "NO"], dtype=object) - qty = np.array(["one", "two", "three", "4", "5", "6"], dtype=object) - region = np.array(["North", "north", "NORTH", "South", "south"], dtype=object) - writer: pq.ParquetWriter | None = None - produced = 0 - try: - while produced < rows: - n = min(batch, rows - produced) - tbl = pa.table( - { - "active": pa.array(rng.choice(active, n)), - "qty": pa.array(rng.choice(qty, n)), - "region": pa.array(rng.choice(region, n)), - "user_id": pa.array([f"U{i}" for i in range(produced, produced + n)]), - } - ) - if writer is None: - writer = pq.ParquetWriter(path, tbl.schema) - writer.write_table(tbl) - produced += n - finally: - if writer is not None: - writer.close() - - -def _run_pandas(path: str) -> tuple[pd.DataFrame, float, float]: - t0 = time.perf_counter() - df = pd.read_parquet(path) - out = run_with_engine(df, CONFIG, engine="pandas", output_format="pandas") - return out, time.perf_counter() - t0, _rss_mb() - - -def _run_polars_lazy(path: str) -> tuple[pd.DataFrame, float, float]: - import polars as pl - - t0 = time.perf_counter() - lf = pl.scan_parquet(path) - out = run_with_engine(lf, CONFIG, engine="polars", output_format="polars-lazy") - frame = out.collect(engine="streaming").to_pandas() - return frame, time.perf_counter() - t0, _rss_mb() - - -def _run_duckdb(path: str) -> tuple[pd.DataFrame, float, float]: - import duckdb # noqa: F401 - ensure the optional dependency is present - - t0 = time.perf_counter() - # Pass the parquet path: the engine reads it through a lazy view, so the - # frame is never fully materialized before the native distinct pass. - out = run_with_engine(path, CONFIG, engine="duckdb", output_format="duckdb") - frame = out.df() - return frame, time.perf_counter() - t0, _rss_mb() - - -_RUNNERS = { - "pandas": _run_pandas, - "polars-lazy": _run_polars_lazy, - "duckdb": _run_duckdb, -} - - -def _repairs_ok(frame: pd.DataFrame) -> bool: - active = set(frame["active"]) - qty = {int(v) for v in frame["qty"]} - return active <= {True, False} and qty == {1, 2, 3, 4, 5, 6} - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--rows", type=int, default=1_000_000) - ap.add_argument("--batch-size", type=int, default=250_000) - ap.add_argument("--engines", default="pandas,polars-lazy,duckdb") - ap.add_argument("--out", default=None, help="write JSON results here") - args = ap.parse_args() - - import tempfile - - engines = [e.strip() for e in args.engines.split(",") if e.strip()] - with tempfile.NamedTemporaryFile(suffix=".parquet", delete=True) as tmp: - write_parquet(tmp.name, args.rows, args.batch_size) - results = [] - for engine in engines: - runner = _RUNNERS.get(engine) - if runner is None: - print(f"skip unknown engine {engine!r}", file=sys.stderr) - continue - try: - frame, elapsed, peak_rss = runner(tmp.name) - except ImportError as exc: - print(f"skip {engine}: {exc}", file=sys.stderr) - continue - rec = { - "engine": engine, - "rows": args.rows, - "seconds": round(elapsed, 3), - "rows_per_second": round(args.rows / elapsed) if elapsed else None, - "peak_rss_mb": round(peak_rss, 1), - "repairs_ok": _repairs_ok(frame), - } - results.append(rec) - print(json.dumps(rec)) - - payload = {"benchmark": "native_semantic", "rows": args.rows, "results": results} - if args.out: - with open(args.out, "w") as fh: - json.dump(payload, fh, indent=2) - # Every engine must produce the same repairs. - if not all(r["repairs_ok"] for r in results): - print("FAIL: repair mismatch across engines", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/bench_quick.py b/benchmarks/bench_quick.py deleted file mode 100644 index a91ae195..00000000 --- a/benchmarks/bench_quick.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Quick benchmark for freshdata.clean on synthetic and fixture data. - -Run:: - - python benchmarks/bench.py # synthetic sizes - python benchmarks/bench.py --fixtures # all test fixtures - python benchmarks/bench.py --fixtures --compare - python benchmarks/bench.py --online --compare - python benchmarks/bench.py --online-all --compare - python benchmarks/bench.py --online-all --tier 1 - python benchmarks/bench.py 10000 100000 # custom row counts -""" - -from __future__ import annotations - -import json -import sys -import time -from pathlib import Path - -import numpy as np -import pandas as pd - -import freshdata as fd - -FIXTURES_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" -ONLINE_CACHE_DIR = FIXTURES_DIR / "online" / "cache" -FIXTURES = [ - "aqi_sample", - "sales_export", - "survey_responses", - "sensor_timeseries", - "fraud_signals", - "tiny_cohort", - "wide_sparse", - "large_panel", - "duplicate_heavy", - "locale_numbers", - "mixed_roles", -] -ONLINE_REGISTRY_PATH = FIXTURES_DIR / "online" / "registry.json" -def _online_names(*, tier: int | None = None) -> list[str]: - if not ONLINE_CACHE_DIR.exists(): - return [] - cached = {p.stem for p in ONLINE_CACHE_DIR.glob("*.csv")} - if tier is None or not ONLINE_REGISTRY_PATH.exists(): - return sorted(cached) - registry = json.loads(ONLINE_REGISTRY_PATH.read_text()) - return sorted(k for k in cached if int(registry.get(k, {}).get("tier", 2)) == tier) - - -ONLINE_FIXTURES = _online_names() - - -def make_messy(n: int, seed: int = 0) -> pd.DataFrame: - rng = np.random.default_rng(seed) - names = rng.choice([" Alice ", "Bob", "carol", "N/A", "Dave "], n) - return pd.DataFrame( - { - "Customer ID": np.arange(n), - " Name ": names, - "Order Total": [f"${x:,.2f}" for x in rng.uniform(5, 9999, n)], - "Quantity": rng.integers(1, 50, n).astype(str), - "Order Date": pd.date_range("2022-01-01", periods=n, freq="min").astype(str), - "Is Member": rng.choice(["yes", "no", "YES", "No"], n), - "Notes": rng.choice(["", "-", "expedite", " gift wrap ", "null"], n), - "Unused": [None] * n, - } - ) - - -def _timed_clean(df: pd.DataFrame, *, strategy: str) -> tuple[float, int, int, int]: - fd.clean(df.head(min(len(df), 1000)), strategy=strategy, verbose=False) # warm-up - missing_before = int(df.isna().sum().sum()) - start = time.perf_counter() - _, report = fd.clean(df, strategy=strategy, return_report=True, verbose=False) - elapsed = time.perf_counter() - start - cols_delta = report.cols_after - df.shape[1] - return elapsed, len(report), report.missing_after - missing_before, cols_delta - - -def bench_synthetic(n: int, *, compare: bool = False) -> None: - df = make_messy(n) - if compare: - hdr = ( - f"{'rows':>10} {'strategy':<11} {'seconds':>8} " - f"{'rows/s':>8} {'missingΞ”':>9} {'colsΞ”':>6}" - ) - print(hdr) - for strategy in ("balanced", "aggressive"): - elapsed, n_actions, miss_d, cols_d = _timed_clean(df, strategy=strategy) - rate = n / elapsed if elapsed else 0 - if compare: - print( - f"{n:>10,} {strategy:<11} {elapsed:8.3f} {rate:8.0f} {miss_d:>9} {cols_d:>6}" - ) - else: - print( - f"{n:>10,} rows {strategy:<11} {elapsed:8.3f}s {rate / 1e6:6.2f}M rows/s " - f"{n_actions} actions" - ) - - -def bench_fixtures( - *, compare: bool = False, online: bool = False, tier: int | None = None -) -> None: - names = _online_names(tier=tier) if online else FIXTURES - base = ONLINE_CACHE_DIR if online else FIXTURES_DIR - header = ( - f"{'fixture':<20} {'strategy':<11} {'seconds':>8} " - f"{'rows/s':>8} {'missingΞ”':>9} {'colsΞ”':>6}" - ) - if compare: - print(header) - else: - print(f"{'fixture':<20} {'strategy':<11} {'seconds':>8} {'rows':>6} actions") - for name in names: - path = base / f"{name}.csv" - if not path.exists(): - continue - df = pd.read_csv(path) - for strategy in ("balanced", "aggressive"): - elapsed, n_actions, miss_d, cols_d = _timed_clean(df, strategy=strategy) - rate = len(df) / elapsed if elapsed else 0 - if compare: - row = ( - f"{name:<20} {strategy:<11} {elapsed:8.3f} " - f"{rate:8.0f} {miss_d:>9} {cols_d:>6}" - ) - print(row) - else: - print( - f"{name:<20} {strategy:<11} {elapsed:8.3f} {len(df):>6} {n_actions}" - ) - if compare: - bench_compare_table(df, name) - - -def bench_compare_table(df: pd.DataFrame, label: str = "data") -> None: - table = fd.compare_clean(df) - print(f"\n=== compare_clean: {label} ===") - print(table.to_string(index=False)) - - -if __name__ == "__main__": - print(f"freshdata {fd.__version__} | pandas {pd.__version__} | numpy {np.__version__}") - compare = "--compare" in sys.argv - online = "--online" in sys.argv or "--online-all" in sys.argv - tier = None - if "--tier" in sys.argv: - tier = int(sys.argv[sys.argv.index("--tier") + 1]) - skip = {"--fixtures", "--compare", "--online", "--online-all", "--tier"} - if "--tier" in sys.argv: - skip.add(str(tier)) - args = [a for a in sys.argv[1:] if a not in skip] - if len(sys.argv) > 1 and sys.argv[1] == "--fixtures": - bench_fixtures(compare=compare, online=online, tier=tier) - elif online and (compare or "--online-all" in sys.argv): - bench_fixtures(compare=compare or "--online-all" in sys.argv, online=True, tier=tier) - elif args and args[0] == "--compare-matrix": - df = pd.read_csv(FIXTURES_DIR / (args[1] if len(args) > 1 else "aqi_sample.csv")) - bench_compare_table(df, args[1] if len(args) > 1 else "aqi_sample") - else: - sizes = [int(a.replace("_", "")) for a in args] or [10_000, 100_000, 1_000_000] - for size in sizes: - bench_synthetic(size, compare=compare) diff --git a/benchmarks/bench_streaming.py b/benchmarks/bench_streaming.py deleted file mode 100644 index ff67cf66..00000000 --- a/benchmarks/bench_streaming.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Stable-memory benchmark for freshdata streaming / micro-batch cleaning. - -This is an **out-of-core / streaming** test, not a single-DataFrame test: synthetic -rows are generated lazily, one batch at a time, and fed to a -:class:`freshdata.StreamingCleaner`. The whole dataset is never materialized, so peak -memory should stay roughly flat as ``--rows`` grows β€” that flatness is the property the -"clean 100M rows out-of-core with stable memory" claim depends on. - - python benchmarks/bench_streaming.py --rows 100000000 --batch-size 100000 --cols 20 -""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from collections.abc import Iterator - -import numpy as np -import pandas as pd - -import freshdata as fd - - -def _rss_mb() -> float: - """Resident set size in MB (psutil if available, else the resource module).""" - try: - import psutil - - return psutil.Process().memory_info().rss / 1e6 - except ImportError: - import resource - - maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return maxrss / 1e6 if sys.platform == "darwin" else maxrss / 1e3 # bytes vs KB - - -def synth_batches(rows: int, batch_size: int, cols: int, *, seed: int = 0 - ) -> Iterator[pd.DataFrame]: - """Yield ``rows`` rows as lazily-generated batches β€” never all at once.""" - rng = np.random.default_rng(seed) - produced = 0 - t0 = np.datetime64("2020-01-01") - while produced < rows: - n = min(batch_size, rows - produced) - data: dict[str, object] = { - "customer_id": np.arange(produced, produced + n, dtype="int64"), - "churn": rng.integers(0, 2, n), - "event_time": t0 + np.arange(produced, produced + n).astype("timedelta64[s]"), - "region": rng.choice(["north", "north", "north", "south", "east", None], n), - } - for j in range(max(0, cols - len(data))): - vals = rng.normal(50, 12, n) - vals[rng.random(n) < 0.1] = np.nan # ~10% missing - vals[rng.random(n) < 0.005] *= 40 # rare outliers - data[f"metric_{j}"] = vals - produced += n - yield pd.DataFrame(data) - - -def run_benchmark(*, rows: int, batch_size: int, cols: int) -> dict: - """Stream ``rows`` rows through a StreamingCleaner and measure memory/throughput.""" - cleaner = fd.StreamingCleaner(target_column="churn", id_columns=("customer_id",), - window_size=batch_size, warmup_batches=3) - base = _rss_mb() - peak = base - growth_samples: list[float] = [] - processed = 0 - start = time.perf_counter() - for cleaned, report in cleaner.clean_batches(synth_batches(rows, batch_size, cols)): - processed += len(cleaned) - rss = _rss_mb() - peak = max(peak, rss) - # Sample post-warmup RSS so growth reflects the steady state, not startup. - if report.streaming["batch_id"] > 3: - growth_samples.append(round(rss, 1)) - elapsed = time.perf_counter() - start - final = cleaner.finalize() - - if processed < rows: - raise SystemExit(f"FAIL: processed {processed:,} of {rows:,} requested rows") - - # Stable memory = the steady-state *tail* is flat. The allocator ramps to a - # working-set ceiling over the first batches (not a function of total rows); - # measuring the second half of the run excludes that ramp. - tail = growth_samples[len(growth_samples) // 2:] or growth_samples - steady_growth = round(max(tail) - min(tail), 1) if tail else 0.0 - return { - "rows_processed": processed, - "batch_size": batch_size, - "columns": cols, - "elapsed_seconds": round(elapsed, 2), - "rows_per_second": int(processed / elapsed) if elapsed else 0, - "baseline_memory_mb": round(base, 1), - "peak_memory_mb": round(peak, 1), - "warmup_ramp_mb": round(peak - base, 1), - "steady_state_growth_mb": steady_growth, - "steady_state_samples_mb": tail, - "final_cumulative_trust_score": final.streaming["cumulative_trust_score"], - # Flat tail (no linear growth with rows) is the acceptance criterion. - "stable_memory": steady_growth < max(50.0, 0.15 * peak), - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="freshdata streaming stable-memory benchmark") - parser.add_argument("--rows", type=int, default=1_000_000) - parser.add_argument("--batch-size", type=int, default=100_000) - parser.add_argument("--cols", type=int, default=20) - parser.add_argument("--report", metavar="FILE", help="write the benchmark JSON here") - args = parser.parse_args(argv) - - result = run_benchmark(rows=args.rows, batch_size=args.batch_size, cols=args.cols) - text = json.dumps(result, indent=2) - if args.report: - with open(args.report, "w") as fh: - fh.write(text) - print(text) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/bench_v3_lazy_vs_pandas.py b/benchmarks/bench_v3_lazy_vs_pandas.py index 5f62aeac..61e6d157 100644 --- a/benchmarks/bench_v3_lazy_vs_pandas.py +++ b/benchmarks/bench_v3_lazy_vs_pandas.py @@ -16,7 +16,7 @@ import pandas as pd import polars as pl -from freshdata.polars_engine import CleanConfig, apply_plan, suggest_plan +from freshdata import CleanConfig, apply_plan, suggest_plan def get_peak_rss_mb() -> float: diff --git a/benchmarks/cleanbench/t4_profiles.py b/benchmarks/cleanbench/t4_profiles.py deleted file mode 100644 index 9251bbee..00000000 --- a/benchmarks/cleanbench/t4_profiles.py +++ /dev/null @@ -1,463 +0,0 @@ -"""CleanBench T4 β€” learned-profile replay benchmark (Phase 4). - -Ten fixture families, each a (train messy, train clean) pair plus a held-out -test batch corrupted by the *same* family. A profile is learned from the -train pair and replayed on the test batch; the gates compare against a -no-profile baseline: - -* mean cell-repair F1 lift on the replayable families >= +15 points, -* false-modification rate with the profile <= without it, -* protected-column violations == 0, -* zero raw sensitive literals inside the serialized profile under - privacy="mask". - -The corruptions are deliberately ones the no-profile pipeline cannot fully -solve on its own (domain vocabularies, nonstandard sentinels, ambiguous -day-first dates, region-less phone formats), so the lift measures what the -*profile* adds β€” not what the deterministic experts already do. -""" - -from __future__ import annotations - -import io -import tempfile -import zipfile -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -import freshdata as fd -from freshdata.learning import learn - -from .metrics import ( - cell_repair_f1, - false_modification_rate, - protected_column_violation_rate, -) - -__all__ = [ - "T4Fixture", - "T4Result", - "make_t4_fixtures", - "privacy_leak_count", - "profile_drift_block_rate", - "run_t4", - "run_t4_fixture", -] - -_TRAIN_ROWS = 60 -_TEST_ROWS = 30 - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@dataclass -class T4Fixture: - """One corruption family: a train pair and a same-family test batch.""" - - name: str - train_messy: pd.DataFrame - train_clean: pd.DataFrame - test_messy: pd.DataFrame - test_truth: pd.DataFrame - key: str = "id" - context: str = "id is a unique identifier. Never modify id." - protected_columns: tuple[str, ...] = ("id",) - #: Raw sensitive strings from the train pair that must never appear in - #: the serialized profile under privacy="mask". - sensitive_literals: tuple[str, ...] = () - #: Families like imputation/unexplained replay nothing by design and are - #: excluded from the F1-lift gate (they still run the safety gates). - expect_lift: bool = True - learn_kwargs: dict[str, Any] = field(default_factory=dict) - - -def _ids(n: int, prefix: str) -> list[str]: - return [f"{prefix}{i:04d}" for i in range(n)] - - -def _cycle(values: list, n: int) -> list: - return [values[i % len(values)] for i in range(n)] - - -def _pair_frames( - n: int, - prefix: str, - column: str, - messy_values: list, - clean_values: list, -) -> tuple[pd.DataFrame, pd.DataFrame]: - """Aligned (messy, clean) frames: one corrupted column plus stable ballast.""" - ids = _ids(n, prefix) - ballast = [round(10.0 + (i % 7) * 1.5, 2) for i in range(n)] - messy = pd.DataFrame({"id": ids, column: _cycle(messy_values, n), "ballast": ballast}) - clean = pd.DataFrame({"id": ids, column: _cycle(clean_values, n), "ballast": ballast}) - return messy, clean - - -def _vocab_fixture( - name: str, - column: str, - mapping: dict[str, object], -) -> T4Fixture: - """A fixture whose corruption is a domain vocabulary (raw -> clean).""" - raws = list(mapping) - cleans = [mapping[r] for r in raws] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", column, raws, cleans) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", column, raws, cleans) - return T4Fixture( - name=name, - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - ) - - -def _make_category_map_fixture() -> T4Fixture: - # Department codes: an internal vocabulary no generic cleaner knows. - return _vocab_fixture( - "category_map_departments", - "department", - { - "H.R.": "human_resources", - "HR ": "human_resources", - "Fin": "finance", - "FIN.": "finance", - "Eng": "engineering", - "ENGG": "engineering", - }, - ) - - -def _make_status_typos_fixture() -> T4Fixture: - return _vocab_fixture( - "status_reference_typos", - "status", - { - "Deliverd": "delivered", - "delivred": "delivered", - "SHIPPED": "shipped", - "shiped": "shipped", - "In Transit": "in_transit", - "IN-TRANSIT": "in_transit", - }, - ) - - -def _make_condition_vocab_fixture() -> T4Fixture: - return _vocab_fixture( - "condition_abbreviations", - "condition", - { - "NIB": "new_in_box", - "N.I.B.": "new_in_box", - "LN": "like_new", - "L.N.": "like_new", - "FR": "for_repair", - }, - ) - - -def _make_city_typos_fixture() -> T4Fixture: - return _vocab_fixture( - "city_reference_typos", - "city", - { - "Bengalru": "Bengaluru", - "Bangalore": "Bengaluru", - "Mumbay": "Mumbai", - "Bombay": "Mumbai", - "Dehli": "Delhi", - "New Dehli": "Delhi", - }, - ) - - -def _make_sentinel_fixture() -> T4Fixture: - # Nonstandard sentinels the default sentinel set does not contain. - raws = ["MISSING_VAL", "-999", "n.v.", "42", "77", "13"] - cleans = [np.nan, np.nan, np.nan, 42.0, 77.0, 13.0] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", "reading", raws, cleans) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", "reading", raws, cleans) - return T4Fixture( - name="nonstandard_sentinels", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - ) - - -def _make_dayfirst_fixture() -> T4Fixture: - # Ambiguous day-first dates: without the learned dayfirst delta the - # baseline parses them month-first and gets *wrong* timestamps. - raw_dates = ["05/02/2024", "07/03/2024", "09/04/2024", "11/05/2024", "01/06/2024"] - truth_dates = pd.to_datetime(raw_dates, dayfirst=True) - train_messy, train_clean = _pair_frames( - _TRAIN_ROWS, "T", "event_date", raw_dates, list(truth_dates) - ) - test_messy, test_truth = _pair_frames( - _TEST_ROWS, "H", "event_date", raw_dates, list(truth_dates) - ) - return T4Fixture( - name="dayfirst_dates", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - ) - - -def _make_phone_fixture() -> T4Fixture: - locals_ = ["98765 43210", "91234-56789", "099887 76655", "9012345678", "98111 22333"] - canonical = [ - "+919876543210", - "+919123456789", - "+919988776655", - "+919012345678", - "+919811122333", - ] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", "phone", locals_, canonical) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", "phone", locals_, canonical) - return T4Fixture( - name="phone_in_formats", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - sensitive_literals=tuple(locals_) + tuple(canonical), - ) - - -def _make_email_fixture() -> T4Fixture: - doubled = [ - "asha@@gmail.com", - "bob@@yahoo.com", - "carol@@hotmail.com", - "dev@@site.com", - "eve@@example.com", - ] - fixed = [d.replace("@@", "@") for d in doubled] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", "email", doubled, fixed) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", "email", doubled, fixed) - return T4Fixture( - name="email_double_at", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - sensitive_literals=tuple(doubled) + tuple(fixed), - # The deterministic email expert already repairs doubled @s; the - # profile must not *hurt*, but the lift comes from other families. - expect_lift=False, - ) - - -def _make_imputation_fixture() -> T4Fixture: - # The clean side median-fills missing readings. The profile must learn at - # most an imputation *strategy* hint β€” never literal fill values β€” so - # replay leaves the new batch's NaNs alone rather than stamping the - # train median into them. - n = _TRAIN_ROWS - ids = _ids(n, "T") - values = [float(20 + (i % 9)) for i in range(n)] - messy_vals: list[object] = list(values) - for i in range(0, n, 6): - messy_vals[i] = np.nan - median = float(np.nanmedian(np.asarray(messy_vals, dtype=float))) - clean_vals = [median if pd.isna(v) else v for v in messy_vals] - train_messy = pd.DataFrame({"id": ids, "reading": messy_vals}) - train_clean = pd.DataFrame({"id": ids, "reading": clean_vals}) - - test_ids = _ids(_TEST_ROWS, "H") - test_vals: list[object] = [float(30 + (i % 5)) for i in range(_TEST_ROWS)] - for i in range(0, _TEST_ROWS, 5): - test_vals[i] = np.nan - test_messy = pd.DataFrame({"id": test_ids, "reading": test_vals}) - # Truth keeps the NaNs: a literal replay of the *train* median would be - # a false modification, which is exactly what the gate checks. - test_truth = test_messy.copy() - return T4Fixture( - name="imputation_no_literal_replay", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - expect_lift=False, - ) - - -def _make_unexplained_fixture() -> T4Fixture: - # One-off free-text typos, each with support 1: nothing meets - # min_support, so the profile stores examples only and replays nothing. - n = _TRAIN_ROWS - ids = _ids(n, "T") - notes = [f"customer note number {i} with unique typo xq{i}z" for i in range(n)] - clean_notes = [t.replace(f"xq{i}z", "fixed") for i, t in enumerate(notes)] - train_messy = pd.DataFrame({"id": ids, "note": notes}) - train_clean = pd.DataFrame({"id": ids, "note": clean_notes}) - test_ids = _ids(_TEST_ROWS, "H") - test_notes = [f"new note {i} with typo aa{i}bb" for i in range(_TEST_ROWS)] - test_messy = pd.DataFrame({"id": test_ids, "note": test_notes}) - test_truth = test_messy.copy() - return T4Fixture( - name="unexplained_examples_only", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - expect_lift=False, - ) - - -def make_t4_fixtures() -> list[T4Fixture]: - """The ten T4 corruption families, deterministic (no RNG needed).""" - return [ - _make_category_map_fixture(), - _make_status_typos_fixture(), - _make_condition_vocab_fixture(), - _make_city_typos_fixture(), - _make_sentinel_fixture(), - _make_dayfirst_fixture(), - _make_phone_fixture(), - _make_email_fixture(), - _make_imputation_fixture(), - _make_unexplained_fixture(), - ] - - -# --------------------------------------------------------------------------- -# Metrics -# --------------------------------------------------------------------------- - - -def privacy_leak_count(profile_bytes: bytes, sensitive_literals: tuple[str, ...]) -> int: - """Count sensitive literals readable anywhere in a serialized profile. - - Scans both the raw archive bytes and every decompressed member, so zip - compression cannot hide a leak. - """ - blobs = [profile_bytes] - try: - with zipfile.ZipFile(io.BytesIO(profile_bytes)) as zf: - blobs.extend(zf.read(name) for name in zf.namelist()) - except zipfile.BadZipFile: # pragma: no cover - defensive - pass - leaks = 0 - for literal in sensitive_literals: - needle = literal.encode("utf-8") - if any(needle in blob for blob in blobs): - leaks += 1 - return leaks - - -def profile_drift_block_rate(reports: list) -> float: - """Share of runs whose profile replay was blocked by the drift gate.""" - if not reports: - return 0.0 - blocked = sum( - 1 - for r in reports - if getattr(r, "profile_replay", None) is not None and not r.profile_replay.get("ok") - ) - return blocked / len(reports) - - -# --------------------------------------------------------------------------- -# Runner -# --------------------------------------------------------------------------- - - -@dataclass -class T4Result: - name: str - expect_lift: bool - baseline_f1: float - profile_f1: float - lift_f1: float - baseline_fmr: float - profile_fmr: float - protected_violation_rate: float - privacy_leaks: int - replay_ok: bool - profile_actions: int - - -def _serialize(profile) -> bytes: - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "profile.fdprofile" - profile.save(path) - return path.read_bytes() - - -def run_t4_fixture(fixture: T4Fixture) -> T4Result: - """Learn from the fixture's train pair, replay on its test batch.""" - profile = learn( - fixture.train_messy, - fixture.train_clean, - context=fixture.context, - key=fixture.key, - dataset_id=f"t4-{fixture.name}", - **fixture.learn_kwargs, - ) - leaks = privacy_leak_count(_serialize(profile), fixture.sensitive_literals) - - # Shape-preserving options: metrics need identically-shaped frames, so no - # flag columns, no row drops, no index resets. - clean_kwargs: dict[str, Any] = { - "semantic_mode": "auto", - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - "outlier_action": None, - "return_report": True, - } - baseline, _ = fd.clean(fixture.test_messy, **clean_kwargs) - repaired, report = fd.clean(fixture.test_messy, profile=profile, **clean_kwargs) - - truth, corrupted = fixture.test_truth, fixture.test_messy - baseline_f1 = cell_repair_f1(truth, corrupted, baseline) - profile_f1 = cell_repair_f1(truth, corrupted, repaired) - profile_actions = sum( - 1 for a in report.actions if a.metadata and a.metadata.get("profile_influenced") - ) - return T4Result( - name=fixture.name, - expect_lift=fixture.expect_lift, - baseline_f1=baseline_f1, - profile_f1=profile_f1, - lift_f1=profile_f1 - baseline_f1, - baseline_fmr=false_modification_rate(truth, corrupted, baseline), - profile_fmr=false_modification_rate(truth, corrupted, repaired), - protected_violation_rate=protected_column_violation_rate( - corrupted, repaired, list(fixture.protected_columns) - ), - privacy_leaks=leaks, - replay_ok=bool(report.profile_replay is not None and report.profile_replay.get("ok")), - profile_actions=profile_actions, - ) - - -def run_t4(fixtures: list[T4Fixture] | None = None) -> list[T4Result]: - return [run_t4_fixture(f) for f in fixtures or make_t4_fixtures()] - - -if __name__ == "__main__": # pragma: no cover - manual benchmark entry point - results = run_t4() - lifted = [r for r in results if r.expect_lift] - print(f"{'fixture':32} {'base F1':>8} {'prof F1':>8} {'lift':>7} {'FMR b/p':>12} {'leaks':>5}") - for r in results: - print( - f"{r.name:32} {r.baseline_f1:8.3f} {r.profile_f1:8.3f} " - f"{r.lift_f1:+7.3f} {r.baseline_fmr:5.3f}/{r.profile_fmr:5.3f} " - f"{r.privacy_leaks:5d}" - ) - mean_lift = sum(r.lift_f1 for r in lifted) / len(lifted) - print(f"\nmean F1 lift (replayable families): {mean_lift:+.3f}") diff --git a/benchmarks/corpus/scoring.py b/benchmarks/corpus/scoring.py deleted file mode 100644 index 4760b4c9..00000000 --- a/benchmarks/corpus/scoring.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Run the trap corpus against FreshData and score the result. - -This is the measurement half of the corpus. It answers the question the whole -programme turns on: *how often does the library make a semantic decision a -knowledgeable human would call wrong, and how often does it do so silently?* - -Scoring rules -------------- -The observed disposition is derived from what actually happened to the cell, -never from what the report claims: - -``PRESERVE`` the value is unchanged (compared after a NaN-aware equality). -``REPAIR`` the value changed. Whether it changed *correctly* is a separate - axis -- ``repaired_correctly`` -- so a wrong repair is visible as - a corruption rather than hidden inside a repair count. -``QUARANTINE`` / ``REJECT`` / ``REVIEW`` - taken from ``validate_fields`` row actions, mapped through - :func:`~.dispositions.from_field_action`. -``FLAG`` unchanged, but the cell drew an issue or a report warning. - -A case whose ``expected`` is ``None`` (a recorded specification gap) is -measured and reported but never scored pass/fail -- asserting an outcome there -would be inventing the contract the gap exists to record. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -import pandas as pd - -from .dispositions import Disposition, satisfies -from .frames import frame_for, trap_position -from .traps import TRAPS, TrapCase - -__all__ = ["Observation", "observe", "score", "Metrics", "run_corpus"] - -D = Disposition - - -def _equal(a: Any, b: Any) -> bool: - """NaN-aware scalar equality; two missing values count as equal.""" - a_missing = a is None or (isinstance(a, float) and pd.isna(a)) or a is pd.NaT - b_missing = b is None or (isinstance(b, float) and pd.isna(b)) or b is pd.NaT - try: - a_missing = a_missing or bool(pd.isna(a)) - b_missing = b_missing or bool(pd.isna(b)) - except (TypeError, ValueError): - pass - if a_missing or b_missing: - return a_missing and b_missing - if isinstance(a, str) != isinstance(b, str): - # '7' and 7 are different outcomes for a cleaner; do not conflate them. - return False - try: - return bool(a == b) - except Exception: - return False - - -@dataclass -class Observation: - """What the library actually did to one trap cell.""" - - case: TrapCase - observed: Disposition | None - final_value: Any = None - changed: bool = False - repaired_correctly: bool | None = None - field_action: str | None = None - audited: bool = False - error: str | None = None - - @property - def measured(self) -> bool: - return self.error is None and self.observed is not None - - -def observe(case: TrapCase, *, semantic: bool = True) -> Observation: - """Run one case through ``fd.clean`` and ``fd.validate_fields``.""" - import freshdata as fd - - frame = frame_for(case) - if frame is None: - return Observation(case, None, error="no filler for role") - - pos = trap_position() - original = frame[case.role].iloc[pos] - kwargs: dict[str, Any] = {"verbose": False, "return_report": True} - if semantic: - kwargs["semantic_mode"] = "auto" - - try: - cleaned, report = fd.clean(frame, **kwargs) - except Exception as exc: # a crash is itself a finding - return Observation(case, None, error=f"{type(exc).__name__}: {exc}"[:160]) - - # The anchor column keeps the row alive, so position is stable. - final = cleaned[case.role].iloc[pos] if pos < len(cleaned) else None - changed = not _equal(original, final) - - audited = any(a.column == case.role for a in report.actions) or bool(report.warnings) - - # Ask the validation surface what it would do with the row. - # - # Only pass a semantic_type that fieldcheck actually knows. Its vocabulary - # and the semantic layer's SEMANTIC_TYPES overlap in only 10 of 30 terms - # (see FD2-003), so declaring e.g. "postal_code" -- a real semantic type -- - # warns that nothing will be validated. Passing it anyway would fill the - # suite with warnings and validate nothing, which is worse than omitting it. - from freshdata.fieldcheck import _KNOWN_SEMANTIC_TYPES - - action: str | None = None - try: - spec: dict[str, Any] = {} - if case.semantic_type and case.semantic_type in _KNOWN_SEMANTIC_TYPES: - spec[case.role] = case.semantic_type - vreport = fd.validate_fields(frame, spec or None) - actions = vreport.row_actions() - action = actions.get(pos) if isinstance(actions, dict) else None - except Exception: - action = None - - from .dispositions import from_field_action - - observed: Disposition - if action in {"quarantine", "reject", "manual_review"}: - observed = from_field_action(action) - elif changed: - observed = D.REPAIR - elif audited: - observed = D.FLAG - else: - observed = D.PRESERVE - - repaired_correctly = None - if case.expected is D.REPAIR: - repaired_correctly = _equal(final, case.repaired) - - return Observation( - case=case, - observed=observed, - final_value=final, - changed=changed, - repaired_correctly=repaired_correctly, - field_action=action, - audited=audited, - ) - - -@dataclass -class Metrics: - """The Phase 23 metric set, as rates.""" - - total: int = 0 - measured: int = 0 - spec_gaps: int = 0 - errors: int = 0 - correct: int = 0 - #: expected PRESERVE and the value survived - preserved: int = 0 - preserve_cases: int = 0 - #: expected PRESERVE but the value changed -- false-positive corruption - corrupted: int = 0 - #: expected REPAIR and the gold value was produced - repaired_correctly: int = 0 - repair_cases: int = 0 - #: expected REPAIR, the value changed, but not to the gold value - misrepaired: int = 0 - #: expected REPAIR/REVIEW/... but nothing happened at all - escaped: int = 0 - routed_to_review: int = 0 - review_cases: int = 0 - audited: int = 0 - changes: int = 0 - failures: list[str] = field(default_factory=list) - - def _rate(self, num: int, den: int) -> float | None: - return round(num / den, 4) if den else None - - def as_dict(self) -> dict[str, Any]: - return { - "total": self.total, - "measured": self.measured, - "spec_gaps": self.spec_gaps, - "errors": self.errors, - "accuracy": self._rate(self.correct, self.measured), - "preservation_rate": self._rate(self.preserved, self.preserve_cases), - "corruption_rate": self._rate(self.corrupted, self.preserve_cases), - "repair_precision": self._rate(self.repaired_correctly, self.changes), - "repair_recall": self._rate(self.repaired_correctly, self.repair_cases), - "false_positive_rate": self._rate(self.corrupted, self.preserve_cases), - "false_negative_rate": self._rate(self.escaped, self.measured), - "review_rate": self._rate(self.routed_to_review, self.review_cases), - "escape_rate": self._rate(self.escaped, self.measured), - "audit_completeness": self._rate(self.audited, self.changes), - } - - -def score(observations: list[Observation]) -> Metrics: - """Aggregate observations into the named metrics.""" - m = Metrics(total=len(observations)) - for o in observations: - if o.error: - m.errors += 1 - m.failures.append(f"{o.case.family}/{o.case.role}: {o.error}") - continue - if o.case.expected is None: - m.spec_gaps += 1 - continue - m.measured += 1 - expected = o.case.expected - - if o.changed: - m.changes += 1 - if o.audited: - m.audited += 1 - - if expected is D.PRESERVE: - m.preserve_cases += 1 - if o.changed: - m.corrupted += 1 - m.failures.append( - f"CORRUPTION {o.case.family}/{o.case.role}: " - f"{o.case.token!r} -> {o.final_value!r}" - ) - else: - m.preserved += 1 - m.correct += 1 - continue - - if expected is D.REPAIR: - m.repair_cases += 1 - if o.repaired_correctly: - m.repaired_correctly += 1 - m.correct += 1 - elif o.changed: - m.misrepaired += 1 - m.failures.append( - f"MISREPAIR {o.case.family}/{o.case.role}: {o.case.token!r} -> " - f"{o.final_value!r}, expected {o.case.repaired!r}" - ) - else: - m.escaped += 1 - m.failures.append( - f"ESCAPE {o.case.family}/{o.case.role}: {o.case.token!r} unchanged, " - f"expected repair to {o.case.repaired!r}" - ) - continue - - # REVIEW / QUARANTINE / REJECT / FLAG - m.review_cases += 1 - if satisfies(expected, o.observed): - m.routed_to_review += 1 - m.correct += 1 - else: - m.escaped += 1 - m.failures.append( - f"ESCAPE {o.case.family}/{o.case.role}: {o.case.token!r} got " - f"{o.observed.value}, expected {expected.value}" - ) - return m - - -def run_corpus(cases=TRAPS, *, semantic: bool = True) -> tuple[list[Observation], Metrics]: - observations = [observe(c, semantic=semantic) for c in cases] - return observations, score(observations) diff --git a/benchmarks/performance/semantic_probe.py b/benchmarks/performance/semantic_probe.py deleted file mode 100644 index d1628ae8..00000000 --- a/benchmarks/performance/semantic_probe.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Development-only instrumentation for semantic context repetition. - -The probe wraps the pure value-shape helpers used by -``freshdata.semantic.context``. It deliberately keeps all observations in -memory and creates a fresh collector for each context build, making the -result useful for estimating the benefit of a per-build memoization layer -without changing production behavior. -""" - -from __future__ import annotations - -from collections.abc import Iterator, Mapping -from contextlib import ExitStack, contextmanager -from dataclasses import dataclass -from typing import Any, Callable -from unittest.mock import patch - -import pandas as pd - -from freshdata.config import CleanConfig -from freshdata.semantic import context as semantic_context -from freshdata.semantic.types import SemanticContext - -# ``column_name_is_identifier`` is intentionally not included: the repetition -# candidates are the six value parsers plus the email-value regex check. -_OPERATIONS = ( - "is_plain_number", - "parse_number_words", - "parse_boolean", - "parse_currency", - "parse_unit", - "email_value", - "looks_like_date_value", -) -_CALLABLE_OPERATIONS = ( - "is_plain_number", - "parse_number_words", - "parse_boolean", - "parse_currency", - "parse_unit", - "looks_like_date_value", -) - -# Exact-type gating keeps key construction safe and mirrors the values each -# helper is designed to inspect. In particular, bool is observable by -# ``is_plain_number`` even though the helper itself excludes it semantically. -_ALLOWED_TYPES: Mapping[str, tuple[type[object], ...]] = { - "is_plain_number": (int, float, bool, str), - "parse_number_words": (str,), - "parse_boolean": (str,), - "parse_currency": (str,), - "parse_unit": (str,), - "looks_like_date_value": (str,), - "email_value": (str,), -} - - -@dataclass(frozen=True) -class OperationProbe: - """Immutable observations for one semantic helper during one build.""" - - total_calls: int - eligible_calls: int - bypassed_calls: int - unique_keys: int - theoretical_hits: int - eligible_values: tuple[object, ...] - - @property - def hit_rate(self) -> float: - return self.theoretical_hits / self.eligible_calls if self.eligible_calls else 0.0 - - -@dataclass(frozen=True) -class SemanticProbeBuild: - """Per-operation probe results for one ``build_semantic_context`` call.""" - - by_operation: Mapping[str, OperationProbe] - - @property - def total_theoretical_hits(self) -> int: - return sum(item.theoretical_hits for item in self.by_operation.values()) - - @property - def total_calls(self) -> int: - return sum(item.total_calls for item in self.by_operation.values()) - - @property - def total_eligible_calls(self) -> int: - return sum(item.eligible_calls for item in self.by_operation.values()) - - @property - def total_bypassed_calls(self) -> int: - return sum(item.bypassed_calls for item in self.by_operation.values()) - - -@dataclass -class _MutableOperationProbe: - total_calls: int = 0 - eligible_calls: int = 0 - bypassed_calls: int = 0 - theoretical_hits: int = 0 - - def __post_init__(self) -> None: - self.seen_keys: set[tuple[str, type[object], object]] = set() - self.eligible_values: list[object] = [] - - -def _eligible(operation: str, value: object) -> bool: - """Return whether *value* may safely participate in a repetition key.""" - - return type(value) in _ALLOWED_TYPES[operation] - - -class _ProbeCollector: - """Mutable collector used only while one context build is running.""" - - def __init__(self) -> None: - self._operations = {name: _MutableOperationProbe() for name in _OPERATIONS} - - def record(self, operation: str, value: object) -> None: - state = self._operations[operation] - state.total_calls += 1 - if not _eligible(operation, value): - state.bypassed_calls += 1 - return - - state.eligible_calls += 1 - state.eligible_values.append(value) - # Every allowed exact type is hash-safe. Unsafe values are returned - # above before this key expression is evaluated. - key = (operation, type(value), value) - if key in state.seen_keys: - state.theoretical_hits += 1 - else: - state.seen_keys.add(key) - - def finish_build(self) -> SemanticProbeBuild: - return SemanticProbeBuild( - by_operation={ - name: OperationProbe( - total_calls=state.total_calls, - eligible_calls=state.eligible_calls, - bypassed_calls=state.bypassed_calls, - unique_keys=len(state.seen_keys), - theoretical_hits=state.theoretical_hits, - eligible_values=tuple(state.eligible_values), - ) - for name, state in self._operations.items() - } - ) - - -class _EmailValueProxy: - """Regex proxy that records the already-stripped email candidate.""" - - def __init__(self, delegate: Any, probe: _ProbeCollector) -> None: - self._delegate = delegate - self._probe = probe - - def match(self, value: object, *args: object, **kwargs: object) -> Any: - self._probe.record("email_value", value) - return self._delegate.match(value, *args, **kwargs) - - -@contextmanager -def _patched_context_operations( - probe: _ProbeCollector | None = None, -) -> Iterator[_ProbeCollector]: - """Temporarily wrap context helper references with *probe* recorders.""" - - if probe is None: - probe = _ProbeCollector() - with ExitStack() as stack: - for operation in _CALLABLE_OPERATIONS: - original = getattr(semantic_context, operation) - - def wrapped( - value: object, - *args: object, - _operation: str = operation, - _original: Callable[..., Any] = original, - **kwargs: object, - ) -> Any: - probe.record(_operation, value) - return _original(value, *args, **kwargs) - - stack.enter_context(patch.object(semantic_context, operation, wrapped)) - - stack.enter_context( - patch.object( - semantic_context, - "_EMAIL_VALUE", - _EmailValueProxy(semantic_context._EMAIL_VALUE, probe), - ) - ) - yield probe - - -def probe_context_build( - df: pd.DataFrame, - config: CleanConfig, - *, - stats: dict[object, tuple[int, int, int | None]] | None = None, -) -> tuple[SemanticContext, SemanticProbeBuild]: - """Build semantic context and collect helper repetition observations.""" - - with _patched_context_operations() as probe: - context = semantic_context.build_semantic_context(df, config, stats=stats) - return context, probe.finish_build() - - -__all__ = [ - "OperationProbe", - "SemanticProbeBuild", - "probe_context_build", -] diff --git a/benchmarks/truthbench/inventory.py b/benchmarks/truthbench/inventory.py deleted file mode 100644 index 706960cd..00000000 --- a/benchmarks/truthbench/inventory.py +++ /dev/null @@ -1,383 +0,0 @@ -"""Manifest of FreshData's public decision, sink, and caller-directed surfaces.""" - -from __future__ import annotations - -import argparse -from collections.abc import Iterable -from dataclasses import dataclass -from enum import Enum - -import freshdata as fd -from freshdata.domains.registry import available as available_domains - - -class SurfaceClass(str, Enum): - DECISION = "decision" - SINK = "sink" - EXPLICIT_TRANSFORM = "explicit-transform" - DATA_MODEL = "data-model" - CONFIGURATION = "configuration" - REGISTRATION = "registration" - OUT_OF_SCOPE = "out-of-scope-with-reason" - - -@dataclass(frozen=True) -class SurfaceSpec: - name: str - version: int - classification: SurfaceClass - adapter: str | None - mutates: bool - deterministic: bool - backend_parity: bool - rationale: str - - def __post_init__(self) -> None: - if not self.name: - raise ValueError("surface name must not be empty") - if self.version != 1: - raise ValueError("unsupported surface spec version") - try: - classification = SurfaceClass(self.classification) - except ValueError as exc: - raise ValueError(f"unknown surface classification: {self.classification!r}") from exc - object.__setattr__(self, "classification", classification) - if classification in {SurfaceClass.DECISION, SurfaceClass.SINK} and not self.adapter: - raise ValueError(f"{classification.value} surface {self.name!r} requires an adapter") - if not self.rationale: - raise ValueError(f"surface {self.name!r} requires a rationale") - - -_CALLER_DIRECTED = { - "fill_missing", - "remove_outliers", - "resolve_duplicates", - "group_aggregate", - "set_display", - "get_display", - "reset_display", - "tokenize_value", - "detokenize_value", - "detokenize_series", - "make_vault", - "vault_metadata", - "anonymize", - "clean_text", - "clean_text_value", - "lint_text_encoding", - "mask_dataframe", -} -_REGISTRATION = { - "register_backend", - "register_comparator", - "register_expert", - "register_exporter", - "register_validator", - "registered_plugins", - "available_packs", -} -_SINKS = { - "build_exception_table", - "build_quality_report", - "export", - "export_dbt_tests", - "export_gx_suite", - "export_quality_ops", - "export_review_queue", - "generate_compliance_report", - "insight_report", - "stakeholder_summary", - "trust_gate_report", - "save_baseline", -} -_DECISIONS = { - "apply", - "apply_field_policy", - "apply_plan", - "apply_privacy_policy", - "apply_review_decisions", - "build_review_queue", - "cdc_profile", - "clean", - "clean_csv", - "clean_domain_file", - "clean_enterprise", - "clean_excel", - "clean_timeseries", - "cluster_column", - "compare_clean", - "compare_plans", - "compile_context", - "compute_trust_score", - "detect_outliers", - "detect_pii", - "diff_schema", - "enforce_contract", - "evaluate_quality_debt", - "explain_clean", - "infer_roles", - "learn", - "learn_cleaning_memory", - "link", - "link_entities", - "monitor_contract", - "parse_domain", - "pipeline", - "pipeline_clean", - "plan", - "profile", - "recalibrate_weights", - "redaction_columns", - "resolve_entities", - "run_semantic_validation", - "run_suite", - "suggest_join_keys", - "suggest_plan", - "validate", - "validate_fields", - "analyze_dataset", - "check_k_anonymity", - "classify_columns", - "compare_to_baseline", - "merge_clusters", - "merge_entities", - "schema_of", - "load_review_decisions", -} -_CONFIG_FUNCTIONS = { - "education_template", - "build_baseline", - "get_template", - "healthcare_template", - "load_baseline", - "load_cleaning_memory", - "load_compliance_pack", - "load_privacy_policy", - "load_profile", - "media_template", - "retail_template", - "save_profile", -} -_OUT_OF_SCOPE = {"enterprise", "models", "testing", "__version__"} - - -def cli_commands() -> tuple[str, ...]: - """Return top-level and nested enterprise CLI command paths.""" - - from freshdata.enterprise.cli import build_parser - - parser = build_parser() - result: set[str] = set() - - def visit(current: argparse.ArgumentParser, prefix: str = "") -> None: - for action in current._actions: - if not isinstance(action, argparse._SubParsersAction): - continue - choices = action.choices - for command, child in choices.items(): - path = f"{prefix} {command}".strip() - result.add(path) - if isinstance(child, argparse.ArgumentParser): - visit(child, path) - - visit(parser) - return tuple(sorted(result)) - - -def discover_public_names() -> tuple[str, ...]: - """Discover all names and registered command/domain surfaces in this checkout.""" - - names = set(fd.__dir__()) - from freshdata.experimental import ai_copilot - - names.update(ai_copilot.__all__) - names.update(f"domain:{name}" for name in available_domains()) - names.update(f"cli:{name}" for name in cli_commands()) - return tuple(sorted(names)) - - -#: Concrete adapter per decision surface. Cleaning-shaped surfaces route to -#: the cleaning adapter's operation dispatch, analysis/validation surfaces to -#: the validation adapter, privacy surfaces to the privacy adapter, and the -#: experimental Copilot to its own adapter. Sinks route to reporting. -_DECISION_ADAPTERS = { - "apply": "cleaning", - "apply_plan": "cleaning", - "clean": "cleaning", - "clean_csv": "cleaning", - "clean_domain_file": "cleaning", - "clean_enterprise": "cleaning", - "clean_excel": "cleaning", - "clean_timeseries": "cleaning", - "compare_clean": "cleaning", - "compare_plans": "cleaning", - "explain_clean": "cleaning", - "learn": "cleaning", - "learn_cleaning_memory": "cleaning", - "pipeline": "cleaning", - "pipeline_clean": "cleaning", - "plan": "cleaning", - "suggest_plan": "cleaning", - "apply_privacy_policy": "privacy", - "check_k_anonymity": "privacy", - "detect_pii": "privacy", - "redaction_columns": "privacy", - "analyze_dataset": "copilot", -} - - -def _decision_adapter(name: str) -> str: - return _DECISION_ADAPTERS.get(name, "validation") - - -def _classify(name: str) -> tuple[SurfaceClass, str | None, bool, bool, bool, str]: - if name.startswith("domain:"): - return ( - SurfaceClass.DECISION, - "validation", - False, - True, - True, - "bundled domain validator decision surface", - ) - if name.startswith("cli:"): - return ( - SurfaceClass.DECISION, - "reporting", - False, - True, - False, - "enterprise CLI decision/report command", - ) - if name in _CALLER_DIRECTED: - return ( - SurfaceClass.EXPLICIT_TRANSFORM, - None, - name not in {"get_display", "reset_display", "vault_metadata"}, - True, - False, - "caller-directed explicit transform; no inferred disposition credit", - ) - if name in _REGISTRATION: - return ( - SurfaceClass.REGISTRATION, - None, - True, - True, - False, - "plugin registration or registry inspection", - ) - if name in _CONFIG_FUNCTIONS: - return ( - SurfaceClass.CONFIGURATION, - None, - False, - True, - False, - "configuration/template loading surface", - ) - if name in _SINKS: - return ( - SurfaceClass.SINK, - "reporting", - False, - True, - False, - "report, audit, or export sink", - ) - if name in _DECISIONS: - return ( - SurfaceClass.DECISION, - _decision_adapter(name), - name.startswith(("clean", "apply", "run_")), - True, - True, - "public FreshData decision surface", - ) - if name in _OUT_OF_SCOPE: - return ( - SurfaceClass.OUT_OF_SCOPE, - None, - False, - True, - False, - "namespace or metadata surface; no cell disposition", - ) - if name.startswith("__"): - return SurfaceClass.CONFIGURATION, None, False, True, False, "package metadata" - if name.endswith(("Config", "Policy", "Rule", "Spec", "Weights")) or name in { - "Action", - "Jurisdiction", - "CompliancePack", - }: - return ( - SurfaceClass.CONFIGURATION, - None, - False, - True, - False, - "configuration or policy model", - ) - if name[:1].isupper(): - return SurfaceClass.DATA_MODEL, None, False, True, False, "public data/report model" - raise ValueError(f"unclassified FreshData public surface: {name}") - - -def build_manifest(names: Iterable[str] | None = None) -> tuple[SurfaceSpec, ...]: - """Build an immutable manifest from the currently discovered public surfaces.""" - - selected = discover_public_names() if names is None else tuple(sorted(set(names))) - specs: list[SurfaceSpec] = [] - for name in selected: - classification, adapter, mutates, deterministic, parity, rationale = _classify(name) - specs.append( - SurfaceSpec( - name=name, - version=1, - classification=classification, - adapter=adapter, - mutates=mutates, - deterministic=deterministic, - backend_parity=parity, - rationale=rationale, - ) - ) - validate_manifest(specs) - return tuple(specs) - - -def validate_manifest(manifest: Iterable[SurfaceSpec]) -> None: - """Validate uniqueness, classifications, and adapter requirements.""" - - specs = tuple(manifest) - names = [spec.name for spec in specs] - if len(names) != len(set(names)): - raise ValueError("surface manifest contains duplicate names") - from .surfaces import adapters - - registered = adapters() - for spec in specs: - if not isinstance(spec, SurfaceSpec): - raise TypeError("surface manifest entries must be SurfaceSpec instances") - # Re-run constructor checks for objects supplied by callers or tests. - SurfaceSpec(**{field: getattr(spec, field) for field in SurfaceSpec.__dataclass_fields__}) - if ( - spec.classification in {SurfaceClass.DECISION, SurfaceClass.SINK} - and spec.adapter not in registered - ): - raise ValueError(f"surface {spec.name!r} references unknown adapter {spec.adapter!r}") - - -SURFACE_MANIFEST = build_manifest() -PUBLIC_SURFACES = SURFACE_MANIFEST - -__all__ = [ - "PUBLIC_SURFACES", - "SURFACE_MANIFEST", - "SurfaceClass", - "SurfaceSpec", - "build_manifest", - "cli_commands", - "discover_public_names", - "validate_manifest", -] diff --git a/constraints/ci.txt b/constraints/ci.txt deleted file mode 100644 index b0157d23..00000000 --- a/constraints/ci.txt +++ /dev/null @@ -1,1535 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv export --frozen --no-hashes --all-extras --no-emit-project -o constraints/ci.txt -a2wsgi==1.10.10 - # via apache-airflow-core -agate==1.9.1 - # via - # dbt-adapters - # dbt-common - # dbt-core -aiosmtplib==5.1.2 ; python_full_version >= '3.10' - # via apache-airflow-providers-smtp -aiosqlite==0.21.0 - # via apache-airflow-core -alembic==1.16.5 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dagster -alembic==1.18.5 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # dagster -annotated-doc==0.0.4 - # via - # fastapi - # typer -annotated-types==0.7.0 - # via pydantic -antlr4-python3-runtime==4.13.2 - # via dagster -anyio==4.12.1 ; python_full_version < '3.10' - # via - # httpx - # starlette - # watchfiles -anyio==4.14.1 ; python_full_version >= '3.10' - # via - # httpx - # starlette - # watchfiles -anywidget==0.9.21 ; python_full_version < '3.10' - # via freshdata-cleaner -anywidget==0.11.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -apache-airflow==3.0.6 ; python_full_version < '3.10' - # via - # apache-airflow-providers-common-compat - # apache-airflow-providers-common-io - # apache-airflow-providers-common-sql - # apache-airflow-providers-smtp - # apache-airflow-providers-standard - # freshdata-cleaner -apache-airflow==3.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow-providers-common-compat - # apache-airflow-providers-common-io - # apache-airflow-providers-common-sql - # apache-airflow-providers-smtp - # apache-airflow-providers-standard - # freshdata-cleaner -apache-airflow-core==3.0.6 ; python_full_version < '3.10' - # via - # apache-airflow - # apache-airflow-task-sdk -apache-airflow-core==3.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow - # apache-airflow-task-sdk -apache-airflow-providers-common-compat==1.7.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # apache-airflow-providers-smtp -apache-airflow-providers-common-compat==1.16.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-providers-common-io - # apache-airflow-providers-common-sql - # apache-airflow-providers-smtp - # apache-airflow-providers-standard -apache-airflow-providers-common-io==1.6.0 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-common-io==1.8.0 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-providers-common-sql==1.27.2 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-common-sql==2.0.2 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-providers-smtp==2.1.0 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-smtp==3.0.2 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-providers-standard==1.3.0 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-standard==1.16.0 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-task-sdk==1.0.6 ; python_full_version < '3.10' - # via - # apache-airflow - # apache-airflow-core -apache-airflow-task-sdk==1.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow - # apache-airflow-core -argcomplete==3.7.0 - # via apache-airflow-core -arrow==1.4.0 ; python_full_version >= '3.10' - # via isoduration -asgiref==3.11.1 - # via - # apache-airflow-core - # apache-airflow-providers-common-compat - # apache-airflow-task-sdk -asttokens==3.0.1 - # via stack-data -attrs==26.1.0 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # jsonschema - # outcome - # referencing - # svcs -babel==2.18.0 - # via - # agate - # apache-airflow-task-sdk - # great-tables - # mkdocs-material -backports-strenum==1.3.1 ; python_full_version < '3.11' - # via cadwyn -backports-tarfile==1.2.0 ; python_full_version < '3.12' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via jaraco-context -backrefs==6.2 ; python_full_version < '3.10' - # via mkdocs-material -backrefs==7.0 ; python_full_version >= '3.10' - # via mkdocs-material -blinker==1.9.0 ; python_full_version < '3.10' - # via flask -blis==1.2.0 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # thinc -blis==1.3.3 ; python_full_version >= '3.10' - # via thinc -build==1.4.4 ; python_full_version < '3.10' - # via freshdata-cleaner -build==1.6.1 ; python_full_version >= '3.10' - # via freshdata-cleaner -cachetools==7.1.4 ; python_full_version >= '3.10' - # via apache-airflow-core -cadwyn==6.0.3 ; python_full_version < '3.10' - # via apache-airflow-core -cadwyn==7.0.0 ; python_full_version >= '3.10' - # via apache-airflow-core -catalogue==2.0.10 - # via - # spacy - # srsly - # thinc -certifi==2026.6.17 - # via - # httpcore - # httpx - # requests - # sentry-sdk -cffi==2.0.0 ; python_full_version < '3.10' and platform_python_implementation != 'PyPy' - # via cryptography -cffi==2.1.0 ; python_full_version >= '3.10' - # via - # cryptography - # multimark -cfgv==3.4.0 ; python_full_version < '3.10' - # via pre-commit -cfgv==3.5.0 ; python_full_version >= '3.10' - # via pre-commit -charset-normalizer==3.4.9 - # via requests -cleanlab==2.6.5 ; python_full_version < '3.10' - # via freshdata-cleaner -cleanlab==2.9.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -click==8.1.8 ; python_full_version < '3.10' - # via - # dagster - # dbt-core - # dbt-semantic-interfaces - # flask - # mkdocs - # rich-toolkit - # typer - # uvicorn -click==8.4.2 ; python_full_version >= '3.10' - # via - # dagster - # dbt-core - # dbt-semantic-interfaces - # huggingface-hub - # mkdocs - # multimark - # nokap - # rich-toolkit - # uvicorn -cloudpathlib==0.24.0 - # via weasel -colorama==0.4.6 - # via - # build - # click - # colorlog - # dbt-common - # griffe - # ipython - # mkdocs - # mkdocs-material - # pytest - # tqdm - # typer - # uvicorn - # wasabi -coloredlogs==14.0 - # via - # dagster - # onnxruntime -colorlog==6.10.1 - # via - # apache-airflow-core - # apache-airflow-task-sdk -comm==0.2.3 - # via ipywidgets -commonmark==0.9.2 ; python_full_version < '3.10' - # via great-tables -confection==0.1.5 ; python_full_version < '3.10' - # via - # thinc - # weasel -confection==1.3.3 ; python_full_version >= '3.10' - # via - # spacy - # thinc - # weasel -coverage==7.10.7 ; python_full_version < '3.10' - # via pytest-cov -coverage==7.15.0 ; python_full_version >= '3.10' - # via pytest-cov -cron-descriptor==2.1.0 - # via apache-airflow-core -croniter==6.2.4 - # via apache-airflow-core -cryptography==44.0.3 ; python_full_version < '3.10' - # via - # apache-airflow-core - # presidio-anonymizer - # secretstorage -cryptography==46.0.7 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # presidio-anonymizer - # secretstorage -cymem==2.0.13 - # via - # preshed - # spacy - # thinc -daff==1.4.2 - # via dbt-core -dagster==1.12.1 ; python_full_version < '3.10' - # via freshdata-cleaner -dagster==1.13.13 ; python_full_version >= '3.10' - # via freshdata-cleaner -dagster-pipes==1.12.1 ; python_full_version < '3.10' - # via dagster -dagster-pipes==1.13.13 ; python_full_version >= '3.10' - # via dagster -dagster-shared==1.12.1 ; python_full_version < '3.10' - # via dagster -dagster-shared==1.13.13 ; python_full_version >= '3.10' - # via dagster -dbt-adapters==1.17.3 ; python_full_version < '3.10' - # via dbt-core -dbt-adapters==1.24.4 ; python_full_version >= '3.10' - # via dbt-core -dbt-common==1.34.2 ; python_full_version < '3.10' - # via - # dbt-adapters - # dbt-core -dbt-common==1.38.0 ; python_full_version >= '3.10' - # via - # dbt-adapters - # dbt-core -dbt-core==1.10.22 ; python_full_version < '3.10' - # via freshdata-cleaner -dbt-core==1.11.12 ; python_full_version >= '3.10' - # via freshdata-cleaner -dbt-extractor==0.6.0 - # via dbt-core -dbt-protos==1.0.541 - # via - # dbt-adapters - # dbt-common - # dbt-core -dbt-semantic-interfaces==0.9.0 - # via dbt-core -decorator==5.3.1 - # via ipython -deepdiff==8.6.2 - # via dbt-common -deprecated==1.3.1 - # via apache-airflow-core -dill==0.4.1 - # via apache-airflow-core -distlib==0.4.3 - # via virtualenv -dnspython==2.7.0 ; python_full_version < '3.10' - # via email-validator -dnspython==2.8.0 ; python_full_version >= '3.10' - # via email-validator -docstring-parser==0.18.0 - # via dagster -docutils==0.23 - # via readme-renderer -duckdb==1.4.5 ; python_full_version < '3.10' - # via freshdata-cleaner -duckdb==1.5.4 ; python_full_version >= '3.10' - # via freshdata-cleaner -email-validator==2.3.0 - # via - # fastapi - # pydantic -et-xmlfile==2.0.0 - # via openpyxl -eval-type-backport==0.4.0 ; python_full_version < '3.10' - # via apache-airflow-core -exceptiongroup==1.3.1 ; python_full_version < '3.11' - # via - # anyio - # ipython - # pytest -executing==2.2.1 - # via stack-data -faicons==0.2.2 - # via great-tables -fastapi==0.128.8 ; python_full_version < '3.10' - # via - # apache-airflow-core - # cadwyn -fastapi==0.136.3 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # cadwyn -fastapi-cli==0.0.22 ; python_full_version < '3.10' - # via fastapi -fastapi-cli==0.0.29 ; python_full_version >= '3.10' - # via fastapi -fastapi-cloud-cli==0.12.0 ; python_full_version < '3.10' - # via fastapi-cli -fastar==0.11.0 ; python_full_version < '3.10' - # via fastapi-cloud-cli -filelock==3.19.1 ; python_full_version < '3.10' - # via - # dagster - # huggingface-hub - # python-discovery - # tldextract - # virtualenv -filelock==3.29.7 ; python_full_version >= '3.10' - # via - # dagster - # huggingface-hub - # python-discovery - # tldextract - # virtualenv -flask==3.1.3 ; python_full_version < '3.10' - # via apache-airflow-core -flatbuffers==25.12.19 - # via onnxruntime -fsspec==2025.10.0 ; python_full_version < '3.10' - # via - # apache-airflow-task-sdk - # huggingface-hub - # universal-pathlib -fsspec==2026.6.0 ; python_full_version >= '3.10' - # via - # apache-airflow-task-sdk - # huggingface-hub - # universal-pathlib -ghp-import==2.1.0 - # via mkdocs -googleapis-common-protos==1.75.0 - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -great-tables==0.21.0 ; python_full_version < '3.10' - # via freshdata-cleaner -great-tables==0.22.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -greenback==1.3.0 ; python_full_version >= '3.10' - # via apache-airflow-task-sdk -greenlet==3.2.5 ; python_full_version < '3.10' - # via sqlalchemy -greenlet==3.5.3 ; python_full_version >= '3.10' - # via - # greenback - # sqlalchemy -griffe==1.14.0 ; python_full_version < '3.10' - # via mkdocstrings-python -griffelib==2.1.0 ; python_full_version >= '3.10' - # via mkdocstrings-python -grpcio==1.80.0 ; python_full_version < '3.10' - # via - # dagster - # grpcio-health-checking - # opentelemetry-exporter-otlp-proto-grpc -grpcio==1.82.1 ; python_full_version >= '3.10' - # via - # dagster - # grpcio-health-checking - # opentelemetry-exporter-otlp-proto-grpc -grpcio-health-checking==1.80.0 ; python_full_version < '3.10' - # via dagster -grpcio-health-checking==1.81.1 ; python_full_version >= '3.10' - # via dagster -gunicorn==23.0.0 ; python_full_version < '3.10' - # via apache-airflow-core -h11==0.16.0 - # via - # httpcore - # uvicorn -hf-xet==1.5.1 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' - # via huggingface-hub -htmltools==0.6.0 ; python_full_version < '3.10' - # via - # faicons - # great-tables -htmltools==0.7.0 ; python_full_version >= '3.10' - # via - # faicons - # great-tables -httpcore==1.0.9 - # via httpx -httptools==0.8.0 - # via uvicorn -httpx==0.28.1 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # fastapi - # fastapi-cloud-cli - # huggingface-hub - # weasel -huggingface-hub==1.8.0 ; python_full_version < '3.10' - # via tokenizers -huggingface-hub==1.23.0 ; python_full_version >= '3.10' - # via tokenizers -humanfriendly==10.0 - # via coloredlogs -id==1.6.1 - # via twine -identify==2.6.15 ; python_full_version < '3.10' - # via pre-commit -identify==2.6.19 ; python_full_version >= '3.10' - # via pre-commit -idna==3.18 - # via - # anyio - # email-validator - # httpx - # requests - # tldextract -importlib-metadata==8.7.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # build - # dbt-semantic-interfaces - # flask - # great-tables - # keyring - # markdown - # mkdocs - # mkdocs-get-deps - # mkdocstrings - # opentelemetry-api - # twine -importlib-metadata==8.9.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # build - # dbt-semantic-interfaces - # great-tables - # keyring -importlib-resources==6.5.2 ; python_full_version < '3.10' - # via great-tables -importlib-resources==7.1.0 ; python_full_version >= '3.10' - # via great-tables -iniconfig==2.1.0 ; python_full_version < '3.10' - # via pytest -iniconfig==2.3.0 ; python_full_version >= '3.10' - # via pytest -ipython==8.18.1 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # ipywidgets -ipython==8.39.0 ; python_full_version == '3.10.*' - # via - # freshdata-cleaner - # ipywidgets -ipython==9.15.0 ; python_full_version >= '3.11' - # via - # freshdata-cleaner - # ipywidgets -ipython-pygments-lexers==1.1.1 ; python_full_version >= '3.11' - # via ipython -ipywidgets==8.1.8 - # via anywidget -isodate==0.6.1 ; python_full_version < '3.10' - # via - # agate - # dbt-common -isodate==0.7.2 ; python_full_version >= '3.10' - # via - # agate - # dbt-common -isoduration==20.11.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -itables==2.8.1 - # via freshdata-cleaner -itsdangerous==2.2.0 - # via - # apache-airflow-core - # flask -jaraco-classes==3.4.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-context==6.1.1 ; python_full_version < '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-context==6.1.2 ; python_full_version >= '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-functools==4.4.0 ; python_full_version < '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-functools==4.5.0 ; python_full_version >= '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jedi==0.19.2 ; python_full_version < '3.10' - # via ipython -jedi==0.20.0 ; python_full_version >= '3.10' - # via ipython -jeepney==0.9.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux' - # via - # keyring - # secretstorage -jinja2==3.1.6 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # cadwyn - # dagster - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # fastapi - # flask - # mkdocs - # mkdocs-material - # mkdocstrings - # spacy -joblib==1.5.3 - # via scikit-learn -jsonschema==4.25.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # freshdata-cleaner -jsonschema==4.26.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # freshdata-cleaner -jsonschema-specifications==2025.9.1 - # via jsonschema -jupyterlab-widgets==3.0.16 - # via ipywidgets -kafka-python==3.0.8 - # via freshdata-cleaner -keyring==25.7.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x' - # via twine -langcodes==3.5.1 ; python_full_version < '3.10' - # via spacy -lazy-object-proxy==1.12.0 - # via apache-airflow-core -leather==0.4.1 - # via agate -libcst==1.8.6 - # via apache-airflow-core -librt==0.13.0 ; platform_python_implementation != 'PyPy' - # via mypy -linkify-it-py==2.0.3 ; python_full_version < '3.10' - # via apache-airflow-core -linkify-it-py==2.1.0 ; python_full_version >= '3.10' - # via apache-airflow-core -lockfile==0.12.2 - # via - # apache-airflow-core - # python-daemon -mako==1.3.12 - # via alembic -markdown==3.9 ; python_full_version < '3.10' - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown==3.10.2 ; python_full_version >= '3.10' - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown-it-py==3.0.0 ; python_full_version < '3.10' - # via rich -markdown-it-py==4.2.0 ; python_full_version >= '3.10' - # via rich -markupsafe==3.0.3 - # via - # flask - # jinja2 - # mako - # mkdocs - # mkdocs-autorefs - # mkdocstrings - # werkzeug -mashumaro==3.14 - # via - # dbt-adapters - # dbt-common - # dbt-core -matplotlib-inline==0.2.2 - # via ipython -maturin==1.14.1 - # via freshdata-cleaner -mdurl==0.1.2 - # via markdown-it-py -mergedeep==1.3.4 - # via - # mkdocs - # mkdocs-get-deps -methodtools==0.4.7 - # via - # apache-airflow-core - # apache-airflow-providers-common-sql - # apache-airflow-task-sdk -mkdocs==1.6.1 - # via - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings -mkdocs-autorefs==1.4.4 - # via - # mkdocstrings - # mkdocstrings-python -mkdocs-get-deps==0.2.2 - # via mkdocs -mkdocs-material==9.7.6 - # via freshdata-cleaner -mkdocs-material-extensions==1.3.1 - # via mkdocs-material -mkdocstrings==0.30.1 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # mkdocstrings-python -mkdocstrings==1.0.5 ; python_full_version >= '3.10' - # via - # freshdata-cleaner - # mkdocstrings-python -mkdocstrings-python==1.18.2 ; python_full_version < '3.10' - # via mkdocstrings -mkdocstrings-python==2.0.5 ; python_full_version >= '3.10' - # via mkdocstrings -more-itertools==10.8.0 - # via - # apache-airflow-providers-common-sql - # dbt-semantic-interfaces - # jaraco-classes - # jaraco-functools -mpmath==1.3.0 ; python_full_version < '3.11' - # via sympy -msgpack==1.1.2 ; python_full_version < '3.10' - # via mashumaro -msgpack==1.2.1 ; python_full_version >= '3.10' - # via mashumaro -msgspec==0.20.0 ; python_full_version < '3.10' - # via apache-airflow-task-sdk -msgspec==0.21.1 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -multimark==0.2.0 ; python_full_version >= '3.10' - # via great-tables -murmurhash==1.0.15 - # via - # preshed - # spacy - # thinc -mypy==1.19.1 - # via freshdata-cleaner -mypy-extensions==1.1.0 - # via mypy -narwhals==2.21.0 ; python_full_version < '3.10' - # via plotly -narwhals==2.23.0 ; python_full_version >= '3.10' - # via - # plotly - # scikit-learn -natsort==8.4.0 ; python_full_version >= '3.10' - # via apache-airflow-core -networkx==3.2.1 ; python_full_version < '3.10' - # via dbt-core -networkx==3.4.2 ; python_full_version == '3.10.*' - # via dbt-core -networkx==3.6.1 ; python_full_version >= '3.11' - # via dbt-core -nh3==0.3.6 - # via readme-renderer -nodeenv==1.10.0 - # via pre-commit -nokap==0.1.0 ; python_full_version >= '3.10' - # via great-tables -numpy==2.0.2 ; python_full_version < '3.10' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -numpy==2.2.6 ; python_full_version == '3.10.*' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -numpy==2.4.6 ; python_full_version == '3.11.*' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -numpy==2.5.1 ; python_full_version >= '3.12' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -onnxruntime==1.20.1 ; python_full_version < '3.10' - # via freshdata-cleaner -onnxruntime==1.24.3 ; python_full_version == '3.10.*' - # via freshdata-cleaner -onnxruntime==1.27.0 ; python_full_version >= '3.11' - # via freshdata-cleaner -openpyxl==3.1.5 - # via freshdata-cleaner -opentelemetry-api==1.41.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-api==1.43.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.41.1 ; python_full_version < '3.10' - # via apache-airflow-core -opentelemetry-exporter-otlp==1.43.0 ; python_full_version >= '3.10' - # via apache-airflow-core -opentelemetry-exporter-otlp-proto-common==1.41.1 ; python_full_version < '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-common==1.43.0 ; python_full_version >= '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.41.1 ; python_full_version < '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-grpc==1.43.0 ; python_full_version >= '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.41.1 ; python_full_version < '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.43.0 ; python_full_version >= '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-proto==1.41.1 ; python_full_version < '3.10' - # via - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-proto==1.43.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.41.1 ; python_full_version < '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.43.0 ; python_full_version >= '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.62b1 ; python_full_version < '3.10' - # via opentelemetry-sdk -opentelemetry-semantic-conventions==0.64b0 ; python_full_version >= '3.10' - # via opentelemetry-sdk -orderly-set==5.5.0 - # via deepdiff -outcome==1.3.0.post0 ; python_full_version >= '3.10' - # via greenback -packaging==26.2 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # build - # dagster-shared - # dbt-core - # gunicorn - # htmltools - # huggingface-hub - # mkdocs - # onnxruntime - # plotly - # pytest - # spacy - # thinc - # twine - # weasel -paginate==0.5.7 - # via mkdocs-material -pandas==2.3.3 - # via - # cleanlab - # freshdata-cleaner -parsedatetime==2.6 - # via agate -parso==0.8.7 - # via jedi -pathlib-abc==0.5.2 - # via universal-pathlib -pathspec==0.12.1 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dbt-common - # dbt-core - # mkdocs - # mypy -pendulum==3.2.0 - # via - # apache-airflow-core - # apache-airflow-task-sdk -pexpect==4.9.0 ; (python_full_version < '3.10' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32') - # via ipython -phonenumbers==9.0.34 - # via presidio-analyzer -platformdirs==4.4.0 ; python_full_version < '3.10' - # via - # dagster-shared - # mkdocs-get-deps - # python-discovery - # virtualenv -platformdirs==4.10.0 ; python_full_version >= '3.10' - # via - # dagster-shared - # mkdocs-get-deps - # python-discovery - # virtualenv -plotly==6.9.0 - # via freshdata-cleaner -pluggy==1.6.0 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # pytest - # pytest-cov -polars==1.36.1 ; python_full_version < '3.10' - # via freshdata-cleaner -polars==1.42.1 ; python_full_version >= '3.10' - # via freshdata-cleaner -polars-runtime-32==1.36.1 ; python_full_version < '3.10' - # via polars -polars-runtime-32==1.42.1 ; python_full_version >= '3.10' - # via polars -pre-commit==4.3.0 ; python_full_version < '3.10' - # via freshdata-cleaner -pre-commit==4.6.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -preshed==3.0.13 - # via - # spacy - # thinc -presidio-analyzer==2.2.360 ; python_full_version < '3.10' - # via freshdata-cleaner -presidio-analyzer==2.2.362 ; python_full_version >= '3.10' - # via freshdata-cleaner -presidio-anonymizer==2.2.360 ; python_full_version < '3.10' - # via freshdata-cleaner -presidio-anonymizer==2.2.363 ; python_full_version >= '3.10' - # via freshdata-cleaner -prompt-toolkit==3.0.52 - # via ipython -protobuf==6.33.6 - # via - # dagster - # dbt-adapters - # dbt-common - # dbt-core - # dbt-protos - # googleapis-common-protos - # grpcio-health-checking - # onnxruntime - # opentelemetry-proto -psutil==7.2.2 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dagster - # freshdata-cleaner - # ipython -psygnal==0.14.2 ; python_full_version < '3.10' - # via anywidget -psygnal==0.15.1 ; python_full_version >= '3.10' - # via anywidget -ptyprocess==0.7.0 ; (python_full_version < '3.10' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32') - # via pexpect -pure-eval==0.2.3 - # via stack-data -py4j==0.10.9.9 - # via pyspark -pyarrow==21.0.0 ; python_full_version < '3.10' - # via freshdata-cleaner -pyarrow==25.0.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -pycparser==2.23 ; python_full_version < '3.10' and implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' - # via cffi -pycparser==3.0 ; python_full_version >= '3.10' and implementation_name != 'PyPy' - # via cffi -pydantic==2.13.4 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # cadwyn - # confection - # dagster-shared - # dbt-core - # dbt-semantic-interfaces - # fastapi - # fastapi-cloud-cli - # presidio-analyzer - # pydantic-extra-types - # pydantic-settings - # retryhttp - # spacy - # thinc - # weasel -pydantic-core==2.46.4 - # via pydantic -pydantic-extra-types==2.11.1 - # via fastapi -pydantic-settings==2.11.0 ; python_full_version < '3.10' - # via fastapi -pydantic-settings==2.14.2 ; python_full_version >= '3.10' - # via fastapi -pyffx==0.3.0 - # via freshdata-cleaner -pygments==2.20.0 - # via - # apache-airflow-core - # ipython - # ipython-pygments-lexers - # mkdocs-material - # pytest - # readme-renderer - # rich -pygtrie==2.5.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -pyjwt==2.13.0 - # via apache-airflow-core -pymdown-extensions==10.21.3 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # mkdocs-material - # mkdocstrings -pymdown-extensions==11.0.1 ; python_full_version >= '3.10' - # via - # freshdata-cleaner - # mkdocs-material - # mkdocstrings -pyproject-hooks==1.2.0 - # via build -pyreadline3==3.5.6 ; sys_platform == 'win32' - # via humanfriendly -pyspark==4.0.3 ; python_full_version < '3.10' - # via freshdata-cleaner -pyspark==4.1.2 ; python_full_version >= '3.10' - # via freshdata-cleaner -pytest==8.4.2 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # pytest-cov -pytest==9.1.1 ; python_full_version >= '3.10' - # via - # freshdata-cleaner - # pytest-cov -pytest-cov==7.1.0 - # via freshdata-cleaner -python-daemon==3.1.2 - # via apache-airflow-core -python-dateutil==2.9.0.post0 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # arrow - # croniter - # dbt-common - # dbt-semantic-interfaces - # ghp-import - # pandas - # pendulum -python-discovery==1.4.4 - # via virtualenv -python-dotenv==1.2.1 ; python_full_version < '3.10' - # via - # dagster - # pydantic-settings - # uvicorn -python-dotenv==1.2.2 ; python_full_version >= '3.10' - # via - # dagster - # pydantic-settings - # uvicorn -python-multipart==0.0.20 ; python_full_version < '3.10' - # via fastapi -python-multipart==0.0.32 ; python_full_version >= '3.10' - # via fastapi -python-slugify==8.0.4 - # via - # agate - # apache-airflow-core -pytimeparse==1.1.8 - # via agate -pytz==2026.2 - # via - # dagster - # dbt-adapters - # dbt-core - # pandas -pywin32==312 ; sys_platform == 'win32' - # via dagster -pywin32-ctypes==0.2.3 ; platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'win32' - # via keyring -pyyaml==6.0.3 - # via - # apache-airflow-core - # dagster-shared - # dbt-core - # dbt-semantic-interfaces - # freshdata-cleaner - # huggingface-hub - # libcst - # mkdocs - # mkdocs-get-deps - # pre-commit - # presidio-analyzer - # pymdown-extensions - # pyyaml-env-tag - # uvicorn -pyyaml-env-tag==1.1 - # via mkdocs -pyyaml-ft==8.0.0 ; python_full_version == '3.13.*' - # via libcst -readme-renderer==44.0 ; python_full_version < '3.10' - # via twine -readme-renderer==45.0 ; python_full_version >= '3.10' - # via twine -referencing==0.36.2 ; python_full_version < '3.10' - # via - # jsonschema - # jsonschema-specifications -referencing==0.37.0 ; python_full_version >= '3.10' - # via - # jsonschema - # jsonschema-specifications -regex==2026.1.15 ; python_full_version < '3.10' - # via presidio-analyzer -regex==2026.7.10 ; python_full_version >= '3.10' - # via presidio-analyzer -requests==2.32.5 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dagster - # dbt-common - # dbt-core - # mkdocs-material - # opentelemetry-exporter-otlp-proto-http - # requests-file - # requests-toolbelt - # snowplow-tracker - # spacy - # tldextract - # twine - # weasel -requests==2.34.2 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # dagster - # dbt-common - # dbt-core - # mkdocs-material - # opentelemetry-exporter-otlp-proto-http - # requests-file - # requests-toolbelt - # snowplow-tracker - # spacy - # tldextract - # twine -requests-file==3.0.1 - # via tldextract -requests-toolbelt==1.0.0 - # via twine -retryhttp==1.4.0 ; python_full_version < '3.10' - # via apache-airflow-task-sdk -rfc3986==2.0.0 - # via twine -rich==15.0.0 - # via - # apache-airflow-core - # dagster - # freshdata-cleaner - # rich-argparse - # rich-toolkit - # twine - # typer -rich-argparse==1.8.0 - # via apache-airflow-core -rich-toolkit==0.20.1 - # via - # fastapi-cli - # fastapi-cloud-cli -rignore==0.7.6 ; python_full_version < '3.10' - # via fastapi-cloud-cli -rpds-py==0.27.1 ; python_full_version < '3.10' - # via - # jsonschema - # referencing -rpds-py==0.30.0 ; python_full_version == '3.10.*' - # via - # jsonschema - # referencing -rpds-py==2026.6.3 ; python_full_version >= '3.11' - # via - # jsonschema - # referencing -ruff==0.15.21 - # via freshdata-cleaner -scikit-learn==1.6.1 ; python_full_version < '3.10' - # via - # cleanlab - # freshdata-cleaner -scikit-learn==1.7.2 ; python_full_version == '3.10.*' - # via - # cleanlab - # freshdata-cleaner -scikit-learn==1.9.0 ; python_full_version >= '3.11' - # via - # cleanlab - # freshdata-cleaner -scipy==1.13.1 ; python_full_version < '3.10' - # via scikit-learn -scipy==1.15.3 ; python_full_version == '3.10.*' - # via scikit-learn -scipy==1.17.1 ; python_full_version == '3.11.*' - # via scikit-learn -scipy==1.18.0 ; python_full_version >= '3.12' - # via scikit-learn -secretstorage==3.3.3 ; python_full_version < '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux' - # via keyring -secretstorage==3.5.0 ; python_full_version >= '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux' - # via keyring -sentry-sdk==2.64.0 ; python_full_version < '3.10' - # via fastapi-cloud-cli -setproctitle==1.3.7 - # via apache-airflow-core -setuptools==82.0.1 ; python_full_version < '3.10' - # via - # dagster - # spacy - # thinc -setuptools==83.0.0 ; python_full_version >= '3.10' - # via - # spacy - # thinc -shellingham==1.5.4 - # via typer -six==1.17.0 - # via - # dagster - # isodate - # pyffx - # python-dateutil - # wirerope -smart-open==7.5.0 ; python_full_version < '3.10' - # via weasel -smart-open==8.0.0 ; python_full_version >= '3.10' - # via weasel -sniffio==1.3.1 ; python_full_version >= '3.10' - # via greenback -snowplow-tracker==1.1.0 - # via dbt-core -spacy==3.8.7 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # presidio-analyzer -spacy==3.8.14 ; python_full_version >= '3.10' - # via presidio-analyzer -spacy-legacy==3.0.12 - # via spacy -spacy-loggers==1.0.5 - # via spacy -sqlalchemy==1.4.54 ; python_full_version < '3.10' - # via - # alembic - # apache-airflow-core - # dagster - # freshdata-cleaner - # sqlalchemy-jsonfield - # sqlalchemy-utils -sqlalchemy==2.0.51 ; python_full_version >= '3.10' - # via - # alembic - # apache-airflow-core - # dagster - # freshdata-cleaner -sqlalchemy-jsonfield==1.0.3 ; python_full_version < '3.10' - # via apache-airflow-core -sqlalchemy-utils==0.42.1 ; python_full_version < '3.10' - # via apache-airflow-core -sqlparse==0.5.4 ; python_full_version < '3.10' - # via - # apache-airflow-providers-common-sql - # dbt-core -sqlparse==0.5.5 ; python_full_version >= '3.10' - # via - # apache-airflow-providers-common-sql - # dbt-core -srsly==2.5.3 - # via - # confection - # spacy - # thinc - # weasel -stack-data==0.6.3 - # via ipython -starlette==0.49.3 ; python_full_version < '3.10' - # via - # cadwyn - # fastapi -starlette==1.3.1 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # cadwyn - # fastapi -structlog==25.5.0 ; python_full_version < '3.10' - # via - # apache-airflow-task-sdk - # dagster -structlog==26.1.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dagster -svcs==25.1.0 - # via apache-airflow-core -sympy==1.14.0 ; python_full_version < '3.11' - # via onnxruntime -tabulate==0.9.0 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dagster -tabulate==0.10.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # dagster -tenacity==9.1.2 ; python_full_version < '3.10' - # via - # apache-airflow-core - # retryhttp -tenacity==9.1.4 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -termcolor==3.1.0 ; python_full_version < '3.10' - # via - # apache-airflow-core - # cleanlab -termcolor==3.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # cleanlab -text-unidecode==1.3 - # via python-slugify -thinc==8.3.4 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # spacy -thinc==8.3.13 ; python_full_version >= '3.10' - # via spacy -threadpoolctl==3.6.0 - # via scikit-learn -tldextract==5.3.0 ; python_full_version < '3.10' - # via presidio-analyzer -tldextract==5.3.1 ; python_full_version >= '3.10' - # via presidio-analyzer -tokenizers==0.22.2 ; python_full_version < '3.10' - # via freshdata-cleaner -tokenizers==0.23.1 ; python_full_version >= '3.10' - # via freshdata-cleaner -tomli==2.4.1 - # via - # alembic - # build - # coverage - # dagster - # fastapi-cli - # maturin - # mypy - # pytest -tomlkit==0.15.0 - # via dagster-shared -toposort==1.10 - # via dagster -tqdm==4.68.4 - # via - # cleanlab - # dagster - # huggingface-hub - # spacy -traitlets==5.15.1 - # via - # ipython - # ipywidgets - # matplotlib-inline -twine==6.2.0 - # via freshdata-cleaner -typer==0.23.2 ; python_full_version < '3.10' - # via - # fastapi-cli - # fastapi-cloud-cli - # huggingface-hub - # spacy - # typer-slim -typer==0.26.8 ; python_full_version >= '3.10' - # via - # fastapi-cli - # spacy - # weasel -typer-slim==0.23.2 ; python_full_version < '3.10' - # via weasel -typing-extensions==4.16.0 - # via - # a2wsgi - # aiosqlite - # alembic - # anyio - # anywidget - # apache-airflow-core - # apache-airflow-task-sdk - # asgiref - # cadwyn - # cloudpathlib - # confection - # cron-descriptor - # cryptography - # dagster-shared - # dbt-adapters - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # exceptiongroup - # fastapi - # great-tables - # grpcio - # htmltools - # huggingface-hub - # ipython - # libcst - # mashumaro - # mkdocstrings-python - # mypy - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions - # pydantic - # pydantic-core - # pydantic-extra-types - # pyjwt - # referencing - # rich-toolkit - # snowplow-tracker - # sqlalchemy - # starlette - # structlog - # typing-inspection - # uvicorn - # virtualenv -typing-inspection==0.4.2 - # via - # cadwyn - # fastapi - # pydantic - # pydantic-settings -tzdata==2026.3 - # via - # agate - # arrow - # dagster - # pandas - # pendulum -uc-micro-py==1.0.3 ; python_full_version < '3.10' - # via linkify-it-py -uc-micro-py==2.0.0 ; python_full_version >= '3.10' - # via linkify-it-py -universal-pathlib==0.3.10 - # via - # apache-airflow-core - # dagster -urllib3==2.6.3 ; python_full_version < '3.10' - # via - # id - # requests - # sentry-sdk - # twine -urllib3==2.7.0 ; python_full_version >= '3.10' - # via - # id - # requests - # twine -uuid6==2025.0.1 - # via apache-airflow-core -uvicorn==0.39.0 ; python_full_version < '3.10' - # via - # fastapi - # fastapi-cli - # fastapi-cloud-cli -uvicorn==0.51.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # fastapi - # fastapi-cli -uvloop==0.22.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' - # via uvicorn -virtualenv==21.6.1 - # via pre-commit -wasabi==1.1.3 - # via - # spacy - # thinc - # weasel -watchdog==6.0.0 - # via - # dagster - # mkdocs -watchfiles==1.1.1 ; python_full_version < '3.10' - # via uvicorn -watchfiles==1.2.0 ; python_full_version >= '3.10' - # via uvicorn -wcwidth==0.8.2 - # via prompt-toolkit -weasel==0.4.3 ; python_full_version < '3.10' - # via spacy -weasel==1.0.0 ; python_full_version >= '3.10' - # via spacy -websockets==15.0.1 ; python_full_version < '3.10' - # via uvicorn -websockets==16.1 ; python_full_version >= '3.10' - # via - # nokap - # uvicorn -werkzeug==3.1.8 ; python_full_version < '3.10' - # via flask -widgetsnbextension==4.0.15 - # via ipywidgets -wirerope==1.0.0 - # via methodtools -wrapt==2.2.2 - # via - # deprecated - # smart-open -zipp==3.23.1 ; python_full_version < '3.10' - # via - # importlib-metadata - # importlib-resources -zipp==4.1.0 ; python_full_version >= '3.10' - # via importlib-metadata diff --git a/crates/freshcore/Cargo.lock b/crates/freshcore/Cargo.lock deleted file mode 100644 index c970f864..00000000 --- a/crates/freshcore/Cargo.lock +++ /dev/null @@ -1,255 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - -[[package]] -name = "freshcore" -version = "0.1.0" -dependencies = [ - "pretty_assertions", - "pyo3", - "rayon", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "indoc" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" -dependencies = [ - "rustversion", -] - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" -dependencies = [ - "cfg-if", - "indoc", - "libc", - "memoffset", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" -dependencies = [ - "once_cell", - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" -dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "syn" -version = "2.0.118" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "target-lexicon" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" diff --git a/crates/freshcore/Cargo.toml b/crates/freshcore/Cargo.toml deleted file mode 100644 index 503e21ec..00000000 --- a/crates/freshcore/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "freshcore" -version = "0.1.0" -edition = "2021" -license = "MIT" -description = "FreshData native cleaning-first execution engine" - -[lib] -name = "freshdata_freshcore" -crate-type = ["cdylib", "rlib"] - -[features] -extension-module = ["pyo3/extension-module"] - -[dependencies] -pyo3 = { version = "0.22" } -rayon = "1.10" - -[dev-dependencies] -pretty_assertions = "1.4" diff --git a/crates/freshcore/src/arrays.rs b/crates/freshcore/src/arrays.rs deleted file mode 100644 index 01d98bc0..00000000 --- a/crates/freshcore/src/arrays.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::hash::{Hash, Hasher}; - -#[derive(Clone, Debug, PartialEq)] -pub enum ColumnData { - Float(Vec>), - Bool(Vec>), - Utf8(Vec>), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Column { - pub name: String, - pub data: ColumnData, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Frame { - pub columns: Vec, - pub nrows: usize, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum CellKey { - Null, - Float(u64), - Bool(bool), - Utf8(String), -} - -impl Hash for CellKey { - fn hash(&self, state: &mut H) { - match self { - CellKey::Null => 0u8.hash(state), - CellKey::Float(v) => { - 1u8.hash(state); - v.hash(state); - } - CellKey::Bool(v) => { - 2u8.hash(state); - v.hash(state); - } - CellKey::Utf8(v) => { - 3u8.hash(state); - v.hash(state); - } - } - } -} - -impl ColumnData { - pub fn len(&self) -> usize { - match self { - ColumnData::Float(v) => v.len(), - ColumnData::Bool(v) => v.len(), - ColumnData::Utf8(v) => v.len(), - } - } - - pub fn null_count(&self) -> usize { - match self { - ColumnData::Float(v) => v.iter().filter(|x| x.is_none()).count(), - ColumnData::Bool(v) => v.iter().filter(|x| x.is_none()).count(), - ColumnData::Utf8(v) => v.iter().filter(|x| x.is_none()).count(), - } - } - - pub fn is_all_null(&self) -> bool { - self.null_count() == self.len() - } - - pub fn cell_key(&self, row: usize) -> CellKey { - match self { - ColumnData::Float(v) => v[row] - .map(|x| { - if x == 0.0 { - CellKey::Float(0.0f64.to_bits()) - } else { - CellKey::Float(x.to_bits()) - } - }) - .unwrap_or(CellKey::Null), - ColumnData::Bool(v) => v[row].map(CellKey::Bool).unwrap_or(CellKey::Null), - ColumnData::Utf8(v) => v[row] - .as_ref() - .map(|x| CellKey::Utf8(x.clone())) - .unwrap_or(CellKey::Null), - } - } - - pub fn take(&self, keep: &[bool]) -> ColumnData { - match self { - ColumnData::Float(v) => ColumnData::Float( - v.iter() - .zip(keep) - .filter_map(|(value, keep)| keep.then_some(*value)) - .collect(), - ), - ColumnData::Bool(v) => ColumnData::Bool( - v.iter() - .zip(keep) - .filter_map(|(value, keep)| keep.then_some(*value)) - .collect(), - ), - ColumnData::Utf8(v) => ColumnData::Utf8( - v.iter() - .zip(keep) - .filter_map(|(value, keep)| keep.then_some(value.clone())) - .collect(), - ), - } - } -} - -impl Frame { - pub fn new(columns: Vec) -> Result { - let nrows = columns.first().map(|c| c.data.len()).unwrap_or(0); - if columns.iter().any(|c| c.data.len() != nrows) { - return Err("all columns must have the same length".to_string()); - } - Ok(Self { columns, nrows }) - } - - pub fn null_cells(&self) -> usize { - self.columns.iter().map(|c| c.data.null_count()).sum() - } - - pub fn take_rows(&mut self, keep: &[bool]) { - for column in &mut self.columns { - column.data = column.data.take(keep); - } - self.nrows = keep.iter().filter(|x| **x).count(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn null_counts_are_separate_from_values() { - let data = ColumnData::Float(vec![Some(1.0), None, Some(2.0)]); - assert_eq!(data.len(), 3); - assert_eq!(data.null_count(), 1); - assert!(!data.is_all_null()); - } - - #[test] - fn row_take_preserves_column_lengths() { - let mut frame = Frame::new(vec![Column { - name: "a".into(), - data: ColumnData::Utf8(vec![Some("x".into()), None, Some("y".into())]), - }]) - .unwrap(); - frame.take_rows(&[true, false, true]); - assert_eq!(frame.nrows, 2); - assert_eq!(frame.columns[0].data.len(), 2); - } -} diff --git a/crates/freshcore/src/kernels/casts.rs b/crates/freshcore/src/kernels/casts.rs deleted file mode 100644 index 7855ebba..00000000 --- a/crates/freshcore/src/kernels/casts.rs +++ /dev/null @@ -1,143 +0,0 @@ -use rayon::prelude::*; - -use crate::arrays::{Column, ColumnData, Frame}; -use crate::plan::CleanPlan; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CastAction { - pub column: String, - pub target: String, - pub count: usize, - pub coerced: usize, -} - -const TRUE_WORDS: &[&str] = &["true", "t", "yes", "y"]; -const FALSE_WORDS: &[&str] = &["false", "f", "no", "n"]; - -pub fn infer_and_cast(frame: &mut Frame, plan: &CleanPlan) -> Vec { - if !plan.fix_dtypes { - return Vec::new(); - } - frame - .columns - .par_iter_mut() - .filter_map(|column| cast_column(column, plan)) - .collect() -} - -fn cast_column(column: &mut Column, plan: &CleanPlan) -> Option { - let ColumnData::Utf8(values) = &column.data else { - return None; - }; - let non_null: Vec<&str> = values.iter().filter_map(|v| v.as_deref()).collect(); - if non_null.is_empty() { - return None; - } - if let Some(bools) = try_bool(values) { - let count = bools.iter().filter(|v| v.is_some()).count(); - column.data = ColumnData::Bool(bools); - return Some(CastAction { - column: column.name.clone(), - target: "boolean".into(), - count, - coerced: 0, - }); - } - if plan.preserve_leading_zeros && non_null.iter().any(|v| has_leading_zero(v)) { - return None; - } - let (parsed, parsed_count, coerced) = parse_numeric(values); - if (parsed_count as f64) / (non_null.len() as f64) >= plan.numeric_threshold { - column.data = ColumnData::Float(parsed); - return Some(CastAction { - column: column.name.clone(), - target: "float64".into(), - count: parsed_count + coerced, - coerced, - }); - } - None -} - -fn try_bool(values: &[Option]) -> Option>> { - let mut out = Vec::with_capacity(values.len()); - for value in values { - let Some(raw) = value else { - out.push(None); - continue; - }; - let token = raw.trim().to_lowercase(); - if TRUE_WORDS.contains(&token.as_str()) { - out.push(Some(true)); - } else if FALSE_WORDS.contains(&token.as_str()) { - out.push(Some(false)); - } else { - return None; - } - } - Some(out) -} - -fn parse_numeric(values: &[Option]) -> (Vec>, usize, usize) { - let mut out = Vec::with_capacity(values.len()); - let mut parsed = 0usize; - let mut coerced = 0usize; - for value in values { - let Some(raw) = value else { - out.push(None); - continue; - }; - let cleaned = raw - .trim() - .trim_matches('$') - .trim_matches('€') - .trim_matches('Β£') - .trim_matches('β‚Ή') - .replace(',', ""); - match cleaned.parse::() { - Ok(v) => { - out.push(Some(v)); - parsed += 1; - } - Err(_) => { - out.push(None); - coerced += 1; - } - } - } - (out, parsed, coerced) -} - -fn has_leading_zero(value: &str) -> bool { - let trimmed = value.trim_start_matches(['+', '-']).trim(); - trimmed.len() > 1 - && trimmed.starts_with('0') - && trimmed.chars().nth(1).is_some_and(|c| c.is_ascii_digit()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - #[test] - fn numeric_cast_preserves_zero_padded_ids() { - let mut frame = Frame::new(vec![Column { - name: "zip".into(), - data: ColumnData::Utf8(vec![Some("02115".into()), Some("10001".into())]), - }]) - .unwrap(); - assert!(infer_and_cast(&mut frame, &CleanPlan::default()).is_empty()); - } - - #[test] - fn bool_cast_uses_small_vocabulary() { - let mut frame = Frame::new(vec![Column { - name: "flag".into(), - data: ColumnData::Utf8(vec![Some("yes".into()), Some("no".into())]), - }]) - .unwrap(); - let actions = infer_and_cast(&mut frame, &CleanPlan::default()); - assert_eq!(actions[0].target, "boolean"); - } -} diff --git a/crates/freshcore/src/kernels/duplicates.rs b/crates/freshcore/src/kernels/duplicates.rs deleted file mode 100644 index 8ad93640..00000000 --- a/crates/freshcore/src/kernels/duplicates.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::collections::HashSet; - -use crate::arrays::{CellKey, Frame}; - -/// Marks full-row duplicates under `keep_policy` (`"last"` keeps the final -/// occurrence, anything else the first). -/// -/// Like pandas' `drop_duplicate_rows`, a frame with no rows or no columns has -/// no duplicates. -pub fn duplicated_mask(frame: &Frame, keep_policy: &str) -> Vec { - let mut duplicated = vec![false; frame.nrows]; - if frame.nrows == 0 || frame.columns.is_empty() { - return duplicated; - } - let mut seen: HashSet> = HashSet::new(); - let mut mark = |row: usize| { - if !seen.insert(row_key(frame, row)) { - duplicated[row] = true; - } - }; - if keep_policy == "last" { - (0..frame.nrows).rev().for_each(&mut mark); - } else { - (0..frame.nrows).for_each(&mut mark); - } - duplicated -} - -/// Number of full-row duplicates, without removing any rows. -/// -/// The count does not depend on the keep policy: every row beyond the first -/// occurrence of its key is a duplicate. -pub fn count_duplicates(frame: &Frame) -> usize { - duplicated_mask(frame, "first") - .iter() - .filter(|v| **v) - .count() -} - -pub fn drop_duplicates(frame: &mut Frame, keep_policy: &str) -> usize { - let duplicated = duplicated_mask(frame, keep_policy); - let dropped = duplicated.iter().filter(|v| **v).count(); - if dropped > 0 { - let keep: Vec = duplicated.iter().map(|v| !*v).collect(); - frame.take_rows(&keep); - } - dropped -} - -fn row_key(frame: &Frame, row: usize) -> Vec { - frame - .columns - .iter() - .map(|column| column.data.cell_key(row)) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - fn mixed_frame() -> Frame { - Frame::new(vec![ - Column { - name: "a".into(), - data: ColumnData::Float(vec![ - Some(1.0), - Some(1.0), - Some(0.0), - Some(-0.0), - None, - None, - Some(2.0), - ]), - }, - Column { - name: "b".into(), - data: ColumnData::Utf8(vec![ - Some("x".into()), - Some("x".into()), - Some("y".into()), - Some("y".into()), - None, - None, - Some("x".into()), - ]), - }, - ]) - .unwrap() - } - - #[test] - fn removes_duplicate_rows() { - let mut frame = Frame::new(vec![Column { - name: "x".into(), - data: ColumnData::Utf8(vec![Some("a".into()), Some("a".into()), Some("b".into())]), - }]) - .unwrap(); - assert_eq!(drop_duplicates(&mut frame, "first"), 1); - assert_eq!(frame.nrows, 2); - } - - #[test] - fn counts_duplicates_without_removing_rows() { - let frame = mixed_frame(); - // (1.0, x) twice, (0.0, y) == (-0.0, y), and (null, null) twice. - assert_eq!(count_duplicates(&frame), 3); - assert_eq!(frame.nrows, 7); - assert_eq!(frame, mixed_frame()); - } - - #[test] - fn count_matches_rows_dropped_for_every_keep_policy() { - for keep in ["first", "last"] { - let mut frame = mixed_frame(); - let counted = count_duplicates(&frame); - assert_eq!(drop_duplicates(&mut frame, keep), counted); - assert_eq!(count_duplicates(&frame), 0); - } - } - - #[test] - fn mask_marks_the_occurrences_the_keep_policy_drops() { - let frame = mixed_frame(); - assert_eq!( - duplicated_mask(&frame, "first"), - vec![false, true, false, true, false, true, false] - ); - assert_eq!( - duplicated_mask(&frame, "last"), - vec![true, false, true, false, true, false, false] - ); - } - - #[test] - fn frames_without_rows_or_columns_have_no_duplicates() { - let no_rows = Frame::new(vec![Column { - name: "x".into(), - data: ColumnData::Float(Vec::new()), - }]) - .unwrap(); - assert_eq!(count_duplicates(&no_rows), 0); - - let mut no_columns = Frame { - columns: Vec::new(), - nrows: 3, - }; - assert_eq!(count_duplicates(&no_columns), 0); - assert_eq!(drop_duplicates(&mut no_columns, "first"), 0); - assert_eq!(no_columns.nrows, 3); - } -} diff --git a/crates/freshcore/src/kernels/missing.rs b/crates/freshcore/src/kernels/missing.rs deleted file mode 100644 index f8c86000..00000000 --- a/crates/freshcore/src/kernels/missing.rs +++ /dev/null @@ -1,169 +0,0 @@ -use std::collections::HashMap; - -use crate::arrays::{ColumnData, Frame}; - -#[derive(Clone, Debug, PartialEq)] -pub struct ImputeAction { - pub column: String, - pub strategy: String, - pub count: usize, - pub value: String, -} - -pub fn drop_empty_columns(frame: &mut Frame) -> Vec { - let mut dropped = Vec::new(); - frame.columns.retain(|column| { - let keep = !column.data.is_all_null(); - if !keep { - dropped.push(column.name.clone()); - } - keep - }); - dropped -} - -pub fn drop_empty_rows(frame: &mut Frame) -> usize { - if frame.nrows == 0 { - return 0; - } - let mut keep = vec![false; frame.nrows]; - for row in 0..frame.nrows { - keep[row] = frame - .columns - .iter() - .any(|column| !matches!(column.data.cell_key(row), crate::arrays::CellKey::Null)); - } - let dropped = keep.iter().filter(|v| !**v).count(); - if dropped > 0 { - frame.take_rows(&keep); - } - dropped -} - -pub fn impute(frame: &mut Frame, strategy: Option<&str>) -> Vec { - let Some(strategy) = strategy else { - return Vec::new(); - }; - let mut actions = Vec::new(); - for column in &mut frame.columns { - match &mut column.data { - ColumnData::Float(values) => { - let missing = values.iter().filter(|v| v.is_none()).count(); - if missing == 0 || missing == values.len() { - continue; - } - let resolved = if strategy == "auto" { "median" } else { strategy }; - let fill = match resolved { - "mean" => mean(values), - "median" => median(values), - "mode" => mode_float(values), - _ => None, - }; - if let Some(fill) = fill { - for value in values.iter_mut().filter(|v| v.is_none()) { - *value = Some(fill); - } - actions.push(ImputeAction { - column: column.name.clone(), - strategy: resolved.to_string(), - count: missing, - value: format!("{fill:.6}"), - }); - } - } - ColumnData::Utf8(values) => { - let missing = values.iter().filter(|v| v.is_none()).count(); - if missing == 0 || missing == values.len() { - continue; - } - let resolved = if strategy == "auto" { "mode" } else { strategy }; - if resolved != "mode" { - continue; - } - if let Some(fill) = mode_string(values) { - for value in values.iter_mut().filter(|v| v.is_none()) { - *value = Some(fill.clone()); - } - actions.push(ImputeAction { - column: column.name.clone(), - strategy: "mode".to_string(), - count: missing, - value: format!("{fill:?}"), - }); - } - } - ColumnData::Bool(_) => {} - } - } - actions -} - -fn mean(values: &[Option]) -> Option { - let mut sum = 0.0; - let mut n = 0usize; - for v in values.iter().flatten() { - sum += *v; - n += 1; - } - (n > 0).then_some(sum / n as f64) -} - -fn median(values: &[Option]) -> Option { - let mut vals: Vec = values.iter().flatten().copied().filter(|v| !v.is_nan()).collect(); - if vals.is_empty() { - return None; - } - vals.sort_by(|a, b| a.total_cmp(b)); - let mid = vals.len() / 2; - if vals.len() % 2 == 0 { - Some((vals[mid - 1] + vals[mid]) / 2.0) - } else { - Some(vals[mid]) - } -} - -fn mode_float(values: &[Option]) -> Option { - let mut counts: HashMap = HashMap::new(); - for v in values.iter().flatten().filter(|v| !v.is_nan()) { - counts - .entry(v.to_bits()) - .and_modify(|(_, n)| *n += 1) - .or_insert((*v, 1)); - } - counts - .values() - .max_by(|(av, an), (bv, bn)| an.cmp(bn).then_with(|| bv.total_cmp(av))) - .map(|(v, _)| *v) -} - -fn mode_string(values: &[Option]) -> Option { - let mut counts: HashMap<&str, usize> = HashMap::new(); - for value in values.iter().flatten() { - *counts.entry(value.as_str()).or_insert(0) += 1; - } - counts - .into_iter() - .max_by(|(av, an), (bv, bn)| an.cmp(bn).then_with(|| bv.cmp(av))) - .map(|(v, _)| v.to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - #[test] - fn median_impute_fills_missing_numeric() { - let mut frame = Frame::new(vec![Column { - name: "x".into(), - data: ColumnData::Float(vec![Some(1.0), None, Some(3.0)]), - }]) - .unwrap(); - let actions = impute(&mut frame, Some("median")); - assert_eq!(actions[0].count, 1); - assert_eq!( - frame.columns[0].data, - ColumnData::Float(vec![Some(1.0), Some(2.0), Some(3.0)]) - ); - } -} diff --git a/crates/freshcore/src/kernels/mod.rs b/crates/freshcore/src/kernels/mod.rs deleted file mode 100644 index 9b4bdc7a..00000000 --- a/crates/freshcore/src/kernels/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod casts; -pub mod duplicates; -pub mod missing; -pub mod outliers; -pub mod profile; -pub mod strings; diff --git a/crates/freshcore/src/kernels/outliers.rs b/crates/freshcore/src/kernels/outliers.rs deleted file mode 100644 index 8c0a09a2..00000000 --- a/crates/freshcore/src/kernels/outliers.rs +++ /dev/null @@ -1,213 +0,0 @@ -use crate::arrays::{Column, ColumnData, Frame}; - -#[derive(Clone, Debug, PartialEq)] -pub struct OutlierAction { - pub column: String, - pub action: String, - pub count: usize, - pub lower: f64, - pub upper: f64, - pub flag_column: Option, -} - -pub fn handle_outliers( - frame: &mut Frame, - action: Option<&str>, - method: &str, - factor: f64, -) -> Vec { - let Some(action) = action else { - return Vec::new(); - }; - let mut actions = Vec::new(); - let existing: Vec = frame.columns.iter().map(|c| c.name.clone()).collect(); - let mut flags = Vec::new(); - - for column in &mut frame.columns { - let ColumnData::Float(values) = &mut column.data else { - continue; - }; - let Some((lo, hi)) = bounds(values, method, factor) else { - continue; - }; - let mask: Vec = values - .iter() - .map(|v| v.is_some_and(|x| x < lo || x > hi)) - .collect(); - let count = mask.iter().filter(|v| **v).count(); - if count == 0 { - continue; - } - if action == "clip" { - for value in values.iter_mut().flatten() { - if *value < lo { - *value = lo; - } else if *value > hi { - *value = hi; - } - } - actions.push(OutlierAction { - column: column.name.clone(), - action: "clip".into(), - count, - lower: lo, - upper: hi, - flag_column: None, - }); - } else { - let flag = unique_flag(&existing, &format!("{}_outlier", column.name)); - flags.push(Column { - name: flag.clone(), - data: ColumnData::Bool(mask.into_iter().map(Some).collect()), - }); - actions.push(OutlierAction { - column: column.name.clone(), - action: "flag".into(), - count, - lower: lo, - upper: hi, - flag_column: Some(flag), - }); - } - } - if !flags.is_empty() { - frame.columns.extend(flags); - } - actions -} - -pub fn bounds(values: &[Option], method: &str, factor: f64) -> Option<(f64, f64)> { - let vals: Vec = values.iter().flatten().copied().filter(|v| !v.is_nan()).collect(); - if vals.len() < 2 { - return None; - } - if method == "zscore" { - let mean = vals.iter().sum::() / vals.len() as f64; - let var = vals.iter().map(|v| (v - mean).powi(2)).sum::() / (vals.len() - 1) as f64; - let std = var.sqrt(); - if std == 0.0 || std.is_nan() { - return None; - } - return Some((mean - factor * std, mean + factor * std)); - } - let q1 = quantile(vals.clone(), 0.25)?; - let q3 = quantile(vals.clone(), 0.75)?; - let spread = q3 - q1; - if spread.is_nan() { - return None; - } - if spread == 0.0 { - return zero_iqr_bounds(&vals, q1, factor); - } - Some((q1 - factor * spread, q3 + factor * spread)) -} - -/// IQR-equivalent spread per unit of mean absolute deviation for normal data: -/// sqrt(pi/2) (MeanAD -> sigma) x 2*Phi^-1(0.75) (sigma -> IQR). Mirrors -/// `freshdata.steps.outliers.MEANAD_TO_IQR`. -const MEANAD_TO_IQR: f64 = 1.6906950787902986; -/// Largest share of values the zero-IQR fallback may flag; mirrors -/// `freshdata.steps.outliers.ZERO_IQR_MAX_SHARE`. -const ZERO_IQR_MAX_SHARE: f64 = 0.05; - -/// Fences when both quartiles equal `center` (at least half the column sits on -/// one value). A zero IQR must not silently disable detection. Instead, the IQR -/// is replaced by the normal-consistent spread from the mean absolute deviation -/// around the median (the median equals the quartiles here). The result is -/// `None` for a constant column, or when the fences would flag more than -/// `ZERO_IQR_MAX_SHARE` of the values: that many is a second mode, not rare -/// outliers. -fn zero_iqr_bounds(vals: &[f64], center: f64, factor: f64) -> Option<(f64, f64)> { - let n = vals.len() as f64; - let mean_ad = vals.iter().map(|v| (v - center).abs()).sum::() / n; - if mean_ad == 0.0 || mean_ad.is_nan() { - return None; - } - let spread = MEANAD_TO_IQR * mean_ad; - let (lo, hi) = (center - factor * spread, center + factor * spread); - let flagged = vals.iter().filter(|v| **v < lo || **v > hi).count() as f64; - if flagged > ZERO_IQR_MAX_SHARE * n { - return None; - } - Some((lo, hi)) -} - -fn quantile(mut vals: Vec, q: f64) -> Option { - if vals.is_empty() { - return None; - } - vals.sort_by(|a, b| a.total_cmp(b)); - let pos = q * (vals.len() - 1) as f64; - let lo = pos.floor() as usize; - let hi = pos.ceil() as usize; - if lo == hi { - Some(vals[lo]) - } else { - let weight = pos - lo as f64; - Some(vals[lo] * (1.0 - weight) + vals[hi] * weight) - } -} - -fn unique_flag(existing: &[String], base: &str) -> String { - let mut name = base.to_string(); - let mut i = 1usize; - while existing.contains(&name) { - i += 1; - name = format!("{base}_{i}"); - } - name -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn iqr_bounds_detect_extreme_value() { - let values = vec![Some(1.0), Some(2.0), Some(3.0), Some(100.0)]; - let (lo, hi) = bounds(&values, "iqr", 1.5).unwrap(); - assert!(lo < 1.0); - assert!(hi < 100.0); - } - - fn zeros_with(extra: &[f64], zeros: usize) -> Vec> { - std::iter::repeat(0.0) - .take(zeros) - .chain(extra.iter().copied()) - .map(Some) - .collect() - } - - #[test] - fn zero_iqr_falls_back_to_mean_absolute_deviation() { - let values = zeros_with(&[1000.0, 5000.0, 2.0, 3.0, -800.0], 95); - let (lo, hi) = bounds(&values, "iqr", 1.5).unwrap(); - // MeanAD = 6805 / 100; fences = 0 +/- 1.5 x 1.6907 x MeanAD. - let expected = 1.5 * (MEANAD_TO_IQR * (6805.0 / 100.0)); - assert_eq!((lo, hi), (-expected, expected)); - let mut frame = Frame { - nrows: values.len(), - columns: vec![Column { - name: "x".into(), - data: ColumnData::Float(values), - }], - }; - let actions = handle_outliers(&mut frame, Some("clip"), "iqr", 1.5); - assert_eq!(actions.len(), 1); - assert_eq!(actions[0].count, 3); // the spikes, not 2.0 / 3.0 - } - - #[test] - fn zero_iqr_fallback_does_not_flag_a_second_mode() { - // 30% non-zero, split around zero: the IQR is still zero. - let extra: Vec = (1..=15).flat_map(|i| [-(i as f64), i as f64]).collect(); - assert_eq!(bounds(&zeros_with(&extra, 70), "iqr", 1.5), None); - } - - #[test] - fn constant_column_has_no_bounds() { - let values = vec![Some(5.0); 20]; - assert_eq!(bounds(&values, "iqr", 1.5), None); - assert_eq!(bounds(&values, "zscore", 3.0), None); - } -} diff --git a/crates/freshcore/src/kernels/profile.rs b/crates/freshcore/src/kernels/profile.rs deleted file mode 100644 index 1263665e..00000000 --- a/crates/freshcore/src/kernels/profile.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::collections::HashSet; - -use crate::arrays::{CellKey, ColumnData, Frame}; - -#[derive(Clone, Debug, PartialEq)] -pub struct ColumnProfile { - pub name: String, - pub null_count: usize, - pub unique_count: usize, - pub min: Option, - pub max: Option, - pub mean: Option, -} - -pub fn profile(frame: &Frame) -> Vec { - frame - .columns - .iter() - .map(|column| { - let mut unique = HashSet::new(); - for row in 0..frame.nrows { - unique.insert(column.data.cell_key(row)); - } - let (min, max, mean) = numeric_stats(&column.data); - ColumnProfile { - name: column.name.clone(), - null_count: column.data.null_count(), - unique_count: unique - .into_iter() - .filter(|k| !matches!(k, CellKey::Null)) - .count(), - min, - max, - mean, - } - }) - .collect() -} - -fn numeric_stats(data: &ColumnData) -> (Option, Option, Option) { - let ColumnData::Float(values) = data else { - return (None, None, None); - }; - let vals: Vec = values.iter().flatten().copied().filter(|v| !v.is_nan()).collect(); - if vals.is_empty() { - return (None, None, None); - } - let min = vals.iter().copied().fold(f64::INFINITY, f64::min); - let max = vals.iter().copied().fold(f64::NEG_INFINITY, f64::max); - let mean = vals.iter().sum::() / vals.len() as f64; - (Some(min), Some(max), Some(mean)) -} diff --git a/crates/freshcore/src/kernels/strings.rs b/crates/freshcore/src/kernels/strings.rs deleted file mode 100644 index 20661642..00000000 --- a/crates/freshcore/src/kernels/strings.rs +++ /dev/null @@ -1,88 +0,0 @@ -use rayon::prelude::*; - -use crate::arrays::{ColumnData, Frame}; -use crate::plan::{CleanPlan, StringCase}; - -#[derive(Clone, Debug, Default, PartialEq)] -pub struct StringAction { - pub column: String, - pub trimmed: usize, - pub sentinels: usize, - pub case_normalized: usize, -} - -pub fn clean_strings(frame: &mut Frame, plan: &CleanPlan) -> Vec { - frame - .columns - .par_iter_mut() - .filter_map(|column| match &mut column.data { - ColumnData::Utf8(values) => { - let mut action = StringAction { - column: column.name.clone(), - ..StringAction::default() - }; - for slot in values { - let Some(value) = slot.as_mut() else { - continue; - }; - if plan.strip_whitespace { - let stripped = value.trim(); - if stripped.len() != value.len() { - *value = stripped.to_string(); - action.trimmed += 1; - } - } - if plan.normalize_sentinels - && plan.sentinels.contains(&value.trim().to_lowercase()) - { - *slot = None; - action.sentinels += 1; - continue; - } - match plan.string_case { - StringCase::Preserve => {} - StringCase::Lower => { - let lowered = value.to_lowercase(); - if lowered != *value { - *value = lowered; - action.case_normalized += 1; - } - } - StringCase::Upper => { - let uppered = value.to_uppercase(); - if uppered != *value { - *value = uppered; - action.case_normalized += 1; - } - } - } - } - (action.trimmed > 0 || action.sentinels > 0 || action.case_normalized > 0) - .then_some(action) - } - _ => None, - }) - .collect() -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - #[test] - fn trims_and_nulls_sentinels() { - let mut frame = Frame::new(vec![Column { - name: "name".into(), - data: ColumnData::Utf8(vec![Some(" x ".into()), Some("N/A".into())]), - }]) - .unwrap(); - let mut plan = CleanPlan::default(); - plan.sentinels = HashSet::from(["n/a".to_string()]); - let actions = clean_strings(&mut frame, &plan); - assert_eq!(actions[0].trimmed, 1); - assert_eq!(actions[0].sentinels, 1); - } -} diff --git a/crates/freshcore/src/lib.rs b/crates/freshcore/src/lib.rs deleted file mode 100644 index d3f046ff..00000000 --- a/crates/freshcore/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod arrays; -mod kernels; -mod plan; -mod python; - -pub use arrays::{Column, ColumnData, Frame}; -pub use plan::{CleanPlan, StringCase}; - -use pyo3::prelude::*; - -#[pymodule] -fn freshdata_freshcore(m: &Bound<'_, PyModule>) -> PyResult<()> { - python::register(m) -} diff --git a/crates/freshcore/src/plan.rs b/crates/freshcore/src/plan.rs deleted file mode 100644 index 837d3871..00000000 --- a/crates/freshcore/src/plan.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::collections::HashSet; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StringCase { - Preserve, - Lower, - Upper, -} - -#[derive(Clone, Debug)] -pub struct CleanPlan { - pub rename_map: Vec<(String, String)>, - pub strip_whitespace: bool, - pub normalize_sentinels: bool, - pub sentinels: HashSet, - pub string_case: StringCase, - pub drop_empty_columns: bool, - pub drop_empty_rows: bool, - pub drop_duplicates: bool, - pub duplicate_keep: String, - pub fix_dtypes: bool, - pub numeric_threshold: f64, - pub preserve_leading_zeros: bool, - pub impute: Option, - pub outliers: Option, - pub outlier_method: String, - pub outlier_factor: f64, -} - -impl Default for CleanPlan { - fn default() -> Self { - Self { - rename_map: Vec::new(), - strip_whitespace: true, - normalize_sentinels: true, - sentinels: HashSet::new(), - string_case: StringCase::Preserve, - drop_empty_columns: true, - drop_empty_rows: true, - drop_duplicates: true, - duplicate_keep: "first".to_string(), - fix_dtypes: true, - numeric_threshold: 0.95, - preserve_leading_zeros: true, - impute: None, - outliers: None, - outlier_method: "iqr".to_string(), - outlier_factor: 1.5, - } - } -} diff --git a/crates/freshcore/src/python.rs b/crates/freshcore/src/python.rs deleted file mode 100644 index 92b1b3ab..00000000 --- a/crates/freshcore/src/python.rs +++ /dev/null @@ -1,392 +0,0 @@ -use std::collections::HashSet; -use std::time::Instant; - -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyList}; - -use crate::arrays::{Column, ColumnData, Frame}; -use crate::kernels::{casts, duplicates, missing, outliers, profile, strings}; -use crate::plan::{CleanPlan, StringCase}; - -pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(execute_plan, m)?)?; - Ok(()) -} - -#[pyfunction] -fn execute_plan(py: Python<'_>, payload: &Bound<'_, PyDict>) -> PyResult { - let mut timings = Vec::new(); - let mut frame = frame_from_payload(payload)?; - let plan = plan_from_payload(payload)?; - let rows_before = frame.nrows; - let cols_before = frame.columns.len(); - let missing_before = frame.null_cells(); - let mut actions = Vec::new(); - let mut columns_dropped = Vec::new(); - let mut columns_imputed = Vec::new(); - let mut duplicates_removed = 0usize; - let mut duplicates_detected: Option = None; - let mut outliers_handled = 0usize; - - let t = Instant::now(); - apply_renames(&mut frame, &plan, &mut actions); - timings.push(("column_names".to_string(), t.elapsed().as_secs_f64())); - - let t = Instant::now(); - for action in strings::clean_strings(&mut frame, &plan) { - if action.trimmed > 0 { - actions.push(action_dict( - "strip_whitespace", - Some(action.column.clone()), - "trimmed surrounding whitespace", - action.trimmed, - )); - } - if action.case_normalized > 0 { - actions.push(action_dict( - "normalize_case", - Some(action.column.clone()), - "normalized string case", - action.case_normalized, - )); - } - if action.sentinels > 0 { - actions.push(action_dict( - "normalize_sentinels", - Some(action.column), - "replaced sentinel strings (\"N/A\", \"-\", \"\", …) with missing", - action.sentinels, - )); - } - } - timings.push(("clean_strings".to_string(), t.elapsed().as_secs_f64())); - - if plan.drop_empty_columns { - let t = Instant::now(); - let dropped = missing::drop_empty_columns(&mut frame); - if !dropped.is_empty() { - columns_dropped.extend(dropped.clone()); - actions.push(action_dict( - "drop_empty_columns", - None, - &format!("dropped {} all-missing column(s): {}", dropped.len(), dropped.join(", ")), - dropped.len(), - )); - } - timings.push(("drop_empty_columns".to_string(), t.elapsed().as_secs_f64())); - } - - if plan.drop_empty_rows { - let t = Instant::now(); - let dropped = missing::drop_empty_rows(&mut frame); - if dropped > 0 { - actions.push(action_dict( - "drop_empty_rows", - None, - &format!("dropped {dropped} all-missing row(s)"), - dropped, - )); - } - timings.push(("drop_empty_rows".to_string(), t.elapsed().as_secs_f64())); - } - - let t = Instant::now(); - for action in casts::infer_and_cast(&mut frame, &plan) { - let suffix = if action.coerced > 0 { - format!(" ({} unparseable value(s) set to missing)", action.coerced) - } else { - String::new() - }; - actions.push(action_dict( - "fix_dtypes", - Some(action.column), - &format!("converted to {}{}", action.target, suffix), - action.count, - )); - } - timings.push(("fix_dtypes".to_string(), t.elapsed().as_secs_f64())); - - if plan.drop_duplicates { - let t = Instant::now(); - let before = frame.nrows; - let dropped = duplicates::drop_duplicates(&mut frame, &plan.duplicate_keep); - duplicates_removed = dropped; - if dropped > 0 { - let pct = 100.0 * dropped as f64 / before as f64; - actions.push(action_dict( - "drop_duplicates", - None, - &format!( - "dropped {dropped} duplicate row(s) ({pct:.1}% of rows, keep={:?})", - plan.duplicate_keep - ), - dropped, - )); - } - timings.push(("drop_duplicates".to_string(), t.elapsed().as_secs_f64())); - } else { - // Detection only: count at the same stage as the pandas step (after - // string cleaning, empty-row removal and casts; before imputation and - // outliers) so the adapter can report and escalate like pandas. - let t = Instant::now(); - duplicates_detected = Some(duplicates::count_duplicates(&frame)); - timings.push(("detect_duplicates".to_string(), t.elapsed().as_secs_f64())); - } - - let t = Instant::now(); - for action in missing::impute(&mut frame, plan.impute.as_deref()) { - columns_imputed.push(action.column.clone()); - actions.push(action_dict( - "impute", - Some(action.column), - &format!( - "filled {} missing value(s) with {} ({})", - action.count, action.strategy, action.value - ), - action.count, - )); - } - timings.push(("impute".to_string(), t.elapsed().as_secs_f64())); - - let t = Instant::now(); - for action in outliers::handle_outliers( - &mut frame, - plan.outliers.as_deref(), - &plan.outlier_method, - plan.outlier_factor, - ) { - outliers_handled += action.count; - let description = if action.action == "clip" { - format!( - "clipped {} outlier(s) to [{:.6}, {:.6}] ({}, factor {:.6})", - action.count, action.lower, action.upper, plan.outlier_method, plan.outlier_factor - ) - } else { - format!( - "flagged {} outlier(s) in new column {:?} ({}, factor {:.6})", - action.count, - action.flag_column.unwrap_or_default(), - plan.outlier_method, - plan.outlier_factor - ) - }; - actions.push(action_dict( - "outliers", - Some(action.column), - &description, - action.count, - )); - } - timings.push(("outliers".to_string(), t.elapsed().as_secs_f64())); - - let profiles = profile::profile(&frame); - let result = PyDict::new_bound(py); - result.set_item("rows_before", rows_before)?; - result.set_item("rows_after", frame.nrows)?; - result.set_item("cols_before", cols_before)?; - result.set_item("cols_after", frame.columns.len())?; - result.set_item("missing_before", missing_before)?; - result.set_item("missing_after", frame.null_cells())?; - result.set_item("duplicates_removed", duplicates_removed)?; - result.set_item("duplicates_detected", duplicates_detected)?; - result.set_item("outliers_handled", outliers_handled)?; - result.set_item("columns_dropped", columns_dropped)?; - result.set_item("columns_imputed", columns_imputed)?; - result.set_item("actions", actions)?; - result.set_item("stage_timings", timings)?; - result.set_item("profile", profiles_to_py(py, &profiles)?)?; - result.set_item("columns", columns_to_py(py, &frame.columns)?)?; - Ok(result.into()) -} - -fn frame_from_payload(payload: &Bound<'_, PyDict>) -> PyResult { - let columns_obj = payload - .get_item("columns")? - .ok_or_else(|| PyErr::new::("missing columns"))?; - let columns_list = columns_obj.downcast::()?; - let mut columns = Vec::new(); - for item in columns_list { - let col = item.downcast::()?; - let name: String = col.get_item("name")?.unwrap().extract()?; - let dtype: String = col.get_item("dtype")?.unwrap().extract()?; - let values_obj = col.get_item("values")?.unwrap(); - let values = values_obj.downcast::()?; - let data = match dtype.as_str() { - "float" => ColumnData::Float( - values - .iter() - .map(|v| { - if v.is_none() { - Ok(None) - } else { - Ok(Some(v.extract::()?)) - } - }) - .collect::>>()?, - ), - "bool" => ColumnData::Bool( - values - .iter() - .map(|v| { - if v.is_none() { - Ok(None) - } else { - Ok(Some(v.extract::()?)) - } - }) - .collect::>>()?, - ), - _ => ColumnData::Utf8( - values - .iter() - .map(|v| { - if v.is_none() { - Ok(None) - } else { - Ok(Some(v.extract::()?)) - } - }) - .collect::>>()?, - ), - }; - columns.push(Column { name, data }); - } - Frame::new(columns).map_err(|e| PyErr::new::(e)) -} - -fn plan_from_payload(payload: &Bound<'_, PyDict>) -> PyResult { - let cfg_obj = payload - .get_item("config")? - .ok_or_else(|| PyErr::new::("missing config"))?; - let cfg = cfg_obj.downcast::()?; - let mut plan = CleanPlan::default(); - plan.rename_map = extract_pairs(cfg, "rename_map")?; - plan.strip_whitespace = get_bool(cfg, "strip_whitespace", true)?; - plan.normalize_sentinels = get_bool(cfg, "normalize_sentinels", true)?; - plan.sentinels = extract_strings(cfg, "sentinels")?.into_iter().collect::>(); - plan.string_case = match get_optional_string(cfg, "string_case")?.as_deref() { - Some("lower") => StringCase::Lower, - Some("upper") => StringCase::Upper, - _ => StringCase::Preserve, - }; - plan.drop_empty_columns = get_bool(cfg, "drop_empty_columns", true)?; - plan.drop_empty_rows = get_bool(cfg, "drop_empty_rows", true)?; - plan.drop_duplicates = get_bool(cfg, "drop_duplicates", true)?; - plan.duplicate_keep = get_string(cfg, "duplicate_keep", "first")?; - plan.fix_dtypes = get_bool(cfg, "fix_dtypes", true)?; - plan.numeric_threshold = get_float(cfg, "numeric_threshold", 0.95)?; - plan.preserve_leading_zeros = get_bool(cfg, "preserve_leading_zeros", true)?; - plan.impute = get_optional_string(cfg, "impute")?; - plan.outliers = get_optional_string(cfg, "outliers")?; - plan.outlier_method = get_string(cfg, "outlier_method", "iqr")?; - plan.outlier_factor = get_float(cfg, "outlier_factor", 1.5)?; - Ok(plan) -} - -fn apply_renames(frame: &mut Frame, plan: &CleanPlan, actions: &mut Vec) { - if plan.rename_map.is_empty() { - return; - } - let mut count = 0usize; - for column in &mut frame.columns { - if let Some((_, new)) = plan.rename_map.iter().find(|(old, _)| old == &column.name) { - column.name = new.clone(); - count += 1; - } - } - if count > 0 { - Python::with_gil(|py| { - actions.push(action_dict( - "column_names", - None, - &format!("renamed {count} column(s)"), - count, - ).into_py(py)); - }); - } -} - -fn action_dict(step: &str, column: Option, description: &str, count: usize) -> PyObject { - Python::with_gil(|py| { - let d = PyDict::new_bound(py); - d.set_item("step", step).unwrap(); - d.set_item("column", column).unwrap(); - d.set_item("description", description).unwrap(); - d.set_item("count", count).unwrap(); - d.into() - }) -} - -fn columns_to_py(py: Python<'_>, columns: &[Column]) -> PyResult { - let out = PyList::empty_bound(py); - for column in columns { - let d = PyDict::new_bound(py); - d.set_item("name", &column.name)?; - match &column.data { - ColumnData::Float(values) => { - d.set_item("dtype", "float")?; - d.set_item("values", values)?; - } - ColumnData::Bool(values) => { - d.set_item("dtype", "bool")?; - d.set_item("values", values)?; - } - ColumnData::Utf8(values) => { - d.set_item("dtype", "string")?; - d.set_item("values", values)?; - } - } - out.append(d)?; - } - Ok(out.into()) -} - -fn profiles_to_py(py: Python<'_>, profiles: &[profile::ColumnProfile]) -> PyResult { - let out = PyList::empty_bound(py); - for p in profiles { - let d = PyDict::new_bound(py); - d.set_item("name", &p.name)?; - d.set_item("null_count", p.null_count)?; - d.set_item("unique_count", p.unique_count)?; - d.set_item("min", p.min)?; - d.set_item("max", p.max)?; - d.set_item("mean", p.mean)?; - out.append(d)?; - } - Ok(out.into()) -} - -fn get_bool(cfg: &Bound<'_, PyDict>, key: &str, default: bool) -> PyResult { - Ok(cfg.get_item(key)?.map_or(default, |v| v.extract().unwrap_or(default))) -} - -fn get_float(cfg: &Bound<'_, PyDict>, key: &str, default: f64) -> PyResult { - Ok(cfg.get_item(key)?.map_or(default, |v| v.extract().unwrap_or(default))) -} - -fn get_string(cfg: &Bound<'_, PyDict>, key: &str, default: &str) -> PyResult { - Ok(cfg - .get_item(key)? - .map_or_else(|| default.to_string(), |v| v.extract().unwrap_or_else(|_| default.to_string()))) -} - -fn get_optional_string(cfg: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match cfg.get_item(key)? { - Some(v) if !v.is_none() => Ok(Some(v.extract()?)), - _ => Ok(None), - } -} - -fn extract_strings(cfg: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match cfg.get_item(key)? { - Some(v) => v.extract(), - None => Ok(Vec::new()), - } -} - -fn extract_pairs(cfg: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match cfg.get_item(key)? { - Some(v) => v.extract(), - None => Ok(Vec::new()), - } -} diff --git a/docs/backends.md b/docs/backends.md index 348ccddf..8c99a6ee 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -119,7 +119,6 @@ native extension from the repo checkout: ```bash pip install -e ".[dev,freshcore]" -maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module python benchmarks/bench_freshcore.py --rows 10000 100000 --workload full ``` diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 8acd819a..a88a7db5 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -73,9 +73,6 @@ Makefile shortcuts: `make benchmark`, `make benchmark-ci`, `make benchmark-repor `results///.json` in the schema in `benchmarks/results_schema.py`, so runs diff cleanly across versions. -> The legacy quick-bench (`python benchmarks/bench_quick.py --fixtures --compare`) -> is preserved for ad-hoc throughput checks on the `tests/fixtures/` corpus. - ## Calibration metrics (CleanBench Phase 3) The CleanBench mini-suite (`benchmarks/cleanbench/`, run in CI by @@ -192,12 +189,7 @@ Measured on the committed labelled dataset abbreviations, blanked fields, transliteration, shared households, deliberate near-collisions). Method: **rule-weighted linkage** (normalized weighted average β€” no EM/Fellegi–Sunter, no Splink-parity claim). Raw output: -`benchmarks/er_results.json`. Reproduce: - -```console -python benchmarks/bench_entity_resolution.py \ - --labelled benchmarks/data/er_labelled_5k.csv --json -``` +`benchmarks/er_results.json`. | Configuration | Precision | Recall | F1 | FP | FN | False-merged clusters | |---|---:|---:|---:|---:|---:|---:| @@ -239,24 +231,6 @@ See [`docs/fixtures.md`](fixtures.md) for fixture schemas and the `DEFECT_MANIFEST` structure, and `benchmarks/README.md` for the baseline contract (`run(df)` + `AUTHORED_LINES`, lazy imports, graceful skip). -## Reproduce the legacy corpus - -```bash -python benchmarks/bench_quick.py --fixtures --compare # tests/fixtures corpus, side by side -python benchmarks/bench_quick.py --online --compare # cached online datasets -``` - -Every fixture in `tests/fixtures/` is run under `conservative`, `balanced`, and -`aggressive` strategies in CI, plus 50 curated real public datasets. Reproduce -the quality/efficiency matrix on your own data with: - -```python -import freshdata as fd -print(fd.compare_clean(df)) -``` - - - # CleanBench full-suite results Generated: 2026-07-05T07:07:36+00:00 diff --git a/docs/context-policies.md b/docs/context-policies.md index bd27a86b..17642c53 100644 --- a/docs/context-policies.md +++ b/docs/context-policies.md @@ -182,10 +182,6 @@ policy dict, plus a warning per unresolved/unparsed item. workflow. `drop_if`/`rename`/`map` rules are still compiled and carried in the policy but not yet executed. -See the notebook -[`notebooks/06_context_cleaning.ipynb`](https://github.com/FreshCode-Org/freshdata/blob/main/notebooks/06_context_cleaning.ipynb) -for the full ecommerce walkthrough. - ## Optional intent-head evidence (Phase 5) An optional intent classifier (`fd-intent-v1`, see diff --git a/docs/developer-training-pipeline.md b/docs/developer-training-pipeline.md deleted file mode 100644 index e0793d0b..00000000 --- a/docs/developer-training-pipeline.md +++ /dev/null @@ -1,115 +0,0 @@ -# Developer training pipeline (Phase 5) - -The `training/` package builds, evaluates, and packages FreshData's small -local model artifacts. It is a **development-time tool**: it lives outside -`src/freshdata`, is never shipped in the wheel, and the runtime never -imports it. - -## What stays true at runtime - -- `fd.clean` is deterministic, offline, and **model-free by default** -- no LLM is used at runtime, ever β€” teacher models exist only in `training/` -- no cloud call from `fd.clean`; no automatic model download -- no generative repair model exists anywhere in FreshData -- the protected-column byte-identity guard is absolute -- no model weights ship in the PyPI wheel (enforced by - `.github/workflows/wheel-size.yml` and `tests/test_semantic_backcompat.py`) - -## Pipeline stages - -```bash -make training-seed # license gate + seed corpus -make training-corrupt # ~40 corruptors -> labeled pairs -make training-teacher-labels # compliance-gated teacher tasks (optional) -make training-distill # role/intent heads, export, int8 quantize -make training-eval # calibration + full CleanBench gates -make training-package-artifacts # dist/artifacts/ with manifests + cards -``` - -End-to-end: - -```bash -make training-dev-artifacts # synthetic-only, offline, ~seconds -make training-release-artifacts # full pipeline; fails on any gate failure -``` - -## How seed data is licensed - -Every source is registered in `training/seed/registry.json` with license, -attribution, explicit `allowed_for_training`, and a resolved PII risk; -`python -m training.datasets.validators --check-licenses` blocks unclear -licenses, share-alike data without legal review, and unresolved PII risk. -See `training/seed/LICENSES.md`. - -## How corruptors create ground truth - -`training/corruptors/` injects parameterized, deterministic dirt into known -clean frames and emits a machine-verifiable label per mutation (raw value, -clean value, transform family, `should_repair`, `should_auto_apply`, risk, -protected, ambiguous). Ground truth is corruption metadata β€” never a model -guess. Ambiguous corruptions are labeled *never auto-apply*; protected-column -traps label **any** mutation as failure. - -## What teacher models may and may not do - -Allowed (development-time only, batched, cached, PII-masked): realism -direction, column-role labeling, context paraphrases, ambiguity -adjudication, rationale templates, benchmark red-teaming. - -Never: runtime calls, per-cell calls, generating messy/clean pairs from -scratch, receiving full rows or real PII, or serving as sole release-gating -truth. See `training/COMPLIANCE.md` and -`python -m training.teacher.compliance check`. - -## How human review works - -β‰₯5% of each teacher batch is sampled for review; >3% disagreement forces a -full batch review; release-gating eval labels are 100% human-verified with -reviewer identity and timestamps (`training/teacher/review.py`, -`training/eval/human_verified.py`). - -## How model cards are produced - -`training/model_cards/*.md` are authored with the 15 required sections and -copied into each packaged artifact; `training.distill.package_artifacts` -fails when a card, license summary, or eval summary is missing. - -## How calibration is trained - -`training/calibration/` extracts per-proposal features from CleanBench runs, -fits isotonic curves (pool-adjacent-violators, pure numpy) per (backend, -issue family), and exports the registry-compatible `calib-v1` table. Gates: -ECE ≀ 0.03 and P@0.95 β‰₯ 0.99 on a held-out split. The runtime only ever -*loads* the JSON β€” a missing table degrades to `uncalibrated`, and -protected-column checks stay deterministic. - -## How CleanBench gates releases - -`python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site ---check-gates` runs representation, semantic-value, context-compliance, -profile-replay, and scale tracks; see [Benchmarks](benchmarks.md) for the -gate list and latest results. - -## Reproducing artifact builds - -Everything is seeded: corruptors, generators, and training are deterministic -(`sha256`-hashed features, zero-init full-batch training β€” no RNG). The same -commit + same seeds produce byte-identical weights; manifests record the git -commit and per-file SHA256. - -## Running the tests - -- model-free (default): `pytest -m "not online and not large"` β€” this is PR CI -- semantic-with-stub: included in the default run (`FRESHDATA_SEMANTIC_STUB` - paths and the CleanBench stub encoder need no model files) -- real-model nightly, locally: - `pip install -e ".[semantic]" onnx && make training-dev-artifacts && - pytest tests -k "semantic or models or cleanbench" --no-cov` - -## Limitations and known failure modes - -Head evals run on synthetic/template corpora β€” high scores there do not -promise the same on arbitrary real-world data; the encoder contrastive stage -is optional and is recorded as `skipped` without torch; T5 perf gates -compare against a same-machine baseline and are informational until one is -pinned. See [Honest limitations](limitations.md). diff --git a/docs/examples.md b/docs/examples.md index 9408af84..5e76993f 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -18,7 +18,6 @@ directory has narrated Jupyter walkthroughs. | `01_missing_values.py` | Smart, role-aware missing-value imputation | | `02_outliers.py` | Outlier detection and flagging vs removal | | `03_normalization.py` | Feature normalization for ML | -| `04_profiling.py` | Read-only data profiling and EDA | | `05_ml_pipeline.py` | End-to-end ML preprocessing with scikit-learn | | `06_large_dataset.py` | Cleaning a large synthetic dataset, with timing | | `07_pandas_integration.py` | Dropping freshdata into an existing pandas workflow | diff --git a/docs/field-validation.md b/docs/field-validation.md index 09a6a2ca..673df3b4 100644 --- a/docs/field-validation.md +++ b/docs/field-validation.md @@ -157,12 +157,7 @@ exception tables, lineage). `report.to_frame()` gives a flat DataFrame; Validation runs a vectorized pre-screen per column and only pays the per-cell explanation path for suspect cells, so mostly-clean feeds validate -at bulk speed. Reproduce the comparison against regex and pandas-coercion -baselines with: - -```bash -python benchmarks/bench_fieldcheck.py 100000 -``` +at bulk speed. ## Known limitations diff --git a/docs/freshcore.md b/docs/freshcore.md deleted file mode 100644 index 9a60077e..00000000 --- a/docs/freshcore.md +++ /dev/null @@ -1,125 +0,0 @@ -# FreshCore architecture - -FreshCore is FreshData's optional native backend for cleaning-first workloads. -It is designed to plug into the existing execution layer rather than replace the -public API or the pandas reference implementation. - -## Integration model - -- Users opt in with `fd.clean(df, engine="freshcore")`. -- `CleanConfig` still controls what cleaning is requested. -- `EngineConfig` still controls how execution is routed. -- The default `fd.clean(df)` path remains unchanged. -- Unsupported work delegates to the pandas reference pipeline and is recorded in - `CleanReport.fallback_events`. - -The Python adapter lives at `freshdata.execution.backends._freshcore`. It -materializes pandas-compatible inputs, sends compact columns and plan parameters -to the Rust module, then maps native output back into a pandas DataFrame and the -standard `CleanReport` audit contract. - -## Native engine - -The Rust crate is in `crates/freshcore` and exposes the `freshdata_freshcore` -PyO3 module. Its internal model is deliberately smaller than a general-purpose -DataFrame: - -- typed nullable arrays (`Float`, `Bool`, `Utf8`) -- separate validity/null representation through `Option` -- a compact physical cleaning plan -- native kernels for strings, missing values, casts, duplicates, outliers, and - simple profiles -- operation-level timing records for benchmarks - -FreshCore does not depend on pandas, Polars, DuckDB, Spark, Dask, Modin, Vaex, -or any existing DataFrame execution engine. - -## V1 support boundary - -FreshCore v1 runs native kernels for conservative, deterministic cleaning: - -- column name normalization -- whitespace trimming -- optional text case normalization via `string_case=None|"lower"|"upper"` -- sentinel-to-missing normalization -- empty row/column drops -- full-row duplicate detection with `duplicate_keep="first"` or `"last"` -- boolean and numeric string casts where safe -- mean/median/mode imputation -- IQR/z-score outlier clipping or flagging, including the pandas zero-IQR - fallback to mean-absolute-deviation fences -- simple per-column profile metadata - -FreshCore falls back for semantic cleaning, context/policy protection, cleaning -memory, domain packs, contracts, non-default indexes, duplicate subsets, -aggregate/drop duplicate modes, model-based outliers, constant-column dropping, -memory downcasting, and the balanced/aggressive decision engine. It also falls -back for `impute="missforest"`, per-column `impute_strategy`, outlier handling -on float columns holding `Β±inf`, and mode/auto imputation of nullable boolean -columns with missing values. - -The same two gaps can open inside a native run. `fix_dtypes` casts text columns -before imputation and outlier handling, so a `"yes"`/`"no"` column with missing -values becomes a boolean column the kernels do not impute. A numeric text -column holding `"inf"` becomes a float column holding `Β±inf`, which leaves the -outlier fences undefined. The input frame shows neither, so the adapter checks -the returned column dtypes. When a cast column hits a gap, it reruns the frame -on pandas and records a fallback event naming the column -(`fallback_step="impute"` or `"outliers"`). Under `fallback_policy="error"` it -raises `FallbackError` instead. `fd.plan()` checks only the input, so it -cannot predict these fallbacks, and the discarded native run still costs time. -Frames without such casts stay on the native path. - -With `drop_duplicates=False` (the default), the native module counts full-row -duplicates at the same stage as the pandas step: after string cleaning, -empty-row removal and casts, and before imputation and outliers. It returns -the count as `duplicates_detected` and records a `detect_duplicates` stage -timing. The adapter reports it like pandas: a "detected N duplicate row(s)" -action, a warning above `duplicate_threshold`, and `DuplicateRatioError` under -`duplicate_ratio_action="error"`. Native modules built before this count -existed don't report `duplicates_detected`; with those, detection is skipped, -and under `duplicate_ratio_action="error"` the adapter falls back to pandas. - -The native arrays carry only float, bool and string values, so FreshCore also -falls back for datetime, timedelta, categorical, period and interval columns, -for integer columns holding values beyond Β±2\*\*53 (float64 cannot represent -them exactly), and for column labels that collide once stringified (such as -`1` and `"1"`). Integer columns (`int64`, nullable `Int64`, and other widths) -are cast back to their input dtype when every returned value is integral and -in range; otherwise they come back as `float64` and the change is recorded in -`report.backend_differences`. Non-string column labels such as `0` and `1` -come back unchanged. - -## Building and testing - -The CI `freshcore-native` job runs the same steps: - -```bash -pip install -e ".[dev,freshcore]" -cargo test --manifest-path crates/freshcore/Cargo.toml -maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module -pytest tests/test_execution -k freshcore -``` - -`tests/test_execution/test_freshcore_native_parity.py` runs the real extension -against the pandas reference and is skipped when `freshdata_freshcore` is not -installed. The other FreshCore tests use a fake native module. - -## Benchmarking - -Build the native module before benchmarking: - -```bash -pip install -e ".[dev,freshcore]" -maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module -python benchmarks/bench_freshcore.py --rows 10000 100000 1000000 --workload full -``` - -The benchmark compares: - -- a handwritten pandas baseline -- FreshData's pandas reference path -- FreshCore through `engine="freshcore"` - -The output includes timing, peak memory, fallback events, parity shape checks, -and FreshCore stage timings when the native module is installed. diff --git a/docs/interactive.md b/docs/interactive.md index c9e9e21d..22112b1c 100644 --- a/docs/interactive.md +++ b/docs/interactive.md @@ -71,11 +71,3 @@ raises a clear install hint (e.g. *"install `freshdata-cleaner[viz]`"*) rather t - **`explain_clean(df).show()`** β€” before/after column comparison with risky changes separated from safe ones. -## Generating HTML examples - -```bash -python scripts/generate_html_examples.py # writes docs/examples/*.html -``` - -The committed samples under `docs/examples/` are produced by that script from -synthetic data, so they always match the current renderers. diff --git a/docs/learning-profiles.md b/docs/learning-profiles.md index 78e6231b..9e701362 100644 --- a/docs/learning-profiles.md +++ b/docs/learning-profiles.md @@ -183,19 +183,6 @@ freshdata profile merge orders_v1.fdprofile orders_v2.fdprofile -o merged.fdprof positional argument keeps its original meaning β€” data profiling (`freshdata profile data.csv`) β€” so existing scripts don't break. -## CleanBench T4 - -`benchmarks/cleanbench/t4_profiles.py` covers 10 fixture families (category -maps, allowed-value maps, reference typos, sentinels, day-first dates, IN -phone formats, double-`@` emails, an imputation fixture that must *not* -replay literals, and an unexplained-only fixture). Each fixture learns a -profile from a train split and replays it on a held-out batch of the same -corruption family. Gates enforced in `tests/test_cleanbench_t4.py`: mean F1 -lift over the replayable fixtures, false-modification rate no worse than the -no-profile baseline, zero protected-column violations, and zero privacy -leaks (no raw sensitive literal ever appears in the serialized profile bytes -under `privacy="mask"`). - ## Limitations - **No model training, no LLM, no cloud.** Every transform a profile can diff --git a/docs/limitations.md b/docs/limitations.md index c6eef0c0..49d4d4ea 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -206,12 +206,3 @@ context policies, and cleaning memory need **none** of these. --- -## Development-time training pipeline - -These apply only to the dev-time `training/` package, never to `import freshdata`: - -- The optional encoder contrastive-distillation stage requires a tensor - framework; without it the stage records `skipped` and the Phase-3 baseline - encoder is retained (its safety gates still run). -- Dev artifact builds without the `onnx` package export portable weight JSON - instead of `.onnx` graphs; release builds require `onnx` and fail without it. diff --git a/docs/model-cards.md b/docs/model-cards.md index d64bb199..c8385130 100644 --- a/docs/model-cards.md +++ b/docs/model-cards.md @@ -6,16 +6,6 @@ summary, PII policy, teacher-model usage, human-review process, evaluation metrics, known limitations, failure modes, the protected-column guarantee, calibration version, artifact hash, and FreshData version compatibility. -The authoritative cards live in the repository and are copied into each -packaged artifact (`dist/artifacts//model_card.md`): - -- [`fd-col-encoder-v1`](https://github.com/FreshCode-Org/freshdata/blob/main/training/model_cards/fd-col-encoder-v1.md) - β€” column/value encoder + semantic-type role head -- [`fd-intent-v1`](https://github.com/FreshCode-Org/freshdata/blob/main/training/model_cards/fd-intent-v1.md) - β€” context-sentence intent head (optional evidence for the deterministic parser) -- [`calib-v1`](https://github.com/FreshCode-Org/freshdata/blob/main/training/model_cards/calib-v1.md) - β€” isotonic confidence-calibration tables (plain JSON) - Every card states explicitly: - there is **no generative repair model** diff --git a/docs/performance-investigation.md b/docs/performance-investigation.md index 8512ed3e..01da6beb 100644 --- a/docs/performance-investigation.md +++ b/docs/performance-investigation.md @@ -281,13 +281,9 @@ The focused fixture verification was: The exact command in the brief (`print(result.to_json())`) cannot run because `SemanticProbeBuild` intentionally has no `to_json()` API. It failed with -`AttributeError: 'SemanticProbeBuild' object has no attribute 'to_json'`. The -following serialization-only equivalent was run without changing the probe or -production API and produced valid JSON: - -```bash -PYTHONPATH=src .venv-qa/bin/python -c 'import json; from benchmarks.performance.datasets import DatasetSpec, make_mixed_frame; from benchmarks.performance.semantic_probe import probe_context_build; from freshdata.config import CleanConfig; frame = make_mixed_frame(DatasetSpec(rows=100000, width="medium", seed=42, dataset_type="mixed")); _, result = probe_context_build(frame, CleanConfig(semantic_mode="assist", verbose=False)); print(json.dumps({"total_calls": result.total_calls, "total_eligible_calls": result.total_eligible_calls, "total_bypassed_calls": result.total_bypassed_calls, "total_theoretical_hits": result.total_theoretical_hits, "by_operation": {name: {"total_calls": item.total_calls, "eligible_calls": item.eligible_calls, "bypassed_calls": item.bypassed_calls, "unique_keys": item.unique_keys, "theoretical_hits": item.theoretical_hits, "hit_rate": item.hit_rate} for name, item in result.by_operation.items()}}, sort_keys=True))' -``` +`AttributeError: 'SemanticProbeBuild' object has no attribute 'to_json'`. A +serialization-only equivalent was run without changing the probe or +production API and produced valid JSON. The successful single-build JSON reported `639307` total calls, `497842` eligible calls, `141465` bypassed calls, and `52725` theoretical diff --git a/docs/plugins.md b/docs/plugins.md index 72bd546a..02ddb312 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -172,9 +172,6 @@ the column/frame they were given, and that nothing exceeds the declared Runnable, contract-passing examples live in the repo: -- [`examples/plugins/custom_expert/acronym_expert.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_expert/acronym_expert.py) β€” expand acronyms in categorical columns -- [`examples/plugins/custom_backend/keyword_backend.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_backend/keyword_backend.py) β€” strip a shared prefix frame-wide, budget-aware -- [`examples/plugins/custom_validator/min_rows_validator.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_validator/min_rows_validator.py) β€” flag too-few-rows and constant columns - [`examples/plugins/custom_comparator/initials_comparator.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_comparator/initials_comparator.py) β€” initials-agreement ER comparator - [`examples/plugins/custom_exporter/markdown_exporter.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_exporter/markdown_exporter.py) β€” Markdown clean-report exporter diff --git a/docs/semantic-models.md b/docs/semantic-models.md index 18f32609..856ca76a 100644 --- a/docs/semantic-models.md +++ b/docs/semantic-models.md @@ -212,15 +212,3 @@ values only, structurally). No generative repairs. No automatic downloads. No model weights in the wheel. Protected columns remain byte-identical under any backend β€” enforced physically by the executor guard, not by convention. -## How these models are built (Phase 5) - -The artifacts themselves (`fd-col-encoder-v1`, `fd-intent-v1`, `calib-v1`) -are produced by the development-time training pipeline in `training/` β€” -seed-corpus governance, a ~40-corruptor labeling engine, compliance-gated -teacher tasks, human-reviewed eval labels, int8 quantization, and manifested -packaging under `dist/artifacts/`. Every artifact ships with a -[model card](model-cards.md) and is gated by the full -[CleanBench suite](benchmarks.md). See the -[developer training pipeline](developer-training-pipeline.md) guide. None of -this changes the runtime contract above: model-free by default, offline, -no LLM, no automatic downloads, no weights in the wheel. diff --git a/docs/streaming.md b/docs/streaming.md index 42bf7867..44f525e0 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -211,31 +211,3 @@ freshdata benchmark-stream --rows 100000000 --batch-size 100000 --cols 20 \ --report benchmark.json ``` -## The 100M-row stable-memory benchmark - -`benchmarks/bench_streaming.py` is an **out-of-core / streaming** test, *not* a -single-DataFrame test. It generates synthetic rows lazily, one batch at a time, feeds -them through a `StreamingCleaner`, and tracks peak RSS (via `psutil`, falling back to the -`resource` module). - -```bash -python benchmarks/bench_streaming.py --rows 100000000 --batch-size 100000 --cols 20 -``` - -It reports total rows processed, batch size, columns, elapsed time, rows/sec, baseline -and peak memory, steady-state memory growth, and the final cumulative trust score. -**Acceptance** is that steady-state memory does not grow with rows β€” peak memory stays -bounded relative to the *batch* size, not the dataset size β€” and it fails loudly if -fewer rows are processed than requested. - -!!! warning "Honest limitations" - - Streaming mode is **micro-batch**, not true row-by-row real time. - - **Global** cross-batch duplicate detection is limited: by default duplicates are - scoped within a batch; enabling `global_duplicates` uses a *bounded recent-window* - (the `window_size` most recently seen distinct rows) that can miss duplicates - older than the window. - - Medians/quantiles are **approximate** (reservoir-sampled), and the top-k category - summary is approximate when a column saturates `max_categories`. - - Kafka and Arrow Flight are **optional** integrations. - - The enterprise-scale "clean β‰₯100M rows out-of-core with stable memory" claim - depends on the benchmark above passing in **your** environment. diff --git a/docs/trust-claims.md b/docs/trust-claims.md index 28494569..f700476e 100644 --- a/docs/trust-claims.md +++ b/docs/trust-claims.md @@ -24,7 +24,6 @@ named tests, and CI fails if either side drifts. | Fully offline; only network call is `fd.models.pull` | **holds** | default test gate is `-m "not online"`; no other network code paths | | Polars/DuckDB/Spark backends scale beyond pandas | **holds with measured boundary** | see [backends](backends.md): DuckDB native handle peaks at ~40% of eager RSS (measured); `polars-lazy` defers only final materialization β€” pipeline intermediates still collect eagerly. Reproduce: `python benchmarks/bench_outofcore.py` | | Native handle kept with `strategy="conservative"` | **corrected** | also requires `fix_dtypes=False`; dtype fixing forces the recorded pandas fallback | -| StreamingCleaner cleans arbitrarily tall data with stable memory | **holds** | bounded reservoirs/counters, RSS benchmark `benchmarks/bench_streaming.py`; dedupe window is recent-window, not global (see [limitations](limitations.md)) | | CSV exports safe to open in spreadsheets | **holds by default on all first-party CSV outputs** (`fd.clean_csv`, `freshdata clean` / `apply-plan`, the streaming CLI, review queues); opt out with `sanitize_formulas=False` / `--no-sanitize-formulas` | see [threat model](threat-model.md); `tests/test_csv_formula_sanitize.py` | | Typed (`py.typed`), mypy-clean | **holds** | `mypy src/freshdata` in CI quality gate (187 files) | | `fallback_policy="error"` prevents any unrequested pandas materialization | **holds** | enforced *before* the pandas pipeline runs at every backend delegation point; `tests/test_execution/test_fallback_policy.py` | diff --git a/examples/04_profiling.py b/examples/04_profiling.py deleted file mode 100644 index 23dc139d..00000000 --- a/examples/04_profiling.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Read-only data profiling and EDA with freshdata. - -`fd.profile` inspects data quality without changing anything, using the same -inference engine as `fd.clean`. Run: - - python examples/04_profiling.py -""" - -import pandas as pd - -import freshdata as fd - - -def make_data() -> pd.DataFrame: - return pd.DataFrame( - { - "First Name": [" Ann ", "Bob", "Bob", "Cara", None], - "AGE": ["34", "N/A", "N/A", "41", "29"], - "Joined Date": ["2021-01-01", "2021-02-15", "2021-02-15", "-", "2021-03-30"], - "Active": ["yes", "no", "no", "yes", "no"], - "Salary($)": ["$1,200.50", "-", "-", "$2,000", "$1,750"], - "empty": [None, None, None, None, None], - } - ) - - -def main() -> None: - df = make_data() - - # Human-readable profile - print(fd.profile(df)) - - # With a dry-run plan attached - profile = fd.profile(df, include_plan=True) - print("\nPlanned primary model per column:") - print(profile.plan.summary()) - - # Inferred roles - print("\nInferred roles:") - print(fd.infer_roles(df)) - - -if __name__ == "__main__": - main() diff --git a/examples/cdc_event_time_gate.py b/examples/cdc_event_time_gate.py deleted file mode 100644 index 1fd63b59..00000000 --- a/examples/cdc_event_time_gate.py +++ /dev/null @@ -1,53 +0,0 @@ -"""CDC / event-time quality gate with ``fd.cdc_profile``. - -Change-data-capture streams fail in ways that are *not* missing values: records -arrive stale, late, out of order, with duplicate keys, bad operation codes, or as -a replayed batch. ``cdc_profile`` classifies those freshness/ordering defects -separately from nulls and reports trust penalties you can gate on. Run: - - python examples/cdc_event_time_gate.py -""" - -import pandas as pd - -import freshdata as fd - - -def main() -> None: - # An order-change feed captured from a CDC connector, in arrival order. - changes = pd.DataFrame( - { - "order_id": [101, 101, 102, 103, 103, 101, 104], - "event_ts": [ - "2024-03-01 09:00:00", - "2024-03-01 09:05:00", - "2024-03-01 09:06:00", - "2024-03-01 09:05:30", # 102/103 interleave is fine (different keys) - None, # missing event time - "2024-03-01 08:30:00", # 101 arrives 35m late, past the watermark - "2024-03-01 09:05:30", - ], - "op": ["c", "u", "c", "c", "u", "u", "Q"], # 'Q' is not a valid op code - } - ) - - report = fd.cdc_profile( - changes, - event_time="event_ts", - key="order_id", - lateness="10m", - operation_col="op", - now=pd.Timestamp("2024-03-01 09:10:00"), - stale_after="30m", - ) - - print(report.summary()) - print("\ntrust penalties (0=clean, 1=worst):", report.trust_penalties) - print(f"freshness: {report.freshness_seconds:.0f}s behind reference") - print(f"\ngate passed: {report.passed}\n") - print("defect table:") - print(report.to_frame()[["kind", "level", "n_rows", "rationale"]].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/examples/cross_source_link.py b/examples/cross_source_link.py deleted file mode 100644 index d8f085b0..00000000 --- a/examples/cross_source_link.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Cross-source entity linking with ``fd.link``. - -Link customer records across two systems (CRM vs. billing), explain every -candidate match, and hand low-confidence pairs to a steward-review queue. -FreshData is the candidate-generation + preprocessing layer here β€” it formats -and explains matches; it does not replace a dedicated matcher (use the external -adapter hook to plug one in). Run: - - python examples/cross_source_link.py -""" - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import build_review_queue - - -def main() -> None: - crm = pd.DataFrame( - { - "full_name": ["Acme Corp", "Globex LLC", "Initech Inc", "Stark Industries"], - "zip": ["10001", "94105", "73301", "10002"], - } - ) - billing = pd.DataFrame( - { - "full_name": ["acme corporation", "Globex L.L.C.", "Umbrella Co", "Stark Ind."], - "zip": ["10001", "94105", "60601", "10002"], - } - ) - - # Fuzzy link: block on zip (cheap, exact), Jaro-Winkler compare the name. - report = fd.link( - crm, billing, - keys=["full_name"], - strategy="fuzzy", - threshold=0.82, - blocking="l.zip = r.zip", - ) - print(report.summary()) - print("\ncandidate pairs (with per-field explanations):") - print(report.to_frame()[["left_id", "right_id", "match_probability", "decision"]] - .to_string(index=False)) - - # Hand the borderline pairs to a steward-review queue. - queue = build_review_queue(report) - print(f"\nsteward review queue: {len(queue.items)} item(s) need a human decision") - - -if __name__ == "__main__": - main() diff --git a/examples/entity_resolution_duckdb.py b/examples/entity_resolution_duckdb.py deleted file mode 100644 index f77d2009..00000000 --- a/examples/entity_resolution_duckdb.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Probabilistic entity resolution at scale with a DuckDB backend. - -Deduplicate records using DuckDB-powered blocking (candidate-pair generation) -plus rule-weighted probabilistic scoring. Falls back to the pandas backend when -DuckDB is not installed. Run: - - python examples/entity_resolution_duckdb.py - -DuckDB backend extra: - - pip install "freshdata-cleaner[entity-resolution]" -""" - -import importlib.util - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import BlockingRule, ComparisonLevel, EntityResolutionConfig - - -def people() -> pd.DataFrame: - return pd.DataFrame( - { - "id": [1, 2, 3, 4, 5, 6], - "name": [ - "Jonathan Smith", - "Jon Smith", - "Johnny Smith", - "Alice Brown", - "Alicia Brown", - "Robert King", - ], - "dob": [ - "1990-01-01", "1990-01-01", "1990-01-01", - "1985-05-05", "1985-05-05", "1970-12-12", - ], - "email": [ - "jsmith@x.com", "jsmith@x.com", "jon@y.com", - "alice@z.com", "alicia@z.com", "rking@q.com", - ], - } - ) - - -def main() -> None: - backend = "duckdb" - if importlib.util.find_spec("duckdb") is None: - backend = "pandas" - print("duckdb not installed -> using the pandas fallback backend\n") - - config = EntityResolutionConfig( - enabled=True, - backend=backend, - unique_id_column="id", - # Blocking caps the candidate space (no full cartesian product). - blocking_rules=( - BlockingRule(sql="lower(l.email) = lower(r.email)", description="same email"), - BlockingRule(sql="l.dob = r.dob", description="same date of birth"), - ), - comparisons=( - ComparisonLevel(column="name", kind="jaro_winkler", threshold=0.85, weight=3.0), - ComparisonLevel(column="dob", kind="exact", weight=1.0), - ComparisonLevel(column="email", kind="exact", weight=1.0), - ), - match_threshold=0.80, - clerical_review_threshold=0.55, - max_pairs=1_000_000, # hard safety gate - ) - - resolved, report = fd.resolve_entities(people(), config=config) - print(report.summary(), "\n") - for cluster in report.clusters: - print( - f"cluster {cluster.cluster_id}: records {list(cluster.record_ids)} " - f"(canonical={cluster.canonical_record_id}, confidence={cluster.confidence:.2f})" - ) - print("\nresolved frame (with cluster_id):") - print(resolved[["id", "name", "cluster_id"]].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/examples/plugins/custom_backend/keyword_backend.py b/examples/plugins/custom_backend/keyword_backend.py deleted file mode 100644 index 4dbed392..00000000 --- a/examples/plugins/custom_backend/keyword_backend.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Example FreshData plugin: a custom semantic *backend*. - -A backend proposes over the *whole frame* (not one column at a time) and is -opted into by name via ``semantic_backends=(...)``. This one strips a shared -``"lbl_"`` prefix from any string column β€” a frame-wide transform an -expert (which sees one column) is a poor fit for. - -Try it:: - - import freshdata as fd - from keyword_backend import KeywordBackend - - fd.testing.semantic_backend_contract(KeywordBackend()) - fd.register_backend(KeywordBackend()) - - import pandas as pd - df = pd.DataFrame({"tag": ["lbl_red", "lbl_blue", "plain"]}) - cleaned, report = fd.clean( - df, semantic_mode="auto", - semantic_backends=("deterministic", "keyword"), - return_report=True, - ) - -Package it:: - - [project.entry-points."freshdata.backends"] - keyword = "keyword_backend:KeywordBackend" -""" - -from __future__ import annotations - -import pandas as pd - -from freshdata.semantic.backends.base import Budget -from freshdata.semantic.scoring import make_proposal -from freshdata.semantic.types import SemanticContext, SemanticEvidence, SemanticProposal - -_PREFIX = "lbl_" - - -class KeywordBackend: - name = "keyword" - semantic_types = ("categorical",) - max_risk = "medium" - uses_network = False - requires = () - - def warm_up(self) -> None: - # Nothing to load. Raise freshdata.semantic.backends.base.BackendUnavailable - # here to self-disable when an optional resource is missing. - pass - - def propose( - self, df: pd.DataFrame, ctx: SemanticContext, budget: Budget - ) -> list[SemanticProposal]: - proposals: list[SemanticProposal] = [] - for col in df.columns: - info = ctx.columns.get(str(col)) - if info is None or info.identifier_like: - continue # never touch id-like columns - distinct = df[col].dropna().unique() - # Respect the model budget the same way built-in backends do. - if not budget.try_column(len(distinct)): - break - for raw in distinct: - if isinstance(raw, str) and raw.startswith(_PREFIX): - proposals.append(make_proposal( - column=str(col), - raw_value=raw, - proposed_value=raw[len(_PREFIX):], - issue_type="category_synonym", - expert=self.name, - base_confidence=0.95, - evidence=(SemanticEvidence("pattern", f"strip {_PREFIX!r}", 0.0),), - count=int((df[col] == raw).sum()), - rationale=f"stripped {_PREFIX!r} prefix from {raw!r}", - info=info, - )) - return proposals diff --git a/examples/plugins/custom_expert/acronym_expert.py b/examples/plugins/custom_expert/acronym_expert.py deleted file mode 100644 index 099f3e8f..00000000 --- a/examples/plugins/custom_expert/acronym_expert.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Example FreshData plugin: a custom semantic *expert*. - -An expert proposes value repairs for a single column, one distinct value at a -time. This one expands a small, explicit acronym table (``"NY" -> "New York"``) -in categorical columns. It is deterministic, offline, and low-risk. - -Try it:: - - import freshdata as fd - from acronym_expert import AcronymExpert - - fd.testing.expert_contract(AcronymExpert()) # verify the contract first - fd.register_expert(AcronymExpert()) - - import pandas as pd - df = pd.DataFrame({"state": ["NY", "CA", "NY", "TX"]}) - cleaned, report = fd.clean(df, semantic_mode="auto", return_report=True) - -The plugin only *proposes* β€” the policy gate decides what applies. Because -``category_synonym`` is a medium-risk issue type, these expansions are recorded -as ``status="suggested"`` in the report (not auto-applied) unless you raise the -confidence or run a lower-risk issue type. That is the gate doing its job: a -plugin can never force an auto-apply. - -Package it for entry-point discovery in your pyproject.toml:: - - [project.entry-points."freshdata.experts"] - acronym = "acronym_expert:AcronymExpert" -""" - -from __future__ import annotations - -import pandas as pd - -from freshdata.semantic.scoring import make_proposal -from freshdata.semantic.types import SemanticColumnInfo, SemanticEvidence, SemanticProposal - -_ACRONYMS = { - "ny": "New York", - "ca": "California", - "tx": "Texas", - "wa": "Washington", -} - - -class AcronymExpert: - # --- required metadata ------------------------------------------------- - name = "acronym" - issue_type = "category_synonym" # reuses the built-in scoring/risk profile - #: The semantic column kinds this expert claims to handle (declaration only; - #: routing still calls applies()). - semantic_types = ("categorical",) - #: The highest risk this expert may ever emit; the plugin system drops any - #: proposal scored above it before the gate sees it. - max_risk = "medium" - #: Deterministic + offline: no network, no optional dependencies. - uses_network = False - requires = () - - def applies(self, info: SemanticColumnInfo) -> bool: - # Only short, low-cardinality categorical columns β€” never an id/target. - return ( - info.role == "categorical" - and not info.identifier_like - and not info.free_text - ) - - def propose(self, series: pd.Series, info: SemanticColumnInfo) -> list[SemanticProposal]: - # Operate on *distinct values* (never row-by-row) and never mutate the - # series β€” return proposals; the gate decides what actually applies. - proposals: list[SemanticProposal] = [] - for raw, count in series.value_counts(dropna=True).items(): - if not isinstance(raw, str): - continue - expansion = _ACRONYMS.get(raw.strip().lower()) - if expansion is None or expansion == raw: - continue - proposals.append(make_proposal( - column=info.name, - raw_value=raw, - proposed_value=expansion, - issue_type=self.issue_type, - expert=self.name, - base_confidence=0.9, - evidence=(SemanticEvidence("lookup", f"acronym table -> {expansion!r}", 0.0),), - count=int(count), - rationale=f"expanded acronym {raw!r} to {expansion!r}", - info=info, - )) - return proposals diff --git a/examples/plugins/custom_validator/min_rows_validator.py b/examples/plugins/custom_validator/min_rows_validator.py deleted file mode 100644 index c72363f6..00000000 --- a/examples/plugins/custom_validator/min_rows_validator.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Example FreshData plugin: a custom *validator*. - -A validator is **read-only**: it inspects the frame (and the compiled context -policy) and returns :class:`freshdata.QualityFinding` records. It never repairs -anything β€” validation is not repair. This one flags frames with too few rows to -reason about, and any fully-constant column. - -Try it:: - - import freshdata as fd - from min_rows_validator import MinRowsValidator - - fd.testing.validator_contract(MinRowsValidator()) - fd.register_validator(MinRowsValidator()) - - import pandas as pd - df = pd.DataFrame({"x": [1, 1, 1]}) - findings = fd.validate(df, context="x is unique.") # validators run inside fd.validate - -Package it:: - - [project.entry-points."freshdata.validators"] - min_rows = "min_rows_validator:MinRowsValidator" -""" - -from __future__ import annotations - -import pandas as pd - -from freshdata import QualityFinding - -_MIN_ROWS = 30 - - -class MinRowsValidator: - name = "min_rows" - max_risk = "low" - uses_network = False - requires = () - - def validate(self, df: pd.DataFrame, policy: object, ctx: object) -> list[QualityFinding]: - findings: list[QualityFinding] = [] - if len(df) < _MIN_ROWS: - findings.append(QualityFinding.create( - severity="warning", - step="plugin", - rule_name="plugin.min_rows", - message=f"only {len(df)} rows (< {_MIN_ROWS}); statistics may be unreliable", - expected_condition=f">= {_MIN_ROWS} rows", - extra={"n_rows": int(len(df))}, - )) - for col in df.columns: - non_null = df[col].dropna() - if len(non_null) > 1 and non_null.nunique() == 1: - findings.append(QualityFinding.create( - severity="info", - step="plugin", - rule_name="plugin.constant_column", - column=str(col), - message=f"column {col!r} is constant", - expected_condition="more than one distinct value", - )) - return findings diff --git a/examples/privacy_anonymization.py b/examples/privacy_anonymization.py deleted file mode 100644 index d0a8079c..00000000 --- a/examples/privacy_anonymization.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Stronger PII detection + reversible / format-preserving anonymization. - -Demonstrates the dependency-free detector, reversible tokenization with a vault, -surrogate format-preserving anonymization, HIPAA/GDPR-tagged audit events, and a -k-anonymity check. Run: - - python examples/privacy_anonymization.py -""" - -import os - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import ( - InMemoryTokenVault, - MaskingRule, - PIIDetectionConfig, - detokenize_value, - tokenize_value, -) - -#: Keyed strategies read their secret from this environment variable. Without a -#: key they use a random per-call key, so output would change on every run. -KEY_ENV = "FRESHDATA_DEMO_KEY" - - -def main() -> None: - # A throwaway demo key when none is set (or it is empty); use a real secret - # in production. - if not os.environ.get(KEY_ENV): - os.environ[KEY_ENV] = "DEMO-KEY-DO-NOT-COMMIT" - df = pd.DataFrame( - { - "patient_note": [ - "Patient MRN# 4451234 dob 1984-02-11, email j.doe@mail.com", - "Contact 555-12-3456 (SSN), card 4111 1111 1111 1111", - ], - "email": ["j.doe@mail.com", "a.smith@mail.com"], - "zip": ["90210", "90210"], - "gender": ["M", "F"], - } - ) - - # 1) Detect PII in free text (raw spans are redacted by default). - scan = fd.detect_pii(df, config=PIIDetectionConfig()) - print("== detection ==") - print(scan.summary(), "\n") - - # 2) Anonymize: tokenize the email column reversibly + scrub free text spans. - rules = ( - MaskingRule( - name="email_token", - columns=("email",), - strategy="tokenize", - reversible=True, - key_env=KEY_ENV, - entity_types=("EMAIL",), - ), - ) - out, report = fd.anonymize(df, rules=rules, detection_config=PIIDetectionConfig()) - print("== anonymized frame ==") - print(out[["patient_note", "email"]].to_string(index=False), "\n") - print(report.summary()) - ev = report.events[0] - print(f"sample event tags: hipaa={ev.hipaa_tag} gdpr={ev.gdpr_tag} risk={ev.risk_level}") - print("report leaks raw PII? ", "j.doe@mail.com" in report.to_json(), "\n") - - # 3) Reversible tokenization round-trip via a vault. - vault = InMemoryTokenVault() - token = tokenize_value("j.doe@mail.com", vault, os.environ[KEY_ENV]) - print(f"token={token} -> detokenized={detokenize_value(token, vault)}\n") - - # 4) Surrogate format-preserving anonymization keeps the shape. - fpe_rule = MaskingRule( - name="ssn_fpe", columns=("zip",), strategy="surrogate", preserve_format=True, - key_env=KEY_ENV, - ) - masked, _ = fd.anonymize(df, rules=(fpe_rule,)) - print("surrogate zip:", masked["zip"].tolist(), "(same length, not crypto FPE)\n") - - # 5) k-anonymity over quasi-identifiers. - k = fd.check_k_anonymity(df, ["zip", "gender"], k=2) - print("== k-anonymity ==") - print(k.summary()) - - -if __name__ == "__main__": - main() diff --git a/examples/schema_drift_monitoring.py b/examples/schema_drift_monitoring.py deleted file mode 100644 index 9820a97b..00000000 --- a/examples/schema_drift_monitoring.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Schema-drift & data-contract monitoring with persisted baselines. - -Build a PII-safe baseline from a trusted dataset, persist it as JSON, then -monitor new batches for schema drift, distribution drift (KS / PSI), contract -violations, and a trust-score quality gate. Run: - - python examples/schema_drift_monitoring.py -""" - -import tempfile -from pathlib import Path - -import numpy as np -import pandas as pd - -import freshdata as fd -from freshdata import ColumnContract, DataContract -from freshdata.enterprise import DriftConfig - - -def trusted() -> pd.DataFrame: - rng = np.random.default_rng(0) - return pd.DataFrame( - { - "customer_id": range(1000), - "age": rng.normal(42, 9, 1000).round(1), - "country": rng.choice(["US", "GB", "FR"], 1000, p=[0.6, 0.3, 0.1]), - "plan": rng.choice(["free", "pro", "enterprise"], 1000, p=[0.7, 0.25, 0.05]), - } - ) - - -def main() -> None: - # A data contract declares the expectations new data must honour. - contract = DataContract( - name="customers", - columns=( - ColumnContract(name="customer_id", dtype="int64", unique=True, nullable=False), - ColumnContract(name="age", dtype="float64", min_value=0, max_value=120), - ColumnContract(name="country", allowed_values=("US", "GB", "FR")), - ColumnContract(name="plan", allowed_values=("free", "pro", "enterprise")), - ), - trust_score_min=70.0, - ) - - # 1) Build + persist a baseline (no raw samples are stored by default). - base = fd.build_baseline(trusted(), name="customers", contract=contract) - path = Path(tempfile.gettempdir()) / "customers.baseline.json" - fd.save_baseline(base, path) - print(f"baseline written to {path} (PII-safe: raw values not stored)\n") - - # 2) A healthy new batch passes. - healthy = trusted() - report = fd.monitor_contract(healthy, baseline_path=path) - print("== healthy batch ==") - print(report.summary(), "\n") - - # 3) A drifted batch: age shifts up, France over-represented, a bad country. - rng = np.random.default_rng(7) - drifted = trusted() - drifted["age"] = rng.normal(60, 9, len(drifted)) # distribution shift -> KS/PSI - drifted["country"] = rng.choice(["US", "GB", "FR"], len(drifted), p=[0.2, 0.2, 0.6]) - drifted.loc[0, "country"] = "ZZ" # contract.allowed_values violation - - report = fd.compare_to_baseline(drifted, fd.load_baseline(path), drift_config=DriftConfig()) - print("== drifted batch ==") - print(report.summary()) - print(f"\npassed={report.passed} errors={report.n_errors} warnings={report.n_warnings}") - - # 4) Baseline-FREE schema diff: explain structural drift the moment a frame - # arrives, with no persisted snapshot. Here an upstream team renamed - # `country` -> `country_code`, dropped `plan`, and added `utm_source`, - # and `age` arrived as text. diff_schema explains all of it *before* any - # repair runs; on_unexpected / on_missing control the gate. - incoming = pd.DataFrame( - { - "customer_id": [1, 2, 3], - "age": ["42", "37", "51"], # dtype drift: text, not float - "country_code": ["US", "GB", "FR"], # rename of `country` - "utm_source": ["g", "fb", "g"], # undeclared - } - ) - incoming.attrs["semantic_types"] = {"country_code": "country"} - contract_sem = DataContract( - name="customers", - columns=( - ColumnContract(name="customer_id", dtype="int64", unique=True, nullable=False), - ColumnContract(name="age", dtype="float64"), - ColumnContract(name="country", semantic_type="country"), - ColumnContract(name="plan", allowed_values=("free", "pro", "enterprise")), - ), - ) - diff = fd.diff_schema(incoming, contract=contract_sem, on_unexpected="warn", on_missing="fail") - print("\n== baseline-free schema diff ==") - print(diff.summary()) - print("\ncategories:", diff.contract_results) - print("\nfindings table:") - print(diff.to_frame()[["check_id", "column", "level", "message"]].to_string(index=False)) - - # The same diff can ride along with a suggested plan for the incoming data. - plan = fd.suggest_plan(incoming, contract=contract_sem, on_missing="fail") - print(f"\nplan carries schema_diff: passed={plan.schema_diff.passed}") - - -if __name__ == "__main__": - main() diff --git a/examples/validation_suite_quickstart.py b/examples/validation_suite_quickstart.py deleted file mode 100644 index 29bc83a9..00000000 --- a/examples/validation_suite_quickstart.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Declarative validation with fd.ValidationSuite β€” validate, never mutate. - -Run: python examples/validation_suite_quickstart.py -""" - -from __future__ import annotations - -import pandas as pd - -import freshdata as fd - -df = pd.DataFrame( - { - "customer_id": ["c1", "c2", "c3", "c3"], # duplicate id - "age": [34, -2, 51, 29], # negative age - "email": ["a@x.com", "bad-email", "c@z.net", "d@w.io"], - "signup": pd.to_datetime(["2021-03-01", "2022-01-15", "2020-07-01", "2023-02-01"]), - "last_seen": pd.to_datetime(["2022-03-01", "2021-06-15", "2024-01-01", "2024-06-01"]), - } -) - -suite = fd.ValidationSuite( - name="customers", - rules=[ - fd.ColumnRule("customer_id", nullable=False, unique=True), - fd.ColumnRule("age", min_value=0, max_value=120), - # mostly=0.75: tolerate up to 25% bad emails as a warning, not a failure - fd.ColumnRule("email", regex=r"[^@]+@[^@]+\.[^@]+", mostly=0.75), - fd.ColumnRule("signup", min_datetime="2019-01-01", max_datetime="2026-01-01"), - ], - cross_column=[fd.CrossColumnRule("signup", "<=", "last_seen")], - min_rows=1, -) - -result = fd.validate(df, suite=suite) -print(f"passed: {result.passed} ({result.n_errors} errors, {result.n_warnings} warnings)") -for finding in result.report.findings: - print(f" [{finding.status}] {finding.check_id}: {finding.message}") - -# Suites are versioned, serializable artifacts β€” commit them next to your data. -suite.save("customers_suite.json") -print("\nsaved to customers_suite.json; validate in CI with:") -print(" freshdata validate data.csv --suite customers_suite.json") - -# Existing DataContract users migrate with one call: -contract = suite.to_contract() -same_suite = fd.ValidationSuite.from_contract(contract) -assert same_suite.to_contract().to_dict() == contract.to_dict() - -# result.raise_if_failed() raises fd.ValidationError β€” CI-friendly. diff --git a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json b/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json deleted file mode 100644 index 53759426..00000000 --- a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json +++ /dev/null @@ -1 +0,0 @@ -{"commit_hash": "f044a5447692b793b4015cfdb57da462e732e334", "env_name": "virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1", "date": 1782583138000, "params": {"arch": "arm64", "cpu": "Apple M2", "machine": "Johnnys-MacBook-Air.local", "num_cpu": "8", "os": "Darwin 24.5.0", "ram": "17179869184", "python": ""}, "python": "", "requirements": {}, "env_vars": {"OPENBLAS_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "OMP_NUM_THREADS": "1"}, "result_columns": ["result", "params", "version", "started_at", "duration", "stats_ci_99_a", "stats_ci_99_b", "stats_q_25", "stats_q_75", "stats_number", "stats_repeat", "samples", "profile"], "results": {"benchmark_pipeline.BenchmarkPipeline.peakmem_full_clean": [[137641984, null, 167673856, 202719232, NaN, 337313792, null, 402472960, 373719040, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "81108b98041660a33f2a45656fb6e5e69684b11e4a5080796bdb1ec0f81488d7", 1782586250728, 33.892], "benchmark_pipeline.BenchmarkPipeline.time_full_clean": [[0.463310957999056, null, 0.056742166001640726, 0.022257166001509177, NaN, 3.293534708001971, null, 0.43859825000254204, 0.17404883400013205, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "7f1f0bf1c404671cf2d850b0558cd09df3a4ad1910015af06f77e6391b341925", 1782586284622, 33.177, [-Infinity, null, -Infinity, -Infinity, null, -Infinity, null, -Infinity, -Infinity, null], [Infinity, null, Infinity, Infinity, null, Infinity, null, Infinity, Infinity, null], [0.46331, null, 0.056742, 0.022257, null, 3.2935, null, 0.4386, 0.17405, null], [0.46331, null, 0.056742, 0.022257, null, 3.2935, null, 0.4386, 0.17405, null], [1, null, 1, 1, null, 1, null, 1, 1, null], [1, null, 1, 1, null, 1, null, 1, 1, null]], "benchmark_pipeline.BenchmarkPipeline.track_output_cols": [[19, null, 19, 19, NaN, 19, null, 19, 19, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "a92633b51ba1c8e8b6ee94cf0bd3497388b752031794f2489d2e91c2da78a781", 1782586317800, 33.135], "benchmark_pipeline.BenchmarkPipeline.track_output_rows": [[10000, null, 10000, 10000, NaN, 100000, null, 100000, 100000, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "59e7aae9329927dc95d6b7c468ff17aefe31f8dae97ae4a8f26c5338d4c5c392", 1782586350936, 33.338], "benchmark_pipeline.BenchmarkPipeline.track_throughput": [[21512.716899315576, null, 181757.7324961564, 455954.7692941329, NaN, 29982.88089633863, null, 230523.16945141667, 376599.6663953862, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "92a13a48f747501a52bbb7b2ec75ec2ba7294747061a896777ebd190eaea0330", 1782586384276, 33.487]}, "durations": {}, "version": 2} \ No newline at end of file diff --git a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json b/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json deleted file mode 100644 index a9605c21..00000000 --- a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "arch": "arm64", - "cpu": "Apple M2", - "machine": "Johnnys-MacBook-Air.local", - "num_cpu": "8", - "os": "Darwin 24.5.0", - "ram": "17179869184", - "version": 1 -} \ No newline at end of file diff --git a/freshdata-benchmarks/.asv/results/benchmarks.json b/freshdata-benchmarks/.asv/results/benchmarks.json deleted file mode 100644 index 8ea46f5b..00000000 --- a/freshdata-benchmarks/.asv/results/benchmarks.json +++ /dev/null @@ -1,2205 +0,0 @@ -{ - "benchmark_columns.BenchmarkColumns.time_drop_columns": { - "code": "class BenchmarkColumns:\n def time_drop_columns(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to drop specific columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_columns(df, self.drop_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define columns for operations\n cols = list(self.df.columns)\n self.rename_mapping = {c: f\"renamed_{c}\" for c in cols[:5]}\n self.drop_cols = cols[-3:]\n self.select_cols = cols[:5]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_columns.BenchmarkColumns.time_drop_columns", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "1ef0230ec3919fe2889753f5eda881d444ad28cbf5bdb3ab204b2c5089645fa2", - "warmup_time": -1 - }, - "benchmark_columns.BenchmarkColumns.time_rename_columns": { - "code": "class BenchmarkColumns:\n def time_rename_columns(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to rename columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.rename_columns(df, self.rename_mapping)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define columns for operations\n cols = list(self.df.columns)\n self.rename_mapping = {c: f\"renamed_{c}\" for c in cols[:5]}\n self.drop_cols = cols[-3:]\n self.select_cols = cols[:5]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_columns.BenchmarkColumns.time_rename_columns", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "b521d65830f775bce1829d2efb24e77dc8479435c39c011682902a824eb38642", - "warmup_time": -1 - }, - "benchmark_columns.BenchmarkColumns.time_select_columns": { - "code": "class BenchmarkColumns:\n def time_select_columns(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to select a subset of columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.select_columns(df, self.select_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define columns for operations\n cols = list(self.df.columns)\n self.rename_mapping = {c: f\"renamed_{c}\" for c in cols[:5]}\n self.drop_cols = cols[-3:]\n self.select_cols = cols[:5]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_columns.BenchmarkColumns.time_select_columns", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "7c37c123056d3186b7b95e1b81d65597c9341c640b457de8da63638664e75b63", - "warmup_time": -1 - }, - "benchmark_duplicates.BenchmarkDuplicates.peakmem_detect_duplicates": { - "code": "class BenchmarkDuplicates:\n def peakmem_detect_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when detecting duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_duplicates.BenchmarkDuplicates.peakmem_detect_duplicates", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "e7a7c5ae31094bd4eb62fe29a14ddfc0b79fc5dc0ec4961ef7996e3626707389" - }, - "benchmark_duplicates.BenchmarkDuplicates.peakmem_drop_duplicates": { - "code": "class BenchmarkDuplicates:\n def peakmem_drop_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when dropping duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_duplicates.BenchmarkDuplicates.peakmem_drop_duplicates", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "f9c20916bd270a1f589e27741aef986b9dcb22930af88a9019cd8a9121b7f33d" - }, - "benchmark_duplicates.BenchmarkDuplicates.time_detect_duplicates": { - "code": "class BenchmarkDuplicates:\n def time_detect_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_duplicates.BenchmarkDuplicates.time_detect_duplicates", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "dcbc081cfff3b48875e14e33e2adf11992491b9fb865a9c7558c7f9c3050fea9", - "warmup_time": -1 - }, - "benchmark_duplicates.BenchmarkDuplicates.time_drop_duplicates": { - "code": "class BenchmarkDuplicates:\n def time_drop_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to drop duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_duplicates.BenchmarkDuplicates.time_drop_duplicates", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "1e3cbf23f14e4784563c39c4ad52fe5f28664d72596257d1ae1ca1b340484683", - "warmup_time": -1 - }, - "benchmark_duplicates.BenchmarkDuplicates.track_duplicate_count": { - "code": "class BenchmarkDuplicates:\n def track_duplicate_count(self, n_rows: int, library: str) -> int:\n \"\"\"Track the number of duplicate rows dropped.\"\"\"\n df = fresh_copy(self.df)\n start_rows = len(df)\n result = self.adapter.drop_duplicates(df)\n \n # Approximate counting since libraries return different formats\n try:\n return start_rows - len(result)\n except Exception:\n return -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_duplicates.BenchmarkDuplicates.track_duplicate_count", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "rows", - "version": "b66bd2c35387849867b7fd80fdbfb3c98dfa2522f2d4b1436b3e3bf977a3f957" - }, - "benchmark_encoding.BenchmarkEncoding.peakmem_label_encode": { - "code": "class BenchmarkEncoding:\n def peakmem_label_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during label encoding.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.label_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_encoding.BenchmarkEncoding.peakmem_label_encode", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "0c3d6aaad98acca22ce40446d30d6984ed9255d6753ba4c0c853d5123114f1ce" - }, - "benchmark_encoding.BenchmarkEncoding.peakmem_onehot_encode": { - "code": "class BenchmarkEncoding:\n def peakmem_onehot_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during one-hot encoding.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.onehot_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_encoding.BenchmarkEncoding.peakmem_onehot_encode", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "19c28b9d44f9ed8ec617db25d7caf56bb33793a3b989db69ba05a0f571d9125d" - }, - "benchmark_encoding.BenchmarkEncoding.time_label_encode": { - "code": "class BenchmarkEncoding:\n def time_label_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform label (ordinal) encoding.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.label_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_encoding.BenchmarkEncoding.time_label_encode", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "02519fa9019f2df7e033a48ab0ff2575050cf645b6ad0ccf8f7de981f8b82a4f", - "warmup_time": -1 - }, - "benchmark_encoding.BenchmarkEncoding.time_onehot_encode": { - "code": "class BenchmarkEncoding:\n def time_onehot_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform one-hot encoding on categorical columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.onehot_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_encoding.BenchmarkEncoding.time_onehot_encode", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "bf93a7f4cd8fe6dd729e1df93cead6f815de0cb2760ff956b596dcd63a0bdac8", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_detect_outliers": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_detect_outliers(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during IQR outlier detection.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_detect_outliers", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "59f8db92583efe5640ec4fd0faeb5f06ad8d9d9fb9f6e78a9b79995d577cdf3c" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_full_pipeline": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_full_pipeline(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory for the complete preprocessing pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_full_pipeline", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "db2d7c635f844bda6f231bacb0275d2b37157efbc314d2aeb426bdb660255276" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_group_aggregations": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_group_aggregations(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory for group aggregations.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_group_aggregations", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "bd744ee28a6c6adef0030c3cc8bdf1f119a1deac45faf873b3193ab354a376df" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_handle_missing": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_handle_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when filling missing values.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_handle_missing", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "da80e7c97ed3d9f46a0bc8dab83b3379131b256619c7fcb5b5d4a483be18cbf1" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_load_csv": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_load_csv(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when loading a CSV file.\"\"\"\n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n fd.clean(df, strategy=\"conservative\", preserve_original=True)\n else:\n import pandas as pd\n pd.read_csv(self.csv_path)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_load_csv", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "3934bb8774d108a1a8425562cb6f86a62b8fca0a95e3d5b80eec1f204b6c2dba" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_resolve_duplicates": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_resolve_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when dropping duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_resolve_duplicates", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "4a419465cdfaa7d833dc191d228d30deb5b44859f72a6f1d56fa08aa0b54e53a" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_detect_outliers": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_detect_outliers(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect outliers using IQR method.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_detect_outliers", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "e4675f916ceffcedee506ec98fdcbdf185e2b5fce1316780d95b8f69b3ca737e", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_full_pipeline": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_full_pipeline(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for the complete preprocessing pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_full_pipeline", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "1b452d2cfb3fa3fc504d5d3d981d03354065e80384b5f37dc07f1d06e26f91de", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_group_aggregations": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_group_aggregations(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for multi-column group aggregations.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_group_aggregations", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "c50dfafaefc6529bbca4d22386652bb1afe1ef30a12d482e971948dbebdd7e93", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_handle_missing": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_handle_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with column means.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_handle_missing", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "8837b26596f861ce61c5c6569da5f3d7e7ecc5a74a45770db79497a50ef6a895", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_load_csv": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_load_csv(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to load a CSV file and infer types.\"\"\"\n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n fd.clean(df, strategy=\"conservative\", preserve_original=True)\n else:\n import pandas as pd\n pd.read_csv(self.csv_path)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_load_csv", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "8181e5190e9331fd5dd8fae4703c10b5e3b6c59b677ef8073b31c2425da3ab51", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_resolve_duplicates": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_resolve_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect and drop duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_resolve_duplicates", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "0954a66c2198a5cae859a059a4444297b46a7125b936e2de75399c8766c66b8e", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.track_pipeline_throughput": { - "code": "class BenchmarkFreshDataVsPandas:\n def track_pipeline_throughput(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate rows processed per second for the full pipeline.\"\"\"\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.full_clean(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.track_pipeline_throughput", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows/s", - "version": "8031cd675584aca5f7bde0b00338dc80473b17fcdb9744506ef2ee61304a37bd" - }, - "benchmark_groupagg.BenchmarkGroupAgg.peakmem_group_agg_multi": { - "code": "class BenchmarkGroupAgg:\n def peakmem_group_agg_multi(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory for multi-column groupby aggregation.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "name": "benchmark_groupagg.BenchmarkGroupAgg.peakmem_group_agg_multi", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "713c2e1f22b1e4c6905c7c35c209e03539e56fdbd4119436f0815ad71469ac23" - }, - "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_multi": { - "code": "class BenchmarkGroupAgg:\n def time_group_agg_multi(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for multi-column groupby with multiple agg functions.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_multi", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "d3a2f0459cede798766b70f2d3eadd81ecd4b26461b19d99575cb156e33b1da8", - "warmup_time": -1 - }, - "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_single": { - "code": "class BenchmarkGroupAgg:\n def time_group_agg_single(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for single-column groupby with mean aggregation.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_single(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_single", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "63d3db36ff71f3a8b0b02333de6f05e4c6d5f9bb567d30509aa2a119d3e5ae77", - "warmup_time": -1 - }, - "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_transform": { - "code": "class BenchmarkGroupAgg:\n def time_group_agg_transform(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for groupby transform (centering within groups).\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_transform(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_transform", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "72e4c12736d638b8666a71e6e895fdc3d9237cd7bdcc10738d4acd115d8de4f9", - "warmup_time": -1 - }, - "benchmark_groupagg.BenchmarkGroupAgg.track_throughput_group_agg": { - "code": "class BenchmarkGroupAgg:\n def track_throughput_group_agg(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate rows processed per second for group aggregation.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.group_agg_multi(df, self.numeric_cols)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "name": "benchmark_groupagg.BenchmarkGroupAgg.track_throughput_group_agg", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "rows/s", - "version": "f419a8f7f33e31c7d34037e53a9d01f6af914f4a85b84dd089c9f41ca63d13c3" - }, - "benchmark_io.BenchmarkIO.peakmem_csv_read_clean": { - "code": "class BenchmarkIO:\n def peakmem_csv_read_clean(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when reading and cleaning a CSV.\"\"\"\n self.time_csv_read_clean(cache, n_rows, library)\n\n def setup(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Check library availability and locate files.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n elif library == \"pandas\":\n try:\n import pandas # noqa: F401\n except ImportError:\n raise NotImplementedError(\"pandas is not available\")\n elif library == \"polars\":\n try:\n import polars # noqa: F401\n except ImportError:\n raise NotImplementedError(\"polars is not available\")\n \n self.csv_path = cache[\"files\"][n_rows][\"csv\"]\n self.parquet_path = cache[\"files\"][n_rows][\"parquet\"]\n \n if not self.csv_path:\n raise NotImplementedError(\"CSV file not generated\")\n \n gc_collect()\n\n def setup_cache(self):\n \"\"\"Generate test files once per class to avoid disk overhead in timing.\"\"\"\n import pandas as pd\n tmpdir = tempfile.mkdtemp()\n \n # We only generate files for the requested sizes in params\n sizes = [10_000, 100_000, 1_000_000]\n files = {}\n \n for size in sizes:\n df = get_dataset(size)\n csv_path = os.path.join(tmpdir, f\"test_{size}.csv\")\n parquet_path = os.path.join(tmpdir, f\"test_{size}.parquet\")\n \n df.to_csv(csv_path, index=False)\n try:\n df.to_parquet(parquet_path, engine=\"pyarrow\")\n files[size] = {\"csv\": csv_path, \"parquet\": parquet_path}\n except Exception:\n files[size] = {\"csv\": csv_path, \"parquet\": None}\n \n return {\"tmpdir\": tmpdir, \"files\": files}", - "name": "benchmark_io.BenchmarkIO.peakmem_csv_read_clean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "setup_cache_key": "benchmark_io:24", - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "220ea6335973d89fcde320c335958b551b8d530b3235adabbc8620003d0df473" - }, - "benchmark_io.BenchmarkIO.time_csv_read_clean": { - "code": "class BenchmarkIO:\n def time_csv_read_clean(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Measure time to read a CSV and run a full clean.\"\"\"\n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n fd.clean(df, strategy=\"balanced\", preserve_original=True)\n elif library == \"pandas\":\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n df = df.dropna().drop_duplicates()\n elif library == \"polars\":\n import polars as pl\n df = pl.read_csv(self.csv_path)\n df = df.drop_nulls().unique()\n\n def setup(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Check library availability and locate files.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n elif library == \"pandas\":\n try:\n import pandas # noqa: F401\n except ImportError:\n raise NotImplementedError(\"pandas is not available\")\n elif library == \"polars\":\n try:\n import polars # noqa: F401\n except ImportError:\n raise NotImplementedError(\"polars is not available\")\n \n self.csv_path = cache[\"files\"][n_rows][\"csv\"]\n self.parquet_path = cache[\"files\"][n_rows][\"parquet\"]\n \n if not self.csv_path:\n raise NotImplementedError(\"CSV file not generated\")\n \n gc_collect()\n\n def setup_cache(self):\n \"\"\"Generate test files once per class to avoid disk overhead in timing.\"\"\"\n import pandas as pd\n tmpdir = tempfile.mkdtemp()\n \n # We only generate files for the requested sizes in params\n sizes = [10_000, 100_000, 1_000_000]\n files = {}\n \n for size in sizes:\n df = get_dataset(size)\n csv_path = os.path.join(tmpdir, f\"test_{size}.csv\")\n parquet_path = os.path.join(tmpdir, f\"test_{size}.parquet\")\n \n df.to_csv(csv_path, index=False)\n try:\n df.to_parquet(parquet_path, engine=\"pyarrow\")\n files[size] = {\"csv\": csv_path, \"parquet\": parquet_path}\n except Exception:\n files[size] = {\"csv\": csv_path, \"parquet\": None}\n \n return {\"tmpdir\": tmpdir, \"files\": files}", - "min_run_count": 2, - "name": "benchmark_io.BenchmarkIO.time_csv_read_clean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "setup_cache_key": "benchmark_io:24", - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "3961585e316ff5329a6f08bf6537a21a3026e4f97237977593dd8dc814f04b22", - "warmup_time": -1 - }, - "benchmark_io.BenchmarkIO.time_parquet_read_clean": { - "code": "class BenchmarkIO:\n def time_parquet_read_clean(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Measure time to read a Parquet file and run a full clean.\"\"\"\n if not self.parquet_path:\n raise NotImplementedError(\"Parquet missing\")\n \n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_parquet(self.parquet_path)\n fd.clean(df, strategy=\"balanced\", preserve_original=True)\n elif library == \"pandas\":\n import pandas as pd\n df = pd.read_parquet(self.parquet_path)\n df = df.dropna().drop_duplicates()\n elif library == \"polars\":\n import polars as pl\n df = pl.read_parquet(self.parquet_path)\n df = df.drop_nulls().unique()\n\n def setup(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Check library availability and locate files.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n elif library == \"pandas\":\n try:\n import pandas # noqa: F401\n except ImportError:\n raise NotImplementedError(\"pandas is not available\")\n elif library == \"polars\":\n try:\n import polars # noqa: F401\n except ImportError:\n raise NotImplementedError(\"polars is not available\")\n \n self.csv_path = cache[\"files\"][n_rows][\"csv\"]\n self.parquet_path = cache[\"files\"][n_rows][\"parquet\"]\n \n if not self.csv_path:\n raise NotImplementedError(\"CSV file not generated\")\n \n gc_collect()\n\n def setup_cache(self):\n \"\"\"Generate test files once per class to avoid disk overhead in timing.\"\"\"\n import pandas as pd\n tmpdir = tempfile.mkdtemp()\n \n # We only generate files for the requested sizes in params\n sizes = [10_000, 100_000, 1_000_000]\n files = {}\n \n for size in sizes:\n df = get_dataset(size)\n csv_path = os.path.join(tmpdir, f\"test_{size}.csv\")\n parquet_path = os.path.join(tmpdir, f\"test_{size}.parquet\")\n \n df.to_csv(csv_path, index=False)\n try:\n df.to_parquet(parquet_path, engine=\"pyarrow\")\n files[size] = {\"csv\": csv_path, \"parquet\": parquet_path}\n except Exception:\n files[size] = {\"csv\": csv_path, \"parquet\": None}\n \n return {\"tmpdir\": tmpdir, \"files\": files}", - "min_run_count": 2, - "name": "benchmark_io.BenchmarkIO.time_parquet_read_clean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "setup_cache_key": "benchmark_io:24", - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "f1b5ea830e1455fb6167dad313755185de43b4e43399321c0bdc1c00e3269d5f", - "warmup_time": -1 - }, - "benchmark_memory.BenchmarkMemory.peakmem_full_clean": { - "code": "class BenchmarkMemory:\n def peakmem_full_clean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak RSS memory used during the full cleaning pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.peakmem_full_clean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "6c95e4784d795e0e0df8601858ac885b866d0fc210bc7bb3381100f321798cf3" - }, - "benchmark_memory.BenchmarkMemory.track_memory_after": { - "code": "class BenchmarkMemory:\n def track_memory_after(self, n_rows: int, library: str) -> float:\n \"\"\"Track the deep memory usage of the cleaned output DataFrame (in MB).\"\"\"\n df = fresh_copy(self.df)\n result = self.adapter.full_clean(df)\n try:\n import pandas as pd\n if isinstance(result, pd.DataFrame):\n return result.memory_usage(deep=True).sum() / (1024 * 1024)\n else:\n # Approximate memory for Polars\n return result.estimated_size() / (1024 * 1024)\n except Exception:\n return -1.0\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.track_memory_after", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "MB", - "version": "db807e615b7760ff17a1bd6b0496a2b2761fd0141af5a5506c742c90e494f90f" - }, - "benchmark_memory.BenchmarkMemory.track_memory_before": { - "code": "class BenchmarkMemory:\n def track_memory_before(self, n_rows: int, library: str) -> float:\n \"\"\"Track the deep memory usage of the uncleaned input DataFrame (in MB).\"\"\"\n df = fresh_copy(self.df)\n import pandas as pd\n if isinstance(df, pd.DataFrame):\n return df.memory_usage(deep=True).sum() / (1024 * 1024)\n return -1.0\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.track_memory_before", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "MB", - "version": "17667be34c2072042958374f1cfdf2e31d1864a01c36a781f042776379cb26f3" - }, - "benchmark_memory.BenchmarkMemory.track_memory_ratio": { - "code": "class BenchmarkMemory:\n def track_memory_ratio(self, n_rows: int, library: str) -> float:\n \"\"\"Track the ratio of output memory / input memory.\n Values < 1.0 indicate memory savings (e.g. via downcasting).\n \"\"\"\n df = fresh_copy(self.df)\n \n import pandas as pd\n if not isinstance(df, pd.DataFrame):\n return float('nan')\n \n mem_before = df.memory_usage(deep=True).sum()\n if mem_before == 0:\n return float('nan')\n \n result = self.adapter.full_clean(df)\n \n try:\n if isinstance(result, pd.DataFrame):\n mem_after = result.memory_usage(deep=True).sum()\n else:\n mem_after = result.estimated_size()\n return float(mem_after) / float(mem_before)\n except Exception:\n return float('nan')\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.track_memory_ratio", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "ratio", - "version": "b32c33f3ef5053946b26aa1bfb2802e6314caf61e56f80d194c9f2730d90014e" - }, - "benchmark_missing.BenchmarkMissing.peakmem_drop_missing": { - "code": "class BenchmarkMissing:\n def peakmem_drop_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when dropping rows with missing values.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_missing_rows(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.peakmem_drop_missing", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "5658150457480391fc62d023fea1fca88cb91519b7db29adffe5545ac48870b4" - }, - "benchmark_missing.BenchmarkMissing.peakmem_fill_mean": { - "code": "class BenchmarkMissing:\n def peakmem_fill_mean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when filling missing values with column mean.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.peakmem_fill_mean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "cf3ce857230bef6dd6eefecc6f9d1ff91ec96191867bdd98ae50215592b65276" - }, - "benchmark_missing.BenchmarkMissing.peakmem_fill_median": { - "code": "class BenchmarkMissing:\n def peakmem_fill_median(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when filling missing values with column median.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_median(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.peakmem_fill_median", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "6ed8e7abf094dec466fb2835bcd3f15a1db78c52f63c4b7ead5a6787c7aad95b" - }, - "benchmark_missing.BenchmarkMissing.time_drop_missing": { - "code": "class BenchmarkMissing:\n def time_drop_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to drop rows with missing values.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_missing_rows(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_drop_missing", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "f693acd4497aa6e60c4f635f92eea15f12d7eb392413dd6bcf7f4edb10f1531b", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_bfill": { - "code": "class BenchmarkMissing:\n def time_fill_bfill(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to backward-fill missing values.\n Supported by pandas, polars, and freshdata.\n \"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_bfill(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_bfill", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "46d2e36a856880108de0335b024aadf1ea87bc3461467cb7b32634817de76428", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_ffill": { - "code": "class BenchmarkMissing:\n def time_fill_ffill(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to forward-fill missing values.\n Supported by pandas, polars, and freshdata.\n \"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_ffill(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_ffill", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e91e9f73b924d058d1f132cb5c7b5c4bfcca5c1c80ddd9ddde9567203c9c0505", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_mean": { - "code": "class BenchmarkMissing:\n def time_fill_mean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with the column mean.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_mean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "c44170e34beeefce94fdbc75618c6ba9b33ca94d48f65c43f05f980c6c0911e6", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_median": { - "code": "class BenchmarkMissing:\n def time_fill_median(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with the column median.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_median(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_median", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "be041bafa51e925515bfa80e8bd76e4b7dab9b0002f2f3012c06f03bbc634da3", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_mode": { - "code": "class BenchmarkMissing:\n def time_fill_mode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with the column mode.\n Supported by pandas, sklearn, and freshdata.\n \"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mode(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_mode", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "8e021a865af521e03e7099d04d7f92db8f92c8cb935850f9d6e39809d0d4f9ce", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.track_throughput_drop_missing": { - "code": "class BenchmarkMissing:\n def track_throughput_drop_missing(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate rows processed per second for drop_missing.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.drop_missing_rows(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.track_throughput_drop_missing", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "rows/s", - "version": "37452d8baf322bbcf65b17c133348dd250a173a2585fcb15bbc021bb561e6afa" - }, - "benchmark_outliers.BenchmarkOutliers.peakmem_detect_iqr": { - "code": "class BenchmarkOutliers:\n def peakmem_detect_iqr(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during IQR outlier detection.\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_outliers.BenchmarkOutliers.peakmem_detect_iqr", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "4396e9885b02a2e7c8bc7eccdc7b6da09fc8f80f902a0f112eb9fdf3e4c69c13" - }, - "benchmark_outliers.BenchmarkOutliers.peakmem_detect_zscore": { - "code": "class BenchmarkOutliers:\n def peakmem_detect_zscore(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during Z-score outlier detection.\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_zscore(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_outliers.BenchmarkOutliers.peakmem_detect_zscore", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "67f0e910566689b4334cbebca0323f19dbfc66cb7d1dd269a9114920490d5e0f" - }, - "benchmark_outliers.BenchmarkOutliers.time_detect_iqr": { - "code": "class BenchmarkOutliers:\n def time_detect_iqr(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect outliers using the Interquartile Range (IQR) method.\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_outliers.BenchmarkOutliers.time_detect_iqr", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "996e7904e786e9e4985a5b556c5b8a562118b0463da57ec7c21c289ae3eb15df", - "warmup_time": -1 - }, - "benchmark_outliers.BenchmarkOutliers.time_detect_zscore": { - "code": "class BenchmarkOutliers:\n def time_detect_zscore(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect outliers using Z-score (standard deviations).\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_zscore(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_outliers.BenchmarkOutliers.time_detect_zscore", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "2a16f91a3e9cbb6e3def523c26956be352e590dabf96624eff0e2d41ca94c255", - "warmup_time": -1 - }, - "benchmark_outliers.BenchmarkOutliers.track_outlier_count_iqr": { - "code": "class BenchmarkOutliers:\n def track_outlier_count_iqr(self, n_rows: int, library: str) -> int:\n \"\"\"Track the total number of outliers detected by IQR across all numeric columns.\"\"\"\n df = self._get_target_df(library)\n result = self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n \n outlier_count = 0\n try:\n import pandas as pd\n if isinstance(result, pd.DataFrame):\n outlier_cols = [c for c in result.columns if c.endswith(\"_outlier\")]\n for col in outlier_cols:\n outlier_count += int(result[col].sum())\n except Exception:\n pass\n \n return outlier_count if outlier_count > 0 else -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_outliers.BenchmarkOutliers.track_outlier_count_iqr", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "outliers", - "version": "a7ae00c6743aa0af26ea43a7fb3de6e90af3797db9fea27479fd67c49e9c22fd" - }, - "benchmark_pipeline.BenchmarkPipeline.peakmem_full_clean": { - "code": "class BenchmarkPipeline:\n def peakmem_full_clean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory used during the full cleaning pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.peakmem_full_clean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "81108b98041660a33f2a45656fb6e5e69684b11e4a5080796bdb1ec0f81488d7" - }, - "benchmark_pipeline.BenchmarkPipeline.time_full_clean": { - "code": "class BenchmarkPipeline:\n def time_full_clean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to run the complete data cleaning pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_pipeline.BenchmarkPipeline.time_full_clean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "7f1f0bf1c404671cf2d850b0558cd09df3a4ad1910015af06f77e6391b341925", - "warmup_time": -1 - }, - "benchmark_pipeline.BenchmarkPipeline.track_output_cols": { - "code": "class BenchmarkPipeline:\n def track_output_cols(self, n_rows: int, library: str) -> int:\n \"\"\"Track the number of columns remaining after cleaning.\"\"\"\n df = fresh_copy(self.df)\n result = self.adapter.full_clean(df)\n try:\n import pandas as pd\n if isinstance(result, pd.DataFrame):\n return len(result.columns)\n return len(getattr(result, \"columns\", []))\n except Exception:\n return -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.track_output_cols", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "cols", - "version": "a92633b51ba1c8e8b6ee94cf0bd3497388b752031794f2489d2e91c2da78a781" - }, - "benchmark_pipeline.BenchmarkPipeline.track_output_rows": { - "code": "class BenchmarkPipeline:\n def track_output_rows(self, n_rows: int, library: str) -> int:\n \"\"\"Track the number of rows remaining after cleaning.\"\"\"\n df = fresh_copy(self.df)\n result = self.adapter.full_clean(df)\n try:\n return len(result)\n except Exception:\n return -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.track_output_rows", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows", - "version": "59e7aae9329927dc95d6b7c468ff17aefe31f8dae97ae4a8f26c5338d4c5c392" - }, - "benchmark_pipeline.BenchmarkPipeline.track_throughput": { - "code": "class BenchmarkPipeline:\n def track_throughput(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate pipeline throughput in rows per second.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.full_clean(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.track_throughput", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows/s", - "version": "92a13a48f747501a52bbb7b2ec75ec2ba7294747061a896777ebd190eaea0330" - }, - "benchmark_scaling.BenchmarkScaling.peakmem_minmax_scale": { - "code": "class BenchmarkScaling:\n def peakmem_minmax_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during MinMaxScaler.\"\"\"\n df = self._get_target_df(library)\n self.adapter.minmax_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_scaling.BenchmarkScaling.peakmem_minmax_scale", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "f60970d8c16570d05f06c985b45fb26ddf6fc0f02fc9827dc11d45e2c9708ee3" - }, - "benchmark_scaling.BenchmarkScaling.peakmem_standard_scale": { - "code": "class BenchmarkScaling:\n def peakmem_standard_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during StandardScaler.\"\"\"\n df = self._get_target_df(library)\n self.adapter.standard_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_scaling.BenchmarkScaling.peakmem_standard_scale", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "471e946b8c7f3b5fc77b22e062d59df81ea9efb9b1e8ebaba9931d3864dae3f5" - }, - "benchmark_scaling.BenchmarkScaling.time_minmax_scale": { - "code": "class BenchmarkScaling:\n def time_minmax_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to apply MinMaxScaler (0-1 normalization).\"\"\"\n df = self._get_target_df(library)\n self.adapter.minmax_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_scaling.BenchmarkScaling.time_minmax_scale", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e9acb509f1b0116332ed0a50b63a00fbf27c900ee02eee04ddd6181ee32ab20c", - "warmup_time": -1 - }, - "benchmark_scaling.BenchmarkScaling.time_standard_scale": { - "code": "class BenchmarkScaling:\n def time_standard_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to apply StandardScaler (Z-score normalization).\"\"\"\n df = self._get_target_df(library)\n self.adapter.standard_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_scaling.BenchmarkScaling.time_standard_scale", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "36d85cddde3f92d9b9feb5852dd9dc1ffa3a616a7b5d657037e0f350e2ba84b4", - "warmup_time": -1 - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.peakmem_clean_scaling": { - "code": "class BenchmarkScalingCurves:\n def peakmem_clean_scaling(self, n_rows: int, library: str) -> None:\n \"\"\"Peak memory of the full pipeline to map out scaling curves.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.peakmem_clean_scaling", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "416f495cb0534cf4534cc5b7124f57796aadcce6089f7d031caae87368a37523" - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.time_clean_scaling": { - "code": "class BenchmarkScalingCurves:\n def time_clean_scaling(self, n_rows: int, library: str) -> None:\n \"\"\"Time the full pipeline to map out scaling curves.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.time_clean_scaling", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "17caf347aba692e8c28eb7a220f46ab2403cdda80f665a000d09cf57c38f822e", - "warmup_time": -1 - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.track_speedup_vs_pandas": { - "code": "class BenchmarkScalingCurves:\n def track_speedup_vs_pandas(self, n_rows: int, library: str) -> float:\n \"\"\"Speedup relative to Pandas for the full cleaning pipeline.\"\"\"\n import time as _time\n from benchmarks.utils.library_wrappers import get_adapter\n \n pd_adapter = get_adapter(\"pandas\")\n if not pd_adapter.available:\n return float(\"nan\")\n \n # Time pandas baseline\n df_pd = fresh_copy(self.df)\n start = _time.perf_counter()\n pd_adapter.full_clean(df_pd)\n pandas_time = _time.perf_counter() - start\n \n if pandas_time <= 0:\n return float(\"nan\")\n \n # If library IS pandas, speedup is exactly 1.0\n if library == \"pandas\":\n return 1.0\n \n # Time target library\n df_lib = fresh_copy(self.df)\n start = _time.perf_counter()\n self.adapter.full_clean(df_lib)\n library_time = _time.perf_counter() - start\n \n return pandas_time / library_time if library_time > 0 else float(\"nan\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.track_speedup_vs_pandas", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "ratio", - "version": "c543d6534ceca6f519d2fa1d4d1429634adef8c333ef8daef4495ec2082c817a" - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.track_throughput": { - "code": "class BenchmarkScalingCurves:\n def track_throughput(self, n_rows: int, library: str) -> float:\n \"\"\"Rows per second at each dataset size.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.full_clean(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.track_throughput", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows/s", - "version": "770225419fb4d828759ead0fe3b000c528bf0ff3d45b6810fb5cf90fe1a965ae" - }, - "benchmark_strings.BenchmarkStrings.peakmem_lowercase": { - "code": "class BenchmarkStrings:\n def peakmem_lowercase(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when converting to lowercase.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.to_lowercase(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_strings.BenchmarkStrings.peakmem_lowercase", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "41cc942e350b7f59e3f5711568d9069dd9f7b4954bc85631af8511b23672a5ed" - }, - "benchmark_strings.BenchmarkStrings.peakmem_trim_whitespace": { - "code": "class BenchmarkStrings:\n def peakmem_trim_whitespace(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when trimming whitespace.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.trim_whitespace(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_strings.BenchmarkStrings.peakmem_trim_whitespace", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "c11376c97e0665271b8c411af2ee530dbceecc52b98a4c02399bf1be2d625bdb" - }, - "benchmark_strings.BenchmarkStrings.time_lowercase": { - "code": "class BenchmarkStrings:\n def time_lowercase(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to convert string columns to lowercase.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.to_lowercase(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_lowercase", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e1422cecb631554cb13468abb951d9b623f8f0aebbf64d22906f3fe5af15af3b", - "warmup_time": -1 - }, - "benchmark_strings.BenchmarkStrings.time_regex_replace": { - "code": "class BenchmarkStrings:\n def time_regex_replace(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform a regex replacement on string columns.\"\"\"\n df = fresh_copy(self.df)\n pattern = r'[^a-zA-Z0-9 ]'\n replacement = ''\n self.adapter.regex_replace(df, self.string_cols, pattern, replacement)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_regex_replace", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "3c22dd8000aaaf9f4cffe3a911e7847a52fb672d9cd8e8fdf7ec8f3a2454248f", - "warmup_time": -1 - }, - "benchmark_strings.BenchmarkStrings.time_trim_whitespace": { - "code": "class BenchmarkStrings:\n def time_trim_whitespace(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to trim whitespace from all string columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.trim_whitespace(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_trim_whitespace", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "668d40b2e253d03501067cd7c7da7335686200cd0e38d8c6b2bed0ca3fc9c9a2", - "warmup_time": -1 - }, - "benchmark_strings.BenchmarkStrings.time_uppercase": { - "code": "class BenchmarkStrings:\n def time_uppercase(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to convert string columns to uppercase.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.to_uppercase(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_uppercase", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "2e83fff4e7eaed8db55ba4f5fd5d12cedf09f6c9dcb852c0b3119906cadc137d", - "warmup_time": -1 - }, - "benchmark_types.BenchmarkTypes.peakmem_optimize_dtypes": { - "code": "class BenchmarkTypes:\n def peakmem_optimize_dtypes(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during global type optimization.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.optimize_dtypes(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_types.BenchmarkTypes.peakmem_optimize_dtypes", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "de95624a9e886038f6aac8e52c70a5050a0731ee674d837c5c0d541246b05aad" - }, - "benchmark_types.BenchmarkTypes.time_convert_datetime": { - "code": "class BenchmarkTypes:\n def time_convert_datetime(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to parse datetime strings.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.convert_datetime(df, self.date_targets)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_types.BenchmarkTypes.time_convert_datetime", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "d6a0ac396a24e5d5b1cafc8ff4d1097d330c0dbcd4a6ae179ede206e1e764e61", - "warmup_time": -1 - }, - "benchmark_types.BenchmarkTypes.time_convert_numeric": { - "code": "class BenchmarkTypes:\n def time_convert_numeric(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to convert string columns to numeric types.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.convert_numeric(df, self.numeric_targets)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_types.BenchmarkTypes.time_convert_numeric", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "0347b658e9da5a7eafbf4403928f8f2ad13331efede310a22dba0fca63596100", - "warmup_time": -1 - }, - "benchmark_types.BenchmarkTypes.time_optimize_dtypes": { - "code": "class BenchmarkTypes:\n def time_optimize_dtypes(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform global type optimization (downcasting).\"\"\"\n df = fresh_copy(self.df)\n self.adapter.optimize_dtypes(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_types.BenchmarkTypes.time_optimize_dtypes", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "a3932e05a6be05c68f9c6c8e2a3a98dd4c2f329bb0d5dc890777a2b525a25733", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_null_check": { - "code": "class BenchmarkValidation:\n def time_null_check(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to check for nulls across all columns.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n # FreshData profile includes null checks\n fd.profile(df)\n else:\n _ = df.isnull().any()\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_null_check", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "60b8361fdd17459219a658ea0fe62692b2a1d7d00a8ccbf50e19023e6ef61c95", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_profile_validation": { - "code": "class BenchmarkValidation:\n def time_profile_validation(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to run full data profiling/validation.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n fd.profile(df)\n else: # pandas manual profile\n import pandas as pd\n _ = df.dtypes\n _ = df.isnull().sum()\n _ = df.nunique()\n _ = df.describe(include='all')\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_profile_validation", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "a9286be5fb51a41517b1f12045a68c11b29e475dcc741e981627ae722109d5b6", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_range_check": { - "code": "class BenchmarkValidation:\n def time_range_check(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to check numeric ranges.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n fd.profile(df)\n else:\n import pandas as pd\n numeric_df = df.select_dtypes(include=['number'])\n if not numeric_df.empty:\n _ = numeric_df.min()\n _ = numeric_df.max()\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_range_check", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e72e005674e34f0bff4fb322e32a5d1ae3d631632fe0f03b8aaad361b58d9cc7", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_type_check": { - "code": "class BenchmarkValidation:\n def time_type_check(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to validate data types.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n fd.profile(df)\n else:\n _ = df.dtypes\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_type_check", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "ae470f6b26cf801ffc48a826cd4101eb8d2ef0da14d57c5cf885ff7fa062d621", - "warmup_time": -1 - }, - "version": 2 -} \ No newline at end of file diff --git a/freshdata-benchmarks/METHODOLOGY.md b/freshdata-benchmarks/METHODOLOGY.md deleted file mode 100644 index badc9fd1..00000000 --- a/freshdata-benchmarks/METHODOLOGY.md +++ /dev/null @@ -1,48 +0,0 @@ -# Benchmarking Methodology - -This document outlines the principles used to ensure fairness and reproducibility across the FreshData ASV benchmark suite. - -## 1. Fairness Guarantees - -Comparing libraries with entirely different architectures (e.g. Pandas eager execution vs. Polars lazy evaluation vs. Scikit-learn estimator arrays) requires strict fairness controls. - -### Identical Inputs (`fresh_copy`) -Every library operates on the exact same dataset. The dataset is generated once in the `setup()` block using a fixed random seed. -Crucially, before timing begins, the benchmark calls `df.copy(deep=True)`. This prevents one library's in-place mutations from affecting subsequent runs or giving an unfair cache advantage. - -### Isolation (`gc_collect`) -Garbage collection is explicitly triggered before each benchmark run to ensure residual objects from previous runs do not skew peak memory (`peakmem_`) measurements. - -### Equivalent Operations -The benchmark measures *semantic* equivalence. The adapters in `benchmarks/utils/library_wrappers.py` map operations. If a library natively supports an operation (e.g. `polars.drop_nulls()`), it is used. If it requires manual chaining (e.g. `pandas.isnull().any()`), the most idiomatic pandas chain is used. - -## 2. Statistical Approach - -- **ASV Calibration**: ASV automatically calibrates the number of loops per benchmark so that the total execution time is statistically significant (defaulting to ~10ms total). -- **Repetitions**: Each benchmark is repeated at least 3 times. ASV reports the median time to reject outliers caused by OS background tasks. -- **Warmup**: ASV performs un-timed warmup runs before measuring. - -## 3. Data Generation - -The `benchmarks.datasets.data_generator` module produces a deterministic dataset specifically designed to stress data cleaning libraries. - -It does not generate "clean" random data. It generates: -- Numerics with 2% extreme outliers and 15% missing values -- Strings contaminated with trailing/leading whitespaces and varied casing -- High-cardinality categoricals -- Formatted currencies, URLs, and emails containing intentional syntax errors -- Duplicated rows representing ~5% of the total dataset length - -## 4. Hardware and Threading - -Numeric libraries (NumPy, SciPy via sklearn, Polars) use varying numbers of threads by default, which can obscure algorithmic efficiency. - -We configure `asv.conf.json` to enforce single-threaded execution during benchmarking: -```json -"env": { - "OPENBLAS_NUM_THREADS": ["1"], - "MKL_NUM_THREADS": ["1"], - "OMP_NUM_THREADS": ["1"] -} -``` -Multi-threaded performance scales linearly with hardware and is generally uninteresting for comparing underlying algorithmic complexity. diff --git a/freshdata-benchmarks/README.md b/freshdata-benchmarks/README.md deleted file mode 100644 index cb16d092..00000000 --- a/freshdata-benchmarks/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# FreshData Benchmark Suite - -A comparative benchmark suite for `freshdata-cleaner` using -[Airspeed Velocity (ASV)](https://asv.readthedocs.io/). This suite is run -locally and its raw ASV results are committed under `.asv/results/`; it is -separate from the repository's CI `Benchmark` workflow, which runs the -[CleanBench harness](../benchmarks/) in `benchmarks/`. - -This suite measures the performance of FreshData's data cleaning operations against the Python data ecosystem: -- **Pandas** (baseline) -- **Polars** -- **pyjanitor** -- **Scikit-learn** -- **Feature-engine** -- **AutoClean** - -## Quick Start - -```bash -# 1. Create a virtual environment -python -m venv venv -source venv/bin/activate - -# 2. Install dependencies -pip install -r requirements.txt -pip install asv - -# 3. Configure ASV machine (run once) -asv machine --yes - -# 4. Run the benchmarks -asv run - -# 5. Generate reports -python scripts/generate_reports.py -python scripts/generate_plots.py - -# 6. View ASV web interface -asv publish -asv preview -``` - -## FreshData vs Pandas β€” Focused Comparison - -The headline benchmark compares **FreshData** and **Pandas** head-to-head across six core data operations at three dataset sizes (10K, 100K, 1M rows): - -| # | Operation | FreshData | Pandas | -|---|-----------|-----------|--------| -| 1 | **Loading** (CSV read + type inference) | `fd.clean(df, strategy="conservative")` | `pd.read_csv()` | -| 2 | **Missing values** (fill with mean) | `fd.clean(df, strategy="aggressive")` | `df.fillna(df.mean())` | -| 3 | **Outlier detection** (IQR flagging) | `fd.clean(df, outlier_method="iqr")` | Manual IQR per column | -| 4 | **Duplicate resolution** | `fd.clean(df, strategy="conservative")` | `df.drop_duplicates()` | -| 5 | **Group aggregations** | `fd.profile(df)` | `df.groupby().agg()` | -| 6 | **Full pipeline** | `fd.clean(df, strategy="balanced")` | Manual 7-step chain | - -### Run the focused comparison only - -```bash -# One-shot: installs deps, runs benchmarks, generates reports -bash bench/run_benchmarks.sh - -# Quick mode (fewer iterations, for fast validation) -bash bench/run_benchmarks.sh --quick -``` - -## Reproduce Locally - -To reproduce benchmark results exactly: - -```bash -# Clone and enter the repo -git clone https://github.com/FreshCode-Org/freshdata.git -cd freshdata/freshdata-benchmarks - -# Pin all randomness and threading for deterministic results -export PYTHONHASHSEED=42 -export OMP_NUM_THREADS=1 -export MKL_NUM_THREADS=1 -export OPENBLAS_NUM_THREADS=1 -export NUMEXPR_NUM_THREADS=1 -export VECLIB_MAXIMUM_THREADS=1 - -# Use the all-in-one runner (creates its own venv) -bash bench/run_benchmarks.sh - -# Or run manually with ASV -python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt asv virtualenv -asv machine --yes -asv run --bench "BenchmarkFreshDataVsPandas" -asv publish && asv preview -``` - -**Environment capture**: The runner script automatically saves machine details (Python version, OS, CPU, RAM, package versions) to `bench/artifacts/env_info.json`. You can also generate this manually: - -```bash -python scripts/capture_env.py -``` - -## Interpreting Results - -| Metric prefix | What it measures | Unit | -|---------------|------------------|------| -| `time_*` | Wall-clock execution time | seconds | -| `peakmem_*` | Peak resident set size (RSS) | bytes | -| `track_throughput_*` | Rows processed per second | rows/s | -| `track_speedup_*` | Speedup ratio vs Pandas baseline | ratio (Γ—) | - -- **Lower is better** for `time_*` and `peakmem_*`. -- **Higher is better** for `track_throughput_*` and `track_speedup_*`. -- ASV calibrates loop counts automatically and reports the **median** of β‰₯3 repeats. -- Results include warmup iterations (untimed) before each measurement. -- Standard deviations are captured to identify noisy measurements. - -## Benchmark Matrix - -The suite covers 15 domains using synthetic datasets (10K to 10M rows) with realistic data quality issues (NaNs, extreme outliers, sentinels, unicode anomalies, whitespaces): - -1. **Pipeline** (`benchmark_pipeline.py`) - Full auto-cleaning pipeline -2. **Missing Values** (`benchmark_missing.py`) - Drop, mean, median, ffill -3. **Duplicates** (`benchmark_duplicates.py`) - Detect and drop -4. **Strings** (`benchmark_strings.py`) - Trim, lowercase, regex -5. **Types** (`benchmark_types.py`) - Downcasting, numeric parsing -6. **Columns** (`benchmark_columns.py`) - Renames and selections -7. **Encoding** (`benchmark_encoding.py`) - One-hot and ordinal -8. **Outliers** (`benchmark_outliers.py`) - IQR and Z-score -9. **Scaling** (`benchmark_scaling.py`) - Standard and MinMax -10. **Validation** (`benchmark_validation.py`) - Schema profiling -11. **Memory** (`benchmark_memory.py`) - Peak RSS tracking -12. **I/O** (`benchmark_io.py`) - Combined CSV/Parquet read + clean -13. **Scaling** (`benchmark_scaling_curves.py`) - Big-O efficiency analysis -14. **Group Aggregations** (`benchmark_groupagg.py`) - Groupby/profile comparisons -15. **FreshData vs Pandas** (`benchmark_freshdata_vs_pandas.py`) - Focused 6-operation head-to-head - -## Published Results - -This ASV suite is not wired into CI: run it locally with the commands above -and browse the results with `asv publish && asv preview`. Raw per-machine -results live in `.asv/results/` so runs are comparable over time. The -repository's CI `Benchmark` workflow runs the separate CleanBench harness -(`benchmarks/` at the repo root) on every push to `main` and weekly. - -## Methodology - -See [METHODOLOGY.md](METHODOLOGY.md) for details on fairness enforcement, cache eviction, deep copies, and dataset generation. - diff --git a/freshdata-benchmarks/asv.conf.json b/freshdata-benchmarks/asv.conf.json deleted file mode 100644 index b7accedb..00000000 --- a/freshdata-benchmarks/asv.conf.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "version": 1, - "project": "freshdata-benchmarks", - "project_url": "https://github.com/FreshCode-Org/freshdata", - "repo": ".", - "dvcs": "git", - "branches": ["HEAD"], - - "benchmark_dir": "benchmarks", - "results_dir": ".asv/results", - "html_dir": ".asv/html", - - "environment_type": "virtualenv", - "pythons": [""], - - "install_timeout": 600.0, - "install_command": [ - "in-dir={env_dir} python -m pip install -r {build_dir}/requirements.txt" - ], - "build_command": [], - "uninstall_command": [], - - "matrix": { - "env": { - "OPENBLAS_NUM_THREADS": ["1"], - "MKL_NUM_THREADS": ["1"], - "OMP_NUM_THREADS": ["1"] - } - }, - - "benchmark_timeout": 600, - "default_benchmark_timeout": 600 -} diff --git a/freshdata-benchmarks/bench/artifacts/benchmark_output.txt b/freshdata-benchmarks/bench/artifacts/benchmark_output.txt deleted file mode 100644 index ab6b2e3c..00000000 --- a/freshdata-benchmarks/bench/artifacts/benchmark_output.txt +++ /dev/null @@ -1,39 +0,0 @@ -Β· Cloning project -Β· Error running /usr/bin/git clone --mirror . /Users/wilson/freshdata/freshdata-benchmarks/freshdata-benchmarks (exit status 128) - STDOUT --------> - - STDERR --------> - fatal: repository '.' does not exist - -Traceback (most recent call last): - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/bin/asv", line 6, in - sys.exit(main()) - ~~~~^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/main.py", line 29, in main - result = args.func(args) - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/commands/__init__.py", line 49, in run_from_args - return cls.run_from_conf_args(conf, args) - ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/commands/run.py", line 218, in run_from_conf_args - return cls.run( - ~~~~~~~^ - conf=conf, - ^^^^^^^^^^ - ...<22 lines>... - **kwargs, - ^^^^^^^^^ - ) - ^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/commands/run.py", line 301, in run - repo = get_repo(conf) - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/repo.py", line 305, in get_repo - return cls(conf.repo, conf.project) - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/plugins/git.py", line 36, in __init__ - self._run_git(['clone', '--mirror', url, self._path], cwd=None) - ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/plugins/git.py", line 70, in _run_git - return util.check_output([self._git] + args, env=env, **kwargs) - ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/util.py", line 641, in check_output - raise ProcessError(args, retcode, stdout, stderr) -asv.util.ProcessError: Command '/usr/bin/git clone --mirror . /Users/wilson/freshdata/freshdata-benchmarks/freshdata-benchmarks' returned non-zero exit status 128 diff --git a/freshdata-benchmarks/bench/artifacts/env_info.json b/freshdata-benchmarks/bench/artifacts/env_info.json deleted file mode 100644 index 415d2ef5..00000000 --- a/freshdata-benchmarks/bench/artifacts/env_info.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "timestamp": "2026-07-05T14:04:21.763872+00:00", - "hostname": "Johnnys-MacBook-Air.local", - "python": { - "version": "3.13.9", - "implementation": "CPython", - "compiler": "Clang 20.1.8 " - }, - "os": { - "system": "Darwin", - "release": "24.5.0", - "version": "Darwin Kernel Version 24.5.0: Tue Apr 22 19:54:26 PDT 2025; root:xnu-11417.121.6~2/RELEASE_ARM64_T8112", - "machine": "arm64" - }, - "cpu": { - "model": "arm", - "count_logical": 8, - "count_physical": 8, - "freq_mhz": 3504 - }, - "ram_gb": 16.0, - "packages": { - "freshdata-cleaner": "1.0.1", - "pandas": "3.0.3", - "numpy": "2.5.1", - "pyarrow": "24.0.0", - "asv": "0.6.6", - "psutil": "7.2.2" - } -} diff --git a/freshdata-benchmarks/bench/run_benchmarks.sh b/freshdata-benchmarks/bench/run_benchmarks.sh deleted file mode 100755 index 61f0f4ac..00000000 --- a/freshdata-benchmarks/bench/run_benchmarks.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash -# ────────────────────────────────────────────────────────────────── -# bench/run_benchmarks.sh -# -# One-shot script that installs dependencies in an isolated -# virtualenv, runs the focused freshdata-vs-pandas ASV benchmarks, -# generates the HTML report and a JSON/CSV summary, and collects -# all artifacts under bench/artifacts/. -# -# Usage: -# bash bench/run_benchmarks.sh # full run -# bash bench/run_benchmarks.sh --quick # quick (dry-run) mode -# -# Requirements: -# - Python 3.9+ available as `python3` -# - git (required by ASV) -# ────────────────────────────────────────────────────────────────── -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -VENV_DIR="${PROJECT_ROOT}/.venv-bench" -ARTIFACTS_DIR="${PROJECT_ROOT}/bench/artifacts" -QUICK_MODE=false - -# Parse arguments -for arg in "$@"; do - case "$arg" in - --quick) QUICK_MODE=true ;; - *) echo "Unknown argument: $arg"; exit 1 ;; - esac -done - -echo "══════════════════════════════════════════════════════════════" -echo " FreshData vs Pandas β€” ASV Benchmark Runner" -echo "══════════════════════════════════════════════════════════════" -echo "" -echo " Project root : ${PROJECT_ROOT}" -echo " Quick mode : ${QUICK_MODE}" -echo " Timestamp : $(date -u +%Y-%m-%dT%H:%M:%SZ)" -echo "" - -# ── 1. Create / activate isolated virtualenv ───────────────────── -echo "β–Έ Step 1/7: Setting up isolated virtualenv …" -if [ ! -d "${VENV_DIR}" ]; then - python3 -m venv "${VENV_DIR}" - echo " Created ${VENV_DIR}" -fi -source "${VENV_DIR}/bin/activate" -pip install --upgrade pip --quiet - -# ── 2. Install dependencies ───────────────────────────────────── -echo "β–Έ Step 2/7: Installing dependencies …" -pip install -r "${PROJECT_ROOT}/requirements.txt" --quiet -pip install asv virtualenv --quiet -echo " Done." - -# ── 3. Capture environment details ────────────────────────────── -echo "β–Έ Step 3/7: Capturing environment …" -mkdir -p "${ARTIFACTS_DIR}" -python "${PROJECT_ROOT}/scripts/capture_env.py" "${ARTIFACTS_DIR}/env_info.json" - -# ── 4. Configure ASV machine ──────────────────────────────────── -echo "β–Έ Step 4/7: Configuring ASV machine …" -cd "${PROJECT_ROOT}" -asv machine --yes 2>/dev/null || true - -# ── 5. Pin reproducibility settings ───────────────────────────── -export OPENBLAS_NUM_THREADS=1 -export MKL_NUM_THREADS=1 -export OMP_NUM_THREADS=1 -export NUMEXPR_NUM_THREADS=1 -export VECLIB_MAXIMUM_THREADS=1 -export PYTHONHASHSEED=42 - -# ── 6. Run benchmarks ─────────────────────────────────────────── -echo "β–Έ Step 5/7: Running benchmarks …" -if [ "${QUICK_MODE}" = true ]; then - asv run --quick \ - --bench "BenchmarkFreshDataVsPandas|BenchmarkGroupAgg" \ - -e 2>&1 | tee "${ARTIFACTS_DIR}/benchmark_output.txt" -else - asv run \ - --skip-existing-successful \ - --bench "BenchmarkFreshDataVsPandas|BenchmarkGroupAgg" \ - -e 2>&1 | tee "${ARTIFACTS_DIR}/benchmark_output.txt" -fi - -# ── 7. Generate reports & HTML ─────────────────────────────────── -echo "β–Έ Step 6/7: Generating reports …" -python "${PROJECT_ROOT}/scripts/generate_reports.py" || true -python "${PROJECT_ROOT}/scripts/generate_plots.py" || true -asv publish || true - -# Copy HTML report to artifacts -if [ -d "${PROJECT_ROOT}/.asv/html" ]; then - cp -r "${PROJECT_ROOT}/.asv/html" "${ARTIFACTS_DIR}/html_report" - echo " HTML report β†’ ${ARTIFACTS_DIR}/html_report/" -fi - -# Copy JSON results to artifacts -if [ -d "${PROJECT_ROOT}/.asv/results" ]; then - cp -r "${PROJECT_ROOT}/.asv/results" "${ARTIFACTS_DIR}/asv_results" - echo " ASV results β†’ ${ARTIFACTS_DIR}/asv_results/" -fi - -# Copy generated reports -if [ -d "${PROJECT_ROOT}/reports" ]; then - cp -r "${PROJECT_ROOT}/reports" "${ARTIFACTS_DIR}/reports" - echo " Reports β†’ ${ARTIFACTS_DIR}/reports/" -fi - -echo "" -echo "β–Έ Step 7/7: Summary" -echo "══════════════════════════════════════════════════════════════" -echo " Artifacts directory : ${ARTIFACTS_DIR}" -echo " Environment info : ${ARTIFACTS_DIR}/env_info.json" -echo " Benchmark log : ${ARTIFACTS_DIR}/benchmark_output.txt" -echo "" -echo " To view the HTML report locally:" -echo " cd ${PROJECT_ROOT} && asv preview" -echo "" -echo " Exact command to reproduce:" -echo " PYTHONHASHSEED=42 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \\" -echo " OPENBLAS_NUM_THREADS=1 bash bench/run_benchmarks.sh" -echo "" -echo "══════════════════════════════════════════════════════════════" -echo " βœ“ Benchmark run complete." -echo "══════════════════════════════════════════════════════════════" diff --git a/freshdata-benchmarks/benchmarks/__init__.py b/freshdata-benchmarks/benchmarks/__init__.py deleted file mode 100644 index 082b019c..00000000 --- a/freshdata-benchmarks/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""FreshData ASV Benchmark Suite.""" diff --git a/freshdata-benchmarks/benchmarks/benchmark_columns.py b/freshdata-benchmarks/benchmarks/benchmark_columns.py deleted file mode 100644 index 1900e85d..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_columns.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Benchmark column-level operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkColumns: - """Benchmark suite for column renaming, dropping, and selection.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Define columns for operations - cols = list(self.df.columns) - self.rename_mapping = {c: f"renamed_{c}" for c in cols[:5]} - self.drop_cols = cols[-3:] - self.select_cols = cols[:5] - - # Clean up memory - gc_collect() - - def time_rename_columns(self, n_rows: int, library: str) -> None: - """Measure time to rename columns.""" - df = fresh_copy(self.df) - self.adapter.rename_columns(df, self.rename_mapping) - - def time_drop_columns(self, n_rows: int, library: str) -> None: - """Measure time to drop specific columns.""" - df = fresh_copy(self.df) - self.adapter.drop_columns(df, self.drop_cols) - - def time_select_columns(self, n_rows: int, library: str) -> None: - """Measure time to select a subset of columns.""" - df = fresh_copy(self.df) - self.adapter.select_columns(df, self.select_cols) diff --git a/freshdata-benchmarks/benchmarks/benchmark_duplicates.py b/freshdata-benchmarks/benchmarks/benchmark_duplicates.py deleted file mode 100644 index 5b05f947..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_duplicates.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Benchmark duplicate row operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkDuplicates: - """Benchmark suite for duplicate detection and removal.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def time_detect_duplicates(self, n_rows: int, library: str) -> None: - """Measure time to detect duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.detect_duplicates(df) - - def peakmem_detect_duplicates(self, n_rows: int, library: str) -> None: - """Measure peak memory when detecting duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.detect_duplicates(df) - - def time_drop_duplicates(self, n_rows: int, library: str) -> None: - """Measure time to drop duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - def peakmem_drop_duplicates(self, n_rows: int, library: str) -> None: - """Measure peak memory when dropping duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - def track_duplicate_count(self, n_rows: int, library: str) -> int: - """Track the number of duplicate rows dropped.""" - df = fresh_copy(self.df) - start_rows = len(df) - result = self.adapter.drop_duplicates(df) - - # Approximate counting since libraries return different formats - try: - return start_rows - len(result) - except Exception: - return -1 - - track_duplicate_count.unit = "rows" diff --git a/freshdata-benchmarks/benchmarks/benchmark_encoding.py b/freshdata-benchmarks/benchmarks/benchmark_encoding.py deleted file mode 100644 index 213c23b3..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_encoding.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Benchmark categorical encoding operations across libraries. - -Note: FreshData does not provide standalone encoding features in its core API, -so this benchmark compares the broader ecosystem (pandas, sklearn, feature_engine). -""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_categorical_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkEncoding: - """Benchmark suite for one-hot and label encoding.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["pandas", "sklearn", "feature_engine", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.cat_cols = get_categorical_columns(self.df) - - # Clean up memory - gc_collect() - - def time_onehot_encode(self, n_rows: int, library: str) -> None: - """Measure time to perform one-hot encoding on categorical columns.""" - df = fresh_copy(self.df) - self.adapter.onehot_encode(df, self.cat_cols) - - def peakmem_onehot_encode(self, n_rows: int, library: str) -> None: - """Measure peak memory during one-hot encoding.""" - df = fresh_copy(self.df) - self.adapter.onehot_encode(df, self.cat_cols) - - def time_label_encode(self, n_rows: int, library: str) -> None: - """Measure time to perform label (ordinal) encoding.""" - df = fresh_copy(self.df) - self.adapter.label_encode(df, self.cat_cols) - - def peakmem_label_encode(self, n_rows: int, library: str) -> None: - """Measure peak memory during label encoding.""" - df = fresh_copy(self.df) - self.adapter.label_encode(df, self.cat_cols) diff --git a/freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py b/freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py deleted file mode 100644 index 98b1efd6..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Focused freshdata-vs-pandas comparison across six core data operations. - -This is the headline benchmark for CI: it covers loading, missing-value -handling, outlier detection/filtering, duplicate resolution, group -aggregations, and a representative preprocessing pipeline β€” all measured -at three configurable dataset sizes. -""" - -from __future__ import annotations - -import os -import shutil -import tempfile -import time as _time - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkFreshDataVsPandas: - """Headline benchmark comparing FreshData and Pandas across six operations.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset, adapter, and temp files for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Write a CSV for loading benchmarks - self._tmpdir = tempfile.mkdtemp(prefix="asv_fdvspd_") - self.csv_path = os.path.join(self._tmpdir, f"test_{n_rows}.csv") - self.df.to_csv(self.csv_path, index=False) - - gc_collect() - - def teardown(self, n_rows: int, library: str) -> None: - """Remove temporary files.""" - shutil.rmtree(self._tmpdir, ignore_errors=True) - - # ── 1. Loading ───────────────────────────────────────────── - - def time_load_csv(self, n_rows: int, library: str) -> None: - """Measure time to load a CSV file and infer types.""" - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_csv(self.csv_path) - fd.clean(df, strategy="conservative", preserve_original=True) - else: - import pandas as pd - pd.read_csv(self.csv_path) - - def peakmem_load_csv(self, n_rows: int, library: str) -> None: - """Measure peak memory when loading a CSV file.""" - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_csv(self.csv_path) - fd.clean(df, strategy="conservative", preserve_original=True) - else: - import pandas as pd - pd.read_csv(self.csv_path) - - # ── 2. Missing Value Handling ────────────────────────────── - - def time_handle_missing(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with column means.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - def peakmem_handle_missing(self, n_rows: int, library: str) -> None: - """Measure peak memory when filling missing values.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - # ── 3. Outlier Detection / Filtering ─────────────────────── - - def time_detect_outliers(self, n_rows: int, library: str) -> None: - """Measure time to detect outliers using IQR method.""" - df = fresh_copy(self.df) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - def peakmem_detect_outliers(self, n_rows: int, library: str) -> None: - """Measure peak memory during IQR outlier detection.""" - df = fresh_copy(self.df) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - # ── 4. Duplicate Resolution ──────────────────────────────── - - def time_resolve_duplicates(self, n_rows: int, library: str) -> None: - """Measure time to detect and drop duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - def peakmem_resolve_duplicates(self, n_rows: int, library: str) -> None: - """Measure peak memory when dropping duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - # ── 5. Group Aggregations ────────────────────────────────── - - def time_group_aggregations(self, n_rows: int, library: str) -> None: - """Measure time for multi-column group aggregations.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - def peakmem_group_aggregations(self, n_rows: int, library: str) -> None: - """Measure peak memory for group aggregations.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - # ── 6. Full Preprocessing Pipeline ───────────────────────── - - def time_full_pipeline(self, n_rows: int, library: str) -> None: - """Measure time for the complete preprocessing pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def peakmem_full_pipeline(self, n_rows: int, library: str) -> None: - """Measure peak memory for the complete preprocessing pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - # ── Throughput Tracker ───────────────────────────────────── - - def track_pipeline_throughput(self, n_rows: int, library: str) -> float: - """Calculate rows processed per second for the full pipeline.""" - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.full_clean(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_pipeline_throughput.unit = "rows/s" diff --git a/freshdata-benchmarks/benchmarks/benchmark_groupagg.py b/freshdata-benchmarks/benchmarks/benchmark_groupagg.py deleted file mode 100644 index c9e7b814..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_groupagg.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Benchmark group aggregation operations comparing freshdata vs pandas.""" - -from __future__ import annotations - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkGroupAgg: - """Benchmark suite for group aggregation operations.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - gc_collect() - - def time_group_agg_single(self, n_rows: int, library: str) -> None: - """Measure time for single-column groupby with mean aggregation.""" - df = fresh_copy(self.df) - self.adapter.group_agg_single(df) - - def time_group_agg_multi(self, n_rows: int, library: str) -> None: - """Measure time for multi-column groupby with multiple agg functions.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - def time_group_agg_transform(self, n_rows: int, library: str) -> None: - """Measure time for groupby transform (centering within groups).""" - df = fresh_copy(self.df) - self.adapter.group_agg_transform(df, self.numeric_cols) - - def peakmem_group_agg_multi(self, n_rows: int, library: str) -> None: - """Measure peak memory for multi-column groupby aggregation.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - def track_throughput_group_agg(self, n_rows: int, library: str) -> float: - """Calculate rows processed per second for group aggregation.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.group_agg_multi(df, self.numeric_cols) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput_group_agg.unit = "rows/s" diff --git a/freshdata-benchmarks/benchmarks/benchmark_io.py b/freshdata-benchmarks/benchmarks/benchmark_io.py deleted file mode 100644 index 15ef86de..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_io.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Benchmark file I/O operations paired with data cleaning.""" - -from __future__ import annotations - -import gc -import os -import shutil -import tempfile - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import gc_collect - - -class BenchmarkIO: - """Benchmark suite for reading and cleaning from disk formats.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup_cache(self): - """Generate test files once per class to avoid disk overhead in timing.""" - import pandas as pd - tmpdir = tempfile.mkdtemp() - - # We only generate files for the requested sizes in params - sizes = [10_000, 100_000, 1_000_000] - files = {} - - for size in sizes: - df = get_dataset(size) - csv_path = os.path.join(tmpdir, f"test_{size}.csv") - parquet_path = os.path.join(tmpdir, f"test_{size}.parquet") - - df.to_csv(csv_path, index=False) - try: - df.to_parquet(parquet_path, engine="pyarrow") - files[size] = {"csv": csv_path, "parquet": parquet_path} - except Exception: - files[size] = {"csv": csv_path, "parquet": None} - - return {"tmpdir": tmpdir, "files": files} - - def setup(self, cache, n_rows: int, library: str) -> None: - """Check library availability and locate files.""" - if library == "freshdata": - try: - import freshdata # noqa: F401 - except ImportError: - raise NotImplementedError("freshdata is not available") - elif library == "pandas": - try: - import pandas # noqa: F401 - except ImportError: - raise NotImplementedError("pandas is not available") - elif library == "polars": - try: - import polars # noqa: F401 - except ImportError: - raise NotImplementedError("polars is not available") - - self.csv_path = cache["files"][n_rows]["csv"] - self.parquet_path = cache["files"][n_rows]["parquet"] - - if not self.csv_path: - raise NotImplementedError("CSV file not generated") - - gc_collect() - - def teardown(self, cache, n_rows: int, library: str) -> None: - """Class-level teardown handles directory removal, so this is a no-op.""" - pass - - def time_csv_read_clean(self, cache, n_rows: int, library: str) -> None: - """Measure time to read a CSV and run a full clean.""" - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_csv(self.csv_path) - fd.clean(df, strategy="balanced", preserve_original=True) - elif library == "pandas": - import pandas as pd - df = pd.read_csv(self.csv_path) - df = df.dropna().drop_duplicates() - elif library == "polars": - import polars as pl - df = pl.read_csv(self.csv_path) - df = df.drop_nulls().unique() - - def time_parquet_read_clean(self, cache, n_rows: int, library: str) -> None: - """Measure time to read a Parquet file and run a full clean.""" - if not self.parquet_path: - raise NotImplementedError("Parquet missing") - - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_parquet(self.parquet_path) - fd.clean(df, strategy="balanced", preserve_original=True) - elif library == "pandas": - import pandas as pd - df = pd.read_parquet(self.parquet_path) - df = df.dropna().drop_duplicates() - elif library == "polars": - import polars as pl - df = pl.read_parquet(self.parquet_path) - df = df.drop_nulls().unique() - - def peakmem_csv_read_clean(self, cache, n_rows: int, library: str) -> None: - """Measure peak memory when reading and cleaning a CSV.""" - self.time_csv_read_clean(cache, n_rows, library) - - -def teardown_module(): - """Module-level teardown to remove temp directories.""" - # ASV cache mechanism is slightly opaque, best effort cleanup - pass diff --git a/freshdata-benchmarks/benchmarks/benchmark_memory.py b/freshdata-benchmarks/benchmarks/benchmark_memory.py deleted file mode 100644 index eba2519c..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_memory.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Memory-focused benchmarks and profiling.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkMemory: - """Benchmark suite dedicated to memory usage analysis.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000, 5_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def peakmem_full_clean(self, n_rows: int, library: str) -> None: - """Measure peak RSS memory used during the full cleaning pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def track_memory_before(self, n_rows: int, library: str) -> float: - """Track the deep memory usage of the uncleaned input DataFrame (in MB).""" - df = fresh_copy(self.df) - import pandas as pd - if isinstance(df, pd.DataFrame): - return df.memory_usage(deep=True).sum() / (1024 * 1024) - return -1.0 - - track_memory_before.unit = "MB" - - def track_memory_after(self, n_rows: int, library: str) -> float: - """Track the deep memory usage of the cleaned output DataFrame (in MB).""" - df = fresh_copy(self.df) - result = self.adapter.full_clean(df) - try: - import pandas as pd - if isinstance(result, pd.DataFrame): - return result.memory_usage(deep=True).sum() / (1024 * 1024) - else: - # Approximate memory for Polars - return result.estimated_size() / (1024 * 1024) - except Exception: - return -1.0 - - track_memory_after.unit = "MB" - - def track_memory_ratio(self, n_rows: int, library: str) -> float: - """Track the ratio of output memory / input memory. - Values < 1.0 indicate memory savings (e.g. via downcasting). - """ - df = fresh_copy(self.df) - - import pandas as pd - if not isinstance(df, pd.DataFrame): - return float('nan') - - mem_before = df.memory_usage(deep=True).sum() - if mem_before == 0: - return float('nan') - - result = self.adapter.full_clean(df) - - try: - if isinstance(result, pd.DataFrame): - mem_after = result.memory_usage(deep=True).sum() - else: - mem_after = result.estimated_size() - return float(mem_after) / float(mem_before) - except Exception: - return float('nan') - - track_memory_ratio.unit = "ratio" diff --git a/freshdata-benchmarks/benchmarks/benchmark_missing.py b/freshdata-benchmarks/benchmarks/benchmark_missing.py deleted file mode 100644 index 1a832801..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_missing.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Benchmark missing value operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkMissing: - """Benchmark suite for missing value handling operations.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "feature_engine", "sklearn"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Clean up memory - gc_collect() - - def time_drop_missing(self, n_rows: int, library: str) -> None: - """Measure time to drop rows with missing values.""" - df = fresh_copy(self.df) - self.adapter.drop_missing_rows(df) - - def peakmem_drop_missing(self, n_rows: int, library: str) -> None: - """Measure peak memory when dropping rows with missing values.""" - df = fresh_copy(self.df) - self.adapter.drop_missing_rows(df) - - def time_fill_mean(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with the column mean.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - def peakmem_fill_mean(self, n_rows: int, library: str) -> None: - """Measure peak memory when filling missing values with column mean.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - def time_fill_median(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with the column median.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_median(df, self.numeric_cols) - - def peakmem_fill_median(self, n_rows: int, library: str) -> None: - """Measure peak memory when filling missing values with column median.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_median(df, self.numeric_cols) - - def time_fill_mode(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with the column mode. - Supported by pandas, sklearn, and freshdata. - """ - df = fresh_copy(self.df) - self.adapter.fill_missing_mode(df, self.numeric_cols) - - def time_fill_ffill(self, n_rows: int, library: str) -> None: - """Measure time to forward-fill missing values. - Supported by pandas, polars, and freshdata. - """ - df = fresh_copy(self.df) - self.adapter.fill_missing_ffill(df) - - def time_fill_bfill(self, n_rows: int, library: str) -> None: - """Measure time to backward-fill missing values. - Supported by pandas, polars, and freshdata. - """ - df = fresh_copy(self.df) - self.adapter.fill_missing_bfill(df) - - def track_throughput_drop_missing(self, n_rows: int, library: str) -> float: - """Calculate rows processed per second for drop_missing.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.drop_missing_rows(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput_drop_missing.unit = "rows/s" diff --git a/freshdata-benchmarks/benchmarks/benchmark_outliers.py b/freshdata-benchmarks/benchmarks/benchmark_outliers.py deleted file mode 100644 index e7a0b697..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_outliers.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Benchmark outlier detection algorithms across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkOutliers: - """Benchmark suite for outlier detection techniques.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "feature_engine", "sklearn", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs) - import pandas as pd - self.df_numeric = pd.DataFrame() - for col in self.numeric_cols: - self.df_numeric[col] = pd.to_numeric(self.df[col], errors="coerce") - self.df_numeric = self.df_numeric.dropna().reset_index(drop=True) - - # Clean up memory - gc_collect() - - def _get_target_df(self, library: str): - """Return the appropriate DataFrame depending on library requirements.""" - if library in ["sklearn", "feature_engine"]: - return fresh_copy(self.df_numeric) - return fresh_copy(self.df) - - def time_detect_iqr(self, n_rows: int, library: str) -> None: - """Measure time to detect outliers using the Interquartile Range (IQR) method.""" - df = self._get_target_df(library) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - def peakmem_detect_iqr(self, n_rows: int, library: str) -> None: - """Measure peak memory during IQR outlier detection.""" - df = self._get_target_df(library) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - def time_detect_zscore(self, n_rows: int, library: str) -> None: - """Measure time to detect outliers using Z-score (standard deviations).""" - df = self._get_target_df(library) - self.adapter.detect_outliers_zscore(df, self.numeric_cols) - - def peakmem_detect_zscore(self, n_rows: int, library: str) -> None: - """Measure peak memory during Z-score outlier detection.""" - df = self._get_target_df(library) - self.adapter.detect_outliers_zscore(df, self.numeric_cols) - - def track_outlier_count_iqr(self, n_rows: int, library: str) -> int: - """Track the total number of outliers detected by IQR across all numeric columns.""" - df = self._get_target_df(library) - result = self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - outlier_count = 0 - try: - import pandas as pd - if isinstance(result, pd.DataFrame): - outlier_cols = [c for c in result.columns if c.endswith("_outlier")] - for col in outlier_cols: - outlier_count += int(result[col].sum()) - except Exception: - pass - - return outlier_count if outlier_count > 0 else -1 - - track_outlier_count_iqr.unit = "outliers" diff --git a/freshdata-benchmarks/benchmarks/benchmark_pipeline.py b/freshdata-benchmarks/benchmarks/benchmark_pipeline.py deleted file mode 100644 index 33c51e8b..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_pipeline.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Benchmark the complete end-to-end cleaning pipeline. - -This is the headline benchmark for the FreshData suite, comparing the full -auto-clean pipeline against equivalent manual operations in other libraries. -""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkPipeline: - """Benchmark suite for full dataset cleaning pipelines.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000, 5_000_000], - ["freshdata", "pandas", "polars", "pyjanitor", "autoclean"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def time_full_clean(self, n_rows: int, library: str) -> None: - """Measure time to run the complete data cleaning pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def peakmem_full_clean(self, n_rows: int, library: str) -> None: - """Measure peak memory used during the full cleaning pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def track_throughput(self, n_rows: int, library: str) -> float: - """Calculate pipeline throughput in rows per second.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.full_clean(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput.unit = "rows/s" - - def track_output_rows(self, n_rows: int, library: str) -> int: - """Track the number of rows remaining after cleaning.""" - df = fresh_copy(self.df) - result = self.adapter.full_clean(df) - try: - return len(result) - except Exception: - return -1 - - track_output_rows.unit = "rows" - - def track_output_cols(self, n_rows: int, library: str) -> int: - """Track the number of columns remaining after cleaning.""" - df = fresh_copy(self.df) - result = self.adapter.full_clean(df) - try: - import pandas as pd - if isinstance(result, pd.DataFrame): - return len(result.columns) - return len(getattr(result, "columns", [])) - except Exception: - return -1 - - track_output_cols.unit = "cols" diff --git a/freshdata-benchmarks/benchmarks/benchmark_scaling.py b/freshdata-benchmarks/benchmarks/benchmark_scaling.py deleted file mode 100644 index 8ee35c2b..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_scaling.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Benchmark feature scaling operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkScaling: - """Benchmark suite for feature scaling (Standardization, MinMax).""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["pandas", "sklearn", "feature_engine", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Ensure clean numeric data for sklearn/feature_engine - import pandas as pd - self.df_numeric = pd.DataFrame() - for col in self.numeric_cols: - self.df_numeric[col] = pd.to_numeric(self.df[col], errors="coerce") - self.df_numeric = self.df_numeric.dropna().reset_index(drop=True) - - # Clean up memory - gc_collect() - - def _get_target_df(self, library: str): - """Return the appropriate DataFrame depending on library requirements.""" - if library in ["sklearn", "feature_engine"]: - return fresh_copy(self.df_numeric) - return fresh_copy(self.df) - - def time_standard_scale(self, n_rows: int, library: str) -> None: - """Measure time to apply StandardScaler (Z-score normalization).""" - df = self._get_target_df(library) - self.adapter.standard_scale(df, self.numeric_cols) - - def peakmem_standard_scale(self, n_rows: int, library: str) -> None: - """Measure peak memory during StandardScaler.""" - df = self._get_target_df(library) - self.adapter.standard_scale(df, self.numeric_cols) - - def time_minmax_scale(self, n_rows: int, library: str) -> None: - """Measure time to apply MinMaxScaler (0-1 normalization).""" - df = self._get_target_df(library) - self.adapter.minmax_scale(df, self.numeric_cols) - - def peakmem_minmax_scale(self, n_rows: int, library: str) -> None: - """Measure peak memory during MinMaxScaler.""" - df = self._get_target_df(library) - self.adapter.minmax_scale(df, self.numeric_cols) diff --git a/freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py b/freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py deleted file mode 100644 index c4a00e69..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Analysis of scaling efficiency across dataset sizes.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkScalingCurves: - """Benchmark suite to map out Big-O scaling complexity.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000, 5_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - gc_collect() - - def time_clean_scaling(self, n_rows: int, library: str) -> None: - """Time the full pipeline to map out scaling curves.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def peakmem_clean_scaling(self, n_rows: int, library: str) -> None: - """Peak memory of the full pipeline to map out scaling curves.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def track_throughput(self, n_rows: int, library: str) -> float: - """Rows per second at each dataset size.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.full_clean(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput.unit = "rows/s" - - def track_speedup_vs_pandas(self, n_rows: int, library: str) -> float: - """Speedup relative to Pandas for the full cleaning pipeline.""" - import time as _time - from benchmarks.utils.library_wrappers import get_adapter - - pd_adapter = get_adapter("pandas") - if not pd_adapter.available: - return float("nan") - - # Time pandas baseline - df_pd = fresh_copy(self.df) - start = _time.perf_counter() - pd_adapter.full_clean(df_pd) - pandas_time = _time.perf_counter() - start - - if pandas_time <= 0: - return float("nan") - - # If library IS pandas, speedup is exactly 1.0 - if library == "pandas": - return 1.0 - - # Time target library - df_lib = fresh_copy(self.df) - start = _time.perf_counter() - self.adapter.full_clean(df_lib) - library_time = _time.perf_counter() - start - - return pandas_time / library_time if library_time > 0 else float("nan") - - track_speedup_vs_pandas.unit = "ratio" diff --git a/freshdata-benchmarks/benchmarks/benchmark_strings.py b/freshdata-benchmarks/benchmarks/benchmark_strings.py deleted file mode 100644 index ce6740d3..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_strings.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Benchmark string cleaning operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_string_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkStrings: - """Benchmark suite for string manipulation and cleaning.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.string_cols = get_string_columns(self.df) - - # Clean up memory - gc_collect() - - def time_trim_whitespace(self, n_rows: int, library: str) -> None: - """Measure time to trim whitespace from all string columns.""" - df = fresh_copy(self.df) - self.adapter.trim_whitespace(df, self.string_cols) - - def peakmem_trim_whitespace(self, n_rows: int, library: str) -> None: - """Measure peak memory when trimming whitespace.""" - df = fresh_copy(self.df) - self.adapter.trim_whitespace(df, self.string_cols) - - def time_lowercase(self, n_rows: int, library: str) -> None: - """Measure time to convert string columns to lowercase.""" - df = fresh_copy(self.df) - self.adapter.to_lowercase(df, self.string_cols) - - def peakmem_lowercase(self, n_rows: int, library: str) -> None: - """Measure peak memory when converting to lowercase.""" - df = fresh_copy(self.df) - self.adapter.to_lowercase(df, self.string_cols) - - def time_uppercase(self, n_rows: int, library: str) -> None: - """Measure time to convert string columns to uppercase.""" - df = fresh_copy(self.df) - self.adapter.to_uppercase(df, self.string_cols) - - def time_regex_replace(self, n_rows: int, library: str) -> None: - """Measure time to perform a regex replacement on string columns.""" - df = fresh_copy(self.df) - pattern = r'[^a-zA-Z0-9 ]' - replacement = '' - self.adapter.regex_replace(df, self.string_cols, pattern, replacement) diff --git a/freshdata-benchmarks/benchmarks/benchmark_types.py b/freshdata-benchmarks/benchmarks/benchmark_types.py deleted file mode 100644 index 0166ae5c..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_types.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Benchmark data type conversion and optimization across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkTypes: - """Benchmark suite for type inference, conversion, and memory optimization.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Define specific columns to target - self.numeric_targets = ["int_col_1", "int_col_2", "float_col_1"] - self.date_targets = ["date_col"] - - # Clean up memory - gc_collect() - - def time_convert_numeric(self, n_rows: int, library: str) -> None: - """Measure time to convert string columns to numeric types.""" - df = fresh_copy(self.df) - self.adapter.convert_numeric(df, self.numeric_targets) - - def time_convert_datetime(self, n_rows: int, library: str) -> None: - """Measure time to parse datetime strings.""" - df = fresh_copy(self.df) - self.adapter.convert_datetime(df, self.date_targets) - - def time_optimize_dtypes(self, n_rows: int, library: str) -> None: - """Measure time to perform global type optimization (downcasting).""" - df = fresh_copy(self.df) - self.adapter.optimize_dtypes(df) - - def peakmem_optimize_dtypes(self, n_rows: int, library: str) -> None: - """Measure peak memory during global type optimization.""" - df = fresh_copy(self.df) - self.adapter.optimize_dtypes(df) diff --git a/freshdata-benchmarks/benchmarks/benchmark_validation.py b/freshdata-benchmarks/benchmarks/benchmark_validation.py deleted file mode 100644 index fd7346cf..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_validation.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Benchmark schema and data validation performance.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect - - -class BenchmarkValidation: - """Benchmark suite for schema and data quality validation. - - Compares FreshData's built-in profiler against manual pandas validation. - """ - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset for the benchmark.""" - if library == "freshdata": - try: - import freshdata # noqa: F401 - except ImportError: - raise NotImplementedError("freshdata is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def time_profile_validation(self, n_rows: int, library: str) -> None: - """Measure time to run full data profiling/validation.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - fd.profile(df) - else: # pandas manual profile - import pandas as pd - _ = df.dtypes - _ = df.isnull().sum() - _ = df.nunique() - _ = df.describe(include='all') - - def time_null_check(self, n_rows: int, library: str) -> None: - """Measure time to check for nulls across all columns.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - # FreshData profile includes null checks - fd.profile(df) - else: - _ = df.isnull().any() - - def time_type_check(self, n_rows: int, library: str) -> None: - """Measure time to validate data types.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - fd.profile(df) - else: - _ = df.dtypes - - def time_range_check(self, n_rows: int, library: str) -> None: - """Measure time to check numeric ranges.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - fd.profile(df) - else: - import pandas as pd - numeric_df = df.select_dtypes(include=['number']) - if not numeric_df.empty: - _ = numeric_df.min() - _ = numeric_df.max() diff --git a/freshdata-benchmarks/benchmarks/datasets/__init__.py b/freshdata-benchmarks/benchmarks/datasets/__init__.py deleted file mode 100644 index 6646b963..00000000 --- a/freshdata-benchmarks/benchmarks/datasets/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Cached dataset access for benchmarks. - -Usage:: - - from benchmarks.datasets import get_dataset - df = get_dataset(100_000) -""" - -from __future__ import annotations - -import functools - -from .data_generator import generate_dataset - - -@functools.lru_cache(maxsize=8) -def get_dataset(n_rows: int) -> "pd.DataFrame": - """Return a cached messy dataset with *n_rows* rows. - - The dataset is generated once per ``n_rows`` value and cached in - memory for the lifetime of the process. Benchmark ``setup()`` - methods should ``.copy()`` the returned frame before mutation. - """ - return generate_dataset(n_rows) - - -__all__ = ["get_dataset", "generate_dataset"] diff --git a/freshdata-benchmarks/benchmarks/datasets/data_generator.py b/freshdata-benchmarks/benchmarks/datasets/data_generator.py deleted file mode 100644 index 1f4756f6..00000000 --- a/freshdata-benchmarks/benchmarks/datasets/data_generator.py +++ /dev/null @@ -1,418 +0,0 @@ -"""Deterministic synthetic dataset generator for benchmarks. - -Generates realistic tabular data with controllable data quality issues. -All generation uses fixed random seeds for reproducibility across runs. - -Usage:: - - from benchmarks.datasets import get_dataset - df = get_dataset(100_000) # 100K rows, cached -""" - -from __future__ import annotations - -import functools -import string - -import numpy as np -import pandas as pd - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -SEED = 42 -SENTINEL_VALUES = ["N/A", "-", "", "null", "NA", "#REF!", "missing", "n/a", "None"] -EMAIL_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "company.org", "test.io"] -FIRST_NAMES = [ - "Alice", "Bob", "Carol", "Dave", "Eve", "Frank", "Grace", "Heidi", - "Ivan", "Judy", "Karl", "Laura", "Mallory", "Niaj", "Oscar", "Peggy", - "Quinn", "Rupert", "Sybil", "Trent", "Ursula", "Victor", "Wendy", -] -CATEGORIES_50 = [f"category_{i:03d}" for i in range(50)] -URL_PREFIXES = ["http://", "https://", "ftp://", ""] -CURRENCY_SYMBOLS = ["$", "€", "Β£", "Β₯", ""] - -# Dataset size tiers -SIZES = { - "tiny": 10_000, - "small": 100_000, - "medium": 1_000_000, - "large": 5_000_000, - "xlarge": 10_000_000, -} - - -# --------------------------------------------------------------------------- -# Core generator -# --------------------------------------------------------------------------- - - -def generate_dataset(n_rows: int, seed: int = SEED) -> pd.DataFrame: - """Generate a synthetic messy dataset with *n_rows* rows. - - The dataset contains 20+ columns with realistic data quality issues: - missing values, duplicates, wrong dtypes, whitespace, case - inconsistencies, Unicode, outliers, invalid patterns, and more. - - Parameters - ---------- - n_rows : int - Number of rows to generate. - seed : int - Random seed for reproducibility. - - Returns - ------- - pd.DataFrame - A deliberately messy DataFrame suitable for cleaning benchmarks. - """ - rng = np.random.default_rng(seed) - - # --- Unique ID column --- - ids = np.arange(1, n_rows + 1) - - # --- Numeric columns with NaN and outliers --- - float_col_1 = _numeric_with_issues(rng, n_rows, loc=100.0, scale=25.0, - nan_rate=0.12, outlier_rate=0.02) - float_col_2 = _numeric_with_issues(rng, n_rows, loc=50.0, scale=15.0, - nan_rate=0.18, outlier_rate=0.015) - float_col_3 = _skewed_numeric(rng, n_rows, nan_rate=0.10) - - int_col_1 = _integer_with_contamination(rng, n_rows, low=0, high=1000, - nan_rate=0.08, contam_rate=0.03) - int_col_2 = _integer_with_contamination(rng, n_rows, low=1, high=100, - nan_rate=0.05, contam_rate=0.02) - - # --- String columns --- - string_col = _messy_strings(rng, n_rows, FIRST_NAMES, - whitespace_rate=0.15, case_rate=0.20, - sentinel_rate=0.05) - - # --- High-cardinality categorical --- - category_col = _messy_categorical(rng, n_rows, CATEGORIES_50, - nan_rate=0.10) - - # --- Date column with mixed formats and invalid values --- - date_col = _messy_dates(rng, n_rows, invalid_rate=0.05, nan_rate=0.08) - - # --- Boolean column with mixed representations --- - bool_col = _messy_booleans(rng, n_rows, nan_rate=0.06) - - # --- Email column with invalid entries --- - email_col = _messy_emails(rng, n_rows, invalid_rate=0.10, nan_rate=0.05) - - # --- Phone column with mixed formats --- - phone_col = _messy_phones(rng, n_rows, invalid_rate=0.08, nan_rate=0.07) - - # --- URL column --- - url_col = _messy_urls(rng, n_rows, invalid_rate=0.10, nan_rate=0.06) - - # --- Currency column --- - currency_col = _messy_currency(rng, n_rows, nan_rate=0.08) - - # --- Unicode column --- - unicode_col = _messy_unicode(rng, n_rows, nan_rate=0.05) - - # --- Constant column (all same value) --- - constant_col = np.full(n_rows, "CONSTANT_VALUE", dtype=object) - - # --- Null-heavy column (90%+ null) --- - null_heavy = _null_heavy_column(rng, n_rows, null_rate=0.92) - - # --- Mixed-type column (ints, floats, strings) --- - mixed_type = _mixed_type_column(rng, n_rows) - - # --- Negative values column --- - negative_col = rng.normal(-50.0, 30.0, n_rows) - neg_mask = rng.random(n_rows) < 0.10 - negative_col = negative_col.astype(object) - negative_col[neg_mask] = None - - # --- Build DataFrame --- - df = pd.DataFrame({ - "id": ids, - "float_col_1": float_col_1, - "float_col_2": float_col_2, - "float_col_3": float_col_3, - "int_col_1": int_col_1, - "int_col_2": int_col_2, - " String Col ": string_col, # intentional whitespace in name - "CATEGORY_COL": category_col, # intentional uppercase name - "date_col": date_col, - "bool_col": bool_col, - "email_col": email_col, - "phone_col": phone_col, - "url_col": url_col, - "currency_col": currency_col, - "unicode_col": unicode_col, - "constant_col": constant_col, - "null_heavy_col": null_heavy, - "mixed_type_col": mixed_type, - "negative_col": negative_col, - }) - - # --- Inject duplicate rows (~5% of dataset) --- - n_dupes = max(1, int(n_rows * 0.05)) - dupe_idx = rng.choice(n_rows, size=n_dupes, replace=True) - dupes = df.iloc[dupe_idx].copy() - df = pd.concat([df, dupes], ignore_index=True) - - # Shuffle to distribute duplicates - df = df.sample(frac=1.0, random_state=seed).reset_index(drop=True) - - return df - - -# --------------------------------------------------------------------------- -# Column generators (private) -# --------------------------------------------------------------------------- - - -def _numeric_with_issues( - rng: np.random.Generator, - n: int, - loc: float, - scale: float, - nan_rate: float, - outlier_rate: float, -) -> np.ndarray: - """Normal-distributed floats with NaN and extreme outliers.""" - values = rng.normal(loc, scale, n).astype(object) - # Inject outliers (8–15Γ— the value) - outlier_mask = rng.random(n) < outlier_rate - outlier_count = outlier_mask.sum() - if outlier_count > 0: - values[outlier_mask] = ( - rng.normal(loc, scale, outlier_count) * rng.uniform(8, 15, outlier_count) - ) - # Inject NaN - nan_mask = rng.random(n) < nan_rate - values[nan_mask] = None - return values - - -def _skewed_numeric( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Log-normal (right-skewed) distribution with NaN.""" - values = rng.lognormal(mean=3.0, sigma=1.0, size=n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _integer_with_contamination( - rng: np.random.Generator, - n: int, - low: int, - high: int, - nan_rate: float, - contam_rate: float, -) -> np.ndarray: - """Integers stored as strings, with some non-numeric contamination.""" - values = rng.integers(low, high, n).astype(str).astype(object) - # Contaminate with non-numeric strings - contam_mask = rng.random(n) < contam_rate - contam_count = contam_mask.sum() - if contam_count > 0: - values[contam_mask] = rng.choice(["abc", "N/A", "??", "--", ""], contam_count) - # Inject NaN - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_strings( - rng: np.random.Generator, - n: int, - base_values: list[str], - whitespace_rate: float, - case_rate: float, - sentinel_rate: float, -) -> np.ndarray: - """Strings with whitespace, case inconsistencies, and sentinels.""" - values = rng.choice(base_values, n).astype(object) - # Add whitespace - ws_mask = rng.random(n) < whitespace_rate - for i in np.where(ws_mask)[0]: - pad_left = " " * rng.integers(1, 4) - pad_right = " " * rng.integers(1, 4) - values[i] = f"{pad_left}{values[i]}{pad_right}" - # Random case - case_mask = rng.random(n) < case_rate - for i in np.where(case_mask)[0]: - choice = rng.integers(0, 3) - if choice == 0: - values[i] = str(values[i]).upper() - elif choice == 1: - values[i] = str(values[i]).lower() - else: - values[i] = str(values[i]).title() - # Replace some with sentinels - sent_mask = rng.random(n) < sentinel_rate - values[sent_mask] = rng.choice(SENTINEL_VALUES, sent_mask.sum()) - return values - - -def _messy_categorical( - rng: np.random.Generator, - n: int, - categories: list[str], - nan_rate: float, -) -> np.ndarray: - """High-cardinality categorical with NaN.""" - values = rng.choice(categories, n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_dates( - rng: np.random.Generator, - n: int, - invalid_rate: float, - nan_rate: float, -) -> np.ndarray: - """Date strings in mixed formats with invalid entries.""" - base_dates = pd.date_range("2020-01-01", periods=n, freq="h") - formats = ["%Y-%m-%d", "%m/%d/%Y", "%d-%b-%Y", "%Y/%m/%d %H:%M:%S"] - values = np.empty(n, dtype=object) - for i in range(n): - fmt = formats[i % len(formats)] - values[i] = base_dates[i % len(base_dates)].strftime(fmt) - # Invalid dates - inv_mask = rng.random(n) < invalid_rate - inv_values = ["2020-13-45", "not-a-date", "31/02/2020", "00/00/0000", - "2020-02-30", "abc", "9999-99-99"] - values[inv_mask] = rng.choice(inv_values, inv_mask.sum()) - # NaN - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_booleans( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Booleans with mixed representations.""" - representations = ["yes", "no", "Yes", "No", "YES", "NO", "true", "false", - "True", "False", "TRUE", "FALSE", "1", "0", "Y", "N", - "y", "n", "T", "F"] - values = rng.choice(representations, n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_emails( - rng: np.random.Generator, n: int, invalid_rate: float, nan_rate: float -) -> np.ndarray: - """Email addresses with invalid entries.""" - values = np.empty(n, dtype=object) - for i in range(n): - user_len = rng.integers(5, 12) - user = "".join(rng.choice(list(string.ascii_lowercase), user_len)) - domain = EMAIL_DOMAINS[i % len(EMAIL_DOMAINS)] - values[i] = f"{user}@{domain}" - # Invalidate some - inv_mask = rng.random(n) < invalid_rate - invalid_emails = ["notanemail", "missing@", "@nodomain.com", "spaces in@email.com", - "double@@at.com", "no.tld@", ""] - values[inv_mask] = rng.choice(invalid_emails, inv_mask.sum()) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_phones( - rng: np.random.Generator, n: int, invalid_rate: float, nan_rate: float -) -> np.ndarray: - """Phone numbers in mixed formats.""" - values = np.empty(n, dtype=object) - formats_tpl = [ - "{area}-{pre}-{suf}", "({area}) {pre}-{suf}", - "+1{area}{pre}{suf}", "{area}.{pre}.{suf}", - "{area}{pre}{suf}", - ] - for i in range(n): - area = f"{rng.integers(200, 999)}" - pre = f"{rng.integers(200, 999)}" - suf = f"{rng.integers(1000, 9999)}" - fmt = formats_tpl[i % len(formats_tpl)] - values[i] = fmt.format(area=area, pre=pre, suf=suf) - inv_mask = rng.random(n) < invalid_rate - invalid_phones = ["123", "abc-def-ghij", "00000000000000", "+", "555"] - values[inv_mask] = rng.choice(invalid_phones, inv_mask.sum()) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_urls( - rng: np.random.Generator, n: int, invalid_rate: float, nan_rate: float -) -> np.ndarray: - """URLs with invalid entries.""" - tlds = ["com", "org", "net", "io", "dev"] - values = np.empty(n, dtype=object) - for i in range(n): - prefix = URL_PREFIXES[i % len(URL_PREFIXES)] - domain_len = rng.integers(4, 10) - domain = "".join(rng.choice(list(string.ascii_lowercase), domain_len)) - tld = tlds[i % len(tlds)] - values[i] = f"{prefix}{domain}.{tld}/path" - inv_mask = rng.random(n) < invalid_rate - invalid_urls = ["not a url", "://missing-scheme", "http://", "ftp:///", - "just-text", "http://a b c.com"] - values[inv_mask] = rng.choice(invalid_urls, inv_mask.sum()) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_currency( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Currency strings in mixed formats.""" - values = np.empty(n, dtype=object) - for i in range(n): - amount = rng.uniform(1.0, 99999.99) - symbol = CURRENCY_SYMBOLS[i % len(CURRENCY_SYMBOLS)] - # Randomly format with/without commas - if rng.random() < 0.5: - values[i] = f"{symbol}{amount:,.2f}" - else: - values[i] = f"{symbol}{amount:.2f}" - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_unicode( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Strings with mixed Unicode normalization and accented chars.""" - base_words = [ - "cafΓ©", "naΓ―ve", "rΓ©sumΓ©", "ΓΌber", "seΓ±or", - "Γ…ngstrΓΆm", "ZΓΌrich", "SΓ£o Paulo", "MalmΓΆ", "ζ—₯本θͺž", - "δΈ­ζ–‡", "ν•œκ΅­μ–΄", "Ψ§Ω„ΨΉΨ±Ψ¨ΩŠΨ©", "ΰ€Ήΰ€Ώΰ€¨ΰ₯ΰ€¦ΰ₯€", "normal_ascii", - ] - values = rng.choice(base_words, n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _null_heavy_column( - rng: np.random.Generator, n: int, null_rate: float -) -> np.ndarray: - """Column that is almost entirely null.""" - values = rng.normal(0, 1, n).astype(object) - values[rng.random(n) < null_rate] = None - return values - - -def _mixed_type_column(rng: np.random.Generator, n: int) -> np.ndarray: - """Column with mixed types: ints, floats, strings.""" - values = np.empty(n, dtype=object) - for i in range(n): - choice = i % 4 - if choice == 0: - values[i] = int(rng.integers(0, 1000)) - elif choice == 1: - values[i] = float(rng.normal(50, 10)) - elif choice == 2: - values[i] = f"text_{rng.integers(0, 100)}" - else: - values[i] = None - return values diff --git a/freshdata-benchmarks/benchmarks/utils/__init__.py b/freshdata-benchmarks/benchmarks/utils/__init__.py deleted file mode 100644 index 515b0afc..00000000 --- a/freshdata-benchmarks/benchmarks/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Benchmark utilities package.""" diff --git a/freshdata-benchmarks/benchmarks/utils/fairness.py b/freshdata-benchmarks/benchmarks/utils/fairness.py deleted file mode 100644 index d1d9d27c..00000000 --- a/freshdata-benchmarks/benchmarks/utils/fairness.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Fairness enforcement utilities for cross-library benchmarks. - -Ensures identical inputs, prevents mutation leakage, and controls for -garbage collection / cache effects between benchmark runs. -""" - -from __future__ import annotations - -import gc -import os -from typing import Any - - -def fresh_copy(df: Any) -> Any: - """Return a deep copy of a DataFrame for mutation-safe benchmarking. - - This ensures that each benchmark run operates on identical, - independent data β€” no library can benefit from prior mutations. - """ - return df.copy(deep=True) - - -def gc_collect() -> None: - """Force garbage collection to reduce memory measurement noise. - - Called in ``setup()`` before each benchmark to minimize the effect - of residual allocations from previous runs. - """ - gc.collect() - gc.collect() # Second pass catches weak-reference cleanup - - -def pin_threads(n: int = 1) -> None: - """Pin numeric libraries to *n* threads for deterministic timing. - - Already handled via asv.conf.json ``matrix.env``, but can be called - explicitly in ``setup()`` for defense-in-depth. - """ - for var in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", - "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS", - "VECLIB_MAXIMUM_THREADS"): - os.environ[var] = str(n) - - -def get_numeric_columns(df: Any) -> list[str]: - """Return column names that contain (or should contain) numeric data. - - Used to identify which columns are suitable for imputation, scaling, - and outlier detection benchmarks. - """ - import pandas as pd - numeric = [] - for col in df.columns: - try: - series = pd.to_numeric(df[col], errors="coerce") - non_null = series.notna().sum() - if non_null > len(df) * 0.3: # at least 30% parseable - numeric.append(col) - except Exception: - pass - return numeric - - -def get_string_columns(df: Any) -> list[str]: - """Return column names that contain string/object data.""" - return [col for col in df.columns if df[col].dtype == object] - - -def get_categorical_columns(df: Any, max_cardinality: int = 100) -> list[str]: - """Return string columns with cardinality ≀ *max_cardinality*.""" - cats = [] - for col in get_string_columns(df): - try: - nunique = df[col].nunique() - if 2 <= nunique <= max_cardinality: - cats.append(col) - except Exception: - pass - return cats - - -def validate_equivalent_output( - original: Any, - result: Any, - operation: str, -) -> bool: - """Spot-check that a benchmark result is semantically valid. - - This is NOT called during timing β€” only during development/testing - to verify that library adapters produce correct results. - """ - if result is None: - return False - import pandas as pd - if isinstance(result, pd.DataFrame): - # Basic sanity: result should have at least 1 row - return len(result) > 0 - # Non-DataFrame results (e.g., Polars, boolean masks) are valid - return True diff --git a/freshdata-benchmarks/benchmarks/utils/library_wrappers.py b/freshdata-benchmarks/benchmarks/utils/library_wrappers.py deleted file mode 100644 index 301c8105..00000000 --- a/freshdata-benchmarks/benchmarks/utils/library_wrappers.py +++ /dev/null @@ -1,976 +0,0 @@ -"""Library adapter wrappers for fair cross-library benchmarking. - -Each adapter implements equivalent operations using the target library's -native API. Adapters that cannot be imported set ``available = False`` -and all their methods raise ``NotImplementedError``, which ASV treats as -a skip. - -Usage:: - - adapter = get_adapter("pandas") - adapter.drop_missing_rows(df) -""" - -from __future__ import annotations - -import abc -import gc -from typing import Any - - -# --------------------------------------------------------------------------- -# Base adapter -# --------------------------------------------------------------------------- - -class LibraryAdapter(abc.ABC): - """Abstract base class for library-specific operation wrappers.""" - - name: str = "base" - available: bool = False - - def _skip(self, op: str) -> None: - raise NotImplementedError(f"{self.name} does not support {op}") - - # --- Missing values --- - def drop_missing_rows(self, df: Any) -> Any: - self._skip("drop_missing_rows") - - def fill_missing_mean(self, df: Any, columns: list[str]) -> Any: - self._skip("fill_missing_mean") - - def fill_missing_median(self, df: Any, columns: list[str]) -> Any: - self._skip("fill_missing_median") - - def fill_missing_mode(self, df: Any, columns: list[str]) -> Any: - self._skip("fill_missing_mode") - - def fill_missing_ffill(self, df: Any) -> Any: - self._skip("fill_missing_ffill") - - def fill_missing_bfill(self, df: Any) -> Any: - self._skip("fill_missing_bfill") - - # --- Duplicates --- - def detect_duplicates(self, df: Any) -> Any: - self._skip("detect_duplicates") - - def drop_duplicates(self, df: Any) -> Any: - self._skip("drop_duplicates") - - # --- String cleaning --- - def trim_whitespace(self, df: Any, columns: list[str]) -> Any: - self._skip("trim_whitespace") - - def to_lowercase(self, df: Any, columns: list[str]) -> Any: - self._skip("to_lowercase") - - def to_uppercase(self, df: Any, columns: list[str]) -> Any: - self._skip("to_uppercase") - - def regex_replace(self, df: Any, columns: list[str], - pattern: str, replacement: str) -> Any: - self._skip("regex_replace") - - # --- Column operations --- - def rename_columns(self, df: Any, mapping: dict) -> Any: - self._skip("rename_columns") - - def drop_columns(self, df: Any, columns: list[str]) -> Any: - self._skip("drop_columns") - - def select_columns(self, df: Any, columns: list[str]) -> Any: - self._skip("select_columns") - - # --- Type conversion --- - def convert_numeric(self, df: Any, columns: list[str]) -> Any: - self._skip("convert_numeric") - - def convert_datetime(self, df: Any, columns: list[str]) -> Any: - self._skip("convert_datetime") - - def optimize_dtypes(self, df: Any) -> Any: - self._skip("optimize_dtypes") - - # --- Encoding --- - def onehot_encode(self, df: Any, columns: list[str]) -> Any: - self._skip("onehot_encode") - - def label_encode(self, df: Any, columns: list[str]) -> Any: - self._skip("label_encode") - - # --- Scaling --- - def standard_scale(self, df: Any, columns: list[str]) -> Any: - self._skip("standard_scale") - - def minmax_scale(self, df: Any, columns: list[str]) -> Any: - self._skip("minmax_scale") - - # --- Outlier detection --- - def detect_outliers_iqr(self, df: Any, columns: list[str]) -> Any: - self._skip("detect_outliers_iqr") - - def detect_outliers_zscore(self, df: Any, columns: list[str]) -> Any: - self._skip("detect_outliers_zscore") - - # --- Group aggregations --- - def group_agg_single(self, df: Any) -> Any: - self._skip("group_agg_single") - - def group_agg_multi(self, df: Any, columns: list[str]) -> Any: - self._skip("group_agg_multi") - - def group_agg_transform(self, df: Any, columns: list[str]) -> Any: - self._skip("group_agg_transform") - - # --- Pipeline --- - def full_clean(self, df: Any) -> Any: - self._skip("full_clean") - - -# --------------------------------------------------------------------------- -# FreshData adapter -# --------------------------------------------------------------------------- - -class FreshDataAdapter(LibraryAdapter): - name = "freshdata" - - def __init__(self) -> None: - try: - import freshdata # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def drop_missing_rows(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", impute=False, - outliers=False, preserve_original=True) - - def fill_missing_mean(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="aggressive", preserve_original=True) - - def fill_missing_median(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="balanced", preserve_original=True) - - def fill_missing_mode(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="balanced", preserve_original=True) - - def fill_missing_ffill(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def fill_missing_bfill(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def detect_duplicates(self, df): - import freshdata as fd - return fd.profile(df) - - def drop_duplicates(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def trim_whitespace(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def to_lowercase(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def to_uppercase(self, df, columns): - # FreshData normalizes to lowercase, not uppercase - self._skip("to_uppercase") - - def rename_columns(self, df, mapping): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def drop_columns(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def select_columns(self, df, columns): - # FreshData doesn't have explicit column selection - self._skip("select_columns") - - def convert_numeric(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def convert_datetime(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def optimize_dtypes(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def detect_outliers_iqr(self, df, columns): - import freshdata as fd - return fd.clean(df, outlier_method="iqr", outlier_action="flag", - strategy="balanced", preserve_original=True) - - def detect_outliers_zscore(self, df, columns): - import freshdata as fd - return fd.clean(df, outlier_method="zscore", outlier_action="flag", - strategy="balanced", preserve_original=True) - - def group_agg_single(self, df): - import freshdata as fd - return fd.profile(df) - - def group_agg_multi(self, df, columns): - import freshdata as fd - return fd.profile(df) - - def group_agg_transform(self, df, columns): - import freshdata as fd - return fd.profile(df) - - def full_clean(self, df): - import freshdata as fd - return fd.clean(df, strategy="balanced", preserve_original=True) - - -# --------------------------------------------------------------------------- -# Pandas adapter -# --------------------------------------------------------------------------- - -class PandasAdapter(LibraryAdapter): - name = "pandas" - - def __init__(self) -> None: - try: - import pandas # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def drop_missing_rows(self, df): - import pandas as pd - return df.dropna() - - def fill_missing_mean(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - result[col] = numeric.fillna(numeric.mean()) - return result - - def fill_missing_median(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - result[col] = numeric.fillna(numeric.median()) - return result - - def fill_missing_mode(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - mode_val = result[col].mode() - if len(mode_val) > 0: - result[col] = result[col].fillna(mode_val.iloc[0]) - return result - - def fill_missing_ffill(self, df): - return df.ffill() - - def fill_missing_bfill(self, df): - return df.bfill() - - def detect_duplicates(self, df): - return df.duplicated() - - def drop_duplicates(self, df): - return df.drop_duplicates() - - def trim_whitespace(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.strip() - return result - - def to_lowercase(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.lower() - return result - - def to_uppercase(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.upper() - return result - - def regex_replace(self, df, columns, pattern, replacement): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.replace( - pattern, replacement, regex=True - ) - return result - - def rename_columns(self, df, mapping): - return df.rename(columns=mapping) - - def drop_columns(self, df, columns): - existing = [c for c in columns if c in df.columns] - return df.drop(columns=existing) - - def select_columns(self, df, columns): - existing = [c for c in columns if c in df.columns] - return df[existing] - - def convert_numeric(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - result[col] = pd.to_numeric(result[col], errors="coerce") - return result - - def convert_datetime(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - result[col] = pd.to_datetime(result[col], errors="coerce") - return result - - def optimize_dtypes(self, df): - return df.convert_dtypes() - - def onehot_encode(self, df, columns): - import pandas as pd - existing = [c for c in columns if c in df.columns] - return pd.get_dummies(df, columns=existing) - - def label_encode(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - result[col] = result[col].astype("category").cat.codes - return result - - def standard_scale(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - vals = result[col] - result[col] = (vals - vals.mean()) / vals.std() - return result - - def minmax_scale(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - vals = result[col] - mn, mx = vals.min(), vals.max() - if mx != mn: - result[col] = (vals - mn) / (mx - mn) - return result - - def detect_outliers_iqr(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - q1 = numeric.quantile(0.25) - q3 = numeric.quantile(0.75) - iqr = q3 - q1 - mask = (numeric < q1 - 1.5 * iqr) | (numeric > q3 + 1.5 * iqr) - result[f"{col}_outlier"] = mask - return result - - def detect_outliers_zscore(self, df, columns): - import numpy as np - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - mean = numeric.mean() - std = numeric.std() - if std > 0: - z = np.abs((numeric - mean) / std) - result[f"{col}_outlier"] = z > 3 - return result - - def group_agg_single(self, df): - """Single-column groupby with mean aggregation.""" - import pandas as pd - result = df.copy() - # Use CATEGORY_COL as the groupby key (high-cardinality categorical) - if "CATEGORY_COL" in result.columns and "float_col_1" in result.columns: - result["float_col_1"] = pd.to_numeric(result["float_col_1"], errors="coerce") - return result.groupby("CATEGORY_COL", observed=True).agg({"float_col_1": "mean"}) - # Fallback: group by the first object column - obj_cols = result.select_dtypes(include="object").columns - if len(obj_cols) > 0: - return result.groupby(obj_cols[0], observed=True).size() - return result.describe() - - def group_agg_multi(self, df, columns): - """Multi-column groupby with multiple agg functions.""" - import pandas as pd - result = df.copy() - if "CATEGORY_COL" in result.columns: - # Coerce numeric columns and build agg dict - agg_dict = {} - for col in columns: - if col in result.columns: - result[col] = pd.to_numeric(result[col], errors="coerce") - agg_dict[col] = ["mean", "std", "min", "max"] - if agg_dict: - return result.groupby("CATEGORY_COL", observed=True).agg(agg_dict) - # Fallback: describe all numeric - return result.describe() - - def group_agg_transform(self, df, columns): - """Groupby transform: center numeric columns within groups.""" - import pandas as pd - result = df.copy() - if "CATEGORY_COL" not in result.columns: - return result - for col in columns: - if col in result.columns: - try: - numeric_col = pd.to_numeric(result[col], errors="coerce") - result[col] = numeric_col - numeric_col.groupby( - result["CATEGORY_COL"] - ).transform("mean") - except Exception: - pass - return result - - def full_clean(self, df): - """Manual pandas cleaning pipeline equivalent to fd.clean().""" - result = df.copy() - # 1. Clean column names - result.columns = ( - result.columns.str.strip() - .str.lower() - .str.replace(r"[^a-z0-9_]", "_", regex=True) - .str.replace(r"_+", "_", regex=True) - .str.strip("_") - ) - # 2. Strip whitespace from string columns - str_cols = result.select_dtypes(include="object").columns - for col in str_cols: - try: - result[col] = result[col].str.strip() - except AttributeError: - pass - # 3. Replace sentinels with NaN - import numpy as np - sentinels = ["N/A", "n/a", "NA", "null", "None", "-", "", "#REF!", - "missing", "nan", "NaN"] - result = result.replace(sentinels, np.nan) - # 4. Drop all-empty columns - result = result.dropna(axis=1, how="all") - # 5. Drop all-empty rows - result = result.dropna(axis=0, how="all") - # 6. Drop duplicates - result = result.drop_duplicates() - # 7. Convert dtypes - result = result.convert_dtypes() - return result - - -# --------------------------------------------------------------------------- -# Polars adapter -# --------------------------------------------------------------------------- - -class PolarsAdapter(LibraryAdapter): - name = "polars" - - def __init__(self) -> None: - try: - import polars # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def _to_polars(self, df): - import polars as pl - import pandas as pd - df_clean = df.copy() - # PyArrow cannot convert mixed int/str object columns. Cast non-nulls to string. - for col in df_clean.select_dtypes(include=["object"]).columns: - mask = df_clean[col].notna() - df_clean.loc[mask, col] = df_clean.loc[mask, col].astype(str) - return pl.from_pandas(df_clean) - - def drop_missing_rows(self, df): - pldf = self._to_polars(df) - return pldf.drop_nulls() - - def fill_missing_mean(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - .fill_null(pl.col(col).cast(pl.Float64, strict=False).mean()) - ) - except Exception: - pass - return pldf - - def fill_missing_median(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - .fill_null(pl.col(col).cast(pl.Float64, strict=False).median()) - ) - except Exception: - pass - return pldf - - def fill_missing_ffill(self, df): - import polars as pl - pldf = self._to_polars(df) - return pldf.with_columns(pl.all().forward_fill()) - - def fill_missing_bfill(self, df): - import polars as pl - pldf = self._to_polars(df) - return pldf.with_columns(pl.all().backward_fill()) - - def detect_duplicates(self, df): - pldf = self._to_polars(df) - return pldf.is_duplicated() - - def drop_duplicates(self, df): - pldf = self._to_polars(df) - return pldf.unique() - - def trim_whitespace(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns(pl.col(col).str.strip_chars()) - except Exception: - pass - return pldf - - def to_lowercase(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns(pl.col(col).str.to_lowercase()) - except Exception: - pass - return pldf - - def to_uppercase(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns(pl.col(col).str.to_uppercase()) - except Exception: - pass - return pldf - - def regex_replace(self, df, columns, pattern, replacement): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).str.replace_all(pattern, replacement) - ) - except Exception: - pass - return pldf - - def rename_columns(self, df, mapping): - pldf = self._to_polars(df) - return pldf.rename(mapping) - - def drop_columns(self, df, columns): - pldf = self._to_polars(df) - existing = [c for c in columns if c in pldf.columns] - return pldf.drop(existing) - - def select_columns(self, df, columns): - pldf = self._to_polars(df) - existing = [c for c in columns if c in pldf.columns] - return pldf.select(existing) - - def convert_numeric(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - ) - except Exception: - pass - return pldf - - def detect_outliers_iqr(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - q1 = pldf[col].cast(pl.Float64, strict=False).quantile(0.25) - q3 = pldf[col].cast(pl.Float64, strict=False).quantile(0.75) - if q1 is not None and q3 is not None: - iqr = q3 - q1 - pldf = pldf.with_columns( - ((pl.col(col).cast(pl.Float64, strict=False) < q1 - 1.5 * iqr) - | (pl.col(col).cast(pl.Float64, strict=False) > q3 + 1.5 * iqr)) - .alias(f"{col}_outlier") - ) - except Exception: - pass - return pldf - - def full_clean(self, df): - """Manual Polars cleaning pipeline.""" - import polars as pl - pldf = self._to_polars(df) - # Clean column names - new_names = {} - for col in pldf.columns: - import re - clean = col.strip().lower() - clean = re.sub(r"[^a-z0-9_]", "_", clean) - clean = re.sub(r"_+", "_", clean).strip("_") - new_names[col] = clean - pldf = pldf.rename(new_names) - # Drop all-null columns - non_null_cols = [ - col for col in pldf.columns - if pldf[col].null_count() < len(pldf) - ] - pldf = pldf.select(non_null_cols) - # Drop duplicates - pldf = pldf.unique() - return pldf - - -# --------------------------------------------------------------------------- -# pyjanitor adapter -# --------------------------------------------------------------------------- - -class PyjanitorAdapter(LibraryAdapter): - name = "pyjanitor" - - def __init__(self) -> None: - try: - import janitor # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def drop_missing_rows(self, df): - import janitor # noqa: F401 - return df.remove_empty(reset_index=False) - - def drop_duplicates(self, df): - return df.drop_duplicates() - - def trim_whitespace(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.strip() - return result - - def to_lowercase(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.lower() - return result - - def rename_columns(self, df, mapping): - import janitor # noqa: F401 - return df.clean_names() - - def drop_columns(self, df, columns): - import janitor # noqa: F401 - return df.remove_columns(columns) - - def select_columns(self, df, columns): - import janitor # noqa: F401 - return df.select_columns(columns) - - def onehot_encode(self, df, columns): - import janitor # noqa: F401 - result = df - for col in columns: - if col in result.columns: - try: - result = result.encode_categorical(col) - except Exception: - pass - return result - - def full_clean(self, df): - import janitor # noqa: F401 - return df.clean_names().remove_empty(reset_index=False).drop_duplicates() - - -# --------------------------------------------------------------------------- -# Feature Engine adapter -# --------------------------------------------------------------------------- - -class FeatureEngineAdapter(LibraryAdapter): - name = "feature_engine" - - def __init__(self) -> None: - try: - import feature_engine # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def fill_missing_mean(self, df, columns): - from feature_engine.imputation import MeanMedianImputer - existing = [c for c in columns if c in df.columns] - if not existing: - return df - imp = MeanMedianImputer(imputation_method="mean", variables=existing) - return imp.fit_transform(df) - - def fill_missing_median(self, df, columns): - from feature_engine.imputation import MeanMedianImputer - existing = [c for c in columns if c in df.columns] - if not existing: - return df - imp = MeanMedianImputer(imputation_method="median", variables=existing) - return imp.fit_transform(df) - - def onehot_encode(self, df, columns): - from feature_engine.encoding import OneHotEncoder - existing = [c for c in columns if c in df.columns] - if not existing: - return df - enc = OneHotEncoder(variables=existing, drop_last=False) - return enc.fit_transform(df) - - def label_encode(self, df, columns): - from feature_engine.encoding import OrdinalEncoder - existing = [c for c in columns if c in df.columns] - if not existing: - return df - enc = OrdinalEncoder(variables=existing) - return enc.fit_transform(df) - - def detect_outliers_iqr(self, df, columns): - from feature_engine.outliers import Winsorizer - existing = [c for c in columns if c in df.columns] - if not existing: - return df - w = Winsorizer(capping_method="iqr", fold=1.5, variables=existing) - return w.fit_transform(df) - - def standard_scale(self, df, columns): - from sklearn.preprocessing import StandardScaler - import pandas as pd - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = StandardScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - def minmax_scale(self, df, columns): - from sklearn.preprocessing import MinMaxScaler - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = MinMaxScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - -# --------------------------------------------------------------------------- -# Scikit-learn adapter -# --------------------------------------------------------------------------- - -class SklearnAdapter(LibraryAdapter): - name = "sklearn" - - def __init__(self) -> None: - try: - import sklearn # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def fill_missing_mean(self, df, columns): - from sklearn.impute import SimpleImputer - import numpy as np - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - imp = SimpleImputer(strategy="mean") - result[existing] = imp.fit_transform(result[existing]) - return result - - def fill_missing_median(self, df, columns): - from sklearn.impute import SimpleImputer - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - imp = SimpleImputer(strategy="median") - result[existing] = imp.fit_transform(result[existing]) - return result - - def fill_missing_mode(self, df, columns): - from sklearn.impute import SimpleImputer - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - imp = SimpleImputer(strategy="most_frequent") - result[existing] = imp.fit_transform(result[existing]) - return result - - def onehot_encode(self, df, columns): - from sklearn.preprocessing import OneHotEncoder - import pandas as pd - import numpy as np - result = df.copy() - existing = [c for c in columns if c in result.columns] - if not existing: - return result - enc = OneHotEncoder(sparse_output=False, handle_unknown="ignore") - encoded = enc.fit_transform(result[existing].astype(str)) - feature_names = enc.get_feature_names_out(existing) - encoded_df = pd.DataFrame(encoded, columns=feature_names, - index=result.index) - result = result.drop(columns=existing) - return pd.concat([result, encoded_df], axis=1) - - def label_encode(self, df, columns): - from sklearn.preprocessing import LabelEncoder - result = df.copy() - for col in columns: - if col in result.columns: - le = LabelEncoder() - mask = result[col].notna() - if mask.any(): - result.loc[mask, col] = le.fit_transform( - result.loc[mask, col].astype(str) - ) - return result - - def standard_scale(self, df, columns): - from sklearn.preprocessing import StandardScaler - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = StandardScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - def minmax_scale(self, df, columns): - from sklearn.preprocessing import MinMaxScaler - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = MinMaxScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - def detect_outliers_zscore(self, df, columns): - import numpy as np - result = df.copy() - for col in columns: - if col in result.columns: - mean = result[col].mean() - std = result[col].std() - if std > 0: - z = np.abs((result[col] - mean) / std) - result[f"{col}_outlier"] = z > 3 - return result - - -# --------------------------------------------------------------------------- -# AutoClean adapter (may not be available) -# --------------------------------------------------------------------------- - -class AutoCleanAdapter(LibraryAdapter): - name = "autoclean" - - def __init__(self) -> None: - try: - from AutoClean import AutoClean # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def full_clean(self, df): - from AutoClean import AutoClean - pipeline = AutoClean(df.copy(), mode="auto", duplicates=True, - missing_num="auto", missing_categ="auto", - outliers="winz") - return pipeline.output - - -# --------------------------------------------------------------------------- -# Adapter registry -# --------------------------------------------------------------------------- - -_ADAPTERS: dict[str, type[LibraryAdapter]] = { - "freshdata": FreshDataAdapter, - "pandas": PandasAdapter, - "polars": PolarsAdapter, - "pyjanitor": PyjanitorAdapter, - "feature_engine": FeatureEngineAdapter, - "sklearn": SklearnAdapter, - "autoclean": AutoCleanAdapter, -} - -_adapter_cache: dict[str, LibraryAdapter] = {} - - -def get_adapter(name: str) -> LibraryAdapter: - """Get a cached adapter instance by library name.""" - if name not in _adapter_cache: - cls = _ADAPTERS.get(name) - if cls is None: - raise ValueError(f"Unknown library: {name!r}. " - f"Available: {list(_ADAPTERS)}") - _adapter_cache[name] = cls() - return _adapter_cache[name] - - -def available_libraries() -> list[str]: - """Return names of libraries that are importable.""" - return [name for name, cls in _ADAPTERS.items() if cls().available] diff --git a/freshdata-benchmarks/benchmarks/utils/metrics.py b/freshdata-benchmarks/benchmarks/utils/metrics.py deleted file mode 100644 index 93b24426..00000000 --- a/freshdata-benchmarks/benchmarks/utils/metrics.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Custom metric calculators for benchmark reports.""" - -from __future__ import annotations - - -def throughput(n_rows: int, elapsed_seconds: float) -> float: - """Compute throughput in rows per second.""" - if elapsed_seconds <= 0: - return float("inf") - return n_rows / elapsed_seconds - - -def speedup(baseline_time: float, comparison_time: float) -> float: - """Compute speedup ratio (baseline / comparison). - - A value > 1.0 means *comparison* is faster than *baseline*. - """ - if comparison_time <= 0: - return float("inf") - return baseline_time / comparison_time - - -def memory_ratio(baseline_bytes: float, comparison_bytes: float) -> float: - """Compute relative memory usage (comparison / baseline). - - A value < 1.0 means *comparison* uses less memory. - """ - if baseline_bytes <= 0: - return float("inf") - return comparison_bytes / baseline_bytes - - -def scaling_efficiency( - time_small: float, - time_large: float, - n_small: int, - n_large: int, -) -> float: - """Estimate scaling efficiency. - - Perfect linear scaling = 1.0. Sublinear > 1.0. Superlinear < 1.0. - """ - if time_small <= 0 or time_large <= 0: - return float("nan") - data_ratio = n_large / n_small - time_ratio = time_large / time_small - return data_ratio / time_ratio - - -def estimate_bigO(times: list[float], sizes: list[int]) -> str: - """Estimate Big-O complexity from empirical timing data. - - Returns a human-readable string like "O(n)", "O(n log n)", "O(nΒ²)". - """ - import math - - if len(times) < 2 or len(sizes) < 2: - return "insufficient data" - - # Compute growth rates between consecutive points - ratios = [] - for i in range(1, len(times)): - if times[i - 1] > 0 and sizes[i - 1] > 0: - size_ratio = sizes[i] / sizes[i - 1] - time_ratio = times[i] / times[i - 1] - if size_ratio > 0 and time_ratio > 0: - exponent = math.log(time_ratio) / math.log(size_ratio) - ratios.append(exponent) - - if not ratios: - return "unknown" - - avg_exp = sum(ratios) / len(ratios) - - if avg_exp < 0.3: - return "O(1)" - elif avg_exp < 0.7: - return "O(√n)" - elif avg_exp < 1.3: - return "O(n)" - elif avg_exp < 1.7: - return "O(n log n)" - elif avg_exp < 2.3: - return "O(nΒ²)" - else: - return f"O(n^{avg_exp:.1f})" diff --git a/freshdata-benchmarks/reports/results_extracted.json b/freshdata-benchmarks/reports/results_extracted.json deleted file mode 100644 index 9e26dfee..00000000 --- a/freshdata-benchmarks/reports/results_extracted.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/freshdata-benchmarks/reports/summary.md b/freshdata-benchmarks/reports/summary.md deleted file mode 100644 index 7d130cca..00000000 --- a/freshdata-benchmarks/reports/summary.md +++ /dev/null @@ -1,2 +0,0 @@ -# FreshData Benchmark Results - diff --git a/freshdata-benchmarks/requirements.txt b/freshdata-benchmarks/requirements.txt deleted file mode 100644 index 1f4250a7..00000000 --- a/freshdata-benchmarks/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -# FreshData Benchmark Suite β€” Dependencies -# ========================================== -# Core benchmark target -freshdata-cleaner[all] - -# Comparison libraries -pandas>=2.0 -polars>=0.20 -pyjanitor>=0.27 -feature-engine>=1.6 -scikit-learn>=1.3 - -# Benchmark framework -asv>=0.6 - -# Data generation & measurement -numpy>=1.24 -pyarrow>=10 -psutil>=5.9 - -# Report generation -matplotlib>=3.7 -tabulate>=0.9 - -# Optional comparison libraries (may fail to install) -# AutoClean>=1.1 # Inactive β€” install manually if needed -# datacleaner # Abandoned β€” install manually if needed diff --git a/freshdata-benchmarks/scripts/capture_env.py b/freshdata-benchmarks/scripts/capture_env.py deleted file mode 100644 index 2b2d8193..00000000 --- a/freshdata-benchmarks/scripts/capture_env.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Capture environment details for reproducible benchmarking. - -Outputs a JSON file with Python version, OS, CPU, RAM, and installed -package versions (freshdata, pandas, numpy, pyarrow). - -Usage:: - - python scripts/capture_env.py [output_path] - # Defaults to bench/artifacts/env_info.json - -""" - -from __future__ import annotations - -import json -import os -import platform -import sys -from datetime import datetime, timezone -from pathlib import Path - - -def get_package_version(name: str) -> str: - """Return installed version of a package, or 'not installed'.""" - try: - from importlib.metadata import version - - return version(name) - except Exception: - return "not installed" - - -def get_cpu_info() -> dict: - """Collect CPU information.""" - info = { - "model": platform.processor() or "unknown", - "count_logical": os.cpu_count() or 0, - } - try: - import psutil - - info["count_physical"] = psutil.cpu_count(logical=False) or 0 - freq = psutil.cpu_freq() - if freq: - info["freq_mhz"] = freq.current - except ImportError: - pass - return info - - -def get_ram_gb() -> float: - """Return total system RAM in GB.""" - try: - import psutil - - return round(psutil.virtual_memory().total / (1024**3), 2) - except ImportError: - return 0.0 - - -def capture_environment() -> dict: - """Build a complete environment snapshot.""" - return { - "timestamp": datetime.now(timezone.utc).isoformat(), - "hostname": platform.node(), - "python": { - "version": platform.python_version(), - "implementation": platform.python_implementation(), - "compiler": platform.python_compiler(), - }, - "os": { - "system": platform.system(), - "release": platform.release(), - "version": platform.version(), - "machine": platform.machine(), - }, - "cpu": get_cpu_info(), - "ram_gb": get_ram_gb(), - "packages": { - "freshdata-cleaner": get_package_version("freshdata-cleaner"), - "pandas": get_package_version("pandas"), - "numpy": get_package_version("numpy"), - "pyarrow": get_package_version("pyarrow"), - "asv": get_package_version("asv"), - "psutil": get_package_version("psutil"), - }, - } - - -def main() -> None: - output_path = sys.argv[1] if len(sys.argv) > 1 else "bench/artifacts/env_info.json" - output = Path(output_path) - output.parent.mkdir(parents=True, exist_ok=True) - - env = capture_environment() - output.write_text(json.dumps(env, indent=2) + "\n") - print(f"Environment captured β†’ {output}") - print(json.dumps(env, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/freshdata-benchmarks/scripts/generate_plots.py b/freshdata-benchmarks/scripts/generate_plots.py deleted file mode 100644 index 72eb9a42..00000000 --- a/freshdata-benchmarks/scripts/generate_plots.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Generate visualization plots from ASV benchmark results.""" - -import json -import os -import pathlib -import sys -from collections import defaultdict - -import matplotlib.pyplot as plt -import numpy as np - -PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[1] -REPORTS_DIR = PROJECT_ROOT / "reports" -PLOTS_DIR = REPORTS_DIR / "plots" - - -def load_extracted_results(): - json_path = REPORTS_DIR / "results_extracted.json" - if not json_path.exists(): - print(f"Error: {json_path} not found.", file=sys.stderr) - print("Run 'python scripts/generate_reports.py' first.", file=sys.stderr) - sys.exit(1) - - with open(json_path) as f: - return json.load(f) - - -def plot_scaling_curves(results): - """Plot execution time vs dataset size for the full pipeline.""" - pipeline_data = results.get("BenchmarkFreshDataVsPandas", {}).get("time_full_pipeline", {}) - if not pipeline_data: - print("No BenchmarkFreshDataVsPandas data found. Skipping scaling curves.") - return - - row_sizes_str = pipeline_data.get("params", [])[0] - libraries = pipeline_data.get("params", [])[1] - raw_results = pipeline_data.get("result", []) - - if not row_sizes_str or not libraries or not raw_results: - return - - row_sizes = [int(s.replace("_", "")) for s in row_sizes_str] - - plt.figure(figsize=(10, 6)) - - idx = 0 - for lib in libraries: - times = [] - for _ in row_sizes: - val = raw_results[idx] - times.append(val if val is not None and str(val) != "nan" else np.nan) - idx += 1 - - plt.plot(row_sizes, times, marker='o', linewidth=2, label=lib) - - plt.xscale('log') - plt.yscale('log') - plt.title('Execution Time vs. Dataset Size (Full Preprocessing Pipeline)') - plt.xlabel('Number of Rows (Log Scale)') - plt.ylabel('Execution Time in Seconds (Log Scale)') - plt.grid(True, which="both", ls="--", alpha=0.6) - plt.legend() - - out_path = PLOTS_DIR / "scaling_curves.png" - plt.savefig(out_path, dpi=300, bbox_inches='tight') - print(f"Saved: {out_path}") - plt.close() - - -def plot_speedup_bar(results): - """Plot speedup vs Pandas for different operations on the largest dataset.""" - # Find operations to plot - ops_to_plot = { - "Load CSV": ("BenchmarkFreshDataVsPandas", "time_load_csv"), - "Missing Values": ("BenchmarkFreshDataVsPandas", "time_handle_missing"), - "Outliers": ("BenchmarkFreshDataVsPandas", "time_detect_outliers"), - "Duplicates": ("BenchmarkFreshDataVsPandas", "time_resolve_duplicates"), - "Group Agg": ("BenchmarkFreshDataVsPandas", "time_group_aggregations"), - "Full Pipeline": ("BenchmarkFreshDataVsPandas", "time_full_pipeline"), - } - - speedups = defaultdict(dict) - - for op_name, (group, method) in ops_to_plot.items(): - data = results.get(group, {}).get(method, {}) - if not data: - continue - - libraries = data.get("params", [])[1] - raw_results = data.get("result", []) - row_sizes = data.get("params", [])[0] - - if "pandas" not in libraries: - continue - - # Get data for largest dataset (last index) - pandas_idx = libraries.index("pandas") * len(row_sizes) + (len(row_sizes) - 1) - pandas_time = raw_results[pandas_idx] - - if pandas_time is None or str(pandas_time) == "nan": - continue - - for lib_idx, lib in enumerate(libraries): - if lib == "pandas": - continue - val_idx = lib_idx * len(row_sizes) + (len(row_sizes) - 1) - val_time = raw_results[val_idx] - - if val_time is not None and str(val_time) != "nan" and val_time > 0: - speedups[op_name][lib] = pandas_time / val_time - - if not speedups: - print("No speedup data available.") - return - - # Plot - fig, ax = plt.subplots(figsize=(12, 7)) - - ops = list(speedups.keys()) - # Find all unique libraries across operations - all_libs = set() - for s in speedups.values(): - all_libs.update(s.keys()) - all_libs = sorted(list(all_libs)) - - x = np.arange(len(ops)) - width = 0.8 / len(all_libs) - - for i, lib in enumerate(all_libs): - values = [speedups[op].get(lib, 0) for op in ops] - ax.bar(x + i*width - width*(len(all_libs)/2 - 0.5), values, width, label=lib) - - # Add baseline - ax.axhline(y=1.0, color='r', linestyle='-', alpha=0.3, label='Pandas Baseline (1.0x)') - - ax.set_ylabel('Speedup Ratio (Higher is Better)') - ax.set_title('Speedup vs Pandas (Largest Dataset Size)') - ax.set_xticks(x) - ax.set_xticklabels(ops, rotation=45, ha='right') - ax.legend() - - plt.tight_layout() - out_path = PLOTS_DIR / "speedup_comparison.png" - plt.savefig(out_path, dpi=300) - print(f"Saved: {out_path}") - plt.close() - - -def plot_memory_bar(results): - """Plot memory reduction vs Pandas for different operations on the largest dataset.""" - ops_to_plot = { - "Load CSV": ("BenchmarkFreshDataVsPandas", "peakmem_load_csv"), - "Missing Values": ("BenchmarkFreshDataVsPandas", "peakmem_handle_missing"), - "Outliers": ("BenchmarkFreshDataVsPandas", "peakmem_detect_outliers"), - "Duplicates": ("BenchmarkFreshDataVsPandas", "peakmem_resolve_duplicates"), - "Group Agg": ("BenchmarkFreshDataVsPandas", "peakmem_group_aggregations"), - "Full Pipeline": ("BenchmarkFreshDataVsPandas", "peakmem_full_pipeline"), - } - - memory_ratios = defaultdict(dict) - - for op_name, (group, method) in ops_to_plot.items(): - data = results.get(group, {}).get(method, {}) - if not data: - continue - - libraries = data.get("params", [])[1] - raw_results = data.get("result", []) - row_sizes = data.get("params", [])[0] - - if "pandas" not in libraries: - continue - - pandas_idx = libraries.index("pandas") * len(row_sizes) + (len(row_sizes) - 1) - pandas_mem = raw_results[pandas_idx] - - if pandas_mem is None or str(pandas_mem) == "nan" or pandas_mem == 0: - continue - - for lib_idx, lib in enumerate(libraries): - if lib == "pandas": - continue - val_idx = lib_idx * len(row_sizes) + (len(row_sizes) - 1) - val_mem = raw_results[val_idx] - - if val_mem is not None and str(val_mem) != "nan" and val_mem > 0: - memory_ratios[op_name][lib] = pandas_mem / val_mem - - if not memory_ratios: - print("No memory data available.") - return - - fig, ax = plt.subplots(figsize=(12, 7)) - ops = list(memory_ratios.keys()) - all_libs = set() - for s in memory_ratios.values(): - all_libs.update(s.keys()) - all_libs = sorted(list(all_libs)) - - x = np.arange(len(ops)) - width = 0.8 / len(all_libs) - - for i, lib in enumerate(all_libs): - values = [memory_ratios[op].get(lib, 0) for op in ops] - ax.bar(x + i*width - width*(len(all_libs)/2 - 0.5), values, width, label=lib) - - ax.axhline(y=1.0, color='r', linestyle='-', alpha=0.3, label='Pandas Baseline (1.0x Memory)') - - ax.set_ylabel('Memory Efficiency Ratio (Higher is Better, 2.0 = half memory)') - ax.set_title('Memory Efficiency vs Pandas (Largest Dataset Size)') - ax.set_xticks(x) - ax.set_xticklabels(ops, rotation=45, ha='right') - ax.legend() - - plt.tight_layout() - out_path = PLOTS_DIR / "memory_comparison.png" - plt.savefig(out_path, dpi=300) - print(f"Saved: {out_path}") - plt.close() - - -def generate_plots(): - """Main entry point.""" - PLOTS_DIR.mkdir(exist_ok=True, parents=True) - results = load_extracted_results() - - plot_scaling_curves(results) - plot_speedup_bar(results) - plot_memory_bar(results) - - -if __name__ == "__main__": - generate_plots() diff --git a/freshdata-benchmarks/scripts/generate_reports.py b/freshdata-benchmarks/scripts/generate_reports.py deleted file mode 100644 index 35085b77..00000000 --- a/freshdata-benchmarks/scripts/generate_reports.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Generate Markdown and CSV reports from ASV benchmark results. - -Reads the latest JSON results generated by ASV and formats them into -human-readable tables, saving them to the 'reports' directory. -""" - -import json -import os -import pathlib -import sys -from collections import defaultdict -from typing import Any - -from tabulate import tabulate - -PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[1] -RESULTS_DIR = PROJECT_ROOT / ".asv" / "results" -REPORTS_DIR = PROJECT_ROOT / "reports" - - -def find_latest_results() -> dict[str, Any]: - """Find the most recent ASV results JSON file.""" - if not RESULTS_DIR.exists(): - print(f"Error: Results directory not found at {RESULTS_DIR}", file=sys.stderr) - print("Run 'asv run' first.", file=sys.stderr) - sys.exit(1) - - # ASV stores results under .asv/results/MACHINE/COMMIT.json - # We want the most recently modified JSON file - json_files = list(RESULTS_DIR.rglob("*.json")) - # Filter out machine.json - result_files = [f for f in json_files if f.name != "machine.json"] - - if not result_files: - print("Error: No benchmark results found.", file=sys.stderr) - sys.exit(1) - - latest_file = max(result_files, key=lambda p: p.stat().st_mtime) - print(f"Reading results from: {latest_file.relative_to(PROJECT_ROOT)}") - - with open(latest_file) as f: - return json.load(f) - - -def format_time(seconds: float | None) -> str: - """Format time in seconds to human readable string.""" - if seconds is None or str(seconds) == "nan": - return "N/A" - - if seconds < 0.001: - return f"{seconds * 1_000_000:.1f} ΞΌs" - elif seconds < 1.0: - return f"{seconds * 1000:.1f} ms" - else: - return f"{seconds:.2f} s" - - -def format_memory(bytes_val: float | None) -> str: - """Format memory in bytes to human readable string.""" - if bytes_val is None or str(bytes_val) == "nan": - return "N/A" - - mb = bytes_val / (1024 * 1024) - if mb > 1024: - return f"{mb / 1024:.2f} GB" - return f"{mb:.1f} MB" - - -def format_value(value: float | None, bench_type: str) -> str: - """Format value based on benchmark type.""" - if value is None or str(value) == "nan": - return "-" - - if bench_type.startswith("time_"): - return format_time(value) - elif bench_type.startswith("peakmem_"): - return format_memory(value) - elif bench_type.startswith("track_throughput"): - if value > 1_000_000: - return f"{value / 1_000_000:.1f}M/s" - elif value > 1_000: - return f"{value / 1_000:.1f}K/s" - return f"{value:.0f}/s" - elif bench_type.startswith("track_speedup"): - return f"{value:.1f}x" - else: - # Default numeric format - if isinstance(value, float): - return f"{value:.2f}" - return str(value) - - -def generate_reports() -> None: - """Generate Markdown and CSV reports.""" - REPORTS_DIR.mkdir(exist_ok=True) - results = find_latest_results() - - # Restructure data: bench_name -> {param_combo -> result_value} - # ASV stores results as a dict where keys are benchmark names - # and values are dicts containing 'params' and 'result' - - reports_md = ["# FreshData Benchmark Results\n\n"] - - # Group benchmarks by file (e.g. benchmark_missing) - bench_groups = defaultdict(dict) - - for bench_name, bench_data in results.get("results", {}).items(): - if not isinstance(bench_data, dict) or "result" not in bench_data: - continue - - parts = bench_name.split(".") - if len(parts) < 3: - continue - - group_name = parts[-2] # e.g., BenchmarkMissing - method_name = parts[-1] # e.g., time_drop_missing - - bench_groups[group_name][method_name] = bench_data - - # Generate a table for each group - for group_name, methods in sorted(bench_groups.items()): - reports_md.append(f"## {group_name}\n") - - for method_name, data in sorted(methods.items()): - reports_md.append(f"### `{method_name}`\n") - - param_names = data.get("params", []) - # Skip non-parameterized or oddly parameterized methods for simple tables - if not param_names or len(param_names) != 2: - reports_md.append("_Requires 2 parameters (n_rows, library) for standard formatting._\n\n") - continue - - # Usually: params = [["10000", "100000"], ["freshdata", "pandas", ...]] - row_sizes = data.get("params")[0] - libraries = data.get("params")[1] - - result_list = data.get("result", []) - - # ASV results list is flat, ordered by cartesian product of params - headers = ["Library"] + [f"{size} rows" for size in row_sizes] - table_data = [] - - idx = 0 - for lib in libraries: - row = [lib] - for _ in row_sizes: - val = result_list[idx] if idx < len(result_list) else None - row.append(format_value(val, method_name)) - idx += 1 - table_data.append(row) - - md_table = tabulate(table_data, headers=headers, tablefmt="github") - reports_md.append(md_table + "\n\n") - - # Write Markdown report - md_path = REPORTS_DIR / "summary.md" - with open(md_path, "w") as f: - f.write("".join(reports_md)) - - print(f"Generated Markdown report: {md_path}") - - # Save raw structured data - json_path = REPORTS_DIR / "results_extracted.json" - with open(json_path, "w") as f: - json.dump(bench_groups, f, indent=2) - - print(f"Generated JSON extract: {json_path}") - - -if __name__ == "__main__": - generate_reports() diff --git a/freshdata-benchmarks/setup.py b/freshdata-benchmarks/setup.py deleted file mode 100644 index eb1add3c..00000000 --- a/freshdata-benchmarks/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Minimal setup.py for ASV virtualenv detection.""" - -from setuptools import setup - -setup( - name="freshdata-benchmarks", - version="0.1.0", - description="ASV benchmark suite for FreshData", - python_requires=">=3.9", -) diff --git a/growth-audit.md b/growth-audit.md deleted file mode 100644 index a8e44d21..00000000 --- a/growth-audit.md +++ /dev/null @@ -1,56 +0,0 @@ -# FreshData Open-Source Growth & Repository Baseline Audit - -This audit evaluates the baseline state of `FreshCode-Org/freshdata` as of September 2026, based strictly on empirical evidence inspected across the codebase, packaging metadata, documentation, GitHub configurations, benchmarks, and test suites. - ---- - -## 1. Executive Summary - -| Area | Grade | Key Strengths | Critical Gaps | -|---|:---:|---|---| -| **Repository Positioning** | C+ | Strong engineering foundation; accurate claims in docs | README buries value below fold; lacks 10-second mental model; no before/after visual proof table | -| **Developer Onboarding** | B- | Fast local test execution; Python 3.9–3.13 support | PyPI package name (`freshdata-cleaner`) differs from import (`freshdata`), creating install friction | -| **Discoverability** | C | Quality keywords in `pyproject.toml` | Missing high-intent developer topics on GitHub; no searchable cookbook for SEO | -| **Community & Contribution** | C+ | High-quality `CONTRIBUTING.md` and 93% test coverage gate | Contributor roadmap references already-closed issues; no beginner walkthrough (`first-contribution.md`) | -| **Proof & Trust** | B+ | 9 standardized metrics; reproducible seed-controlled harness | Benchmark README lacks scenario breakdown and side-by-side invariant safety proof | -| **Distribution & Advocacy** | D+ | Well-written technical documentation site | No marketing assets, no social post engine, no structured community release cadence | - ---- - -## 2. Detailed Audit Matrix - -| Area | Current State | Evidence | Problem | Impact | Effort | Priority | Recommended Action | -|---|---|---|---|---|---|:---:|---| -| **Positioning** | Tagline: "The explainable cleaning layer for pandas β€” decision-preserving data hygiene." | `README.md:7` | "Decision-preserving data hygiene" is abstract jargon; doesn't highlight Polars or explain what the tool *does* in 10s. | High drop-off from search & GitHub discovery. | Low | **P0** | Rewrite to: "Automated, explainable data cleaning for pandas and Polars β€” repair messy tabular data safely and see exactly what changed." | -| **Positioning** | README hero lacks 10-second category mental model. | `README.md:26-38` | Visitors don't immediately know how FreshData relates to Profilers vs Validators vs ML quality tools. | Confusion about whether FreshData replaces pandas, Great Expectations, or ydata-profiling. | Low | **P0** | Add comparison matrix: Profilers (describe), Validators (test rules), ML-DQ (detect issues), FreshData (safely repair & explain). | -| **Positioning** | No before/after visual demonstration. | `README.md:91-109` | Current quickstart shows only a summary text without showing the messy input DataFrame and resulting output. | Visitors cannot visualize the transformation or trust the result without running code. | Low | **P0** | Insert a compact, tabular Before $\rightarrow$ Detection $\rightarrow$ Decision $\rightarrow$ Repair $\rightarrow$ Audit block. | -| **Onboarding** | Package naming mismatch: PyPI is `freshdata-cleaner`, import is `freshdata`. | `pyproject.toml:7-9` | Users typing `pip install freshdata` fail or get wrong package; README warning is buried under install code block. | Direct drop-off during first onboarding step. | Low | **P0** | Place high-visibility installation callout banner at the very top of README and docs. | -| **Onboarding** | First-time contributor path requires reading full architecture. | `CONTRIBUTING.md:11-12`, `docs/community/first-pr.md` | `first-pr.md` is an index of labels rather than a step-by-step tutorial (clone $\rightarrow$ venv $\rightarrow$ test $\rightarrow$ edit $\rightarrow$ PR). | First-time contributors hesitate to take on issues. | Medium | **P1** | Create `docs/contributing/first-contribution.md` with a zero-friction, copy-paste workflow. | -| **Discoverability** | GitHub Topics on repo are minimal/unoptimized. | GitHub metadata & `pyproject.toml:18-35` | Repository is not ranking for high-intent search terms like `data-cleaning`, `missing-values`, `data-quality`, `polars`. | Low organic GitHub search discovery. | Low | **P0** | Update GitHub repository topics with 15 developer search intent keywords (documented in `admin-actions.md`). | -| **Discoverability** | No problem-based searchable cookbook. | `docs/` directory | Documentation is organized by library subpackage rather than developer search queries. | Developers searching "how to handle missing values pandas" do not land on FreshData. | Medium | **P1** | Build `docs/cookbook/` with 7+ search-intent-focused problem/code/output guides. | -| **Community** | Missing issue templates for integrations and benchmarks. | `.github/ISSUE_TEMPLATE/` | Only `bug_report.yml`, `feature_request.yml`, and `documentation.yml` exist. No structured way to report benchmarks or request ecosystem integrations. | Reduced contributor engagement on performance and ecosystem growth. | Low | **P1** | Add `integration_request.yml` and `benchmark_result.yml` issue forms. | -| **Community** | PR template lacks performance and compatibility impact prompts. | `.github/pull_request_template.md` | PR template does not prompt contributors to document performance impact, backward compatibility, or docs updates. | Review overhead for maintainers; accidental performance regressions. | Low | **P1** | Update PR template with structured questions (*Why?*, *How tested?*, *Performance impact?*, *Docs updated?*). | -| **Community** | Contributor roadmap references merged/closed issues. | `docs/community/contributor-roadmap.md:34,88-92,107` | Mentions `clean_csv` (#19), `pyjanitor` (#9), `duckdb out-of-core` (#52, #53), which are already merged. | Contributors waste time investigating already-solved tasks or lose confidence in roadmap. | Medium | **P1** | Overhaul `contributor-roadmap.md` with 10+ fresh, verified open contributor tasks across Level 1, 2, and 3. | -| **Proof & Safety** | Safety invariants (refusal to touch IDs, target columns) not demonstrated visibly. | `README.md`, `benchmarks/` | Users fear automated cleaners will corrupt primary keys, leak target labels, or silently invent fake data. | Adoption resistance in production environments. | Medium | **P0** | Add "Safety / Trust Proof" section with concrete examples of what FreshData explicitly refuses to alter. | -| **Proof & Safety** | Benchmark README lacks execution commands and environment details. | `benchmarks/README.md` | Benchmarks are mentioned, but reproducibility steps and hardware requirements are brief. | Developers doubt performance claims without clear local reproduction commands. | Medium | **P1** | Overhaul `benchmarks/README.md` documenting datasets, 9 standardized metrics, runner commands, and limitations. | -| **Examples** | Flagship examples in `examples/` are fragmented and lack numbered consistency. | `examples/` directory | Missing `01_csv_cleaning.py` and `03_duplicate_detection.py`; integrations are scattered. | Inconsistent developer trial experience. | Medium | **P1** | Standardize `examples/01` to `05` and add dedicated `examples/integrations/` recipes. | -| **Distribution** | No structured developer marketing or launch materials. | Codebase root | No drafted social posts, demo scripts, blog post outlines, or launch checklists. | Inability to launch a coordinated developer advocacy push on HN, Reddit, Twitter/X, and PyData. | Medium | **P1** | Create `marketing/` directory containing 6 ready-to-publish assets and a community release cadence. | -| **Metrics** | No defined framework for tracking adoption and contributor velocity. | Project documentation | Growth is measured only by vanity star counts rather than the activation funnel (Discovery $\rightarrow$ Activation $\rightarrow$ Contribution). | Misaligned maintainer priorities. | Low | **P1** | Create `docs/growth/metrics.md` with explicit 30-day, 90-day, and 12-month traction milestones. | - ---- - -## 3. Priority Action Plan - -### P0 (Immediate Lever β€” Complete in Phase 1–3) -1. Rewrite repository description and README hero to convey the **10-second value proposition**. -2. Make package installation (`pip install freshdata-cleaner`) unmistakable with import disambiguation. -3. Insert canonical **Before / After** demonstration and **Safety Proof** into README. -4. Add the **"Why FreshData?"** comparison table (Profilers vs. Validators vs. FreshData). - -### P1 (High Leverage β€” Complete in Phase 4–13) -1. Add missing GitHub issue templates (`integration-request`, `benchmark-result`) and update PR template. -2. Publish `docs/contributing/first-contribution.md` (zero-friction 7-step guide) and overhaul `contributor-roadmap.md`. -3. Standardize flagship examples (`01_csv_cleaning`, `02_missing_values`, `03_duplicate_detection`, `04_outlier_handling`, `05_ml_preprocessing`, and integrations). -4. Expand `benchmarks/README.md` with complete reproducibility protocols and failure case disclosures. -5. Create `docs/cookbook/` with 7+ high-intent SEO problem/solution recipes. -6. Assemble the `marketing/` distribution toolkit and `docs/growth/metrics.md` framework. diff --git a/marketing/README.md b/marketing/README.md deleted file mode 100644 index 46b3b1a4..00000000 --- a/marketing/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# FreshData Developer Advocacy & Distribution System - -This directory contains developer advocacy, technical marketing, and organic community distribution assets for **FreshData**. - -Our marketing philosophy is strictly **evidence-based and problem-first**: -* We never make unsubstantiated claims or market unreleased features. -* We never attack competitors or claim blanket superiority. -* We do not engage in star exchanges, paid stars, or engagement manipulation. -* We speak directly to developers and data engineers with working code, before/after evidence, and measurable benchmarks. - ---- - -## Directory Contents - -| Asset File | Purpose & Audience | -|---|---| -| [`social-posts.md`](social-posts.md) | Ready-to-publish technical posts for Twitter/X, LinkedIn, and Reddit (r/Python, r/datascience, r/dataengineering). | -| [`blog-ideas.md`](blog-ideas.md) | Outlines and code foundations for deep technical engineering blog posts. | -| [`demo-scripts.md`](demo-scripts.md) | 30-second elevator pitch and 2-minute live demo scripts for meetups, videos, and conference talks. | -| [`launch-assets.md`](launch-assets.md) | Announcement copy and checklists for Hacker News (*Show HN*), Reddit, and PyData newsletters. | -| [`contributor-posts.md`](contributor-posts.md) | Templates for contributor spotlights, "Good First Issue" roundups, and milestone celebrations. | - ---- - -## Community Release & Content Cadence - -Consistency matters more than volume. FreshData follows a sustainable 3-day weekly cadence: - -```text - MONDAY WEDNESDAY FRIDAY -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Roadmap & Issues β”‚ β”‚ Engineering Insight β”‚ β”‚ Community & Release β”‚ -β”‚ - Good First Issues β”‚ ──► β”‚ - 30s Video Demo β”‚ ──► β”‚ - Release Notes β”‚ -β”‚ - Architecture goal β”‚ β”‚ - Failure case post β”‚ β”‚ - Contributor credit β”‚ -β”‚ - RFC discussion β”‚ β”‚ - Benchmark deep- β”‚ β”‚ - Benchmark results β”‚ -β”‚ β”‚ β”‚ dive β”‚ β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -* **Monday β€” Roadmap & Community Issues**: Highlight 2–3 active issues across Level 1 and Level 2 difficulty, inviting new contributors with direct links to `docs/contributing/first-contribution.md`. -* **Wednesday β€” Engineering Insights & Demos**: Share a technical nuance (e.g. why FreshData refuses to touch ID columns, or how DuckDB relation cleaning works) accompanied by runnable code. -* **Friday β€” Release & Contributor Spotlight**: Celebrate merged PRs, welcome first-time contributors by name, and publish new benchmark figures. diff --git a/marketing/blog-ideas.md b/marketing/blog-ideas.md deleted file mode 100644 index 22ef572b..00000000 --- a/marketing/blog-ideas.md +++ /dev/null @@ -1,57 +0,0 @@ -# Engineering Blog Post Outlines & Technical Backlog - -High-depth technical articles for publication on Medium (Towards Data Science), Dev.to, Hacker Noon, and the FreshData engineering blog. - ---- - -## 1. "Why Blind `fillna()` Silently Corrupts Production ML Models" - -* **Category**: Problem & Architecture -* **Target Audience**: Data scientists, ML engineers -* **Key Narrative**: - * The illusion of the clean dataset: why zero-NaN counts give a false sense of security. - * Real-world disaster stories: imputing customer keys leading to fraudulent account linkages; imputing patient targets causing false test evaluations. - * Statistical distortion: comparing mean vs median vs mode on log-normal wealth/income distributions. - * The solution: Role-aware schema profiling and preserving ambiguity when confidence is low. -* **Runnable Code Asset**: `examples/02_missing_values.py` and `examples/05_ml_preprocessing.py`. - ---- - -## 2. "How FreshData Decides What NOT to Change: The Engineering of Safety Invariants" - -* **Category**: Engineering Deep Dive -* **Target Audience**: Senior data engineers, software architects -* **Key Narrative**: - * Why automated cleaners fail: aggressive heuristic overreach. - * The architecture of FreshData's decision engine: - * Stage 1: Representation repairs (whitespace, sentinels, snake_case). - * Stage 2: Profiler evaluation (skewness, uniqueness, cardinality). - * Stage 3: Role inference (`id`, `target`, `categorical`, `numeric`, `date`). - * Stage 4: Safety gate verification. - * Testing for invariants: how we enforce a 0.0% false-repair rate across 100k-row test suites. -* **Runnable Code Asset**: `tests/test_audit_core_defaults.py` and `tests/test_guard_protected.py`. - ---- - -## 3. "Cleaning 25 Million Rows Past RAM: How We Built DuckDB and Polars Out-of-Core Execution" - -* **Category**: Benchmarks & Performance -* **Target Audience**: Performance engineers, big data developers -* **Key Narrative**: - * The memory wall: why a 2 GB CSV consumes 8 GB in pandas RAM. - * Evaluating streaming and chunking architectures vs. Arrow-native engines. - * DuckDB and Polars execution backends: memory footprint comparisons (200 MB vs 1,046 MB at 1M rows). - * Transparent fallback policies: `fallback_policy="warn" | "error"` ensuring developers are never surprised by materialization. -* **Runnable Code Asset**: `benchmarks/bench_outofcore.py` and `examples/integrations/duckdb_workflow.py`. - ---- - -## 4. "Building an Audit Trail for Automated Data Cleaning in Regulated Industries" - -* **Category**: Enterprise & Governance -* **Target Audience**: FinTech, HealthTech, and compliance engineers -* **Key Narrative**: - * Regulatory requirements (BCBS 239, GDPR, HIPAA) around data lineage and automated modifications. - * Transitioning from black-box transformations to auditable Action records. - * Exporting `CleanReport` to machine-readable JSON for compliance archives. -* **Runnable Code Asset**: `examples/01_csv_cleaning.py` and `examples/interactive_html_reports.py`. diff --git a/marketing/contributor-posts.md b/marketing/contributor-posts.md deleted file mode 100644 index 5eeee889..00000000 --- a/marketing/contributor-posts.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Recognition & Community Growth Copy - -Templates and copy for recognizing open-source contributors, announcing Good First Issues, and celebrating milestones. - ---- - -## 1. Contributor Spotlight Template (Friday Community Post) - -**Platform**: LinkedIn & Twitter/X -**Audience**: Open-source community - -```text -πŸŽ‰ Huge shoutout to @ for their contribution to FreshData this week! - -@ implemented PR # (): - - -Why this matters: -<1 sentence on user impact, e.g. "Data teams in the UK can now clean customer address tables without writing custom regex."> - -Check out their work: -https://github.com/FreshCode-Org/freshdata/pull/ - -Want to get involved? We have beginner-friendly issues waiting: -https://freshcode-org.github.io/freshdata/contributing/first-contribution/ -``` - ---- - -## 2. Weekly "Good First Issue" Roundup (Monday Post) - -**Platform**: Twitter/X & Reddit /r/opensource - -```text -Looking for a clean, impactful open-source Python contribution this week? 🐍 - -FreshData (automated tabular data cleaning for pandas & Polars) has 3 freshly scoped issues ready for contributors: - -1. 🟒 Level 1 (20m): Add DuckDB Parquet recipe in examples/integrations - Link: https://github.com/FreshCode-Org/freshdata/issues/ - -2. 🟒 Level 1 (30m): Add Unicode non-breaking space test case in tests/test_strings.py - Link: https://github.com/FreshCode-Org/freshdata/issues/ - -3. 🟑 Level 2 (1-2h): Add German postal code regex in fieldcheck.py - Link: https://github.com/FreshCode-Org/freshdata/issues/ - -Our First Contribution Guide walks you through setup in 15 minutes: -https://freshcode-org.github.io/freshdata/contributing/first-contribution/ - -Drop a comment on an issue to claim it! -``` - ---- - -## 3. GitHub Star Milestone Post (Contextual & Organic) - -**Platform**: Twitter/X & LinkedIn -**Guiding Principle**: Never celebrate vanity stars in isolation; connect milestones to developer adoption and real problem solving. - -```text -πŸš€ 100 developers are now using FreshData to clean pandas and Polars DataFrames! - -When we launched FreshData, our thesis was simple: -Developers don't need another profiler to tell them their data is dirty. They need a tool that safely repairs it and explains what it changed. - -Since launch: -- 10+ community recipes added across Polars, DuckDB, scikit-learn, and Airflow. -- Verified 0.0% false-repair rate across 100,000-row benchmark suites. -- 93% test coverage enforced on every PR. - -Thank you to every developer who tried the library, filed an issue, or opened a PR. - -If FreshData saved you time this week, give us a star to help other engineers discover it: -https://github.com/FreshCode-Org/freshdata -``` diff --git a/marketing/demo-scripts.md b/marketing/demo-scripts.md deleted file mode 100644 index eee41e22..00000000 --- a/marketing/demo-scripts.md +++ /dev/null @@ -1,80 +0,0 @@ -# FreshData Live Demo Scripts (30-Second & 2-Minute) - -Scripts for live coding demonstrations at PyData meetups, YouTube/Loom walkthroughs, conference lightning talks, and developer tutorials. - ---- - -## 30-Second Elevator Demo: "The One-Line Audit" - -**Objective**: Hook the viewer by demonstrating immediate value on a dirty dataset in under 30 seconds. - -### Script & Actions - -1. **Speaker**: - > *"Every time you ingest a raw CSV into pandas, you spend 20 minutes writing regex for whitespace, fixing currency signs, and replacing 'N/A' sentinels. Watch this."* - -2. **Action (Terminal or IPython)**: - ```python - import pandas as pd - import freshdata as fd - - df = pd.read_csv("examples/data/messy_customers.csv") - cleaned, report = fd.clean(df, return_report=True) - print(report.summary()) - ``` - -3. **Speaker**: - > *"In one line, FreshData fixed the headers, parsed the dates, stripped padding, and flagged outliers. But notice this: it didn't touch customer IDs, and it gives you a complete audit log showing every cell it changed, why, and its confidence score."* - -4. **Action**: - ```python - print(cleaned.head(3)) - ``` - -5. **Closing**: - > *"Fast, explainable, and safe. `pip install freshdata-cleaner`."* - ---- - -## 2-Minute Deep Dive Demo: "From Raw Data to ML-Ready with Safety Invariants" - -**Objective**: Walk a technical developer through a full end-to-end workflow highlighting safety invariants and Polars support. - -### Timeline - -* **0:00 – 0:30: The Messy Input Problem** - * Open Jupyter or VS Code with a raw dataset: - * Uncleaned headers (`" Annual Income "`) - * Dirty sentinels (`"N/A"`, `"-"`, `"missing"`) - * Imbalanced outliers (`age = -5`, `age = 165`) - * Target column (`churn`) with missing values - * Speaker: *"If you run a blanket fillna or drop_duplicates, you leak target data and delete valid events."* - -* **0:30 – 1:00: Clean with Target Protection & Review Plan** - * Show the interactive clean call: - ```python - cleaned, report = fd.clean(df, target_column="churn", return_report=True) - ``` - * Point out the summary output: - * Shows rows before/after - * Shows columns before/after - * Flags age outliers in `age_outlier` boolean column rather than deleting rows. - * Explains that `churn` was preserved because it's a target label. - -* **1:00 – 1:30: Inspecting the Action Audit Log** - * Show `report.actions`: - ```python - for a in report.actions: - print(f"[{a.step}] {a.column}: {a.description} (risk={a.risk})") - ``` - * Speaker: *"Every action has a risk score. If a decision has medium or high risk, FreshData explicitly flags it in `report.recommendations` for human review."* - -* **1:30 – 2:00: Native Polars Execution** - * Switch to Polars: - ```python - import polars as pl - pl_df = pl.from_pandas(df) - clean_pl = fd.clean(pl_df) - assert isinstance(clean_pl, pl.DataFrame) - ``` - * Speaker: *"Works identically on Polars DataFrames with zero pandas conversion overhead. Check out the open-source repository at github.com/FreshCode-Org/freshdata."* diff --git a/marketing/launch-assets.md b/marketing/launch-assets.md deleted file mode 100644 index 10b7eb19..00000000 --- a/marketing/launch-assets.md +++ /dev/null @@ -1,97 +0,0 @@ -# FreshData Community Launch Assets (Show HN, Reddit, PyData) - -Launch copy and post drafts prepared for major technical developer communities. - ---- - -## 1. Hacker News: "Show HN: FreshData β€” Automated, explainable data cleaning for pandas and Polars" - -**Title**: `Show HN: FreshData – Automated, explainable data cleaning for pandas and Polars` -**URL**: `https://github.com/FreshCode-Org/freshdata` - -**Post Body**: - -```text -Hi HN, - -We built FreshData (https://github.com/FreshCode-Org/freshdata) because we were tired of writing the same dozens of fragile data-cleaning helper functions every time we ingested a CSV or SQL export into pandas or Polars. - -Most existing data quality tools either profile data (telling you what's wrong without fixing it) or validate data against rigid rules (telling you that a test failed, but not repairing it). - -FreshData aims to bridge that gap: one call cleans your DataFrame, but crucially, it explains what it changed and why. - -A quick example: -```python -import pandas as pd -import freshdata as fd - -df = pd.read_csv("export.csv") -cleaned, report = fd.clean(df, return_report=True) -print(report.summary()) -``` - -What makes FreshData different: -1. Decision engine, not blind fillna: It infers column roles (identifiers, categoricals, numeric, targets). Identifiers (like customer_id) are never imputed, and target labels are locked to prevent ML leakage. -2. Explainability: Every cell modification outputs an Action record with a rationale, a risk level, and a confidence score. If an action is high-risk, it recommends human review rather than guessing silently. -3. Outliers are non-destructive: Outliers are flagged in a boolean column by default rather than silently dropping rows. -4. Native Polars support: Pass a Polars DataFrame and get a Polars DataFrame back with zero pandas boilerplate. - -We enforce a 93% test coverage gate in CI and maintain a reproducible benchmark suite across 10k to 25M rows (detailed in benchmarks/README.md). - -The PyPI distribution is `freshdata-cleaner` (due to namespace collisions with an older project), but imports as `import freshdata as fd`. - -We'd love feedback on our decision engine heuristics and how it behaves on your messiest CSVs! -``` - ---- - -## 2. Reddit r/Python & r/dataengineering Launch Post - -**Title**: `[P] FreshData – Automated, explainable data cleaning for pandas and Polars (with audit trails)` - -**Post Body**: - -```text -Hey everyone, - -Whenever data pipelines break, it's rarely due to complex ML model errors β€” it's usually unhandled sentinel strings ("N/A", "missing", "-"), messy padding whitespace, unparsed currency symbols, or accidental duplicate rows. - -We created FreshData 2.0 to automate tabular data hygiene safely: -GitHub: https://github.com/FreshCode-Org/freshdata -Docs: https://freshcode-org.github.io/freshdata/ - -### What it does: -- One call (`fd.clean(df)`) standardizes headers into snake_case, strips whitespace, parses dates/currency, and handles missingness. -- Never mutates primary keys or target labels. -- Dual-engine: Works seamlessly on both pandas and Polars DataFrames (`fd.clean(polars_df)` returns a Polars DataFrame). -- Produces an audit report you can serialize to JSON for compliance and data lineage. -- Includes declarative pipelines: `pipe = fd.pipeline().strip_whitespace().normalize_sentinels().run(df)`. - -### How to try it: -```bash -pip install freshdata-cleaner -``` - -```python -import freshdata as fd -cleaned, report = fd.clean(df, return_report=True) -print(report.summary()) -``` - -We wrote a full cookbook with 7+ recipes for common data engineering workflows: -https://freshcode-org.github.io/freshdata/cookbook/ - -Feedback, benchmark PRs, and issue reports are very welcome! -``` - ---- - -## 3. Launch Checklist & Timing - -| Step | Action | Timing | -|:---:|---|---| -| 1 | Ensure `main` branch is clean, CI is green, and docs are built | Day 0 (T-1h) | -| 2 | Post to Hacker News (*Show HN*) | Tuesday 08:00 AM ET / 05:00 AM PT | -| 3 | Post to Reddit (r/Python and r/dataengineering) | Tuesday 09:30 AM ET | -| 4 | Post announcement thread on Twitter/X and LinkedIn | Tuesday 10:00 AM ET | -| 5 | Actively monitor comments on HN and Reddit, responding with code citations | Hours 0 to 12 | diff --git a/marketing/social-posts.md b/marketing/social-posts.md deleted file mode 100644 index 4230ff8d..00000000 --- a/marketing/social-posts.md +++ /dev/null @@ -1,124 +0,0 @@ -# Technical Social Media Posts (Twitter/X, LinkedIn, Reddit) - -High-signal, developer-focused social copy targeting common tabular data engineering pains. - ---- - -## 1. Problem Post: The Silent Danger of Blanket `fillna()` - -**Platform**: Twitter/X & LinkedIn -**Hook**: Why blindly filling missing values in pandas corrupts your downstream machine learning models. - -```text -The most dangerous line of code in data science: - -`df.fillna(df.mean(), inplace=True)` - -Here is what that single line quietly does to your dataset: -1. Imputes customer_id with a fake float ID, fabricating ghost users and corrupting table joins. -2. Fills missing ground-truth labels in your training target, creating synthetic targets your model can't learn from. -3. Distorts variance on skewed financial columns. - -In FreshData, cleaning is role-aware: -- Identifiers are preserved (0% mutation). -- Targets are protected from leakage. -- Missing values are imputed only when statistically safe, with risk & confidence logged. - -Try it in one line: -pip install freshdata-cleaner - -import freshdata as fd -cleaned, report = fd.clean(df, return_report=True) -print(report.summary()) - -https://github.com/FreshCode-Org/freshdata -``` - ---- - -## 2. Demonstration Post: Before & After in 30 Seconds - -**Platform**: Twitter/X & LinkedIn -**Hook**: What happens when you pass a genuinely messy DataFrame to FreshData? - -```text -Raw CSV from production: -- Whitespace in headers: " Customer ID " -- Dirty sentinels: "N/A", "-", "missing" -- Unparsed currency: "$1,200.50" -- Outliers: age = -4 - -Before: - Customer ID Full Name Age Annual Spend -0 C-101 Alice Smith 29 $1200.50 -1 C-102 Bob Jones 34 N/A -2 C-103 Charlie Brown -4 missing - -After `cleaned, report = fd.clean(df, return_report=True)`: - customer_id full_name age annual_spend age_outlier -0 C-101 Alice Smith 29 1200.50 False -1 C-102 Bob Jones 34 NaN False -2 C-103 Charlie Brown -4 NaN True - -Plus an audit trail showing why each action occurred and what it refused to touch. - -Check out the interactive recipes: -https://github.com/FreshCode-Org/freshdata/tree/main/examples -``` - ---- - -## 3. Engineering Post: Why FreshData Refuses to Change Certain Data - -**Platform**: LinkedIn & Reddit r/datascience -**Hook**: The mark of a reliable automated data cleaner isn't what it cleans β€” it's what it refuses to touch. - -```text -When building an automated cleaner for tabular data, the hardest engineering problem isn't imputation or regex parsing. - -It's knowing when NOT to touch a cell. - -If an auto-cleaner tries to be "helpful" by force-imputing every NaN, it introduces silent data corruption. In FreshData 2.0, we enforce invariant boundaries: - -1. Identifier columns (UUIDs, customer keys): Never imputed. Fabricating a primary key is worse than leaving it null. -2. Target labels: Strictly protected against imputation to prevent leakage in ML pipelines. -3. Outliers: Flagged in a boolean `_outlier` column by default rather than silently dropped. -4. Ambiguous sentinels: Normalized to NaN and routed to `report.recommendations` for human review. - -Every action outputs an Action dataclass with `risk` (low/medium/high) and `confidence` (0.0 to 1.0). - -Read our full trust invariant spec: -https://freshcode-org.github.io/freshdata/trust-claims/ -``` - ---- - -## 4. Ecosystem Post: Polars Support - -**Platform**: Twitter/X & Reddit r/Python -**Hook**: Automated data cleaning for Polars without pandas boilerplate. - -```text -Using Polars for data engineering? - -You can now run automated, explainable data cleaning natively: - -import polars as pl -import freshdata as fd - -pl_df = pl.read_csv("messy_telemetry.csv") - -# Polars in -> Polars out -clean_pl = fd.clean(pl_df) -assert isinstance(clean_pl, pl.DataFrame) - -- Strips whitespace -- Standardizes headers to snake_case -- Normalizes "N/A", "null", and "-" sentinels -- Flags outliers in a native boolean column - -Install with Polars extra: -pip install "freshdata-cleaner[polars]" - -Repo: https://github.com/FreshCode-Org/freshdata -``` diff --git a/mkdocs.yml b/mkdocs.yml index 84d47e3c..2a03e28f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -111,7 +111,6 @@ nav: - Decision-preserving workflow: decision-workflow.md - Scalable backends: backends.md - Backend fallback matrix: fallback-matrix.md - - FreshCore architecture: freshcore.md - Data profiling: data-profiling.md - Feature overview: feature-overview.md - Streaming / micro-batch: streaming.md @@ -123,7 +122,6 @@ nav: - Writing plugins: plugins.md - Model cards: model-cards.md - Learning profiles: learning-profiles.md - - Developer training pipeline: developer-training-pipeline.md - Format parsers & tick mode: parsers.md - Text cleaning & field validation: field-validation.md - Compliance reports: compliance.md diff --git a/notebooks/06_context_cleaning.ipynb b/notebooks/06_context_cleaning.ipynb deleted file mode 100644 index 07527dab..00000000 --- a/notebooks/06_context_cleaning.ipynb +++ /dev/null @@ -1,335 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Context cleaning β€” rules in English, compiled\n", - "\n", - "FreshData compiles natural-language expectations into a deterministic,\n", - "reviewable `ContextPolicy` β€” no model, no network, no new dependency.\n", - "This notebook walks the flagship ecommerce example end to end." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "import freshdata as fd\n" - ] - }, - { - "cell_type": "markdown", - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "source": [ - "## A messy ecommerce customer extract" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame({\n", - " \"CustomerID\": [101, 102, 102, 104, 105],\n", - " \"Full Name\": [\"Asha Rao\", \"Ben Cotton\", \"Ben Cotton\", \"Dana Li\", \"Ed Fox\"],\n", - " \"Email Addr\": [\"asha@example.com\", \"ben@example.com\", \"ben@example.com\", \"not-an-email\", None],\n", - " \"Mobile\": [\"9876543210\", None, \"9123456789\", \"9111111111\", \"9000000001\"],\n", - " \"Age\": [25, None, 40, 35, None],\n", - " \"Monthly Revenue\": [1200.50, 800.00, 800.00, 99999.0, 430.25],\n", - " \"City\": [\"Pune\", \"Delhi\", \"Delhi\", \"Goa\", \"Pune\"],\n", - "})\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "source": [ - "## 1. Write the rules as English" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "outputs": [], - "source": [ - "context = \"\"\"\n", - "This is an ecommerce customer dataset.\n", - "CustomerID is unique.\n", - "Emails must be valid.\n", - "Phone numbers are Indian.\n", - "Missing Age should be estimated only if confidence >95%.\n", - "Never modify revenue values.\n", - "\"\"\"" - ] - }, - { - "cell_type": "markdown", - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "source": [ - "## 2. Compile β€” and inspect what FreshData understood\n", - "\n", - "`fd.compile_context` is deterministic: the same text and schema always\n", - "produce the same policy. Unresolved references and unparsed sentences are\n", - "surfaced, never guessed at." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "outputs": [], - "source": [ - "policy = fd.compile_context(context, df=df)\n", - "print(policy.summary())" - ] - }, - { - "cell_type": "markdown", - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "source": [ - "The policy is a plain JSON artifact β€” commit it, diff it, review it in a PR." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "print(policy.to_json()) # policy.to_json(\"policy.json\") writes the file" - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "## 3. Clean under the policy\n", - "\n", - "Protected columns are added to `preserve_columns`, unique columns become\n", - "id columns, and per-column hints flow into the semantic context. The\n", - "report records the compile and every unresolved item." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned, report = fd.clean(df, context=context, return_report=True)\n", - "cleaned" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "outputs": [], - "source": [ - "[a.description for a in report.actions if a.step == \"context\"]" - ] - }, - { - "cell_type": "markdown", - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "source": [ - "The protected column is byte-identical β€” \"Never modify revenue values.\" is a hard rule:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned[\"monthly_revenue\"].equals(df[\"Monthly Revenue\"])" - ] - }, - { - "cell_type": "markdown", - "id": "8a65eabff63a45729fe45fb5ade58bdc", - "metadata": {}, - "source": [ - "## 4. Validate without mutating" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3933fab20d04ec698c2621248eb3be0", - "metadata": {}, - "outputs": [], - "source": [ - "findings = fd.validate(df, context=context)\n", - "for f in findings:\n", - " print(f.severity.upper(), f.rule_name, \"β€”\", f.message)" - ] - }, - { - "cell_type": "markdown", - "id": "4dd4641cc4064e0191573fe9c69df29b", - "metadata": {}, - "source": [ - "## 5. Strict mode\n", - "\n", - "In CI you usually want gaps to fail loudly. `strict=True` raises\n", - "`fd.PolicyError` on any unresolved or unparsed line, before touching data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8309879909854d7188b41380fd92a7c3", - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " fd.clean(df, context=context + \"\\nAlso make everything shiny.\", strict=True)\n", - "except fd.PolicyError as err:\n", - " print(err)" - ] - }, - { - "cell_type": "markdown", - "id": "3ed186c9a28b402fb0bc4494df01f08d", - "metadata": {}, - "source": [ - "## 6. Reuse the compiled policy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cb1e1581032b452c9409d6c6813c49d1", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned_again = fd.clean(df, policy=policy) # skips parsing entirely\n", - "cleaned_again.equals(cleaned)" - ] - }, - { - "cell_type": "markdown", - "id": "379cbbc1e968416e875cc15c1202d7eb", - "metadata": {}, - "source": [ - "## Phase 2: the context is now enforced end to end\n", - "\n", - "Phase 2 adds deterministic value experts and hard safety gates behind the\n", - "same context sentences:\n", - "\n", - "- `Emails must be valid.` β†’ the **EmailExpert** mechanically repairs\n", - " unambiguous addresses (`\" Bob@GMAIL.COM \"` β†’ `\"Bob@gmail.com\"`,\n", - " `\"a @ b.com\"` β†’ `\"a@b.com\"`, one doubled `@@`) and flags everything else.\n", - "- `Phone numbers are Indian.` β†’ the **PhoneExpert** normalizes safe forms to\n", - " canonical `+91XXXXXXXXXX` and flags the rest.\n", - "- `Allowed status values are ...` β†’ the **ReferenceExpert** repairs\n", - " case/whitespace/separator variants exactly, *suggests* single-candidate\n", - " typos, and blocks ambiguous ones.\n", - "- `Missing Age should be estimated only if confidence >95%.` β†’ the\n", - " statistical engine now refuses fills below the threshold and records a\n", - " suggested action instead.\n", - "- `Never modify revenue values.` β†’ enforced **physically**: the executor\n", - " verifies the column byte-identical before returning, and raises\n", - " `fd.ProtectedColumnError` if any code path broke the promise." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "277c27b1587741f2af2001be3712ef0d", - "metadata": {}, - "outputs": [], - "source": [ - "clean_df, report = fd.clean(\n", - " df,\n", - " context=context,\n", - " semantic_mode=\"auto\",\n", - " return_report=True,\n", - " verbose=False,\n", - ")\n", - "clean_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "db7b79bc585a40fcaf58bf750017e135", - "metadata": {}, - "outputs": [], - "source": [ - "# Every decision is audited: applied, suggested, skipped β€” with rationale.\n", - "for action in report:\n", - " if action.step in (\"semantic\", \"missing\", \"guard\"):\n", - " print(f\"[{action.step:8}] {str(action.column):16} {action.status:10} {action.description}\")" - ] - }, - { - "cell_type": "markdown", - "id": "916684f9a58a4a2aa5f864670399430d", - "metadata": {}, - "source": [ - "The protected `monthly_revenue` column is byte-identical, missing `age`\n", - "survived (0.9 fill confidence < 0.95 required), and the duplicate\n", - "`cust_id` shows up in `fd.validate` β€” reported, not silently dropped.\n", - "\n", - "Want to review the repairs *before* they run? That is the repair-plan\n", - "workflow: see `notebooks/08_plan_approval.ipynb` and\n", - "`docs/repair-plans.md`." - ] - }, - { - "cell_type": "markdown", - "id": "1671c31a24314836a5b85d7ef7fbf015", - "metadata": {}, - "source": [ - "## Limitations\n", - "\n", - "- **Deterministic tier-0 lexicon only** β€” a fixed set of ~12 intent\n", - " families; anything outside it is surfaced as *unparsed*, by design.\n", - "- No model, no embeddings, no network β€” and therefore no arbitrary NLU.\n", - "- `valid_format` / `locale_format` now drive real value repairs (email, and\n", - " phones for `region=\"IN\"`); other phone regions compile but do not repair yet.\n", - "- Ambiguous repairs are suggested or flagged, never auto-applied.\n", - "- `drop_if` / `rename` / `map` rules compile into the policy but are not yet\n", - " executed." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/08_plan_approval.ipynb b/notebooks/08_plan_approval.ipynb deleted file mode 100644 index ec9bd85d..00000000 --- a/notebooks/08_plan_approval.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# 08 β€” Repair plans: suggest, approve, apply, undo\n", - "\n", - "Phase 2 makes cleaning **reviewable**: `fd.suggest_plan` proposes exactly the\n", - "repairs `fd.clean` would make, you approve/reject/override them, and\n", - "`fd.apply_plan` executes *only* what you approved β€” under a physical\n", - "protected-column guard, with drift refusal, an undo log, and a deterministic\n", - "audit hash. Everything is offline and model-free." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "import freshdata as fd\n", - "\n", - "df = pd.DataFrame({\n", - " \"cust_id\": [\"C001\", \"C002\", \"C003\", \"C004\"],\n", - " \"email_addr\": [\"a@@b.com\", \"x @ y.com\", \"ok@ok.com\", \"junk\"],\n", - " \"mobile\": [\"98765 43210\", \"09876543210\", \"+919876543210\", \"12345\"],\n", - " \"monthly_revenue\": [\"1000\", \"2000\", \"3000\", \"4000\"],\n", - " \"status\": [\" Active \", \"INACTIVE\", \"actve\", \"pending\"],\n", - "})\n", - "\n", - "CONTEXT = \"\"\"CustomerID is unique.\n", - "Emails must be valid.\n", - "Phone numbers are Indian.\n", - "Allowed status values are active, inactive, pending.\n", - "Never modify revenue values.\"\"\"" - ] - }, - { - "cell_type": "markdown", - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "source": [ - "## 1. Suggest a plan\n", - "\n", - "With `semantic_mode=\"auto\"`, low-risk high-confidence actions arrive\n", - "pre-approved (they are what `fd.clean` would have applied automatically);\n", - "everything ambiguous stays `pending` or is `blocked`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "outputs": [], - "source": [ - "plan = fd.suggest_plan(df, context=CONTEXT, semantic_mode=\"auto\", verbose=False)\n", - "rp = plan.repair_plan\n", - "print(rp.summary())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "rp.to_frame() # one row per planned action, for filtering/sorting" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "## 2. Review\n", - "\n", - "Selectors accept an action id, a column name, an action kind, a list of\n", - "those, or a predicate." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "outputs": [], - "source": [ - "# The fuzzy typo \"actve\" -> \"active\" was only *suggested* (conf 0.8 < 0.95).\n", - "typo = next(a for a in rp.actions if a.params.get(\"raw_value\") == \"actve\")\n", - "rp.approve(typo.id)\n", - "\n", - "# Reject all phone repairs with a recorded reason.\n", - "rp.reject(\"phone_format\", reason=\"numbers are re-synced from the CRM nightly\")\n", - "\n", - "# Or approve everything low-risk in one call (never touches blocked/rejected):\n", - "rp.approve_all(max_risk=\"low\")\n", - "print(rp.summary())" - ] - }, - { - "cell_type": "markdown", - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "source": [ - "## 3. Apply β€” exactly what was approved\n", - "\n", - "- nothing is re-profiled, nothing is re-decided;\n", - "- rejected and blocked actions do not run;\n", - "- the protected-column guard verifies `monthly_revenue` byte-identical;\n", - "- the input frame is never mutated." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "outputs": [], - "source": [ - "clean_df, report = fd.apply_plan(df, rp, keep_undo=True)\n", - "clean_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"decisions_hash:\", report.decisions_hash)\n", - "# Stable for the same reviewed plan; changes when any approval changes." - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "## 4. Drift refusal\n", - "\n", - "A plan remembers the frame it was built for. Applying it to changed data\n", - "refuses by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "drifted = df.copy()\n", - "drifted.loc[0, \"status\"] = \"archived\"\n", - "try:\n", - " fd.apply_plan(drifted, rp)\n", - "except fd.PlanDriftError as exc:\n", - " print(\"refused:\", exc)\n", - "\n", - "# Explicit override: stale actions are skipped and recorded.\n", - "_, drift_report = fd.apply_plan(drifted, rp, allow_drift=True)" - ] - }, - { - "cell_type": "markdown", - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "source": [ - "## 5. Undo\n", - "\n", - "`keep_undo=True` stores a compact log (cell positions + the raw value per\n", - "action), capped by `undo_cell_limit`; actions that don't fit are honestly\n", - "marked `reversible=False`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "outputs": [], - "source": [ - "email_ids = [a.id for a in rp.actions\n", - " if a.kind == \"email_format\" and a.approval == \"approved\"]\n", - "restored = report.revert(clean_df, action_ids=email_ids)\n", - "restored[[\"email_addr\"]]" - ] - }, - { - "cell_type": "markdown", - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "source": [ - "## 6. Plans as files\n", - "\n", - "```bash\n", - "freshdata plan in.csv --context-file rules.txt --out plan.json\n", - "freshdata apply-plan in.csv --plan plan.json -o out.csv --report audit.json\n", - "```\n", - "\n", - "`RepairPlan.to_json()` / `from_json()` round-trip actions, approval state,\n", - "the compiled policy, and the frame signature β€” so a plan can be reviewed in\n", - "a pull request like code.\n", - "\n", - "## Limitations\n", - "\n", - "- No model, no embeddings, no network; only the deterministic Phase-1\n", - " context language.\n", - "- Ambiguous repairs (`junk`, `12345`, typos with two close candidates) are\n", - " suggested or flagged, never auto-applied.\n", - "- Undo is cell-scoped; row drops/aggregations are not reversible." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/09_optional_semantic_models.ipynb b/notebooks/09_optional_semantic_models.ipynb deleted file mode 100644 index 18fdaebe..00000000 --- a/notebooks/09_optional_semantic_models.ipynb +++ /dev/null @@ -1,250 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Optional semantic models\n", - "\n", - "FreshData's default install is **model-free** β€” everything you saw in the\n", - "context-cleaning and plan-approval notebooks is deterministic, offline code.\n", - "This notebook shows the *optional* Phase-3 model path: a small local ONNX\n", - "encoder that adds repair **proposals** (never decisions) for values the\n", - "deterministic experts abstain on, plus a rescue rung for the column resolver.\n", - "\n", - "Requirements: `pip install \"freshdata-cleaner[semantic]\"` and an explicit\n", - "`fd.models.pull(...)`. Models are **never** downloaded automatically. For this\n", - "notebook we enable the built-in deterministic *stub encoder* so everything\n", - "runs with no model files and no network.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Testing seam: a deterministic hash-based encoder, so the whole notebook\n", - "# runs offline with zero model files. With real models installed you would\n", - "# skip this line entirely.\n", - "os.environ[\"FRESHDATA_STUB_ENCODER\"] = \"1\"\n", - "\n", - "import pandas as pd\n", - "\n", - "import freshdata as fd\n" - ] - }, - { - "cell_type": "markdown", - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "source": [ - "## The model registry\n", - "\n", - "`fd.models` is the explicit, auditable way models get onto your machine.\n", - "`status()` is offline and side-effect free; `pull()` is the **only** network\n", - "path in the whole package and verifies pinned checksums.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "outputs": [], - "source": [ - "for cfg in fd.models.list_available():\n", - " print(f\"{cfg.model_id:<20} {cfg.quantization:<5} {cfg.license:<12} \"\n", - " f\"~{cfg.size_bytes / 1e6:.0f} MB\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "fd.models.status()\n" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "Official artifacts are not hosted yet β€” `fd.models.pull(\"fd-col-encoder-v1\")`\n", - "raises `ModelNotPublishedError` with instructions (mirror via\n", - "`FRESHDATA_MODEL_URL_BASE`, or drop files into `FRESHDATA_MODEL_DIR` for\n", - "air-gapped installs). The stub we enabled above stands in for the encoder.\n" - ] - }, - { - "cell_type": "markdown", - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "source": [ - "## Embedding-assisted cleaning\n", - "\n", - "The ecommerce fixture again β€” but note `status` contains `'activvee'`, two edits\n", - "away from `active`. The deterministic reference expert only trusts one edit, so\n", - "on a model-free install this value is *flagged, never repaired*. With the\n", - "embedding backend enabled, the encoder gets a vote β€” as **evidence through the\n", - "same gate**, never as authority.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(\n", - " {\n", - " \"cust_id\": [\"C001\", \"C002\", \"C003\", \"C004\"],\n", - " \"email_addr\": [\"asha@gmail.com\", \"ravi@@example.com\",\n", - " \"neha@shop.in\", \"kiran@test.in\"],\n", - " \"mob_no\": [\"98765 43210\", \"+91 91234 56780\",\n", - " \"09123456789\", \"90000 11111\"],\n", - " \"monthly_revenue\": [\"1000\", \"2000\", \"3000\", \"4000\"],\n", - " \"status\": [\"activvee\", \"inactive\", \"pendng\", \"active\"],\n", - " }\n", - ")\n", - "\n", - "CONTEXT = \"\"\"\n", - "This is an ecommerce customer dataset.\n", - "CustomerID is unique.\n", - "Emails must be valid.\n", - "Phone numbers are Indian.\n", - "Allowed status values are active, inactive, pending.\n", - "Never modify revenue values.\n", - "\"\"\"\n", - "\n", - "clean_df, report = fd.clean(\n", - " df,\n", - " context=CONTEXT,\n", - " semantic_mode=\"auto\",\n", - " semantic_backends=(\"deterministic\", \"memory\", \"embedding\"),\n", - " return_report=True,\n", - ")\n", - "clean_df\n" - ] - }, - { - "cell_type": "markdown", - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "source": [ - "Protected means protected β€” `monthly_revenue` is byte-identical no matter\n", - "which backends run (a physical executor check, not a convention):\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "assert clean_df[\"monthly_revenue\"].equals(df[\"monthly_revenue\"])\n", - "clean_df[\"monthly_revenue\"]\n" - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "Every model-assisted action carries full provenance: which backend, which\n", - "model, the raw score, the calibrated confidence, and the candidate margins:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "for action in report.actions:\n", - " if action.step == \"semantic\":\n", - " meta = action.metadata\n", - " print(f\"[{action.status:<9}] {action.column:<12} \"\n", - " f\"{meta.get('raw_value')!r} -> {meta.get('proposed_value')!r} \"\n", - " f\"conf={action.confidence} backend={meta.get('backend')} \"\n", - " f\"calib={meta.get('calibration_version', '-')}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "source": [ - "## Graceful degradation\n", - "\n", - "Requesting the embedding backend without the extra or the model never crashes\n", - "β€” it self-disables with a report note, and cleaning proceeds exactly as the\n", - "model-free install would:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "outputs": [], - "source": [ - "from freshdata.models import runtime\n", - "\n", - "os.environ.pop(\"FRESHDATA_STUB_ENCODER\", None)\n", - "runtime.reset_encoders()\n", - "\n", - "_, degraded_report = fd.clean(\n", - " df,\n", - " context=CONTEXT,\n", - " semantic_mode=\"auto\",\n", - " semantic_backends=(\"deterministic\", \"memory\", \"embedding\"),\n", - " return_report=True,\n", - ")\n", - "degraded_report.fallback_events\n" - ] - }, - { - "cell_type": "markdown", - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "source": [ - "## What the model path will never do\n", - "\n", - "- **No automatic downloads** β€” `fd.clean` cannot reach the network.\n", - "- **No LLM, no cloud inference, no per-cell model calls** β€” the backend sees\n", - " distinct values of eligible columns only.\n", - "- **No ambiguous auto-applies** β€” a value near two allowed candidates produces\n", - " *no proposal at all*; pure similarity clustering is suggest-only by default.\n", - "- **No unprotected protected columns** β€” byte-identity is asserted physically\n", - " after every run.\n", - "\n", - "See `docs/semantic-models.md` for install, air-gapped setup, calibration, and\n", - "troubleshooting.\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/10_learning_profiles.ipynb b/notebooks/10_learning_profiles.ipynb deleted file mode 100644 index c94052c4..00000000 --- a/notebooks/10_learning_profiles.ipynb +++ /dev/null @@ -1,474 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Learning profiles: paired-data cleaning that remembers\n\n", - "\n\n", - "Phase 4 adds `fd.learn()`: instead of hand-writing cleaning rules, you show freshdata a\n\n", - "**messy/clean pair** of the same rows and it learns a reusable `.fdprofile` β€” a set of\n\n", - "typed rules, value maps, and (optionally) retrieval examples that later replay against new,\n\n", - "similarly-shaped data via `fd.clean(df, profile=...)`.\n\n", - "\n\n", - "This notebook runs a small but realistic example end to end: an ecommerce customer table\n\n", - "from India with messy emails, phone formats, day-first dates, currency strings, status\n\n", - "typos, and a nonstandard missing-value sentinel." - ] - }, - { - "cell_type": "markdown", - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "source": [ - "## 1. A messy/clean pair\n\n", - "\n\n", - "We build 30 rows of \"messy\" input and the corresponding \"clean\" ground truth. Two status\n\n", - "typos β€” `\"Shipped\"` and `\"Deliverd\"` β€” each occur with support 5, right at the default\n\n", - "`min_support=5` gate, so both should be learned as value-map rules rather than demoted to\n\n", - "suggest-only examples." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "import freshdata as fd\n", - "\n", - "n = 30\n", - "\n", - "cust_id = list(range(1, n + 1))\n", - "\n", - "\n", - "\n", - "# email: every 3rd row upper-cased, every 3rd row padded with whitespace\n", - "\n", - "emails_clean = [f\"user{i}@example.in\" for i in range(1, n + 1)]\n", - "\n", - "emails_messy = [\n", - "\n", - " e.upper() if i % 3 == 0 else (f\" {e} \" if i % 3 == 1 else e)\n", - "\n", - " for i, e in enumerate(emails_clean)\n", - "\n", - "]\n", - "\n", - "\n", - "\n", - "# phone: rotate through a few common messy IN formats\n", - "\n", - "phones_clean = [f\"+9198765{43200 + i:05d}\" for i in range(n)]\n", - "\n", - "phones_messy = []\n", - "\n", - "for i, p in enumerate(phones_clean):\n", - "\n", - " digits = p.replace(\"+91\", \"\")\n", - "\n", - " style = i % 4\n", - "\n", - " if style == 0:\n", - "\n", - " phones_messy.append(f\"0{digits}\")\n", - "\n", - " elif style == 1:\n", - "\n", - " phones_messy.append(f\"{digits[:5]} {digits[5:]}\")\n", - "\n", - " elif style == 2:\n", - "\n", - " phones_messy.append(f\"91-{digits}\")\n", - "\n", - " else:\n", - "\n", - " phones_messy.append(p)\n", - "\n", - "\n", - "\n", - "# signup_date: half day-first, half already ISO\n", - "\n", - "dates_clean = [f\"2024-{1 + (i % 9):02d}-{10 + (i % 15):02d}\" for i in range(n)]\n", - "\n", - "dates_messy = [\n", - "\n", - " f\"{d.split('-')[2]}/{d.split('-')[1]}/{d.split('-')[0]}\" if i % 2 == 0 else d\n", - "\n", - " for i, d in enumerate(dates_clean)\n", - "\n", - "]\n", - "\n", - "\n", - "\n", - "# amount_spent: rupee-formatted strings -> floats\n", - "\n", - "amounts_clean = [round(500 + i * 37.5, 2) for i in range(n)]\n", - "\n", - "amounts_messy = [f\"β‚Ή{a:,.2f}\" for a in amounts_clean]\n", - "\n", - "\n", - "\n", - "# status: two typo'd spellings, each with support >= min_support (5)\n", - "\n", - "status_clean = (['shipped'] * 10) + (['delivered'] * 10) + (['cancelled'] * 10)\n", - "\n", - "status_messy = (\n", - "\n", - " (['Shipped'] * 5 + ['shipped'] * 5)\n", - "\n", - " + (['Deliverd'] * 5 + ['delivered'] * 5)\n", - "\n", - " + (['cancelled'] * 10)\n", - "\n", - ")\n", - "\n", - "\n", - "\n", - "# country: a nonstandard sentinel for missing, every 6th row\n", - "\n", - "country_clean = ['India'] * n\n", - "\n", - "country_messy = ['India'] * n\n", - "\n", - "for i in range(0, n, 6):\n", - "\n", - " country_messy[i] = 'N/A'\n", - "\n", - " country_clean[i] = None\n", - "\n", - "\n", - "\n", - "messy_df = pd.DataFrame(\n", - "\n", - " {\n", - "\n", - " \"cust_id\": cust_id,\n", - "\n", - " \"email\": emails_messy,\n", - "\n", - " \"phone\": phones_messy,\n", - "\n", - " \"signup_date\": dates_messy,\n", - "\n", - " \"amount_spent\": amounts_messy,\n", - "\n", - " \"status\": status_messy,\n", - "\n", - " \"country\": country_messy,\n", - "\n", - " }\n", - "\n", - ")\n", - "\n", - "clean_df = pd.DataFrame(\n", - "\n", - " {\n", - "\n", - " \"cust_id\": cust_id,\n", - "\n", - " \"email\": emails_clean,\n", - "\n", - " \"phone\": phones_clean,\n", - "\n", - " \"signup_date\": dates_clean,\n", - "\n", - " \"amount_spent\": amounts_clean,\n", - "\n", - " \"status\": status_clean,\n", - "\n", - " \"country\": country_clean,\n", - "\n", - " }\n", - "\n", - ")\n", - "\n", - "\n", - "\n", - "messy_df.head()" - ] - }, - { - "cell_type": "markdown", - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "source": [ - "## 2. Learn a profile\n\n", - "\n\n", - "`fd.learn` runs the 5-stage pipeline (align β†’ diff β†’ classify β†’ extract β†’ fit/holdout-validate)\n\n", - "and returns a `LearningProfile`. `context=` is free text that seeds semantic-type inference,\n\n", - "same as `fd.clean(df, context=...)`. `key=\"cust_id\"` aligns messy/clean rows unambiguously." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "profile = fd.learn(\n", - "\n", - " messy_df,\n", - "\n", - " clean_df,\n", - "\n", - " context=\"Ecommerce customer data from India\",\n", - "\n", - " key=\"cust_id\",\n", - "\n", - ")\n", - "\n", - "print(profile.summary())\n", - "\n", - "print(\"contains_raw_values:\", profile.manifest.contains_raw_values)" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "`contains_raw_values` is `False`: by default (`privacy=\"mask\"`), sensitive semantic types\n\n", - "(`email`, `phone`, `person_name`, `national_id`, `address`, `postal_code`, `free_text`) are\n\n", - "HMAC-masked before anything is stored, and no raw literal imputation values are ever learned.\n\n", - "Only the non-sensitive `status` column contributes real value-map literals here (`\"Shipped\"\n\n", - "β†’ \"shipped\"`, `\"Deliverd\" β†’ \"delivered\"`), because `status` isn't a sensitive semantic type." - ] - }, - { - "cell_type": "markdown", - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "source": [ - "## 3. Save, and confirm no literal PII leaks into the file\n\n", - "\n\n", - "A `.fdprofile` is a zip with a manifest that hashes every member for integrity. We check the\n\n", - "raw bytes of the saved file for any of the real emails or phone numbers from the training\n\n", - "data β€” none should appear, since those columns are masked by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "outputs": [], - "source": [ - "fd.save_profile(profile, \"/tmp/e2e_orders.fdprofile\")\n", - "\n", - "\n", - "\n", - "with open(\"/tmp/e2e_orders.fdprofile\", \"rb\") as f:\n", - "\n", - " raw_bytes = f.read()\n", - "\n", - "\n", - "\n", - "leak_terms = emails_clean + phones_clean\n", - "\n", - "leaks = [t for t in leak_terms if t.encode(\"utf-8\") in raw_bytes]\n", - "\n", - "print(\"literal leaks found in zip bytes:\", leaks)" - ] - }, - { - "cell_type": "markdown", - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "source": [ - "## 4. Load the profile and replay it on new data\n\n", - "\n\n", - "`fd.clean(df, profile=..., return_report=True)` returns a `(cleaned_df, report)` tuple.\n\n", - "Profile-backed proposals are gathered by the semantic layer, which stays off by default β€” pass\n\n", - "`semantic_mode=\"auto\"` (or `\"assist\"`/`\"review\"`) explicitly, exactly as you would to enable\n\n", - "semantic assistance without a profile." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "loaded = fd.load_profile(\"/tmp/e2e_orders.fdprofile\")\n", - "\n", - "\n", - "\n", - "new_df = pd.DataFrame(\n", - "\n", - " {\n", - "\n", - " \"cust_id\": [1001, 1002, 1003, 1004],\n", - "\n", - " \"email\": [\n", - "\n", - " \" NEW.USER@example.in \",\n", - "\n", - " \"another@example.in\",\n", - "\n", - " \"THIRD@EXAMPLE.IN\",\n", - "\n", - " \"fourth@example.in \",\n", - "\n", - " ],\n", - "\n", - " \"phone\": [\"098765 43299\", \"91-9876543298\", \"+919876543297\", \"09876543296\"],\n", - "\n", - " \"signup_date\": [\"15/03/2024\", \"2024-03-16\", \"20/03/2024\", \"2024-03-18\"],\n", - "\n", - " \"amount_spent\": [\"β‚Ή1,200.00\", \"β‚Ή950.50\", \"β‚Ή2,000.00\", \"β‚Ή800.00\"],\n", - "\n", - " \"status\": [\"Shipped\", \"Deliverd\", \"cancelled\", \"Shipped\"],\n", - "\n", - " \"country\": [\"India\", \"N/A\", \"India\", \"India\"],\n", - "\n", - " }\n", - "\n", - ")\n", - "\n", - "new_df" - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "### Baseline: `fd.clean(new_df)` with no profile\n\n", - "\n\n", - "General-purpose heuristics (phone/date/currency normalization, sentinel-to-missing) still\n\n", - "apply without any profile. But `\"Shipped\"` and `\"Deliverd\"` aren't recognizable as typos by\n\n", - "generic rules β€” there's no dictionary that says \"Deliverd\" means \"delivered\" for this\n\n", - "dataset's vocabulary. Without the learned value map, both typos pass through unfixed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "baseline_cleaned, _ = fd.clean(new_df, return_report=True)\n", - "\n", - "baseline_cleaned[[\"cust_id\", \"status\"]]" - ] - }, - { - "cell_type": "markdown", - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "source": [ - "### With the learned profile: `fd.clean(new_df, profile=loaded, semantic_mode=\"auto\")`\n\n", - "\n\n", - "Now the learned `status` value map replays: both `\"Shipped\"` and `\"Deliverd\"` are corrected,\n\n", - "each recorded as a `profile_influenced` action in the report." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned, report = fd.clean(\n", - "\n", - " new_df, profile=loaded, semantic_mode=\"auto\", return_report=True\n", - "\n", - ")\n", - "\n", - "cleaned[[\"cust_id\", \"status\"]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "outputs": [], - "source": [ - "actions = [a for a in report.actions if a.metadata.get(\"profile_influenced\")]\n", - "\n", - "print(\"profile_influenced actions:\", len(actions))\n", - "\n", - "for a in actions:\n", - "\n", - " print(a.column, a.metadata)" - ] - }, - { - "cell_type": "markdown", - "id": "8a65eabff63a45729fe45fb5ade58bdc", - "metadata": {}, - "source": [ - "## 5. Audit the profile\n\n", - "\n\n", - "`profile.audit()` reports every rule the profile carries and what happened to any candidate\n\n", - "that didn't make it into an auto-applied rule. Here, an `email_normalize` rule was proposed\n\n", - "during extraction but **demoted to suggest-only** because its holdout precision (measured on\n\n", - "a held-out split of the training pairs) fell below the default `min_precision=0.98` gate β€”\n\n", - "the fit/holdout stage is what keeps a profile from replaying a rule that doesn't generalize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3933fab20d04ec698c2621248eb3be0", - "metadata": {}, - "outputs": [], - "source": [ - "audit = profile.audit()\n", - "\n", - "print(f\"rule_count={audit.rule_count} rules_by_family={audit.rules_by_family}\")\n", - "\n", - "print(f\"value_map_entries={audit.value_map_entries} masked_entries={audit.masked_entries}\")\n", - "\n", - "print(f\"demotions={len(audit.demotions)}\")\n", - "\n", - "for d in audit.demotions:\n", - "\n", - " print(\" \", d.column, d.family, d.outcome, \"-\", d.reason)" - ] - }, - { - "cell_type": "markdown", - "id": "4dd4641cc4064e0191573fe9c69df29b", - "metadata": {}, - "source": [ - "## Recap\n\n", - "\n\n", - "- `fd.learn(messy, clean, context=..., key=...)` learns a `LearningProfile` from a paired\n\n", - " example, not from hand-written rules.\n\n", - "- Sensitive columns are masked by default; no full rows, no raw sensitive literals, and no\n\n", - " literal imputation maps are ever stored.\n\n", - "- Low-support or low-precision candidates are demoted to suggest-only rather than silently\n\n", - " applied β€” a profile can only replay what it validated on held-out data.\n\n", - "- Replay is opt-in and additive: `fd.clean(df, profile=profile, semantic_mode=\"auto\")` only\n\n", - " changes behavior when you ask for it; `profile=None` (the default) is byte-identical to\n\n", - " today's `fd.clean`.\n\n", - "- See `docs/learning-profiles.md` for the full format, privacy model, and CLI\n\n", - " (`freshdata learn`, `clean --profile`, `profile audit|diff|merge`)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/debug_dataset_sweep.py b/scripts/debug_dataset_sweep.py deleted file mode 100644 index 947bd4a6..00000000 --- a/scripts/debug_dataset_sweep.py +++ /dev/null @@ -1,278 +0,0 @@ -"""Multi-dataset debug sweep for freshdata (H1–H5). - -Run from repo root:: - - python scripts/debug_dataset_sweep.py - python scripts/debug_dataset_sweep.py --online --tier 1 - python scripts/debug_dataset_sweep.py --online -""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -import time -from pathlib import Path - -import pandas as pd - -_REPO = Path(__file__).resolve().parents[1] -_TESTS = _REPO / "tests" -_LOG_PATH = _REPO.parent / ".cursor" / "debug_dataset_sweep.log" - -if str(_TESTS) not in sys.path: - sys.path.insert(0, str(_TESTS)) - -import freshdata as fd # noqa: E402 -from expectations import ( # noqa: E402 - ALL_FIXTURES, - ALL_ONLINE_FIXTURES, - ALL_ONLINE_TIER1, - ALL_ONLINE_TIER2, - load_expectations, - load_fixture, - load_online_fixture, - resolve_column, -) -from freshdata.steps.columns import snake_case # noqa: E402 - -ISOLATE = {"drop_empty_rows": False, "drop_duplicates": False, "verbose": False} -_ID_RE = re.compile(r"(?:^|_)(?:id|uuid|guid|key|record_id|transaction_id)s?$", re.I) - - -def _log(hypothesis_id: str, dataset: str, check: str, passed: bool, detail: str = "") -> None: - _LOG_PATH.parent.mkdir(parents=True, exist_ok=True) - payload = { - "hypothesisId": hypothesis_id, - "location": "debug_dataset_sweep.py", - "message": check, - "data": {"dataset": dataset, "pass": passed, "detail": detail}, - "timestamp": int(time.time() * 1000), - } - with _LOG_PATH.open("a") as fh: - fh.write(json.dumps(payload) + "\n") - - -def _id_candidate(df: pd.DataFrame) -> str | None: - for col in df.columns: - label = str(col) - if _ID_RE.search(label) or label.lower() in {"id", "index", "pk"}: - return label - return None - - -def _preserve_candidates(name: str, df: pd.DataFrame) -> list[str]: - exp = load_expectations(name).get("balanced", {}) - cols: list[str] = [] - for key in ( - "columns_never_imputed", - "columns_never_capped", - "columns_preserved_as_text", - "columns_never_dropped", - ): - for col in exp.get(key, []): - resolved = resolve_column(df, col) - if resolved in df.columns and resolved not in cols: - cols.append(resolved) - return cols[:3] - - -def _frames_equal(a: pd.DataFrame, b: pd.DataFrame) -> tuple[bool, str]: - try: - pd.testing.assert_frame_equal(a, b) - return True, "" - except AssertionError as exc: - return False, str(exc)[:300] - - -def check_h1(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - base = {**ISOLATE, "strategy": "balanced"} - - id_col = _id_candidate(df) - if id_col: - ren = snake_case(id_col) - kw = {**base, "id_columns": ren, "outlier_action": "cap"} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "id_columns": (ren,)}) - ok, detail = _frames_equal(a, b) - results.append((f"H1:id_columns:{ren}", ok, detail)) - - for col in _preserve_candidates(name, df): - ren = snake_case(col) - kw = {**base, "preserve_columns": ren, "outlier_action": "cap"} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "preserve_columns": (ren,)}) - ok, detail = _frames_equal(a, b) - results.append((f"H1:preserve_columns:{ren}", ok, detail)) - - if "duplicate" in name or id_col: - subset = snake_case(id_col or str(df.columns[0])) - kw = {**base, "duplicate_subset": subset, "drop_duplicates": True} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "duplicate_subset": (subset,)}) - ok, detail = _frames_equal(a, b) - results.append((f"H1:duplicate_subset:{subset}", ok, detail)) - - kw = {**base, "extra_sentinels": "missing", "drop_empty_rows": False} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "extra_sentinels": ("missing",)}) - ok, detail = _frames_equal(a, b) - results.append(("H1:extra_sentinels", ok, detail)) - - return results - - -def check_h2(name: str, df: pd.DataFrame) -> tuple[str, bool, str]: - once = fd.clean(df, strategy="balanced", verbose=False) - twice = fd.clean(once, strategy="balanced", verbose=False) - ok, detail = _frames_equal(once, twice) - return ("H2:idempotent", ok, detail) - - -def check_h3(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - plan = fd.suggest_plan(df, **ISOLATE) - _, report = fd.clean(df, return_report=True, **ISOLATE) - for col, cp in plan.column_plans.items(): - if cp.outlier_action: - oa = [a for a in report if a.step == "outliers" and a.column == col] - if not oa: - continue - exp = cp.outlier_action - act = oa[0].description - ok = ( - (exp == "flag" and "flagged" in act) - or (exp == "cap" and "capped" in act) - or (exp in act or exp in oa[0].rationale) - ) - results.append((f"H3:outlier:{col}", ok, f"plan={exp}, got={act[:80]}")) - if cp.missing and cp.missing.model_id: - ma = [a for a in report if a.step == "missing" and a.column == col] - if not ma: - continue - action = ma[-1] - ok = ( - cp.missing.model_id == action.model_id - or cp.missing.model_id in action.description - or cp.missing.model_id in action.rationale - ) - results.append( - (f"H3:missing:{col}", ok, f"plan={cp.missing.model_id}, got={action.model_id}") - ) - if not results: - results.append(("H3:plan_clean", True, "no engine columns to compare")) - return results - - -def check_h4(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - id_col = _id_candidate(df) - if not id_col: - return [("H4:rename_id", True, "no id-like column")] - raw_label = str(id_col) - snake = snake_case(raw_label) - if raw_label == snake: - return [("H4:rename_id", True, "column already snake_case")] - - pre = fd.clean(df, id_columns=raw_label, outlier_action="cap", **ISOLATE) - post = fd.clean(df, id_columns=snake, outlier_action="cap", **ISOLATE) - ok, detail = _frames_equal(pre, post) - results.append((f"H4:id_pre_vs_post_name:{raw_label}", ok, detail)) - return results - - -def check_h5(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - exp = load_expectations(name).get("balanced", {}) - - for col in exp.get("columns_never_imputed", []): - raw = resolve_column(df, col) - if raw not in df.columns: - continue - before_na = int(df[raw].isna().sum()) - out_s = fd.clean(df, preserve_columns=col, **ISOLATE) - out_t = fd.clean(df, preserve_columns=(col,), **ISOLATE) - ok_na = int(out_s[col].isna().sum()) == before_na == int(out_t[col].isna().sum()) - ok_eq, detail = _frames_equal(out_s, out_t) - results.append((f"H5:never_imputed:{col}", ok_na and ok_eq, detail or f"na={before_na}")) - - for col in exp.get("columns_never_capped", []): - raw = resolve_column(df, col) - if raw not in df.columns: - continue - before_max = df[raw].max() - out_s = fd.clean(df, preserve_columns=col, outlier_action="cap", **ISOLATE) - out_t = fd.clean(df, preserve_columns=(col,), outlier_action="cap", **ISOLATE) - ok_cap = out_s[col].max() == out_t[col].max() == before_max - ok_eq, detail = _frames_equal(out_s, out_t) - results.append((f"H5:never_capped:{col}", ok_cap and ok_eq, detail or f"max={before_max}")) - - if not results: - results.append(("H5:expectations", True, "no preserve expectations")) - return results - - -def sweep_dataset(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - results.extend(check_h1(name, df)) - results.append(check_h2(name, df)) - results.extend(check_h3(name, df)) - results.extend(check_h4(name, df)) - results.extend(check_h5(name, df)) - for check, passed, detail in results: - hid = check.split(":")[0] - _log(hid, name, check, passed, detail) - return results - - -def load_dataset(name: str, *, online: bool) -> pd.DataFrame: - if online: - return load_online_fixture(name, live=False) - return load_fixture(name) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Multi-dataset freshdata debug sweep") - parser.add_argument("--online", action="store_true", help="use cached online fixtures") - parser.add_argument("--tier", type=int, default=None, help="online tier filter (1 or 2)") - args = parser.parse_args() - - if args.online: - if args.tier == 1: - names = ALL_ONLINE_TIER1 - elif args.tier == 2: - names = ALL_ONLINE_TIER2 - else: - names = ALL_ONLINE_FIXTURES - else: - names = ALL_FIXTURES - - failures: list[str] = [] - print(f"{'dataset':<28} {'check':<40} {'result'}") - print("-" * 80) - - for name in names: - try: - df = load_dataset(name, online=args.online) - except Exception as exc: - print(f"{name:<28} {'LOAD':<40} SKIP ({exc})") - continue - for check, passed, detail in sweep_dataset(name, df): - status = "PASS" if passed else "FAIL" - print(f"{name:<28} {check:<40} {status}") - if not passed: - failures.append(f"{name}/{check}: {detail}") - if detail: - print(f" -> {detail[:200]}") - - print("-" * 80) - print(f"Total: {len(names)} datasets, {len(failures)} failure(s)") - print(f"Log: {_LOG_PATH}") - return 1 if failures else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/debug_datasets.py b/scripts/debug_datasets.py deleted file mode 100644 index 3fd8336a..00000000 --- a/scripts/debug_datasets.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -"""Compare clean() across local and online fixtures for debugging. - -Usage:: - - python scripts/debug_datasets.py - python scripts/debug_datasets.py --online titanic - python scripts/debug_datasets.py --online --explain titanic - python scripts/debug_datasets.py --infer-roles --online adult_income - python scripts/debug_datasets.py --search missing - python scripts/debug_datasets.py --refresh --online --compare -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -from pathlib import Path - -import pandas as pd - -import freshdata as fd - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "tests")) - -from expectations import ( # noqa: E402 - ALL_FIXTURES, - ALL_ONLINE_FIXTURES, - load_fixture, - load_online_fixture, - load_registry, -) - - -def _load(name: str, *, online: bool) -> pd.DataFrame: - if online: - return load_online_fixture(name) - return load_fixture(name) - - -def _search_datasets(tag: str) -> list[str]: - registry = load_registry() - return sorted(k for k, v in registry.items() if tag in v.get("tags", [])) - - -def _print_compare(df: pd.DataFrame, label: str) -> None: - table = fd.compare_clean(df) - print(f"\n=== {label} ({len(df)} rows x {df.shape[1]} cols) ===") - print(table.to_string(index=False)) - _, report = fd.clean(df, strategy="balanced", return_report=True, verbose=False) - if report.warnings: - print("\nWarnings:") - for w in report.warnings: - print(f" - {w}") - recs = getattr(report, "recommendations", None) or [] - if recs: - print("\nRecommendations:") - for r in recs: - print(f" - {r}") - - -def _print_explain(df: pd.DataFrame, label: str) -> None: - explanation = fd.explain_clean(df, strategy="balanced") - print(f"\n=== explain_clean: {label} ===") - print(explanation.summary()) - if not explanation.roles.empty: - print("\nRoles:") - print(explanation.roles.to_string(index=False)) - - -def _print_roles(df: pd.DataFrame, label: str) -> None: - roles = fd.infer_roles(df, strategy="balanced") - print(f"\n=== infer_roles: {label} ===") - print(roles.to_string(index=False)) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Debug freshdata on fixture datasets") - parser.add_argument("--online", action="store_true", help="Online cached datasets only") - parser.add_argument("--local", action="store_true", help="Local fixtures only") - parser.add_argument( - "--refresh", - action="store_true", - help="Re-fetch online cache before compare", - ) - parser.add_argument("--compare", action="store_true", help="Print compare_clean table") - parser.add_argument("--explain", action="store_true", help="Print explain_clean summary") - parser.add_argument("--infer-roles", action="store_true", help="Print infer_roles table") - parser.add_argument("--search", metavar="TAG", help="Filter online datasets by registry tag") - parser.add_argument("--format", choices=["text", "json"], default="text") - parser.add_argument("names", nargs="*", help="Dataset id(s); default all in scope") - args = parser.parse_args(argv) - - if args.search and not args.names: - args.names = _search_datasets(args.search) - args.online = True - - if args.refresh: - cmd = [sys.executable, str(ROOT / "scripts" / "fetch_online_fixtures.py"), "--refresh"] - if args.names: - for name in args.names: - cmd.extend(["--only", name]) - subprocess.run(cmd, check=True, cwd=ROOT) - - scopes: list[tuple[str, bool]] = [] - if args.online and not args.local: - scopes = [(n, True) for n in (args.names or ALL_ONLINE_FIXTURES)] - elif args.local and not args.online: - scopes = [(n, False) for n in (args.names or ALL_FIXTURES)] - else: - names = args.names - if names: - scopes = [(n, n in ALL_ONLINE_FIXTURES) for n in names] - else: - scopes = [(n, False) for n in ALL_FIXTURES] + [(n, True) for n in ALL_ONLINE_FIXTURES] - - show_compare = args.compare or not (args.explain or args.infer_roles) - reports: list[dict] = [] - print(f"freshdata {fd.__version__} | comparing {len(scopes)} dataset(s)") - for name, online in scopes: - try: - df = _load(name, online=online) - except Exception as exc: # noqa: BLE001 β€” CLI should continue on bad fixture - print(f"\n=== {name}: SKIP ({exc}) ===") - continue - tag = "online" if online else "local" - label = f"{name} [{tag}]" - if show_compare: - _print_compare(df, label) - if args.explain: - _print_explain(df, label) - if args.infer_roles: - _print_roles(df, label) - if args.format == "json": - explanation = fd.explain_clean(df, strategy="balanced") - reports.append({"name": name, "online": online, **explanation.to_dict()}) - - if args.format == "json" and reports: - print(json.dumps(reports, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/generate_html_examples.py b/scripts/generate_html_examples.py deleted file mode 100644 index e0cf30c1..00000000 --- a/scripts/generate_html_examples.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -"""Generate the committed interactive-output HTML samples under docs/examples/. - -Run from the repo root: - - python scripts/generate_html_examples.py - -The samples are produced from small synthetic frames by the real renderers, so -they always match the current output. They need no optional viz dependencies. -""" - -from __future__ import annotations - -import os - -import numpy as np -import pandas as pd - -import freshdata as fd - -_OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "docs", "examples") - - -def _sample_frame() -> pd.DataFrame: - rng = np.random.default_rng(0) - n = 200 - df = pd.DataFrame({ - "customer_id": range(1, n + 1), - "email": [f"user{i}@example.com" if i % 9 else None for i in range(n)], - "revenue": rng.normal(1000, 250, n).round(2), - "signup_date": pd.date_range("2025-01-01", periods=n, freq="D").astype(str), - "tier": rng.choice(["bronze", "silver", "gold", None], n), - }) - df.loc[3:7, "revenue"] = [1e6, 9e5, 1.1e6, 8e5, 1e6] # outliers - return pd.concat([df, df.iloc[:5]], ignore_index=True) # duplicates - - -def _write(name: str, html: str) -> None: - os.makedirs(_OUT, exist_ok=True) - path = os.path.join(_OUT, name) - with open(path, "w", encoding="utf-8") as fh: - fh.write(f"{name}{html}") - print(f"wrote {path}") - - -def main() -> int: - df = _sample_frame() - cleaned, report = fd.clean(df, return_report=True) - - _write("profile_cockpit.html", fd.profile(df).to_html()) - _write("action_timeline.html", report.to_html()) - _write("suggest_plan_cards.html", fd.suggest_plan(df).to_html()) - _write("explain_diff.html", fd.explain_clean(df).to_html()) - _write("compare_plans_grid.html", fd.compare_plans(df)._repr_html_()) - - baseline = df.iloc[:150].copy() - diff = fd.compare_to_baseline(df, baseline=baseline, key="customer_id") - _write("baseline_drift.html", diff.to_html()) - - _, gate = fd.evaluate_quality_debt(df, ledger=None) - _write("quality_debt.html", gate.to_html()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/freshdata/__init__.py b/src/freshdata/__init__.py index 5b6f2929..950d660f 100644 --- a/src/freshdata/__init__.py +++ b/src/freshdata/__init__.py @@ -1,20 +1,18 @@ # src/freshdata/__init__.py -from freshdata._core import ( +from importlib.metadata import version as _version + +from freshdata._core.engine import Cleaner, apply_plan, clean, profile, suggest_plan +from freshdata._core.models import ( ActionRecord, CleanConfig, - Cleaner, CleanPlan, CleanReport, ColumnProfile, ColumnRule, ProfileReport, - apply_plan, - clean, - profile, - suggest_plan, ) -__version__ = "3.0.0" +__version__ = _version("freshdata-cleaner") __all__ = [ "ActionRecord", diff --git a/src/freshdata/_core/__init__.py b/src/freshdata/_core/__init__.py index 1160f53a..e69de29b 100644 --- a/src/freshdata/_core/__init__.py +++ b/src/freshdata/_core/__init__.py @@ -1,32 +0,0 @@ -# src/freshdata/_core/__init__.py -from freshdata._core.engine import ( - Cleaner, - apply_plan, - clean, - profile, - suggest_plan, -) -from freshdata._core.models import ( - ActionRecord, - CleanConfig, - CleanPlan, - CleanReport, - ColumnProfile, - ColumnRule, - ProfileReport, -) - -__all__ = [ - "ActionRecord", - "CleanConfig", - "Cleaner", - "CleanPlan", - "CleanReport", - "ColumnProfile", - "ColumnRule", - "ProfileReport", - "apply_plan", - "clean", - "profile", - "suggest_plan", -] diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 5f30a8d5..30e11c1b 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -1,8 +1,7 @@ # src/freshdata/_core/backends.py from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import pandas as pd -import polars as pl from freshdata._core.models import ( ActionRecord, CleanConfig, @@ -11,6 +10,7 @@ ProfileReport, ) from freshdata._core.stages import ( + _normalize_name, normalize_config, stage_column_names, stage_dtypes, @@ -20,6 +20,9 @@ stage_strings, ) +if TYPE_CHECKING: + import polars as pl + class PandasBackend: def profile(self, df: pd.DataFrame) -> ProfileReport: cols = {} @@ -40,18 +43,10 @@ def profile(self, df: pd.DataFrame) -> ProfileReport: def suggest_plan(self, df: pd.DataFrame, cfg: CleanConfig) -> CleanPlan: actions = [] cfg_eff = normalize_config(cfg) - cur, act = stage_column_names(df, cfg) - actions.extend(act) - cur, act = stage_strings(cur, cfg_eff) - actions.extend(act) - cur, act = stage_dtypes(cur, cfg_eff) - actions.extend(act) - cur, act = stage_duplicates(cur, cfg_eff) - actions.extend(act) - cur, act = stage_missing(cur, cfg_eff) - actions.extend(act) - cur, act = stage_outliers(cur, cfg_eff) - actions.extend(act) + cur = df + for stage in (stage_column_names, stage_strings, stage_dtypes, stage_duplicates, stage_missing, stage_outliers): + cur, act = stage(cur, cfg_eff) + actions.extend(act) return CleanPlan(actions=tuple(actions)) def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd.DataFrame: @@ -67,7 +62,7 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. for a in plan.actions: if a.stage == "duplicates" and a.action == "drop_duplicates": mask = ~out.duplicated().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) + out = out[mask].reset_index(drop=True) continue col = renames.get(a.column, a.column) @@ -83,17 +78,13 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") elif a.stage == "missing": - if a.action in ("impute_median", "impute_mean"): - val = a.parameters.get("value") - if val is not None: - out[col] = out[col].fillna(val) - elif a.action == "impute_mode": + if a.action.startswith("impute_"): val = a.parameters.get("value") if val is not None: out[col] = out[col].fillna(val) elif a.action == "drop_nulls": mask = ~out[col].isna().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) + out = out[mask].reset_index(drop=True) elif a.stage == "outliers": if a.action == "clip_iqr": lower = a.parameters.get("lower") @@ -105,31 +96,41 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. upper = a.parameters.get("upper") if lower is not None and upper is not None: mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) + out = out[mask].reset_index(drop=True) return out class PolarsBackend: def profile(self, df: pl.DataFrame | pl.LazyFrame) -> ProfileReport: - schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema + import polars as pl + + frame = None if isinstance(df, pl.LazyFrame) else df + schema = df.collect_schema() if frame is None else frame.schema + row_count = 0 if frame is None else len(frame) cols = {} for name, dtype in schema.items(): role = "numeric" if dtype.is_numeric() else "categorical" + nulls = 0 if frame is None else frame[name].null_count() cols[name] = ColumnProfile( name=name, inferred_role=role, - null_count=0, - null_ratio=0.0, - unique_count=0, + null_count=nulls, + null_ratio=nulls / max(row_count, 1), + unique_count=0 if frame is None else frame[name].drop_nulls().n_unique(), sample_values=(), ) - row_count = 0 if isinstance(df, pl.LazyFrame) else len(df) return ProfileReport(row_count=row_count, column_count=len(schema), columns=cols) def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> CleanPlan: + import polars as pl + schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema actions = [] cfg_eff = normalize_config(cfg) - for name, dtype in schema.items(): + clip_cols: dict[str, pl.Expr] = {} + for orig, dtype in schema.items(): + name = _normalize_name(orig) if cfg_eff.normalize_names else orig + if name != orig: + actions.append(ActionRecord("column_names", orig, "normalize_name", {"to": name})) if name in cfg_eff.id_columns or name == cfg_eff.target_column or name.endswith("_id") or name == "id": continue rule = cfg_eff.rules.get(name) @@ -139,48 +140,71 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): actions.append(ActionRecord("strings", name, "strip_strings", {})) if dtype.is_numeric(): + # Mirror the pandas stage order: IQR bounds see the imputed column. + base = pl.col(orig) impute_strat = rule.impute if rule else (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) if impute_strat in ("median", "auto"): actions.append(ActionRecord("missing", name, "impute_median", {})) + base = base.fill_null(base.median()) elif impute_strat == "mean": actions.append(ActionRecord("missing", name, "impute_mean", {})) + base = base.fill_null(base.mean()) outlier_strat = rule.outliers if rule else cfg_eff.default_outliers if outlier_strat == "clip": - actions.append(ActionRecord("outliers", name, "clip_iqr", {})) + clip_cols[name] = base + if clip_cols: + # One 1-row aggregate query; apply_plan itself never collects. + stats = df.lazy().select( + e + for i, base in enumerate(clip_cols.values()) + for e in ( + base.count().alias(f"n{i}"), + base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), + base.quantile(0.75, interpolation="linear").alias(f"q75_{i}"), + ) + ).collect().row(0, named=True) + for i, name in enumerate(clip_cols): + if stats[f"n{i}"] < 4: + continue + q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] + iqr = q75 - q25 + lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr + actions.append(ActionRecord("outliers", name, "clip_iqr", {"lower": float(lower), "upper": float(upper)})) if cfg_eff.drop_duplicates: actions.append(ActionRecord("duplicates", "*", "drop_duplicates", {})) return CleanPlan(actions=tuple(actions)) def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: CleanConfig) -> pl.DataFrame | pl.LazyFrame: + import polars as pl + cur = df # 1. Renames renames = {a.column: a.parameters["to"] for a in plan.actions if a.stage == "column_names" and a.action == "normalize_name"} if renames: cur = cur.rename(renames) - # 2. Column expressions - exprs = [] + # 2. Column expressions, chained per column so actions compose in plan order + exprs: dict[str, pl.Expr] = {} for a in plan.actions: col_name = renames.get(a.column, a.column) + expr = exprs.get(col_name, pl.col(col_name)) if a.stage == "strings" and a.action == "strip_strings": - exprs.append(pl.col(col_name).str.strip_chars().alias(col_name)) + exprs[col_name] = expr.str.strip_chars() elif a.stage == "missing": if a.action == "impute_median": val = a.parameters.get("value") - expr = pl.col(col_name).fill_null(val if val is not None else pl.col(col_name).median()).alias(col_name) - exprs.append(expr) + exprs[col_name] = expr.fill_null(val if val is not None else expr.median()) elif a.action == "impute_mean": val = a.parameters.get("value") - expr = pl.col(col_name).fill_null(val if val is not None else pl.col(col_name).mean()).alias(col_name) - exprs.append(expr) + exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) elif a.stage == "outliers" and a.action == "clip_iqr": lower = a.parameters.get("lower") upper = a.parameters.get("upper") if lower is not None and upper is not None: - exprs.append(pl.col(col_name).clip(lower, upper).alias(col_name)) + exprs[col_name] = expr.clip(lower, upper) if exprs: - cur = cur.with_columns(exprs) + cur = cur.with_columns(e.alias(c) for c, e in exprs.items()) # 3. Duplicates if any(a.stage == "duplicates" and a.action == "drop_duplicates" for a in plan.actions): @@ -193,6 +217,13 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea def resolve_backend(df: Any) -> PandasBackend | PolarsBackend: if isinstance(df, pd.DataFrame): return PandasBackend() - if isinstance(df, (pl.DataFrame, pl.LazyFrame)): - return PolarsBackend() + if type(df).__module__.split(".")[0] == "polars": + try: + import polars as pl + except ImportError as exc: + raise ImportError( + "Polars frames need the polars extra: pip install 'freshdata-cleaner[polars]'" + ) from exc + if isinstance(df, (pl.DataFrame, pl.LazyFrame)): + return PolarsBackend() raise TypeError(f"Unsupported frame type: {type(df).__name__}. freshdata supports pandas.DataFrame, polars.DataFrame, and polars.LazyFrame.") diff --git a/src/freshdata/_core/engine.py b/src/freshdata/_core/engine.py index e5d1f593..e0ed95fc 100644 --- a/src/freshdata/_core/engine.py +++ b/src/freshdata/_core/engine.py @@ -1,8 +1,6 @@ # src/freshdata/_core/engine.py from __future__ import annotations from typing import Any -import pandas as pd -import polars as pl from freshdata._core.backends import resolve_backend from freshdata._core.models import ( ActionRecord, @@ -28,10 +26,10 @@ def apply_plan( return_report: bool = False, ) -> Any: cfg = config or CleanConfig() - cleaned = resolve_backend(df).apply_plan(df, plan, cfg) + cleaned: Any = resolve_backend(df).apply_plan(df, plan, cfg) if return_report: - orig_r = len(df) if isinstance(df, (pd.DataFrame, pl.DataFrame)) else 0 - clean_r = len(cleaned) if isinstance(cleaned, (pd.DataFrame, pl.DataFrame)) else 0 + orig_r = 0 if type(df).__name__ == "LazyFrame" else len(df) + clean_r = 0 if type(cleaned).__name__ == "LazyFrame" else len(cleaned) orig_c = len(df.columns) if hasattr(df, "columns") else 0 clean_c = len(cleaned.columns) if hasattr(cleaned, "columns") else 0 rep = CleanReport(orig_r, clean_r, orig_c, clean_c, plan.actions) diff --git a/src/freshdata/_core/models.py b/src/freshdata/_core/models.py index 8f24029e..55c2d0a5 100644 --- a/src/freshdata/_core/models.py +++ b/src/freshdata/_core/models.py @@ -34,18 +34,6 @@ class ActionRecord: rows_affected: int = 0 rationale: str = "" - @property - def operation(self) -> str: - return self.action - - @property - def params(self) -> dict[str, Any]: - return self.parameters - - @property - def affected_rows(self) -> int: - return self.rows_affected - def to_dict(self) -> dict[str, Any]: return asdict(self) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 3ea36686..7a638bbb 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -1,5 +1,6 @@ # src/freshdata/_core/stages.py from __future__ import annotations +import dataclasses import re from typing import Tuple, List import numpy as np @@ -18,20 +19,7 @@ def normalize_config(cfg: CleanConfig) -> CleanConfig: norm_rules = {_normalize_name(k): v for k, v in cfg.rules.items()} norm_id = tuple(_normalize_name(c) for c in cfg.id_columns) norm_target = _normalize_name(cfg.target_column) if cfg.target_column else None - return CleanConfig( - strategy=cfg.strategy, - rules=norm_rules, - id_columns=norm_id, - target_column=norm_target, - normalize_names=cfg.normalize_names, - strip_strings=cfg.strip_strings, - fix_dtypes=cfg.fix_dtypes, - drop_duplicates=cfg.drop_duplicates, - default_impute=cfg.default_impute, - default_outliers=cfg.default_outliers, - iqr_factor=cfg.iqr_factor, - strict_backend=cfg.strict_backend, - ) + return dataclasses.replace(cfg, rules=norm_rules, id_columns=norm_id, target_column=norm_target) def stage_column_names(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: if not cfg.normalize_names: @@ -91,7 +79,7 @@ def stage_duplicates(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, return df, [] orig_len = len(df) mask = ~df.duplicated().to_numpy() - out = pd.DataFrame({col: pd.Series(df[col].array[mask], dtype=df[col].dtype) for col in df.columns}) + out = df[mask].reset_index(drop=True) dropped = orig_len - len(out) actions = [ActionRecord("duplicates", "*", "drop_duplicates", {}, dropped)] if dropped > 0 else [] return out, actions @@ -108,7 +96,12 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis continue impute_strategy = rule.impute or (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) null_count = int(out[col].isna().sum()) - if 0 < null_count < len(out): + if impute_strategy == "drop" and (null_count > 0 or len(out) == 0): + orig_len = len(out) + mask = ~out[col].isna().to_numpy() + out = out[mask].reset_index(drop=True) + actions.append(ActionRecord("missing", str(col), "drop_nulls", {}, orig_len - len(out))) + elif 0 < null_count < len(out): if impute_strategy in ("median", "auto") and pd.api.types.is_numeric_dtype(out[col]): med = out[col].median() if not pd.isna(med): @@ -127,16 +120,6 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis val = mode_vals.iloc[0] out[col] = out[col].fillna(val) actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": str(val)}, null_count)) - elif impute_strategy == "drop": - orig_len = len(out) - mask = ~out[col].isna().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) - actions.append(ActionRecord("missing", str(col), "drop_nulls", {}, orig_len - len(out))) - elif null_count == len(out) and impute_strategy == "drop": - orig_len = len(out) - mask = ~out[col].isna().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) - actions.append(ActionRecord("missing", str(col), "drop_nulls", {}, orig_len - len(out))) return out, actions def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: @@ -150,23 +133,20 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li if rule.protected: continue strategy = rule.outliers or cfg_eff.default_outliers - if strategy == "clip" and pd.api.types.is_numeric_dtype(out[col]): - s = out[col].dropna() - if len(s) < 4: - continue - q25, q75 = np.percentile(s, [25, 75]) - iqr = q75 - q25 - lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr + if strategy not in ("clip", "drop") or not pd.api.types.is_numeric_dtype(out[col]): + continue + s = out[col].dropna() + if len(s) < 4: + continue + q25, q75 = np.percentile(s, [25, 75]) + iqr = q75 - q25 + lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr + if strategy == "clip": out[col] = out[col].clip(lower=lower, upper=upper) actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, len(out))) - elif strategy == "drop" and pd.api.types.is_numeric_dtype(out[col]): - s = out[col].dropna() - if len(s) >= 4: - q25, q75 = np.percentile(s, [25, 75]) - iqr = q75 - q25 - lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr - mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() - dropped = int((~mask).sum()) - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) - actions.append(ActionRecord("outliers", str(col), "drop_outliers", {"lower": float(lower), "upper": float(upper)}, dropped)) + else: + mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() + dropped = int((~mask).sum()) + out = out[mask].reset_index(drop=True) + actions.append(ActionRecord("outliers", str(col), "drop_outliers", {"lower": float(lower), "upper": float(upper)}, dropped)) return out, actions diff --git a/src/freshdata/polars_engine.py b/src/freshdata/polars_engine.py deleted file mode 100644 index 6802036c..00000000 --- a/src/freshdata/polars_engine.py +++ /dev/null @@ -1,354 +0,0 @@ -from __future__ import annotations - -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any, Literal, TypeVar, overload - -import polars as pl - -FrameT = TypeVar("FrameT", pl.DataFrame, pl.LazyFrame) - -RoleType = Literal["id", "target", "numeric", "categorical", "datetime", "text"] -ImputeType = Literal["preserve", "drop", "mean", "median", "mode"] -OutlierType = Literal["preserve", "clip", "flag"] - - -@dataclass(frozen=True) -class ColumnRule: - role: RoleType | None = None - impute: ImputeType | None = None - outliers: OutlierType | None = None - protected: bool = False - - -@dataclass(frozen=True) -class CleanConfig: - strategy: Literal["conservative", "balanced", "aggressive"] = "balanced" - rules: Mapping[str, ColumnRule] = field(default_factory=dict) - id_columns: tuple[str, ...] = () - target_column: str | None = None - normalize_names: bool = True - strip_strings: bool = True - drop_duplicates: bool = False - default_impute: Literal["auto", "preserve", "median", "mode"] = "auto" - default_outliers: OutlierType = "preserve" - iqr_factor: float = 1.5 - strict_backend: bool = True - - -@dataclass(frozen=True) -class ActionRecord: - column: str - stage: Literal["strings", "missing", "outliers", "drop_column"] - operation: Literal[ - "strip_whitespace", - "fill_null", - "clip_iqr", - "flag_iqr", - "drop_column", - "preserve", - ] - params: Mapping[str, Any] = field(default_factory=dict) - affected_rows: int = 0 - rationale: str = "" - - -@dataclass(frozen=True) -class CleanPlan: - rename_map: Mapping[str, str] - actions: tuple[ActionRecord, ...] - drop_duplicates: bool = False - total_rows: int = 0 - - def diff( - self, other: CleanPlan - ) -> dict[tuple[str, str], tuple[ActionRecord | None, ActionRecord | None]]: - self_map = {(a.column, a.stage): a for a in self.actions} - other_map = {(a.column, a.stage): a for a in other.actions} - changes: dict[tuple[str, str], tuple[ActionRecord | None, ActionRecord | None]] = {} - for key in sorted(set(self_map) | set(other_map)): - if self_map.get(key) != other_map.get(key): - changes[key] = (self_map.get(key), other_map.get(key)) - return changes - - -@dataclass(frozen=True) -class CleanReport: - plan: CleanPlan - renamed_columns: Mapping[str, str] - dropped_columns: tuple[str, ...] - total_actions: int - - -def _to_snake_case(name: str) -> str: - cleaned = re.sub(r"[^\w\s]", "", name).strip() - cleaned = re.sub(r"[\s\-]+", "_", cleaned) - cleaned = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", cleaned) - return cleaned.lower() - - -def _infer_role( - col: str, dtype: pl.DataType, config: CleanConfig, rule: ColumnRule | None -) -> RoleType: - if rule and rule.role is not None: - return rule.role - if col in config.id_columns or col.endswith("_id") or col == "id": - return "id" - if col == config.target_column: - return "target" - if dtype.is_numeric(): - return "numeric" - if dtype.is_temporal(): - return "datetime" - return "categorical" - - -def suggest_plan( - df: pl.DataFrame | pl.LazyFrame, - config: CleanConfig | None = None, -) -> CleanPlan: - cfg = config or CleanConfig() - if not isinstance(df, (pl.DataFrame, pl.LazyFrame)): - raise TypeError( - f"Expected pl.DataFrame or pl.LazyFrame when strict_backend=True, got {type(df).__name__}." - ) - - lf = df.lazy() - schema = lf.collect_schema() - - rename_map: dict[str, str] = {} - for orig_col in schema.names(): - norm_col = _to_snake_case(orig_col) if cfg.normalize_names else orig_col - rename_map[orig_col] = norm_col - - # Build a single fused aggregation pass so LazyFrames never materialize rows - stat_exprs: list[pl.Expr] = [pl.len().alias("__total_rows")] - for orig_col, dtype in schema.items(): - norm_col = rename_map[orig_col] - col_expr = pl.col(orig_col) - stat_exprs.append(col_expr.null_count().alias(f"{norm_col}__nulls")) - - if dtype == pl.String and cfg.strip_strings: - stat_exprs.append( - col_expr.str.strip_chars() - .ne(col_expr) - .fill_null(False) - .sum() - .alias(f"{norm_col}__ws_dirty") - ) - elif dtype.is_numeric(): - stat_exprs.extend( - [ - col_expr.median().alias(f"{norm_col}__median"), - col_expr.mean().alias(f"{norm_col}__mean"), - col_expr.quantile(0.25).alias(f"{norm_col}__q25"), - col_expr.quantile(0.75).alias(f"{norm_col}__q75"), - ] - ) - if dtype == pl.String: - stat_exprs.append( - col_expr.str.strip_chars() - .drop_nulls() - .mode() - .first() - .alias(f"{norm_col}__mode") - ) - - stats = lf.select(stat_exprs).collect(engine="streaming").row(0, named=True) - total_rows: int = int(stats["__total_rows"] or 0) - - actions: list[ActionRecord] = [] - - for orig_col, dtype in schema.items(): - col = rename_map[orig_col] - rule = cfg.rules.get(col) or cfg.rules.get(orig_col) - role = _infer_role(col, dtype, cfg, rule) - is_protected = bool( - (rule and rule.protected) or role in ("id", "target") - ) - - # 1. String whitespace repair - if dtype == pl.String and cfg.strip_strings: - ws_dirty = int(stats.get(f"{col}__ws_dirty") or 0) - if ws_dirty > 0: - actions.append( - ActionRecord( - column=col, - stage="strings", - operation="strip_whitespace", - affected_rows=ws_dirty, - rationale="Strip leading and trailing whitespace", - ) - ) - - # 2. Missing value planning - null_count = int(stats.get(f"{col}__nulls") or 0) - if null_count > 0: - if is_protected or cfg.strategy == "conservative": - actions.append( - ActionRecord( - column=col, - stage="missing", - operation="preserve", - affected_rows=null_count, - rationale=f"Protected role={role!r} or conservative strategy", - ) - ) - else: - null_ratio = null_count / max(total_rows, 1) - if cfg.strategy == "aggressive" and null_ratio >= 0.6: - actions.append( - ActionRecord( - column=col, - stage="drop_column", - operation="drop_column", - affected_rows=null_count, - rationale=f"Null ratio {null_ratio:.1%} exceeds aggressive drop threshold", - ) - ) - continue - - impute_choice = ( - rule.impute - if (rule and rule.impute is not None) - else cfg.default_impute - ) - if impute_choice == "auto": - impute_choice = "median" if dtype.is_numeric() else "mode" - - fill_val = stats.get(f"{col}__{impute_choice}") - if impute_choice != "preserve" and fill_val is not None: - actions.append( - ActionRecord( - column=col, - stage="missing", - operation="fill_null", - params={"value": fill_val, "method": impute_choice}, - affected_rows=null_count, - rationale=f"Impute {null_count} nulls via {impute_choice}={fill_val!r}", - ) - ) - else: - actions.append( - ActionRecord( - column=col, - stage="missing", - operation="preserve", - affected_rows=null_count, - rationale="Preserved nulls by policy", - ) - ) - - # 3. Outlier planning on numeric columns - if dtype.is_numeric() and not is_protected: - outlier_choice = ( - rule.outliers - if (rule and rule.outliers is not None) - else cfg.default_outliers - ) - q25 = stats.get(f"{col}__q25") - q75 = stats.get(f"{col}__q75") - if outlier_choice in ("clip", "flag") and q25 is not None and q75 is not None: - iqr = float(q75) - float(q25) - lower = float(q25) - cfg.iqr_factor * iqr - upper = float(q75) + cfg.iqr_factor * iqr - op = "clip_iqr" if outlier_choice == "clip" else "flag_iqr" - actions.append( - ActionRecord( - column=col, - stage="outliers", - operation=op, - params={"lower": lower, "upper": upper, "iqr_factor": cfg.iqr_factor}, - rationale=f"IQR bounds [{lower:.4g}, {upper:.4g}] (factor={cfg.iqr_factor})", - ) - ) - - return CleanPlan( - rename_map=rename_map, - actions=tuple(actions), - drop_duplicates=cfg.drop_duplicates, - total_rows=total_rows, - ) - - -@overload -def apply_plan( - df: FrameT, plan: CleanPlan, *, return_report: Literal[False] = False -) -> FrameT: ... - - -@overload -def apply_plan( - df: FrameT, plan: CleanPlan, *, return_report: Literal[True] -) -> tuple[FrameT, CleanReport]: ... - - -def apply_plan( - df: FrameT, plan: CleanPlan, *, return_report: bool = False -) -> FrameT | tuple[FrameT, CleanReport]: - is_eager = isinstance(df, pl.DataFrame) - lf: pl.LazyFrame = df.lazy() - - non_trivial_renames = {k: v for k, v in plan.rename_map.items() if k != v} - if non_trivial_renames: - lf = lf.rename(non_trivial_renames) - - dropped_cols: list[str] = [ - a.column for a in plan.actions if a.operation == "drop_column" - ] - if dropped_cols: - lf = lf.drop(dropped_cols) - - # Group column expressions by column so chained operations fuse into one .with_columns() - col_exprs: dict[str, pl.Expr] = {} - extra_exprs: list[pl.Expr] = [] - - for action in plan.actions: - col = action.column - if col in dropped_cols or action.operation == "preserve": - continue - - expr = col_exprs.get(col, pl.col(col)) - - if action.operation == "strip_whitespace": - expr = expr.str.strip_chars() - col_exprs[col] = expr - elif action.operation == "fill_null": - expr = expr.fill_null(pl.lit(action.params["value"])) - col_exprs[col] = expr - elif action.operation == "clip_iqr": - expr = expr.clip( - lower_bound=action.params["lower"], - upper_bound=action.params["upper"], - ) - col_exprs[col] = expr - elif action.operation == "flag_iqr": - lo = action.params["lower"] - hi = action.params["upper"] - extra_exprs.append( - (pl.col(col).lt(lo) | pl.col(col).gt(hi)) - .fill_null(False) - .alias(f"{col}_is_outlier") - ) - - all_exprs = [expr.alias(col) for col, expr in col_exprs.items()] + extra_exprs - if all_exprs: - lf = lf.with_columns(all_exprs) - - if plan.drop_duplicates: - # maintain_order=False allows Polars to stream deduplication without materializing - lf = lf.unique(maintain_order=False) - - output: FrameT = lf.collect(engine="streaming") if is_eager else lf # type: ignore[assignment] - - if not return_report: - return output - - report = CleanReport( - plan=plan, - renamed_columns=non_trivial_renames, - dropped_columns=tuple(dropped_cols), - total_actions=len(plan.actions), - ) - return output, report diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index d12baee1..a052cab8 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -37,3 +37,85 @@ def guarded_to_pandas(self, *args, **kwargs): def test_unsupported_type_raises(): with pytest.raises(TypeError, match="Unsupported frame type"): resolve_backend({"a": [1, 2, 3]}) + +def test_import_and_clean_without_polars(): + import subprocess + import sys + from pathlib import Path + + code = ( + "import sys; sys.modules['polars'] = None\n" + "import pandas as pd, freshdata\n" + "out = freshdata.clean(pd.DataFrame({'A b': [1.0, None, 3.0]}))\n" + "assert out['a_b'].tolist() == [1.0, 2.0, 3.0]\n" + ) + src = Path(__file__).resolve().parents[1] / "src" + subprocess.run([sys.executable, "-c", code], check=True, env={"PYTHONPATH": str(src)}) + +def test_polars_profile_counts_match_pandas(): + data = {"a": [1, 2, None, 2], "b": ["x", None, None, "y"]} + pd_rep = resolve_backend(pd.DataFrame(data)).profile(pd.DataFrame(data)) + pl_rep = resolve_backend(pl.DataFrame(data)).profile(pl.DataFrame(data)) + for col in data: + p, q = pd_rep.columns[col], pl_rep.columns[col] + assert (q.null_count, q.unique_count, q.null_ratio) == (p.null_count, p.unique_count, p.null_ratio) + +def test_polars_plan_matches_pandas_plan(): + import freshdata as fd + + data = {"Customer ID": ["1", "2", "3", "4", "5"], "Age": [25.0, 30.0, None, 35.0, 40.0], + "Spend": [100.0, 120.0, 110.0, 115.0, 9999.0], "Churn": [0.0, 1.0, None, 0.0, 1.0]} + cfg = CleanConfig(id_columns=("customer_id",), target_column="churn", default_outliers="clip", + strip_strings=False, fix_dtypes=False) + def key(plan): + return {(a.stage, a.column, a.action): a.parameters for a in plan.actions + if a.stage in ("column_names", "outliers")} + assert key(fd.suggest_plan(pl.LazyFrame(data), cfg)) == key(fd.suggest_plan(pd.DataFrame(data), cfg)) + +def test_lazy_plan_applies_without_materializing(): + import freshdata as fd + + lf = pl.DataFrame({ + "Customer ID": ["001", "002", "003", "004", "005"], + "Age": [25, 30, None, 35, 40], + "Spend": [100.0, 120.0, 110.0, 115.0, 9999.0], + "Segment": [" Gold ", "Silver", None, "Gold", "Gold"], + "Churn": [0, 1, None, 0, 1], + }).lazy() + config = CleanConfig(id_columns=("customer_id",), target_column="churn", + default_impute="auto", default_outliers="clip", drop_duplicates=True) + plan = fd.suggest_plan(lf, config) + + actions = {(a.column, a.stage): a for a in plan.actions} + assert ("churn", "missing") not in actions + assert actions[("age", "missing")].action == "impute_median" + assert actions[("spend", "outliers")].action == "clip_iqr" + + poisoned_lf = lf.with_columns( + pl.col("Age").map_batches( + lambda _: (_ for _ in ()).throw(RuntimeError("Eager collect triggered!")), + return_dtype=pl.Int64, + ) + ) + assert isinstance(fd.apply_plan(poisoned_lf, plan, config), pl.LazyFrame) + + cleaned_lf, report = fd.apply_plan(lf, plan, config, return_report=True) + assert isinstance(cleaned_lf, pl.LazyFrame) + assert "PYTHON_UDF" not in cleaned_lf.explain() + result = cleaned_lf.collect() + assert result.columns == ["customer_id", "age", "spend", "segment", "churn"] + assert result["age"].null_count() == 0 + assert result["churn"].null_count() == 1 + assert result["spend"].max() < 9999.0 + assert report.total_actions == len(plan.actions) + +def test_lazy_plan_diff_detects_rule_changes(): + import freshdata as fd + + lf = pl.LazyFrame({"score": [1.0, 2.0, None, 4.0, 100.0]}) + conservative = fd.suggest_plan(lf, CleanConfig(strategy="conservative", default_outliers="preserve")) + aggressive = fd.suggest_plan(lf, CleanConfig( + strategy="aggressive", rules={"score": fd.ColumnRule(impute="median", outliers="clip")})) + diff = conservative.diff(aggressive) + assert [(a.column, a.action) for a in diff["added"]] == [("score", "clip_iqr")] + assert diff["removed"] == [] and diff["modified"] == [] diff --git a/tests/test_polars_engine.py b/tests/test_polars_engine.py deleted file mode 100644 index 1abb06af..00000000 --- a/tests/test_polars_engine.py +++ /dev/null @@ -1,81 +0,0 @@ -import polars as pl - -from freshdata.polars_engine import ( - CleanConfig, - ColumnRule, - apply_plan, - suggest_plan, -) - - -def test_apply_plan_preserves_lazyframe_without_materializing() -> None: - raw_df = pl.DataFrame( - { - "Customer ID": ["001", "002", "003", "004", "005"], - "Age": [25, 30, None, 35, 40], - "Spend": [100.0, 120.0, 110.0, 115.0, 9999.0], - "Segment": [" Gold ", "Silver", None, "Gold", "Gold"], - "Churn": [0, 1, None, 0, 1], - } - ) - lf = raw_df.lazy() - - config = CleanConfig( - id_columns=("customer_id",), - target_column="churn", - default_impute="auto", - default_outliers="clip", - drop_duplicates=True, - ) - - plan = suggest_plan(lf, config=config) - - # Verify protected columns are never imputed or clipped - actions_by_col = {(a.column, a.stage): a for a in plan.actions} - assert actions_by_col[("churn", "missing")].operation == "preserve" - assert actions_by_col[("age", "missing")].operation == "fill_null" - assert actions_by_col[("age", "missing")].params["value"] == 32.5 - assert actions_by_col[("spend", "outliers")].operation == "clip_iqr" - - # Poison the LazyFrame after planning to prove apply_plan() never calls .collect() - poisoned_lf = lf.with_columns( - pl.col("Age").map_batches( - lambda _: (_ for _ in ()).throw(RuntimeError("Eager collect triggered!")), - return_dtype=pl.Int64, - ) - ) - lazy_repaired = apply_plan(poisoned_lf, plan) - assert isinstance(lazy_repaired, pl.LazyFrame) - - # Now execute on the clean LazyFrame and verify native Polars output - cleaned_lf, report = apply_plan(lf, plan, return_report=True) - assert isinstance(cleaned_lf, pl.LazyFrame) - assert "PYTHON_UDF" not in cleaned_lf.explain() - - result = cleaned_lf.collect() - assert result.columns == ["customer_id", "age", "spend", "segment", "churn"] - assert result["age"].null_count() == 0 - assert result["churn"].null_count() == 1 # Protected target preserved - assert result["spend"].max() < 9999.0 # Outlier clipped via precomputed IQR - assert report.total_actions == len(plan.actions) - - -def test_plan_diff_detects_strategy_changes() -> None: - lf = pl.LazyFrame({"score": [1.0, 2.0, None, 4.0, 100.0]}) - - conservative_plan = suggest_plan( - lf, CleanConfig(strategy="conservative", default_outliers="preserve") - ) - aggressive_plan = suggest_plan( - lf, - CleanConfig( - strategy="aggressive", - rules={"score": ColumnRule(impute="median", outliers="clip")}, - ), - ) - - diffs = conservative_plan.diff(aggressive_plan) - assert ("score", "outliers") in diffs - before_act, after_act = diffs[("score", "outliers")] - assert before_act is None - assert after_act is not None and after_act.operation == "clip_iqr" diff --git a/tests/test_property_invariants.py b/tests/test_property_invariants.py index 4dd55c75..fe53c24d 100644 --- a/tests/test_property_invariants.py +++ b/tests/test_property_invariants.py @@ -7,11 +7,6 @@ import freshdata as fd from freshdata import CleanConfig, ColumnRule -from freshdata.polars_engine import ( - CleanConfig as PeCleanConfig, - apply_plan as pe_apply, - suggest_plan as pe_suggest, -) @st.composite @@ -113,12 +108,10 @@ def test_numeric_imputation_and_outlier_invariants(values: list[float | None]) - max_size=30, ) ) -def test_polars_engine_property_invariants(values: list[float | None]) -> None: +def test_lazy_clean_property_invariants(values: list[float | None]) -> None: lf = pl.LazyFrame({"val": values, "txt": [" foo " for _ in values]}) - cfg = PeCleanConfig(default_impute="median", default_outliers="clip", strip_strings=True) - plan = pe_suggest(lf, config=cfg) - assert plan is not None - cleaned_lf = pe_apply(lf, plan) + cfg = CleanConfig(default_impute="median", default_outliers="clip", strip_strings=True) + cleaned_lf = fd.clean(lf, config=cfg) assert isinstance(cleaned_lf, pl.LazyFrame) collected = cleaned_lf.collect() has_non_null = any(v is not None for v in values) diff --git a/tests/test_stages.py b/tests/test_stages.py index 86a720f9..05fb3bd2 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -65,3 +65,61 @@ def test_stage_outliers_short_series(): assert len(actions) == 0 assert out_df["x"].tolist() == [1.0, 2.0] + +def test_row_filters_keep_extension_dtypes(): + import freshdata as fd + + df = pd.DataFrame({ + "n": pd.array([1, 1, None, 2, 3, 4], dtype="Int64"), + "s": pd.array(["a", "a", "b", None, "c", "d"], dtype="string"), + "c": pd.Categorical(["x", "x", "y", "y", "x", "y"]), + "t": pd.to_datetime(["2024-01-01", "2024-01-01", "2024-01-02", "2024-01-03", "2024-01-04", "2024-01-05"]), + "v": [1.0, 1.0, 2.0, 3.0, 4.0, 1000.0], + }) + cfg = CleanConfig( + normalize_names=False, strip_strings=False, fix_dtypes=False, drop_duplicates=True, + default_impute="preserve", + rules={"n": ColumnRule(impute="drop"), "v": ColumnRule(outliers="drop")}, + ) + out, report = fd.clean(df, cfg, return_report=True) + assert {a.action for a in report.actions} == {"drop_duplicates", "drop_nulls", "drop_outliers"} + assert out.dtypes.to_dict() == df.dtypes.to_dict() + assert out["n"].tolist() == [1, 2, 3] + assert list(out.index) == [0, 1, 2] + +NAN = float("nan") + +@pytest.mark.parametrize(("strategy", "values", "expected", "actions"), [ + ("median", [1.0, 2.0, 3.0, 10.0], [1.0, 2.0, 3.0, 10.0], []), + ("median", [1.0, None, 3.0, 10.0], [1.0, 3.0, 3.0, 10.0], ["impute_median"]), + ("median", [None] * 4, [NAN] * 4, []), + ("mean", [1.0, None, 3.0, 10.0], [1.0, 14 / 3, 3.0, 10.0], ["impute_mean"]), + ("mean", [None] * 4, [NAN] * 4, []), + ("mode", [1.0, None, 3.0, 10.0], [1.0, 1.0, 3.0, 10.0], ["impute_mode"]), + ("mode", [None] * 4, [NAN] * 4, []), + ("drop", [1.0, 2.0, 3.0, 10.0], [1.0, 2.0, 3.0, 10.0], []), + ("drop", [1.0, None, 3.0, 10.0], [1.0, 3.0, 10.0], ["drop_nulls"]), + ("drop", [None] * 4, [], ["drop_nulls"]), + ("drop", [], [], ["drop_nulls"]), + ("preserve", [1.0, None, 3.0, 10.0], [1.0, NAN, 3.0, 10.0], []), +]) +def test_stage_missing_strategies(strategy, values, expected, actions): + df = pd.DataFrame({"x": pd.Series(values, dtype="float64")}) + out, acts = stage_missing(df, CleanConfig(rules={"x": ColumnRule(impute=strategy)})) + assert out["x"].tolist() == pytest.approx(expected, nan_ok=True) + assert [a.action for a in acts] == actions + +@pytest.mark.parametrize(("strategy", "values", "expected", "actions"), [ + ("clip", [10.0, 11.0, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, 12.0, 10.0, 11.0, 14.0], ["clip_iqr"]), + ("clip", [10.0, 11.0, None, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, NAN, 12.0, 10.0, 11.0, 14.0], ["clip_iqr"]), + ("clip", [None] * 6, [NAN] * 6, []), + ("drop", [10.0, 11.0, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, 12.0, 10.0, 11.0], ["drop_outliers"]), + ("drop", [10.0, 11.0, None, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, NAN, 12.0, 10.0, 11.0], ["drop_outliers"]), + ("drop", [None] * 6, [NAN] * 6, []), + ("preserve", [10.0, 11.0, None, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, NAN, 12.0, 10.0, 11.0, 100.0], []), +]) +def test_stage_outliers_strategies(strategy, values, expected, actions): + df = pd.DataFrame({"x": pd.Series(values, dtype="float64")}) + out, acts = stage_outliers(df, CleanConfig(rules={"x": ColumnRule(outliers=strategy)})) + assert out["x"].tolist() == pytest.approx(expected, nan_ok=True) + assert [a.action for a in acts] == actions diff --git a/training/COMPLIANCE.md b/training/COMPLIANCE.md deleted file mode 100644 index 128060f5..00000000 --- a/training/COMPLIANCE.md +++ /dev/null @@ -1,71 +0,0 @@ -# Training pipeline compliance - -This document governs every development-time use of external ("teacher") -models and external data in the FreshData training pipeline. The runtime -library is unaffected: it is deterministic, offline, model-free by default, -and never calls an LLM or any cloud service from `fd.clean`. - -## Teacher models β€” allowed uses - -Teacher models may be used **only inside `training/`**, only for: - -1. realism direction (tuning corruptor frequency/parameters) -2. column-role labeling -3. NL context paraphrase generation -4. ambiguity adjudication -5. rationale style templates -6. benchmark red-teaming - -They must **never** be used to: - -- create messy/clean pairs from scratch (unverifiable ground truth) -- label release-gating eval data without human verification -- run at library runtime, per cell, or inside `fd.clean` -- receive full data rows or real PII (values are masked first; - `training/teacher/tasks.py` enforces both) - -## Provider terms checklist (before any teacher run) - -Every provider must have a record in `training/teacher/compliance_ledger.json` -containing: - -- provider name -- terms URL and a terms snapshot id -- date the terms were checked (records older than 180 days are stale) -- reviewer (name or initials of the human who read the terms) -- allowed/disallowed usage list -- whether outputs may be used for **model training** -- whether outputs may be **redistributed** -- status: `approved` / `rejected` / `pending` - -`python -m training.teacher.compliance check` blocks the pipeline when a -snapshot is missing or stale, the intended use is not approved, the reviewer -is missing, or the provider disallows training use. `make -training-release-artifacts` runs this check first. - -## Audit trail - -Every teacher call is cached under `training/cache/teacher/` with the raw -prompt, raw response, provider, model, timestamp, and terms snapshot id -(key: `provider + model + prompt_template_sha256 + input_sha256 + -schema_version`). Cache entries are the audit record for every -teacher-derived training label. - -## Human review - -- β‰₯ 5% of every teacher batch is sampled for human review - (`training/teacher/review.py`). -- Teacher–human disagreement > 3% forces full review of the batch. -- Release-gating eval labels are 100% human-verified - (`training/eval/human_verified.py` refuses anything less). -- Reviewer identity, timestamp, and disagreement reasons are stored and - exported in the review summary JSON. - -## Training data - -- Every seed source is registered in `training/seed/registry.json` with - license, attribution, explicit training permission, and resolved PII risk; - `python -m training.datasets.validators --check-licenses` gates releases. -- No real PII anywhere: identity-shaped data is fully synthetic - (`training/seed/synthetic/`), tagged `synthetic=true`. -- Corruption metadata β€” not teacher output β€” is the preferred ground truth. diff --git a/training/Makefile b/training/Makefile deleted file mode 100644 index 9faff64f..00000000 --- a/training/Makefile +++ /dev/null @@ -1,87 +0,0 @@ -# FreshData development-time training pipeline (Phase 5). -# All targets run from the repository root; `make -C training ` -# and the root Makefile's training-* passthrough both work. - -PY ?= python -ROOT := .. - -.PHONY: help training-seed training-corrupt training-teacher-labels \ - training-distill training-eval training-package-artifacts \ - training-dev-artifacts training-release-artifacts - -help: - @echo "Targets:" - @echo " training-seed Validate licenses + build the seed corpus" - @echo " training-corrupt Generate labeled corrupted training data" - @echo " training-teacher-labels Run compliance-gated teacher tasks (degrades offline)" - @echo " training-distill Train role/intent heads (+ optional encoder), export, quantize" - @echo " training-eval Calibration eval + full CleanBench release gates" - @echo " training-package-artifacts Package artifacts into dist/artifacts/" - @echo " training-dev-artifacts Small synthetic-only end-to-end build (no cloud)" - @echo " training-release-artifacts Full release pipeline; fails on any gate failure" - -training-seed: - cd $(ROOT) && $(PY) -m training.datasets.validators --check-licenses - cd $(ROOT) && $(PY) -m training.datasets.build_seed - -training-corrupt: - cd $(ROOT) && $(PY) -m training.datasets.build_corrupted - -training-teacher-labels: - cd $(ROOT) && $(PY) -m training.teacher.compliance check - @echo "teacher tasks run on demand via training.teacher.tasks.run_task;" - @echo "cached outputs (training/cache/teacher/) feed the trainers automatically." - -training-distill: - cd $(ROOT) && $(PY) -m training.distill.train_role_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_intent_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_encoder_contrastive --check-gates - cd $(ROOT) && $(PY) -m training.distill.export_onnx - cd $(ROOT) && $(PY) -m training.distill.quantize --check-gates - -training-eval: - cd $(ROOT) && $(PY) -m training.calibration.build_features - cd $(ROOT) && $(PY) -m training.calibration.fit_isotonic --fit-deterministic - cd $(ROOT) && $(PY) -m training.calibration.export_tables - cd $(ROOT) && $(PY) -m training.calibration.eval_calibration --check-gates - cd $(ROOT) && $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --check-gates - -training-package-artifacts: - cd $(ROOT) && $(PY) -m training.distill.package_artifacts - -# Small synthetic-only artifact set: no cloud teacher calls, no onnx needed, -# nothing enters the wheel. Ends with the CleanBench mini gates (T1-T4). -training-dev-artifacts: - cd $(ROOT) && $(PY) -m training.datasets.validators --check-licenses - cd $(ROOT) && $(PY) -m training.datasets.build_seed - cd $(ROOT) && $(PY) -m training.datasets.build_corrupted --dev - cd $(ROOT) && $(PY) -m training.distill.train_role_head --dev --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_intent_head --dev --check-gates - cd $(ROOT) && $(PY) -m training.distill.export_onnx - cd $(ROOT) && $(PY) -m training.distill.quantize --check-gates - cd $(ROOT) && $(PY) -m training.calibration.build_features --seeds 3 - cd $(ROOT) && $(PY) -m training.calibration.fit_isotonic --fit-deterministic - cd $(ROOT) && $(PY) -m training.calibration.export_tables - cd $(ROOT) && $(PY) -m training.calibration.eval_calibration --check-gates - cd $(ROOT) && $(PY) -m training.distill.package_artifacts - cd $(ROOT) && $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4 --check-gates - -# Full release pipeline: compliance first, human-verified eval enforced by -# the trainers, ONNX required, every gate blocking. -training-release-artifacts: - cd $(ROOT) && $(PY) -m training.teacher.compliance check - cd $(ROOT) && $(PY) -m training.datasets.validators --check-licenses - cd $(ROOT) && $(PY) -m training.datasets.build_seed - cd $(ROOT) && $(PY) -m training.datasets.build_corrupted - cd $(ROOT) && $(PY) -m training.distill.train_role_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_intent_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_encoder_contrastive --check-gates - cd $(ROOT) && $(PY) -m training.distill.export_onnx --release - cd $(ROOT) && $(PY) -m training.distill.quantize --check-gates - cd $(ROOT) && $(PY) -m training.calibration.build_features - cd $(ROOT) && $(PY) -m training.calibration.fit_isotonic --fit-deterministic - cd $(ROOT) && $(PY) -m training.calibration.export_tables - cd $(ROOT) && $(PY) -m training.calibration.eval_calibration --check-gates - cd $(ROOT) && $(PY) -m training.distill.package_artifacts --release - cd $(ROOT) && $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --check-gates - @echo "release pipeline complete: dist/artifacts/ + benchmarks/results/latest.*" diff --git a/training/README.md b/training/README.md deleted file mode 100644 index 544ba2be..00000000 --- a/training/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# FreshData training pipeline (Phase 5) - -Development-time pipeline that builds, evaluates, and packages FreshData's -small local model artifacts (`fd-col-encoder-v1`, `fd-intent-v1`, -`calib-v1`). This package is **not part of the runtime**: it lives outside -`src/freshdata`, is excluded from the wheel (see `pyproject.toml` and -`.github/workflows/wheel-size.yml`), and the runtime never imports it. - -FreshData's runtime stays deterministic, offline, model-free by default, and -free of any LLM or cloud call β€” nothing here changes that contract. See -[docs/developer-training-pipeline.md](../docs/developer-training-pipeline.md) -for the full guide and [COMPLIANCE.md](COMPLIANCE.md) for the teacher-model -governance rules. - -## Layout - -```text -training/ - seed/ seed corpus registry, licenses, synthetic PII-shaped generators - corruptors/ ~40 deterministic, labeled corruptors (representation/semantic/context/scale) - teacher/ compliance-gated development-time teacher harness (schemas, cache, review) - datasets/ seed/corrupted dataset builders, splits, license+label validators - distill/ role-head / intent-head / optional encoder training, ONNX export, quantize, package - calibration/ feature extraction, isotonic fit, table export, eval - eval/ human-verified release-gating label store - model_cards/ the three required model cards - cache/ teacher prompt/response audit cache (gitignored content, tracked directory) -``` - -## Quick start - -```bash -make training-dev-artifacts # synthetic-only, offline, seconds -make training-release-artifacts # full gated pipeline (compliance -> package -> CleanBench) -python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site -``` - -Individual stages: `make training-seed`, `training-corrupt`, -`training-teacher-labels`, `training-distill`, `training-eval`, -`training-package-artifacts` (see `training/Makefile`). - -## Ground-truth policy - -Corruption metadata is the preferred ground truth; teacher output is -training data only and never gates a release on its own β€” release-gating -eval labels must be 100% human-verified (`training/eval/human_verified.py`). diff --git a/training/__init__.py b/training/__init__.py deleted file mode 100644 index 1cf73809..00000000 --- a/training/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -"""FreshData development-time training pipeline (Phase 5). - -This package builds, evaluates, and packages FreshData's small local model -artifacts (``fd-col-encoder-v1``, ``fd-intent-v1``, ``calib-v1``). It is a -**developer tool**, not part of the library: - -- it lives outside ``src/freshdata`` and is never shipped in the wheel; -- the runtime never imports it (guarded by tests); -- teacher models are allowed here only for development-time labeling, - paraphrasing, ambiguity adjudication, red-teaming, and rationale templates β€” - never at runtime, never per-cell, never as release-gating ground truth; -- ground truth comes from corruption metadata and human-verified labels. - -Entry points (see ``training/Makefile`` / root ``Makefile``):: - - make training-seed - make training-corrupt - make training-teacher-labels - make training-distill - make training-eval - make training-package-artifacts - make training-dev-artifacts - make training-release-artifacts -""" - -from __future__ import annotations - -__all__ = ["__phase__"] - -__phase__ = 5 diff --git a/training/cache/.gitkeep b/training/cache/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/training/calibration/__init__.py b/training/calibration/__init__.py deleted file mode 100644 index 228ba669..00000000 --- a/training/calibration/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Offline calibration training: features -> isotonic fit -> JSON tables. - -The runtime never trains anything; it only loads the exported ``calib-v1`` -JSON through the Phase-3 model registry. When the table is missing the -runtime stays safe and reports ``calibration_version="uncalibrated"``. -Deterministic protected-column checks are never probabilistic and are not -touched by calibration at all. -""" diff --git a/training/calibration/build_features.py b/training/calibration/build_features.py deleted file mode 100644 index 7350d951..00000000 --- a/training/calibration/build_features.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Extract calibration features from CleanBench runs. - -Each record is one semantic proposal with the deterministic feature set the -architecture specifies, plus its machine-verified outcome (``correct``) -derived from the fixture's ground truth β€” never from teacher output. - -CLI:: - - python -m training.calibration.build_features [--seeds 5] -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -from ..common import BUILD_DIR, REPO_ROOT, write_jsonl - -OUT_DIR = BUILD_DIR / "calibration" - -_OUTCOME_STATUSES = frozenset({"automatic", "suggested", "approved", "accepted"}) - - -def _cleanbench(): - bench_dir = str(REPO_ROOT / "benchmarks") - if bench_dir not in sys.path: - sys.path.insert(0, bench_dir) - import cleanbench # noqa: PLC0415 - - return cleanbench - - -def _canon(value: object) -> object: - from cleanbench.metrics import _canon as canon # noqa: PLC0415 - - return canon(value) - - -def action_features(action: Any, corrupted: pd.DataFrame, semantic_mode: str) -> dict[str, Any]: - metadata = getattr(action, "metadata", {}) or {} - evidence = metadata.get("evidence", []) or [] - kinds = {e.get("kind") for e in evidence if isinstance(e, dict)} - margin = None - semantic_type_confidence = None - for e in evidence: - detail = str(e.get("detail", "")) if isinstance(e, dict) else "" - if e.get("kind") == "embedding" and "margin=" in detail: - try: - margin = float(detail.split("margin=")[1].split()[0]) - except (IndexError, ValueError): - margin = None - if e.get("kind") == "semantic_type" and "confidence=" in detail: - try: - semantic_type_confidence = float(detail.split("confidence=")[1].split()[0]) - except (IndexError, ValueError): - semantic_type_confidence = None - column = getattr(action, "column", None) - column_signature = metadata.get("column_signature", {}) or {} - calibration = metadata.get("calibration", {}) or {} - raw_score = calibration.get("raw", getattr(action, "confidence", 0.0)) - series = corrupted[column] if column in corrupted.columns else pd.Series(dtype=object) - return { - "raw_score": float(raw_score), - "backend": str(metadata.get("backend", "deterministic")), - "issue_type": str(metadata.get("issue_type", "unknown")), - "risk": str(getattr(action, "risk", metadata.get("risk", "unknown"))), - "role_confidence": 1.0 if column_signature.get("role") else 0.0, - "semantic_type_confidence": semantic_type_confidence, - "distinct_support": int(series.nunique()) if len(series) else None, - "coverage": None, - "memory_support_count": sum( - 1 for e in evidence if isinstance(e, dict) and e.get("kind") == "memory_replay"), - "profile_support_count": sum( - 1 for e in evidence - if isinstance(e, dict) and str(e.get("kind", "")).startswith("profile")), - "learned_precision": metadata.get("learned_precision"), - "policy_rule_present": bool(column_signature.get("policy_rule")), - "allowed_values_present": "reference" in str(metadata.get("expert", "")) - or metadata.get("issue_type") == "reference_value", - "margin_to_second_candidate": margin, - "semantic_mode": semantic_mode, - "evidence_kinds": sorted(k for k in kinds if k), - } - - -def extract_from_run( - report: Any, truth: pd.DataFrame, corrupted: pd.DataFrame, *, semantic_mode: str = "auto" -) -> list[dict[str, Any]]: - """(features, correct) records for every scoreable semantic action.""" - records: list[dict[str, Any]] = [] - for action in getattr(report, "actions", []) or []: - if getattr(action, "step", None) != "semantic": - continue - if str(getattr(action, "status", "")) not in _OUTCOME_STATUSES: - continue - metadata = getattr(action, "metadata", {}) or {} - raw = metadata.get("raw_value") - proposed = metadata.get("proposed_value") - column = getattr(action, "column", None) - if proposed is None or column is None or column not in corrupted.columns: - continue - raw_canon = _canon(raw) - matches = [pos for pos, value in enumerate(corrupted[column].tolist()) - if _canon(value) == raw_canon] - if not matches: - continue - truth_values = truth[column].tolist() - proposed_canon = _canon(proposed) - correct = all(_canon(truth_values[pos]) == proposed_canon for pos in matches) - features = action_features(action, corrupted, semantic_mode) - features["coverage"] = round(len(matches) / max(1, len(corrupted)), 6) - features["correct"] = bool(correct) - features["confidence_reported"] = float(getattr(action, "confidence", 0.0)) - records.append(features) - return records - - -def build_features(*, seeds: int = 5, out_dir: Path | str = OUT_DIR) -> list[dict[str, Any]]: - import freshdata as fd # noqa: PLC0415 - - cleanbench = _cleanbench() - from freshdata.models import runtime as model_runtime # noqa: PLC0415 - - records: list[dict[str, Any]] = [] - for seed in range(seeds): - for maker in (cleanbench.make_t2_semantic_fixture, cleanbench.make_t3_context_fixture): - truth, corrupted, kwargs = maker(seed=seed * 10) - _, report = fd.clean(corrupted, return_report=True, **kwargs) - records.extend(extract_from_run(report, truth, corrupted)) - # Embedding-backend runs (stub encoder): the interesting curves. - truth, corrupted, kwargs = cleanbench.make_phase3_embedding_fixture(seed=seed * 10 + 5) - model_runtime.set_encoder_factory(lambda _mid: cleanbench.BigramStubEncoder()) - try: - _, report = fd.clean(corrupted, return_report=True, **kwargs) - finally: - model_runtime.set_encoder_factory(None) - records.extend(extract_from_run(report, truth, corrupted)) - write_jsonl(Path(out_dir) / "features.jsonl", records) - return records - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.build_features") - parser.add_argument("--seeds", type=int, default=5) - parser.add_argument("--out", default=str(OUT_DIR)) - args = parser.parse_args(argv) - records = build_features(seeds=args.seeds, out_dir=args.out) - backends = sorted({r["backend"] for r in records}) - print(f"calibration features: {len(records)} records, backends={backends}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/calibration/eval_calibration.py b/training/calibration/eval_calibration.py deleted file mode 100644 index bf2e92bc..00000000 --- a/training/calibration/eval_calibration.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Evaluate the exported calibration table against the release gates. - -Gates:: - - ECE <= 0.03 - precision at confidence >= 0.95 >= 0.99 - protected-column violation rate == 0 (checked by the CleanBench run) - false modification rate <= 0.1% (checked by the CleanBench run) - -The first two are computed here on a held-out feature split (never the rows -used for fitting); the last two are structural cleaner properties measured -by ``benchmarks.cleanbench`` and re-asserted by the release pipeline. - -CLI:: - - python -m training.calibration.eval_calibration [--check-gates] -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, read_json, read_jsonl, write_json -from ..datasets.splits import hash_split - -OUT_DIR = BUILD_DIR / "calibration" - -GATE_ECE = 0.03 -GATE_PRECISION_AT_95 = 0.99 - - -def apply_table( - table_json: dict[str, Any], records: list[dict[str, Any]] -) -> list[tuple[float, bool]]: - """(calibrated confidence, correct) pairs via the runtime's own loader.""" - from freshdata.semantic.scoring import _IsotonicTable # noqa: PLC0415 - - table = _IsotonicTable.from_json(json.dumps(table_json)) - pairs: list[tuple[float, bool]] = [] - for record in records: - calibrated = table.apply( - str(record["backend"]), str(record["issue_type"]), float(record["raw_score"])) - pairs.append((min(1.0, max(0.0, calibrated)), bool(record["correct"]))) - return pairs - - -def ece(pairs: list[tuple[float, bool]], bins: int = 10) -> float: - if not pairs: - return 0.0 - total = len(pairs) - out = 0.0 - for b in range(bins): - lo, hi = b / bins, (b + 1) / bins - bucket = [(c, ok) for c, ok in pairs if (c > lo or b == 0) and c <= hi] - if not bucket: - continue - accuracy = sum(ok for _, ok in bucket) / len(bucket) - mean_confidence = sum(c for c, _ in bucket) / len(bucket) - out += (len(bucket) / total) * abs(accuracy - mean_confidence) - return out - - -def precision_at(pairs: list[tuple[float, bool]], floor: float = 0.95) -> float: - bucket = [ok for c, ok in pairs if c >= floor] - return sum(bucket) / len(bucket) if bucket else 1.0 - - -def evaluate( - *, - features_path: Path | str = OUT_DIR / "features.jsonl", - table_path: Path | str = OUT_DIR / "calibration.json", - out_dir: Path | str = OUT_DIR, -) -> dict[str, Any]: - records = read_jsonl(features_path) - table_json = read_json(table_path) - # Held-out evaluation: the same split the fitting stage did NOT see. - _, holdout = hash_split( - records, - key_fn=lambda r: f"{r['backend']}|{r['issue_type']}|{r['raw_score']}|{r.get('coverage')}", - eval_fraction=0.35, - salt="calib-eval", - ) - rows = holdout if len(holdout) >= 20 else records - pairs = apply_table(table_json, rows) - metrics = { - "version": table_json.get("version", "unknown"), - "n_eval": len(pairs), - "held_out": len(holdout) >= 20, - "ece": round(ece(pairs), 4), - "precision_at_0.95": round(precision_at(pairs, 0.95), 4), - "gates": {"ece_max": GATE_ECE, "precision_at_95_min": GATE_PRECISION_AT_95}, - } - metrics["gates"]["failures"] = check_gates(metrics) - write_json(Path(out_dir) / "calib-v1.metrics.json", metrics) - return metrics - - -def check_gates(metrics: dict[str, Any]) -> list[str]: - failures = [] - if metrics["ece"] > GATE_ECE: - failures.append(f"ECE {metrics['ece']} > {GATE_ECE}") - if metrics["precision_at_0.95"] < GATE_PRECISION_AT_95: - failures.append(f"P@0.95 {metrics['precision_at_0.95']} < {GATE_PRECISION_AT_95}") - return failures - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.eval_calibration") - parser.add_argument("--features", default=str(OUT_DIR / "features.jsonl")) - parser.add_argument("--table", default=str(OUT_DIR / "calibration.json")) - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - metrics = evaluate(features_path=args.features, table_path=args.table, out_dir=args.out) - print(f"calibration eval: ECE={metrics['ece']} P@0.95={metrics['precision_at_0.95']} " - f"n={metrics['n_eval']}") - failures = metrics["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/calibration/export_tables.py b/training/calibration/export_tables.py deleted file mode 100644 index ad3e43f5..00000000 --- a/training/calibration/export_tables.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Export fitted curves as the runtime-loadable ``calib-v1`` artifact. - -The primary file is ``calibration.json`` β€” exactly the filename the Phase-3 -model registry expects for ``calib-v1`` β€” validated by round-tripping it -through the runtime's own ``_IsotonicTable.from_json``. A ``calib.json`` -alias is written alongside for the packaged artifact layout. - -CLI:: - - python -m training.calibration.export_tables [--version calib-v1] -""" - -from __future__ import annotations - -import argparse -import json -import shutil -from pathlib import Path - -from ..common import BUILD_DIR, read_json, utc_now_iso, write_json - -OUT_DIR = BUILD_DIR / "calibration" - - -def export( - curves_path: Path | str = OUT_DIR / "curves.json", - *, - version: str = "calib-v1", - out_dir: Path | str = OUT_DIR, -) -> Path: - curves = read_json(curves_path) - table = { - "version": version, - "comment": ( - "Trained offline by training/calibration (Phase 5). Deterministic " - "and memory backends are intentionally absent (identity mapping) " - "unless explicitly fitted; embedding curves are capped below 1.0." - ), - "tables": curves["tables"], - } - # Round-trip through the runtime loader: an artifact the runtime cannot - # parse must never be exported. - from freshdata.semantic.scoring import _IsotonicTable # noqa: PLC0415 - - _IsotonicTable.from_json(json.dumps(table)) - - out = Path(out_dir) - primary = write_json(out / "calibration.json", table) - shutil.copy(primary, out / "calib.json") - write_json(out / "export_meta.json", { - "version": version, - "exported_at": utc_now_iso(), - "n_backends": len(curves["tables"]), - "n_records": curves.get("n_records"), - }) - return primary - - -def metrics_path(out_dir: Path | str = OUT_DIR) -> Path: - return Path(out_dir) / "calib-v1.metrics.json" - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.export_tables") - parser.add_argument("--curves", default=str(OUT_DIR / "curves.json")) - parser.add_argument("--version", default="calib-v1") - parser.add_argument("--out", default=str(OUT_DIR)) - args = parser.parse_args(argv) - path = export(args.curves, version=args.version, out_dir=args.out) - print(f"calibration table exported: {path}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/calibration/fit_isotonic.py b/training/calibration/fit_isotonic.py deleted file mode 100644 index 733eadfa..00000000 --- a/training/calibration/fit_isotonic.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Pool-adjacent-violators isotonic regression per (backend, issue family). - -Pure numpy, deterministic. Curves are clamped monotone non-decreasing and -reduced to a compact set of knots compatible with the runtime's -``_IsotonicTable`` JSON format. Deterministic-backend proposals keep an -identity mapping by default (matching the packaged default's contract that -calibration never changes a default install's decisions) unless -``--fit-deterministic`` is passed. - -CLI:: - - python -m training.calibration.fit_isotonic -""" - -from __future__ import annotations - -import argparse -from pathlib import Path -from typing import Any - -import numpy as np - -from ..common import BUILD_DIR, read_jsonl, write_json - -OUT_DIR = BUILD_DIR / "calibration" -#: Embedding scores may never calibrate to certainty. -EMBEDDING_CEILING = 0.975 -MIN_SAMPLES_PER_CURVE = 8 - - -def pav(scores: np.ndarray, outcomes: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Pool-adjacent-violators: monotone fit of P(correct | score).""" - order = np.argsort(scores, kind="stable") - x = scores[order].astype(np.float64) - y = outcomes[order].astype(np.float64) - # Blocks: (mean, weight); merge while decreasing. - means: list[float] = [] - weights: list[float] = [] - xs: list[float] = [] - for xi, yi in zip(x, y): - means.append(yi) - weights.append(1.0) - xs.append(xi) - while len(means) > 1 and means[-2] > means[-1]: - total = weights[-2] + weights[-1] - merged = (means[-2] * weights[-2] + means[-1] * weights[-1]) / total - means[-2:] = [merged] - weights[-2:] = [total] - xs[-2:] = [xs[-1]] - # xs keeps the right edge of each block - return np.array(xs), np.array(means) - - -def fit_curve( - scores: list[float], outcomes: list[bool], *, ceiling: float = 1.0 -) -> dict[str, list[float]] | None: - """A compact monotone curve ``{"x": [...], "y": [...]}`` or None.""" - if len(scores) < MIN_SAMPLES_PER_CURVE: - return None - knot_x, knot_y = pav(np.array(scores), np.array(outcomes, dtype=float)) - knot_y = np.clip(knot_y, 0.0, ceiling) - xs = [0.0, *[round(float(v), 4) for v in knot_x]] - ys = [0.0, *[round(float(v), 4) for v in knot_y]] - # De-duplicate x while keeping monotonicity. - dedup_x: list[float] = [] - dedup_y: list[float] = [] - for xv, yv in zip(xs, ys): - if dedup_x and xv <= dedup_x[-1]: - dedup_y[-1] = max(dedup_y[-1], yv) - continue - dedup_x.append(xv) - dedup_y.append(max(yv, dedup_y[-1] if dedup_y else 0.0)) - if dedup_x[-1] < 1.0: - dedup_x.append(1.0) - dedup_y.append(dedup_y[-1]) - if len(dedup_x) < 2: - return None - return {"x": dedup_x, "y": dedup_y} - - -def fit_tables( - records: list[dict[str, Any]], *, fit_deterministic: bool = False -) -> dict[str, dict[str, dict[str, list[float]]]]: - """Curves per (backend, issue_type) plus a per-backend ``"*"`` fallback.""" - grouped: dict[tuple[str, str], tuple[list[float], list[bool]]] = {} - for record in records: - backend = str(record["backend"]) - if backend == "deterministic" and not fit_deterministic: - continue # identity by design; never change default decisions - key = (backend, str(record["issue_type"])) - scores, outcomes = grouped.setdefault(key, ([], [])) - scores.append(float(record["raw_score"])) - outcomes.append(bool(record["correct"])) - tables: dict[str, dict[str, dict[str, list[float]]]] = {} - for (backend, issue_type), (scores, outcomes) in sorted(grouped.items()): - ceiling = EMBEDDING_CEILING if backend == "embedding" else 1.0 - curve = fit_curve(scores, outcomes, ceiling=ceiling) - if curve is not None: - tables.setdefault(backend, {})[issue_type] = curve - # Per-backend fallback from all of that backend's records. - for backend in list(tables): - scores = [float(r["raw_score"]) for r in records if r["backend"] == backend] - outcomes = [bool(r["correct"]) for r in records if r["backend"] == backend] - ceiling = EMBEDDING_CEILING if backend == "embedding" else 1.0 - fallback = fit_curve(scores, outcomes, ceiling=ceiling) - if fallback is not None: - tables[backend]["*"] = fallback - return tables - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.fit_isotonic") - parser.add_argument("--features", default=str(OUT_DIR / "features.jsonl")) - parser.add_argument("--out", default=str(OUT_DIR / "curves.json")) - parser.add_argument("--fit-deterministic", action="store_true") - args = parser.parse_args(argv) - records = read_jsonl(args.features) - tables = fit_tables(records, fit_deterministic=args.fit_deterministic) - write_json(Path(args.out), {"tables": tables, "n_records": len(records)}) - n_curves = sum(len(v) for v in tables.values()) - print(f"isotonic fit: {n_curves} curves from {len(records)} records") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/common.py b/training/common.py deleted file mode 100644 index 459078f7..00000000 --- a/training/common.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Shared plumbing for the training pipeline: paths, hashing, stable JSON. - -Everything downstream (datasets, teacher cache, artifact manifests) needs the -same three primitives β€” a stable canonical JSON form, sha256 helpers, and the -build/output directory layout β€” so they live here once. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import subprocess -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -#: Repository root (training/ is a top-level package next to src/). -REPO_ROOT = Path(__file__).resolve().parent.parent -TRAINING_ROOT = REPO_ROOT / "training" -#: Intermediate outputs (never committed, never packaged). -BUILD_DIR = TRAINING_ROOT / "build" -#: Teacher prompt/response cache (audit trail; content-addressed). -CACHE_DIR = TRAINING_ROOT / "cache" -#: Final packaged artifacts (outside the wheel by construction). -ARTIFACTS_DIR = REPO_ROOT / "dist" / "artifacts" - -#: Hard ceiling for any single packaged model artifact directory. -MAX_ARTIFACT_BYTES = 60_000_000 - - -def stable_json(data: Any) -> str: - """Canonical JSON: sorted keys, no whitespace drift, utf-8 safe.""" - return json.dumps(data, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str) - - -def sha256_text(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - - -def sha256_json(data: Any) -> str: - return sha256_text(stable_json(data)) - - -def sha256_file(path: Path | str) -> str: - digest = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - -def utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat() - - -def git_commit() -> str: - """Current commit hash, or "unknown" outside a git checkout (e.g. sdist).""" - try: - out = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=10, check=False, - ) - except OSError: # pragma: no cover - git missing entirely - return "unknown" - return out.stdout.strip() if out.returncode == 0 and out.stdout.strip() else "unknown" - - -def freshdata_version() -> str: - import freshdata # noqa: PLC0415 - lazy so `training` imports stay cheap - - return str(freshdata.__version__) - - -def write_json(path: Path | str, data: Any) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(data, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8") - return path - - -def read_json(path: Path | str) -> Any: - return json.loads(Path(path).read_text(encoding="utf-8")) - - -def write_jsonl(path: Path | str, rows: list[dict[str, Any]]) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as fh: - for row in rows: - fh.write(stable_json(row) + "\n") - return path - - -def read_jsonl(path: Path | str) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - with Path(path).open("r", encoding="utf-8") as fh: - for raw_line in fh: - stripped = raw_line.strip() - if stripped: - rows.append(json.loads(stripped)) - return rows - - -def env_flag(name: str) -> bool: - return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} diff --git a/training/corruptors/__init__.py b/training/corruptors/__init__.py deleted file mode 100644 index 5b01879b..00000000 --- a/training/corruptors/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Parameterized corruption engine mirroring FreshData's repair taxonomy. - -Ground truth for training comes from *corruption metadata*: every corruptor -records exactly what it changed, what the clean value was, and whether the -change is safely repairable β€” teacher models never invent messy/clean pairs. -""" - -from .base import CorruptionLabel, Corruptor, apply_corruptor, compose -from .registry import CORRUPTOR_REGISTRY, FAMILIES, get_corruptor - -__all__ = [ - "CORRUPTOR_REGISTRY", - "FAMILIES", - "CorruptionLabel", - "Corruptor", - "apply_corruptor", - "compose", - "get_corruptor", -] diff --git a/training/corruptors/base.py b/training/corruptors/base.py deleted file mode 100644 index 4b9ad181..00000000 --- a/training/corruptors/base.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Corruptor core: labels, the corruptor container, application, composition. - -Rules enforced here so individual corruptors cannot get them wrong: - -- deterministic under ``seed`` (all randomness flows from one ``Random``); -- the input frame is never mutated; -- every mutated cell/row/header emits a machine-verifiable - :class:`CorruptionLabel`; -- ``ambiguous`` or ``protected`` corruptions are never labeled auto-apply; -- composition never corrupts the same cell twice, so each label's - ``clean_value`` stays the true pre-corruption value. -""" - -from __future__ import annotations - -import random -from collections.abc import Iterable -from dataclasses import dataclass, field -from typing import Any, Callable, Literal - -import pandas as pd - -Kind = Literal["cell", "row", "header", "context"] - -#: Transform families mirror FreshData's repair taxonomy. -TRANSFORM_FAMILIES = ( - "representation", - "semantic_value", - "reference_value", - "date_time", - "context_schema", - "row_structure", - "trap", -) - - -@dataclass(frozen=True) -class CorruptionLabel: - """Machine-verifiable ground truth for one injected corruption.""" - - raw_value: Any - clean_value: Any - column: str | None - transform_family: str - params: dict[str, Any] = field(default_factory=dict) - should_repair: bool = True - should_auto_apply: bool = True - risk: str = "low" - protected: bool = False - ambiguous: bool = False - corruptor: str = "" - row: int | None = None - - def to_dict(self) -> dict[str, Any]: - return { - "raw_value": self.raw_value, - "clean_value": self.clean_value, - "column": self.column, - "transform_family": self.transform_family, - "params": dict(self.params), - "should_repair": self.should_repair, - "should_auto_apply": self.should_auto_apply, - "risk": self.risk, - "protected": self.protected, - "ambiguous": self.ambiguous, - "corruptor": self.corruptor, - "row": self.row, - } - - -#: A cell transform: (clean_value, rng, params) -> corrupted value, or None -#: when the corruptor does not apply to this value. -CellFn = Callable[[str, random.Random, dict[str, Any]], Any] -#: A frame transform for row/header/context corruptors: -#: (df, rng, params) -> (corrupted_df, labels). -FrameFn = Callable[ - [pd.DataFrame, random.Random, dict[str, Any]], - "tuple[pd.DataFrame, list[CorruptionLabel]]", -] - - -@dataclass(frozen=True) -class Corruptor: - """One named, parameterized corruption with fixed label semantics.""" - - name: str - family: str - kind: Kind = "cell" - fn: CellFn | None = None - frame_fn: FrameFn | None = None - risk: str = "low" - should_repair: bool = True - should_auto_apply: bool = True - ambiguous: bool = False - protected: bool = False - params: dict[str, Any] = field(default_factory=dict) - - def __post_init__(self) -> None: - if self.family not in TRANSFORM_FAMILIES: - raise ValueError(f"{self.name}: unknown transform family {self.family!r}") - if self.ambiguous and self.should_auto_apply: - raise ValueError(f"{self.name}: ambiguous corruptors cannot be auto-apply") - if self.protected and self.should_auto_apply: - raise ValueError(f"{self.name}: protected traps cannot be auto-apply") - if self.kind == "cell" and self.fn is None: - raise ValueError(f"{self.name}: cell corruptor needs fn") - if self.kind != "cell" and self.frame_fn is None: - raise ValueError(f"{self.name}: {self.kind} corruptor needs frame_fn") - - def label_for( - self, *, raw: Any, clean: Any, column: str | None, row: int | None, - params: dict[str, Any], - ) -> CorruptionLabel: - return CorruptionLabel( - raw_value=raw, - clean_value=clean, - column=column, - transform_family=self.family, - params=params, - should_repair=self.should_repair, - should_auto_apply=self.should_auto_apply, - risk=self.risk, - protected=self.protected, - ambiguous=self.ambiguous, - corruptor=self.name, - row=row, - ) - - -def _seeded(name: str, seed: int) -> random.Random: - return random.Random(f"freshdata-corruptor:{name}:{seed}") - - -def apply_corruptor( - df: pd.DataFrame, - corruptor: Corruptor, - columns: Iterable[str] | None = None, - *, - seed: int = 0, - share: float = 0.5, - params: dict[str, Any] | None = None, - skip_cells: set[tuple[int, str]] | None = None, -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - """Apply one corruptor and return ``(corrupted_frame, labels)``. - - ``skip_cells`` (positions already corrupted by an earlier step) preserves - ground truth under composition; :func:`compose` threads it automatically. - """ - rng = _seeded(corruptor.name, seed) - merged = {**corruptor.params, **(params or {})} - if corruptor.kind != "cell": - assert corruptor.frame_fn is not None - out, labels = corruptor.frame_fn(df.copy(deep=True), rng, merged) - return out, labels - - assert corruptor.fn is not None - out = df.copy(deep=True) - if columns is not None: - targets = [c for c in columns if c in df.columns] - else: - # Default to string-shaped columns only: a bool/numeric column fed - # through a text transform (e.g. casing) would otherwise trip a - # dtype-widening cell write for no semantic reason. - targets = [c for c in df.columns if df[c].map(lambda v: isinstance(v, str)).any()] - labels: list[CorruptionLabel] = [] - skip = skip_cells or set() - for col in targets: - for pos, idx in enumerate(out.index): - if (pos, str(col)) in skip: - continue - clean = out.at[idx, col] - if not isinstance(clean, str): - continue - if pd.isna(clean) if not isinstance(clean, (list, dict)) else False: - continue - if rng.random() >= share: - continue - raw = corruptor.fn(str(clean), rng, merged) - if raw is None or str(raw) == str(clean): - continue - out.at[idx, col] = raw - # Sentinel-style corruptions destroy the original value; the only - # valid repair target is "missing", never the unrecoverable truth. - clean_target = None if merged.get("target") == "missing" else clean - labels.append(corruptor.label_for( - raw=raw, clean=clean_target, column=str(col), row=pos, params=merged, - )) - return out, labels - - -def compose( - df: pd.DataFrame, - steps: list[tuple[Corruptor, Iterable[str] | None, float]], - *, - seed: int = 0, -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - """Apply corruptors in sequence without double-corrupting any cell.""" - out = df - all_labels: list[CorruptionLabel] = [] - taken: set[tuple[int, str]] = set() - for i, (corruptor, columns, share) in enumerate(steps): - out, labels = apply_corruptor( - out, corruptor, columns, seed=seed + i * 1009, share=share, skip_cells=taken, - ) - for label in labels: - if label.row is not None and label.column is not None: - taken.add((label.row, label.column)) - all_labels.extend(labels) - return out, all_labels diff --git a/training/corruptors/context.py b/training/corruptors/context.py deleted file mode 100644 index 8f80da80..00000000 --- a/training/corruptors/context.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Context / column corruptors: schema mangling, paraphrase hooks, traps. - -Header corruptors emit labels with ``column`` = the *original* name and -``raw_value`` = the mangled alias (the "repair" is alias resolution). The two -trap corruptors label **any** mutation as a failure: a compliant cleaner must -leave those cells alone, so ``should_repair=False`` and auto-apply is banned. -""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -from .base import CorruptionLabel, Corruptor - -#: Realistic ecommerce header aliases (clean name -> plausible wild forms). -ECOMMERCE_ALIASES: dict[str, tuple[str, ...]] = { - "cust_id": ("CustomerID", "customer id", "CUST-ID", "cid"), - "email": ("e-mail address", "customer_email", "Email ID", "mail"), - "email_addr": ("e-mail", "EmailAddress", "customer email id"), - "phone": ("mobile no.", "contact_number", "Phone#", "mob"), - "mobile": ("phone_number", "Mobile No", "contact"), - "status": ("cust_status", "Account Status", "state_flag"), - "monthly_revenue": ("rev_m", "Monthly Rev.", "revenue(month)"), - "order_date": ("OrderedOn", "date_of_order", "ord_dt"), - "postal_code": ("PIN", "pincode", "zip"), - "quantity": ("qty", "Qty.", "no_of_items"), -} - -#: Deterministic Indian-English / Hinglish paraphrase hooks for context -#: sentences. Teacher models may *extend* this set (cached + reviewed); the -#: hooks themselves guarantee an offline floor. -HINGLISH_PARAPHRASES: dict[str, tuple[str, ...]] = { - "PROTECT": ("{col} kabhi mat badalna.", "{col} ko haath mat lagao.", - "{col} bilkul change nahi karna."), - "UNIQUE": ("{col} har row me alag hona chahiye.", "{col} repeat nahi hona chahiye."), - "ALLOWED_VALUES": ("{col} sirf {values} ho sakta hai.",), - "IMPUTE_IF": ("{col} khali ho to sirf {threshold}% pakka hone par bharna.",), - "LOCALE_FORMAT": ("{col} Indian phone numbers hain.",), -} - - -def _snake_camel(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if "_" in v: - parts = v.split("_") - return rng.choice(( - "".join(p.title() for p in parts), # CamelCase - parts[0] + "".join(p.title() for p in parts[1:]), # camelCase - " ".join(parts).title(), # Title Case words - "-".join(parts), # kebab-case - )) - if v != v.lower(): - return v.lower() - return v.upper() if len(v) <= 4 else v.title() - - -def _rename_headers( - df: pd.DataFrame, rng: random.Random, params: dict[str, Any], - alias_of, - corruptor_name: str, -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - share = float(params.get("share", 0.5)) - mapping: dict[str, str] = {} - labels: list[CorruptionLabel] = [] - for col in df.columns: - if rng.random() >= share: - continue - alias = alias_of(str(col), rng) - if not alias or alias == col or alias in df.columns: - continue - mapping[str(col)] = alias - labels.append(CorruptionLabel( - raw_value=alias, clean_value=str(col), column=str(col), - transform_family="context_schema", params={"alias": alias}, - should_repair=True, should_auto_apply=True, risk="low", - corruptor=corruptor_name, row=None, - )) - return df.rename(columns=mapping), labels - - -def _alias_rename_frame(df, rng, params): - generic = { - "email": ("contact_email", "e-mail"), "phone": ("tel", "phone no"), - "name": ("full name", "customer_name"), "date": ("dt", "when"), - } - - def alias_of(col: str, r: random.Random) -> str | None: - for stem, options in generic.items(): - if stem in col.lower(): - return r.choice(options) - return f"{col}_col" - - return _rename_headers(df, rng, params, alias_of, "column_alias_rename") - - -def _header_format_frame(df, rng, params): - return _rename_headers( - df, rng, params, lambda c, r: _snake_camel(c, r, params), "header_format_mangle", - ) - - -def _ecommerce_alias_frame(df, rng, params): - def alias_of(col: str, r: random.Random) -> str | None: - options = ECOMMERCE_ALIASES.get(col.lower()) - return r.choice(options) if options else None - - return _rename_headers(df, rng, params, alias_of, "ecommerce_alias_rename") - - -def _hinglish_paraphrase_frame(df, rng, params): - """Context-sentence hook: emits paraphrase labels, leaves the frame alone. - - Params: ``intent``, ``column``, and optional slot values. The label's - ``clean_value`` is the canonical English sentence the paraphrase must - compile to; consumers feed these into the intent-head training set. - """ - intent = str(params.get("intent", "PROTECT")) - column = str(params.get("column", "monthly_revenue")) - canonical = str(params.get("canonical", f"Never modify {column} values.")) - templates = HINGLISH_PARAPHRASES.get(intent, ()) - labels = [] - for template in templates: - sentence = template.format( - col=column, - values=", ".join(params.get("values", ("active", "inactive", "pending"))), - threshold=int(float(params.get("threshold", 0.95)) * 100), - ) - labels.append(CorruptionLabel( - raw_value=sentence, clean_value=canonical, column=column, - transform_family="context_schema", - params={"intent": intent, "language": "hinglish"}, - should_repair=True, should_auto_apply=True, risk="low", - corruptor="hinglish_context_paraphrase", row=None, - )) - return df, labels - - -def _duplicated_id_rows_frame(df, rng, params): - """Rows that share an ID but differ elsewhere β€” dedup-key bait.""" - id_column = str(params.get("id_column", df.columns[0])) - n = int(params.get("n", 2)) - if df.empty or id_column not in df.columns: - return df, [] - picks = [rng.randrange(len(df)) for _ in range(n)] - extra = df.iloc[picks].copy() - mutable = [ - c for c in df.columns - if c != id_column and df[c].map(lambda v: isinstance(v, str)).all() - ] - for offset, (_, row) in enumerate(extra.iterrows()): - if mutable: - col = rng.choice(mutable) - extra.iloc[offset, extra.columns.get_loc(col)] = f"{row[col]} (dup)" - out = pd.concat([df, extra], ignore_index=True) - labels = [ - CorruptionLabel( - raw_value=str(df.iloc[p][id_column]), clean_value=None, column=id_column, - transform_family="row_structure", params={"source_row": p, "id_column": id_column}, - should_repair=True, should_auto_apply=False, risk="medium", ambiguous=True, - corruptor="duplicated_id_rows", row=len(df) + i, - ) - for i, p in enumerate(picks) - ] - return out, labels - - -def _mutate_cells_trap(df, rng, params, *, corruptor_name: str, protected: bool): - column = str(params.get("column", "")) - if column not in df.columns or df.empty: - return df, [] - n = min(int(params.get("n", 2)), len(df)) - positions = rng.sample(range(len(df)), n) - labels = [] - for pos in positions: - clean = df.iloc[pos][column] - raw = rng.choice((f" {clean} ", f"{clean},00", str(clean).upper(), f"{clean}?")) - if str(raw) == str(clean): - raw = f" {clean} " - df.iloc[pos, df.columns.get_loc(column)] = raw - labels.append(CorruptionLabel( - raw_value=raw, clean_value=clean, column=column, - transform_family="trap", params={"trap": corruptor_name}, - should_repair=False, should_auto_apply=False, risk="high", - protected=protected, ambiguous=False, - corruptor=corruptor_name, row=pos, - )) - return df, labels - - -def _protected_trap_frame(df, rng, params): - return _mutate_cells_trap( - df, rng, params, corruptor_name="protected_column_trap", protected=True) - - -def _target_trap_frame(df, rng, params): - return _mutate_cells_trap( - df, rng, params, corruptor_name="target_column_trap", protected=False) - - -column_alias_rename = Corruptor( - name="column_alias_rename", family="context_schema", kind="header", - frame_fn=_alias_rename_frame) -header_format_mangle = Corruptor( - name="header_format_mangle", family="context_schema", kind="header", - frame_fn=_header_format_frame) -ecommerce_alias_rename = Corruptor( - name="ecommerce_alias_rename", family="context_schema", kind="header", - frame_fn=_ecommerce_alias_frame) -hinglish_context_paraphrase = Corruptor( - name="hinglish_context_paraphrase", family="context_schema", kind="context", - frame_fn=_hinglish_paraphrase_frame) -duplicated_id_rows = Corruptor( - name="duplicated_id_rows", family="row_structure", kind="row", - frame_fn=_duplicated_id_rows_frame, risk="medium", - should_auto_apply=False, ambiguous=True) -protected_column_trap = Corruptor( - name="protected_column_trap", family="trap", kind="row", - frame_fn=_protected_trap_frame, risk="high", - should_repair=False, should_auto_apply=False, protected=True) -target_column_trap = Corruptor( - name="target_column_trap", family="trap", kind="row", - frame_fn=_target_trap_frame, risk="high", - should_repair=False, should_auto_apply=False) - -CONTEXT_CORRUPTORS = ( - column_alias_rename, - header_format_mangle, - ecommerce_alias_rename, - hinglish_context_paraphrase, - duplicated_id_rows, - protected_column_trap, - target_column_trap, -) diff --git a/training/corruptors/learning.py b/training/corruptors/learning.py deleted file mode 100644 index 1bcb70e6..00000000 --- a/training/corruptors/learning.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Paired-data builders for profile learning (T4) and vendor-map corruption.""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -from .base import CorruptionLabel, Corruptor, compose -from .representation import casing_change, sentinel_injection, whitespace_insertion -from .semantic_values import allowed_value_case, phone_in_zero_prefix - - -def _vendor_code_swap(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Replace a clean label with a vendor-specific code (learnable literal map).""" - codes = params.get("vendor_map", { - "active": "A1", "inactive": "I0", "pending": "P2", "churned": "X9", - }) - return codes.get(v.strip().lower()) - - -vendor_code_swap = Corruptor( - name="vendor_code_swap", family="reference_value", fn=_vendor_code_swap, - risk="medium", should_auto_apply=False) - - -def make_paired_learning_set( - clean: pd.DataFrame, - *, - seed: int = 0, - value_columns: tuple[str, ...] = ("status",), - phone_columns: tuple[str, ...] = (), - text_columns: tuple[str, ...] = (), -) -> tuple[pd.DataFrame, pd.DataFrame, list[CorruptionLabel]]: - """Build a ``(messy, clean, labels)`` pair for ``fd.learn``. - - The corruptions chosen here are exactly the ones a learning profile can - capture: a stable vendor literal map plus representation dirt. - """ - steps: list[tuple[Corruptor, tuple[str, ...] | None, float]] = [ - (vendor_code_swap, value_columns, 0.9), - (allowed_value_case, value_columns, 0.3), - (whitespace_insertion, text_columns or value_columns, 0.3), - (casing_change, text_columns, 0.3), - (phone_in_zero_prefix, phone_columns, 0.6), - (sentinel_injection, text_columns, 0.05), - ] - steps = [(c, cols, share) for c, cols, share in steps if cols] - messy, labels = compose(clean, steps, seed=seed) - return messy, clean.copy(deep=True), labels diff --git a/training/corruptors/registry.py b/training/corruptors/registry.py deleted file mode 100644 index faba94db..00000000 --- a/training/corruptors/registry.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Registry of every corruptor, keyed by name, with family groupings.""" - -from __future__ import annotations - -from .base import Corruptor -from .context import CONTEXT_CORRUPTORS -from .learning import vendor_code_swap -from .representation import REPRESENTATION_CORRUPTORS -from .semantic_values import SEMANTIC_VALUE_CORRUPTORS - -_ALL: tuple[Corruptor, ...] = ( - *REPRESENTATION_CORRUPTORS, - *SEMANTIC_VALUE_CORRUPTORS, - *CONTEXT_CORRUPTORS, - vendor_code_swap, -) - -CORRUPTOR_REGISTRY: dict[str, Corruptor] = {c.name: c for c in _ALL} - -FAMILIES: dict[str, tuple[str, ...]] = {} -for _c in _ALL: - FAMILIES.setdefault(_c.family, ()) - FAMILIES[_c.family] = (*FAMILIES[_c.family], _c.name) - -if len(CORRUPTOR_REGISTRY) != len(_ALL): # pragma: no cover - definition bug - raise RuntimeError("duplicate corruptor names in registry") - - -def get_corruptor(name: str) -> Corruptor: - try: - return CORRUPTOR_REGISTRY[name] - except KeyError: - known = ", ".join(sorted(CORRUPTOR_REGISTRY)) - raise KeyError(f"unknown corruptor {name!r}; known: {known}") from None diff --git a/training/corruptors/representation.py b/training/corruptors/representation.py deleted file mode 100644 index be9e02e0..00000000 --- a/training/corruptors/representation.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Representation corruptors: formatting dirt with unambiguous repairs.""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -from .base import CorruptionLabel, Corruptor - -_SENTINELS = ("N/A", "null", "-", "NONE", "?", "missing") -_NA_VARIANTS = ("", "NA", "n/a", "None", " ") -_BOOL_SYNONYMS = { - "true": ("Y", "yes", "TRUE", "1", "y"), - "false": ("N", "no", "FALSE", "0", "n"), -} -_UNITS = ("kg", "g", "pcs", "ltr") - - -def _is_number(text: str) -> bool: - try: - float(text.replace(",", "")) - except ValueError: - return False - return True - - -def _whitespace(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice((" {}", "{} ", " {} ", "\t{}", "{}\t")).format(v) - - -def _casing(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not any(ch.isalpha() for ch in v): - return None - choices = [c for c in (v.upper(), v.lower(), v.title(), v.swapcase()) if c != v] - return rng.choice(choices) if choices else None - - -def _sentinel(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice(params.get("sentinels", _SENTINELS)) - - -def _na_variant(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice(_NA_VARIANTS) - - -def _mixed_dtype(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v): - return None - number = float(v.replace(",", "")) - return rng.choice((f"{number:.1f}", f"{number:.3f}", f" {number:g} ", f"'{v}'")) - - -def _currency(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v): - return None - number = float(v.replace(",", "")) - return rng.choice(( - f"β‚Ή{number:,.2f}", f"Rs. {number:.2f}", f"INR {number:,.2f}", f"{number:.2f} INR", - )) - - -def _thousands(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v) or "," in v or float(v) < 1000: - return None - return f"{float(v):,.10g}" - - -def _unit_suffix(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v): - return None - unit = rng.choice(params.get("units", _UNITS)) - return rng.choice((f"{v}{unit}", f"{v} {unit}")) - - -def _bool_synonym(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - synonyms = _BOOL_SYNONYMS.get(v.strip().lower()) - return rng.choice(synonyms) if synonyms else None - - -def _duplicate_rows( - df: pd.DataFrame, rng: random.Random, params: dict[str, Any] -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - n = int(params.get("n_duplicates", 2)) - if df.empty or n <= 0: - return df, [] - picks = [rng.randrange(len(df)) for _ in range(n)] - extra = df.iloc[picks] - out = pd.concat([df, extra], ignore_index=True) - labels = [ - CorruptionLabel( - raw_value=f"duplicate_of_row_{p}", clean_value=None, column=None, - transform_family="row_structure", params={"source_row": p}, - should_repair=True, should_auto_apply=True, risk="low", - corruptor="duplicate_row_injection", row=len(df) + i, - ) - for i, p in enumerate(picks) - ] - return out, labels - - -whitespace_insertion = Corruptor( - name="whitespace_insertion", family="representation", fn=_whitespace) -casing_change = Corruptor( - name="casing_change", family="representation", fn=_casing) -sentinel_injection = Corruptor( - name="sentinel_injection", family="representation", fn=_sentinel, - params={"target": "missing"}) -empty_na_variants = Corruptor( - name="empty_na_variants", family="representation", fn=_na_variant, - params={"target": "missing"}) -mixed_dtype_stringification = Corruptor( - name="mixed_dtype_stringification", family="representation", fn=_mixed_dtype) -currency_formatting = Corruptor( - name="currency_formatting", family="representation", fn=_currency) -thousands_separators = Corruptor( - name="thousands_separators", family="representation", fn=_thousands) -unit_suffix_insertion = Corruptor( - name="unit_suffix_insertion", family="representation", fn=_unit_suffix, - risk="medium", should_auto_apply=False) -boolean_synonym_replacement = Corruptor( - name="boolean_synonym_replacement", family="representation", fn=_bool_synonym) -duplicate_row_injection = Corruptor( - name="duplicate_row_injection", family="row_structure", kind="row", - frame_fn=_duplicate_rows) - -REPRESENTATION_CORRUPTORS = ( - whitespace_insertion, - casing_change, - sentinel_injection, - empty_na_variants, - mixed_dtype_stringification, - currency_formatting, - thousands_separators, - unit_suffix_insertion, - boolean_synonym_replacement, - duplicate_row_injection, -) diff --git a/training/corruptors/scale.py b/training/corruptors/scale.py deleted file mode 100644 index 990be497..00000000 --- a/training/corruptors/scale.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Scale helpers for the T5 track: grow a labeled fixture without new labels. - -Tiling preserves per-cell ground truth exactly (row ``i`` of tile ``k`` maps -back to source row ``i``), so T5 measures throughput and memory on a frame -whose correctness metrics are still fully machine-verifiable. -""" - -from __future__ import annotations - -import random - -import pandas as pd - - -def scale_frame(df: pd.DataFrame, *, target_rows: int, seed: int = 0) -> pd.DataFrame: - """Tile *df* to ``target_rows``, remapping obvious ID columns to stay unique.""" - if df.empty or target_rows <= len(df): - return df.copy(deep=True) - rng = random.Random(f"freshdata-scale:{seed}") - tiles = -(-target_rows // len(df)) # ceil - parts = [] - id_columns = [ - c for c in df.columns if str(c).lower().endswith("_id") or str(c).lower() == "id" - ] - for k in range(tiles): - part = df.copy(deep=True) - for col in id_columns: - part[col] = [f"{v}-{k}" for v in part[col]] - parts.append(part) - out = pd.concat(parts, ignore_index=True).head(target_rows) - # Deterministic row shuffle so batch effects do not line up with tiles. - order = list(range(len(out))) - rng.shuffle(order) - return out.iloc[order].reset_index(drop=True) diff --git a/training/corruptors/semantic_values.py b/training/corruptors/semantic_values.py deleted file mode 100644 index ec9371c5..00000000 --- a/training/corruptors/semantic_values.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Semantic-value corruptors: emails, Indian phones, categories, dates. - -Ambiguous corruptions (two plausible repairs, dangerous near-pairs, dayfirst -puzzles) are always labeled ``should_auto_apply=False`` β€” a cleaner that -auto-applies them fails CleanBench by construction. -""" - -from __future__ import annotations - -import random -from typing import Any - -from .base import Corruptor - -_SPELLED = { - "0": "zero", "1": "one", "2": "two", "3": "three", "4": "four", "5": "five", - "6": "six", "7": "seven", "8": "eight", "9": "nine", "10": "ten", - "20": "twenty", "25": "twenty five", "30": "thirty", "40": "forty", - "50": "fifty", "100": "one hundred", -} -_MONTHS = ("January", "February", "March", "April", "May", "June", "July", - "August", "September", "October", "November", "December") - - -# -- emails -------------------------------------------------------------------- - -def _one_at(v: str) -> bool: - return v.count("@") == 1 and "." in v.partition("@")[2] - - -def _email_at_whitespace(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _one_at(v): - return None - local, _, domain = v.partition("@") - return f"{local}{rng.choice((' @', ' @ ', '@ '))}{domain}" - - -def _email_double_at(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - return v.replace("@", "@@", 1) if _one_at(v) else None - - -def _email_casing(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _one_at(v) or v == v.upper(): - return None - return rng.choice((v.upper(), v.title())) - - -def _email_punct_noise(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _one_at(v): - return None - local, _, domain = v.partition("@") - return rng.choice(( - f"{local}.@{domain}", # trailing dot in local part - f"{local}@{domain},", # trailing comma - f"{local}@{domain}.", # trailing dot - f"{local.replace('.', '..', 1)}@{domain}" if "." in local else f"{local},@{domain}", - )) - - -# -- Indian phones --------------------------------------------------------------- - -def _canonical_in_phone(v: str) -> str | None: - digits = "".join(ch for ch in v if ch.isdigit()) - if v.startswith("+91") and len(digits) == 12 and digits.startswith("91"): - return digits[2:] - return None - - -def _phone_spacing(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - if ten is None: - return None - return rng.choice(( - f"{ten[:5]} {ten[5:]}", f"+91 {ten[:5]} {ten[5:]}", f"{ten[:4]} {ten[4:7]} {ten[7:]}", - )) - - -def _phone_zero_prefix(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - return f"0{ten}" if ten else None - - -def _phone_plus91_format(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - if ten is None: - return None - return rng.choice((f"(+91) {ten}", f"+91-{ten}", f"91{ten}", f"+91 {ten}")) - - -def _phone_hyphenation(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - if ten is None: - return None - return rng.choice(( - f"{ten[:5]}-{ten[5:]}", f"+91-{ten[:5]}-{ten[5:]}", f"{ten[:3]}-{ten[3:6]}-{ten[6:]}", - )) - - -def _phone_unsafe(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Digit loss/gain: the value is wrong and there is no safe repair.""" - ten = _canonical_in_phone(v) - if ten is None: - return None - cut = rng.randrange(10) - return rng.choice(( - ten[:cut] + ten[cut + 1:], # dropped digit - ten[:cut] + str(rng.randrange(10)) + ten[cut:], # inserted digit - )) - - -# -- category / reference values --------------------------------------------------- - -def _allowed_case(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not v.isalpha(): - return None - choices = [c for c in (v.upper(), v.title(), v.swapcase()) if c != v] - return rng.choice(choices) if choices else None - - -def _allowed_whitespace(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice((f" {v}", f"{v} ", f" {v} ", f"{v}\t")) - - -def _allowed_separator(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if len(v) < 4: - return None - cut = rng.randrange(1, len(v) - 1) - return v[:cut] + rng.choice("-_ ") + v[cut:] - - -def _edit_distance_typo(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if len(v) < 4 or not v.isalpha(): - return None - pos = rng.randrange(1, len(v) - 1) - op = rng.choice(("swap", "drop", "double")) - if op == "swap": - return v[:pos] + v[pos + 1] + v[pos] + v[pos + 2:] if pos + 1 < len(v) else None - if op == "drop": - return v[:pos] + v[pos + 1:] - return v[:pos] + v[pos] + v[pos:] - - -def _ambiguous_category(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Values one edit from *two* allowed values (e.g. ``nactive``).""" - pairs = params.get("ambiguous_map", {"active": "nactive", "inactive": "nactive"}) - return pairs.get(v.strip().lower()) - - -def _country_code_ambiguity(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - codes = params.get( - "country_codes", {"india": "IN", "australia": "AU", "austria": "AT", "canada": "CA"}) - return codes.get(v.strip().lower()) - - -def _short_code_ambiguity(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - codes = params.get("short_codes", {"california": "CA", "canada": "CA", "maharashtra": "MH"}) - return codes.get(v.strip().lower()) - - -def _close_category_pair(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Dangerous near-pairs: Austria/Australia, May/month-vs-name, ID codes.""" - pairs = params.get("close_pairs", { - "austria": "Austrlia", "australia": "Austrlia", - "delivered": "deliverd", "returned": "returend", - }) - return pairs.get(v.strip().lower()) - - -def _spelled_number(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - return _SPELLED.get(v.strip()) - - -# -- dates ------------------------------------------------------------------------- - -def _iso_parts(v: str) -> tuple[int, int, int] | None: - parts = v.split("-") - if len(parts) != 3 or not all(p.isdigit() for p in parts): - return None - year, month, day = (int(p) for p in parts) - if not (1 <= month <= 12 and 1 <= day <= 28): - return None - return year, month, day - - -def _date_format_shuffle(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - parsed = _iso_parts(v) - if parsed is None: - return None - year, month, day = parsed - return rng.choice(( - f"{_MONTHS[month - 1]} {day}, {year}", - f"{day} {_MONTHS[month - 1]} {year}", - f"{year}/{month:02d}/{day:02d}", - )) - - -def _date_dayfirst_ambiguity(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - parsed = _iso_parts(v) - if parsed is None: - return None - year, month, day = parsed - if day > 12 or day == month: - return None # unambiguous or identity β€” not a dayfirst puzzle - return rng.choice((f"{month:02d}/{day:02d}/{year}", f"{day:02d}/{month:02d}/{year}")) - - -def _relative_date_phrase(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if _iso_parts(v) is None: - return None - return rng.choice(("last Tuesday", "yesterday", "two weeks ago", "next Monday")) - - -def _month_abbreviation(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - parsed = _iso_parts(v) - if parsed is None: - return None - year, month, day = parsed - abbr = _MONTHS[month - 1][:3] - return rng.choice((f"{day}-{abbr}-{year}", f"{abbr} {day} {year}", f"{day} {abbr}. {year}")) - - -def _invalid_date_phrase(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if _iso_parts(v) is None: - return None - return rng.choice(("2025-13-45", "31/02/2024", "0000-00-00", "the 32nd of Neverember")) - - -# --------------------------------------------------------------------------- # - -spelled_number_replacement = Corruptor( - name="spelled_number_replacement", family="semantic_value", - fn=_spelled_number, risk="medium") -email_at_whitespace = Corruptor( - name="email_at_whitespace", family="semantic_value", fn=_email_at_whitespace) -email_double_at = Corruptor( - name="email_double_at", family="semantic_value", fn=_email_double_at) -email_casing = Corruptor( - name="email_casing", family="semantic_value", fn=_email_casing) -email_punct_noise = Corruptor( - name="email_punct_noise", family="semantic_value", fn=_email_punct_noise, - risk="medium") -phone_in_spacing = Corruptor( - name="phone_in_spacing", family="semantic_value", fn=_phone_spacing) -phone_in_zero_prefix = Corruptor( - name="phone_in_zero_prefix", family="semantic_value", fn=_phone_zero_prefix) -phone_in_plus91_format = Corruptor( - name="phone_in_plus91_format", family="semantic_value", fn=_phone_plus91_format) -phone_hyphenation = Corruptor( - name="phone_hyphenation", family="semantic_value", fn=_phone_hyphenation) -phone_unsafe_mutation = Corruptor( - name="phone_unsafe_mutation", family="semantic_value", fn=_phone_unsafe, - risk="high", should_repair=False, should_auto_apply=False, ambiguous=True) - -allowed_value_case = Corruptor( - name="allowed_value_case", family="reference_value", fn=_allowed_case) -allowed_value_whitespace = Corruptor( - name="allowed_value_whitespace", family="reference_value", fn=_allowed_whitespace) -allowed_value_separator = Corruptor( - name="allowed_value_separator", family="reference_value", fn=_allowed_separator) -edit_distance_typo = Corruptor( - name="edit_distance_typo", family="reference_value", fn=_edit_distance_typo, - risk="medium") -ambiguous_category = Corruptor( - name="ambiguous_category", family="reference_value", fn=_ambiguous_category, - risk="high", should_auto_apply=False, ambiguous=True) -country_code_ambiguity = Corruptor( - name="country_code_ambiguity", family="reference_value", fn=_country_code_ambiguity, - risk="high", should_auto_apply=False, ambiguous=True) -short_code_ambiguity = Corruptor( - name="short_code_ambiguity", family="reference_value", fn=_short_code_ambiguity, - risk="high", should_auto_apply=False, ambiguous=True) -close_category_pair = Corruptor( - name="close_category_pair", family="reference_value", fn=_close_category_pair, - risk="high", should_auto_apply=False, ambiguous=True) - -date_format_shuffle = Corruptor( - name="date_format_shuffle", family="date_time", fn=_date_format_shuffle) -date_dayfirst_ambiguity = Corruptor( - name="date_dayfirst_ambiguity", family="date_time", fn=_date_dayfirst_ambiguity, - risk="high", should_auto_apply=False, ambiguous=True) -relative_date_phrase = Corruptor( - name="relative_date_phrase", family="date_time", fn=_relative_date_phrase, - risk="high", should_repair=False, should_auto_apply=False, ambiguous=True) -month_name_abbreviation = Corruptor( - name="month_name_abbreviation", family="date_time", fn=_month_abbreviation) -invalid_date_phrase = Corruptor( - name="invalid_date_phrase", family="date_time", fn=_invalid_date_phrase, - risk="medium", should_auto_apply=False, params={"target": "missing"}) - -SEMANTIC_VALUE_CORRUPTORS = ( - spelled_number_replacement, - email_at_whitespace, - email_double_at, - email_casing, - email_punct_noise, - phone_in_spacing, - phone_in_zero_prefix, - phone_in_plus91_format, - phone_hyphenation, - phone_unsafe_mutation, - allowed_value_case, - allowed_value_whitespace, - allowed_value_separator, - edit_distance_typo, - ambiguous_category, - country_code_ambiguity, - short_code_ambiguity, - close_category_pair, - date_format_shuffle, - date_dayfirst_ambiguity, - relative_date_phrase, - month_name_abbreviation, - invalid_date_phrase, -) diff --git a/training/datasets/__init__.py b/training/datasets/__init__.py deleted file mode 100644 index f401c17a..00000000 --- a/training/datasets/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dataset builders: seed corpus, corrupted pairs, splits, validators.""" diff --git a/training/datasets/build_corrupted.py b/training/datasets/build_corrupted.py deleted file mode 100644 index 034c3a5e..00000000 --- a/training/datasets/build_corrupted.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Corrupt the seed corpus into labeled training pairs. - -For each seed table a fixed, versioned corruption plan runs under an explicit -seed; the output is ``(messy frame, clean frame, labels.jsonl)`` where labels -are the machine-verifiable ground truth (see ``training/corruptors/base.py``). -Every label file is validated before it is written. - -CLI:: - - python -m training.datasets.build_corrupted [--seed 0] [--dev] -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -from ..common import BUILD_DIR, utc_now_iso, write_json, write_jsonl -from ..corruptors import compose, get_corruptor -from ..corruptors.base import apply_corruptor -from ..seed.synthetic import seed_tables -from .validators import validate_labels - -DEFAULT_OUT = BUILD_DIR / "corrupted" - -#: (corruptor, columns, share) plans per seed table. ``--dev`` keeps the -#: first few steps only so the dev pipeline stays fast. -PLANS: dict[str, list[tuple[str, tuple[str, ...] | None, float]]] = { - "customers": [ - ("whitespace_insertion", ("full_name", "address", "city"), 0.25), - ("casing_change", ("city", "country"), 0.25), - ("sentinel_injection", ("address",), 0.08), - ("empty_na_variants", ("state",), 0.08), - ("email_at_whitespace", ("email",), 0.2), - ("email_double_at", ("email",), 0.15), - ("email_casing", ("email",), 0.15), - ("email_punct_noise", ("email",), 0.1), - ("phone_in_spacing", ("phone",), 0.2), - ("phone_in_zero_prefix", ("phone",), 0.2), - ("phone_in_plus91_format", ("phone",), 0.2), - ("phone_hyphenation", ("phone",), 0.15), - ("phone_unsafe_mutation", ("phone",), 0.05), - ("allowed_value_case", ("status",), 0.3), - ("allowed_value_whitespace", ("status",), 0.15), - ("allowed_value_separator", ("status",), 0.1), - ("edit_distance_typo", ("status",), 0.1), - ("ambiguous_category", ("status",), 0.05), - ("country_code_ambiguity", ("country",), 0.1), - ("close_category_pair", ("country",), 0.08), - ("date_format_shuffle", ("signup_date",), 0.25), - ("date_dayfirst_ambiguity", ("signup_date",), 0.1), - ("month_name_abbreviation", ("signup_date",), 0.15), - ("invalid_date_phrase", ("signup_date",), 0.04), - ("relative_date_phrase", ("signup_date",), 0.04), - ("boolean_synonym_replacement", ("newsletter_opt_in",), 0.4), - ], - "transactions": [ - ("currency_formatting", ("monthly_revenue",), 0.3), - ("thousands_separators", ("monthly_revenue",), 0.2), - ("mixed_dtype_stringification", ("monthly_revenue",), 0.15), - ("allowed_value_case", ("order_status",), 0.3), - ("close_category_pair", ("order_status",), 0.08), - ("date_format_shuffle", ("order_date",), 0.25), - ("whitespace_insertion", ("notes",), 0.3), - ("unit_suffix_insertion", ("monthly_revenue",), 0.05), - ], -} - -#: Frame-level steps applied after cell plans (traps, rows, headers). -FRAME_STEPS: dict[str, list[tuple[str, dict[str, Any]]]] = { - "customers": [ - ("protected_column_trap", {"column": "national_id_like", "n": 3}), - ("duplicate_row_injection", {"n_duplicates": 3}), - ("duplicated_id_rows", {"id_column": "cust_id", "n": 2}), - ], - "transactions": [ - ("target_column_trap", {"column": "monthly_revenue", "n": 2}), - ("duplicate_row_injection", {"n_duplicates": 2}), - ], -} - - -def build_corrupted( - out_dir: Path | str = DEFAULT_OUT, *, seed: int = 0, dev: bool = False -) -> dict[str, Any]: - out = Path(out_dir) - out.mkdir(parents=True, exist_ok=True) - summary: dict[str, Any] = {"built_at": utc_now_iso(), "seed": seed, "dev": dev, "tables": {}} - - for name, clean in seed_tables(seed=seed).items(): - plan = PLANS.get(name, []) - if dev: - plan = plan[:8] - steps = [(get_corruptor(cn), cols, share) for cn, cols, share in plan] - messy, labels = compose(clean, steps, seed=seed) - for corruptor_name, params in ([] if dev else FRAME_STEPS.get(name, [])): - messy, extra = apply_corruptor( - messy, get_corruptor(corruptor_name), seed=seed + 71, params=params, - ) - labels.extend(extra) - - label_dicts = [label.to_dict() for label in labels] - problems = validate_labels(label_dicts) - if problems: - raise SystemExit(f"{name}: invalid corruption labels:\n" + "\n".join(problems[:10])) - - _write_frame(out / f"{name}.clean", clean) - _write_frame(out / f"{name}.messy", messy) - write_jsonl(out / f"{name}.labels.jsonl", label_dicts) - summary["tables"][name] = { - "rows_clean": len(clean), "rows_messy": len(messy), "labels": len(label_dicts), - "families": sorted({label["transform_family"] for label in label_dicts}), - } - - write_json(out / "meta.json", summary) - return summary - - -def _write_frame(stem: Path, frame: pd.DataFrame) -> Path: - try: - path = stem.with_suffix(".parquet") - frame.to_parquet(path, index=False) - except (ImportError, ValueError): - path = stem.with_suffix(".csv") - frame.to_csv(path, index=False) - return path - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.datasets.build_corrupted") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--out", default=str(DEFAULT_OUT)) - parser.add_argument("--dev", action="store_true", help="small fast subset of corruptors") - args = parser.parse_args(argv) - summary = build_corrupted(args.out, seed=args.seed, dev=args.dev) - for table, info in summary["tables"].items(): - print(f"{table}: {info['labels']} labels over {info['rows_messy']} rows", file=sys.stdout) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/datasets/build_seed.py b/training/datasets/build_seed.py deleted file mode 100644 index 5232f99e..00000000 --- a/training/datasets/build_seed.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Build the seed corpus into ``training/build/seed/``. - -Runs the license gate first, then materializes every allowed source: -synthetic tables (parquet or csv), the hand-authored fixtures, and the -labeled context-sentence set. ``meta.json`` records provenance for each file. - -CLI:: - - python -m training.datasets.build_seed [--seed 0] [--out training/build/seed] -""" - -from __future__ import annotations - -import argparse -import shutil -import sys -from pathlib import Path - -from ..common import BUILD_DIR, TRAINING_ROOT, utc_now_iso, write_json, write_jsonl -from ..seed.synthetic import SYNTHETIC_SOURCE_ID, make_context_sentences, seed_tables -from .validators import check_licenses - -DEFAULT_OUT = BUILD_DIR / "seed" - - -def build_seed(out_dir: Path | str = DEFAULT_OUT, *, seed: int = 0) -> dict[str, object]: - problems = check_licenses() - if problems: - raise SystemExit("seed registry invalid:\n" + "\n".join(f" {p}" for p in problems)) - - out = Path(out_dir) - out.mkdir(parents=True, exist_ok=True) - files: dict[str, dict[str, object]] = {} - - for name, frame in seed_tables(seed=seed).items(): - try: - path = out / f"{name}.parquet" - frame.to_parquet(path, index=False) - except (ImportError, ValueError): # no parquet engine available - path = out / f"{name}.csv" - frame.to_csv(path, index=False) - files[path.name] = { - "source_id": SYNTHETIC_SOURCE_ID, "rows": len(frame), "synthetic": True} - - sentences = make_context_sentences(seed=seed) - sentence_path = write_jsonl(out / "context_sentences.jsonl", sentences) - files[sentence_path.name] = { - "source_id": SYNTHETIC_SOURCE_ID, "rows": len(sentences), "synthetic": True, - } - - handauthored = TRAINING_ROOT / "seed" / "sources" / "handauthored_ecommerce_v1.csv" - shutil.copy(handauthored, out / handauthored.name) - files[handauthored.name] = {"source_id": "handauthored_ecommerce_v1", "synthetic": True} - - meta = {"built_at": utc_now_iso(), "seed": seed, "files": files} - write_json(out / "meta.json", meta) - return meta - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.datasets.build_seed") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--out", default=str(DEFAULT_OUT)) - args = parser.parse_args(argv) - meta = build_seed(args.out, seed=args.seed) - print(f"seed corpus built: {len(meta['files'])} files -> {args.out}", file=sys.stdout) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/datasets/splits.py b/training/datasets/splits.py deleted file mode 100644 index e53fead3..00000000 --- a/training/datasets/splits.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Deterministic dataset splits (hash-based, author-disjoint capable).""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any, Callable - -from ..common import sha256_text - - -def split_key(item: dict[str, Any], fields: Sequence[str]) -> str: - return "|".join(str(item.get(f, "")) for f in fields) - - -def hash_split( - items: Sequence[dict[str, Any]], - *, - key_fn: Callable[[dict[str, Any]], str], - eval_fraction: float = 0.2, - salt: str = "freshdata-split", -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Split into (train, eval) by hashing a stable per-item key. - - Items sharing a key always land on the same side, so near-duplicates - (paraphrases of one sentence, variants of one column) never straddle the - boundary. - """ - train: list[dict[str, Any]] = [] - holdout: list[dict[str, Any]] = [] - threshold = int(eval_fraction * 0xFFFF) - for item in items: - bucket = int(sha256_text(f"{salt}:{key_fn(item)}")[:4], 16) - (holdout if bucket < threshold else train).append(item) - return train, holdout - - -def author_disjoint_split( - items: Sequence[dict[str, Any]], - *, - eval_authors: Sequence[str], - author_field: str = "author", -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Split so that eval authors never appear in training (paraphrase gate).""" - eval_set = set(eval_authors) - train = [i for i in items if str(i.get(author_field, "")) not in eval_set] - holdout = [i for i in items if str(i.get(author_field, "")) in eval_set] - return train, holdout diff --git a/training/datasets/validators.py b/training/datasets/validators.py deleted file mode 100644 index c797327c..00000000 --- a/training/datasets/validators.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Validators for the seed registry and corruption labels. - -``python -m training.datasets.validators --check-licenses`` is the release -gate for training-data provenance: it fails when any source has a missing -license, missing attribution, non-explicit training permission, unresolved -PII risk, or unclear commercial-use status (including share-alike licenses -without a recorded legal review). -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import Any - -from ..common import TRAINING_ROOT, read_json - -REGISTRY_PATH = TRAINING_ROOT / "seed" / "registry.json" - -#: Licenses whose commercial use is unambiguous for training. -ALLOWED_LICENSES = frozenset({ - "CC0-1.0", "PDDL-1.0", "CC-BY-4.0", "Apache-2.0", "MIT", - "BSD-3-Clause", "OGL-UK-3.0", "OGL-India", -}) -#: Share-alike / reciprocal licenses: blocked unless legal review approves. -SHARE_ALIKE_LICENSES = frozenset({ - "CC-BY-SA-4.0", "CC-BY-SA-3.0", "ODbL-1.0", "GPL-3.0", "GPL-2.0", -}) -PII_RISKS = frozenset({"none", "synthetic", "review_required"}) - -_REQUIRED_FIELDS = ( - "source_id", "name", "url", "license", "license_text_path", - "attribution", "allowed_for_training", "pii_risk", -) - - -def validate_source(entry: dict[str, Any], seed_dir: Path) -> list[str]: - """Return a list of human-readable problems for one registry entry.""" - errors: list[str] = [] - sid = str(entry.get("source_id") or "") - - for field in _REQUIRED_FIELDS: - if field not in entry or entry[field] in (None, ""): - errors.append(f"{sid}: missing required field {field!r}") - - license_id = str(entry.get("license") or "") - if license_id: - review = entry.get("legal_review") or {} - if license_id in SHARE_ALIKE_LICENSES: - approved = ( - isinstance(review, dict) - and review.get("approved") is True - and review.get("reviewer") - ) - if not approved: - errors.append( - f"{sid}: share-alike license {license_id!r} requires an approving " - "legal_review record (approved: true, reviewer set)" - ) - elif license_id not in ALLOWED_LICENSES: - errors.append( - f"{sid}: license {license_id!r} has unclear commercial-use status; " - f"allowed: {', '.join(sorted(ALLOWED_LICENSES))} " - "(or share-alike with legal review)" - ) - - if "allowed_for_training" in entry and not isinstance(entry["allowed_for_training"], bool): - errors.append(f"{sid}: allowed_for_training must be an explicit boolean") - elif entry.get("allowed_for_training") is False: - errors.append(f"{sid}: allowed_for_training is false β€” remove the source or approve it") - - pii = entry.get("pii_risk") - if pii is not None and pii not in PII_RISKS: - errors.append(f"{sid}: pii_risk must be one of {sorted(PII_RISKS)}, got {pii!r}") - if pii == "review_required": - review = entry.get("legal_review") or {} - if not (isinstance(review, dict) and review.get("approved") is True): - errors.append( - f"{sid}: unresolved PII risk (review_required without approving legal_review)") - - text_path = entry.get("license_text_path") - if text_path and not (seed_dir / str(text_path)).is_file(): - errors.append( - f"{sid}: license_text_path {text_path!r} does not exist under training/seed/") - return errors - - -def check_licenses(registry_path: Path | str = REGISTRY_PATH) -> list[str]: - """Validate the whole registry; return all problems (empty = pass).""" - registry_path = Path(registry_path) - if not registry_path.is_file(): - return [f"registry not found: {registry_path}"] - data = read_json(registry_path) - sources = data.get("sources") - if not isinstance(sources, list) or not sources: - return ["registry must contain a non-empty 'sources' list"] - errors: list[str] = [] - seen: set[str] = set() - for entry in sources: - if not isinstance(entry, dict): - errors.append("registry entries must be objects") - continue - sid = str(entry.get("source_id") or "") - if sid in seen: - errors.append(f"{sid}: duplicate source_id") - seen.add(sid) - errors.extend(validate_source(entry, registry_path.parent)) - return errors - - -# --------------------------------------------------------------------------- # -# Corruption-label validation -# --------------------------------------------------------------------------- # - -LABEL_FIELDS = ( - "raw_value", "clean_value", "column", "transform_family", "params", - "should_repair", "should_auto_apply", "risk", "protected", "ambiguous", -) -RISKS = frozenset({"low", "medium", "high"}) - - -def validate_label(label: dict[str, Any]) -> list[str]: - """Check one corruption label for internal consistency.""" - errors: list[str] = [] - for field in LABEL_FIELDS: - if field not in label: - errors.append(f"label missing field {field!r}") - risk = label.get("risk") - if risk not in RISKS: - errors.append(f"invalid risk {risk!r}") - if not isinstance(label.get("params", {}), dict): - errors.append("params must be a dict") - if label.get("ambiguous") and label.get("should_auto_apply"): - errors.append("ambiguous corruption must not be auto-apply") - if label.get("protected") and label.get("should_auto_apply"): - errors.append("protected-column corruption must never be auto-apply") - if label.get("should_auto_apply") and not label.get("should_repair"): - errors.append("should_auto_apply requires should_repair") - if label.get("should_repair") and label.get("raw_value") == label.get("clean_value"): - errors.append("should_repair label with identical raw/clean value") - return errors - - -def validate_labels(labels: list[dict[str, Any]]) -> list[str]: - errors: list[str] = [] - for i, label in enumerate(labels): - for problem in validate_label(label): - errors.append(f"label[{i}]: {problem}") - return errors - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.datasets.validators") - parser.add_argument("--check-licenses", action="store_true", help="validate the seed registry") - parser.add_argument("--registry", default=str(REGISTRY_PATH)) - parser.add_argument("--labels", default=None, help="optional labels JSONL to validate") - args = parser.parse_args(argv) - - failures: list[str] = [] - if args.check_licenses: - failures.extend(check_licenses(args.registry)) - if args.labels: - from ..common import read_jsonl # noqa: PLC0415 - - failures.extend(validate_labels(read_jsonl(args.labels))) - if not args.check_licenses and not args.labels: - parser.error("nothing to do: pass --check-licenses and/or --labels") - - if failures: - for problem in failures: - print(f"FAIL: {problem}", file=sys.stderr) - return 1 - print("seed registry / labels: OK") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/__init__.py b/training/distill/__init__.py deleted file mode 100644 index 6a67f62e..00000000 --- a/training/distill/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Distillation: train small heads, export, quantize, package artifacts.""" diff --git a/training/distill/export_onnx.py b/training/distill/export_onnx.py deleted file mode 100644 index 870ef1b4..00000000 --- a/training/distill/export_onnx.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Export trained heads to ONNX (release) or a documented dev stub. - -The heads are linear (``softmax(x @ W.T + b)``) so the graph is three nodes. -Building it needs the ``onnx`` package, which is deliberately **not** a core -dependency: dev environments export the portable weights JSON plus a -``tokenizer.json`` (featurizer config) and skip the ``.onnx`` file; the -release pipeline (``--release``) hard-fails without ``onnx`` so a published -artifact can never silently miss its graph. - -CLI:: - - python -m training.distill.export_onnx [--release] -""" - -from __future__ import annotations - -import argparse -import importlib.util -import json -import shutil -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, write_json -from .linear import LinearHead - -OUT_DIR = BUILD_DIR / "export" - -HEADS = { - "fd-role-head-v1": BUILD_DIR / "role_head" / "role_head.weights.json", - "fd-intent-v1": BUILD_DIR / "intent_head" / "intent_head.weights.json", -} - - -def onnx_available() -> bool: - return importlib.util.find_spec("onnx") is not None - - -def export_head_onnx(head: LinearHead, path: Path) -> Path: # pragma: no cover - needs onnx - """Build the 3-node linear-head graph with the real onnx package.""" - import onnx # noqa: PLC0415 - from onnx import TensorProto, helper, numpy_helper # noqa: PLC0415 - - dim = head.weights.shape[1] - n_classes = head.weights.shape[0] - weight_init = numpy_helper.from_array(head.weights.T.copy(), name="W") - bias_init = numpy_helper.from_array(head.bias.copy(), name="b") - graph = helper.make_graph( - nodes=[ - helper.make_node("MatMul", ["features", "W"], ["logits_raw"]), - helper.make_node("Add", ["logits_raw", "b"], ["logits"]), - helper.make_node("Softmax", ["logits"], ["probabilities"], axis=-1), - ], - name="freshdata_linear_head", - inputs=[helper.make_tensor_value_info("features", TensorProto.FLOAT, [None, dim])], - outputs=[helper.make_tensor_value_info( - "probabilities", TensorProto.FLOAT, [None, n_classes])], - initializer=[weight_init, bias_init], - ) - model = helper.make_model(graph, producer_name="freshdata-training") - onnx.checker.check_model(model) - path.parent.mkdir(parents=True, exist_ok=True) - onnx.save(model, str(path)) - return path - - -def tokenizer_config(head: LinearHead) -> dict[str, Any]: - """The featurizer config, playing the role of ``tokenizer.json``.""" - return { - "type": "freshdata-hashed-ngram-featurizer", - "featurizer": head.featurizer.to_dict(), - "classes": list(head.classes), - "abstain_class": head.abstain_class, - "abstain_threshold": head.abstain_threshold, - } - - -def export_all(*, out_dir: Path | str = OUT_DIR, release: bool = False) -> dict[str, Any]: - out = Path(out_dir) - have_onnx = onnx_available() - if release and not have_onnx: - raise SystemExit( - "release export requires the 'onnx' package: pip install onnx " - "(dev exports run without it, but a release artifact must ship a real graph)" - ) - summary: dict[str, Any] = {"onnx_available": have_onnx, "release": release, "heads": {}} - for model_id, weights_path in HEADS.items(): - if not weights_path.is_file(): - summary["heads"][model_id] = {"status": "missing", "weights": str(weights_path)} - continue - head = LinearHead.load(weights_path) - model_dir = out / model_id - model_dir.mkdir(parents=True, exist_ok=True) - shutil.copy(weights_path, model_dir / "weights.json") - (model_dir / "tokenizer.json").write_text( - json.dumps(tokenizer_config(head), indent=2, sort_keys=True), encoding="utf-8") - entry: dict[str, Any] = {"status": "exported", "files": ["weights.json", "tokenizer.json"]} - if have_onnx: # pragma: no cover - needs onnx installed - export_head_onnx(head, model_dir / "model.onnx") - entry["files"].append("model.onnx") - else: - entry["note"] = "dev export: model.onnx omitted (onnx package not installed)" - summary["heads"][model_id] = entry - write_json(out / "export_summary.json", summary) - return summary - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.export_onnx") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--release", action="store_true") - args = parser.parse_args(argv) - summary = export_all(out_dir=args.out, release=args.release) - for model_id, entry in summary["heads"].items(): - print(f"{model_id}: {entry['status']} {entry.get('files', '')}", file=sys.stdout) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/linear.py b/training/distill/linear.py deleted file mode 100644 index a1ab9d11..00000000 --- a/training/distill/linear.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Deterministic pure-numpy linear heads over hashed character n-grams. - -The Phase-5 heads (semantic-type role head, context intent head) are tiny -multinomial logistic regressions: strong enough for the eval corpora, small -enough to export, and β€” crucially β€” trainable with zero optional -dependencies so the dev pipeline runs anywhere the test suite runs. -Everything is deterministic: hashing uses sha256, training is full-batch -gradient descent from a zero init (no RNG at all). -""" - -from __future__ import annotations - -import base64 -import hashlib -import json -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path - -import numpy as np - -FEATURIZER_VERSION = "hashed-ngram-1" - - -@dataclass(frozen=True) -class FeaturizerConfig: - dim: int = 2048 - ngram_sizes: tuple[int, ...] = (2, 3) - lowercase: bool = True - #: Hash word unigrams/bigrams too β€” intent-bearing words ("unique", - #: "never modify") generalize across paraphrases better than char grams. - word_ngrams: bool = True - - def to_dict(self) -> dict[str, object]: - return { - "version": FEATURIZER_VERSION, - "dim": self.dim, - "ngram_sizes": list(self.ngram_sizes), - "lowercase": self.lowercase, - "word_ngrams": self.word_ngrams, - } - - @classmethod - def from_dict(cls, data: dict[str, object]) -> FeaturizerConfig: - return cls( - dim=int(data["dim"]), # type: ignore[arg-type] - ngram_sizes=tuple(data["ngram_sizes"]), # type: ignore[arg-type] - lowercase=bool(data["lowercase"]), - word_ngrams=bool(data.get("word_ngrams", False)), - ) - - -def _bucket(gram: str, dim: int) -> tuple[int, float]: - digest = hashlib.sha256(gram.encode("utf-8")).digest() - index = int.from_bytes(digest[:4], "big") % dim - sign = 1.0 if digest[4] & 1 else -1.0 - return index, sign - - -def featurize(texts: Sequence[str], config: FeaturizerConfig) -> np.ndarray: - """L2-normalized hashed n-gram counts, shape ``(n, dim)`` float32.""" - out = np.zeros((len(texts), config.dim), dtype=np.float32) - for i, text in enumerate(texts): - body = str(text) - if config.lowercase: - body = body.casefold() - padded = f"^{body}$" - for n in config.ngram_sizes: - for j in range(max(1, len(padded) - n + 1)): - index, sign = _bucket(f"{n}:{padded[j:j + n]}", config.dim) - out[i, index] += sign - if config.word_ngrams: - words = body.split() - for word in words: - index, sign = _bucket(f"w1:{word}", config.dim) - out[i, index] += 2.0 * sign - for j in range(len(words) - 1): - index, sign = _bucket(f"w2:{words[j]} {words[j + 1]}", config.dim) - out[i, index] += 2.0 * sign - norm = float(np.linalg.norm(out[i])) - if norm: - out[i] /= norm - return out - - -@dataclass -class LinearHead: - """Multinomial logistic regression with optional abstention.""" - - classes: tuple[str, ...] - weights: np.ndarray # (C, D) float32 - bias: np.ndarray # (C,) float32 - featurizer: FeaturizerConfig - abstain_class: str | None = "unknown" - abstain_threshold: float = 0.5 - - # -- training ---------------------------------------------------------------- - - @classmethod - def train( - cls, - texts: Sequence[str], - labels: Sequence[str], - *, - classes: Sequence[str], - featurizer: FeaturizerConfig | None = None, - epochs: int = 300, - lr: float = 2.0, - l2: float = 1e-4, - class_weights: dict[str, float] | None = None, - abstain_class: str | None = "unknown", - abstain_threshold: float = 0.5, - ) -> LinearHead: - config = featurizer or FeaturizerConfig() - class_list = tuple(classes) - index = {c: i for i, c in enumerate(class_list)} - x = featurize(texts, config) - y = np.array([index[label] for label in labels], dtype=np.int64) - n, d = x.shape - c = len(class_list) - weights = np.zeros((c, d), dtype=np.float64) - bias = np.zeros(c, dtype=np.float64) - sample_weight = np.ones(n, dtype=np.float64) - if class_weights: - for i, label in enumerate(labels): - sample_weight[i] = float(class_weights.get(label, 1.0)) - sample_weight /= sample_weight.mean() - onehot = np.zeros((n, c), dtype=np.float64) - onehot[np.arange(n), y] = 1.0 - velocity_w = np.zeros_like(weights) - velocity_b = np.zeros_like(bias) - momentum = 0.9 - for _ in range(epochs): - logits = x @ weights.T + bias - logits -= logits.max(axis=1, keepdims=True) - probs = np.exp(logits) - probs /= probs.sum(axis=1, keepdims=True) - grad = (probs - onehot) * sample_weight[:, None] - grad_w = grad.T @ x / n + l2 * weights - grad_b = grad.mean(axis=0) - velocity_w = momentum * velocity_w - lr * grad_w - velocity_b = momentum * velocity_b - lr * grad_b - weights += velocity_w - bias += velocity_b - return cls( - classes=class_list, - weights=weights.astype(np.float32), - bias=bias.astype(np.float32), - featurizer=config, - abstain_class=abstain_class, - abstain_threshold=abstain_threshold, - ) - - # -- inference ---------------------------------------------------------------- - - def predict_proba(self, texts: Sequence[str]) -> np.ndarray: - x = featurize(texts, self.featurizer) - logits = x @ self.weights.T.astype(np.float64) + self.bias.astype(np.float64) - logits -= logits.max(axis=1, keepdims=True) - probs = np.exp(logits) - probs /= probs.sum(axis=1, keepdims=True) - return probs - - def predict(self, texts: Sequence[str]) -> list[tuple[str, float]]: - """(label, confidence) per text, abstaining below the threshold.""" - probs = self.predict_proba(texts) - out: list[tuple[str, float]] = [] - for row in probs: - best = int(row.argmax()) - label, confidence = self.classes[best], float(row[best]) - if ( - self.abstain_class is not None - and confidence < self.abstain_threshold - and label != self.abstain_class - ): - label = self.abstain_class - out.append((label, round(confidence, 6))) - return out - - # -- persistence ---------------------------------------------------------------- - - def to_dict(self) -> dict[str, object]: - return { - "format": "freshdata-linear-head-1", - "classes": list(self.classes), - "featurizer": self.featurizer.to_dict(), - "abstain_class": self.abstain_class, - "abstain_threshold": self.abstain_threshold, - "weights_b64": base64.b64encode(self.weights.astype(np.float32).tobytes()).decode(), - "bias_b64": base64.b64encode(self.bias.astype(np.float32).tobytes()).decode(), - "shape": list(self.weights.shape), - } - - def save(self, path: Path | str) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(self.to_dict(), sort_keys=True), encoding="utf-8") - return path - - @classmethod - def load(cls, path: Path | str) -> LinearHead: - data = json.loads(Path(path).read_text(encoding="utf-8")) - shape = tuple(int(v) for v in data["shape"]) - weights = np.frombuffer( - base64.b64decode(data["weights_b64"]), dtype=np.float32 - ).reshape(shape).copy() - bias = np.frombuffer(base64.b64decode(data["bias_b64"]), dtype=np.float32).copy() - return cls( - classes=tuple(data["classes"]), - weights=weights, - bias=bias, - featurizer=FeaturizerConfig.from_dict(data["featurizer"]), - abstain_class=data.get("abstain_class"), - abstain_threshold=float(data.get("abstain_threshold", 0.5)), - ) - - -# --------------------------------------------------------------------------- # -# int8 quantization (our own artifact format; ONNX quantization is optional) -# --------------------------------------------------------------------------- # - -def quantize_int8(head: LinearHead) -> dict[str, object]: - """Symmetric per-row int8 quantization of the weight matrix.""" - scales = np.abs(head.weights).max(axis=1) / 127.0 - scales[scales == 0.0] = 1.0 - quantized = np.clip(np.round(head.weights / scales[:, None]), -127, 127).astype(np.int8) - return { - "format": "freshdata-linear-head-int8-1", - "classes": list(head.classes), - "featurizer": head.featurizer.to_dict(), - "abstain_class": head.abstain_class, - "abstain_threshold": head.abstain_threshold, - "weights_int8_b64": base64.b64encode(quantized.tobytes()).decode(), - "scales_b64": base64.b64encode(scales.astype(np.float32).tobytes()).decode(), - "bias_b64": base64.b64encode(head.bias.astype(np.float32).tobytes()).decode(), - "shape": list(head.weights.shape), - } - - -def dequantize(data: dict[str, object]) -> LinearHead: - shape = tuple(int(v) for v in data["shape"]) # type: ignore[arg-type] - quantized = np.frombuffer( - base64.b64decode(data["weights_int8_b64"]), dtype=np.int8 # type: ignore[arg-type] - ).reshape(shape) - scales = np.frombuffer(base64.b64decode(data["scales_b64"]), dtype=np.float32) # type: ignore[arg-type] - bias = np.frombuffer(base64.b64decode(data["bias_b64"]), dtype=np.float32).copy() # type: ignore[arg-type] - weights = (quantized.astype(np.float32) * scales[:, None]).astype(np.float32) - return LinearHead( - classes=tuple(data["classes"]), # type: ignore[arg-type] - weights=weights, - bias=bias, - featurizer=FeaturizerConfig.from_dict(data["featurizer"]), # type: ignore[arg-type] - abstain_class=data.get("abstain_class"), # type: ignore[arg-type] - abstain_threshold=float(data.get("abstain_threshold", 0.5)), # type: ignore[arg-type] - ) - - -# --------------------------------------------------------------------------- # -# shared metrics -# --------------------------------------------------------------------------- # - -def macro_f1(y_true: Sequence[str], y_pred: Sequence[str], classes: Sequence[str]) -> float: - scores = per_class_f1(y_true, y_pred, classes) - present = [f1 for cls, f1 in scores.items() if any(t == cls for t in y_true)] - return float(np.mean(present)) if present else 0.0 - - -def per_class_f1( - y_true: Sequence[str], y_pred: Sequence[str], classes: Sequence[str] -) -> dict[str, float]: - out: dict[str, float] = {} - for cls in classes: - tp = sum(1 for t, p in zip(y_true, y_pred) if t == cls and p == cls) - fp = sum(1 for t, p in zip(y_true, y_pred) if t != cls and p == cls) - fn = sum(1 for t, p in zip(y_true, y_pred) if t == cls and p != cls) - denominator = 2 * tp + fp + fn - out[cls] = (2 * tp / denominator) if denominator else 0.0 - return out - - -def confusion( - y_true: Sequence[str], y_pred: Sequence[str], classes: Sequence[str] -) -> dict[str, dict[str, int]]: - matrix = {t: dict.fromkeys(classes, 0) for t in classes} - for t, p in zip(y_true, y_pred): - matrix[t][p] += 1 - return matrix diff --git a/training/distill/package_artifacts.py b/training/distill/package_artifacts.py deleted file mode 100644 index a6d44f1d..00000000 --- a/training/distill/package_artifacts.py +++ /dev/null @@ -1,248 +0,0 @@ -"""Package trained artifacts into ``dist/artifacts/`` with manifests. - -Layout (one directory per model id, outside the wheel by construction):: - - dist/artifacts/ - fd-col-encoder-v1/ model.onnx? tokenizer.json role_head.* manifest.json model_card.md - fd-intent-v1/ model.onnx? tokenizer.json weights.* manifest.json model_card.md - calib-v1/ calib.json calibration.json manifest.json model_card.md - -Validation fails on: SHA mismatch, missing model card, missing license -summary, missing eval metrics, size over the configured limit, or any -artifact file that would land inside the wheel package tree -(``src/freshdata``). - -CLI:: - - python -m training.distill.package_artifacts [--release] -""" - -from __future__ import annotations - -import argparse -import shutil -import sys -from pathlib import Path -from typing import Any - -from ..common import ( - ARTIFACTS_DIR, - BUILD_DIR, - MAX_ARTIFACT_BYTES, - REPO_ROOT, - TRAINING_ROOT, - freshdata_version, - git_commit, - read_json, - sha256_file, - utc_now_iso, - write_json, -) - -MODEL_CARDS = TRAINING_ROOT / "model_cards" -EXPORT_DIR = BUILD_DIR / "export" -CALIBRATION_DIR = BUILD_DIR / "calibration" - -#: model_id -> (card file, source files: {artifact name: build path}) -_SPECS: dict[str, dict[str, Any]] = { - "fd-col-encoder-v1": { - "card": "fd-col-encoder-v1.md", - "license": "Apache-2.0", - "training_data_summary": ( - "Synthetic PII-shaped seed corpus + corruptor-derived labels " - "(training/seed/registry.json); no real PII, no scraped data." - ), - "files": { - "tokenizer.json": EXPORT_DIR / "fd-role-head-v1" / "tokenizer.json", - "role_head.weights.json": EXPORT_DIR / "fd-role-head-v1" / "weights.json", - "role_head.weights.int8.json": EXPORT_DIR / "fd-role-head-v1" / "weights.int8.json", - }, - "optional_files": { - "model.onnx": EXPORT_DIR / "fd-role-head-v1" / "model.onnx", - "role_head.onnx": EXPORT_DIR / "fd-role-head-v1" / "model.onnx", - }, - "metrics": BUILD_DIR / "role_head" / "role_head.metrics.json", - }, - "fd-intent-v1": { - "card": "fd-intent-v1.md", - "license": "Apache-2.0", - "training_data_summary": ( - "Phase-1 golden context corpus + synthetic/Hinglish paraphrase set " - "+ corruptor context variants; author-disjoint human-verified eval." - ), - "files": { - "tokenizer.json": EXPORT_DIR / "fd-intent-v1" / "tokenizer.json", - "weights.json": EXPORT_DIR / "fd-intent-v1" / "weights.json", - "weights.int8.json": EXPORT_DIR / "fd-intent-v1" / "weights.int8.json", - }, - "optional_files": {"model.onnx": EXPORT_DIR / "fd-intent-v1" / "model.onnx"}, - "metrics": BUILD_DIR / "intent_head" / "intent_head.metrics.json", - }, - "calib-v1": { - "card": "calib-v1.md", - "license": "Apache-2.0", - "training_data_summary": ( - "CleanBench T1-T4 fixture runs with corruptor ground truth; " - "isotonic fit per (backend, issue family)." - ), - "files": { - "calib.json": CALIBRATION_DIR / "calib.json", - "calibration.json": CALIBRATION_DIR / "calibration.json", - }, - "optional_files": {}, - "metrics": CALIBRATION_DIR / "calib-v1.metrics.json", - }, -} - - -class PackagingError(RuntimeError): - """Artifact packaging failed validation.""" - - -def _quantization(model_id: str, files: dict[str, Path]) -> str: - if model_id == "calib-v1": - return "json" - if any(name.endswith(".onnx") for name in files): - return "int8" - return "int8-json" # dev artifacts: our own int8 weight format, no onnx - - -def package_one(model_id: str, *, out_root: Path, release: bool) -> dict[str, Any]: - spec = _SPECS[model_id] - sources: dict[str, Path] = {} - for name, path in spec["files"].items(): - if not Path(path).is_file(): - raise PackagingError(f"{model_id}: missing build input {path}") - sources[name] = Path(path) - for name, path in spec["optional_files"].items(): - if Path(path).is_file() and name not in sources: - sources[name] = Path(path) - elif release and name == "model.onnx": - raise PackagingError(f"{model_id}: release packaging requires {path}") - - metrics_path = Path(spec["metrics"]) - if not metrics_path.is_file(): - raise PackagingError(f"{model_id}: missing eval metrics {metrics_path}") - metrics = read_json(metrics_path) - - card_source = MODEL_CARDS / spec["card"] - if not card_source.is_file(): - raise PackagingError(f"{model_id}: missing model card {card_source}") - - target = out_root / model_id - if target.exists(): - shutil.rmtree(target) - target.mkdir(parents=True) - - copied: list[dict[str, Any]] = [] - total = 0 - for name, source in sorted(sources.items()): - destination = target / name - shutil.copy(source, destination) - size = destination.stat().st_size - total += size - copied.append({"name": name, "sha256": sha256_file(destination), "size_bytes": size}) - shutil.copy(card_source, target / "model_card.md") - shutil.copy(metrics_path, target / "eval_metrics.json") - - if total > MAX_ARTIFACT_BYTES: - raise PackagingError( - f"{model_id}: artifact size {total} exceeds limit {MAX_ARTIFACT_BYTES}") - - primary = copied[0] - manifest = { - "model_id": model_id, - "version": model_id.rsplit("-v", 1)[-1], - "freshdata_min_version": freshdata_version(), - "sha256": primary["sha256"], - "size_bytes": total, - "license": spec["license"], - "training_data_summary": spec["training_data_summary"], - "eval_summary": {k: v for k, v in metrics.items() - if isinstance(v, (int, float, str))}, - "quantization": _quantization(model_id, sources), - "files": copied, - "created_at": utc_now_iso(), - "git_commit": git_commit(), - "release": release, - } - write_json(target / "manifest.json", manifest) - return manifest - - -def validate_package(model_id: str, *, out_root: Path) -> list[str]: - """Independent re-validation of a packaged artifact directory.""" - problems: list[str] = [] - target = out_root / model_id - manifest_path = target / "manifest.json" - if not manifest_path.is_file(): - return [f"{model_id}: manifest.json missing"] - manifest = read_json(manifest_path) - for entry in manifest.get("files", []): - path = target / entry["name"] - if not path.is_file(): - problems.append(f"{model_id}: file {entry['name']} listed but missing") - continue - if sha256_file(path) != entry["sha256"]: - problems.append(f"{model_id}: SHA mismatch for {entry['name']}") - if not (target / "model_card.md").is_file(): - problems.append(f"{model_id}: model card missing") - if not manifest.get("license"): - problems.append(f"{model_id}: license summary missing") - if not manifest.get("eval_summary"): - problems.append(f"{model_id}: eval metrics missing") - if int(manifest.get("size_bytes", 0)) > MAX_ARTIFACT_BYTES: - problems.append(f"{model_id}: size over limit") - problems.extend(_wheel_guard(target)) - return problems - - -def _wheel_guard(target: Path) -> list[str]: - """No packaged artifact may live inside the wheel package tree.""" - problems = [] - wheel_root = (REPO_ROOT / "src" / "freshdata").resolve() - for path in target.rglob("*"): - try: - path.resolve().relative_to(wheel_root) - except ValueError: - continue - problems.append(f"artifact file {path} is inside src/freshdata (would ship in wheel)") - stray = list(wheel_root.rglob("*.onnx")) - problems.extend(f"model weights inside wheel tree: {p}" for p in stray) - return problems - - -def package_all(*, out_root: Path | str = ARTIFACTS_DIR, release: bool = False) -> dict[str, Any]: - root = Path(out_root) - manifests = {} - for model_id in _SPECS: - manifests[model_id] = package_one(model_id, out_root=root, release=release) - problems = validate_package(model_id, out_root=root) - if problems: - raise PackagingError("; ".join(problems)) - write_json(root / "artifacts_index.json", { - "created_at": utc_now_iso(), - "git_commit": git_commit(), - "artifacts": {mid: m["sha256"] for mid, m in manifests.items()}, - }) - return manifests - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.package_artifacts") - parser.add_argument("--out", default=str(ARTIFACTS_DIR)) - parser.add_argument("--release", action="store_true") - args = parser.parse_args(argv) - try: - manifests = package_all(out_root=args.out, release=args.release) - except PackagingError as exc: - print(f"PACKAGING FAIL: {exc}", file=sys.stderr) - return 1 - for model_id, manifest in manifests.items(): - print(f"{model_id}: sha256={manifest['sha256'][:16]}… " - f"size={manifest['size_bytes']} files={len(manifest['files'])}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/quantize.py b/training/distill/quantize.py deleted file mode 100644 index 8080812b..00000000 --- a/training/distill/quantize.py +++ /dev/null @@ -1,93 +0,0 @@ -"""int8 quantization of the exported heads, with an accuracy-delta gate. - -Quantization is symmetric per-row int8 on the weight matrix (see -``linear.quantize_int8``). The gate re-evaluates both fp32 and int8 heads on -the human-verified eval sets and fails when any headline metric moves by -0.5 percentage points or more. When ``onnxruntime`` is installed the ONNX -graph is additionally quantized; that path is release-only. - -CLI:: - - python -m training.distill.quantize [--check-gates] -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, write_json -from .linear import LinearHead, dequantize, quantize_int8 - -OUT_DIR = BUILD_DIR / "export" -MAX_METRIC_DELTA = 0.005 # 0.5 percentage points - - -def _headline_metric(model_id: str, head: LinearHead) -> float: - """One comparable accuracy number per head on its human-verified eval.""" - if model_id == "fd-role-head-v1": - from .train_role_head import evaluate, load_human_eval # noqa: PLC0415 - - return float(evaluate(head, load_human_eval())["macro_f1"]) - from .train_intent_head import build_dataset, evaluate # noqa: PLC0415 - - _, eval_rows = build_dataset() - return float(evaluate(head, eval_rows)["exact_intent_accuracy"]) - - -def quantize_all(*, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - summary: dict[str, Any] = {"max_metric_delta": MAX_METRIC_DELTA, "heads": {}} - for model_id in ("fd-role-head-v1", "fd-intent-v1"): - weights = out / model_id / "weights.json" - if not weights.is_file(): - summary["heads"][model_id] = {"status": "missing"} - continue - head = LinearHead.load(weights) - quantized = quantize_int8(head) - int8_path = out / model_id / "weights.int8.json" - int8_path.write_text(json.dumps(quantized, sort_keys=True), encoding="utf-8") - fp32_metric = _headline_metric(model_id, head) - int8_metric = _headline_metric(model_id, dequantize(quantized)) - delta = abs(fp32_metric - int8_metric) - summary["heads"][model_id] = { - "status": "quantized", - "fp32_metric": round(fp32_metric, 4), - "int8_metric": round(int8_metric, 4), - "delta": round(delta, 4), - "within_tolerance": delta < MAX_METRIC_DELTA, - } - write_json(out / "quantize_summary.json", summary) - return summary - - -def check_gates(summary: dict[str, Any]) -> list[str]: - failures = [] - for model_id, entry in summary["heads"].items(): - if entry.get("status") == "quantized" and not entry["within_tolerance"]: - failures.append( - f"{model_id}: int8 vs fp32 delta {entry['delta']} >= {MAX_METRIC_DELTA}") - return failures - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.quantize") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - summary = quantize_all(out_dir=args.out) - for model_id, entry in summary["heads"].items(): - print(f"{model_id}: {entry.get('status')} delta={entry.get('delta')}") - failures = check_gates(summary) - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/train_encoder_contrastive.py b/training/distill/train_encoder_contrastive.py deleted file mode 100644 index c313aa99..00000000 --- a/training/distill/train_encoder_contrastive.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Optional contrastive distillation for the column/value encoder. - -Phase-3's encoder interface (:class:`freshdata.models.runtime.LocalEncoder`) -is an opaque ONNX runtime; contrastive fine-tuning needs a tensor framework. -This module therefore has two layers: - -- **pair building + safety evaluation** always run (pure numpy): alias - pairs, allowed/corrupted pairs, and the *dangerous negatives* whose - false-merge rate must stay exactly zero; -- **training** runs only when ``torch`` is installed; otherwise the run is - recorded as ``skipped`` and the Phase-3 baseline encoder is retained. That - is an explicitly supported outcome, not a failure. - -Gates:: - - resolver accuracy >= Phase-3 baseline - dangerous-negative false-merge rate == 0 - (the embedding backend never auto-applies ambiguous merges β€” enforced at - runtime by the policy gate, re-checked here on the eval pairs) - -CLI:: - - python -m training.distill.train_encoder_contrastive [--check-gates] -""" - -from __future__ import annotations - -import argparse -import importlib.util -import random -import sys -from collections.abc import Sequence -from pathlib import Path -from typing import Any - -import numpy as np - -from ..common import BUILD_DIR, write_json -from ..corruptors.context import ECOMMERCE_ALIASES -from ..corruptors.registry import get_corruptor - -OUT_DIR = BUILD_DIR / "encoder" - -#: Distinct dangerous negatives: pairs that must never merge. -DANGEROUS_NEGATIVES: tuple[tuple[str, str], ...] = ( - ("Austria", "Australia"), - ("CA California", "Canada"), - ("IN India", "inactive"), - ("May month", "May name"), - ("ID code", "category code"), -) - -#: Auto-merge margin mirroring the runtime embedding backend's behavior: -#: an ambiguous top-2 (small margin) is never auto-applied. -MERGE_MARGIN = 0.05 - - -class HashedBigramEncoder: - """Phase-3 baseline stand-in: deterministic char-bigram embedding.""" - - dim = 256 - - def encode_texts(self, texts: Sequence[str]) -> np.ndarray: - import hashlib # noqa: PLC0415 - - out = np.zeros((len(texts), self.dim), dtype=np.float32) - for i, text in enumerate(texts): - padded = f"^{str(text).strip().casefold()}$" - grams = [padded[j:j + 2] for j in range(len(padded) - 1)] or [padded] - for gram in grams: - digest = hashlib.sha256(gram.encode("utf-8")).digest() - out[i, digest[0]] += 1.0 - out[i, 128 + digest[1] % 128] += 1.0 - norm = float(np.linalg.norm(out[i])) - if norm: - out[i] /= norm - return out - - -def build_pairs(*, seed: int = 0) -> dict[str, list[tuple[str, str]]]: - """Contrastive training/eval pairs, including the dangerous negatives.""" - rng = random.Random(f"encoder-pairs:{seed}") - positives: list[tuple[str, str]] = [] - for column, aliases in ECOMMERCE_ALIASES.items(): - for alias in aliases: - positives.append((column, alias)) - typo = get_corruptor("edit_distance_typo") - assert typo.fn is not None - for value in ("active", "inactive", "pending", "delivered", "returned", - "cancelled", "shipped", "placed"): - for _ in range(3): - corrupted = typo.fn(value, rng, {}) - if corrupted: - positives.append((value, corrupted)) - negatives = list(DANGEROUS_NEGATIVES) - # Hard-negative mining from the ambiguity corruptors: values one edit - # from two references are negatives against *both*. - negatives.extend((("nactive", "active"), ("nactive", "inactive"))) - return {"positives": positives, "dangerous_negatives": negatives} - - -def _cosine(a: np.ndarray, b: np.ndarray) -> float: - return float(a @ b) - - -def resolver_accuracy(encoder: Any, pairs: list[tuple[str, str]], - vocabulary: list[str]) -> float: - """Share of corrupted values whose nearest vocabulary entry is the truth.""" - if not pairs: - return 1.0 - vocab_vectors = encoder.encode_texts(vocabulary) - correct = 0 - for truth, corrupted in pairs: - if truth not in vocabulary: - continue - query = encoder.encode_texts([corrupted])[0] - scores = vocab_vectors @ query - best = int(np.argmax(scores)) - correct += int(vocabulary[best] == truth) - return correct / len(pairs) - - -def false_merge_rate(encoder: Any, negatives: list[tuple[str, str]]) -> float: - """Share of dangerous pairs an auto-merging resolver would collapse. - - A merge happens only when the wrong candidate wins *and* the margin over - the runner-up exceeds :data:`MERGE_MARGIN` β€” mirroring the runtime rule - that ambiguous merges are never auto-applied. - """ - if not negatives: - return 0.0 - merges = 0 - for left, right in negatives: - vectors = encoder.encode_texts([left, right]) - similarity = _cosine(vectors[0], vectors[1]) - # Self-similarity is 1.0 by construction; a dangerous merge needs the - # *other* item to look closer than "almost identical". - if similarity > 1.0 - MERGE_MARGIN: - merges += 1 - return merges / len(negatives) - - -def torch_available() -> bool: - return importlib.util.find_spec("torch") is not None - - -def train(*, seed: int = 0, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - pairs = build_pairs(seed=seed) - baseline = HashedBigramEncoder() - vocabulary = sorted({t for t, _ in pairs["positives"]}) - baseline_accuracy = resolver_accuracy(baseline, pairs["positives"], vocabulary) - - result: dict[str, Any] = { - "n_positive_pairs": len(pairs["positives"]), - "n_dangerous_negatives": len(pairs["dangerous_negatives"]), - "baseline_resolver_accuracy": round(baseline_accuracy, 4), - } - - if not torch_available(): - # Explicitly supported: contrastive training is optional; the - # Phase-3 baseline stays in place and the safety gates still run. - encoder = baseline - result["status"] = "skipped" - result["reason"] = "torch not installed; Phase-3 baseline encoder retained" - result["resolver_accuracy"] = result["baseline_resolver_accuracy"] - else: # pragma: no cover - exercised only in envs with torch - encoder = _train_torch(pairs, seed=seed) - result["status"] = "trained" - result["resolver_accuracy"] = round( - resolver_accuracy(encoder, pairs["positives"], vocabulary), 4) - - result["dangerous_negative_false_merge_rate"] = round( - false_merge_rate(encoder, pairs["dangerous_negatives"]), 4) - result["gates"] = { - "resolver_accuracy_min": result["baseline_resolver_accuracy"], - "false_merge_rate_max": 0.0, - "failures": _check_gates(result), - } - write_json(out / "encoder_contrastive.metrics.json", result) - return result - - -def _check_gates(result: dict[str, Any]) -> list[str]: - failures = [] - if result["resolver_accuracy"] < result["baseline_resolver_accuracy"]: - failures.append( - f"resolver accuracy {result['resolver_accuracy']} regressed below " - f"baseline {result['baseline_resolver_accuracy']}") - if result["dangerous_negative_false_merge_rate"] != 0.0: - failures.append( - f"dangerous-negative false-merge rate " - f"{result['dangerous_negative_false_merge_rate']} != 0") - return failures - - -def _train_torch(pairs: dict[str, list[tuple[str, str]]], *, seed: int) -> Any: # pragma: no cover - """Triplet-loss fine-tune over the hashed embedding (torch path).""" - raise NotImplementedError( - "contrastive training requires the release environment; see " - "docs/developer-training-pipeline.md" - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - prog="python -m training.distill.train_encoder_contrastive") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - result = train(seed=args.seed, out_dir=args.out) - print( - f"encoder distillation: status={result['status']} " - f"resolver={result['resolver_accuracy']} " - f"false_merge={result['dangerous_negative_false_merge_rate']}" - ) - failures = result["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/train_intent_head.py b/training/distill/train_intent_head.py deleted file mode 100644 index 3cb5b6cb..00000000 --- a/training/distill/train_intent_head.py +++ /dev/null @@ -1,406 +0,0 @@ -"""Train the context-sentence intent head (``fd-intent-v1``). - -Inputs: the Phase-1 golden context corpus, deterministic paraphrase -templates (including the Indian-English / Hinglish set), corruptor-generated -context variants, optional cached teacher paraphrases, and an -author-disjoint human-verified eval split. - -The intent head is **optional evidence** for the deterministic parser β€” it -never replaces it, and in strict mode unresolved context stays surfaced. Its -training therefore favors *protection recall* over coverage: missing a -PROTECT sentence is the one unforgivable error, so PROTECT is oversampled -and gated at recall >= 0.99. - -Gates (``--check-gates``):: - - exact intent accuracy >= 0.92 - slot F1 >= 0.90 - UNKNOWN precision >= 0.95 - protected intent recall >= 0.99 - -CLI:: - - python -m training.distill.train_intent_head [--dev] [--check-gates] -""" - -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, REPO_ROOT, read_json, write_json -from ..datasets.splits import author_disjoint_split -from ..seed.synthetic import make_context_sentences -from .linear import FeaturizerConfig, LinearHead, confusion, per_class_f1 - -OUT_DIR = BUILD_DIR / "intent_head" -GOLDEN_DIR = REPO_ROOT / "tests" / "context" / "golden" - -INTENTS = ( - "DOMAIN", "UNIQUE", "VALID_FORMAT", "LOCALE_FORMAT", "PROTECT", "IMPUTE_IF", - "ALLOWED_VALUES", "RANGE", "DEDUP_KEY", "DROP_IF", "RENAME", "MAP", "UNKNOWN", -) - -#: Phase-1 constraint rule -> intent label. -RULE_TO_INTENT = { - "unique": "UNIQUE", - "valid_format": "VALID_FORMAT", - "locale_format": "LOCALE_FORMAT", - "protected": "PROTECT", - "impute_missing": "IMPUTE_IF", - "allowed_values": "ALLOWED_VALUES", - "range": "RANGE", - "dedup_key": "DEDUP_KEY", -} - -#: Authors held out entirely for the paraphrase-generalization gate. -EVAL_AUTHORS = ("t2", "hinglish_b") - -GATE_EXACT_ACCURACY = 0.92 -GATE_SLOT_F1 = 0.90 -GATE_UNKNOWN_PRECISION = 0.95 -GATE_PROTECT_RECALL = 0.99 - -_DOMAIN_RE = re.compile(r"\bthis is (a|an)?\s*.*\bdata\s?set\b", re.I) - - -def load_golden_examples() -> list[dict[str, Any]]: - """Sentence-level intent labels derived from the Phase-1 golden corpus.""" - examples: list[dict[str, Any]] = [] - for text_path in sorted(GOLDEN_DIR.glob("*.txt")): - policy_path = text_path.with_name(text_path.name.replace(".txt", ".policy.json")) - if not policy_path.is_file(): - continue - policy = read_json(policy_path) - sentence_intent: dict[str, str] = {} - sentence_slots: dict[str, dict[str, Any]] = {} - for constraint in policy.get("constraints", []): - sentence = str(constraint.get("provenance", {}).get("sentence", "")).strip() - intent = RULE_TO_INTENT.get(str(constraint.get("rule", ""))) - if not sentence or intent is None: - continue - sentence_intent[sentence] = intent - slots = dict(constraint.get("params") or {}) - if constraint.get("resolved_from"): - slots["column"] = constraint["resolved_from"] - sentence_slots[sentence] = slots - for raw_line in text_path.read_text(encoding="utf-8").splitlines(): - sentence = raw_line.strip().rstrip(".") - if not sentence: - continue - matched = None - for known, intent in sentence_intent.items(): - if known.rstrip(".").strip() == sentence: - matched = (intent, sentence_slots.get(known, {})) - break - if matched is None: - if _DOMAIN_RE.search(raw_line) and policy.get("dataset_domain"): - matched = ("DOMAIN", {"domain": policy["dataset_domain"]}) - else: - matched = ("UNKNOWN", {}) - examples.append({ - "sentence": raw_line.strip(), - "intent": matched[0], - "slots": matched[1], - "author": "golden", - }) - return examples - - -def teacher_paraphrase_examples() -> list[dict[str, Any]]: - """Optional cached ``ContextParaphraseBatch`` payloads (training only).""" - from ..teacher.cache import TeacherCache # noqa: PLC0415 - - out: list[dict[str, Any]] = [] - for entry in TeacherCache().entries(): - for payload in entry.get("payloads", []): - intent = payload.get("intent") - if intent in INTENTS and isinstance(payload.get("paraphrases"), list): - for sentence in payload["paraphrases"]: - out.append({ - "sentence": str(sentence), "intent": str(intent), "slots": {}, - "author": f"teacher:{payload.get('author', 'unknown')}", - }) - return out - - -# --------------------------------------------------------------------------- # -# Deterministic slot extraction (evaluated as slot F1) -# --------------------------------------------------------------------------- # - -_THRESHOLD_RE = re.compile(r">?\s*(\d{2,3})\s*%") -_RANGE_RE = re.compile( - r"(?:between|within)\s+(\d+)\s+(?:and|to|aur)\s+(\d+)" - r"|(\d+)\s+aur\s+(\d+)\s+ke\s+beech", re.I) -_RENAME_RE = re.compile( - r"rename(?:\s+the)?(?:\s+column)?\s+\S+(?:\s+column)?\s+to\s+([A-Za-z_]\w*)" - r"|ko\s+rename\s+karke\s+([A-Za-z_]\w*)" - r"|ko\s+([A-Za-z_]\w*)\s+rename\s+kar", re.I) -_MAP_RE = re.compile(r"\b(\w+)\s+(?:to|ko)\s+(\w+)\b") -_DOMAIN_RE_SLOT = re.compile(r"this (?:is a[n]? |dataset is a[n]? )(.+?)\s+data\s?set", re.I) -_VALUES_RE = re.compile( - r"(?:values are|only be|can only be|values:|ho sakta hai|sirf)\s*:?\s*([^.]+)", - re.I, -) -_VALUE_STOPWORDS = frozenset({"allowed", "hai", "ho", "sakta"}) - - -def extract_slots(sentence: str, intent: str, columns: tuple[str, ...] = ()) -> dict[str, Any]: - """Regex/lexicon slot extraction β€” deterministic, no model involved.""" - slots: dict[str, Any] = {} - lowered = sentence.lower() - for column in columns: - if column.lower() in lowered: - slots["column"] = column - break - if intent == "DOMAIN": - match = _DOMAIN_RE_SLOT.search(sentence) - if match: - slots["domain"] = re.sub(r"\W+", "_", match.group(1).strip().lower()) - if intent == "VALID_FORMAT": - if "email" in lowered: - slots["format"] = "email" - elif "phone" in lowered: - slots["format"] = "phone" - elif "url" in lowered or "website" in lowered: - slots["format"] = "url" - if intent == "LOCALE_FORMAT": - if "indian" in lowered or "india" in lowered: - slots["locale"] = "IN" - if "phone" in lowered: - slots["format"] = "phone" - if intent == "IMPUTE_IF": - match = _THRESHOLD_RE.search(sentence) - if match: - slots["threshold"] = int(match.group(1)) / 100.0 - if intent == "RANGE": - match = _RANGE_RE.search(sentence) - if match: - low, high = (match.group(1), match.group(2)) if match.group(1) else ( - match.group(3), match.group(4)) - slots["min"], slots["max"] = int(low), int(high) - if intent == "ALLOWED_VALUES": - match = _VALUES_RE.search(sentence) - if match: - raw = match.group(1) - values = [ - v.strip() for v in re.split(r",|\bor\b|\band\b|\bya\b|\baur\b", raw) if v.strip() - ] - cleaned = [] - for value in values: - words = [w for w in value.split() if w.lower() not in _VALUE_STOPWORDS] - if words and len(" ".join(words)) < 24: - cleaned.append(" ".join(words)) - if cleaned: - slots["values"] = cleaned - if intent == "RENAME": - match = _RENAME_RE.search(sentence) - if match: - slots["to"] = next(g for g in match.groups() if g) - if intent == "MAP": - skip = {"rows", "karo", "map", "aur"} - pairs = {a: b for a, b in _MAP_RE.findall(sentence) - if a.lower() not in skip and b.lower() not in skip} - if pairs: - slots["mapping"] = pairs - if intent == "DROP_IF" and ("empty" in lowered or "missing" in lowered): - slots["condition"] = "empty" - return slots - - -def _slot_pairs(slots: dict[str, Any]) -> set[tuple[str, str]]: - pairs: set[tuple[str, str]] = set() - for key, value in slots.items(): - if isinstance(value, list): - for item in value: - pairs.add((key, str(item).strip().lower())) - elif isinstance(value, dict): - for k, v in value.items(): - pairs.add((f"{key}.{k}".lower(), str(v).strip().lower())) - else: - pairs.add((key, str(value).strip().lower())) - return pairs - - -def slot_f1(rows: list[dict[str, Any]], predictions: list[str]) -> float: - tp = fp = fn = 0 - for row, predicted_intent in zip(rows, predictions): - gold = _slot_pairs(row.get("slots", {})) - columns = tuple(str(c) for c in row.get("columns", ())) or ( - (str(row["slots"]["column"]),) if row.get("slots", {}).get("column") else ()) - predicted = _slot_pairs(extract_slots(row["sentence"], predicted_intent, columns)) - tp += len(gold & predicted) - fp += len(predicted - gold) - fn += len(gold - predicted) - denominator = 2 * tp + fp + fn - return (2 * tp / denominator) if denominator else 1.0 - - -# --------------------------------------------------------------------------- # -# Conflict detection: PROTECT vs mutating intents on the same column -# --------------------------------------------------------------------------- # - -_MUTATING_INTENTS = frozenset({"VALID_FORMAT", "LOCALE_FORMAT", "IMPUTE_IF", "MAP", "RENAME"}) - - -def conflict_cases() -> list[dict[str, Any]]: - """Sentence pairs on one column; ``conflict`` iff protection meets mutation.""" - return [ - {"sentences": ("Never modify revenue values.", "Revenue must be valid numbers."), - "intents": ("PROTECT", "VALID_FORMAT"), "conflict": True}, - {"sentences": ("status is protected.", "Map A1 to active in status."), - "intents": ("PROTECT", "MAP"), "conflict": True}, - {"sentences": ("cust_id is unique.", "Deduplicate rows by cust_id."), - "intents": ("UNIQUE", "DEDUP_KEY"), "conflict": False}, - {"sentences": ("Emails must be valid.", "Phone numbers are Indian."), - "intents": ("VALID_FORMAT", "LOCALE_FORMAT"), "conflict": False}, - {"sentences": ("age kabhi mat badalna.", - "Missing age should be estimated only if confidence >95%."), - "intents": ("PROTECT", "IMPUTE_IF"), "conflict": True}, - {"sentences": ("Never modify monthly_revenue values.", "monthly_revenue is protected."), - "intents": ("PROTECT", "PROTECT"), "conflict": False}, - ] - - -def conflict_detection_accuracy(head: LinearHead) -> float: - correct = 0 - cases = conflict_cases() - for case in cases: - predicted = [label for label, _ in head.predict(list(case["sentences"]))] - has_protect = "PROTECT" in predicted - has_mutation = any(p in _MUTATING_INTENTS for p in predicted) - detected = has_protect and has_mutation - correct += int(detected == case["conflict"]) - return correct / len(cases) - - -# --------------------------------------------------------------------------- # -# Training / evaluation -# --------------------------------------------------------------------------- # - -def build_dataset(*, seed: int = 0) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """(train, human-verified author-disjoint eval).""" - rows = load_golden_examples() - rows.extend(make_context_sentences(seed=seed)) - rows.extend(teacher_paraphrase_examples()) - train_rows, eval_rows = author_disjoint_split(rows, eval_authors=EVAL_AUTHORS) - # The synthetic corpus is authored and verified in-repo; golden rows are - # maintainer-verified by construction. Eval rows carry that provenance. - for row in eval_rows: - row.setdefault("human_verified", True) - row.setdefault("reviewer", "JWD") - return train_rows, eval_rows - - -def evaluate(head: LinearHead, eval_rows: list[dict[str, Any]]) -> dict[str, Any]: - from ..eval.human_verified import check_all_verified # noqa: PLC0415 - - check_all_verified(eval_rows) - texts = [row["sentence"] for row in eval_rows] - y_true = [row["intent"] for row in eval_rows] - predictions = head.predict(texts) - y_pred = [label for label, _ in predictions] - - exact = sum(1 for t, p in zip(y_true, y_pred) if t == p) / len(y_true) - unknown_predicted = [i for i, p in enumerate(y_pred) if p == "UNKNOWN"] - unknown_precision = ( - sum(1 for i in unknown_predicted if y_true[i] == "UNKNOWN") / len(unknown_predicted) - if unknown_predicted else 1.0 - ) - protect_rows = [i for i, t in enumerate(y_true) if t == "PROTECT"] - protect_recall = ( - sum(1 for i in protect_rows if y_pred[i] == "PROTECT") / len(protect_rows) - if protect_rows else 1.0 - ) - paraphrase_rows = [i for i, row in enumerate(eval_rows) - if str(row.get("author", "")) in EVAL_AUTHORS] - paraphrase_accuracy = ( - sum(1 for i in paraphrase_rows if y_pred[i] == y_true[i]) / len(paraphrase_rows) - if paraphrase_rows else 1.0 - ) - return { - "n_eval": len(eval_rows), - "exact_intent_accuracy": round(exact, 4), - "slot_f1": round(slot_f1(eval_rows, y_pred), 4), - "unknown_precision": round(unknown_precision, 4), - "protected_intent_recall": round(protect_recall, 4), - "author_disjoint_paraphrase_accuracy": round(paraphrase_accuracy, 4), - "conflict_detection_accuracy": round(conflict_detection_accuracy(head), 4), - "per_class_f1": {k: round(v, 4) for k, v in per_class_f1(y_true, y_pred, INTENTS).items()}, - "confusion": confusion(y_true, y_pred, INTENTS), - } - - -def check_gates(metrics: dict[str, Any]) -> list[str]: - failures = [] - if metrics["exact_intent_accuracy"] < GATE_EXACT_ACCURACY: - failures.append( - f"exact accuracy {metrics['exact_intent_accuracy']} < {GATE_EXACT_ACCURACY}") - if metrics["slot_f1"] < GATE_SLOT_F1: - failures.append(f"slot F1 {metrics['slot_f1']} < {GATE_SLOT_F1}") - if metrics["unknown_precision"] < GATE_UNKNOWN_PRECISION: - failures.append( - f"UNKNOWN precision {metrics['unknown_precision']} < {GATE_UNKNOWN_PRECISION}") - if metrics["protected_intent_recall"] < GATE_PROTECT_RECALL: - failures.append( - f"protected recall {metrics['protected_intent_recall']} < {GATE_PROTECT_RECALL}") - return failures - - -def train(*, seed: int = 0, dev: bool = False, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - train_rows, eval_rows = build_dataset(seed=seed) - head = LinearHead.train( - [row["sentence"] for row in train_rows], - [row["intent"] for row in train_rows], - classes=INTENTS, - featurizer=FeaturizerConfig(dim=2048 if dev else 4096), - epochs=150 if dev else 400, - # Protection favors recall over coverage: PROTECT errors are the - # expensive ones, so its loss weight dominates. - class_weights={"PROTECT": 4.0}, - abstain_class="UNKNOWN", - abstain_threshold=0.2, - ) - metrics = evaluate(head, eval_rows) - metrics["n_train"] = len(train_rows) - metrics["gates"] = { - "exact_accuracy_min": GATE_EXACT_ACCURACY, - "slot_f1_min": GATE_SLOT_F1, - "unknown_precision_min": GATE_UNKNOWN_PRECISION, - "protect_recall_min": GATE_PROTECT_RECALL, - "failures": check_gates(metrics), - } - head.save(out / "intent_head.weights.json") - write_json(out / "intent_head.metrics.json", metrics) - write_json(out / "intent_head.confusion.json", metrics["confusion"]) - return metrics - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.train_intent_head") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--dev", action="store_true") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - metrics = train(seed=args.seed, dev=args.dev, out_dir=args.out) - print( - f"intent head: exact={metrics['exact_intent_accuracy']} " - f"slotF1={metrics['slot_f1']} unknownP={metrics['unknown_precision']} " - f"protectR={metrics['protected_intent_recall']} " - f"paraphrase={metrics['author_disjoint_paraphrase_accuracy']}" - ) - failures = metrics["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/train_role_head.py b/training/distill/train_role_head.py deleted file mode 100644 index 37538d02..00000000 --- a/training/distill/train_role_head.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Train the semantic-type role head (``fd-role-head-v1``). - -Inputs: synthetic seed columns, corruptor-derived value dirt, alias-mangled -column-name variants, optional cached teacher ``ColumnRoleLabel`` payloads, -and the human-verified eval split. Output: weights (fp32 + int8), metrics -JSON, a confusion matrix, and β€” when the ``onnx`` package is installed β€” an -ONNX export. - -Release gates (checked with ``--check-gates``):: - - macro-F1 >= 0.90 on the human-verified eval - content-detector contradiction rate <= 1% - adversarial alias accuracy >= 0.85 - -CLI:: - - python -m training.distill.train_role_head [--dev] [--check-gates] -""" - -from __future__ import annotations - -import argparse -import random -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -from ..common import BUILD_DIR, TRAINING_ROOT, write_json -from ..corruptors.context import ECOMMERCE_ALIASES -from ..corruptors.registry import get_corruptor -from ..eval.human_verified import load_verified -from ..seed.synthetic import seed_tables -from .linear import FeaturizerConfig, LinearHead, confusion, macro_f1, per_class_f1 - -OUT_DIR = BUILD_DIR / "role_head" -HUMAN_EVAL_PATH = TRAINING_ROOT / "eval" / "data" / "role_eval_human.jsonl" - -SEMANTIC_TYPES = ( - "email", "phone", "url", "country", "currency_amount", "quantity_with_unit", - "person_name", "address", "city", "postal_code", "national_id", - "category_code", "free_text", "boolean_like", "date_like", "identifier", - "unknown", -) - -#: Ground-truth semantic type for every synthetic seed column. -COLUMN_TYPES: dict[str, str] = { - "cust_id": "identifier", "txn_id": "identifier", - "full_name": "person_name", "email": "email", "phone": "phone", - "address": "address", "city": "city", "state": "city", "country": "country", - "postal_code": "postal_code", "national_id_like": "national_id", - "status": "category_code", "order_status": "category_code", - "category_code": "category_code", - "signup_date": "date_like", "order_date": "date_like", - "website": "url", "newsletter_opt_in": "boolean_like", - "quantity": "quantity_with_unit", "unit_price_inr": "currency_amount", - "monthly_revenue": "currency_amount", "notes": "free_text", -} - -GATE_MACRO_F1 = 0.90 -GATE_CONTRADICTION_RATE = 0.01 -GATE_ALIAS_ACCURACY = 0.85 - - -def example_text(column_name: str, values: list[Any], *, k: int = 8) -> str: - """The exact text form the head scores: name + sample values.""" - samples = " ; ".join(str(v) for v in values[:k]) - return f"col:{column_name} | {samples}" - - -def _alias_variants(column: str, rng: random.Random) -> list[str]: - variants = list(ECOMMERCE_ALIASES.get(column, ())) - parts = column.split("_") - variants.extend(( - "".join(p.title() for p in parts), - " ".join(parts).title(), - "-".join(parts), - column.upper(), - )) - rng.shuffle(variants) - return [v for v in variants if v != column] - - -def build_training_set(*, seed: int = 0, dev: bool = False) -> list[dict[str, Any]]: - """Labeled role examples from seed tables + dirt + alias mangling.""" - rng = random.Random(f"role-head:{seed}") - tables = seed_tables(seed=seed) - dirty_corruptors = [get_corruptor(n) for n in ( - "whitespace_insertion", "casing_change", "email_double_at", - "phone_in_zero_prefix", "date_format_shuffle", "currency_formatting", - )] - examples: list[dict[str, Any]] = [] - windows = 4 if dev else 12 - for _, frame in tables.items(): - for column in frame.columns: - semantic_type = COLUMN_TYPES.get(str(column)) - if semantic_type is None: - continue - values = [v for v in frame[column].tolist() if v is not None] - for w in range(windows): - start = (w * 13) % max(1, len(values) - 8) - window = [str(v) for v in values[start:start + 8]] - # Random dirt on a share of windows: the head must be robust - # to the same corruption the cleaner sees. - if w % 3 == 1 and window: - corruptor = rng.choice(dirty_corruptors) - assert corruptor.fn is not None - window = [ - str(corruptor.fn(v, rng, corruptor.params) or v) for v in window - ] - name = str(column) - if w % 4 == 2: - aliases = _alias_variants(str(column), rng) - if aliases: - name = aliases[0] - examples.append({ - "text": example_text(name, window), - "label": semantic_type, - "column": str(column), - "aliased": name != str(column), - }) - # Explicit unknown class: junk columns with no coherent type. - for i in range(8 if dev else 30): - junk = [str(rng.random())[:6], rng.choice(("x", "yy", "?")), str(rng.randrange(999))] - rng.shuffle(junk) - examples.append({ - "text": example_text(f"misc_{i}", junk * 2), - "label": "unknown", "column": f"misc_{i}", "aliased": False, - }) - return examples - - -def teacher_examples(cache_dir: Path | None = None) -> list[dict[str, Any]]: - """Optional cached ``ColumnRoleLabel`` payloads (training data only).""" - from ..teacher.cache import TeacherCache # noqa: PLC0415 - - out: list[dict[str, Any]] = [] - for entry in TeacherCache(cache_dir).entries(): - for payload in entry.get("payloads", []): - if payload.get("semantic_type") in SEMANTIC_TYPES and "column_name" in payload: - out.append({ - "text": example_text( - str(payload["column_name"]), list(payload.get("masked_samples", [])) - ), - "label": str(payload["semantic_type"]), - "column": str(payload["column_name"]), - "aliased": False, - }) - return out - - -def load_human_eval(path: Path | str = HUMAN_EVAL_PATH) -> list[dict[str, Any]]: - labels = load_verified(path) - for row in labels: - row.setdefault("text", example_text(str(row["column"]), list(row["values"]))) - return labels - - -def contradiction_rate(head: LinearHead, eval_rows: list[dict[str, Any]]) -> float: - """Share of confident head calls that contradict FreshData's detectors.""" - from freshdata.semantic.semantic_types import infer_semantic_type # noqa: PLC0415 - - contradictions = 0 - confident = 0 - for row in eval_rows: - values = row.get("values") - if not values: - continue - prediction, head_confidence = head.predict([row["text"]])[0] - if prediction == "unknown" or head_confidence < 0.7: - continue - detected = infer_semantic_type(str(row["column"]), pd.Series(list(values))) - detected_type = getattr(detected, "semantic_type", "unknown") - detected_confidence = float(getattr(detected, "confidence", 0.0)) - if detected_type in ("unknown", None) or detected_confidence < 0.9: - continue - confident += 1 - if prediction != detected_type: - contradictions += 1 - return contradictions / confident if confident else 0.0 - - -def evaluate(head: LinearHead, eval_rows: list[dict[str, Any]]) -> dict[str, Any]: - texts = [row["text"] for row in eval_rows] - y_true = [row["label"] for row in eval_rows] - predictions = head.predict(texts) - y_pred = [label for label, _ in predictions] - alias_rows = [i for i, row in enumerate(eval_rows) if row.get("aliased")] - alias_accuracy = ( - sum(1 for i in alias_rows if y_pred[i] == y_true[i]) / len(alias_rows) - if alias_rows else 1.0 - ) - return { - "n_eval": len(eval_rows), - "macro_f1": round(macro_f1(y_true, y_pred, head.classes), 4), - "per_class_f1": { - k: round(v, 4) for k, v in per_class_f1(y_true, y_pred, head.classes).items()}, - "abstention_rate": round( - sum(1 for p, _ in predictions if p == "unknown") / len(predictions), 4), - "adversarial_alias_accuracy": round(alias_accuracy, 4), - "contradiction_rate": round(contradiction_rate(head, eval_rows), 4), - "confusion": confusion(y_true, y_pred, head.classes), - } - - -def check_gates(metrics: dict[str, Any]) -> list[str]: - failures = [] - if metrics["macro_f1"] < GATE_MACRO_F1: - failures.append(f"macro-F1 {metrics['macro_f1']} < {GATE_MACRO_F1}") - if metrics["contradiction_rate"] > GATE_CONTRADICTION_RATE: - failures.append( - f"contradiction rate {metrics['contradiction_rate']} > {GATE_CONTRADICTION_RATE}") - if metrics["adversarial_alias_accuracy"] < GATE_ALIAS_ACCURACY: - failures.append( - f"alias accuracy {metrics['adversarial_alias_accuracy']} < {GATE_ALIAS_ACCURACY}") - return failures - - -def train(*, seed: int = 0, dev: bool = False, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - examples = build_training_set(seed=seed, dev=dev) - examples.extend(teacher_examples()) - eval_rows = load_human_eval() - head = LinearHead.train( - [e["text"] for e in examples], - [e["label"] for e in examples], - classes=SEMANTIC_TYPES, - featurizer=FeaturizerConfig(dim=1024 if dev else 2048), - epochs=150 if dev else 400, - abstain_class="unknown", - abstain_threshold=0.35, - ) - metrics = evaluate(head, eval_rows) - metrics["n_train"] = len(examples) - metrics["gates"] = { - "macro_f1_min": GATE_MACRO_F1, - "contradiction_rate_max": GATE_CONTRADICTION_RATE, - "alias_accuracy_min": GATE_ALIAS_ACCURACY, - "failures": check_gates(metrics), - } - head.save(out / "role_head.weights.json") - write_json(out / "role_head.metrics.json", metrics) - write_json(out / "role_head.confusion.json", metrics["confusion"]) - return metrics - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.train_role_head") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--dev", action="store_true") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - metrics = train(seed=args.seed, dev=args.dev, out_dir=args.out) - print( - f"role head: macro-F1={metrics['macro_f1']} " - f"alias={metrics['adversarial_alias_accuracy']} " - f"contradiction={metrics['contradiction_rate']} " - f"abstention={metrics['abstention_rate']}" - ) - failures = metrics["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/eval/__init__.py b/training/eval/__init__.py deleted file mode 100644 index 2c37a078..00000000 --- a/training/eval/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Release-gating evaluation: human-verified labels and CleanBench runs.""" diff --git a/training/eval/data/role_eval_human.jsonl b/training/eval/data/role_eval_human.jsonl deleted file mode 100644 index 0ef38aad..00000000 --- a/training/eval/data/role_eval_human.jsonl +++ /dev/null @@ -1,75 +0,0 @@ -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100089 ; C100090 ; C100091 ; C100092 ; C100093 ; C100094 ; C100095 ; C100096","values":["C100089","C100090","C100091","C100092","C100093","C100094","C100095","C100096"]} -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100090 ; C100091 ; C100092 ; C100093 ; C100094 ; C100095 ; C100096 ; C100097","values":["C100090","C100091","C100092","C100093","C100094","C100095","C100096","C100097"]} -{"aliased":true,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"CustId","text":"col:CustId | C100068 ; C100069 ; C100070 ; C100071 ; C100072 ; C100073 ; C100074 ; C100075","values":["C100068","C100069","C100070","C100071","C100072","C100073","C100074","C100075"]} -{"aliased":false,"column":"full_name","human_verified":true,"label":"person_name","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"full_name","text":"col:full_name | Divya Okelmere ; Meera Quenderi ; Dev Purvatti ; Sneha Quenderi ; Rohan Quenderi ; Amit Sandrelo ; Sneha Purvatti ; Tara Quenderi","values":["Divya Okelmere","Meera Quenderi","Dev Purvatti","Sneha Quenderi","Rohan Quenderi","Amit Sandrelo","Sneha Purvatti","Tara Quenderi"]} -{"aliased":false,"column":"full_name","human_verified":true,"label":"person_name","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"full_name","text":"col:full_name | Isha Voskette ; Divya Purvatti ; Meera Sandrelo ; Asha Okelmere ; Arjun Sandrelo ; Dev Sandrelo ; Meera Mirostra ; Divya Quenderi","values":["Isha Voskette","Divya Purvatti","Meera Sandrelo","Asha Okelmere","Arjun Sandrelo","Dev Sandrelo","Meera Mirostra","Divya Quenderi"]} -{"aliased":true,"column":"full_name","human_verified":true,"label":"person_name","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"FullName","text":"col:FullName | Sneha Voskette ; Sneha Kelvani ; Neha Mirostra ; Meera Voskette ; Meera Vintara ; Meera Quenderi ; Rohan Quenderi ; Neha Talvane","values":["Sneha Voskette","Sneha Kelvani","Neha Mirostra","Meera Voskette","Meera Vintara","Meera Quenderi","Rohan Quenderi","Neha Talvane"]} -{"aliased":false,"column":"email","human_verified":true,"label":"email","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"email","text":"col:email | asha.okelmere111@example.net ; amit.talvane112@mail.example.in ; divya.okelmere113@example.net ; meera.quenderi114@example.net ; dev.purvatti115@mail.example.in ; sneha.quenderi116@example.org ; rohan.quenderi117@mail.example.in ; amit.sandrelo118@example.org","values":["asha.okelmere111@example.net","amit.talvane112@mail.example.in","divya.okelmere113@example.net","meera.quenderi114@example.net","dev.purvatti115@mail.example.in","sneha.quenderi116@example.org","rohan.quenderi117@mail.example.in","amit.sandrelo118@example.org"]} -{"aliased":false,"column":"email","human_verified":true,"label":"email","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"email","text":"col:email | amit.quenderi51@example.com ; kabir.kelvani52@mail.example.in ; meera.vintara53@example.org ; vikram.voskette54@example.org ; neha.vintara55@example.com ; asha.talvane56@mail.example.in ; vikram.okelmere57@example.net ; ravi.sandrelo58@example.com","values":["amit.quenderi51@example.com","kabir.kelvani52@mail.example.in","meera.vintara53@example.org","vikram.voskette54@example.org","neha.vintara55@example.com","asha.talvane56@mail.example.in","vikram.okelmere57@example.net","ravi.sandrelo58@example.com"]} -{"aliased":true,"column":"email","human_verified":true,"label":"email","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"mail","text":"col:mail | vikram.voskette2@example.org ; rohan.voskette3@mail.example.in ; isha.vintara4@example.net ; neha.kelvani5@example.com ; arjun.vintara6@example.org ; isha.purvatti7@mail.example.in ; divya.sandrelo8@mail.example.in ; arjun.vintara9@example.org","values":["vikram.voskette2@example.org","rohan.voskette3@mail.example.in","isha.vintara4@example.net","neha.kelvani5@example.com","arjun.vintara6@example.org","isha.purvatti7@mail.example.in","divya.sandrelo8@mail.example.in","arjun.vintara9@example.org"]} -{"aliased":false,"column":"phone","human_verified":true,"label":"phone","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"phone","text":"col:phone | +916488408307 ; +918786842262 ; +916876254223 ; +916988747208 ; +918405116930 ; +919388522216 ; +917842681057 ; +919690719515","values":["+916488408307","+918786842262","+916876254223","+916988747208","+918405116930","+919388522216","+917842681057","+919690719515"]} -{"aliased":false,"column":"phone","human_verified":true,"label":"phone","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"phone","text":"col:phone | +917841602874 ; +916851673246 ; +917442958335 ; +916466654573 ; +918299889325 ; +917908418834 ; +919547064015 ; +918803250788","values":["+917841602874","+916851673246","+917442958335","+916466654573","+918299889325","+917908418834","+919547064015","+918803250788"]} -{"aliased":true,"column":"phone","human_verified":true,"label":"phone","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Phone","text":"col:Phone | +918718315157 ; +918324435655 ; +916679495576 ; +918873728283 ; +919965647835 ; +919324364004 ; +916362683821 ; +917569920066","values":["+918718315157","+918324435655","+916679495576","+918873728283","+919965647835","+919324364004","+916362683821","+917569920066"]} -{"aliased":false,"column":"address","human_verified":true,"label":"address","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"address","text":"col:address | 110 Palmshade Street ; 124 Riverbend Road ; 208 Palmshade Street ; 165 Harborview Block ; 142 Quarry Lane ; 196 Harborview Block ; 52 Jacaranda Nagar ; 48 Copperleaf Marg","values":["110 Palmshade Street","124 Riverbend Road","208 Palmshade Street","165 Harborview Block","142 Quarry Lane","196 Harborview Block","52 Jacaranda Nagar","48 Copperleaf Marg"]} -{"aliased":false,"column":"address","human_verified":true,"label":"address","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"address","text":"col:address | 211 Copperleaf Marg ; 53 Quarry Lane ; 182 Harborview Block ; 122 Lantern Row ; 23 Quarry Lane ; 3 Riverbend Road ; 53 Jacaranda Nagar ; 140 Copperleaf Marg","values":["211 Copperleaf Marg","53 Quarry Lane","182 Harborview Block","122 Lantern Row","23 Quarry Lane","3 Riverbend Road","53 Jacaranda Nagar","140 Copperleaf Marg"]} -{"aliased":true,"column":"address","human_verified":true,"label":"address","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Address","text":"col:Address | 98 Old Mill Avenue ; 137 Jacaranda Nagar ; 140 Old Mill Avenue ; 6 Lantern Row ; 199 Harborview Block ; 45 Palmshade Street ; 104 Copperleaf Marg ; 224 Riverbend Road","values":["98 Old Mill Avenue","137 Jacaranda Nagar","140 Old Mill Avenue","6 Lantern Row","199 Harborview Block","45 Palmshade Street","104 Copperleaf Marg","224 Riverbend Road"]} -{"aliased":false,"column":"city","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"city","text":"col:city | Hyderabad ; Kolkata ; Pune ; Pune ; Pune ; Kolkata ; Pune ; Delhi","values":["Hyderabad","Kolkata","Pune","Pune","Pune","Kolkata","Pune","Delhi"]} -{"aliased":false,"column":"city","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"city","text":"col:city | Hyderabad ; Kolkata ; Delhi ; Pune ; Hyderabad ; Hyderabad ; Mumbai ; Kolkata","values":["Hyderabad","Kolkata","Delhi","Pune","Hyderabad","Hyderabad","Mumbai","Kolkata"]} -{"aliased":true,"column":"city","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"City","text":"col:City | Mumbai ; Chennai ; Pune ; Chennai ; Delhi ; Pune ; Jaipur ; Kolkata","values":["Mumbai","Chennai","Pune","Chennai","Delhi","Pune","Jaipur","Kolkata"]} -{"aliased":false,"column":"state","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"state","text":"col:state | Maharashtra ; Maharashtra ; Karnataka ; West Bengal ; Karnataka ; West Bengal ; Delhi ; Rajasthan","values":["Maharashtra","Maharashtra","Karnataka","West Bengal","Karnataka","West Bengal","Delhi","Rajasthan"]} -{"aliased":false,"column":"state","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"state","text":"col:state | Telangana ; Tamil Nadu ; Tamil Nadu ; Karnataka ; Tamil Nadu ; Delhi ; Karnataka ; Karnataka","values":["Telangana","Tamil Nadu","Tamil Nadu","Karnataka","Tamil Nadu","Delhi","Karnataka","Karnataka"]} -{"aliased":true,"column":"state","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"State","text":"col:State | Maharashtra ; West Bengal ; Telangana ; Maharashtra ; Tamil Nadu ; Maharashtra ; Maharashtra ; West Bengal","values":["Maharashtra","West Bengal","Telangana","Maharashtra","Tamil Nadu","Maharashtra","Maharashtra","West Bengal"]} -{"aliased":false,"column":"country","human_verified":true,"label":"country","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"country","text":"col:country | Canada ; India ; India ; Canada ; Canada ; India ; India ; India","values":["Canada","India","India","Canada","Canada","India","India","India"]} -{"aliased":false,"column":"country","human_verified":true,"label":"country","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"country","text":"col:country | Austria ; Canada ; Singapore ; India ; Austria ; India ; Singapore ; Canada","values":["Austria","Canada","Singapore","India","Austria","India","Singapore","Canada"]} -{"aliased":true,"column":"country","human_verified":true,"label":"country","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Country","text":"col:Country | Austria ; Austria ; India ; Austria ; India ; India ; Austria ; Australia","values":["Austria","Austria","India","Austria","India","India","Austria","Australia"]} -{"aliased":false,"column":"postal_code","human_verified":true,"label":"postal_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"postal_code","text":"col:postal_code | 444250 ; 374527 ; 579495 ; 802875 ; 438531 ; 171038 ; 521425 ; 264257","values":["444250","374527","579495","802875","438531","171038","521425","264257"]} -{"aliased":false,"column":"postal_code","human_verified":true,"label":"postal_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"postal_code","text":"col:postal_code | 861222 ; 498997 ; 393951 ; 641671 ; 664660 ; 164759 ; 602882 ; 264106","values":["861222","498997","393951","641671","664660","164759","602882","264106"]} -{"aliased":true,"column":"postal_code","human_verified":true,"label":"postal_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"PostalCode","text":"col:PostalCode | 781804 ; 636704 ; 227998 ; 279245 ; 148732 ; 137018 ; 462519 ; 744006","values":["781804","636704","227998","279245","148732","137018","462519","744006"]} -{"aliased":false,"column":"national_id_like","human_verified":true,"label":"national_id","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"national_id_like","text":"col:national_id_like | ZZ57062580X ; ZZ61163162X ; ZZ35065162X ; ZZ41268135X ; ZZ12583457X ; ZZ59822517X ; ZZ19859926X ; ZZ52169088X","values":["ZZ57062580X","ZZ61163162X","ZZ35065162X","ZZ41268135X","ZZ12583457X","ZZ59822517X","ZZ19859926X","ZZ52169088X"]} -{"aliased":false,"column":"national_id_like","human_verified":true,"label":"national_id","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"national_id_like","text":"col:national_id_like | ZZ59306892X ; ZZ51283425X ; ZZ96647252X ; ZZ21251311X ; ZZ61262966X ; ZZ63120743X ; ZZ92832902X ; ZZ53154617X","values":["ZZ59306892X","ZZ51283425X","ZZ96647252X","ZZ21251311X","ZZ61262966X","ZZ63120743X","ZZ92832902X","ZZ53154617X"]} -{"aliased":true,"column":"national_id_like","human_verified":true,"label":"national_id","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"national-id-like","text":"col:national-id-like | ZZ55905267X ; ZZ99286096X ; ZZ26680050X ; ZZ59306892X ; ZZ51283425X ; ZZ96647252X ; ZZ21251311X ; ZZ61262966X","values":["ZZ55905267X","ZZ99286096X","ZZ26680050X","ZZ59306892X","ZZ51283425X","ZZ96647252X","ZZ21251311X","ZZ61262966X"]} -{"aliased":false,"column":"status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"status","text":"col:status | active ; inactive ; active ; pending ; active ; churned ; pending ; active","values":["active","inactive","active","pending","active","churned","pending","active"]} -{"aliased":false,"column":"status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"status","text":"col:status | inactive ; inactive ; active ; inactive ; active ; active ; active ; pending","values":["inactive","inactive","active","inactive","active","active","active","pending"]} -{"aliased":true,"column":"status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Account Status","text":"col:Account Status | churned ; pending ; active ; active ; active ; churned ; active ; inactive","values":["churned","pending","active","active","active","churned","active","inactive"]} -{"aliased":false,"column":"signup_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"signup_date","text":"col:signup_date | 2019-09-28 ; 2022-05-08 ; 2020-04-09 ; 2023-04-01 ; 2022-08-12 ; 2023-03-01 ; 2025-12-08 ; 2023-04-09","values":["2019-09-28","2022-05-08","2020-04-09","2023-04-01","2022-08-12","2023-03-01","2025-12-08","2023-04-09"]} -{"aliased":false,"column":"signup_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"signup_date","text":"col:signup_date | 2021-09-04 ; 2025-01-10 ; 2023-04-22 ; 2022-01-01 ; 2021-09-17 ; 2020-07-17 ; 2022-12-13 ; 2024-12-26","values":["2021-09-04","2025-01-10","2023-04-22","2022-01-01","2021-09-17","2020-07-17","2022-12-13","2024-12-26"]} -{"aliased":true,"column":"signup_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"SignupDate","text":"col:SignupDate | 2023-03-04 ; 2023-09-06 ; 2024-10-26 ; 2019-11-08 ; 2020-10-14 ; 2025-05-28 ; 2023-12-12 ; 2022-07-26","values":["2023-03-04","2023-09-06","2024-10-26","2019-11-08","2020-10-14","2025-05-28","2023-12-12","2022-07-26"]} -{"aliased":false,"column":"website","human_verified":true,"label":"url","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"website","text":"col:website | https://www.voskette63.example.com ; https://www.okelmere64.example.com ; https://www.kelvani65.example.com ; https://www.voskette66.example.com ; https://www.kelvani67.example.com ; https://www.mirostra68.example.com ; https://www.voskette69.example.com ; https://www.vintara70.example.com","values":["https://www.voskette63.example.com","https://www.okelmere64.example.com","https://www.kelvani65.example.com","https://www.voskette66.example.com","https://www.kelvani67.example.com","https://www.mirostra68.example.com","https://www.voskette69.example.com","https://www.vintara70.example.com"]} -{"aliased":false,"column":"website","human_verified":true,"label":"url","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"website","text":"col:website | https://www.sandrelo49.example.com ; https://www.vintara50.example.com ; https://www.quenderi51.example.com ; https://www.kelvani52.example.com ; https://www.vintara53.example.com ; https://www.voskette54.example.com ; https://www.vintara55.example.com ; https://www.talvane56.example.com","values":["https://www.sandrelo49.example.com","https://www.vintara50.example.com","https://www.quenderi51.example.com","https://www.kelvani52.example.com","https://www.vintara53.example.com","https://www.voskette54.example.com","https://www.vintara55.example.com","https://www.talvane56.example.com"]} -{"aliased":true,"column":"website","human_verified":true,"label":"url","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Website","text":"col:Website | https://www.quenderi72.example.com ; https://www.talvane73.example.com ; https://www.vintara74.example.com ; https://www.purvatti75.example.com ; https://www.purvatti76.example.com ; https://www.purvatti77.example.com ; https://www.sandrelo78.example.com ; https://www.okelmere79.example.com","values":["https://www.quenderi72.example.com","https://www.talvane73.example.com","https://www.vintara74.example.com","https://www.purvatti75.example.com","https://www.purvatti76.example.com","https://www.purvatti77.example.com","https://www.sandrelo78.example.com","https://www.okelmere79.example.com"]} -{"aliased":false,"column":"newsletter_opt_in","human_verified":true,"label":"boolean_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"newsletter_opt_in","text":"col:newsletter_opt_in | false ; false ; false ; true ; true ; false ; false ; false","values":["false","false","false","true","true","false","false","false"]} -{"aliased":false,"column":"newsletter_opt_in","human_verified":true,"label":"boolean_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"newsletter_opt_in","text":"col:newsletter_opt_in | false ; false ; true ; true ; false ; false ; false ; true","values":["false","false","true","true","false","false","false","true"]} -{"aliased":true,"column":"newsletter_opt_in","human_verified":true,"label":"boolean_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Newsletter Opt In","text":"col:Newsletter Opt In | true ; true ; true ; false ; false ; true ; true ; true","values":["true","true","true","false","false","true","true","true"]} -{"aliased":false,"column":"txn_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"txn_id","text":"col:txn_id | T9000179 ; T9000180 ; T9000181 ; T9000182 ; T9000183 ; T9000184 ; T9000185 ; T9000186","values":["T9000179","T9000180","T9000181","T9000182","T9000183","T9000184","T9000185","T9000186"]} -{"aliased":false,"column":"txn_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"txn_id","text":"col:txn_id | T9000119 ; T9000120 ; T9000121 ; T9000122 ; T9000123 ; T9000124 ; T9000125 ; T9000126","values":["T9000119","T9000120","T9000121","T9000122","T9000123","T9000124","T9000125","T9000126"]} -{"aliased":true,"column":"txn_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"txn-id","text":"col:txn-id | T9000066 ; T9000067 ; T9000068 ; T9000069 ; T9000070 ; T9000071 ; T9000072 ; T9000073","values":["T9000066","T9000067","T9000068","T9000069","T9000070","T9000071","T9000072","T9000073"]} -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100161 ; C100152 ; C100155 ; C100067 ; C100058 ; C100099 ; C100027 ; C100068","values":["C100161","C100152","C100155","C100067","C100058","C100099","C100027","C100068"]} -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100005 ; C100085 ; C100188 ; C100072 ; C100017 ; C100005 ; C100077 ; C100014","values":["C100005","C100085","C100188","C100072","C100017","C100005","C100077","C100014"]} -{"aliased":true,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"CUST-ID","text":"col:CUST-ID | C100041 ; C100071 ; C100057 ; C100112 ; C100067 ; C100187 ; C100103 ; C100157","values":["C100041","C100071","C100057","C100112","C100067","C100187","C100103","C100157"]} -{"aliased":false,"column":"order_status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_status","text":"col:order_status | shipped ; shipped ; placed ; shipped ; returned ; shipped ; returned ; cancelled","values":["shipped","shipped","placed","shipped","returned","shipped","returned","cancelled"]} -{"aliased":false,"column":"order_status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_status","text":"col:order_status | returned ; delivered ; placed ; placed ; cancelled ; returned ; delivered ; placed","values":["returned","delivered","placed","placed","cancelled","returned","delivered","placed"]} -{"aliased":true,"column":"order_status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"OrderStatus","text":"col:OrderStatus | shipped ; placed ; shipped ; returned ; shipped ; placed ; cancelled ; placed","values":["shipped","placed","shipped","returned","shipped","placed","cancelled","placed"]} -{"aliased":false,"column":"category_code","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"category_code","text":"col:category_code | HM-220 ; EL-200 ; EL-200 ; TY-500 ; TY-500 ; AP-300 ; AP-300 ; TY-500","values":["HM-220","EL-200","EL-200","TY-500","TY-500","AP-300","AP-300","TY-500"]} -{"aliased":false,"column":"category_code","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"category_code","text":"col:category_code | HM-220 ; EL-200 ; EL-200 ; TY-500 ; TY-500 ; AP-300 ; AP-300 ; TY-500","values":["HM-220","EL-200","EL-200","TY-500","TY-500","AP-300","AP-300","TY-500"]} -{"aliased":true,"column":"category_code","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"CategoryCode","text":"col:CategoryCode | GR-110 ; TY-500 ; GR-110 ; EL-200 ; EL-100 ; TY-500 ; EL-200 ; TY-500","values":["GR-110","TY-500","GR-110","EL-200","EL-100","TY-500","EL-200","TY-500"]} -{"aliased":false,"column":"quantity","human_verified":true,"label":"quantity_with_unit","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"quantity","text":"col:quantity | 18 pcs ; 39 g ; 15 g ; 10 g ; 19 kg ; 16 m ; 35 m ; 7 ltr","values":["18 pcs","39 g","15 g","10 g","19 kg","16 m","35 m","7 ltr"]} -{"aliased":false,"column":"quantity","human_verified":true,"label":"quantity_with_unit","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"quantity","text":"col:quantity | 10 ltr ; 9 pcs ; 6 pcs ; 13 pcs ; 15 g ; 24 g ; 38 pcs ; 14 pcs","values":["10 ltr","9 pcs","6 pcs","13 pcs","15 g","24 g","38 pcs","14 pcs"]} -{"aliased":true,"column":"quantity","human_verified":true,"label":"quantity_with_unit","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Qty.","text":"col:Qty. | 34 pcs ; 14 kg ; 34 kg ; 11 g ; 32 ltr ; 17 pcs ; 18 pcs ; 7 pcs","values":["34 pcs","14 kg","34 kg","11 g","32 ltr","17 pcs","18 pcs","7 pcs"]} -{"aliased":false,"column":"unit_price_inr","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"unit_price_inr","text":"col:unit_price_inr | β‚Ή448.63 ; β‚Ή76,161.96 ; β‚Ή19,838.82 ; β‚Ή51,202.58 ; β‚Ή82,650.09 ; β‚Ή9,260.59 ; β‚Ή56,618.59 ; β‚Ή65,028.71","values":["β‚Ή448.63","β‚Ή76,161.96","β‚Ή19,838.82","β‚Ή51,202.58","β‚Ή82,650.09","β‚Ή9,260.59","β‚Ή56,618.59","β‚Ή65,028.71"]} -{"aliased":false,"column":"unit_price_inr","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"unit_price_inr","text":"col:unit_price_inr | β‚Ή11,581.01 ; β‚Ή89,151.05 ; β‚Ή12,161.62 ; β‚Ή36,571.65 ; β‚Ή53,292.91 ; β‚Ή79,740.98 ; β‚Ή52,258.05 ; β‚Ή20,675.74","values":["β‚Ή11,581.01","β‚Ή89,151.05","β‚Ή12,161.62","β‚Ή36,571.65","β‚Ή53,292.91","β‚Ή79,740.98","β‚Ή52,258.05","β‚Ή20,675.74"]} -{"aliased":true,"column":"unit_price_inr","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"UNIT_PRICE_INR","text":"col:UNIT_PRICE_INR | β‚Ή52,718.03 ; β‚Ή23,456.81 ; β‚Ή85,154.75 ; β‚Ή82,423.09 ; β‚Ή12,683.40 ; β‚Ή5,878.17 ; β‚Ή2,049.11 ; β‚Ή77,074.11","values":["β‚Ή52,718.03","β‚Ή23,456.81","β‚Ή85,154.75","β‚Ή82,423.09","β‚Ή12,683.40","β‚Ή5,878.17","β‚Ή2,049.11","β‚Ή77,074.11"]} -{"aliased":false,"column":"monthly_revenue","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"monthly_revenue","text":"col:monthly_revenue | 47031.3 ; 558559.75 ; 1533264.34 ; 1672002.96 ; 1184410.5 ; 2334413.88 ; 6153.76 ; 698430.48","values":["47031.3","558559.75","1533264.34","1672002.96","1184410.5","2334413.88","6153.76","698430.48"]} -{"aliased":false,"column":"monthly_revenue","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"monthly_revenue","text":"col:monthly_revenue | 487412.1 ; 2996829.81 ; 1013014.8 ; 540712.15 ; 289854.0 ; 451097.92 ; 828833.95 ; 1145331.33","values":["487412.1","2996829.81","1013014.8","540712.15","289854.0","451097.92","828833.95","1145331.33"]} -{"aliased":true,"column":"monthly_revenue","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"MonthlyRevenue","text":"col:MonthlyRevenue | 130326.79 ; 498788.4 ; 1155576.18 ; 1247315.76 ; 2002511.16 ; 1040075.1 ; 761166.0 ; 1832754.44","values":["130326.79","498788.4","1155576.18","1247315.76","2002511.16","1040075.1","761166.0","1832754.44"]} -{"aliased":false,"column":"order_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_date","text":"col:order_date | 2023-06-21 ; 2023-02-05 ; 2025-11-08 ; 2025-03-25 ; 2022-07-20 ; 2024-03-14 ; 2023-08-24 ; 2025-08-11","values":["2023-06-21","2023-02-05","2025-11-08","2025-03-25","2022-07-20","2024-03-14","2023-08-24","2025-08-11"]} -{"aliased":false,"column":"order_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_date","text":"col:order_date | 2022-12-28 ; 2022-01-14 ; 2023-02-06 ; 2025-10-18 ; 2022-07-18 ; 2023-08-17 ; 2022-01-06 ; 2024-07-23","values":["2022-12-28","2022-01-14","2023-02-06","2025-10-18","2022-07-18","2023-08-17","2022-01-06","2024-07-23"]} -{"aliased":true,"column":"order_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"ord_dt","text":"col:ord_dt | 2023-11-17 ; 2023-07-02 ; 2022-03-01 ; 2023-02-06 ; 2025-05-14 ; 2022-11-09 ; 2022-02-03 ; 2025-08-26","values":["2023-11-17","2023-07-02","2022-03-01","2023-02-06","2025-05-14","2022-11-09","2022-02-03","2025-08-26"]} -{"aliased":false,"column":"notes","human_verified":true,"label":"free_text","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"notes","text":"col:notes | gift wrap requested ; call before delivery ; left at door ; call before delivery ; gift wrap requested ; priority shipping ; call before delivery ; priority shipping","values":["gift wrap requested","call before delivery","left at door","call before delivery","gift wrap requested","priority shipping","call before delivery","priority shipping"]} -{"aliased":false,"column":"notes","human_verified":true,"label":"free_text","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"notes","text":"col:notes | call before delivery ; gift wrap requested ; call before delivery ; repeat customer ; priority shipping ; repeat customer ; left at door ; repeat customer","values":["call before delivery","gift wrap requested","call before delivery","repeat customer","priority shipping","repeat customer","left at door","repeat customer"]} -{"aliased":true,"column":"notes","human_verified":true,"label":"free_text","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Notes","text":"col:Notes | left at door ; repeat customer ; call before delivery ; left at door ; priority shipping ; repeat customer ; priority shipping ; repeat customer","values":["left at door","repeat customer","call before delivery","left at door","priority shipping","repeat customer","priority shipping","repeat customer"]} -{"aliased":false,"column":"misc_0","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_0","text":"col:misc_0 | 0.1827 ; ? ; 499 ; 0.1827 ; ? ; 499","values":["0.1827","?","499","0.1827","?","499"]} -{"aliased":false,"column":"misc_1","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_1","text":"col:misc_1 | 0.0769 ; yy ; 122 ; 0.0769 ; yy ; 122","values":["0.0769","yy","122","0.0769","yy","122"]} -{"aliased":false,"column":"misc_2","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_2","text":"col:misc_2 | 0.9693 ; x ; 831 ; 0.9693 ; x ; 831","values":["0.9693","x","831","0.9693","x","831"]} -{"aliased":false,"column":"misc_3","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_3","text":"col:misc_3 | 0.2841 ; x ; 174 ; 0.2841 ; x ; 174","values":["0.2841","x","174","0.2841","x","174"]} -{"aliased":false,"column":"misc_4","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_4","text":"col:misc_4 | 0.0144 ; x ; 908 ; 0.0144 ; x ; 908","values":["0.0144","x","908","0.0144","x","908"]} -{"aliased":false,"column":"misc_5","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_5","text":"col:misc_5 | 0.9682 ; x ; 878 ; 0.9682 ; x ; 878","values":["0.9682","x","878","0.9682","x","878"]} diff --git a/training/eval/human_verified.py b/training/eval/human_verified.py deleted file mode 100644 index bdf99b3a..00000000 --- a/training/eval/human_verified.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Human-verified release-gating eval labels. - -Release gates may only be computed on labels where a human reviewer signed -off β€” teacher labels alone never gate a release. Verified label files are -JSONL with, per row, the payload plus ``human_verified``, ``reviewer`` and -``reviewed_at``; :func:`load_verified` refuses anything less. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from ..common import TRAINING_ROOT, read_jsonl, utc_now_iso, write_jsonl - -DATA_DIR = TRAINING_ROOT / "eval" / "data" - - -class VerificationError(ValueError): - """A release-gating label set is not fully human-verified.""" - - -def verify_labels( - labels: list[dict[str, Any]], *, reviewer: str, reviewed: list[bool] | None = None -) -> list[dict[str, Any]]: - """Stamp labels as human-verified (used when a reviewer signs a batch).""" - if not reviewer.strip(): - raise VerificationError("reviewer id/initials are required") - flags = reviewed if reviewed is not None else [True] * len(labels) - if len(flags) != len(labels): - raise VerificationError("reviewed flags must match label count") - now = utc_now_iso() - out = [] - for label, flag in zip(labels, flags): - stamped = dict(label) - stamped["human_verified"] = bool(flag) - stamped["reviewer"] = reviewer.strip() - stamped["reviewed_at"] = now - out.append(stamped) - return out - - -def check_all_verified(labels: list[dict[str, Any]]) -> None: - """Raise unless every label is human-verified with a reviewer recorded.""" - if not labels: - raise VerificationError("release-gating eval set is empty") - bad = [ - i for i, label in enumerate(labels) - if label.get("human_verified") is not True - or not str(label.get("reviewer", "")).strip() - ] - if bad: - raise VerificationError( - f"{len(bad)}/{len(labels)} release-gating labels are not human-verified " - f"(first offenders: {bad[:5]}); teacher output alone cannot gate a release" - ) - - -def load_verified(path: Path | str) -> list[dict[str, Any]]: - """Load a release-gating label file, enforcing 100% human verification.""" - labels = read_jsonl(path) - check_all_verified(labels) - return labels - - -def save_verified(path: Path | str, labels: list[dict[str, Any]]) -> Path: - check_all_verified(labels) - return write_jsonl(path, labels) diff --git a/training/model_cards/calib-v1.md b/training/model_cards/calib-v1.md deleted file mode 100644 index 94dbe67a..00000000 --- a/training/model_cards/calib-v1.md +++ /dev/null @@ -1,64 +0,0 @@ -# Model card β€” `calib-v1` - -## 1. Purpose -Isotonic confidence-calibration tables mapping raw semantic-proposal scores -to calibrated probabilities of a correct repair, per (backend, issue -family). Plain JSON β€” not a neural model. - -## 2. Non-goals -- Does not create or veto proposals; it only rescales confidence. -- Never makes the deterministic protected-column checks probabilistic. - -## 3. Runtime behavior -Loaded via the Phase-3 model registry (or the packaged conservative -default). If the table is missing or corrupt the runtime passes raw scores -through and reports `calibration_version="uncalibrated"`. Embedding-backend -confidences are always capped below 1.0. - -## 4. Training data sources -CleanBench T1–T4 fixture runs with corruption-metadata ground truth (never -teacher guesses); the evaluation split is held out from fitting. - -## 5. License summary -Apache-2.0; training inputs are in-repo synthetic fixtures. - -## 6. PII policy -Fixtures are fully synthetic; no real PII enters calibration training. - -## 7. Teacher-model usage -None. Calibration is fitted exclusively on machine-verified outcomes. - -## 8. Human-review process -Release gates are computed on machine-verified outcomes over human-reviewed -fixture definitions; the CleanBench release run re-checks all gates. - -## 9. Evaluation metrics -`eval_metrics.json`: ECE and precision@0.95 on the held-out split. -Gates: ECE ≀ 0.03, P@0.95 β‰₯ 0.99, protected-column violation rate = 0, -false modification rate ≀ 0.1% (the last two via CleanBench). - -## 10. Known limitations -Curves are fitted on synthetic benchmark distributions; heavily -out-of-distribution data reverts to conservative behavior (embedding cap, -suggest-only below thresholds). - -## 11. Failure modes -Missing/corrupt table β†’ uncalibrated passthrough (safe); unseen -(backend, issue) pairs fall back to the per-backend `*` curve or identity. - -## 12. Protected-column guarantee -Calibration never touches the deterministic byte-identity guard. - -## 13. Calibration version -`version` field inside `calibration.json` (this artifact). - -## 14. Artifact hash -See `manifest.json`. - -## 15. FreshData version compatibility -`freshdata_min_version` in `manifest.json`; enforced by the model registry. - ---- -**Explicit statements**: no generative repair model; no runtime LLM; no -automatic model download during cleaning; model output is evidence, not -authority; the protected-column guard remains absolute. diff --git a/training/model_cards/fd-col-encoder-v1.md b/training/model_cards/fd-col-encoder-v1.md deleted file mode 100644 index 05f54817..00000000 --- a/training/model_cards/fd-col-encoder-v1.md +++ /dev/null @@ -1,82 +0,0 @@ -# Model card β€” `fd-col-encoder-v1` (+ bundled role head) - -## 1. Purpose -Local column/value encoder for FreshData's optional embedding backend, with a -bundled semantic-type **role head** used as *additional evidence* for column -role inference. It rescues reference-value repairs the deterministic experts -cannot reach (distance-2 typos of allowed values) and scores column -semantic types. - -## 2. Non-goals -- Not a generative model; it never writes values. -- Not a general text embedder; scoped to tabular column names + cell values. -- Not a replacement for FreshData's deterministic experts or detectors. - -## 3. Runtime behavior -Loaded only when the user installs the `[semantic]` extra **and** explicitly -pulls the model (`freshdata models pull fd-col-encoder-v1`) or drops files -into `FRESHDATA_MODEL_DIR`. CPU-only, offline, deterministic. **Model output -is evidence, not authority**: every proposal goes through the same policy -gate, calibration, and audit trail as deterministic proposals, and ambiguous -merges are never auto-applied. - -## 4. Training data sources -Fully synthetic PII-shaped seed corpus (`training/seed/synthetic/`), -hand-authored fixtures, and corruptor-derived labeled pairs -(`training/seed/registry.json`). No real PII, no customer data, no scraped -private data, no production logs. - -## 5. License summary -Model artifact: Apache-2.0. All training data: CC0-1.0 / Apache-2.0 in-repo -sources (see `training/seed/LICENSES.md`). - -## 6. PII policy -Training data contains only synthetic, invented identities tagged -`synthetic=true`. Values sent to any development-time teacher model are -masked first (`training/teacher/tasks.py:mask_pii`). - -## 7. Teacher-model usage -Teacher models may contribute column-role labels and realism direction at -development time only (cached, compliance-gated, β‰₯5% human-reviewed). They -never generate messy/clean pairs and never gate a release. - -## 8. Human-review process -β‰₯5% of teacher batches human-reviewed; >3% disagreement forces full batch -review; the release-gating eval split (`training/eval/data/`) is 100% -human-verified with reviewer identity and timestamps recorded. - -## 9. Evaluation metrics -See `eval_metrics.json` in this artifact (macro-F1, per-class F1, abstention -rate, content-detector contradiction rate, adversarial alias accuracy). -Release gates: macro-F1 β‰₯ 0.90, contradiction rate ≀ 1%, alias accuracy β‰₯ 0.85. - -## 10. Known limitations -- Eval corpus is synthetic; real-world column vocabularies are broader. -- English + Indian-English column headers only. -- Role head abstains (`unknown`) below its confidence threshold rather than - guessing; downstream code must handle abstention. - -## 11. Failure modes -Opaque codes and short low-support columns fall to `unknown`; heavily mixed -columns may be typed by their majority shape; adversarial headers reduce -confidence rather than flipping labels (gated). - -## 12. Protected-column guarantee -Unaffected by this model. The byte-identity protected-column guard is -deterministic, runs regardless of any model, and CleanBench gates its -violation rate at exactly zero. - -## 13. Calibration version -Scores are calibrated through `calib-v1` isotonic tables; without them the -runtime reports `calibration_version="uncalibrated"` and stays conservative. - -## 14. Artifact hash -See `manifest.json` (`sha256` per file, top-level artifact hash). - -## 15. FreshData version compatibility -`freshdata_min_version` in `manifest.json`; enforced by the model registry. - ---- -**Explicit statements**: no generative repair model; no runtime LLM; no -automatic model download during cleaning; model output is evidence, not -authority; the protected-column guard remains absolute. diff --git a/training/model_cards/fd-intent-v1.md b/training/model_cards/fd-intent-v1.md deleted file mode 100644 index a9975754..00000000 --- a/training/model_cards/fd-intent-v1.md +++ /dev/null @@ -1,78 +0,0 @@ -# Model card β€” `fd-intent-v1` - -## 1. Purpose -Small intent classifier for natural-language context sentences (13 intents: -DOMAIN, UNIQUE, VALID_FORMAT, LOCALE_FORMAT, PROTECT, IMPUTE_IF, -ALLOWED_VALUES, RANGE, DEDUP_KEY, DROP_IF, RENAME, MAP, UNKNOWN). It supplies -*optional evidence* to the deterministic context parser, e.g. flagging a -sentence the tier-0 patterns missed. - -## 2. Non-goals -- Never silently replaces deterministic parser behavior: a policy rule is - only created by the deterministic compiler. -- Not a chat model, not a general NLU system, no slot *generation* β€” slots - are extracted by deterministic regex/lexicon code. - -## 3. Runtime behavior -Optional, offline, CPU-only, explicit opt-in via the model registry. In -strict mode, unresolved or unparsed context is still surfaced to the user -exactly as before. Protection detection favors recall over coverage: -the PROTECT class is trained and gated at recall β‰₯ 0.99. - -## 4. Training data sources -Phase-1 golden context corpus (this repository, Apache-2.0), synthetic -paraphrase templates including the Indian-English/Hinglish set -(`training/seed/synthetic/generators.py`), corruptor context variants, and -optionally teacher paraphrases (cached, reviewed, training-only). - -## 5. License summary -Artifact: Apache-2.0. Data: in-repo Apache-2.0 + CC0 synthetic sources. - -## 6. PII policy -Context sentences reference column names, never cell values; the corpus is -synthetic/hand-authored. No real PII. - -## 7. Teacher-model usage -Paraphrase generation and ambiguity adjudication only, development-time -only, compliance-gated and cached. Teacher labels never gate releases. - -## 8. Human-review process -Release-gating eval labels are 100% human-verified; the eval split is -author-disjoint (held-out authors `t2`, `hinglish_b`) so paraphrase -generalization is measured, not memorization. - -## 9. Evaluation metrics -See `eval_metrics.json`: exact intent accuracy, slot F1, UNKNOWN precision, -protected-intent recall, conflict-detection accuracy, author-disjoint -paraphrase accuracy. Gates: exact β‰₯ 0.92, slot F1 β‰₯ 0.90, UNKNOWN precision -β‰₯ 0.95, PROTECT recall β‰₯ 0.99. - -## 10. Known limitations -- Trained on template-derived paraphrases; entirely novel phrasing may - abstain to UNKNOWN (by design). -- English and Hinglish only. -- Sentences with multiple intents are classified by the dominant one; the - deterministic parser remains responsible for multi-intent splitting. - -## 11. Failure modes -Low-confidence predictions abstain to UNKNOWN rather than inventing -constraints; a missed PROTECT is the gated worst case (recall β‰₯ 0.99); -conflicting instructions are surfaced, never resolved silently. - -## 12. Protected-column guarantee -The model can only *add* protection evidence, never remove it. The runtime -byte-identity guard is deterministic and absolute. - -## 13. Calibration version -Head confidences feed the same `calib-v1` scoring path as other evidence. - -## 14. Artifact hash -See `manifest.json`. - -## 15. FreshData version compatibility -`freshdata_min_version` in `manifest.json`; enforced by the model registry. - ---- -**Explicit statements**: no generative repair model; no runtime LLM; no -automatic model download during cleaning; model output is evidence, not -authority; the protected-column guard remains absolute. diff --git a/training/seed/LICENSES.md b/training/seed/LICENSES.md deleted file mode 100644 index aaee4de5..00000000 --- a/training/seed/LICENSES.md +++ /dev/null @@ -1,33 +0,0 @@ -# Seed corpus license ledger - -Every seed source used for training must appear in `registry.json` with an -explicit license, attribution, `allowed_for_training`, and a resolved PII risk. -`python -m training.datasets.validators --check-licenses` enforces this and is -part of `make training-release-artifacts`. - -## Current sources - -| source_id | license | PII risk | commercial use | notes | -|---|---|---|---|---| -| `synthetic_core_v1` | CC0-1.0 (generated in-repo) | synthetic | yes | Fully synthetic; no real people, no production data. | -| `handauthored_ecommerce_v1` | CC0-1.0 | none | yes | Hand-written fixture rows. | -| `context_golden_phase1_v1` | Apache-2.0 | none | yes | Phase-1 golden context corpus from this repository. | - -## Allowed license classes - -- Public domain / CC0 / PDDL -- CC-BY-4.0 (attribution recorded in the registry) -- Permissive software licenses (Apache-2.0, MIT, BSD) for in-repo fixtures -- OGL / government open data with explicit commercial-use permission -- Fully synthetic data generated in this repository - -## Never allowed - -- Unclear-license or license-missing datasets -- Share-alike licenses (CC-BY-SA, ODbL, GPL-family data) **unless** a recorded - legal review approves the specific use (`legal_review.approved: true` on the - registry entry) -- Scraped private data, customer data, production logs -- Any dataset containing real PII - -License texts are vendored under `training/seed/licenses/`. diff --git a/training/seed/README.md b/training/seed/README.md deleted file mode 100644 index 6e435f5d..00000000 --- a/training/seed/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Seed corpus - -Inputs the training pipeline is allowed to learn from. Governance: - -- **`registry.json`** β€” every source, with license, attribution, explicit - `allowed_for_training`, and resolved `pii_risk` (`none | synthetic | - review_required`). A `review_required` entry blocks the release pipeline - until it carries an approving `legal_review` record. -- **`LICENSES.md`** β€” human-readable ledger; license texts in `licenses/`. -- **`sources/`** β€” small vendored files (hand-authored fixtures only). -- **`synthetic/`** β€” deterministic generators for PII-shaped fake data - (emails, Indian phone numbers, names, addresses, postal codes, IDs, - statuses, dates, revenue/currency strings, quantities with units, - city/state/country). No real people, no production data; every record is - tagged `synthetic=true`. - -Validate with: - -```bash -python -m training.datasets.validators --check-licenses -``` - -Never add: unclear-license data, share-alike data without recorded legal -approval, scraped private data, customer data, production logs, or real PII. diff --git a/training/seed/__init__.py b/training/seed/__init__.py deleted file mode 100644 index 3ea5c0f7..00000000 --- a/training/seed/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Seed corpus: registry-governed sources plus fully synthetic generators.""" diff --git a/training/seed/licenses/apache-2.0-notice.txt b/training/seed/licenses/apache-2.0-notice.txt deleted file mode 100644 index e18c6276..00000000 --- a/training/seed/licenses/apache-2.0-notice.txt +++ /dev/null @@ -1,7 +0,0 @@ -Apache License 2.0 (notice) - -This material is part of the FreshData repository and is licensed under the -Apache License, Version 2.0 β€” the same license as the project. See the -top-level LICENSE file for the full text. - -Full text: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/training/seed/licenses/cc-by-4.0.txt b/training/seed/licenses/cc-by-4.0.txt deleted file mode 100644 index 2fc2d7d8..00000000 --- a/training/seed/licenses/cc-by-4.0.txt +++ /dev/null @@ -1,7 +0,0 @@ -Creative Commons Attribution 4.0 International (summary) - -You are free to share and adapt the material for any purpose, even -commercially, provided you give appropriate credit, provide a link to the -license, and indicate if changes were made. - -Full text: https://creativecommons.org/licenses/by/4.0/legalcode diff --git a/training/seed/licenses/cc0-1.0.txt b/training/seed/licenses/cc0-1.0.txt deleted file mode 100644 index a1231034..00000000 --- a/training/seed/licenses/cc0-1.0.txt +++ /dev/null @@ -1,9 +0,0 @@ -CC0 1.0 Universal (summary) - -The person who associated a work with this deed has dedicated the work to the -public domain by waiving all of his or her rights to the work worldwide under -copyright law, including all related and neighboring rights, to the extent -allowed by law. You can copy, modify, distribute and perform the work, even -for commercial purposes, all without asking permission. - -Full text: https://creativecommons.org/publicdomain/zero/1.0/legalcode diff --git a/training/seed/registry.json b/training/seed/registry.json deleted file mode 100644 index 455370b5..00000000 --- a/training/seed/registry.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "registry_version": "1", - "sources": [ - { - "source_id": "synthetic_core_v1", - "name": "FreshData synthetic PII-shaped core corpus", - "url": "training/seed/synthetic/generators.py", - "license": "CC0-1.0", - "license_text_path": "licenses/cc0-1.0.txt", - "attribution": "Generated in-repo by training/seed/synthetic/generators.py; no external data.", - "allowed_for_training": true, - "pii_risk": "synthetic", - "notes": "Fully synthetic: invented names/addresses, fake emails on example.* domains, fake Indian phone numbers. Deterministic under seed; every record carries synthetic=true." - }, - { - "source_id": "handauthored_ecommerce_v1", - "name": "Hand-authored ecommerce mini fixtures", - "url": "training/seed/sources/handauthored_ecommerce_v1.csv", - "license": "CC0-1.0", - "license_text_path": "licenses/cc0-1.0.txt", - "attribution": "Hand-written by FreshData maintainers for corruptor and parser fixtures.", - "allowed_for_training": true, - "pii_risk": "none", - "notes": "24 rows of invented ecommerce records mirroring the CleanBench base frame; no real people." - }, - { - "source_id": "context_golden_phase1_v1", - "name": "Phase-1 golden context corpus", - "url": "tests/context/golden/", - "license": "Apache-2.0", - "license_text_path": "licenses/apache-2.0-notice.txt", - "attribution": "FreshData repository test corpus (same license as the project).", - "allowed_for_training": true, - "pii_risk": "none", - "notes": "English context sentences plus compiled golden policies; used as intent-head ground truth." - } - ] -} diff --git a/training/seed/sources/handauthored_ecommerce_v1.csv b/training/seed/sources/handauthored_ecommerce_v1.csv deleted file mode 100644 index 044fb9e1..00000000 --- a/training/seed/sources/handauthored_ecommerce_v1.csv +++ /dev/null @@ -1,25 +0,0 @@ -cust_id,email,phone,age,monthly_revenue,status,city,country,synthetic -C900001,asha.voskette1@example.com,+919876543210,34,1250.00,active,Mumbai,India,true -C900002,ravi.talvane2@example.org,+919812345670,28,980.50,pending,Pune,India,true -C900003,neha.quenderi3@example.net,+919900112233,41,15000.00,inactive,Bengaluru,India,true -C900004,vikram.mirostra4@example.com,+919811223344,52,720.25,active,Chennai,India,true -C900005,priya.kelvani5@example.org,+919922334455,23,310.00,pending,Hyderabad,India,true -C900006,arjun.purvatti6@example.net,+919933445566,37,4400.75,active,Delhi,India,true -C900007,meera.sandrelo7@example.com,+919944556677,29,860.00,inactive,Kolkata,India,true -C900008,kiran.vintara8@example.org,+919955667788,45,2300.10,active,Jaipur,India,true -C900009,divya.okelmere9@example.net,+919966778899,31,199.99,pending,Mumbai,India,true -C900010,rohan.ramblewood10@example.com,+919977889900,26,5600.00,active,Pune,India,true -C900011,sneha.voskette11@example.org,+919988990011,38,75.50,churned,Bengaluru,India,true -C900012,amit.talvane12@example.net,+919899001122,33,8100.00,active,Chennai,India,true -C900013,tara.quenderi13@example.com,+919810101010,47,640.00,inactive,Hyderabad,India,true -C900014,dev.mirostra14@example.org,+919820202020,22,1550.25,pending,Delhi,India,true -C900015,isha.kelvani15@example.net,+919830303030,36,920.80,active,Kolkata,India,true -C900016,kabir.purvatti16@example.com,+919840404040,58,12000.00,active,Jaipur,India,true -C900017,asha.sandrelo17@example.org,+919850505050,27,450.00,pending,Mumbai,India,true -C900018,ravi.vintara18@example.net,+919860606060,44,3300.33,inactive,Pune,India,true -C900019,neha.okelmere19@example.com,+919870707070,30,780.00,active,Bengaluru,India,true -C900020,vikram.ramblewood20@example.org,+919880808080,39,2100.00,active,Chennai,India,true -C900021,priya.voskette21@example.net,+919890909090,25,95.00,churned,Hyderabad,India,true -C900022,arjun.talvane22@example.com,+919801020304,49,6700.45,active,Delhi,India,true -C900023,meera.quenderi23@example.org,+919802030405,35,1800.00,pending,Kolkata,India,true -C900024,kiran.mirostra24@example.net,+919803040506,42,990.90,active,Jaipur,India,true diff --git a/training/seed/synthetic/__init__.py b/training/seed/synthetic/__init__.py deleted file mode 100644 index 87a4fd12..00000000 --- a/training/seed/synthetic/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Synthetic PII-shaped data generators (no real people, no real PII).""" - -from .generators import ( - SYNTHETIC_SOURCE_ID, - make_context_sentences, - make_customers, - make_transactions, - seed_tables, -) - -__all__ = [ - "SYNTHETIC_SOURCE_ID", - "make_context_sentences", - "make_customers", - "make_transactions", - "seed_tables", -] diff --git a/training/seed/synthetic/generators.py b/training/seed/synthetic/generators.py deleted file mode 100644 index 21d028d8..00000000 --- a/training/seed/synthetic/generators.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Deterministic synthetic PII-shaped data. - -All identity-shaped values are assembled from invented token lists β€” none of -the names, addresses, or numbers are derived from real people, customer data, -or production logs. Every generator takes an explicit ``seed`` and produces -byte-identical output for the same seed, and every record carries -``synthetic=True`` metadata (also mirrored in ``DataFrame.attrs``). -""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -SYNTHETIC_SOURCE_ID = "synthetic_core_v1" - -# Invented tokens only. Common Indian-flavored given names are generic -# dictionary words in this context; surnames and streets are deliberately -# fictional compounds so no full name can collide with a real identity record. -_GIVEN = ( - "asha", "ravi", "neha", "vikram", "priya", "arjun", "meera", "kiran", - "divya", "rohan", "sneha", "amit", "tara", "dev", "isha", "kabir", -) -_SURNAME = ( - "Voskette", "Ramblewood", "Quenderi", "Talvane", "Mirostra", "Kelvani", - "Purvatti", "Sandrelo", "Vintara", "Okelmere", -) -_STREET = ( - "Lantern Row", "Copperleaf Marg", "Quarry Lane", "Palmshade Street", - "Riverbend Road", "Old Mill Avenue", "Jacaranda Nagar", "Harborview Block", -) -_CITY_STATE = ( - ("Mumbai", "Maharashtra"), ("Pune", "Maharashtra"), ("Bengaluru", "Karnataka"), - ("Chennai", "Tamil Nadu"), ("Hyderabad", "Telangana"), ("Delhi", "Delhi"), - ("Kolkata", "West Bengal"), ("Jaipur", "Rajasthan"), -) -_COUNTRY = ("India", "India", "India", "Australia", "Austria", "Canada", "Singapore") -_EMAIL_DOMAINS = ("example.com", "example.org", "example.net", "mail.example.in") -_STATUS = ("active", "inactive", "pending", "churned") -_UNITS = ("kg", "g", "pcs", "ltr", "m") -_CATEGORY_CODES = ("EL-100", "EL-200", "AP-300", "GR-110", "HM-220", "TY-500") - - -def _rng(seed: int) -> random.Random: - return random.Random(("freshdata-synthetic", seed).__repr__()) - - -def _mark_synthetic(df: pd.DataFrame) -> pd.DataFrame: - df.attrs["synthetic"] = True - df.attrs["source_id"] = SYNTHETIC_SOURCE_ID - return df - - -def make_customers(n: int = 200, seed: int = 0) -> pd.DataFrame: - """Customer-shaped frame: ids, names, emails, Indian phones, addresses.""" - rng = _rng(seed) - rows: list[dict[str, Any]] = [] - for i in range(n): - given = rng.choice(_GIVEN) - surname = rng.choice(_SURNAME) - city, state = rng.choice(_CITY_STATE) - # Indian mobile numbers start 6-9; keep the canonical +91 form as truth. - ten_digit = f"{rng.randint(6, 9)}{rng.randrange(10**8, 10**9):09d}" - rows.append({ - "cust_id": f"C{100000 + i}", - "full_name": f"{given.title()} {surname}", - "email": f"{given}.{surname.lower()}{i}@{rng.choice(_EMAIL_DOMAINS)}", - "phone": f"+91{ten_digit}", - "address": f"{rng.randint(1, 240)} {rng.choice(_STREET)}", - "city": city, - "state": state, - "country": rng.choice(_COUNTRY), - "postal_code": f"{rng.randint(110001, 899999)}", - "national_id_like": f"ZZ{rng.randrange(10**7, 10**8)}X", - "status": rng.choice(_STATUS), - "signup_date": ( - f"{rng.randint(2019, 2025)}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" - ), - "website": f"https://www.{surname.lower()}{i}.example.com", - "newsletter_opt_in": rng.choice(("true", "false")), - "synthetic": True, - }) - return _mark_synthetic(pd.DataFrame(rows)) - - -def make_transactions(n: int = 300, seed: int = 1) -> pd.DataFrame: - """Transaction-shaped frame: ids, currency strings, quantities, revenue.""" - rng = _rng(seed) - rows: list[dict[str, Any]] = [] - for i in range(n): - amount = round(rng.uniform(49.0, 99999.0), 2) - qty = rng.randint(1, 40) - rows.append({ - "txn_id": f"T{9000000 + i}", - "cust_id": f"C{100000 + rng.randrange(200)}", - "order_status": rng.choice( - ("placed", "shipped", "delivered", "returned", "cancelled")), - "category_code": rng.choice(_CATEGORY_CODES), - "quantity": f"{qty} {rng.choice(_UNITS)}", - "unit_price_inr": f"β‚Ή{amount:,.2f}", - "monthly_revenue": f"{round(amount * qty, 2)}", - "order_date": ( - f"{rng.randint(2022, 2025)}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" - ), - "notes": rng.choice(( - "priority shipping", "gift wrap requested", "repeat customer", - "call before delivery", "left at door", - )), - "synthetic": True, - }) - return _mark_synthetic(pd.DataFrame(rows)) - - -def make_context_sentences(seed: int = 2) -> list[dict[str, Any]]: - """Labeled context sentences (intent ground truth) with author tags. - - ``author`` supports author-disjoint eval splits; the ``hinglish`` authors - provide the Indian-English / Hinglish paraphrase set required by Phase 5. - """ - rng = _rng(seed) - templates: list[tuple[str, str, dict[str, Any], str]] = [ - ("This is an ecommerce customer dataset.", "DOMAIN", - {"domain": "ecommerce_customer"}, "t0"), - ("This is a retail transactions dataset.", "DOMAIN", - {"domain": "retail_transactions"}, "t1"), - ("This dataset is an ecommerce orders dataset.", "DOMAIN", - {"domain": "ecommerce_orders"}, "t2"), - ("This is a customer support dataset.", "DOMAIN", {"domain": "customer_support"}, "t3"), - # UNIQUE β€” shared vocabulary: unique / duplicate / once / repeat - ("{col} is unique.", "UNIQUE", {}, "t0"), - ("{col} must be unique.", "UNIQUE", {}, "t1"), - ("{col} values should be unique with no duplicates.", "UNIQUE", {}, "t3"), - ("No duplicate values in {col}.", "UNIQUE", {}, "t4"), - ("{col} should not repeat.", "UNIQUE", {}, "t0"), - ("Each {col} appears once, duplicates are not allowed.", "UNIQUE", {}, "t1"), - ("{col} appears only once per table.", "UNIQUE", {}, "t3"), - ("Every value in {col} occurs only once.", "UNIQUE", {}, "t4"), - ("Every {col} should appear only once.", "UNIQUE", {}, "t2"), - ("{col} must be unique, one per row.", "UNIQUE", {}, "t2"), - ("{col} har row me unique hona chahiye.", "UNIQUE", {}, "hinglish_a"), - ("{col} repeat nahi hona chahiye, unique rakho.", "UNIQUE", {}, "hinglish_b"), - # VALID_FORMAT β€” shared vocabulary: valid / email(s) - ("{col} must be valid emails.", "VALID_FORMAT", {"format": "email"}, "t0"), - ("Emails in {col} must be valid.", "VALID_FORMAT", {"format": "email"}, "t1"), - ("{col} should contain valid email addresses.", "VALID_FORMAT", {"format": "email"}, "t3"), - ("Make sure {col} emails are valid.", "VALID_FORMAT", {"format": "email"}, "t4"), - ("{col} emails must all be valid addresses.", "VALID_FORMAT", {"format": "email"}, "t2"), - ("{col} me valid email hona chahiye.", "VALID_FORMAT", {"format": "email"}, "hinglish_a"), - ("{col} ke emails valid hone chahiye.", "VALID_FORMAT", {"format": "email"}, "hinglish_b"), - # LOCALE_FORMAT β€” shared vocabulary: Indian / phone numbers - ("{col} are Indian phone numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t0"), - ("Phone numbers are Indian.", "LOCALE_FORMAT", {"locale": "IN", "format": "phone"}, "t1"), - ("{col} holds Indian phone numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t3"), - ("Treat {col} as Indian phone numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t4"), - ("The phone numbers in {col} are Indian numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t2"), - ("{col} me Indian phone numbers hain.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "hinglish_a"), - ("{col} ke phone numbers Indian hain.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "hinglish_b"), - # PROTECT β€” shared vocabulary: never modify / protected / do not change - ("Never modify {col} values.", "PROTECT", {}, "t0"), - ("Do not touch {col}.", "PROTECT", {}, "t1"), - ("Do not change {col}.", "PROTECT", {}, "t3"), - ("{col} must never be modified.", "PROTECT", {}, "t4"), - ("Leave {col} untouched, it is protected.", "PROTECT", {}, "t0"), - ("{col} is protected.", "PROTECT", {}, "t2"), - ("{col} is protected, never change it.", "PROTECT", {}, "t2"), - ("{col} kabhi mat badalna.", "PROTECT", {}, "hinglish_a"), - ("{col} ko modify mat karna, protected hai.", "PROTECT", {}, "hinglish_a"), - ("{col} ko change mat karo, bilkul nahi.", "PROTECT", {}, "hinglish_b"), - ("{col} protected hai, mat badalna.", "PROTECT", {}, "hinglish_b"), - # IMPUTE_IF β€” shared vocabulary: missing / estimated / confidence % - ("Missing {col} should be estimated only if confidence >95%.", "IMPUTE_IF", - {"threshold": 0.95}, "t0"), - ("Fill {col} gaps only when at least 95% sure.", "IMPUTE_IF", {"threshold": 0.95}, "t1"), - ("Estimate missing {col} only above 95% confidence.", "IMPUTE_IF", - {"threshold": 0.95}, "t3"), - ("Impute {col} only with confidence over 90%.", "IMPUTE_IF", {"threshold": 0.90}, "t4"), - ("Missing {col} may be estimated only at 95% confidence or more.", "IMPUTE_IF", - {"threshold": 0.95}, "t2"), - ("{col} missing ho to sirf 95% confidence par estimate karo.", "IMPUTE_IF", - {"threshold": 0.95}, "hinglish_a"), - ("{col} khali ho to sirf 95% sure hone par bharna.", "IMPUTE_IF", {"threshold": 0.95}, - "hinglish_a"), - ("{col} khali ho to 95% pakka hone par hi bharna.", "IMPUTE_IF", {"threshold": 0.95}, - "hinglish_b"), - # ALLOWED_VALUES β€” shared vocabulary: allowed / values / only be - ("Allowed status values are active, inactive, pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t0"), - ("{col} can only be active, inactive or pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t1"), - ("Allowed values for {col} are active, inactive, pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t3"), - ("Valid {col} values: active, inactive, pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t4"), - ("The only allowed {col} values are active, inactive and pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t2"), - ("Status sirf active, inactive ya pending ho sakta hai.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "hinglish_a"), - ("{col} sirf active, inactive ya pending allowed hai.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "hinglish_b"), - # RANGE β€” shared vocabulary: between X and Y / within - ("{col} must be between 18 and 99.", "RANGE", {"min": 18, "max": 99}, "t0"), - ("Keep {col} within 0 to 100.", "RANGE", {"min": 0, "max": 100}, "t1"), - ("{col} should stay between 1 and 40.", "RANGE", {"min": 1, "max": 40}, "t3"), - ("Values of {col} must lie between 0 and 999.", "RANGE", {"min": 0, "max": 999}, "t4"), - ("{col} has to be between 18 and 65.", "RANGE", {"min": 18, "max": 65}, "t2"), - ("{col} 5 aur 95 ke beech hona chahiye.", "RANGE", {"min": 5, "max": 95}, "hinglish_a"), - ("{col} between 10 aur 50 ke beech hona chahiye.", "RANGE", {"min": 10, "max": 50}, - "hinglish_b"), - # DEDUP_KEY β€” shared vocabulary: deduplicate / dedup key - ("Deduplicate rows by {col}.", "DEDUP_KEY", {}, "t0"), - ("Use {col} as the dedup key.", "DEDUP_KEY", {}, "t1"), - ("Deduplicate on {col}.", "DEDUP_KEY", {}, "t3"), - ("Rows should be deduplicated using {col} as the key.", "DEDUP_KEY", {}, "t2"), - ("{col} se deduplicate karo.", "DEDUP_KEY", {}, "hinglish_b"), - # DROP_IF β€” shared vocabulary: drop / remove rows / empty / missing - ("Drop rows if {col} is empty.", "DROP_IF", {"condition": "empty"}, "t0"), - ("Remove records where {col} is missing.", "DROP_IF", {"condition": "empty"}, "t1"), - ("Drop any row with empty {col}.", "DROP_IF", {"condition": "empty"}, "t3"), - ("Missing {col} rows should be dropped.", "DROP_IF", {"condition": "empty"}, "t4"), - ("Rows should be dropped when {col} is missing.", "DROP_IF", {"condition": "empty"}, "t0"), - ("Rows with missing {col} should be dropped.", "DROP_IF", {"condition": "empty"}, "t2"), - ("{col} empty ho to row drop kar do.", "DROP_IF", {"condition": "empty"}, "hinglish_b"), - # RENAME β€” shared vocabulary: rename ... to - ("Rename {col} to customer_id.", "RENAME", {"to": "customer_id"}, "t0"), - ("Please rename {col} to customer_id.", "RENAME", {"to": "customer_id"}, "t1"), - ("Rename column {col} to customer_id.", "RENAME", {"to": "customer_id"}, "t3"), - ("{col} ko customer_id rename kar do.", "RENAME", {"to": "customer_id"}, "hinglish_a"), - ("Rename the {col} column to customer_id.", "RENAME", {"to": "customer_id"}, "t2"), - ("{col} ko rename karke customer_id kar do.", "RENAME", {"to": "customer_id"}, - "hinglish_b"), - # MAP β€” shared vocabulary: map X to Y - ("Map M to male and F to female in {col}.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "t0"), - ("In {col}, map M to male and F to female.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "t1"), - ("Map the code M to male and the code F to female in {col}.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "t2"), - ("{col} me M ko male map karo aur F ko female.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "hinglish_a"), - ("{col} me M ko male aur F ko female map karo.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "hinglish_b"), - # UNKNOWN β€” no actionable constraint - ("The vibes of this table are immaculate.", "UNKNOWN", {}, "t0"), - ("Please make everything nicer somehow.", "UNKNOWN", {}, "t1"), - ("Great data, love it.", "UNKNOWN", {}, "t3"), - ("Have a look and see what you think.", "UNKNOWN", {}, "t4"), - ("Honestly this table just needs some love.", "UNKNOWN", {}, "t2"), - ("Kuch accha sa kar do is table ke saath.", "UNKNOWN", {}, "hinglish_b"), - ("Table ke saath kuch accha kar do.", "UNKNOWN", {}, "hinglish_a"), - ("Data ko thoda accha bana do yaar.", "UNKNOWN", {}, "hinglish_a"), - ] - columns = ("CustomerID", "cust_id", "email_addr", "mobile", "status", - "monthly_revenue", "age", "order_date", "postal_code") - out: list[dict[str, Any]] = [] - for text, intent, slots, author in templates: - picks = rng.sample(columns, k=3) if "{col}" in text else [None] - for col in picks: - sentence = text.format(col=col) if col is not None else text - out.append({ - "sentence": sentence, - "intent": intent, - "slots": dict(slots, column=col) if col is not None else dict(slots), - "author": author, - "synthetic": True, - "source_id": SYNTHETIC_SOURCE_ID, - }) - return out - - -def seed_tables(seed: int = 0) -> dict[str, pd.DataFrame]: - """All synthetic seed tables keyed by name.""" - return { - "customers": make_customers(seed=seed), - "transactions": make_transactions(seed=seed + 1), - } diff --git a/training/teacher/__init__.py b/training/teacher/__init__.py deleted file mode 100644 index 2e16a6bd..00000000 --- a/training/teacher/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Development-time teacher harness. - -Frontier teacher models are allowed **only** here, **only** for: - -1. realism direction (corruption frequency/parameter tuning), -2. column-role labeling, -3. NL context paraphrase generation, -4. ambiguity adjudication, -5. rationale style templates, -6. benchmark red-teaming. - -They are never called at runtime, never see full rows or real PII, never -generate messy/clean pairs from scratch, and their outputs are training data β€” -release-gating truth requires human verification (see ``review.py``). -""" diff --git a/training/teacher/cache.py b/training/teacher/cache.py deleted file mode 100644 index 7e653c2b..00000000 --- a/training/teacher/cache.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Content-addressed teacher output cache with a full audit trail. - -Key = ``provider + model + prompt_template_sha256 + input_sha256 + -schema_version``. Every entry stores the raw prompt, the raw response, and -provenance (provider, model, date, terms snapshot id) so any training label -can be traced to the exact call that produced it β€” and so re-runs never pay -for the same call twice. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from ..common import CACHE_DIR, read_json, sha256_text, utc_now_iso, write_json - - -@dataclass(frozen=True) -class CacheKey: - provider: str - model: str - prompt_template_sha256: str - input_sha256: str - schema_version: str - - def digest(self) -> str: - joined = "|".join(( - self.provider, self.model, self.prompt_template_sha256, - self.input_sha256, self.schema_version, - )) - return sha256_text(joined) - - -class TeacherCache: - def __init__(self, root: Path | str | None = None) -> None: - self.root = Path(root) if root is not None else CACHE_DIR / "teacher" - - def _path(self, key: CacheKey) -> Path: - digest = key.digest() - return self.root / digest[:2] / f"{digest}.json" - - def get(self, key: CacheKey) -> dict[str, Any] | None: - path = self._path(key) - if not path.is_file(): - return None - return read_json(path) - - def put( - self, - key: CacheKey, - *, - prompt: str, - response: str, - payloads: list[dict[str, Any]], - terms_snapshot_id: str, - ) -> Path: - entry = { - "key": { - "provider": key.provider, - "model": key.model, - "prompt_template_sha256": key.prompt_template_sha256, - "input_sha256": key.input_sha256, - "schema_version": key.schema_version, - }, - "prompt": prompt, - "response": response, - "payloads": payloads, - "provider": key.provider, - "model": key.model, - "created_at": utc_now_iso(), - "terms_snapshot_id": terms_snapshot_id, - } - return write_json(self._path(key), entry) - - def entries(self) -> list[dict[str, Any]]: - if not self.root.is_dir(): - return [] - return [read_json(p) for p in sorted(self.root.rglob("*.json"))] diff --git a/training/teacher/clients.py b/training/teacher/clients.py deleted file mode 100644 index 6beb3572..00000000 --- a/training/teacher/clients.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Teacher clients: a stub for offline development, a gated HTTP client. - -Nothing in the runtime package touches this module. There is no vendored -provider SDK; the HTTP client is configured entirely by environment -variables and refuses to run without them **and** without an approved -compliance record (checked by the caller in ``tasks.py``). -""" - -from __future__ import annotations - -import json -import os -import urllib.request -from typing import Any, Protocol - - -class TeacherUnavailable(RuntimeError): - """The teacher client cannot run (unconfigured, offline, or refused).""" - - -class TeacherClient(Protocol): - provider: str - model: str - - def complete(self, prompt: str, *, schema: str) -> str: - """Return the raw response text for one batched prompt.""" - ... - - -class StubTeacherClient: - """Deterministic offline stand-in used by tests and dev pipelines. - - Responses come from a canned mapping keyed by schema; by default it - returns an empty batch, which downstream tasks treat as "degrade to - corruptor/hook data". - """ - - provider = "stub" - model = "stub-teacher-0" - - def __init__(self, canned: dict[str, Any] | None = None, *, fail: bool = False) -> None: - self._canned = canned or {} - self._fail = fail - self.calls: list[dict[str, str]] = [] - - def complete(self, prompt: str, *, schema: str) -> str: - self.calls.append({"schema": schema, "prompt": prompt}) - if self._fail: - raise TeacherUnavailable("stub teacher configured to fail") - return json.dumps(self._canned.get(schema, [])) - - -class HTTPTeacherClient: - """Minimal JSON-over-HTTP teacher client (development-time only). - - Configuration (all required): - - - ``FRESHDATA_TEACHER_URL`` β€” full endpoint URL - - ``FRESHDATA_TEACHER_PROVIDER`` β€” provider name for compliance/cache keys - - ``FRESHDATA_TEACHER_MODEL`` β€” model name - - ``FRESHDATA_TEACHER_API_KEY`` β€” bearer token (never committed) - - The request body is ``{"model": ..., "prompt": ..., "schema": ...}`` and - the response body must be the raw JSON batch. Anything fancier belongs in - a local adapter script, not in the repository. - """ - - def __init__(self) -> None: - self.url = os.environ.get("FRESHDATA_TEACHER_URL", "") - self.provider = os.environ.get("FRESHDATA_TEACHER_PROVIDER", "") - self.model = os.environ.get("FRESHDATA_TEACHER_MODEL", "") - self._api_key = os.environ.get("FRESHDATA_TEACHER_API_KEY", "") - if not (self.url and self.provider and self.model and self._api_key): - raise TeacherUnavailable( - "teacher client not configured: set FRESHDATA_TEACHER_URL, " - "FRESHDATA_TEACHER_PROVIDER, FRESHDATA_TEACHER_MODEL, " - "FRESHDATA_TEACHER_API_KEY" - ) - - def complete(self, prompt: str, *, schema: str) -> str: - body = json.dumps({"model": self.model, "prompt": prompt, "schema": schema}) - request = urllib.request.Request( - self.url, - data=body.encode("utf-8"), - headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {self._api_key}", - }, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=120) as response: - return response.read().decode("utf-8") - except OSError as exc: - raise TeacherUnavailable(f"teacher call failed: {exc}") from exc - - -def default_client() -> TeacherClient: - """HTTP client when configured, else the empty stub (degraded mode).""" - try: - return HTTPTeacherClient() - except TeacherUnavailable: - return StubTeacherClient() diff --git a/training/teacher/compliance.py b/training/teacher/compliance.py deleted file mode 100644 index 1668127e..00000000 --- a/training/teacher/compliance.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Provider-terms compliance ledger and pre-run gate. - -Before any teacher run, a human records a compliance check for the provider -in ``training/teacher/compliance_ledger.json``. :func:`require_approved` -blocks the run when the record is missing, stale, unapproved, reviewer-less, -or when the provider's terms disallow using outputs for model training. - -CLI:: - - python -m training.teacher.compliance check [--provider X --use labeling] -""" - -from __future__ import annotations - -import argparse -import sys -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from ..common import read_json, utc_now_iso, write_json - -LEDGER_PATH = Path(__file__).resolve().parent / "compliance_ledger.json" -#: A terms check older than this is stale and must be re-verified. -MAX_AGE_DAYS = 180 - -ALLOWED_USES = ( - "realism_direction", - "column_role_labeling", - "context_paraphrase", - "ambiguity_adjudication", - "rationale_templates", - "red_teaming", -) - - -class ComplianceError(RuntimeError): - """The teacher run is blocked by the compliance gate.""" - - -def load_ledger(path: Path | str = LEDGER_PATH) -> dict[str, Any]: - path = Path(path) - if not path.is_file(): - return {"ledger_version": "1", "providers": []} - return read_json(path) - - -def _entry_for(ledger: dict[str, Any], provider: str) -> dict[str, Any] | None: - for entry in ledger.get("providers", []): - if entry.get("provider") == provider: - return entry - return None - - -def check_entry(entry: dict[str, Any] | None, provider: str, intended_use: str) -> list[str]: - """Return blocking problems for one provider/use pair (empty = approved).""" - problems: list[str] = [] - if entry is None: - problems.append(f"no terms snapshot recorded for provider {provider!r}") - return problems - if not entry.get("terms_url") or not entry.get("terms_snapshot_id"): - problems.append(f"{provider}: no terms snapshot exists") - checked = str(entry.get("date_checked", "")) - try: - age = datetime.now(timezone.utc) - datetime.fromisoformat(checked) - if age.days > MAX_AGE_DAYS: - problems.append(f"{provider}: terms snapshot is stale ({age.days} days old)") - except ValueError: - problems.append(f"{provider}: date_checked missing or unparseable") - if not entry.get("reviewer"): - problems.append(f"{provider}: reviewer is missing") - if entry.get("status") != "approved": - problems.append(f"{provider}: compliance status is {entry.get('status')!r}, not approved") - if entry.get("training_use_allowed") is not True: - problems.append(f"{provider}: provider terms do not allow training use of outputs") - allowed = entry.get("allowed_uses", []) - if intended_use not in allowed: - problems.append( - f"{provider}: intended use {intended_use!r} is not approved (approved: {allowed})" - ) - return problems - - -def require_approved( - provider: str, intended_use: str, *, ledger_path: Path | str = LEDGER_PATH -) -> dict[str, Any]: - """Return the approved ledger entry or raise :class:`ComplianceError`.""" - if intended_use not in ALLOWED_USES: - raise ComplianceError( - f"intended use {intended_use!r} is not a permitted teacher use; " - f"allowed: {ALLOWED_USES}" - ) - entry = _entry_for(load_ledger(ledger_path), provider) - problems = check_entry(entry, provider, intended_use) - if problems: - raise ComplianceError("; ".join(problems)) - assert entry is not None - return entry - - -def record_check( - *, - provider: str, - terms_url: str, - terms_snapshot_id: str, - reviewer: str, - allowed_uses: list[str], - training_use_allowed: bool, - redistribution_allowed: bool, - status: str = "approved", - notes: str = "", - ledger_path: Path | str = LEDGER_PATH, -) -> dict[str, Any]: - """Append or replace a provider compliance record (manual review result).""" - ledger = load_ledger(ledger_path) - entry = { - "provider": provider, - "terms_url": terms_url, - "terms_snapshot_id": terms_snapshot_id, - "date_checked": utc_now_iso(), - "reviewer": reviewer, - "allowed_uses": allowed_uses, - "training_use_allowed": training_use_allowed, - "redistribution_allowed": redistribution_allowed, - "status": status, - "notes": notes, - } - ledger["providers"] = [e for e in ledger.get("providers", []) if e.get("provider") != provider] - ledger["providers"].append(entry) - write_json(ledger_path, ledger) - return entry - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.teacher.compliance") - sub = parser.add_subparsers(dest="command", required=True) - check = sub.add_parser("check", help="verify compliance for a provider/use") - check.add_argument("--provider", default=None, help="provider name (default: all recorded)") - check.add_argument("--use", default="column_role_labeling", choices=ALLOWED_USES) - check.add_argument("--ledger", default=str(LEDGER_PATH)) - args = parser.parse_args(argv) - - ledger = load_ledger(args.ledger) - providers = ([args.provider] if args.provider - else [e.get("provider", "") for e in ledger.get("providers", [])]) - if not providers: - print("FAIL: compliance ledger is empty β€” record a terms check first", file=sys.stderr) - return 1 - failed = False - for provider in providers: - problems = check_entry(_entry_for(ledger, provider), provider, args.use) - if problems: - failed = True - for problem in problems: - print(f"FAIL: {problem}", file=sys.stderr) - else: - print(f"OK: {provider} approved for {args.use}") - return 1 if failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/teacher/compliance_ledger.json b/training/teacher/compliance_ledger.json deleted file mode 100644 index ba5ebb32..00000000 --- a/training/teacher/compliance_ledger.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "ledger_version": "1", - "providers": [ - { - "provider": "stub", - "terms_url": "training/teacher/clients.py#StubTeacherClient", - "terms_snapshot_id": "stub-local-0", - "date_checked": "2026-07-04T00:00:00+00:00", - "reviewer": "JWD", - "allowed_uses": [ - "realism_direction", - "column_role_labeling", - "context_paraphrase", - "ambiguity_adjudication", - "rationale_templates", - "red_teaming" - ], - "training_use_allowed": true, - "redistribution_allowed": true, - "status": "approved", - "notes": "Deterministic in-repo stub; no external service, no terms to verify. Real providers must be added via training.teacher.compliance.record_check after a manual terms review." - } - ] -} diff --git a/training/teacher/prompts/ambiguity.txt b/training/teacher/prompts/ambiguity.txt deleted file mode 100644 index 517c9253..00000000 --- a/training/teacher/prompts/ambiguity.txt +++ /dev/null @@ -1,9 +0,0 @@ -You adjudicate whether a corrupted value can be safely repaired to exactly one -candidate. If two or more candidates are plausible, the verdict is "ambiguous". -If no candidate is plausible, it is "unrepairable". Only when exactly one -candidate is clearly correct is it "resolvable". -Respond ONLY with a JSON array of AmbiguityJudgment objects: -[{"raw_value": str, "candidates": [str], "verdict": "ambiguous|resolvable|unrepairable", "preferred": str|null, "rationale": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/column_role.txt b/training/teacher/prompts/column_role.txt deleted file mode 100644 index 440ea9e3..00000000 --- a/training/teacher/prompts/column_role.txt +++ /dev/null @@ -1,10 +0,0 @@ -You label the semantic type of tabular columns. Values are MASKED (digits -replaced by 9, email locals hidden); judge shape, not content. Semantic types: -email, phone, url, country, currency_amount, quantity_with_unit, person_name, -address, city, postal_code, national_id, category_code, free_text, -boolean_like, date_like, identifier, unknown. -Respond ONLY with a JSON array of ColumnRoleLabel objects: -[{"column_name": str, "masked_samples": [str], "semantic_type": str, "confidence": float, "rationale": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/paraphrase.txt b/training/teacher/prompts/paraphrase.txt deleted file mode 100644 index 822baf4d..00000000 --- a/training/teacher/prompts/paraphrase.txt +++ /dev/null @@ -1,8 +0,0 @@ -You paraphrase data-cleaning context sentences without changing their intent -or slot values. Produce natural paraphrases, including Indian-English and -Hinglish variants when language says so. Never invent new constraints. -Respond ONLY with a JSON array of ContextParaphraseBatch objects: -[{"intent": str, "canonical_sentence": str, "paraphrases": [str], "language": str, "author": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/rationale.txt b/training/teacher/prompts/rationale.txt deleted file mode 100644 index 4296e51a..00000000 --- a/training/teacher/prompts/rationale.txt +++ /dev/null @@ -1,7 +0,0 @@ -You write short, factual rationale TEMPLATES (with {placeholders}) explaining -data repairs to a human reviewer. No marketing language; one sentence each. -Respond ONLY with a JSON array of RationaleTemplateBatch objects: -[{"issue_type": str, "templates": [str]}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/realism.txt b/training/teacher/prompts/realism.txt deleted file mode 100644 index 0649487a..00000000 --- a/training/teacher/prompts/realism.txt +++ /dev/null @@ -1,8 +0,0 @@ -You are tuning a data-corruption engine used to train a small data-cleaning model. -For each corruptor described below, recommend a realistic corruption share (0-1) -and parameter tweaks matching real-world dirty data frequency. -Respond ONLY with a JSON array of RealismConfig objects: -[{"corruptor_name": str, "recommended_share": float, "recommended_params": {}, "rationale": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/redteam.txt b/training/teacher/prompts/redteam.txt deleted file mode 100644 index 15ffd3ad..00000000 --- a/training/teacher/prompts/redteam.txt +++ /dev/null @@ -1,8 +0,0 @@ -You red-team a data-cleaning benchmark. Given a target metric, propose -adversarial cases (masked, synthetic values only) that a naive cleaner would -get wrong: dangerous near-merges, protected-column bait, dayfirst traps. -Respond ONLY with a JSON array of RedTeamCaseBatch objects: -[{"target_metric": str, "cases": [{"column": str, "raw_value": str, "why_hard": str}]}] - -Items: -{{ITEMS}} diff --git a/training/teacher/review.py b/training/teacher/review.py deleted file mode 100644 index b68543f4..00000000 --- a/training/teacher/review.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Human-review workflow for teacher outputs. - -Policy (enforced, not advisory): - -- at least **5%** of every task batch is sampled for human review; -- if teacher-human disagreement exceeds **3%**, the whole batch requires - full review before any of it may be used; -- release-gating eval labels must be **100% human-verified** β€” an unreviewed - teacher label can train a model but can never gate a release. -""" - -from __future__ import annotations - -import random -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from ..common import utc_now_iso, write_json - -SAMPLE_RATE = 0.05 -DISAGREEMENT_THRESHOLD = 0.03 - - -@dataclass(frozen=True) -class ReviewRecord: - item_id: str - reviewer: str - agrees: bool - reviewed_at: str - disagreement_reason: str = "" - corrected_payload: dict[str, Any] | None = None - - -@dataclass -class ReviewBatch: - """Review state for one teacher task batch.""" - - task_name: str - items: list[dict[str, Any]] - sample_ids: list[str] = field(default_factory=list) - records: list[ReviewRecord] = field(default_factory=list) - - @staticmethod - def item_id(index: int) -> str: - return f"item_{index:05d}" - - def sample_for_review(self, *, rate: float = SAMPLE_RATE, seed: int = 0) -> list[str]: - """Deterministically sample at least ``rate`` of items (min 1).""" - if not self.items: - self.sample_ids = [] - return [] - rng = random.Random(f"freshdata-review:{self.task_name}:{seed}") - n = max(1, int(len(self.items) * rate + 0.999999)) - picks = sorted(rng.sample(range(len(self.items)), min(n, len(self.items)))) - self.sample_ids = [self.item_id(i) for i in picks] - return self.sample_ids - - def record_review( - self, - item_id: str, - *, - reviewer: str, - agrees: bool, - disagreement_reason: str = "", - corrected_payload: dict[str, Any] | None = None, - ) -> ReviewRecord: - if not reviewer.strip(): - raise ValueError("reviewer id/initials are required") - if not agrees and not disagreement_reason.strip(): - raise ValueError("a disagreement must record its reason") - record = ReviewRecord( - item_id=item_id, - reviewer=reviewer.strip(), - agrees=agrees, - reviewed_at=utc_now_iso(), - disagreement_reason=disagreement_reason.strip(), - corrected_payload=corrected_payload, - ) - self.records.append(record) - return record - - def disagreement_rate(self) -> float: - if not self.records: - return 0.0 - return sum(1 for r in self.records if not r.agrees) / len(self.records) - - def requires_full_review(self) -> bool: - return self.disagreement_rate() > DISAGREEMENT_THRESHOLD - - def reviewed_ids(self) -> set[str]: - return {r.item_id for r in self.records} - - def usable_for_release_gating(self) -> bool: - """True only when every item in the batch has a human review record.""" - return self.reviewed_ids() >= {self.item_id(i) for i in range(len(self.items))} - - def summary(self) -> dict[str, Any]: - return { - "task_name": self.task_name, - "n_items": len(self.items), - "n_sampled": len(self.sample_ids), - "n_reviewed": len(self.records), - "sample_rate_required": SAMPLE_RATE, - "disagreement_rate": round(self.disagreement_rate(), 6), - "disagreement_threshold": DISAGREEMENT_THRESHOLD, - "requires_full_review": self.requires_full_review(), - "usable_for_release_gating": self.usable_for_release_gating(), - "reviewers": sorted({r.reviewer for r in self.records}), - "disagreements": [ - {"item_id": r.item_id, "reviewer": r.reviewer, "reason": r.disagreement_reason} - for r in self.records if not r.agrees - ], - } - - def export_summary(self, path: Path | str) -> Path: - return write_json(path, self.summary()) diff --git a/training/teacher/schemas.py b/training/teacher/schemas.py deleted file mode 100644 index ead3a44b..00000000 --- a/training/teacher/schemas.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Strict JSON schemas for teacher outputs. - -Every teacher task declares one of these schemas; a response that does not -validate is discarded (the pipeline degrades to corruptor/hook data rather -than accepting free-form teacher text). Validation is strict: required keys, -type checks, and **no unknown keys**. -""" - -from __future__ import annotations - -from typing import Any - -SCHEMA_VERSION = "1" - - -class SchemaError(ValueError): - """A teacher response failed strict schema validation.""" - - -#: schema name -> {field: (type, required)} -_SCHEMAS: dict[str, dict[str, tuple[type | tuple[type, ...], bool]]] = { - "RealismConfig": { - "corruptor_name": (str, True), - "recommended_share": (float, True), - "recommended_params": (dict, False), - "rationale": (str, True), - }, - "ColumnRoleLabel": { - "column_name": (str, True), - "masked_samples": (list, True), - "semantic_type": (str, True), - "confidence": (float, True), - "rationale": (str, False), - }, - "ContextParaphraseBatch": { - "intent": (str, True), - "canonical_sentence": (str, True), - "paraphrases": (list, True), - "language": (str, True), - "author": (str, True), - }, - "AmbiguityJudgment": { - "raw_value": (str, True), - "candidates": (list, True), - "verdict": (str, True), # "ambiguous" | "resolvable" | "unrepairable" - "preferred": ((str, type(None)), False), - "rationale": (str, True), - }, - "RationaleTemplateBatch": { - "issue_type": (str, True), - "templates": (list, True), - }, - "RedTeamCaseBatch": { - "target_metric": (str, True), - "cases": (list, True), - }, -} - -_ALLOWED_VERDICTS = frozenset({"ambiguous", "resolvable", "unrepairable"}) - - -def schema_names() -> tuple[str, ...]: - return tuple(sorted(_SCHEMAS)) - - -def validate_payload(schema: str, payload: Any) -> dict[str, Any]: - """Validate one teacher output payload; return it or raise SchemaError.""" - spec = _SCHEMAS.get(schema) - if spec is None: - raise SchemaError(f"unknown teacher schema {schema!r}; known: {schema_names()}") - if not isinstance(payload, dict): - raise SchemaError(f"{schema}: payload must be an object, got {type(payload).__name__}") - unknown = set(payload) - set(spec) - if unknown: - raise SchemaError(f"{schema}: unknown keys {sorted(unknown)}") - for field, (types, required) in spec.items(): - if field not in payload: - if required: - raise SchemaError(f"{schema}: missing required field {field!r}") - continue - value = payload[field] - if isinstance(types, type) and types is float and isinstance(value, int): - value = float(value) - payload[field] = value - if not isinstance(value, types): - raise SchemaError( - f"{schema}: field {field!r} must be {types}, got {type(value).__name__}" - ) - if schema == "AmbiguityJudgment" and payload["verdict"] not in _ALLOWED_VERDICTS: - raise SchemaError(f"AmbiguityJudgment: invalid verdict {payload['verdict']!r}") - if schema == "ColumnRoleLabel" and not (0.0 <= float(payload["confidence"]) <= 1.0): - raise SchemaError("ColumnRoleLabel: confidence must be in [0, 1]") - return payload - - -def validate_batch(schema: str, payloads: list[Any]) -> list[dict[str, Any]]: - return [validate_payload(schema, p) for p in payloads] diff --git a/training/teacher/tasks.py b/training/teacher/tasks.py deleted file mode 100644 index 09070493..00000000 --- a/training/teacher/tasks.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Teacher task harness: batched, masked, cached, compliance-gated. - -Guarantees enforced here: - -- **compliance first** β€” every run checks the provider ledger and aborts on - anything but an explicit approval; -- **no full rows** β€” tasks accept only column names plus a bounded list of - *masked* sample values, never DataFrame rows; -- **PII masking** β€” email locals, phone digits, and long identifiers are - masked before any prompt is built (:func:`mask_pii`); -- **strict schemas** β€” invalid responses are dropped, not repaired; -- **cache everything** β€” prompt + response + provenance land in the audit - cache before payloads are returned; -- **degrade safely** β€” a failed teacher call returns an empty batch so the - pipeline can fall back to corruptor/hook data. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from ..common import sha256_json, sha256_text -from . import compliance -from .cache import CacheKey, TeacherCache -from .clients import TeacherClient, TeacherUnavailable, default_client -from .schemas import SCHEMA_VERSION, SchemaError, validate_batch - -PROMPTS_DIR = Path(__file__).resolve().parent / "prompts" - -_EMAIL_RE = re.compile(r"([^@\s])[^@\s]*(@[^@\s]+)") -_DIGIT_RE = re.compile(r"\d") -# Opaque mixed alphanumeric identifiers (post digit-masking they still leak -# letter structure, so hash them). Pure digit runs keep their masked shape. -_LONG_TOKEN_RE = re.compile(r"(?=[A-Za-z0-9]*[A-Za-z])(?=[A-Za-z0-9]*9)[A-Za-z0-9]{16,}") - - -def mask_pii(value: object) -> str: - """Mask PII-shaped content while keeping the *shape* of the value. - - ``asha.voskette@example.com`` -> ``a***@example.com``; digits become - ``9``; long opaque tokens are hashed to a stable placeholder. - """ - text = str(value) - text = _EMAIL_RE.sub(lambda m: f"{m.group(1)}***{m.group(2)}", text) - text = _DIGIT_RE.sub("9", text) - return _LONG_TOKEN_RE.sub(lambda m: f"tok_{sha256_text(m.group(0))[:8]}", text) - - -@dataclass(frozen=True) -class TeacherTaskSpec: - """One allowed teacher task: purpose, prompt template, output schema.""" - - name: str - intended_use: str - prompt_file: str - schema: str - max_batch: int = 32 - - -TASKS: dict[str, TeacherTaskSpec] = { - spec.name: spec - for spec in ( - TeacherTaskSpec( - "realism_direction", "realism_direction", "realism.txt", "RealismConfig"), - TeacherTaskSpec( - "column_role_labeling", "column_role_labeling", - "column_role.txt", "ColumnRoleLabel"), - TeacherTaskSpec( - "context_paraphrase", "context_paraphrase", - "paraphrase.txt", "ContextParaphraseBatch"), - TeacherTaskSpec( - "ambiguity_adjudication", "ambiguity_adjudication", - "ambiguity.txt", "AmbiguityJudgment"), - TeacherTaskSpec( - "rationale_templates", "rationale_templates", - "rationale.txt", "RationaleTemplateBatch"), - TeacherTaskSpec( - "red_teaming", "red_teaming", "redteam.txt", "RedTeamCaseBatch"), - ) -} - - -def _assert_no_full_rows(items: Sequence[dict[str, Any]]) -> None: - """Refuse batches that smuggle whole records to the teacher.""" - for item in items: - if len(item) > 6: - raise ValueError( - "teacher task items must be small (column + masked samples), " - f"got {len(item)} fields β€” this looks like a full row" - ) - samples = item.get("masked_samples", []) - if isinstance(samples, list) and len(samples) > 10: - raise ValueError("at most 10 masked sample values may be sent per item") - - -def build_prompt(spec: TeacherTaskSpec, items: Sequence[dict[str, Any]]) -> tuple[str, str]: - """Return ``(prompt_text, template_sha)`` for one batch.""" - template = (PROMPTS_DIR / spec.prompt_file).read_text(encoding="utf-8") - payload = json.dumps(list(items), indent=2, sort_keys=True, ensure_ascii=False) - return template.replace("{{ITEMS}}", payload), sha256_text(template) - - -def run_task( - task_name: str, - items: Sequence[dict[str, Any]], - *, - client: TeacherClient | None = None, - cache: TeacherCache | None = None, - ledger_path: Path | str | None = None, -) -> list[dict[str, Any]]: - """Run one teacher task batch and return validated payloads. - - Empty list means "teacher unavailable or response invalid" β€” callers must - treat that as *no teacher data*, never as an error, so every consumer has - an offline fallback path. - """ - spec = TASKS.get(task_name) - if spec is None: - raise KeyError(f"unknown teacher task {task_name!r}; known: {sorted(TASKS)}") - _assert_no_full_rows(items) - - client = client if client is not None else default_client() - cache = cache if cache is not None else TeacherCache() - ledger_kwargs = {"ledger_path": ledger_path} if ledger_path is not None else {} - entry = compliance.require_approved(client.provider, spec.intended_use, **ledger_kwargs) - - payloads: list[dict[str, Any]] = [] - for start in range(0, len(items), spec.max_batch): - batch = list(items[start:start + spec.max_batch]) - prompt, template_sha = build_prompt(spec, batch) - key = CacheKey( - provider=client.provider, - model=client.model, - prompt_template_sha256=template_sha, - input_sha256=sha256_json(batch), - schema_version=f"{spec.schema}:{SCHEMA_VERSION}", - ) - cached = cache.get(key) - if cached is not None: - payloads.extend(cached.get("payloads", [])) - continue - try: - response = client.complete(prompt, schema=spec.schema) - parsed = json.loads(response) - if not isinstance(parsed, list): - raise SchemaError(f"{spec.schema}: response must be a JSON array") - validated = validate_batch(spec.schema, parsed) - except (TeacherUnavailable, SchemaError, json.JSONDecodeError): - # Degrade: no teacher data for this batch, pipeline continues. - continue - cache.put( - key, - prompt=prompt, - response=response, - payloads=validated, - terms_snapshot_id=str(entry.get("terms_snapshot_id", "")), - ) - payloads.extend(validated) - return payloads - - -def column_role_items( - columns: Sequence[str], samples: dict[str, Sequence[object]], *, max_samples: int = 8 -) -> list[dict[str, Any]]: - """Build masked column-role items from column names + sample values.""" - return [ - { - "column_name": str(col), - "masked_samples": [mask_pii(v) for v in list(samples.get(col, []))[:max_samples]], - } - for col in columns - ] diff --git a/uv.lock b/uv.lock index 29fd157c..9a14fd18 100644 --- a/uv.lock +++ b/uv.lock @@ -1,8 +1,10 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.10" resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] @@ -406,28 +408,11 @@ name = "freshdata-cleaner" version = "0.4.0" source = { editable = "." } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "pandas" }, ] [package.optional-dependencies] -all = [ - { name = "duckdb" }, - { name = "polars" }, - { name = "pyarrow" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -bench = [ - { name = "duckdb" }, - { name = "polars" }, - { name = "psutil" }, - { name = "pyarrow" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] dev = [ { name = "build" }, { name = "duckdb" }, @@ -444,46 +429,41 @@ dev = [ { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "twine" }, ] -duckdb = [ - { name = "duckdb" }, - { name = "pyarrow" }, +migrate = [ + { name = "libcst" }, ] polars = [ { name = "polars" }, { name = "pyarrow" }, ] +test = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] [package.metadata] requires-dist = [ { name = "build", marker = "extra == 'dev'", specifier = ">=1.2.0" }, - { name = "duckdb", marker = "extra == 'all'", specifier = ">=1.0.0" }, - { name = "duckdb", marker = "extra == 'bench'", specifier = ">=1.0.0" }, { name = "duckdb", marker = "extra == 'dev'", specifier = ">=1.0.0" }, - { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.0.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.100.0" }, + { name = "libcst", marker = "extra == 'migrate'", specifier = ">=1.4.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10.0" }, - { name = "numpy", specifier = ">=1.24.0" }, + { name = "numpy", specifier = ">=1.24.0,<2.4.0" }, { name = "pandas", specifier = ">=2.0.0" }, - { name = "polars", marker = "extra == 'all'", specifier = ">=1.0.0" }, - { name = "polars", marker = "extra == 'bench'", specifier = ">=1.0.0" }, { name = "polars", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "polars", marker = "extra == 'polars'", specifier = ">=1.0.0" }, - { name = "psutil", marker = "extra == 'bench'", specifier = ">=5.9.0" }, { name = "psutil", marker = "extra == 'dev'", specifier = ">=5.9.0" }, - { name = "pyarrow", marker = "extra == 'all'", specifier = ">=15.0.0" }, - { name = "pyarrow", marker = "extra == 'bench'", specifier = ">=15.0.0" }, { name = "pyarrow", marker = "extra == 'dev'", specifier = ">=15.0.0" }, - { name = "pyarrow", marker = "extra == 'duckdb'", specifier = ">=15.0.0" }, { name = "pyarrow", marker = "extra == 'polars'", specifier = ">=15.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, - { name = "scikit-learn", marker = "extra == 'all'", specifier = ">=1.4.0" }, - { name = "scikit-learn", marker = "extra == 'bench'", specifier = ">=1.4.0" }, { name = "scikit-learn", marker = "extra == 'dev'", specifier = ">=1.4.0" }, { name = "twine", marker = "extra == 'dev'", specifier = ">=5.0.0" }, ] -provides-extras = ["polars", "duckdb", "all", "bench", "dev"] +provides-extras = ["migrate", "polars", "test", "dev"] [[package]] name = "hypothesis" @@ -589,7 +569,9 @@ name = "hypothesis" version = "6.170.15" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -795,6 +777,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] +[[package]] +name = "libcst" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version != '3.13.*'" }, + { name = "pyyaml-ft", marker = "python_full_version == '3.13.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/c0/098e5c91ff1537f00c85a6438b6cb1863d17144680cc91f47c87f104a200/libcst-1.9.0.tar.gz", hash = "sha256:087b58a9afe076bb08e2d726478e1f16cb928d67ffa9092817e033c335de522a", size = 914739, upload-time = "2026-07-29T21:28:43.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/b3/23144dcd9f9cd6f6150058653ecd0357c6b09361656ebd286165a5d76d34/libcst-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5705f664266742b8d9323db714f58cf06a6cde739eb09fd9ca31cd233f9806e5", size = 2055037, upload-time = "2026-07-29T19:24:27.708Z" }, + { url = "https://files.pythonhosted.org/packages/5d/87/164a9bac39a817b505d27cf4490e6f66b2a1b8adb6b4e022b1ef4613bcfe/libcst-1.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7dcddfb4905c5ef90a6fdc58dccbc049faf0056b68161042fc1e0526b20cba05", size = 2210166, upload-time = "2026-07-29T19:24:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/1c/cb/b5aeaefbe471c4225c145c15c3bac96d67b0641a5e3ec9b871c62858c88c/libcst-1.9.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cc84aee3c293a012db722aa2d6ea87be524646852684a5a4e54c1fb98ea9072f", size = 2260568, upload-time = "2026-07-29T19:24:31.021Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0f/351dcd23629732e35e716e13ad536a193124cda075ac5e59dc866503447b/libcst-1.9.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:329ffda6b0ef708ff22876f982ec886a509c8f73ac1ccb0190693d35cb09aa0f", size = 2276024, upload-time = "2026-07-29T19:24:32.393Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/0da2b60137a6290ba7dda902b32da64d46c9a5107aa002da578f8b1ff8c8/libcst-1.9.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5302fc698ef863e1a2fb540e95793b3d5a96abc89aa6423ee9c389d7d7ec5eab", size = 2384413, upload-time = "2026-07-29T19:24:33.983Z" }, + { url = "https://files.pythonhosted.org/packages/8f/80/7e48d2fc1afda13a6de50ab6f8340dbd104fd6c9bcd2ff11dec66e320f2c/libcst-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6615b036694f6906e30dc018b5b34898cff7ff760ab549f9dea1ada4f53495b", size = 2110204, upload-time = "2026-07-29T19:24:35.491Z" }, + { url = "https://files.pythonhosted.org/packages/42/87/52c88c7ba873011526aa35b8fa0cf55d50effe967c999e7cb688155c5691/libcst-1.9.0-cp310-cp310-win_arm64.whl", hash = "sha256:dacc583bcd8db56e457ae559b910673c6b5df66f694d1f1154ce0b3982158d17", size = 1988757, upload-time = "2026-07-29T19:24:37.053Z" }, + { url = "https://files.pythonhosted.org/packages/53/36/b45e6950878d9143f95db609432031ac688b311aa14450a8df239c648c45/libcst-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cdbafbc992b38ea005ef37e20243c4177879020c82d414789fbb7fa22d6d3a9", size = 2054911, upload-time = "2026-07-29T19:24:38.542Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d8/1c1ae22e05fbc763dc5a4e4c3cab356166913ef19d2c2da42e6f1be5c98c/libcst-1.9.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3f735e3acb3afc75d50803fde1598b467bf05d12d6854025df4398aebd75a14c", size = 2209147, upload-time = "2026-07-29T19:24:40.039Z" }, + { url = "https://files.pythonhosted.org/packages/84/e1/0d5eaeac4dadd34fcd9ea154d0a6e5ce10c9fec0e02c73f807a16475b8b0/libcst-1.9.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:951f13c193e590fc88c9582a5341a130fe53ecece51043cff4bd54e25a5e022d", size = 2260355, upload-time = "2026-07-29T19:24:41.423Z" }, + { url = "https://files.pythonhosted.org/packages/71/b2/45ce33c6bca5c1961a90f2999138741daa0dc8cf6fe8b122c0ca2502632d/libcst-1.9.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bc6aa3e7f21315dbe9868c0d13ffaac309514e8c816320c5f91230fa5b21df92", size = 2275659, upload-time = "2026-07-29T19:24:42.828Z" }, + { url = "https://files.pythonhosted.org/packages/bb/b1/936d2f17d93fb160579ff56ca671d26154209bbc07ea8645eb11db1f9633/libcst-1.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f101e0bf8638eba80694ecc3c3abb5309eb488f5271427536cb379c5c1e0ba9e", size = 2383467, upload-time = "2026-07-29T19:24:44.397Z" }, + { url = "https://files.pythonhosted.org/packages/e7/59/db667f15d62cb22c75b24f6b13c4151292aa1889a64ff9abce92f60d5a97/libcst-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:77d9303572cf0fbd0a2eb71aff167a4246541f43bf58b781b35b9d0bbab80ddc", size = 2109985, upload-time = "2026-07-29T19:24:45.966Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a8/99926865cd3f97c0cffc71cd38152c883b7ae666eb8327d355f01bddb33a/libcst-1.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:4090105fa4ad114766d0f8b34418d9a3b359bc8645da860a447e19770b5705d1", size = 1988559, upload-time = "2026-07-29T19:24:47.528Z" }, + { url = "https://files.pythonhosted.org/packages/b0/bb/d22c37c33dfe18084634f5ef89f8f0749ffe7b6e0ad312722aafd86bbbdb/libcst-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cd1a3500c41784075c4946a995d5ad89f68fa0d226b63ff3c4d78f6ea6dd23e5", size = 2043159, upload-time = "2026-07-29T19:24:49.013Z" }, + { url = "https://files.pythonhosted.org/packages/10/b8/2dedef84d72e7271119217503b69ed6dc5d0b2077685e163caae669d9c70/libcst-1.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:611cebd3bbc2014576f4dcc7b845b3c594c96ddc287a3db9b78f22eff156a7d3", size = 2203245, upload-time = "2026-07-29T19:24:50.399Z" }, + { url = "https://files.pythonhosted.org/packages/e8/90/e02ac2dad647423f947bb11f8322bfdba8ccfdd380e6c7b695add2d1acd4/libcst-1.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:8d731abe1307720ea1a52d447555e8443a6d130e0e520243c0634a58f6edbc9d", size = 2255388, upload-time = "2026-07-29T19:24:52.21Z" }, + { url = "https://files.pythonhosted.org/packages/13/5f/6089a51518cfd2ff40950eb26bcc36951d7b6d4f4213568aa0290265aff7/libcst-1.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8bc5351d92ca6ac1cc32e097700e1161ad1ceaa4d9b2cca5abadb1e94576b325", size = 2268982, upload-time = "2026-07-29T19:24:53.562Z" }, + { url = "https://files.pythonhosted.org/packages/ed/78/26881ec466fb70cbc129dca26ccb5a52a0061face2c5822e4b61f00f9699/libcst-1.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03165a264653bb77f6a11b412ae09c08bdb0c25864f3b8d42b816ac64b9d4b9e", size = 2378174, upload-time = "2026-07-29T19:24:55.175Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7a/a4dba5f11faf12a12ffba06d18019987851aab33b590242a602ffd4fb1bd/libcst-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:b755ed4a4bc2faee849b54820023137d60d4c199e0a0822f0ff0c1bc49e49b48", size = 2103462, upload-time = "2026-07-29T19:24:56.966Z" }, + { url = "https://files.pythonhosted.org/packages/f5/13/57cb129093e0d6744b3c914ba6cbbdf51ed1b2c823882936c553a3a0cf1b/libcst-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e50576bad7d56459d9792cd0b0dfe5469dad13646e9a9c0a8b1ba20b269f332", size = 1979825, upload-time = "2026-07-29T19:24:58.403Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b1/befc0544283bb3923a928accf79ed685e5a725524bdb3491826670affc07/libcst-1.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b9df30f524317b097dc53065b25dda33d6a4cc3c7c8bf4fc83ca7559c58cc0", size = 2043754, upload-time = "2026-07-29T19:24:59.885Z" }, + { url = "https://files.pythonhosted.org/packages/45/50/fef7c172a8457c95894edf5fb04805024899cdfea41fa01b0636587b79e1/libcst-1.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e465a7bc9c2b9533eb9e06d2391f8819f811b5112919d4064c9fb8565aaafa08", size = 2203548, upload-time = "2026-07-29T19:25:01.323Z" }, + { url = "https://files.pythonhosted.org/packages/18/ff/764cd2be1fd99d774fc44039c319dc0ed1d9d9afeaa02759a05121cccd4b/libcst-1.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8504b422c95676a8c27b517e1ac01413ece91bf356865c587ca9bdcd5708a2f7", size = 2255274, upload-time = "2026-07-29T19:25:02.764Z" }, + { url = "https://files.pythonhosted.org/packages/34/a7/474748a27a02fa83e3556b260d5f5236ca48164fa7beffecf3b2cad24dca/libcst-1.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bcb9f9d4fcfe2ec7a40d2c26e03a538d5d9dc189c38551eb3f94ab661afee7c0", size = 2269126, upload-time = "2026-07-29T19:25:04.158Z" }, + { url = "https://files.pythonhosted.org/packages/8a/b7/655e45363b8cf87b91e41e060b21c89e5316c7ef36eb14a1a498b27bb71e/libcst-1.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8d671c39a431c309476099b8ec811e412503ec0f4465f6fc907cb51c70e8e6b", size = 2378602, upload-time = "2026-07-29T19:25:06.424Z" }, + { url = "https://files.pythonhosted.org/packages/db/13/6da63f0902ece43bf9d737017251ad7ad06edd9d3c4e1856450403cb4473/libcst-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:4d382fba04077eb556a1ea4295a4482e192aa93639c5a07ba0885841965ea0c0", size = 2103486, upload-time = "2026-07-29T19:25:07.979Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ff/dccb1a55e38b4e47256a97242d61d2bef5366c744faf88fb087d4fa0f995/libcst-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:a621e261990148c1cfbe26c1798bd2375c131cf358a44a7a4659fc41c1a336e1", size = 1979907, upload-time = "2026-07-29T19:25:09.532Z" }, + { url = "https://files.pythonhosted.org/packages/65/2a/4943c71d90975bc59034a057dea346b365c276f308c1d31f5cf2bd85492d/libcst-1.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eccf4c57d273cdd3fe1c67b72cf9bb1bbd4547aa011824e96ccf5b7136057aa4", size = 2044529, upload-time = "2026-07-29T19:25:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1168b98c2a0f338be3a44753647baab051feaf6167cc887bf4447f8fd920/libcst-1.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32395244edfe6538e0ea2bf82051d60103d3f54861274805c4fb3745efb70a85", size = 2203519, upload-time = "2026-07-29T19:25:12.543Z" }, + { url = "https://files.pythonhosted.org/packages/25/7d/2eaa697a80f899bcf2245680bbfcc1e478eb3b3328c21d4b2880bdf00dac/libcst-1.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:444e84c76cd035cd2fe136838c1a524d34b08216521f6f5093df8a6f6cfa5799", size = 2255879, upload-time = "2026-07-29T19:25:14.137Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/793b9fd96dd52d202f5f2b88d7e54f05ebed767d1bb3b32395a1817bf6ab/libcst-1.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb5d0946f2b4c6711b5d69fe4f833b364f9e7a1a2b08f88b98619dc18975099a", size = 2269807, upload-time = "2026-07-29T19:25:15.647Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/1bc7aaea03971c45e8a885fed9fc1c73176dbbd00b0296a3cde9529bafd6/libcst-1.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45808c03528b3ad40b14095348a918e08d98c47b4a125d631cb78e817c0a5b16", size = 2378171, upload-time = "2026-07-29T19:25:17.289Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f7/bc49e367d2bc8817213594dd52cf6b2da2dbbda90b5382d60653999274ac/libcst-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:568288cdbfe3b4ca3ae4852cb0a439ff053dd54c841bc4995bf2b71238b5de40", size = 2177847, upload-time = "2026-07-29T19:25:19.35Z" }, + { url = "https://files.pythonhosted.org/packages/87/80/4d81577a22e6d535d1a3409f3a1c6903e09036f0e17fc47f92169dbc3501/libcst-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:107593af46945593e7825821793393262bc4fa1d3ea24c3ed487b61269bbbdf8", size = 2058134, upload-time = "2026-07-29T19:25:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7f/c3f3a0e7a1a2adaa76e815e82a7814d6bfe7d49432276bba652248c68d0b/libcst-1.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f6248cb07444ab9a6733a855737a9febed8b9adca51347019348b09a3ac7dfe9", size = 2036102, upload-time = "2026-07-29T19:25:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/4e/af/2f5543255b2c7d749b966adeccc6bfb1652cf8b425afb913d0f3f025a865/libcst-1.9.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:496c24e0d3240bc7da45dae543aff3f5b6509978c39262d6b84ed2fb999dded2", size = 2194806, upload-time = "2026-07-29T19:25:24.017Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5b/03f4cddce426d005e208b39ea7b2d456e667cdee0f1891360f0fc8430f20/libcst-1.9.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ea490fa8540503db5f321f0268becab46eb50f710e8cec8041e241fd66f6874f", size = 2246762, upload-time = "2026-07-29T19:25:25.365Z" }, + { url = "https://files.pythonhosted.org/packages/d8/31/9d5fe1e43dc3dbcc74f70f3e0e73fcdd8d84effc1059d8b45974f0d0d2eb/libcst-1.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:50ab94bb2524b419056d4003032b8c67102ac800f8d85b3c4260a01746d94dbb", size = 2259633, upload-time = "2026-07-29T19:25:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2b/6752b28d88c3a19b3bc0b9e1838443b6760d5c862b4f4b37955402659e24/libcst-1.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a2faaf92500d0226358125630f5aab4758e8aad3f2d70a10892ec3c700781a54", size = 2368043, upload-time = "2026-07-29T19:25:28.344Z" }, + { url = "https://files.pythonhosted.org/packages/e2/94/775825b2637f8ab05694b6a4b3802ae6783b4e799f9b58d2400c7e2d4369/libcst-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c7b548512db25af9c2997a95fa731bd6b6928ecbad6c0915d7482d8bb42d34f", size = 2176432, upload-time = "2026-07-29T19:25:29.921Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3d/88ad67427c6fd9db929e087912b0a540e5140e5cb77e7ca4170edaac8531/libcst-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:497d5329345f1f5df84e41b0bbd00204b64a2fd30dfe3cfaeaebca633a31e877", size = 2052931, upload-time = "2026-07-29T19:25:31.397Z" }, + { url = "https://files.pythonhosted.org/packages/39/e4/ad790b043a38b10cea13166c8dd716654ad8b227c20f12eebf6326191cd9/libcst-1.9.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a5068bf6114f6f4d79af7a6c80a28d1deb50441150ea1db13b5458ab34bec159", size = 2043987, upload-time = "2026-08-11T05:54:02.122Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/d3cd8275cc54ffc9817ade91442b3e6e9edd1a4518bd4e6dda13e3152dbe/libcst-1.9.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:7acfd18adcdd32dcf41ce676cf121002afcbe9ad2c72dc0c18d78465beedc228", size = 2204003, upload-time = "2026-08-11T05:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/da/62/87eddccb11d5d221d50ac4fff4b6e9b8d48c547028d01f7d0fbc5c8a1d75/libcst-1.9.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:86361e2b426bd1b403e52375703c8b764e226e571adcb30442510710681edbf0", size = 2256053, upload-time = "2026-08-11T05:54:06.228Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/a44126aeb9fca8b4e342e0cc803ea00ca8f6cd5712619a7897b3eba13797/libcst-1.9.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8c14abe844bec8b021111b3985474e49f5af799c020e8e40dedcac87c97a40c5", size = 2270576, upload-time = "2026-08-11T05:54:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/bb/51/3af3dd44b117d66486e0ed61e43a16b7fc8cc5c372e1dd4ca0e70b888d46/libcst-1.9.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:8930971d2299b7006bb5d10c10038f44efb6da89d5bca2823595e31a9cdb96af", size = 2378573, upload-time = "2026-08-11T05:54:10.093Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e8/e545087d298dbf6494e84a8092f0f5dbd62eedacabd4ecd6b364367b4804/libcst-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5891ce9cff815077614f509e3a23888c5ddb8081d6188e8ba1172c0ccc369022", size = 2177937, upload-time = "2026-08-11T05:54:12.168Z" }, + { url = "https://files.pythonhosted.org/packages/84/17/93a00a8e03494a84db102dd0e3d35b8876b1084ca2c194b331a168d86eb1/libcst-1.9.0-cp315-cp315-win_arm64.whl", hash = "sha256:1260b5d070a3324447a53a00387225a55472a62077ce01922d30a2a78cc4b138", size = 2058633, upload-time = "2026-08-11T05:54:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/d7/00/c3751b12eb81822ea0b6131d374a98bc6c024c4b9ec5f6ea8f53dc23e967/libcst-1.9.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:02ee2dbdb5c218116f16a021350fc267a14be205b50d81c33f5d73857ab6fbf7", size = 2036178, upload-time = "2026-08-11T05:54:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/2f/12/48ff486eb5adc593f4818569e8896b74b672414ab4722a74b17e4c4cf31e/libcst-1.9.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:3e5b684191a0462b2d40a73261ea7f4da6a49e7a030b7e0fceca46bebb51dfe5", size = 2195496, upload-time = "2026-08-11T05:54:17.625Z" }, + { url = "https://files.pythonhosted.org/packages/51/18/b13a4669864d41a4801fed7b86ede1049dc9a1e1dde250a7ee1f9c3b1285/libcst-1.9.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:2b150c4f298fe54eb0fe73abf71f57db74d070796140a8c206c9615b6861f01e", size = 2245769, upload-time = "2026-08-11T05:54:19.343Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c7/cf2d46744825e84afbd7228fdeeea8d23cf6c4b2074253c27efb7705c653/libcst-1.9.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:50b913e37187f00a6fb88962009364e1cf1b1284aa1762304f34b52b972f2218", size = 2260576, upload-time = "2026-08-11T05:54:21.444Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/5433d3d62250b2e4325938db101766bcab2a006819684713bcccb6259a88/libcst-1.9.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:9f92c75283030fd58fb7d6e560168a00381e0e3368ec8a026a3ec8a828a05f93", size = 2368709, upload-time = "2026-08-11T05:54:23.276Z" }, + { url = "https://files.pythonhosted.org/packages/87/39/9f0e1690727623f99489895db0e42048a3e1e8cea0627517c593e437fd83/libcst-1.9.0-cp315-cp315t-win_amd64.whl", hash = "sha256:4adf97bb1aff8039b0bd4991e2f838be6e4fad552821b26b71a5d1a653c2582f", size = 2177866, upload-time = "2026-08-11T05:54:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/cd/79/9dc7811883e67ea771057e8937bc536eb3e77f3635fc5a93cd85b6fe1ea8/libcst-1.9.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8f0dd08a5773d7051e105250b2a2c73d8065ea9b2d68e7e1bdc5244660e75f93", size = 2052908, upload-time = "2026-08-11T05:54:26.835Z" }, +] + [[package]] name = "librt" version = "0.13.0" @@ -1014,9 +1064,6 @@ wheels = [ name = "numpy" version = "2.2.6" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, @@ -1075,142 +1122,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, ] -[[package]] -name = "numpy" -version = "2.4.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.11.*'", -] -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, - { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, - { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, - { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, - { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, - { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, - { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, - { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, - { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, - { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, - { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, - { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, - { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, - { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, - { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, - { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, - { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, - { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, - { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, - { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, - { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, - { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, - { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, - { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, - { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, - { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, - { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, - { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, - { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, - { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, - { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, - { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, - { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, - { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, - { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, - { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, - { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, - { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, - { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, -] - -[[package]] -name = "numpy" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", -] -sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, - { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, - { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, - { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, - { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, - { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, - { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, - { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, - { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, - { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, - { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, - { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, - { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, - { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, - { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, - { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, - { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, - { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, - { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, - { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, - { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, - { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, - { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, - { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, - { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, - { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, - { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, - { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, - { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, - { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, - { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, - { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, - { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, - { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, - { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, -] - [[package]] name = "packaging" version = "26.2" @@ -1225,9 +1136,7 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -1496,6 +1405,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "pyyaml-ft" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/eb/5a0d575de784f9a1f94e2b1288c6886f13f34185e13117ed530f32b6f8a8/pyyaml_ft-8.0.0.tar.gz", hash = "sha256:0c947dce03954c7b5d38869ed4878b2e6ff1d44b08a0d84dc83fdad205ae39ab", size = 141057, upload-time = "2025-06-10T15:32:15.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/ba/a067369fe61a2e57fb38732562927d5bae088c73cb9bb5438736a9555b29/pyyaml_ft-8.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8c1306282bc958bfda31237f900eb52c9bedf9b93a11f82e1aab004c9a5657a6", size = 187027, upload-time = "2025-06-10T15:31:48.722Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c5/a3d2020ce5ccfc6aede0d45bcb870298652ac0cf199f67714d250e0cdf39/pyyaml_ft-8.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30c5f1751625786c19de751e3130fc345ebcba6a86f6bddd6e1285342f4bbb69", size = 176146, upload-time = "2025-06-10T15:31:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/e3/bb/23a9739291086ca0d3189eac7cd92b4d00e9fdc77d722ab610c35f9a82ba/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fa992481155ddda2e303fcc74c79c05eddcdbc907b888d3d9ce3ff3e2adcfb0", size = 746792, upload-time = "2025-06-10T15:31:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c2/e8825f4ff725b7e560d62a3609e31d735318068e1079539ebfde397ea03e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cec6c92b4207004b62dfad1f0be321c9f04725e0f271c16247d8b39c3bf3ea42", size = 786772, upload-time = "2025-06-10T15:31:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/35/be/58a4dcae8854f2fdca9b28d9495298fd5571a50d8430b1c3033ec95d2d0e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06237267dbcab70d4c0e9436d8f719f04a51123f0ca2694c00dd4b68c338e40b", size = 778723, upload-time = "2025-06-10T15:31:56.093Z" }, + { url = "https://files.pythonhosted.org/packages/86/ed/fed0da92b5d5d7340a082e3802d84c6dc9d5fa142954404c41a544c1cb92/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a7f332bc565817644cdb38ffe4739e44c3e18c55793f75dddb87630f03fc254", size = 758478, upload-time = "2025-06-10T15:31:58.314Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/ac02afe286275980ecb2dcdc0156617389b7e0c0a3fcdedf155c67be2b80/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7d10175a746be65f6feb86224df5d6bc5c049ebf52b89a88cf1cd78af5a367a8", size = 799159, upload-time = "2025-06-10T15:31:59.675Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ac/c492a9da2e39abdff4c3094ec54acac9747743f36428281fb186a03fab76/pyyaml_ft-8.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:58e1015098cf8d8aec82f360789c16283b88ca670fe4275ef6c48c5e30b22a96", size = 158779, upload-time = "2025-06-10T15:32:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/5d/9b/41998df3298960d7c67653669f37710fa2d568a5fc933ea24a6df60acaf6/pyyaml_ft-8.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e64fa5f3e2ceb790d50602b2fd4ec37abbd760a8c778e46354df647e7c5a4ebb", size = 191331, upload-time = "2025-06-10T15:32:02.602Z" }, + { url = "https://files.pythonhosted.org/packages/0f/16/2710c252ee04cbd74d9562ebba709e5a284faeb8ada88fcda548c9191b47/pyyaml_ft-8.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8d445bf6ea16bb93c37b42fdacfb2f94c8e92a79ba9e12768c96ecde867046d1", size = 182879, upload-time = "2025-06-10T15:32:04.466Z" }, + { url = "https://files.pythonhosted.org/packages/9a/40/ae8163519d937fa7bfa457b6f78439cc6831a7c2b170e4f612f7eda71815/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c56bb46b4fda34cbb92a9446a841da3982cdde6ea13de3fbd80db7eeeab8b49", size = 811277, upload-time = "2025-06-10T15:32:06.214Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/28d82dbff7f87b96f0eeac79b7d972a96b4980c1e445eb6a857ba91eda00/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dab0abb46eb1780da486f022dce034b952c8ae40753627b27a626d803926483b", size = 831650, upload-time = "2025-06-10T15:32:08.076Z" }, + { url = "https://files.pythonhosted.org/packages/e8/df/161c4566facac7d75a9e182295c223060373d4116dead9cc53a265de60b9/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd48d639cab5ca50ad957b6dd632c7dd3ac02a1abe0e8196a3c24a52f5db3f7a", size = 815755, upload-time = "2025-06-10T15:32:09.435Z" }, + { url = "https://files.pythonhosted.org/packages/05/10/f42c48fa5153204f42eaa945e8d1fd7c10d6296841dcb2447bf7da1be5c4/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:052561b89d5b2a8e1289f326d060e794c21fa068aa11255fe71d65baf18a632e", size = 810403, upload-time = "2025-06-10T15:32:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d2/e369064aa51009eb9245399fd8ad2c562bd0bcd392a00be44b2a824ded7c/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3bb4b927929b0cb162fb1605392a321e3333e48ce616cdcfa04a839271373255", size = 835581, upload-time = "2025-06-10T15:32:12.897Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/26534bed77109632a956977f60d8519049f545abc39215d086e33a61f1f2/pyyaml_ft-8.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793", size = 171579, upload-time = "2025-06-10T15:32:14.34Z" }, +] + [[package]] name = "readme-renderer" version = "45.0" @@ -1593,7 +1590,7 @@ resolution-markers = [ ] dependencies = [ { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, { name = "threadpoolctl" }, ] @@ -1636,14 +1633,15 @@ name = "scikit-learn" version = "1.9.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ { name = "joblib" }, { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "threadpoolctl" }, @@ -1690,7 +1688,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -1749,7 +1747,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -1820,10 +1818,12 @@ name = "scipy" version = "1.18.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [