From 1252483d6244e4d394c352d5c98f76c2d82ff593 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 12:47:30 +0530 Subject: [PATCH 01/11] fix(core): make polars optional at import and compute polars profile counts - Import polars only in the Polars code paths. A polars-module frame without an importable polars raises ImportError that names the polars extra. - PolarsBackend.profile wrote literal zeros into null and unique counts. It now reads them from the eager DataFrame. LazyFrame profiling still does not collect. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 35 ++++++++++++++++++++++++--------- src/freshdata/_core/engine.py | 8 +++----- tests/test_backend_honesty.py | 22 +++++++++++++++++++++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 5f30a8d5..5874c7d7 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -1,8 +1,7 @@ # src/freshdata/_core/backends.py from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import pandas as pd -import polars as pl from freshdata._core.models import ( ActionRecord, CleanConfig, @@ -20,6 +19,9 @@ stage_strings, ) +if TYPE_CHECKING: + import polars as pl + class PandasBackend: def profile(self, df: pd.DataFrame) -> ProfileReport: cols = {} @@ -110,22 +112,28 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. class PolarsBackend: def profile(self, df: pl.DataFrame | pl.LazyFrame) -> ProfileReport: - schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema + import polars as pl + + frame = None if isinstance(df, pl.LazyFrame) else df + schema = df.collect_schema() if frame is None else frame.schema + row_count = 0 if frame is None else len(frame) cols = {} for name, dtype in schema.items(): role = "numeric" if dtype.is_numeric() else "categorical" + nulls = 0 if frame is None else frame[name].null_count() cols[name] = ColumnProfile( name=name, inferred_role=role, - null_count=0, - null_ratio=0.0, - unique_count=0, + null_count=nulls, + null_ratio=nulls / max(row_count, 1), + unique_count=0 if frame is None else frame[name].drop_nulls().n_unique(), sample_values=(), ) - row_count = 0 if isinstance(df, pl.LazyFrame) else len(df) return ProfileReport(row_count=row_count, column_count=len(schema), columns=cols) def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> CleanPlan: + import polars as pl + schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema actions = [] cfg_eff = normalize_config(cfg) @@ -153,6 +161,8 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle return CleanPlan(actions=tuple(actions)) def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: CleanConfig) -> pl.DataFrame | pl.LazyFrame: + import polars as pl + cur = df # 1. Renames renames = {a.column: a.parameters["to"] for a in plan.actions if a.stage == "column_names" and a.action == "normalize_name"} @@ -193,6 +203,13 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea def resolve_backend(df: Any) -> PandasBackend | PolarsBackend: if isinstance(df, pd.DataFrame): return PandasBackend() - if isinstance(df, (pl.DataFrame, pl.LazyFrame)): - return PolarsBackend() + if type(df).__module__.split(".")[0] == "polars": + try: + import polars as pl + except ImportError as exc: + raise ImportError( + "Polars frames need the polars extra: pip install 'freshdata-cleaner[polars]'" + ) from exc + if isinstance(df, (pl.DataFrame, pl.LazyFrame)): + return PolarsBackend() raise TypeError(f"Unsupported frame type: {type(df).__name__}. freshdata supports pandas.DataFrame, polars.DataFrame, and polars.LazyFrame.") diff --git a/src/freshdata/_core/engine.py b/src/freshdata/_core/engine.py index e5d1f593..e0ed95fc 100644 --- a/src/freshdata/_core/engine.py +++ b/src/freshdata/_core/engine.py @@ -1,8 +1,6 @@ # src/freshdata/_core/engine.py from __future__ import annotations from typing import Any -import pandas as pd -import polars as pl from freshdata._core.backends import resolve_backend from freshdata._core.models import ( ActionRecord, @@ -28,10 +26,10 @@ def apply_plan( return_report: bool = False, ) -> Any: cfg = config or CleanConfig() - cleaned = resolve_backend(df).apply_plan(df, plan, cfg) + cleaned: Any = resolve_backend(df).apply_plan(df, plan, cfg) if return_report: - orig_r = len(df) if isinstance(df, (pd.DataFrame, pl.DataFrame)) else 0 - clean_r = len(cleaned) if isinstance(cleaned, (pd.DataFrame, pl.DataFrame)) else 0 + orig_r = 0 if type(df).__name__ == "LazyFrame" else len(df) + clean_r = 0 if type(cleaned).__name__ == "LazyFrame" else len(cleaned) orig_c = len(df.columns) if hasattr(df, "columns") else 0 clean_c = len(cleaned.columns) if hasattr(cleaned, "columns") else 0 rep = CleanReport(orig_r, clean_r, orig_c, clean_c, plan.actions) diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index d12baee1..2d3c1e75 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -37,3 +37,25 @@ def guarded_to_pandas(self, *args, **kwargs): def test_unsupported_type_raises(): with pytest.raises(TypeError, match="Unsupported frame type"): resolve_backend({"a": [1, 2, 3]}) + +def test_import_and_clean_without_polars(): + import subprocess + import sys + from pathlib import Path + + code = ( + "import sys; sys.modules['polars'] = None\n" + "import pandas as pd, freshdata\n" + "out = freshdata.clean(pd.DataFrame({'A b': [1.0, None, 3.0]}))\n" + "assert out['a_b'].tolist() == [1.0, 2.0, 3.0]\n" + ) + src = Path(__file__).resolve().parents[1] / "src" + subprocess.run([sys.executable, "-c", code], check=True, env={"PYTHONPATH": str(src)}) + +def test_polars_profile_counts_match_pandas(): + data = {"a": [1, 2, None, 2], "b": ["x", None, None, "y"]} + pd_rep = resolve_backend(pd.DataFrame(data)).profile(pd.DataFrame(data)) + pl_rep = resolve_backend(pl.DataFrame(data)).profile(pl.DataFrame(data)) + for col in data: + p, q = pd_rep.columns[col], pl_rep.columns[col] + assert (q.null_count, q.unique_count, q.null_ratio) == (p.null_count, p.unique_count, p.null_ratio) From ca20b17081cf602d7f60d1ccf4cbca7b10e18308 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 12:52:06 +0530 Subject: [PATCH 02/11] refactor(core): simplify stages, backends and package init - Replace the 7 hand-built row filters with out[mask].reset_index(drop=True). - Use dataclasses.replace in normalize_config. - Merge the impute_* branches in PandasBackend.apply_plan. - Compute IQR bounds once in stage_outliers. Merge the two drop branches in stage_missing (empty frames still record drop_nulls). - Run the six suggest_plan stages in a loop. - Delete unused ActionRecord.params and ActionRecord.affected_rows. - Import from _core submodules and empty _core/__init__.py. - Read __version__ from package metadata. It now reports 0.4.0, not 3.0.0. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/__init__.py | 12 +++--- src/freshdata/_core/__init__.py | 32 ---------------- src/freshdata/_core/backends.py | 28 ++++---------- src/freshdata/_core/models.py | 8 ---- src/freshdata/_core/stages.py | 66 ++++++++++++--------------------- tests/test_stages.py | 58 +++++++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 110 deletions(-) diff --git a/src/freshdata/__init__.py b/src/freshdata/__init__.py index 5b6f2929..950d660f 100644 --- a/src/freshdata/__init__.py +++ b/src/freshdata/__init__.py @@ -1,20 +1,18 @@ # src/freshdata/__init__.py -from freshdata._core import ( +from importlib.metadata import version as _version + +from freshdata._core.engine import Cleaner, apply_plan, clean, profile, suggest_plan +from freshdata._core.models import ( ActionRecord, CleanConfig, - Cleaner, CleanPlan, CleanReport, ColumnProfile, ColumnRule, ProfileReport, - apply_plan, - clean, - profile, - suggest_plan, ) -__version__ = "3.0.0" +__version__ = _version("freshdata-cleaner") __all__ = [ "ActionRecord", diff --git a/src/freshdata/_core/__init__.py b/src/freshdata/_core/__init__.py index 1160f53a..e69de29b 100644 --- a/src/freshdata/_core/__init__.py +++ b/src/freshdata/_core/__init__.py @@ -1,32 +0,0 @@ -# src/freshdata/_core/__init__.py -from freshdata._core.engine import ( - Cleaner, - apply_plan, - clean, - profile, - suggest_plan, -) -from freshdata._core.models import ( - ActionRecord, - CleanConfig, - CleanPlan, - CleanReport, - ColumnProfile, - ColumnRule, - ProfileReport, -) - -__all__ = [ - "ActionRecord", - "CleanConfig", - "Cleaner", - "CleanPlan", - "CleanReport", - "ColumnProfile", - "ColumnRule", - "ProfileReport", - "apply_plan", - "clean", - "profile", - "suggest_plan", -] diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 5874c7d7..b2f91af3 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -42,18 +42,10 @@ def profile(self, df: pd.DataFrame) -> ProfileReport: def suggest_plan(self, df: pd.DataFrame, cfg: CleanConfig) -> CleanPlan: actions = [] cfg_eff = normalize_config(cfg) - cur, act = stage_column_names(df, cfg) - actions.extend(act) - cur, act = stage_strings(cur, cfg_eff) - actions.extend(act) - cur, act = stage_dtypes(cur, cfg_eff) - actions.extend(act) - cur, act = stage_duplicates(cur, cfg_eff) - actions.extend(act) - cur, act = stage_missing(cur, cfg_eff) - actions.extend(act) - cur, act = stage_outliers(cur, cfg_eff) - actions.extend(act) + cur = df + for stage in (stage_column_names, stage_strings, stage_dtypes, stage_duplicates, stage_missing, stage_outliers): + cur, act = stage(cur, cfg_eff) + actions.extend(act) return CleanPlan(actions=tuple(actions)) def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd.DataFrame: @@ -69,7 +61,7 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. for a in plan.actions: if a.stage == "duplicates" and a.action == "drop_duplicates": mask = ~out.duplicated().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) + out = out[mask].reset_index(drop=True) continue col = renames.get(a.column, a.column) @@ -85,17 +77,13 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") elif a.stage == "missing": - if a.action in ("impute_median", "impute_mean"): - val = a.parameters.get("value") - if val is not None: - out[col] = out[col].fillna(val) - elif a.action == "impute_mode": + if a.action.startswith("impute_"): val = a.parameters.get("value") if val is not None: out[col] = out[col].fillna(val) elif a.action == "drop_nulls": mask = ~out[col].isna().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) + out = out[mask].reset_index(drop=True) elif a.stage == "outliers": if a.action == "clip_iqr": lower = a.parameters.get("lower") @@ -107,7 +95,7 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. upper = a.parameters.get("upper") if lower is not None and upper is not None: mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) + out = out[mask].reset_index(drop=True) return out class PolarsBackend: diff --git a/src/freshdata/_core/models.py b/src/freshdata/_core/models.py index 8f24029e..14f1f229 100644 --- a/src/freshdata/_core/models.py +++ b/src/freshdata/_core/models.py @@ -38,14 +38,6 @@ class ActionRecord: def operation(self) -> str: return self.action - @property - def params(self) -> dict[str, Any]: - return self.parameters - - @property - def affected_rows(self) -> int: - return self.rows_affected - def to_dict(self) -> dict[str, Any]: return asdict(self) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 3ea36686..7a638bbb 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -1,5 +1,6 @@ # src/freshdata/_core/stages.py from __future__ import annotations +import dataclasses import re from typing import Tuple, List import numpy as np @@ -18,20 +19,7 @@ def normalize_config(cfg: CleanConfig) -> CleanConfig: norm_rules = {_normalize_name(k): v for k, v in cfg.rules.items()} norm_id = tuple(_normalize_name(c) for c in cfg.id_columns) norm_target = _normalize_name(cfg.target_column) if cfg.target_column else None - return CleanConfig( - strategy=cfg.strategy, - rules=norm_rules, - id_columns=norm_id, - target_column=norm_target, - normalize_names=cfg.normalize_names, - strip_strings=cfg.strip_strings, - fix_dtypes=cfg.fix_dtypes, - drop_duplicates=cfg.drop_duplicates, - default_impute=cfg.default_impute, - default_outliers=cfg.default_outliers, - iqr_factor=cfg.iqr_factor, - strict_backend=cfg.strict_backend, - ) + return dataclasses.replace(cfg, rules=norm_rules, id_columns=norm_id, target_column=norm_target) def stage_column_names(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: if not cfg.normalize_names: @@ -91,7 +79,7 @@ def stage_duplicates(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, return df, [] orig_len = len(df) mask = ~df.duplicated().to_numpy() - out = pd.DataFrame({col: pd.Series(df[col].array[mask], dtype=df[col].dtype) for col in df.columns}) + out = df[mask].reset_index(drop=True) dropped = orig_len - len(out) actions = [ActionRecord("duplicates", "*", "drop_duplicates", {}, dropped)] if dropped > 0 else [] return out, actions @@ -108,7 +96,12 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis continue impute_strategy = rule.impute or (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) null_count = int(out[col].isna().sum()) - if 0 < null_count < len(out): + if impute_strategy == "drop" and (null_count > 0 or len(out) == 0): + orig_len = len(out) + mask = ~out[col].isna().to_numpy() + out = out[mask].reset_index(drop=True) + actions.append(ActionRecord("missing", str(col), "drop_nulls", {}, orig_len - len(out))) + elif 0 < null_count < len(out): if impute_strategy in ("median", "auto") and pd.api.types.is_numeric_dtype(out[col]): med = out[col].median() if not pd.isna(med): @@ -127,16 +120,6 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis val = mode_vals.iloc[0] out[col] = out[col].fillna(val) actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": str(val)}, null_count)) - elif impute_strategy == "drop": - orig_len = len(out) - mask = ~out[col].isna().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) - actions.append(ActionRecord("missing", str(col), "drop_nulls", {}, orig_len - len(out))) - elif null_count == len(out) and impute_strategy == "drop": - orig_len = len(out) - mask = ~out[col].isna().to_numpy() - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) - actions.append(ActionRecord("missing", str(col), "drop_nulls", {}, orig_len - len(out))) return out, actions def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: @@ -150,23 +133,20 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li if rule.protected: continue strategy = rule.outliers or cfg_eff.default_outliers - if strategy == "clip" and pd.api.types.is_numeric_dtype(out[col]): - s = out[col].dropna() - if len(s) < 4: - continue - q25, q75 = np.percentile(s, [25, 75]) - iqr = q75 - q25 - lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr + if strategy not in ("clip", "drop") or not pd.api.types.is_numeric_dtype(out[col]): + continue + s = out[col].dropna() + if len(s) < 4: + continue + q25, q75 = np.percentile(s, [25, 75]) + iqr = q75 - q25 + lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr + if strategy == "clip": out[col] = out[col].clip(lower=lower, upper=upper) actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, len(out))) - elif strategy == "drop" and pd.api.types.is_numeric_dtype(out[col]): - s = out[col].dropna() - if len(s) >= 4: - q25, q75 = np.percentile(s, [25, 75]) - iqr = q75 - q25 - lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr - mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() - dropped = int((~mask).sum()) - out = pd.DataFrame({c: pd.Series(out[c].array[mask], dtype=out[c].dtype) for c in out.columns}) - actions.append(ActionRecord("outliers", str(col), "drop_outliers", {"lower": float(lower), "upper": float(upper)}, dropped)) + else: + mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() + dropped = int((~mask).sum()) + out = out[mask].reset_index(drop=True) + actions.append(ActionRecord("outliers", str(col), "drop_outliers", {"lower": float(lower), "upper": float(upper)}, dropped)) return out, actions diff --git a/tests/test_stages.py b/tests/test_stages.py index 86a720f9..05fb3bd2 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -65,3 +65,61 @@ def test_stage_outliers_short_series(): assert len(actions) == 0 assert out_df["x"].tolist() == [1.0, 2.0] + +def test_row_filters_keep_extension_dtypes(): + import freshdata as fd + + df = pd.DataFrame({ + "n": pd.array([1, 1, None, 2, 3, 4], dtype="Int64"), + "s": pd.array(["a", "a", "b", None, "c", "d"], dtype="string"), + "c": pd.Categorical(["x", "x", "y", "y", "x", "y"]), + "t": pd.to_datetime(["2024-01-01", "2024-01-01", "2024-01-02", "2024-01-03", "2024-01-04", "2024-01-05"]), + "v": [1.0, 1.0, 2.0, 3.0, 4.0, 1000.0], + }) + cfg = CleanConfig( + normalize_names=False, strip_strings=False, fix_dtypes=False, drop_duplicates=True, + default_impute="preserve", + rules={"n": ColumnRule(impute="drop"), "v": ColumnRule(outliers="drop")}, + ) + out, report = fd.clean(df, cfg, return_report=True) + assert {a.action for a in report.actions} == {"drop_duplicates", "drop_nulls", "drop_outliers"} + assert out.dtypes.to_dict() == df.dtypes.to_dict() + assert out["n"].tolist() == [1, 2, 3] + assert list(out.index) == [0, 1, 2] + +NAN = float("nan") + +@pytest.mark.parametrize(("strategy", "values", "expected", "actions"), [ + ("median", [1.0, 2.0, 3.0, 10.0], [1.0, 2.0, 3.0, 10.0], []), + ("median", [1.0, None, 3.0, 10.0], [1.0, 3.0, 3.0, 10.0], ["impute_median"]), + ("median", [None] * 4, [NAN] * 4, []), + ("mean", [1.0, None, 3.0, 10.0], [1.0, 14 / 3, 3.0, 10.0], ["impute_mean"]), + ("mean", [None] * 4, [NAN] * 4, []), + ("mode", [1.0, None, 3.0, 10.0], [1.0, 1.0, 3.0, 10.0], ["impute_mode"]), + ("mode", [None] * 4, [NAN] * 4, []), + ("drop", [1.0, 2.0, 3.0, 10.0], [1.0, 2.0, 3.0, 10.0], []), + ("drop", [1.0, None, 3.0, 10.0], [1.0, 3.0, 10.0], ["drop_nulls"]), + ("drop", [None] * 4, [], ["drop_nulls"]), + ("drop", [], [], ["drop_nulls"]), + ("preserve", [1.0, None, 3.0, 10.0], [1.0, NAN, 3.0, 10.0], []), +]) +def test_stage_missing_strategies(strategy, values, expected, actions): + df = pd.DataFrame({"x": pd.Series(values, dtype="float64")}) + out, acts = stage_missing(df, CleanConfig(rules={"x": ColumnRule(impute=strategy)})) + assert out["x"].tolist() == pytest.approx(expected, nan_ok=True) + assert [a.action for a in acts] == actions + +@pytest.mark.parametrize(("strategy", "values", "expected", "actions"), [ + ("clip", [10.0, 11.0, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, 12.0, 10.0, 11.0, 14.0], ["clip_iqr"]), + ("clip", [10.0, 11.0, None, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, NAN, 12.0, 10.0, 11.0, 14.0], ["clip_iqr"]), + ("clip", [None] * 6, [NAN] * 6, []), + ("drop", [10.0, 11.0, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, 12.0, 10.0, 11.0], ["drop_outliers"]), + ("drop", [10.0, 11.0, None, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, NAN, 12.0, 10.0, 11.0], ["drop_outliers"]), + ("drop", [None] * 6, [NAN] * 6, []), + ("preserve", [10.0, 11.0, None, 12.0, 10.0, 11.0, 100.0], [10.0, 11.0, NAN, 12.0, 10.0, 11.0, 100.0], []), +]) +def test_stage_outliers_strategies(strategy, values, expected, actions): + df = pd.DataFrame({"x": pd.Series(values, dtype="float64")}) + out, acts = stage_outliers(df, CleanConfig(rules={"x": ColumnRule(outliers=strategy)})) + assert out["x"].tolist() == pytest.approx(expected, nan_ok=True) + assert [a.action for a in acts] == actions From ec069cbf92e5eae729c3595a47060a1c56309676 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 12:58:33 +0530 Subject: [PATCH 03/11] chore: delete code outside src that targets the removed v1/v2 API - Delete 30 benchmark, example, notebook, script and training files that use freshdata names that no longer exist. - Delete training/, freshdata-benchmarks/ (with committed .asv results), crates/freshcore/, marketing/, growth-audit.md and admin-actions.md. - Delete docs/freshcore.md and docs/developer-training-pipeline.md, which only documented deleted paths. - Remove references to the deleted paths from docs, mkdocs.yml, .gitignore, MANIFEST.in and Makefile. Files that live code imports or runs stay, even when they use the removed API (for example benchmarks/bench_report.py and the truthbench surfaces). Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 - MANIFEST.in | 1 - Makefile | 7 - admin-actions.md | 71 - benchmarks/bench_entity_resolution.py | 375 --- benchmarks/bench_fieldcheck.py | 161 -- benchmarks/bench_native_semantic.py | 172 -- benchmarks/bench_quick.py | 166 -- benchmarks/bench_streaming.py | 124 - benchmarks/cleanbench/t4_profiles.py | 463 ---- benchmarks/corpus/scoring.py | 268 -- benchmarks/performance/semantic_probe.py | 217 -- benchmarks/truthbench/inventory.py | 383 --- crates/freshcore/Cargo.lock | 255 -- crates/freshcore/Cargo.toml | 20 - crates/freshcore/src/arrays.rs | 158 -- crates/freshcore/src/kernels/casts.rs | 143 -- crates/freshcore/src/kernels/duplicates.rs | 153 -- crates/freshcore/src/kernels/missing.rs | 169 -- crates/freshcore/src/kernels/mod.rs | 6 - crates/freshcore/src/kernels/outliers.rs | 213 -- crates/freshcore/src/kernels/profile.rs | 52 - crates/freshcore/src/kernels/strings.rs | 88 - crates/freshcore/src/lib.rs | 14 - crates/freshcore/src/plan.rs | 51 - crates/freshcore/src/python.rs | 392 --- docs/backends.md | 1 - docs/benchmarks.md | 28 +- docs/context-policies.md | 4 - docs/developer-training-pipeline.md | 115 - docs/examples.md | 1 - docs/field-validation.md | 7 +- docs/freshcore.md | 125 - docs/interactive.md | 8 - docs/learning-profiles.md | 13 - docs/limitations.md | 9 - docs/model-cards.md | 10 - docs/performance-investigation.md | 10 +- docs/plugins.md | 3 - docs/semantic-models.md | 12 - docs/streaming.md | 28 - docs/trust-claims.md | 1 - examples/04_profiling.py | 44 - examples/cdc_event_time_gate.py | 53 - examples/cross_source_link.py | 51 - examples/entity_resolution_duckdb.py | 83 - .../plugins/custom_backend/keyword_backend.py | 79 - .../plugins/custom_expert/acronym_expert.py | 90 - .../custom_validator/min_rows_validator.py | 63 - examples/privacy_anonymization.py | 89 - examples/schema_drift_monitoring.py | 106 - examples/validation_suite_quickstart.py | 50 - ...MP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json | 1 - .../Johnnys-MacBook-Air.local/machine.json | 9 - .../.asv/results/benchmarks.json | 2205 ----------------- freshdata-benchmarks/METHODOLOGY.md | 48 - freshdata-benchmarks/README.md | 146 -- freshdata-benchmarks/asv.conf.json | 33 - .../bench/artifacts/benchmark_output.txt | 39 - .../bench/artifacts/env_info.json | 30 - freshdata-benchmarks/bench/run_benchmarks.sh | 129 - freshdata-benchmarks/benchmarks/__init__.py | 1 - .../benchmarks/benchmark_columns.py | 54 - .../benchmarks/benchmark_duplicates.py | 67 - .../benchmarks/benchmark_encoding.py | 58 - .../benchmark_freshdata_vs_pandas.py | 148 -- .../benchmarks/benchmark_groupagg.py | 63 - .../benchmarks/benchmark_io.py | 120 - .../benchmarks/benchmark_memory.py | 91 - .../benchmarks/benchmark_missing.py | 98 - .../benchmarks/benchmark_outliers.py | 86 - .../benchmarks/benchmark_pipeline.py | 85 - .../benchmarks/benchmark_scaling.py | 67 - .../benchmarks/benchmark_scaling_curves.py | 87 - .../benchmarks/benchmark_strings.py | 66 - .../benchmarks/benchmark_types.py | 57 - .../benchmarks/benchmark_validation.py | 82 - .../benchmarks/datasets/__init__.py | 27 - .../benchmarks/datasets/data_generator.py | 418 ---- .../benchmarks/utils/__init__.py | 1 - .../benchmarks/utils/fairness.py | 99 - .../benchmarks/utils/library_wrappers.py | 976 -------- .../benchmarks/utils/metrics.py | 86 - .../reports/results_extracted.json | 1 - freshdata-benchmarks/reports/summary.md | 2 - freshdata-benchmarks/requirements.txt | 27 - freshdata-benchmarks/scripts/capture_env.py | 102 - .../scripts/generate_plots.py | 233 -- .../scripts/generate_reports.py | 171 -- freshdata-benchmarks/setup.py | 10 - growth-audit.md | 56 - marketing/README.md | 42 - marketing/blog-ideas.md | 57 - marketing/contributor-posts.md | 76 - marketing/demo-scripts.md | 80 - marketing/launch-assets.md | 97 - marketing/social-posts.md | 124 - mkdocs.yml | 2 - notebooks/06_context_cleaning.ipynb | 335 --- notebooks/08_plan_approval.ipynb | 235 -- notebooks/09_optional_semantic_models.ipynb | 250 -- notebooks/10_learning_profiles.ipynb | 474 ---- scripts/debug_dataset_sweep.py | 278 --- scripts/debug_datasets.py | 147 -- scripts/generate_html_examples.py | 67 - training/COMPLIANCE.md | 71 - training/Makefile | 87 - training/README.md | 46 - training/__init__.py | 30 - training/cache/.gitkeep | 0 training/calibration/__init__.py | 8 - training/calibration/build_features.py | 158 -- training/calibration/eval_calibration.py | 130 - training/calibration/export_tables.py | 75 - training/calibration/fit_isotonic.py | 126 - training/common.py | 107 - training/corruptors/__init__.py | 19 - training/corruptors/base.py | 209 -- training/corruptors/context.py | 235 -- training/corruptors/learning.py | 51 - training/corruptors/registry.py | 34 - training/corruptors/representation.py | 139 -- training/corruptors/scale.py | 34 - training/corruptors/semantic_values.py | 314 --- training/datasets/__init__.py | 1 - training/datasets/build_corrupted.py | 146 -- training/datasets/build_seed.py | 71 - training/datasets/splits.py | 47 - training/datasets/validators.py | 179 -- training/distill/__init__.py | 1 - training/distill/export_onnx.py | 121 - training/distill/linear.py | 287 --- training/distill/package_artifacts.py | 248 -- training/distill/quantize.py | 93 - training/distill/train_encoder_contrastive.py | 227 -- training/distill/train_intent_head.py | 406 --- training/distill/train_role_head.py | 271 -- training/eval/__init__.py | 1 - training/eval/data/role_eval_human.jsonl | 75 - training/eval/human_verified.py | 68 - training/model_cards/calib-v1.md | 64 - training/model_cards/fd-col-encoder-v1.md | 82 - training/model_cards/fd-intent-v1.md | 78 - training/seed/LICENSES.md | 33 - training/seed/README.md | 24 - training/seed/__init__.py | 1 - training/seed/licenses/apache-2.0-notice.txt | 7 - training/seed/licenses/cc-by-4.0.txt | 7 - training/seed/licenses/cc0-1.0.txt | 9 - training/seed/registry.json | 38 - .../sources/handauthored_ecommerce_v1.csv | 25 - training/seed/synthetic/__init__.py | 17 - training/seed/synthetic/generators.py | 283 --- training/teacher/__init__.py | 15 - training/teacher/cache.py | 79 - training/teacher/clients.py | 103 - training/teacher/compliance.py | 162 -- training/teacher/compliance_ledger.json | 24 - training/teacher/prompts/ambiguity.txt | 9 - training/teacher/prompts/column_role.txt | 10 - training/teacher/prompts/paraphrase.txt | 8 - training/teacher/prompts/rationale.txt | 7 - training/teacher/prompts/realism.txt | 8 - training/teacher/prompts/redteam.txt | 8 - training/teacher/review.py | 117 - training/teacher/schemas.py | 97 - training/teacher/tasks.py | 177 -- 167 files changed, 5 insertions(+), 19153 deletions(-) delete mode 100644 admin-actions.md delete mode 100644 benchmarks/bench_entity_resolution.py delete mode 100644 benchmarks/bench_fieldcheck.py delete mode 100644 benchmarks/bench_native_semantic.py delete mode 100644 benchmarks/bench_quick.py delete mode 100644 benchmarks/bench_streaming.py delete mode 100644 benchmarks/cleanbench/t4_profiles.py delete mode 100644 benchmarks/corpus/scoring.py delete mode 100644 benchmarks/performance/semantic_probe.py delete mode 100644 benchmarks/truthbench/inventory.py delete mode 100644 crates/freshcore/Cargo.lock delete mode 100644 crates/freshcore/Cargo.toml delete mode 100644 crates/freshcore/src/arrays.rs delete mode 100644 crates/freshcore/src/kernels/casts.rs delete mode 100644 crates/freshcore/src/kernels/duplicates.rs delete mode 100644 crates/freshcore/src/kernels/missing.rs delete mode 100644 crates/freshcore/src/kernels/mod.rs delete mode 100644 crates/freshcore/src/kernels/outliers.rs delete mode 100644 crates/freshcore/src/kernels/profile.rs delete mode 100644 crates/freshcore/src/kernels/strings.rs delete mode 100644 crates/freshcore/src/lib.rs delete mode 100644 crates/freshcore/src/plan.rs delete mode 100644 crates/freshcore/src/python.rs delete mode 100644 docs/developer-training-pipeline.md delete mode 100644 docs/freshcore.md delete mode 100644 examples/04_profiling.py delete mode 100644 examples/cdc_event_time_gate.py delete mode 100644 examples/cross_source_link.py delete mode 100644 examples/entity_resolution_duckdb.py delete mode 100644 examples/plugins/custom_backend/keyword_backend.py delete mode 100644 examples/plugins/custom_expert/acronym_expert.py delete mode 100644 examples/plugins/custom_validator/min_rows_validator.py delete mode 100644 examples/privacy_anonymization.py delete mode 100644 examples/schema_drift_monitoring.py delete mode 100644 examples/validation_suite_quickstart.py delete mode 100644 freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json delete mode 100644 freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json delete mode 100644 freshdata-benchmarks/.asv/results/benchmarks.json delete mode 100644 freshdata-benchmarks/METHODOLOGY.md delete mode 100644 freshdata-benchmarks/README.md delete mode 100644 freshdata-benchmarks/asv.conf.json delete mode 100644 freshdata-benchmarks/bench/artifacts/benchmark_output.txt delete mode 100644 freshdata-benchmarks/bench/artifacts/env_info.json delete mode 100755 freshdata-benchmarks/bench/run_benchmarks.sh delete mode 100644 freshdata-benchmarks/benchmarks/__init__.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_columns.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_duplicates.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_encoding.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_groupagg.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_io.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_memory.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_missing.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_outliers.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_pipeline.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_scaling.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_strings.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_types.py delete mode 100644 freshdata-benchmarks/benchmarks/benchmark_validation.py delete mode 100644 freshdata-benchmarks/benchmarks/datasets/__init__.py delete mode 100644 freshdata-benchmarks/benchmarks/datasets/data_generator.py delete mode 100644 freshdata-benchmarks/benchmarks/utils/__init__.py delete mode 100644 freshdata-benchmarks/benchmarks/utils/fairness.py delete mode 100644 freshdata-benchmarks/benchmarks/utils/library_wrappers.py delete mode 100644 freshdata-benchmarks/benchmarks/utils/metrics.py delete mode 100644 freshdata-benchmarks/reports/results_extracted.json delete mode 100644 freshdata-benchmarks/reports/summary.md delete mode 100644 freshdata-benchmarks/requirements.txt delete mode 100644 freshdata-benchmarks/scripts/capture_env.py delete mode 100644 freshdata-benchmarks/scripts/generate_plots.py delete mode 100644 freshdata-benchmarks/scripts/generate_reports.py delete mode 100644 freshdata-benchmarks/setup.py delete mode 100644 growth-audit.md delete mode 100644 marketing/README.md delete mode 100644 marketing/blog-ideas.md delete mode 100644 marketing/contributor-posts.md delete mode 100644 marketing/demo-scripts.md delete mode 100644 marketing/launch-assets.md delete mode 100644 marketing/social-posts.md delete mode 100644 notebooks/06_context_cleaning.ipynb delete mode 100644 notebooks/08_plan_approval.ipynb delete mode 100644 notebooks/09_optional_semantic_models.ipynb delete mode 100644 notebooks/10_learning_profiles.ipynb delete mode 100644 scripts/debug_dataset_sweep.py delete mode 100644 scripts/debug_datasets.py delete mode 100644 scripts/generate_html_examples.py delete mode 100644 training/COMPLIANCE.md delete mode 100644 training/Makefile delete mode 100644 training/README.md delete mode 100644 training/__init__.py delete mode 100644 training/cache/.gitkeep delete mode 100644 training/calibration/__init__.py delete mode 100644 training/calibration/build_features.py delete mode 100644 training/calibration/eval_calibration.py delete mode 100644 training/calibration/export_tables.py delete mode 100644 training/calibration/fit_isotonic.py delete mode 100644 training/common.py delete mode 100644 training/corruptors/__init__.py delete mode 100644 training/corruptors/base.py delete mode 100644 training/corruptors/context.py delete mode 100644 training/corruptors/learning.py delete mode 100644 training/corruptors/registry.py delete mode 100644 training/corruptors/representation.py delete mode 100644 training/corruptors/scale.py delete mode 100644 training/corruptors/semantic_values.py delete mode 100644 training/datasets/__init__.py delete mode 100644 training/datasets/build_corrupted.py delete mode 100644 training/datasets/build_seed.py delete mode 100644 training/datasets/splits.py delete mode 100644 training/datasets/validators.py delete mode 100644 training/distill/__init__.py delete mode 100644 training/distill/export_onnx.py delete mode 100644 training/distill/linear.py delete mode 100644 training/distill/package_artifacts.py delete mode 100644 training/distill/quantize.py delete mode 100644 training/distill/train_encoder_contrastive.py delete mode 100644 training/distill/train_intent_head.py delete mode 100644 training/distill/train_role_head.py delete mode 100644 training/eval/__init__.py delete mode 100644 training/eval/data/role_eval_human.jsonl delete mode 100644 training/eval/human_verified.py delete mode 100644 training/model_cards/calib-v1.md delete mode 100644 training/model_cards/fd-col-encoder-v1.md delete mode 100644 training/model_cards/fd-intent-v1.md delete mode 100644 training/seed/LICENSES.md delete mode 100644 training/seed/README.md delete mode 100644 training/seed/__init__.py delete mode 100644 training/seed/licenses/apache-2.0-notice.txt delete mode 100644 training/seed/licenses/cc-by-4.0.txt delete mode 100644 training/seed/licenses/cc0-1.0.txt delete mode 100644 training/seed/registry.json delete mode 100644 training/seed/sources/handauthored_ecommerce_v1.csv delete mode 100644 training/seed/synthetic/__init__.py delete mode 100644 training/seed/synthetic/generators.py delete mode 100644 training/teacher/__init__.py delete mode 100644 training/teacher/cache.py delete mode 100644 training/teacher/clients.py delete mode 100644 training/teacher/compliance.py delete mode 100644 training/teacher/compliance_ledger.json delete mode 100644 training/teacher/prompts/ambiguity.txt delete mode 100644 training/teacher/prompts/column_role.txt delete mode 100644 training/teacher/prompts/paraphrase.txt delete mode 100644 training/teacher/prompts/rationale.txt delete mode 100644 training/teacher/prompts/realism.txt delete mode 100644 training/teacher/prompts/redteam.txt delete mode 100644 training/teacher/review.py delete mode 100644 training/teacher/schemas.py delete mode 100644 training/teacher/tasks.py diff --git a/.gitignore b/.gitignore index a37d06cc..dcad72b5 100644 --- a/.gitignore +++ b/.gitignore @@ -34,12 +34,9 @@ benchmarks/results/performance/* benchmarks/generated_fixtures/ # Rust native backend build output -crates/freshcore/target/ # Phase 5 training pipeline: teacher prompt/response audit cache is a local # dev artifact (regenerated by running teacher tasks), not committed content. -training/cache/* -!training/cache/.gitkeep benchmarks/gauntlet/results/ # TruthBench: the full run output is large and regenerated by the release # command; only the compact README and the regression baseline are committed. diff --git a/MANIFEST.in b/MANIFEST.in index 3384efb1..a07d895a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,6 @@ include README.md include LICENSE include pyproject.toml recursive-include src/freshdata *.py py.typed -recursive-include crates/freshcore Cargo.toml Cargo.lock *.rs prune tests prune benchmarks prune scripts diff --git a/Makefile b/Makefile index eb686466..61dfea26 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,6 @@ PY ?= python -# training-* targets are matched by the pattern rule below (pattern rules -# cannot be .PHONY; the delegated targets are .PHONY inside training/Makefile). .PHONY: help benchmark benchmark-ci benchmark-report benchmark-fixtures benchmark-test \ cleanbench-full truthbench-release truthbench-pr automation-pr performance-ci \ performance-baseline performance-profile performance-report coverage-report \ @@ -25,7 +23,6 @@ help: @echo " performance-profile Profile one 100k-row performance case" @echo " performance-report Analyze and render compact performance evidence" @echo " coverage-report Show per-module coverage for the fast test lane" - @echo " training-* Phase-5 training pipeline (see training/Makefile)" @echo " constraints Refresh uv.lock and the pinned constraints/ci.txt CI installs with" # Refresh the lockfile and the pip constraints the gating CI jobs install with. @@ -69,10 +66,6 @@ truthbench-pr: automation-pr: $(PY) scripts/automate_freshdata.py --output-dir scripts/.automation_out -# Phase-5 training pipeline targets delegate to training/Makefile. -training-%: - $(MAKE) -C training PY=$(PY) $@ - # Full-scale local run. Override sizes per fixture by editing DEFAULT_SIZES in # benchmarks/bench.py, or call bench.py single --size for the 5M+ variants. benchmark: diff --git a/admin-actions.md b/admin-actions.md deleted file mode 100644 index 2a549a36..00000000 --- a/admin-actions.md +++ /dev/null @@ -1,71 +0,0 @@ -# FreshData GitHub Repository Administrative Actions - -The following administrative actions require GitHub repository administrator privileges on [`FreshCode-Org/freshdata`](https://github.com/FreshCode-Org/freshdata). These settings cannot be altered directly via Git commits. - ---- - -## 1. Repository Details & Description - -Navigate to the repository homepage `https://github.com/FreshCode-Org/freshdata` and click the **⚙️ (Edit repository details)** button on the top-right of the "About" box: - -* **Description**: - ```text - Automated, explainable data cleaning for pandas and Polars — repair messy tabular data safely and see exactly what changed. - ``` -* **Website**: - ```text - https://freshcode-org.github.io/freshdata/ - ``` - ---- - -## 2. GitHub Repository Topics (Discoverability & Search Intent) - -In the same "About" settings dialog, set the following **15 developer-search-intent topics** (replace any stale or generic tags): - -```text -data-cleaning -data-quality -data-preprocessing -dataframe -pandas -polars -python -tabular-data -data-validation -etl -data-engineering -data-science -machine-learning -missing-values -outlier-detection -``` - -> **Why these tags**: Developers actively search GitHub for `data-cleaning`, `missing-values`, and `pandas` when looking for libraries to replace fragile manual preprocessing scripts. - ---- - -## 3. GitHub Discussions Setup - -Navigate to **Settings** $\rightarrow$ **General** $\rightarrow$ **Features** $\rightarrow$ Check **Discussions**. - -Once enabled, navigate to the **Discussions** tab $\rightarrow$ **Categories** (click edit pencil) and configure these 6 official categories: - -| Category | Format | Description | -|---|---|---| -| 📢 **Announcements** | Announcement (maintainers only) | Project releases, benchmarks, and community roadmap updates | -| 💬 **Q&A** | Q&A (mark answers) | Ask how to clean specific messy datasets or configure rules | -| 💡 **Ideas** | Open discussion | Share ideas for new cleaning heuristics, formats, or algorithms | -| 🙌 **Show and Tell** | Open discussion | Share datasets, pipelines, and projects you cleaned using FreshData | -| ⚡ **Benchmarks** | Open discussion | Share benchmark runs across different machines and dataset sizes | -| 🔌 **Integrations** | Open discussion | Discuss integrations with Polars, DuckDB, Airflow, dbt, and GX | - ---- - -## 4. Public Issues & Pull Requests Settings - -Ensure the following are enabled under **Settings** $\rightarrow$ **General**: - -* **Issues**: Checked (Enabled) -* **Automatically delete head branches**: Checked (keeps repository clean after PR merges) -* **Squash and merge**: Allowed (recommended with PR title and commit description) diff --git a/benchmarks/bench_entity_resolution.py b/benchmarks/bench_entity_resolution.py deleted file mode 100644 index 5cdb8ef8..00000000 --- a/benchmarks/bench_entity_resolution.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Benchmark for freshdata probabilistic entity resolution. - -Generates a synthetic dataset with *known* duplicate labels, runs the -rule-weighted linkage pipeline, and reports the metrics that matter for a -record-linkage system: - -* **candidate-pair reduction ratio** — how much blocking shrinks the O(n^2) - comparison space (1.0 means everything was pruned, 0.0 means full cartesian). -* **runtime** — wall-clock for the resolve call. -* **precision / recall / F1** — measured against the ground-truth duplicate - pairs (available because the synthetic generator tags each row's true entity). - -It runs both the ``pandas`` and ``duckdb`` backends so they can be compared -side by side (DuckDB is skipped automatically if not installed). - -> **Splink-class linkage target; current implementation is rule-weighted, not -> EM-trained unless calibration is enabled.** - - python benchmarks/bench_entity_resolution.py --rows 5000 --dup-rate 0.3 --json -""" - -from __future__ import annotations - -import argparse -import importlib.util -import json -import sys -import time -from itertools import combinations - -import numpy as np -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import BlockingRule, ComparisonLevel, EntityResolutionConfig - -_FIRST = ["alice", "bob", "carol", "dan", "erin", "frank", "grace", "heidi"] -_LAST = ["smith", "jones", "lee", "patel", "kim", "garcia", "khan", "obrien"] - - -def synth(rows: int, dup_rate: float, *, seed: int = 0) -> pd.DataFrame: - """Build *rows* records; a ``dup_rate`` fraction are noisy copies of earlier - records. ``entity_label`` is the ground-truth identity. - """ - rng = np.random.default_rng(seed) - records: list[dict] = [] - labels: list[int] = [] - next_label = 0 - for i in range(rows): - make_dup = records and rng.random() < dup_rate - if make_dup: - src_idx = int(rng.integers(0, len(records))) - base = records[src_idx] - label = labels[src_idx] - email = base["email"] - name = base["name"] - if rng.random() < 0.4: # perturb the name slightly - name = name + "e" if not name.endswith("e") else name[:-1] - if rng.random() < 0.3: # perturb the email casing - email = email.upper() - rec = { - "name": name, - "email": email, - "dob": base["dob"], - "phone": base["phone"], - } - else: - label = next_label - next_label += 1 - fn = rng.choice(_FIRST) - ln = rng.choice(_LAST) - rec = { - "name": f"{fn} {ln}", - "email": f"{fn}.{ln}{label}@mail.test", - "dob": f"19{rng.integers(60, 99):02d}-{rng.integers(1, 13):02d}-" - f"{rng.integers(1, 28):02d}", - "phone": f"555{rng.integers(1000000, 9999999)}", - } - rec["id"] = i - rec["entity_label"] = label - records.append(rec) - labels.append(label) - df = pd.DataFrame(records) - return df.sample(frac=1.0, random_state=seed).reset_index(drop=True) - - -def _config(backend: str) -> EntityResolutionConfig: - return EntityResolutionConfig( - enabled=True, - backend=backend, # type: ignore[arg-type] - unique_id_column="id", - blocking_rules=( - BlockingRule("lower(l.email) = lower(r.email)", "same email"), - BlockingRule( - "l.dob = r.dob and substr(lower(l.name), 1, 3) = " - "substr(lower(r.name), 1, 3)", - "same dob + name prefix", - ), - ), - comparisons=( - ComparisonLevel("name", "jaro_winkler", threshold=0.85, weight=2.0), - ComparisonLevel("email", "jaro_winkler", threshold=0.90, weight=3.0), - ComparisonLevel("dob", "exact", weight=2.0), - ComparisonLevel("phone", "levenshtein", threshold=0.85, weight=1.0), - ), - match_threshold=0.82, - clerical_review_threshold=0.6, - ) - - -def truth_pairs(df: pd.DataFrame) -> set[tuple]: - """All ground-truth duplicate id-pairs (order-independent).""" - out: set[tuple] = set() - for _, group in df.groupby("entity_label"): - ids = sorted(group["id"].tolist()) - for a, b in combinations(ids, 2): - out.add((a, b)) - return out - - -def _norm(a, b) -> tuple: - return (a, b) if a <= b else (b, a) - - -def evaluate(df: pd.DataFrame, backend: str) -> dict: - config = _config(backend) - t0 = time.perf_counter() - _, report = fd.resolve_entities(df, config=config) - runtime = time.perf_counter() - t0 - - n = len(df) - total_pairs = n * (n - 1) // 2 - reduction = 1.0 - (report.n_candidate_pairs / total_pairs) if total_pairs else 0.0 - - predicted = {_norm(p.left_id, p.right_id) for p in report.pairs if p.decision == "match"} - truth = truth_pairs(df) - tp = len(predicted & truth) - fp = len(predicted - truth) - fn = len(truth - predicted) - precision = tp / (tp + fp) if (tp + fp) else 0.0 - recall = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 - - return { - "backend": backend, - "n_records": n, - "total_pairs": total_pairs, - "candidate_pairs": report.n_candidate_pairs, - "reduction_ratio": round(reduction, 6), - "runtime_s": round(runtime, 4), - "matches": report.n_matches, - "clusters": report.n_clusters, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - } - - -# ===================================================================== -# Labelled-dataset mode (benchmarks/data/er_labelled_*.csv) -# ===================================================================== - - -def _labelled_config(null_policy: str, mode: str) -> EntityResolutionConfig: - """Linkage config for the labelled dataset schema (see gen_er_labelled.py).""" - return EntityResolutionConfig( - enabled=True, - backend="pandas", - unique_id_column="id", - blocking_rules=( - BlockingRule("lower(l.email) = lower(r.email)", "same email"), - BlockingRule( - "l.dob = r.dob and substr(lower(l.last_name), 1, 2) = " - "substr(lower(r.last_name), 1, 2)", - "same dob + surname prefix", - ), - BlockingRule("l.phone = r.phone", "same phone"), - ), - comparisons=( - ComparisonLevel("full_name", "token_set", weight=2.0), - ComparisonLevel("full_name", "jaro_winkler", threshold=0.85, weight=1.5), - ComparisonLevel("last_name", "metaphone", weight=1.0), - ComparisonLevel("email", "jaro_winkler", threshold=0.90, weight=2.5), - ComparisonLevel("dob", "exact", weight=2.0), - ComparisonLevel("first_name", "jaro_winkler", threshold=0.85, weight=1.0), - ), - null_policy=null_policy, # type: ignore[arg-type] - mode=mode, # type: ignore[arg-type] - ) - - -def _false_merge_count(report, df: pd.DataFrame) -> int: - """Clusters that merged more than one true entity (the costly failure).""" - label_of = dict(zip(df["id"], df["entity_id"])) - return sum( - 1 - for c in report.clusters - if len({label_of[rid] for rid in c.record_ids if rid in label_of}) > 1 - ) - - -def _peak_rss_mb() -> float | None: - try: - import resource - - peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return round(peak / (1 << 20) if sys.platform == "darwin" else peak / 1024, 1) - except ImportError: # pragma: no cover - return None - - -def evaluate_labelled(df: pd.DataFrame, null_policy: str, mode: str) -> dict: - config = _labelled_config(null_policy, mode) - t0 = time.perf_counter() - _, report = fd.resolve_entities(df, config=config) - runtime = time.perf_counter() - t0 - - n = len(df) - total_pairs = n * (n - 1) // 2 - predicted = {_norm(p.left_id, p.right_id) for p in report.pairs if p.decision == "match"} - truth = truth_pairs(df.rename(columns={"entity_id": "entity_label"})) - tp, fp, fn = ( - len(predicted & truth), - len(predicted - truth), - len(truth - predicted), - ) - precision = tp / (tp + fp) if (tp + fp) else 0.0 - recall = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 - return { - "config": f"null_policy={null_policy}, mode={mode}", - "null_policy": null_policy, - "mode": mode, - "n_records": n, - "candidate_pairs": report.n_candidate_pairs, - "reduction_ratio": round(1.0 - report.n_candidate_pairs / total_pairs, 6) - if total_pairs - else 0.0, - "true_pairs": len(truth), - "tp": tp, - "fp": fp, - "fn": fn, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "false_merged_clusters": _false_merge_count(report, df), - "n_clusters": report.n_clusters, - "runtime_s": round(runtime, 4), - "peak_rss_mb": _peak_rss_mb(), - } - - -def evaluate_exact_baseline(df: pd.DataFrame) -> dict: - """pandas drop_duplicates on identity fields — the honesty baseline.""" - fields = ["full_name", "email", "phone", "dob"] - t0 = time.perf_counter() - dup_mask = df.duplicated(subset=fields, keep=False) - runtime = time.perf_counter() - t0 - predicted: set[tuple] = set() - for _, group in df[dup_mask].groupby(fields, dropna=False): - ids = sorted(group["id"].tolist()) - for a, b in combinations(ids, 2): - predicted.add((a, b)) - truth = truth_pairs(df.rename(columns={"entity_id": "entity_label"})) - tp, fp, fn = ( - len(predicted & truth), - len(predicted - truth), - len(truth - predicted), - ) - precision = tp / (tp + fp) if (tp + fp) else 1.0 - recall = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 - return { - "config": "pandas exact drop_duplicates (baseline)", - "n_records": len(df), - "true_pairs": len(truth), - "tp": tp, - "fp": fp, - "fn": fn, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "false_merged_clusters": 0, - "runtime_s": round(runtime, 4), - } - - -_SWEEP = [ - ("penalize", "balanced"), - ("neutral", "balanced"), - ("neutral", "precision"), - ("neutral", "recall"), -] - - -def run_labelled(path: str, write_results: str | None) -> list[dict]: - df = pd.read_csv(path) - results = [evaluate_exact_baseline(df)] - results += [evaluate_labelled(df, np_, m) for np_, m in _SWEEP] - if write_results: - import platform - - payload = { - "dataset": path, - "generator": "benchmarks/gen_er_labelled.py", - "machine": f"{platform.system()} {platform.machine()}", - "python": platform.python_version(), - "pandas": pd.__version__, - "freshdata": fd.__version__, - "method": "rule-weighted linkage (normalized weighted average; " - "no EM / Fellegi-Sunter)", - "results": results, - } - with open(write_results, "w", encoding="utf-8") as fh: - json.dump(payload, fh, indent=2) - return results - - -def main(argv: list[str] | None = None) -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--rows", type=int, default=2000) - ap.add_argument("--dup-rate", type=float, default=0.3) - ap.add_argument("--seed", type=int, default=0) - ap.add_argument("--backend", choices=["pandas", "duckdb", "both"], default="both") - ap.add_argument("--json", action="store_true", help="emit machine-readable JSON") - ap.add_argument( - "--labelled", - metavar="CSV", - help="run the labelled-dataset accuracy sweep (see gen_er_labelled.py)", - ) - ap.add_argument( - "--write-results", metavar="JSON", help="with --labelled: write results here" - ) - args = ap.parse_args(argv) - - if args.labelled: - results = run_labelled(args.labelled, args.write_results) - if args.json: - print(json.dumps(results, indent=2)) - else: - for r in results: - print( - f"[{r['config']:<44}] P={r['precision']:.4f} " - f"R={r['recall']:.4f} F1={r['f1']:.4f} " - f"FP={r['fp']} FN={r['fn']} " - f"false-merges={r['false_merged_clusters']}" - ) - return 0 - - df = synth(args.rows, args.dup_rate, seed=args.seed) - backends = ["pandas", "duckdb"] if args.backend == "both" else [args.backend] - - results = [] - for backend in backends: - if backend == "duckdb" and importlib.util.find_spec("duckdb") is None: - print("duckdb not installed — skipping duckdb backend", file=sys.stderr) - continue - results.append(evaluate(df, backend)) - - if args.json: - print(json.dumps(results, indent=2)) - else: - for r in results: - print( - f"[{r['backend']:>6}] n={r['n_records']} " - f"pairs {r['candidate_pairs']}/{r['total_pairs']} " - f"(reduction {r['reduction_ratio']:.4f}) " - f"runtime {r['runtime_s']}s | " - f"P={r['precision']} R={r['recall']} F1={r['f1']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/bench_fieldcheck.py b/benchmarks/bench_fieldcheck.py deleted file mode 100644 index 86ea32b1..00000000 --- a/benchmarks/bench_fieldcheck.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Benchmark fd.validate_fields against regex and pandas-coercion baselines. - -Reproducible (seeded) invalid-cell detection benchmark on a synthetic -financial feed with labeled corruptions:: - - python benchmarks/bench_fieldcheck.py # default 20k rows - python benchmarks/bench_fieldcheck.py 100000 # custom row count - -For each validator we report precision / recall / F1 over the labeled -invalid cells, plus wall time and throughput. Baselines: - -* ``regex`` — per-column regular expressions only (no context, no ranges); -* ``pandas`` — ``pd.to_numeric`` / ``pd.to_datetime`` coercion NaN-diffing - (types only: no vocabulary, format or semantic checks); -* ``fieldcheck`` — ``fd.validate_fields`` with a declared schema. -""" - -from __future__ import annotations - -import json -import random -import re -import sys -import time - -import pandas as pd - -from freshdata.fieldcheck import FieldSpec, validate_fields - -SEED = 20260711 -CORRUPTION_RATE = 0.02 - -SCHEMA = { - "transaction_id": FieldSpec(semantic_type="identifier", required=True, nullable=False), - "transaction_amount": FieldSpec(semantic_type="currency_amount"), - "currency": FieldSpec(allowed_values=frozenset({"USD", "EUR", "GBP", "INR"})), - "stock_ticker": FieldSpec(semantic_type="ticker"), - "interest_rate": FieldSpec(semantic_type="rate", min_value=0, max_value=1), - "transaction_date": FieldSpec(semantic_type="date"), -} - -#: (column, corrupt value) pool — all are invalid for their column. -CORRUPTIONS = [ - ("transaction_amount", "apple"), - ("transaction_amount", "approved"), - ("transaction_amount", "12O.50"), - ("currency", "BTC"), - ("currency", "dollars"), - ("stock_ticker", "apple"), - ("stock_ticker", "AА PL"), - ("interest_rate", "high"), - ("interest_rate", "5"), # out of [0, 1] - ("transaction_date", "2026-02-30"), - ("transaction_date", "not a date"), -] - - -def make_dataset(n: int) -> tuple[pd.DataFrame, set]: - rng = random.Random(SEED) - tickers = ["AAPL", "MSFT", "GOOG", "TSLA", "BRK.B", "AMZN"] - rows = [] - for i in range(n): - rows.append({ - "transaction_id": f"T{i:07d}", - "transaction_amount": f"{rng.uniform(1, 5000):.2f}", - "currency": rng.choice(["USD", "EUR", "GBP", "INR"]), - "stock_ticker": rng.choice(tickers), - "interest_rate": f"{rng.uniform(0.001, 0.2):.4f}", - "transaction_date": f"2026-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}", - }) - df = pd.DataFrame(rows) - truth: set = set() - n_bad = max(1, int(n * CORRUPTION_RATE)) - for row in rng.sample(range(n), n_bad): - col, bad = rng.choice(CORRUPTIONS) - df.loc[row, col] = bad - truth.add((row, col)) - return df, truth - - -# --- baselines --------------------------------------------------------------- - -REGEXES = { - "transaction_id": re.compile(r"^T\d{7}$"), - "transaction_amount": re.compile(r"^-?\d+(\.\d+)?$"), - "currency": re.compile(r"^[A-Z]{3}$"), - "stock_ticker": re.compile(r"^[A-Z]{1,6}([.\-][A-Z0-9]{1,4})?$"), - "interest_rate": re.compile(r"^-?\d+(\.\d+)?$"), - "transaction_date": re.compile(r"^\d{4}-\d{2}-\d{2}$"), -} - - -def regex_validator(df: pd.DataFrame) -> set: - flagged = set() - for col, rx in REGEXES.items(): - bad = ~df[col].astype(str).str.fullmatch(rx) - flagged.update((row, col) for row in df.index[bad]) - return flagged - - -def pandas_validator(df: pd.DataFrame) -> set: - flagged = set() - for col in ("transaction_amount", "interest_rate"): - bad = pd.to_numeric(df[col], errors="coerce").isna() & df[col].notna() - flagged.update((row, col) for row in df.index[bad]) - dates = pd.to_datetime(df["transaction_date"], errors="coerce", format="%Y-%m-%d") - bad = dates.isna() & df["transaction_date"].notna() - flagged.update((row, "transaction_date") for row in df.index[bad]) - return flagged - - -def fieldcheck_validator(df: pd.DataFrame) -> set: - report = validate_fields(df, SCHEMA) - return {(i.row, i.column) for i in report.issues if i.severity == "error"} - - -# --- scoring ----------------------------------------------------------------- - - -def score(name: str, fn, df: pd.DataFrame, truth: set) -> dict: - start = time.perf_counter() - flagged = fn(df) - elapsed = time.perf_counter() - start - tp = len(flagged & truth) - fp = len(flagged - truth) - fn_ = len(truth - flagged) - precision = tp / (tp + fp) if tp + fp else 0.0 - recall = tp / (tp + fn_) if tp + fn_ else 0.0 - f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 - return { - "validator": name, - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "false_accepts": fn_, - "false_rejects": fp, - "seconds": round(elapsed, 3), - "rows_per_s": int(len(df) / elapsed) if elapsed else None, - } - - -def main() -> None: - n = int(sys.argv[1]) if len(sys.argv) > 1 else 20_000 - df, truth = make_dataset(n) - print(f"rows={n} corrupted_cells={len(truth)} seed={SEED}\n") - results = [ - score("regex", regex_validator, df, truth), - score("pandas_coercion", pandas_validator, df, truth), - score("fd.validate_fields", fieldcheck_validator, df, truth), - ] - print(json.dumps(results, indent=2)) - print() - header = f"{'validator':<20}{'precision':>10}{'recall':>8}{'f1':>8}{'sec':>8}{'rows/s':>10}" - print(header) - for r in results: - print(f"{r['validator']:<20}{r['precision']:>10}{r['recall']:>8}" - f"{r['f1']:>8}{r['seconds']:>8}{r['rows_per_s']:>10}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/bench_native_semantic.py b/benchmarks/bench_native_semantic.py deleted file mode 100644 index 3c5a4605..00000000 --- a/benchmarks/bench_native_semantic.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Scale benchmark for the native distinct-value semantic path (Phase 6). - -The semantic stage only reasons about a column's *distinct values*. On a native -engine we extract that bounded distinct table natively (Polars group-by / DuckDB -GROUP BY), score it through the pandas gate, and map repairs back natively — so -peak memory tracks the *distinct* cardinality, not the row count. This benchmark -generates a wide, mostly-low-cardinality frame and compares: - - * pandas — reference path; whole frame in memory, semantics in pandas - * polars-lazy — native distinct path; frame scanned lazily from parquet - * duckdb — native distinct path; frame scanned from parquet - -For each it records rows/sec, peak RSS, and confirms the same repairs landed -(boolean synonyms -> Boolean, spelled numbers -> Int). The headline property: -polars-lazy / duckdb peak RSS stays far below the pandas path as ``--rows`` grows. - - python benchmarks/bench_native_semantic.py --rows 1000000 - python benchmarks/bench_native_semantic.py --rows 10000000 --engines polars-lazy,duckdb -""" - -from __future__ import annotations - -import argparse -import json -import sys -import time - -import numpy as np -import pandas as pd - -from freshdata.config import CleanConfig -from freshdata.execution import run_with_engine - -# A native-compatible config: conservative engine + no sampled dtype fixes, so -# the deterministic stages run natively and only the semantic stage is measured. -CONFIG = CleanConfig( - strategy="conservative", fix_dtypes=False, semantic_mode="auto", verbose=False -) - - -def _rss_mb() -> float: - try: - import psutil - - return psutil.Process().memory_info().rss / 1e6 - except ImportError: - import resource - - maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return maxrss / 1e6 if sys.platform == "darwin" else maxrss / 1e3 - - -def write_parquet(path: str, rows: int, batch: int, seed: int = 0) -> None: - """Write ``rows`` rows to parquet in batches — never all in memory at once.""" - import pyarrow as pa - import pyarrow.parquet as pq - - rng = np.random.default_rng(seed) - active = np.array(["yes", "y", "YES", "no", "n", "NO"], dtype=object) - qty = np.array(["one", "two", "three", "4", "5", "6"], dtype=object) - region = np.array(["North", "north", "NORTH", "South", "south"], dtype=object) - writer: pq.ParquetWriter | None = None - produced = 0 - try: - while produced < rows: - n = min(batch, rows - produced) - tbl = pa.table( - { - "active": pa.array(rng.choice(active, n)), - "qty": pa.array(rng.choice(qty, n)), - "region": pa.array(rng.choice(region, n)), - "user_id": pa.array([f"U{i}" for i in range(produced, produced + n)]), - } - ) - if writer is None: - writer = pq.ParquetWriter(path, tbl.schema) - writer.write_table(tbl) - produced += n - finally: - if writer is not None: - writer.close() - - -def _run_pandas(path: str) -> tuple[pd.DataFrame, float, float]: - t0 = time.perf_counter() - df = pd.read_parquet(path) - out = run_with_engine(df, CONFIG, engine="pandas", output_format="pandas") - return out, time.perf_counter() - t0, _rss_mb() - - -def _run_polars_lazy(path: str) -> tuple[pd.DataFrame, float, float]: - import polars as pl - - t0 = time.perf_counter() - lf = pl.scan_parquet(path) - out = run_with_engine(lf, CONFIG, engine="polars", output_format="polars-lazy") - frame = out.collect(engine="streaming").to_pandas() - return frame, time.perf_counter() - t0, _rss_mb() - - -def _run_duckdb(path: str) -> tuple[pd.DataFrame, float, float]: - import duckdb # noqa: F401 - ensure the optional dependency is present - - t0 = time.perf_counter() - # Pass the parquet path: the engine reads it through a lazy view, so the - # frame is never fully materialized before the native distinct pass. - out = run_with_engine(path, CONFIG, engine="duckdb", output_format="duckdb") - frame = out.df() - return frame, time.perf_counter() - t0, _rss_mb() - - -_RUNNERS = { - "pandas": _run_pandas, - "polars-lazy": _run_polars_lazy, - "duckdb": _run_duckdb, -} - - -def _repairs_ok(frame: pd.DataFrame) -> bool: - active = set(frame["active"]) - qty = {int(v) for v in frame["qty"]} - return active <= {True, False} and qty == {1, 2, 3, 4, 5, 6} - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--rows", type=int, default=1_000_000) - ap.add_argument("--batch-size", type=int, default=250_000) - ap.add_argument("--engines", default="pandas,polars-lazy,duckdb") - ap.add_argument("--out", default=None, help="write JSON results here") - args = ap.parse_args() - - import tempfile - - engines = [e.strip() for e in args.engines.split(",") if e.strip()] - with tempfile.NamedTemporaryFile(suffix=".parquet", delete=True) as tmp: - write_parquet(tmp.name, args.rows, args.batch_size) - results = [] - for engine in engines: - runner = _RUNNERS.get(engine) - if runner is None: - print(f"skip unknown engine {engine!r}", file=sys.stderr) - continue - try: - frame, elapsed, peak_rss = runner(tmp.name) - except ImportError as exc: - print(f"skip {engine}: {exc}", file=sys.stderr) - continue - rec = { - "engine": engine, - "rows": args.rows, - "seconds": round(elapsed, 3), - "rows_per_second": round(args.rows / elapsed) if elapsed else None, - "peak_rss_mb": round(peak_rss, 1), - "repairs_ok": _repairs_ok(frame), - } - results.append(rec) - print(json.dumps(rec)) - - payload = {"benchmark": "native_semantic", "rows": args.rows, "results": results} - if args.out: - with open(args.out, "w") as fh: - json.dump(payload, fh, indent=2) - # Every engine must produce the same repairs. - if not all(r["repairs_ok"] for r in results): - print("FAIL: repair mismatch across engines", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/bench_quick.py b/benchmarks/bench_quick.py deleted file mode 100644 index a91ae195..00000000 --- a/benchmarks/bench_quick.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Quick benchmark for freshdata.clean on synthetic and fixture data. - -Run:: - - python benchmarks/bench.py # synthetic sizes - python benchmarks/bench.py --fixtures # all test fixtures - python benchmarks/bench.py --fixtures --compare - python benchmarks/bench.py --online --compare - python benchmarks/bench.py --online-all --compare - python benchmarks/bench.py --online-all --tier 1 - python benchmarks/bench.py 10000 100000 # custom row counts -""" - -from __future__ import annotations - -import json -import sys -import time -from pathlib import Path - -import numpy as np -import pandas as pd - -import freshdata as fd - -FIXTURES_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" -ONLINE_CACHE_DIR = FIXTURES_DIR / "online" / "cache" -FIXTURES = [ - "aqi_sample", - "sales_export", - "survey_responses", - "sensor_timeseries", - "fraud_signals", - "tiny_cohort", - "wide_sparse", - "large_panel", - "duplicate_heavy", - "locale_numbers", - "mixed_roles", -] -ONLINE_REGISTRY_PATH = FIXTURES_DIR / "online" / "registry.json" -def _online_names(*, tier: int | None = None) -> list[str]: - if not ONLINE_CACHE_DIR.exists(): - return [] - cached = {p.stem for p in ONLINE_CACHE_DIR.glob("*.csv")} - if tier is None or not ONLINE_REGISTRY_PATH.exists(): - return sorted(cached) - registry = json.loads(ONLINE_REGISTRY_PATH.read_text()) - return sorted(k for k in cached if int(registry.get(k, {}).get("tier", 2)) == tier) - - -ONLINE_FIXTURES = _online_names() - - -def make_messy(n: int, seed: int = 0) -> pd.DataFrame: - rng = np.random.default_rng(seed) - names = rng.choice([" Alice ", "Bob", "carol", "N/A", "Dave "], n) - return pd.DataFrame( - { - "Customer ID": np.arange(n), - " Name ": names, - "Order Total": [f"${x:,.2f}" for x in rng.uniform(5, 9999, n)], - "Quantity": rng.integers(1, 50, n).astype(str), - "Order Date": pd.date_range("2022-01-01", periods=n, freq="min").astype(str), - "Is Member": rng.choice(["yes", "no", "YES", "No"], n), - "Notes": rng.choice(["", "-", "expedite", " gift wrap ", "null"], n), - "Unused": [None] * n, - } - ) - - -def _timed_clean(df: pd.DataFrame, *, strategy: str) -> tuple[float, int, int, int]: - fd.clean(df.head(min(len(df), 1000)), strategy=strategy, verbose=False) # warm-up - missing_before = int(df.isna().sum().sum()) - start = time.perf_counter() - _, report = fd.clean(df, strategy=strategy, return_report=True, verbose=False) - elapsed = time.perf_counter() - start - cols_delta = report.cols_after - df.shape[1] - return elapsed, len(report), report.missing_after - missing_before, cols_delta - - -def bench_synthetic(n: int, *, compare: bool = False) -> None: - df = make_messy(n) - if compare: - hdr = ( - f"{'rows':>10} {'strategy':<11} {'seconds':>8} " - f"{'rows/s':>8} {'missingΔ':>9} {'colsΔ':>6}" - ) - print(hdr) - for strategy in ("balanced", "aggressive"): - elapsed, n_actions, miss_d, cols_d = _timed_clean(df, strategy=strategy) - rate = n / elapsed if elapsed else 0 - if compare: - print( - f"{n:>10,} {strategy:<11} {elapsed:8.3f} {rate:8.0f} {miss_d:>9} {cols_d:>6}" - ) - else: - print( - f"{n:>10,} rows {strategy:<11} {elapsed:8.3f}s {rate / 1e6:6.2f}M rows/s " - f"{n_actions} actions" - ) - - -def bench_fixtures( - *, compare: bool = False, online: bool = False, tier: int | None = None -) -> None: - names = _online_names(tier=tier) if online else FIXTURES - base = ONLINE_CACHE_DIR if online else FIXTURES_DIR - header = ( - f"{'fixture':<20} {'strategy':<11} {'seconds':>8} " - f"{'rows/s':>8} {'missingΔ':>9} {'colsΔ':>6}" - ) - if compare: - print(header) - else: - print(f"{'fixture':<20} {'strategy':<11} {'seconds':>8} {'rows':>6} actions") - for name in names: - path = base / f"{name}.csv" - if not path.exists(): - continue - df = pd.read_csv(path) - for strategy in ("balanced", "aggressive"): - elapsed, n_actions, miss_d, cols_d = _timed_clean(df, strategy=strategy) - rate = len(df) / elapsed if elapsed else 0 - if compare: - row = ( - f"{name:<20} {strategy:<11} {elapsed:8.3f} " - f"{rate:8.0f} {miss_d:>9} {cols_d:>6}" - ) - print(row) - else: - print( - f"{name:<20} {strategy:<11} {elapsed:8.3f} {len(df):>6} {n_actions}" - ) - if compare: - bench_compare_table(df, name) - - -def bench_compare_table(df: pd.DataFrame, label: str = "data") -> None: - table = fd.compare_clean(df) - print(f"\n=== compare_clean: {label} ===") - print(table.to_string(index=False)) - - -if __name__ == "__main__": - print(f"freshdata {fd.__version__} | pandas {pd.__version__} | numpy {np.__version__}") - compare = "--compare" in sys.argv - online = "--online" in sys.argv or "--online-all" in sys.argv - tier = None - if "--tier" in sys.argv: - tier = int(sys.argv[sys.argv.index("--tier") + 1]) - skip = {"--fixtures", "--compare", "--online", "--online-all", "--tier"} - if "--tier" in sys.argv: - skip.add(str(tier)) - args = [a for a in sys.argv[1:] if a not in skip] - if len(sys.argv) > 1 and sys.argv[1] == "--fixtures": - bench_fixtures(compare=compare, online=online, tier=tier) - elif online and (compare or "--online-all" in sys.argv): - bench_fixtures(compare=compare or "--online-all" in sys.argv, online=True, tier=tier) - elif args and args[0] == "--compare-matrix": - df = pd.read_csv(FIXTURES_DIR / (args[1] if len(args) > 1 else "aqi_sample.csv")) - bench_compare_table(df, args[1] if len(args) > 1 else "aqi_sample") - else: - sizes = [int(a.replace("_", "")) for a in args] or [10_000, 100_000, 1_000_000] - for size in sizes: - bench_synthetic(size, compare=compare) diff --git a/benchmarks/bench_streaming.py b/benchmarks/bench_streaming.py deleted file mode 100644 index ff67cf66..00000000 --- a/benchmarks/bench_streaming.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Stable-memory benchmark for freshdata streaming / micro-batch cleaning. - -This is an **out-of-core / streaming** test, not a single-DataFrame test: synthetic -rows are generated lazily, one batch at a time, and fed to a -:class:`freshdata.StreamingCleaner`. The whole dataset is never materialized, so peak -memory should stay roughly flat as ``--rows`` grows — that flatness is the property the -"clean 100M rows out-of-core with stable memory" claim depends on. - - python benchmarks/bench_streaming.py --rows 100000000 --batch-size 100000 --cols 20 -""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from collections.abc import Iterator - -import numpy as np -import pandas as pd - -import freshdata as fd - - -def _rss_mb() -> float: - """Resident set size in MB (psutil if available, else the resource module).""" - try: - import psutil - - return psutil.Process().memory_info().rss / 1e6 - except ImportError: - import resource - - maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return maxrss / 1e6 if sys.platform == "darwin" else maxrss / 1e3 # bytes vs KB - - -def synth_batches(rows: int, batch_size: int, cols: int, *, seed: int = 0 - ) -> Iterator[pd.DataFrame]: - """Yield ``rows`` rows as lazily-generated batches — never all at once.""" - rng = np.random.default_rng(seed) - produced = 0 - t0 = np.datetime64("2020-01-01") - while produced < rows: - n = min(batch_size, rows - produced) - data: dict[str, object] = { - "customer_id": np.arange(produced, produced + n, dtype="int64"), - "churn": rng.integers(0, 2, n), - "event_time": t0 + np.arange(produced, produced + n).astype("timedelta64[s]"), - "region": rng.choice(["north", "north", "north", "south", "east", None], n), - } - for j in range(max(0, cols - len(data))): - vals = rng.normal(50, 12, n) - vals[rng.random(n) < 0.1] = np.nan # ~10% missing - vals[rng.random(n) < 0.005] *= 40 # rare outliers - data[f"metric_{j}"] = vals - produced += n - yield pd.DataFrame(data) - - -def run_benchmark(*, rows: int, batch_size: int, cols: int) -> dict: - """Stream ``rows`` rows through a StreamingCleaner and measure memory/throughput.""" - cleaner = fd.StreamingCleaner(target_column="churn", id_columns=("customer_id",), - window_size=batch_size, warmup_batches=3) - base = _rss_mb() - peak = base - growth_samples: list[float] = [] - processed = 0 - start = time.perf_counter() - for cleaned, report in cleaner.clean_batches(synth_batches(rows, batch_size, cols)): - processed += len(cleaned) - rss = _rss_mb() - peak = max(peak, rss) - # Sample post-warmup RSS so growth reflects the steady state, not startup. - if report.streaming["batch_id"] > 3: - growth_samples.append(round(rss, 1)) - elapsed = time.perf_counter() - start - final = cleaner.finalize() - - if processed < rows: - raise SystemExit(f"FAIL: processed {processed:,} of {rows:,} requested rows") - - # Stable memory = the steady-state *tail* is flat. The allocator ramps to a - # working-set ceiling over the first batches (not a function of total rows); - # measuring the second half of the run excludes that ramp. - tail = growth_samples[len(growth_samples) // 2:] or growth_samples - steady_growth = round(max(tail) - min(tail), 1) if tail else 0.0 - return { - "rows_processed": processed, - "batch_size": batch_size, - "columns": cols, - "elapsed_seconds": round(elapsed, 2), - "rows_per_second": int(processed / elapsed) if elapsed else 0, - "baseline_memory_mb": round(base, 1), - "peak_memory_mb": round(peak, 1), - "warmup_ramp_mb": round(peak - base, 1), - "steady_state_growth_mb": steady_growth, - "steady_state_samples_mb": tail, - "final_cumulative_trust_score": final.streaming["cumulative_trust_score"], - # Flat tail (no linear growth with rows) is the acceptance criterion. - "stable_memory": steady_growth < max(50.0, 0.15 * peak), - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="freshdata streaming stable-memory benchmark") - parser.add_argument("--rows", type=int, default=1_000_000) - parser.add_argument("--batch-size", type=int, default=100_000) - parser.add_argument("--cols", type=int, default=20) - parser.add_argument("--report", metavar="FILE", help="write the benchmark JSON here") - args = parser.parse_args(argv) - - result = run_benchmark(rows=args.rows, batch_size=args.batch_size, cols=args.cols) - text = json.dumps(result, indent=2) - if args.report: - with open(args.report, "w") as fh: - fh.write(text) - print(text) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cleanbench/t4_profiles.py b/benchmarks/cleanbench/t4_profiles.py deleted file mode 100644 index 9251bbee..00000000 --- a/benchmarks/cleanbench/t4_profiles.py +++ /dev/null @@ -1,463 +0,0 @@ -"""CleanBench T4 — learned-profile replay benchmark (Phase 4). - -Ten fixture families, each a (train messy, train clean) pair plus a held-out -test batch corrupted by the *same* family. A profile is learned from the -train pair and replayed on the test batch; the gates compare against a -no-profile baseline: - -* mean cell-repair F1 lift on the replayable families >= +15 points, -* false-modification rate with the profile <= without it, -* protected-column violations == 0, -* zero raw sensitive literals inside the serialized profile under - privacy="mask". - -The corruptions are deliberately ones the no-profile pipeline cannot fully -solve on its own (domain vocabularies, nonstandard sentinels, ambiguous -day-first dates, region-less phone formats), so the lift measures what the -*profile* adds — not what the deterministic experts already do. -""" - -from __future__ import annotations - -import io -import tempfile -import zipfile -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -import freshdata as fd -from freshdata.learning import learn - -from .metrics import ( - cell_repair_f1, - false_modification_rate, - protected_column_violation_rate, -) - -__all__ = [ - "T4Fixture", - "T4Result", - "make_t4_fixtures", - "privacy_leak_count", - "profile_drift_block_rate", - "run_t4", - "run_t4_fixture", -] - -_TRAIN_ROWS = 60 -_TEST_ROWS = 30 - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@dataclass -class T4Fixture: - """One corruption family: a train pair and a same-family test batch.""" - - name: str - train_messy: pd.DataFrame - train_clean: pd.DataFrame - test_messy: pd.DataFrame - test_truth: pd.DataFrame - key: str = "id" - context: str = "id is a unique identifier. Never modify id." - protected_columns: tuple[str, ...] = ("id",) - #: Raw sensitive strings from the train pair that must never appear in - #: the serialized profile under privacy="mask". - sensitive_literals: tuple[str, ...] = () - #: Families like imputation/unexplained replay nothing by design and are - #: excluded from the F1-lift gate (they still run the safety gates). - expect_lift: bool = True - learn_kwargs: dict[str, Any] = field(default_factory=dict) - - -def _ids(n: int, prefix: str) -> list[str]: - return [f"{prefix}{i:04d}" for i in range(n)] - - -def _cycle(values: list, n: int) -> list: - return [values[i % len(values)] for i in range(n)] - - -def _pair_frames( - n: int, - prefix: str, - column: str, - messy_values: list, - clean_values: list, -) -> tuple[pd.DataFrame, pd.DataFrame]: - """Aligned (messy, clean) frames: one corrupted column plus stable ballast.""" - ids = _ids(n, prefix) - ballast = [round(10.0 + (i % 7) * 1.5, 2) for i in range(n)] - messy = pd.DataFrame({"id": ids, column: _cycle(messy_values, n), "ballast": ballast}) - clean = pd.DataFrame({"id": ids, column: _cycle(clean_values, n), "ballast": ballast}) - return messy, clean - - -def _vocab_fixture( - name: str, - column: str, - mapping: dict[str, object], -) -> T4Fixture: - """A fixture whose corruption is a domain vocabulary (raw -> clean).""" - raws = list(mapping) - cleans = [mapping[r] for r in raws] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", column, raws, cleans) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", column, raws, cleans) - return T4Fixture( - name=name, - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - ) - - -def _make_category_map_fixture() -> T4Fixture: - # Department codes: an internal vocabulary no generic cleaner knows. - return _vocab_fixture( - "category_map_departments", - "department", - { - "H.R.": "human_resources", - "HR ": "human_resources", - "Fin": "finance", - "FIN.": "finance", - "Eng": "engineering", - "ENGG": "engineering", - }, - ) - - -def _make_status_typos_fixture() -> T4Fixture: - return _vocab_fixture( - "status_reference_typos", - "status", - { - "Deliverd": "delivered", - "delivred": "delivered", - "SHIPPED": "shipped", - "shiped": "shipped", - "In Transit": "in_transit", - "IN-TRANSIT": "in_transit", - }, - ) - - -def _make_condition_vocab_fixture() -> T4Fixture: - return _vocab_fixture( - "condition_abbreviations", - "condition", - { - "NIB": "new_in_box", - "N.I.B.": "new_in_box", - "LN": "like_new", - "L.N.": "like_new", - "FR": "for_repair", - }, - ) - - -def _make_city_typos_fixture() -> T4Fixture: - return _vocab_fixture( - "city_reference_typos", - "city", - { - "Bengalru": "Bengaluru", - "Bangalore": "Bengaluru", - "Mumbay": "Mumbai", - "Bombay": "Mumbai", - "Dehli": "Delhi", - "New Dehli": "Delhi", - }, - ) - - -def _make_sentinel_fixture() -> T4Fixture: - # Nonstandard sentinels the default sentinel set does not contain. - raws = ["MISSING_VAL", "-999", "n.v.", "42", "77", "13"] - cleans = [np.nan, np.nan, np.nan, 42.0, 77.0, 13.0] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", "reading", raws, cleans) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", "reading", raws, cleans) - return T4Fixture( - name="nonstandard_sentinels", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - ) - - -def _make_dayfirst_fixture() -> T4Fixture: - # Ambiguous day-first dates: without the learned dayfirst delta the - # baseline parses them month-first and gets *wrong* timestamps. - raw_dates = ["05/02/2024", "07/03/2024", "09/04/2024", "11/05/2024", "01/06/2024"] - truth_dates = pd.to_datetime(raw_dates, dayfirst=True) - train_messy, train_clean = _pair_frames( - _TRAIN_ROWS, "T", "event_date", raw_dates, list(truth_dates) - ) - test_messy, test_truth = _pair_frames( - _TEST_ROWS, "H", "event_date", raw_dates, list(truth_dates) - ) - return T4Fixture( - name="dayfirst_dates", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - ) - - -def _make_phone_fixture() -> T4Fixture: - locals_ = ["98765 43210", "91234-56789", "099887 76655", "9012345678", "98111 22333"] - canonical = [ - "+919876543210", - "+919123456789", - "+919988776655", - "+919012345678", - "+919811122333", - ] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", "phone", locals_, canonical) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", "phone", locals_, canonical) - return T4Fixture( - name="phone_in_formats", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - sensitive_literals=tuple(locals_) + tuple(canonical), - ) - - -def _make_email_fixture() -> T4Fixture: - doubled = [ - "asha@@gmail.com", - "bob@@yahoo.com", - "carol@@hotmail.com", - "dev@@site.com", - "eve@@example.com", - ] - fixed = [d.replace("@@", "@") for d in doubled] - train_messy, train_clean = _pair_frames(_TRAIN_ROWS, "T", "email", doubled, fixed) - test_messy, test_truth = _pair_frames(_TEST_ROWS, "H", "email", doubled, fixed) - return T4Fixture( - name="email_double_at", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - sensitive_literals=tuple(doubled) + tuple(fixed), - # The deterministic email expert already repairs doubled @s; the - # profile must not *hurt*, but the lift comes from other families. - expect_lift=False, - ) - - -def _make_imputation_fixture() -> T4Fixture: - # The clean side median-fills missing readings. The profile must learn at - # most an imputation *strategy* hint — never literal fill values — so - # replay leaves the new batch's NaNs alone rather than stamping the - # train median into them. - n = _TRAIN_ROWS - ids = _ids(n, "T") - values = [float(20 + (i % 9)) for i in range(n)] - messy_vals: list[object] = list(values) - for i in range(0, n, 6): - messy_vals[i] = np.nan - median = float(np.nanmedian(np.asarray(messy_vals, dtype=float))) - clean_vals = [median if pd.isna(v) else v for v in messy_vals] - train_messy = pd.DataFrame({"id": ids, "reading": messy_vals}) - train_clean = pd.DataFrame({"id": ids, "reading": clean_vals}) - - test_ids = _ids(_TEST_ROWS, "H") - test_vals: list[object] = [float(30 + (i % 5)) for i in range(_TEST_ROWS)] - for i in range(0, _TEST_ROWS, 5): - test_vals[i] = np.nan - test_messy = pd.DataFrame({"id": test_ids, "reading": test_vals}) - # Truth keeps the NaNs: a literal replay of the *train* median would be - # a false modification, which is exactly what the gate checks. - test_truth = test_messy.copy() - return T4Fixture( - name="imputation_no_literal_replay", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - expect_lift=False, - ) - - -def _make_unexplained_fixture() -> T4Fixture: - # One-off free-text typos, each with support 1: nothing meets - # min_support, so the profile stores examples only and replays nothing. - n = _TRAIN_ROWS - ids = _ids(n, "T") - notes = [f"customer note number {i} with unique typo xq{i}z" for i in range(n)] - clean_notes = [t.replace(f"xq{i}z", "fixed") for i, t in enumerate(notes)] - train_messy = pd.DataFrame({"id": ids, "note": notes}) - train_clean = pd.DataFrame({"id": ids, "note": clean_notes}) - test_ids = _ids(_TEST_ROWS, "H") - test_notes = [f"new note {i} with typo aa{i}bb" for i in range(_TEST_ROWS)] - test_messy = pd.DataFrame({"id": test_ids, "note": test_notes}) - test_truth = test_messy.copy() - return T4Fixture( - name="unexplained_examples_only", - train_messy=train_messy, - train_clean=train_clean, - test_messy=test_messy, - test_truth=test_truth, - expect_lift=False, - ) - - -def make_t4_fixtures() -> list[T4Fixture]: - """The ten T4 corruption families, deterministic (no RNG needed).""" - return [ - _make_category_map_fixture(), - _make_status_typos_fixture(), - _make_condition_vocab_fixture(), - _make_city_typos_fixture(), - _make_sentinel_fixture(), - _make_dayfirst_fixture(), - _make_phone_fixture(), - _make_email_fixture(), - _make_imputation_fixture(), - _make_unexplained_fixture(), - ] - - -# --------------------------------------------------------------------------- -# Metrics -# --------------------------------------------------------------------------- - - -def privacy_leak_count(profile_bytes: bytes, sensitive_literals: tuple[str, ...]) -> int: - """Count sensitive literals readable anywhere in a serialized profile. - - Scans both the raw archive bytes and every decompressed member, so zip - compression cannot hide a leak. - """ - blobs = [profile_bytes] - try: - with zipfile.ZipFile(io.BytesIO(profile_bytes)) as zf: - blobs.extend(zf.read(name) for name in zf.namelist()) - except zipfile.BadZipFile: # pragma: no cover - defensive - pass - leaks = 0 - for literal in sensitive_literals: - needle = literal.encode("utf-8") - if any(needle in blob for blob in blobs): - leaks += 1 - return leaks - - -def profile_drift_block_rate(reports: list) -> float: - """Share of runs whose profile replay was blocked by the drift gate.""" - if not reports: - return 0.0 - blocked = sum( - 1 - for r in reports - if getattr(r, "profile_replay", None) is not None and not r.profile_replay.get("ok") - ) - return blocked / len(reports) - - -# --------------------------------------------------------------------------- -# Runner -# --------------------------------------------------------------------------- - - -@dataclass -class T4Result: - name: str - expect_lift: bool - baseline_f1: float - profile_f1: float - lift_f1: float - baseline_fmr: float - profile_fmr: float - protected_violation_rate: float - privacy_leaks: int - replay_ok: bool - profile_actions: int - - -def _serialize(profile) -> bytes: - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "profile.fdprofile" - profile.save(path) - return path.read_bytes() - - -def run_t4_fixture(fixture: T4Fixture) -> T4Result: - """Learn from the fixture's train pair, replay on its test batch.""" - profile = learn( - fixture.train_messy, - fixture.train_clean, - context=fixture.context, - key=fixture.key, - dataset_id=f"t4-{fixture.name}", - **fixture.learn_kwargs, - ) - leaks = privacy_leak_count(_serialize(profile), fixture.sensitive_literals) - - # Shape-preserving options: metrics need identically-shaped frames, so no - # flag columns, no row drops, no index resets. - clean_kwargs: dict[str, Any] = { - "semantic_mode": "auto", - "verbose": False, - "drop_duplicates": False, - "reset_index": False, - "outlier_action": None, - "return_report": True, - } - baseline, _ = fd.clean(fixture.test_messy, **clean_kwargs) - repaired, report = fd.clean(fixture.test_messy, profile=profile, **clean_kwargs) - - truth, corrupted = fixture.test_truth, fixture.test_messy - baseline_f1 = cell_repair_f1(truth, corrupted, baseline) - profile_f1 = cell_repair_f1(truth, corrupted, repaired) - profile_actions = sum( - 1 for a in report.actions if a.metadata and a.metadata.get("profile_influenced") - ) - return T4Result( - name=fixture.name, - expect_lift=fixture.expect_lift, - baseline_f1=baseline_f1, - profile_f1=profile_f1, - lift_f1=profile_f1 - baseline_f1, - baseline_fmr=false_modification_rate(truth, corrupted, baseline), - profile_fmr=false_modification_rate(truth, corrupted, repaired), - protected_violation_rate=protected_column_violation_rate( - corrupted, repaired, list(fixture.protected_columns) - ), - privacy_leaks=leaks, - replay_ok=bool(report.profile_replay is not None and report.profile_replay.get("ok")), - profile_actions=profile_actions, - ) - - -def run_t4(fixtures: list[T4Fixture] | None = None) -> list[T4Result]: - return [run_t4_fixture(f) for f in fixtures or make_t4_fixtures()] - - -if __name__ == "__main__": # pragma: no cover - manual benchmark entry point - results = run_t4() - lifted = [r for r in results if r.expect_lift] - print(f"{'fixture':32} {'base F1':>8} {'prof F1':>8} {'lift':>7} {'FMR b/p':>12} {'leaks':>5}") - for r in results: - print( - f"{r.name:32} {r.baseline_f1:8.3f} {r.profile_f1:8.3f} " - f"{r.lift_f1:+7.3f} {r.baseline_fmr:5.3f}/{r.profile_fmr:5.3f} " - f"{r.privacy_leaks:5d}" - ) - mean_lift = sum(r.lift_f1 for r in lifted) / len(lifted) - print(f"\nmean F1 lift (replayable families): {mean_lift:+.3f}") diff --git a/benchmarks/corpus/scoring.py b/benchmarks/corpus/scoring.py deleted file mode 100644 index 4760b4c9..00000000 --- a/benchmarks/corpus/scoring.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Run the trap corpus against FreshData and score the result. - -This is the measurement half of the corpus. It answers the question the whole -programme turns on: *how often does the library make a semantic decision a -knowledgeable human would call wrong, and how often does it do so silently?* - -Scoring rules -------------- -The observed disposition is derived from what actually happened to the cell, -never from what the report claims: - -``PRESERVE`` the value is unchanged (compared after a NaN-aware equality). -``REPAIR`` the value changed. Whether it changed *correctly* is a separate - axis -- ``repaired_correctly`` -- so a wrong repair is visible as - a corruption rather than hidden inside a repair count. -``QUARANTINE`` / ``REJECT`` / ``REVIEW`` - taken from ``validate_fields`` row actions, mapped through - :func:`~.dispositions.from_field_action`. -``FLAG`` unchanged, but the cell drew an issue or a report warning. - -A case whose ``expected`` is ``None`` (a recorded specification gap) is -measured and reported but never scored pass/fail -- asserting an outcome there -would be inventing the contract the gap exists to record. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -import pandas as pd - -from .dispositions import Disposition, satisfies -from .frames import frame_for, trap_position -from .traps import TRAPS, TrapCase - -__all__ = ["Observation", "observe", "score", "Metrics", "run_corpus"] - -D = Disposition - - -def _equal(a: Any, b: Any) -> bool: - """NaN-aware scalar equality; two missing values count as equal.""" - a_missing = a is None or (isinstance(a, float) and pd.isna(a)) or a is pd.NaT - b_missing = b is None or (isinstance(b, float) and pd.isna(b)) or b is pd.NaT - try: - a_missing = a_missing or bool(pd.isna(a)) - b_missing = b_missing or bool(pd.isna(b)) - except (TypeError, ValueError): - pass - if a_missing or b_missing: - return a_missing and b_missing - if isinstance(a, str) != isinstance(b, str): - # '7' and 7 are different outcomes for a cleaner; do not conflate them. - return False - try: - return bool(a == b) - except Exception: - return False - - -@dataclass -class Observation: - """What the library actually did to one trap cell.""" - - case: TrapCase - observed: Disposition | None - final_value: Any = None - changed: bool = False - repaired_correctly: bool | None = None - field_action: str | None = None - audited: bool = False - error: str | None = None - - @property - def measured(self) -> bool: - return self.error is None and self.observed is not None - - -def observe(case: TrapCase, *, semantic: bool = True) -> Observation: - """Run one case through ``fd.clean`` and ``fd.validate_fields``.""" - import freshdata as fd - - frame = frame_for(case) - if frame is None: - return Observation(case, None, error="no filler for role") - - pos = trap_position() - original = frame[case.role].iloc[pos] - kwargs: dict[str, Any] = {"verbose": False, "return_report": True} - if semantic: - kwargs["semantic_mode"] = "auto" - - try: - cleaned, report = fd.clean(frame, **kwargs) - except Exception as exc: # a crash is itself a finding - return Observation(case, None, error=f"{type(exc).__name__}: {exc}"[:160]) - - # The anchor column keeps the row alive, so position is stable. - final = cleaned[case.role].iloc[pos] if pos < len(cleaned) else None - changed = not _equal(original, final) - - audited = any(a.column == case.role for a in report.actions) or bool(report.warnings) - - # Ask the validation surface what it would do with the row. - # - # Only pass a semantic_type that fieldcheck actually knows. Its vocabulary - # and the semantic layer's SEMANTIC_TYPES overlap in only 10 of 30 terms - # (see FD2-003), so declaring e.g. "postal_code" -- a real semantic type -- - # warns that nothing will be validated. Passing it anyway would fill the - # suite with warnings and validate nothing, which is worse than omitting it. - from freshdata.fieldcheck import _KNOWN_SEMANTIC_TYPES - - action: str | None = None - try: - spec: dict[str, Any] = {} - if case.semantic_type and case.semantic_type in _KNOWN_SEMANTIC_TYPES: - spec[case.role] = case.semantic_type - vreport = fd.validate_fields(frame, spec or None) - actions = vreport.row_actions() - action = actions.get(pos) if isinstance(actions, dict) else None - except Exception: - action = None - - from .dispositions import from_field_action - - observed: Disposition - if action in {"quarantine", "reject", "manual_review"}: - observed = from_field_action(action) - elif changed: - observed = D.REPAIR - elif audited: - observed = D.FLAG - else: - observed = D.PRESERVE - - repaired_correctly = None - if case.expected is D.REPAIR: - repaired_correctly = _equal(final, case.repaired) - - return Observation( - case=case, - observed=observed, - final_value=final, - changed=changed, - repaired_correctly=repaired_correctly, - field_action=action, - audited=audited, - ) - - -@dataclass -class Metrics: - """The Phase 23 metric set, as rates.""" - - total: int = 0 - measured: int = 0 - spec_gaps: int = 0 - errors: int = 0 - correct: int = 0 - #: expected PRESERVE and the value survived - preserved: int = 0 - preserve_cases: int = 0 - #: expected PRESERVE but the value changed -- false-positive corruption - corrupted: int = 0 - #: expected REPAIR and the gold value was produced - repaired_correctly: int = 0 - repair_cases: int = 0 - #: expected REPAIR, the value changed, but not to the gold value - misrepaired: int = 0 - #: expected REPAIR/REVIEW/... but nothing happened at all - escaped: int = 0 - routed_to_review: int = 0 - review_cases: int = 0 - audited: int = 0 - changes: int = 0 - failures: list[str] = field(default_factory=list) - - def _rate(self, num: int, den: int) -> float | None: - return round(num / den, 4) if den else None - - def as_dict(self) -> dict[str, Any]: - return { - "total": self.total, - "measured": self.measured, - "spec_gaps": self.spec_gaps, - "errors": self.errors, - "accuracy": self._rate(self.correct, self.measured), - "preservation_rate": self._rate(self.preserved, self.preserve_cases), - "corruption_rate": self._rate(self.corrupted, self.preserve_cases), - "repair_precision": self._rate(self.repaired_correctly, self.changes), - "repair_recall": self._rate(self.repaired_correctly, self.repair_cases), - "false_positive_rate": self._rate(self.corrupted, self.preserve_cases), - "false_negative_rate": self._rate(self.escaped, self.measured), - "review_rate": self._rate(self.routed_to_review, self.review_cases), - "escape_rate": self._rate(self.escaped, self.measured), - "audit_completeness": self._rate(self.audited, self.changes), - } - - -def score(observations: list[Observation]) -> Metrics: - """Aggregate observations into the named metrics.""" - m = Metrics(total=len(observations)) - for o in observations: - if o.error: - m.errors += 1 - m.failures.append(f"{o.case.family}/{o.case.role}: {o.error}") - continue - if o.case.expected is None: - m.spec_gaps += 1 - continue - m.measured += 1 - expected = o.case.expected - - if o.changed: - m.changes += 1 - if o.audited: - m.audited += 1 - - if expected is D.PRESERVE: - m.preserve_cases += 1 - if o.changed: - m.corrupted += 1 - m.failures.append( - f"CORRUPTION {o.case.family}/{o.case.role}: " - f"{o.case.token!r} -> {o.final_value!r}" - ) - else: - m.preserved += 1 - m.correct += 1 - continue - - if expected is D.REPAIR: - m.repair_cases += 1 - if o.repaired_correctly: - m.repaired_correctly += 1 - m.correct += 1 - elif o.changed: - m.misrepaired += 1 - m.failures.append( - f"MISREPAIR {o.case.family}/{o.case.role}: {o.case.token!r} -> " - f"{o.final_value!r}, expected {o.case.repaired!r}" - ) - else: - m.escaped += 1 - m.failures.append( - f"ESCAPE {o.case.family}/{o.case.role}: {o.case.token!r} unchanged, " - f"expected repair to {o.case.repaired!r}" - ) - continue - - # REVIEW / QUARANTINE / REJECT / FLAG - m.review_cases += 1 - if satisfies(expected, o.observed): - m.routed_to_review += 1 - m.correct += 1 - else: - m.escaped += 1 - m.failures.append( - f"ESCAPE {o.case.family}/{o.case.role}: {o.case.token!r} got " - f"{o.observed.value}, expected {expected.value}" - ) - return m - - -def run_corpus(cases=TRAPS, *, semantic: bool = True) -> tuple[list[Observation], Metrics]: - observations = [observe(c, semantic=semantic) for c in cases] - return observations, score(observations) diff --git a/benchmarks/performance/semantic_probe.py b/benchmarks/performance/semantic_probe.py deleted file mode 100644 index d1628ae8..00000000 --- a/benchmarks/performance/semantic_probe.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Development-only instrumentation for semantic context repetition. - -The probe wraps the pure value-shape helpers used by -``freshdata.semantic.context``. It deliberately keeps all observations in -memory and creates a fresh collector for each context build, making the -result useful for estimating the benefit of a per-build memoization layer -without changing production behavior. -""" - -from __future__ import annotations - -from collections.abc import Iterator, Mapping -from contextlib import ExitStack, contextmanager -from dataclasses import dataclass -from typing import Any, Callable -from unittest.mock import patch - -import pandas as pd - -from freshdata.config import CleanConfig -from freshdata.semantic import context as semantic_context -from freshdata.semantic.types import SemanticContext - -# ``column_name_is_identifier`` is intentionally not included: the repetition -# candidates are the six value parsers plus the email-value regex check. -_OPERATIONS = ( - "is_plain_number", - "parse_number_words", - "parse_boolean", - "parse_currency", - "parse_unit", - "email_value", - "looks_like_date_value", -) -_CALLABLE_OPERATIONS = ( - "is_plain_number", - "parse_number_words", - "parse_boolean", - "parse_currency", - "parse_unit", - "looks_like_date_value", -) - -# Exact-type gating keeps key construction safe and mirrors the values each -# helper is designed to inspect. In particular, bool is observable by -# ``is_plain_number`` even though the helper itself excludes it semantically. -_ALLOWED_TYPES: Mapping[str, tuple[type[object], ...]] = { - "is_plain_number": (int, float, bool, str), - "parse_number_words": (str,), - "parse_boolean": (str,), - "parse_currency": (str,), - "parse_unit": (str,), - "looks_like_date_value": (str,), - "email_value": (str,), -} - - -@dataclass(frozen=True) -class OperationProbe: - """Immutable observations for one semantic helper during one build.""" - - total_calls: int - eligible_calls: int - bypassed_calls: int - unique_keys: int - theoretical_hits: int - eligible_values: tuple[object, ...] - - @property - def hit_rate(self) -> float: - return self.theoretical_hits / self.eligible_calls if self.eligible_calls else 0.0 - - -@dataclass(frozen=True) -class SemanticProbeBuild: - """Per-operation probe results for one ``build_semantic_context`` call.""" - - by_operation: Mapping[str, OperationProbe] - - @property - def total_theoretical_hits(self) -> int: - return sum(item.theoretical_hits for item in self.by_operation.values()) - - @property - def total_calls(self) -> int: - return sum(item.total_calls for item in self.by_operation.values()) - - @property - def total_eligible_calls(self) -> int: - return sum(item.eligible_calls for item in self.by_operation.values()) - - @property - def total_bypassed_calls(self) -> int: - return sum(item.bypassed_calls for item in self.by_operation.values()) - - -@dataclass -class _MutableOperationProbe: - total_calls: int = 0 - eligible_calls: int = 0 - bypassed_calls: int = 0 - theoretical_hits: int = 0 - - def __post_init__(self) -> None: - self.seen_keys: set[tuple[str, type[object], object]] = set() - self.eligible_values: list[object] = [] - - -def _eligible(operation: str, value: object) -> bool: - """Return whether *value* may safely participate in a repetition key.""" - - return type(value) in _ALLOWED_TYPES[operation] - - -class _ProbeCollector: - """Mutable collector used only while one context build is running.""" - - def __init__(self) -> None: - self._operations = {name: _MutableOperationProbe() for name in _OPERATIONS} - - def record(self, operation: str, value: object) -> None: - state = self._operations[operation] - state.total_calls += 1 - if not _eligible(operation, value): - state.bypassed_calls += 1 - return - - state.eligible_calls += 1 - state.eligible_values.append(value) - # Every allowed exact type is hash-safe. Unsafe values are returned - # above before this key expression is evaluated. - key = (operation, type(value), value) - if key in state.seen_keys: - state.theoretical_hits += 1 - else: - state.seen_keys.add(key) - - def finish_build(self) -> SemanticProbeBuild: - return SemanticProbeBuild( - by_operation={ - name: OperationProbe( - total_calls=state.total_calls, - eligible_calls=state.eligible_calls, - bypassed_calls=state.bypassed_calls, - unique_keys=len(state.seen_keys), - theoretical_hits=state.theoretical_hits, - eligible_values=tuple(state.eligible_values), - ) - for name, state in self._operations.items() - } - ) - - -class _EmailValueProxy: - """Regex proxy that records the already-stripped email candidate.""" - - def __init__(self, delegate: Any, probe: _ProbeCollector) -> None: - self._delegate = delegate - self._probe = probe - - def match(self, value: object, *args: object, **kwargs: object) -> Any: - self._probe.record("email_value", value) - return self._delegate.match(value, *args, **kwargs) - - -@contextmanager -def _patched_context_operations( - probe: _ProbeCollector | None = None, -) -> Iterator[_ProbeCollector]: - """Temporarily wrap context helper references with *probe* recorders.""" - - if probe is None: - probe = _ProbeCollector() - with ExitStack() as stack: - for operation in _CALLABLE_OPERATIONS: - original = getattr(semantic_context, operation) - - def wrapped( - value: object, - *args: object, - _operation: str = operation, - _original: Callable[..., Any] = original, - **kwargs: object, - ) -> Any: - probe.record(_operation, value) - return _original(value, *args, **kwargs) - - stack.enter_context(patch.object(semantic_context, operation, wrapped)) - - stack.enter_context( - patch.object( - semantic_context, - "_EMAIL_VALUE", - _EmailValueProxy(semantic_context._EMAIL_VALUE, probe), - ) - ) - yield probe - - -def probe_context_build( - df: pd.DataFrame, - config: CleanConfig, - *, - stats: dict[object, tuple[int, int, int | None]] | None = None, -) -> tuple[SemanticContext, SemanticProbeBuild]: - """Build semantic context and collect helper repetition observations.""" - - with _patched_context_operations() as probe: - context = semantic_context.build_semantic_context(df, config, stats=stats) - return context, probe.finish_build() - - -__all__ = [ - "OperationProbe", - "SemanticProbeBuild", - "probe_context_build", -] diff --git a/benchmarks/truthbench/inventory.py b/benchmarks/truthbench/inventory.py deleted file mode 100644 index 706960cd..00000000 --- a/benchmarks/truthbench/inventory.py +++ /dev/null @@ -1,383 +0,0 @@ -"""Manifest of FreshData's public decision, sink, and caller-directed surfaces.""" - -from __future__ import annotations - -import argparse -from collections.abc import Iterable -from dataclasses import dataclass -from enum import Enum - -import freshdata as fd -from freshdata.domains.registry import available as available_domains - - -class SurfaceClass(str, Enum): - DECISION = "decision" - SINK = "sink" - EXPLICIT_TRANSFORM = "explicit-transform" - DATA_MODEL = "data-model" - CONFIGURATION = "configuration" - REGISTRATION = "registration" - OUT_OF_SCOPE = "out-of-scope-with-reason" - - -@dataclass(frozen=True) -class SurfaceSpec: - name: str - version: int - classification: SurfaceClass - adapter: str | None - mutates: bool - deterministic: bool - backend_parity: bool - rationale: str - - def __post_init__(self) -> None: - if not self.name: - raise ValueError("surface name must not be empty") - if self.version != 1: - raise ValueError("unsupported surface spec version") - try: - classification = SurfaceClass(self.classification) - except ValueError as exc: - raise ValueError(f"unknown surface classification: {self.classification!r}") from exc - object.__setattr__(self, "classification", classification) - if classification in {SurfaceClass.DECISION, SurfaceClass.SINK} and not self.adapter: - raise ValueError(f"{classification.value} surface {self.name!r} requires an adapter") - if not self.rationale: - raise ValueError(f"surface {self.name!r} requires a rationale") - - -_CALLER_DIRECTED = { - "fill_missing", - "remove_outliers", - "resolve_duplicates", - "group_aggregate", - "set_display", - "get_display", - "reset_display", - "tokenize_value", - "detokenize_value", - "detokenize_series", - "make_vault", - "vault_metadata", - "anonymize", - "clean_text", - "clean_text_value", - "lint_text_encoding", - "mask_dataframe", -} -_REGISTRATION = { - "register_backend", - "register_comparator", - "register_expert", - "register_exporter", - "register_validator", - "registered_plugins", - "available_packs", -} -_SINKS = { - "build_exception_table", - "build_quality_report", - "export", - "export_dbt_tests", - "export_gx_suite", - "export_quality_ops", - "export_review_queue", - "generate_compliance_report", - "insight_report", - "stakeholder_summary", - "trust_gate_report", - "save_baseline", -} -_DECISIONS = { - "apply", - "apply_field_policy", - "apply_plan", - "apply_privacy_policy", - "apply_review_decisions", - "build_review_queue", - "cdc_profile", - "clean", - "clean_csv", - "clean_domain_file", - "clean_enterprise", - "clean_excel", - "clean_timeseries", - "cluster_column", - "compare_clean", - "compare_plans", - "compile_context", - "compute_trust_score", - "detect_outliers", - "detect_pii", - "diff_schema", - "enforce_contract", - "evaluate_quality_debt", - "explain_clean", - "infer_roles", - "learn", - "learn_cleaning_memory", - "link", - "link_entities", - "monitor_contract", - "parse_domain", - "pipeline", - "pipeline_clean", - "plan", - "profile", - "recalibrate_weights", - "redaction_columns", - "resolve_entities", - "run_semantic_validation", - "run_suite", - "suggest_join_keys", - "suggest_plan", - "validate", - "validate_fields", - "analyze_dataset", - "check_k_anonymity", - "classify_columns", - "compare_to_baseline", - "merge_clusters", - "merge_entities", - "schema_of", - "load_review_decisions", -} -_CONFIG_FUNCTIONS = { - "education_template", - "build_baseline", - "get_template", - "healthcare_template", - "load_baseline", - "load_cleaning_memory", - "load_compliance_pack", - "load_privacy_policy", - "load_profile", - "media_template", - "retail_template", - "save_profile", -} -_OUT_OF_SCOPE = {"enterprise", "models", "testing", "__version__"} - - -def cli_commands() -> tuple[str, ...]: - """Return top-level and nested enterprise CLI command paths.""" - - from freshdata.enterprise.cli import build_parser - - parser = build_parser() - result: set[str] = set() - - def visit(current: argparse.ArgumentParser, prefix: str = "") -> None: - for action in current._actions: - if not isinstance(action, argparse._SubParsersAction): - continue - choices = action.choices - for command, child in choices.items(): - path = f"{prefix} {command}".strip() - result.add(path) - if isinstance(child, argparse.ArgumentParser): - visit(child, path) - - visit(parser) - return tuple(sorted(result)) - - -def discover_public_names() -> tuple[str, ...]: - """Discover all names and registered command/domain surfaces in this checkout.""" - - names = set(fd.__dir__()) - from freshdata.experimental import ai_copilot - - names.update(ai_copilot.__all__) - names.update(f"domain:{name}" for name in available_domains()) - names.update(f"cli:{name}" for name in cli_commands()) - return tuple(sorted(names)) - - -#: Concrete adapter per decision surface. Cleaning-shaped surfaces route to -#: the cleaning adapter's operation dispatch, analysis/validation surfaces to -#: the validation adapter, privacy surfaces to the privacy adapter, and the -#: experimental Copilot to its own adapter. Sinks route to reporting. -_DECISION_ADAPTERS = { - "apply": "cleaning", - "apply_plan": "cleaning", - "clean": "cleaning", - "clean_csv": "cleaning", - "clean_domain_file": "cleaning", - "clean_enterprise": "cleaning", - "clean_excel": "cleaning", - "clean_timeseries": "cleaning", - "compare_clean": "cleaning", - "compare_plans": "cleaning", - "explain_clean": "cleaning", - "learn": "cleaning", - "learn_cleaning_memory": "cleaning", - "pipeline": "cleaning", - "pipeline_clean": "cleaning", - "plan": "cleaning", - "suggest_plan": "cleaning", - "apply_privacy_policy": "privacy", - "check_k_anonymity": "privacy", - "detect_pii": "privacy", - "redaction_columns": "privacy", - "analyze_dataset": "copilot", -} - - -def _decision_adapter(name: str) -> str: - return _DECISION_ADAPTERS.get(name, "validation") - - -def _classify(name: str) -> tuple[SurfaceClass, str | None, bool, bool, bool, str]: - if name.startswith("domain:"): - return ( - SurfaceClass.DECISION, - "validation", - False, - True, - True, - "bundled domain validator decision surface", - ) - if name.startswith("cli:"): - return ( - SurfaceClass.DECISION, - "reporting", - False, - True, - False, - "enterprise CLI decision/report command", - ) - if name in _CALLER_DIRECTED: - return ( - SurfaceClass.EXPLICIT_TRANSFORM, - None, - name not in {"get_display", "reset_display", "vault_metadata"}, - True, - False, - "caller-directed explicit transform; no inferred disposition credit", - ) - if name in _REGISTRATION: - return ( - SurfaceClass.REGISTRATION, - None, - True, - True, - False, - "plugin registration or registry inspection", - ) - if name in _CONFIG_FUNCTIONS: - return ( - SurfaceClass.CONFIGURATION, - None, - False, - True, - False, - "configuration/template loading surface", - ) - if name in _SINKS: - return ( - SurfaceClass.SINK, - "reporting", - False, - True, - False, - "report, audit, or export sink", - ) - if name in _DECISIONS: - return ( - SurfaceClass.DECISION, - _decision_adapter(name), - name.startswith(("clean", "apply", "run_")), - True, - True, - "public FreshData decision surface", - ) - if name in _OUT_OF_SCOPE: - return ( - SurfaceClass.OUT_OF_SCOPE, - None, - False, - True, - False, - "namespace or metadata surface; no cell disposition", - ) - if name.startswith("__"): - return SurfaceClass.CONFIGURATION, None, False, True, False, "package metadata" - if name.endswith(("Config", "Policy", "Rule", "Spec", "Weights")) or name in { - "Action", - "Jurisdiction", - "CompliancePack", - }: - return ( - SurfaceClass.CONFIGURATION, - None, - False, - True, - False, - "configuration or policy model", - ) - if name[:1].isupper(): - return SurfaceClass.DATA_MODEL, None, False, True, False, "public data/report model" - raise ValueError(f"unclassified FreshData public surface: {name}") - - -def build_manifest(names: Iterable[str] | None = None) -> tuple[SurfaceSpec, ...]: - """Build an immutable manifest from the currently discovered public surfaces.""" - - selected = discover_public_names() if names is None else tuple(sorted(set(names))) - specs: list[SurfaceSpec] = [] - for name in selected: - classification, adapter, mutates, deterministic, parity, rationale = _classify(name) - specs.append( - SurfaceSpec( - name=name, - version=1, - classification=classification, - adapter=adapter, - mutates=mutates, - deterministic=deterministic, - backend_parity=parity, - rationale=rationale, - ) - ) - validate_manifest(specs) - return tuple(specs) - - -def validate_manifest(manifest: Iterable[SurfaceSpec]) -> None: - """Validate uniqueness, classifications, and adapter requirements.""" - - specs = tuple(manifest) - names = [spec.name for spec in specs] - if len(names) != len(set(names)): - raise ValueError("surface manifest contains duplicate names") - from .surfaces import adapters - - registered = adapters() - for spec in specs: - if not isinstance(spec, SurfaceSpec): - raise TypeError("surface manifest entries must be SurfaceSpec instances") - # Re-run constructor checks for objects supplied by callers or tests. - SurfaceSpec(**{field: getattr(spec, field) for field in SurfaceSpec.__dataclass_fields__}) - if ( - spec.classification in {SurfaceClass.DECISION, SurfaceClass.SINK} - and spec.adapter not in registered - ): - raise ValueError(f"surface {spec.name!r} references unknown adapter {spec.adapter!r}") - - -SURFACE_MANIFEST = build_manifest() -PUBLIC_SURFACES = SURFACE_MANIFEST - -__all__ = [ - "PUBLIC_SURFACES", - "SURFACE_MANIFEST", - "SurfaceClass", - "SurfaceSpec", - "build_manifest", - "cli_commands", - "discover_public_names", - "validate_manifest", -] diff --git a/crates/freshcore/Cargo.lock b/crates/freshcore/Cargo.lock deleted file mode 100644 index c970f864..00000000 --- a/crates/freshcore/Cargo.lock +++ /dev/null @@ -1,255 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - -[[package]] -name = "freshcore" -version = "0.1.0" -dependencies = [ - "pretty_assertions", - "pyo3", - "rayon", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "indoc" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" -dependencies = [ - "rustversion", -] - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" -dependencies = [ - "cfg-if", - "indoc", - "libc", - "memoffset", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" -dependencies = [ - "once_cell", - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" -dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "syn" -version = "2.0.118" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "target-lexicon" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" diff --git a/crates/freshcore/Cargo.toml b/crates/freshcore/Cargo.toml deleted file mode 100644 index 503e21ec..00000000 --- a/crates/freshcore/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "freshcore" -version = "0.1.0" -edition = "2021" -license = "MIT" -description = "FreshData native cleaning-first execution engine" - -[lib] -name = "freshdata_freshcore" -crate-type = ["cdylib", "rlib"] - -[features] -extension-module = ["pyo3/extension-module"] - -[dependencies] -pyo3 = { version = "0.22" } -rayon = "1.10" - -[dev-dependencies] -pretty_assertions = "1.4" diff --git a/crates/freshcore/src/arrays.rs b/crates/freshcore/src/arrays.rs deleted file mode 100644 index 01d98bc0..00000000 --- a/crates/freshcore/src/arrays.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::hash::{Hash, Hasher}; - -#[derive(Clone, Debug, PartialEq)] -pub enum ColumnData { - Float(Vec>), - Bool(Vec>), - Utf8(Vec>), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Column { - pub name: String, - pub data: ColumnData, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Frame { - pub columns: Vec, - pub nrows: usize, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum CellKey { - Null, - Float(u64), - Bool(bool), - Utf8(String), -} - -impl Hash for CellKey { - fn hash(&self, state: &mut H) { - match self { - CellKey::Null => 0u8.hash(state), - CellKey::Float(v) => { - 1u8.hash(state); - v.hash(state); - } - CellKey::Bool(v) => { - 2u8.hash(state); - v.hash(state); - } - CellKey::Utf8(v) => { - 3u8.hash(state); - v.hash(state); - } - } - } -} - -impl ColumnData { - pub fn len(&self) -> usize { - match self { - ColumnData::Float(v) => v.len(), - ColumnData::Bool(v) => v.len(), - ColumnData::Utf8(v) => v.len(), - } - } - - pub fn null_count(&self) -> usize { - match self { - ColumnData::Float(v) => v.iter().filter(|x| x.is_none()).count(), - ColumnData::Bool(v) => v.iter().filter(|x| x.is_none()).count(), - ColumnData::Utf8(v) => v.iter().filter(|x| x.is_none()).count(), - } - } - - pub fn is_all_null(&self) -> bool { - self.null_count() == self.len() - } - - pub fn cell_key(&self, row: usize) -> CellKey { - match self { - ColumnData::Float(v) => v[row] - .map(|x| { - if x == 0.0 { - CellKey::Float(0.0f64.to_bits()) - } else { - CellKey::Float(x.to_bits()) - } - }) - .unwrap_or(CellKey::Null), - ColumnData::Bool(v) => v[row].map(CellKey::Bool).unwrap_or(CellKey::Null), - ColumnData::Utf8(v) => v[row] - .as_ref() - .map(|x| CellKey::Utf8(x.clone())) - .unwrap_or(CellKey::Null), - } - } - - pub fn take(&self, keep: &[bool]) -> ColumnData { - match self { - ColumnData::Float(v) => ColumnData::Float( - v.iter() - .zip(keep) - .filter_map(|(value, keep)| keep.then_some(*value)) - .collect(), - ), - ColumnData::Bool(v) => ColumnData::Bool( - v.iter() - .zip(keep) - .filter_map(|(value, keep)| keep.then_some(*value)) - .collect(), - ), - ColumnData::Utf8(v) => ColumnData::Utf8( - v.iter() - .zip(keep) - .filter_map(|(value, keep)| keep.then_some(value.clone())) - .collect(), - ), - } - } -} - -impl Frame { - pub fn new(columns: Vec) -> Result { - let nrows = columns.first().map(|c| c.data.len()).unwrap_or(0); - if columns.iter().any(|c| c.data.len() != nrows) { - return Err("all columns must have the same length".to_string()); - } - Ok(Self { columns, nrows }) - } - - pub fn null_cells(&self) -> usize { - self.columns.iter().map(|c| c.data.null_count()).sum() - } - - pub fn take_rows(&mut self, keep: &[bool]) { - for column in &mut self.columns { - column.data = column.data.take(keep); - } - self.nrows = keep.iter().filter(|x| **x).count(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn null_counts_are_separate_from_values() { - let data = ColumnData::Float(vec![Some(1.0), None, Some(2.0)]); - assert_eq!(data.len(), 3); - assert_eq!(data.null_count(), 1); - assert!(!data.is_all_null()); - } - - #[test] - fn row_take_preserves_column_lengths() { - let mut frame = Frame::new(vec![Column { - name: "a".into(), - data: ColumnData::Utf8(vec![Some("x".into()), None, Some("y".into())]), - }]) - .unwrap(); - frame.take_rows(&[true, false, true]); - assert_eq!(frame.nrows, 2); - assert_eq!(frame.columns[0].data.len(), 2); - } -} diff --git a/crates/freshcore/src/kernels/casts.rs b/crates/freshcore/src/kernels/casts.rs deleted file mode 100644 index 7855ebba..00000000 --- a/crates/freshcore/src/kernels/casts.rs +++ /dev/null @@ -1,143 +0,0 @@ -use rayon::prelude::*; - -use crate::arrays::{Column, ColumnData, Frame}; -use crate::plan::CleanPlan; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CastAction { - pub column: String, - pub target: String, - pub count: usize, - pub coerced: usize, -} - -const TRUE_WORDS: &[&str] = &["true", "t", "yes", "y"]; -const FALSE_WORDS: &[&str] = &["false", "f", "no", "n"]; - -pub fn infer_and_cast(frame: &mut Frame, plan: &CleanPlan) -> Vec { - if !plan.fix_dtypes { - return Vec::new(); - } - frame - .columns - .par_iter_mut() - .filter_map(|column| cast_column(column, plan)) - .collect() -} - -fn cast_column(column: &mut Column, plan: &CleanPlan) -> Option { - let ColumnData::Utf8(values) = &column.data else { - return None; - }; - let non_null: Vec<&str> = values.iter().filter_map(|v| v.as_deref()).collect(); - if non_null.is_empty() { - return None; - } - if let Some(bools) = try_bool(values) { - let count = bools.iter().filter(|v| v.is_some()).count(); - column.data = ColumnData::Bool(bools); - return Some(CastAction { - column: column.name.clone(), - target: "boolean".into(), - count, - coerced: 0, - }); - } - if plan.preserve_leading_zeros && non_null.iter().any(|v| has_leading_zero(v)) { - return None; - } - let (parsed, parsed_count, coerced) = parse_numeric(values); - if (parsed_count as f64) / (non_null.len() as f64) >= plan.numeric_threshold { - column.data = ColumnData::Float(parsed); - return Some(CastAction { - column: column.name.clone(), - target: "float64".into(), - count: parsed_count + coerced, - coerced, - }); - } - None -} - -fn try_bool(values: &[Option]) -> Option>> { - let mut out = Vec::with_capacity(values.len()); - for value in values { - let Some(raw) = value else { - out.push(None); - continue; - }; - let token = raw.trim().to_lowercase(); - if TRUE_WORDS.contains(&token.as_str()) { - out.push(Some(true)); - } else if FALSE_WORDS.contains(&token.as_str()) { - out.push(Some(false)); - } else { - return None; - } - } - Some(out) -} - -fn parse_numeric(values: &[Option]) -> (Vec>, usize, usize) { - let mut out = Vec::with_capacity(values.len()); - let mut parsed = 0usize; - let mut coerced = 0usize; - for value in values { - let Some(raw) = value else { - out.push(None); - continue; - }; - let cleaned = raw - .trim() - .trim_matches('$') - .trim_matches('€') - .trim_matches('£') - .trim_matches('₹') - .replace(',', ""); - match cleaned.parse::() { - Ok(v) => { - out.push(Some(v)); - parsed += 1; - } - Err(_) => { - out.push(None); - coerced += 1; - } - } - } - (out, parsed, coerced) -} - -fn has_leading_zero(value: &str) -> bool { - let trimmed = value.trim_start_matches(['+', '-']).trim(); - trimmed.len() > 1 - && trimmed.starts_with('0') - && trimmed.chars().nth(1).is_some_and(|c| c.is_ascii_digit()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - #[test] - fn numeric_cast_preserves_zero_padded_ids() { - let mut frame = Frame::new(vec![Column { - name: "zip".into(), - data: ColumnData::Utf8(vec![Some("02115".into()), Some("10001".into())]), - }]) - .unwrap(); - assert!(infer_and_cast(&mut frame, &CleanPlan::default()).is_empty()); - } - - #[test] - fn bool_cast_uses_small_vocabulary() { - let mut frame = Frame::new(vec![Column { - name: "flag".into(), - data: ColumnData::Utf8(vec![Some("yes".into()), Some("no".into())]), - }]) - .unwrap(); - let actions = infer_and_cast(&mut frame, &CleanPlan::default()); - assert_eq!(actions[0].target, "boolean"); - } -} diff --git a/crates/freshcore/src/kernels/duplicates.rs b/crates/freshcore/src/kernels/duplicates.rs deleted file mode 100644 index 8ad93640..00000000 --- a/crates/freshcore/src/kernels/duplicates.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::collections::HashSet; - -use crate::arrays::{CellKey, Frame}; - -/// Marks full-row duplicates under `keep_policy` (`"last"` keeps the final -/// occurrence, anything else the first). -/// -/// Like pandas' `drop_duplicate_rows`, a frame with no rows or no columns has -/// no duplicates. -pub fn duplicated_mask(frame: &Frame, keep_policy: &str) -> Vec { - let mut duplicated = vec![false; frame.nrows]; - if frame.nrows == 0 || frame.columns.is_empty() { - return duplicated; - } - let mut seen: HashSet> = HashSet::new(); - let mut mark = |row: usize| { - if !seen.insert(row_key(frame, row)) { - duplicated[row] = true; - } - }; - if keep_policy == "last" { - (0..frame.nrows).rev().for_each(&mut mark); - } else { - (0..frame.nrows).for_each(&mut mark); - } - duplicated -} - -/// Number of full-row duplicates, without removing any rows. -/// -/// The count does not depend on the keep policy: every row beyond the first -/// occurrence of its key is a duplicate. -pub fn count_duplicates(frame: &Frame) -> usize { - duplicated_mask(frame, "first") - .iter() - .filter(|v| **v) - .count() -} - -pub fn drop_duplicates(frame: &mut Frame, keep_policy: &str) -> usize { - let duplicated = duplicated_mask(frame, keep_policy); - let dropped = duplicated.iter().filter(|v| **v).count(); - if dropped > 0 { - let keep: Vec = duplicated.iter().map(|v| !*v).collect(); - frame.take_rows(&keep); - } - dropped -} - -fn row_key(frame: &Frame, row: usize) -> Vec { - frame - .columns - .iter() - .map(|column| column.data.cell_key(row)) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - fn mixed_frame() -> Frame { - Frame::new(vec![ - Column { - name: "a".into(), - data: ColumnData::Float(vec![ - Some(1.0), - Some(1.0), - Some(0.0), - Some(-0.0), - None, - None, - Some(2.0), - ]), - }, - Column { - name: "b".into(), - data: ColumnData::Utf8(vec![ - Some("x".into()), - Some("x".into()), - Some("y".into()), - Some("y".into()), - None, - None, - Some("x".into()), - ]), - }, - ]) - .unwrap() - } - - #[test] - fn removes_duplicate_rows() { - let mut frame = Frame::new(vec![Column { - name: "x".into(), - data: ColumnData::Utf8(vec![Some("a".into()), Some("a".into()), Some("b".into())]), - }]) - .unwrap(); - assert_eq!(drop_duplicates(&mut frame, "first"), 1); - assert_eq!(frame.nrows, 2); - } - - #[test] - fn counts_duplicates_without_removing_rows() { - let frame = mixed_frame(); - // (1.0, x) twice, (0.0, y) == (-0.0, y), and (null, null) twice. - assert_eq!(count_duplicates(&frame), 3); - assert_eq!(frame.nrows, 7); - assert_eq!(frame, mixed_frame()); - } - - #[test] - fn count_matches_rows_dropped_for_every_keep_policy() { - for keep in ["first", "last"] { - let mut frame = mixed_frame(); - let counted = count_duplicates(&frame); - assert_eq!(drop_duplicates(&mut frame, keep), counted); - assert_eq!(count_duplicates(&frame), 0); - } - } - - #[test] - fn mask_marks_the_occurrences_the_keep_policy_drops() { - let frame = mixed_frame(); - assert_eq!( - duplicated_mask(&frame, "first"), - vec![false, true, false, true, false, true, false] - ); - assert_eq!( - duplicated_mask(&frame, "last"), - vec![true, false, true, false, true, false, false] - ); - } - - #[test] - fn frames_without_rows_or_columns_have_no_duplicates() { - let no_rows = Frame::new(vec![Column { - name: "x".into(), - data: ColumnData::Float(Vec::new()), - }]) - .unwrap(); - assert_eq!(count_duplicates(&no_rows), 0); - - let mut no_columns = Frame { - columns: Vec::new(), - nrows: 3, - }; - assert_eq!(count_duplicates(&no_columns), 0); - assert_eq!(drop_duplicates(&mut no_columns, "first"), 0); - assert_eq!(no_columns.nrows, 3); - } -} diff --git a/crates/freshcore/src/kernels/missing.rs b/crates/freshcore/src/kernels/missing.rs deleted file mode 100644 index f8c86000..00000000 --- a/crates/freshcore/src/kernels/missing.rs +++ /dev/null @@ -1,169 +0,0 @@ -use std::collections::HashMap; - -use crate::arrays::{ColumnData, Frame}; - -#[derive(Clone, Debug, PartialEq)] -pub struct ImputeAction { - pub column: String, - pub strategy: String, - pub count: usize, - pub value: String, -} - -pub fn drop_empty_columns(frame: &mut Frame) -> Vec { - let mut dropped = Vec::new(); - frame.columns.retain(|column| { - let keep = !column.data.is_all_null(); - if !keep { - dropped.push(column.name.clone()); - } - keep - }); - dropped -} - -pub fn drop_empty_rows(frame: &mut Frame) -> usize { - if frame.nrows == 0 { - return 0; - } - let mut keep = vec![false; frame.nrows]; - for row in 0..frame.nrows { - keep[row] = frame - .columns - .iter() - .any(|column| !matches!(column.data.cell_key(row), crate::arrays::CellKey::Null)); - } - let dropped = keep.iter().filter(|v| !**v).count(); - if dropped > 0 { - frame.take_rows(&keep); - } - dropped -} - -pub fn impute(frame: &mut Frame, strategy: Option<&str>) -> Vec { - let Some(strategy) = strategy else { - return Vec::new(); - }; - let mut actions = Vec::new(); - for column in &mut frame.columns { - match &mut column.data { - ColumnData::Float(values) => { - let missing = values.iter().filter(|v| v.is_none()).count(); - if missing == 0 || missing == values.len() { - continue; - } - let resolved = if strategy == "auto" { "median" } else { strategy }; - let fill = match resolved { - "mean" => mean(values), - "median" => median(values), - "mode" => mode_float(values), - _ => None, - }; - if let Some(fill) = fill { - for value in values.iter_mut().filter(|v| v.is_none()) { - *value = Some(fill); - } - actions.push(ImputeAction { - column: column.name.clone(), - strategy: resolved.to_string(), - count: missing, - value: format!("{fill:.6}"), - }); - } - } - ColumnData::Utf8(values) => { - let missing = values.iter().filter(|v| v.is_none()).count(); - if missing == 0 || missing == values.len() { - continue; - } - let resolved = if strategy == "auto" { "mode" } else { strategy }; - if resolved != "mode" { - continue; - } - if let Some(fill) = mode_string(values) { - for value in values.iter_mut().filter(|v| v.is_none()) { - *value = Some(fill.clone()); - } - actions.push(ImputeAction { - column: column.name.clone(), - strategy: "mode".to_string(), - count: missing, - value: format!("{fill:?}"), - }); - } - } - ColumnData::Bool(_) => {} - } - } - actions -} - -fn mean(values: &[Option]) -> Option { - let mut sum = 0.0; - let mut n = 0usize; - for v in values.iter().flatten() { - sum += *v; - n += 1; - } - (n > 0).then_some(sum / n as f64) -} - -fn median(values: &[Option]) -> Option { - let mut vals: Vec = values.iter().flatten().copied().filter(|v| !v.is_nan()).collect(); - if vals.is_empty() { - return None; - } - vals.sort_by(|a, b| a.total_cmp(b)); - let mid = vals.len() / 2; - if vals.len() % 2 == 0 { - Some((vals[mid - 1] + vals[mid]) / 2.0) - } else { - Some(vals[mid]) - } -} - -fn mode_float(values: &[Option]) -> Option { - let mut counts: HashMap = HashMap::new(); - for v in values.iter().flatten().filter(|v| !v.is_nan()) { - counts - .entry(v.to_bits()) - .and_modify(|(_, n)| *n += 1) - .or_insert((*v, 1)); - } - counts - .values() - .max_by(|(av, an), (bv, bn)| an.cmp(bn).then_with(|| bv.total_cmp(av))) - .map(|(v, _)| *v) -} - -fn mode_string(values: &[Option]) -> Option { - let mut counts: HashMap<&str, usize> = HashMap::new(); - for value in values.iter().flatten() { - *counts.entry(value.as_str()).or_insert(0) += 1; - } - counts - .into_iter() - .max_by(|(av, an), (bv, bn)| an.cmp(bn).then_with(|| bv.cmp(av))) - .map(|(v, _)| v.to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - #[test] - fn median_impute_fills_missing_numeric() { - let mut frame = Frame::new(vec![Column { - name: "x".into(), - data: ColumnData::Float(vec![Some(1.0), None, Some(3.0)]), - }]) - .unwrap(); - let actions = impute(&mut frame, Some("median")); - assert_eq!(actions[0].count, 1); - assert_eq!( - frame.columns[0].data, - ColumnData::Float(vec![Some(1.0), Some(2.0), Some(3.0)]) - ); - } -} diff --git a/crates/freshcore/src/kernels/mod.rs b/crates/freshcore/src/kernels/mod.rs deleted file mode 100644 index 9b4bdc7a..00000000 --- a/crates/freshcore/src/kernels/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod casts; -pub mod duplicates; -pub mod missing; -pub mod outliers; -pub mod profile; -pub mod strings; diff --git a/crates/freshcore/src/kernels/outliers.rs b/crates/freshcore/src/kernels/outliers.rs deleted file mode 100644 index 8c0a09a2..00000000 --- a/crates/freshcore/src/kernels/outliers.rs +++ /dev/null @@ -1,213 +0,0 @@ -use crate::arrays::{Column, ColumnData, Frame}; - -#[derive(Clone, Debug, PartialEq)] -pub struct OutlierAction { - pub column: String, - pub action: String, - pub count: usize, - pub lower: f64, - pub upper: f64, - pub flag_column: Option, -} - -pub fn handle_outliers( - frame: &mut Frame, - action: Option<&str>, - method: &str, - factor: f64, -) -> Vec { - let Some(action) = action else { - return Vec::new(); - }; - let mut actions = Vec::new(); - let existing: Vec = frame.columns.iter().map(|c| c.name.clone()).collect(); - let mut flags = Vec::new(); - - for column in &mut frame.columns { - let ColumnData::Float(values) = &mut column.data else { - continue; - }; - let Some((lo, hi)) = bounds(values, method, factor) else { - continue; - }; - let mask: Vec = values - .iter() - .map(|v| v.is_some_and(|x| x < lo || x > hi)) - .collect(); - let count = mask.iter().filter(|v| **v).count(); - if count == 0 { - continue; - } - if action == "clip" { - for value in values.iter_mut().flatten() { - if *value < lo { - *value = lo; - } else if *value > hi { - *value = hi; - } - } - actions.push(OutlierAction { - column: column.name.clone(), - action: "clip".into(), - count, - lower: lo, - upper: hi, - flag_column: None, - }); - } else { - let flag = unique_flag(&existing, &format!("{}_outlier", column.name)); - flags.push(Column { - name: flag.clone(), - data: ColumnData::Bool(mask.into_iter().map(Some).collect()), - }); - actions.push(OutlierAction { - column: column.name.clone(), - action: "flag".into(), - count, - lower: lo, - upper: hi, - flag_column: Some(flag), - }); - } - } - if !flags.is_empty() { - frame.columns.extend(flags); - } - actions -} - -pub fn bounds(values: &[Option], method: &str, factor: f64) -> Option<(f64, f64)> { - let vals: Vec = values.iter().flatten().copied().filter(|v| !v.is_nan()).collect(); - if vals.len() < 2 { - return None; - } - if method == "zscore" { - let mean = vals.iter().sum::() / vals.len() as f64; - let var = vals.iter().map(|v| (v - mean).powi(2)).sum::() / (vals.len() - 1) as f64; - let std = var.sqrt(); - if std == 0.0 || std.is_nan() { - return None; - } - return Some((mean - factor * std, mean + factor * std)); - } - let q1 = quantile(vals.clone(), 0.25)?; - let q3 = quantile(vals.clone(), 0.75)?; - let spread = q3 - q1; - if spread.is_nan() { - return None; - } - if spread == 0.0 { - return zero_iqr_bounds(&vals, q1, factor); - } - Some((q1 - factor * spread, q3 + factor * spread)) -} - -/// IQR-equivalent spread per unit of mean absolute deviation for normal data: -/// sqrt(pi/2) (MeanAD -> sigma) x 2*Phi^-1(0.75) (sigma -> IQR). Mirrors -/// `freshdata.steps.outliers.MEANAD_TO_IQR`. -const MEANAD_TO_IQR: f64 = 1.6906950787902986; -/// Largest share of values the zero-IQR fallback may flag; mirrors -/// `freshdata.steps.outliers.ZERO_IQR_MAX_SHARE`. -const ZERO_IQR_MAX_SHARE: f64 = 0.05; - -/// Fences when both quartiles equal `center` (at least half the column sits on -/// one value). A zero IQR must not silently disable detection. Instead, the IQR -/// is replaced by the normal-consistent spread from the mean absolute deviation -/// around the median (the median equals the quartiles here). The result is -/// `None` for a constant column, or when the fences would flag more than -/// `ZERO_IQR_MAX_SHARE` of the values: that many is a second mode, not rare -/// outliers. -fn zero_iqr_bounds(vals: &[f64], center: f64, factor: f64) -> Option<(f64, f64)> { - let n = vals.len() as f64; - let mean_ad = vals.iter().map(|v| (v - center).abs()).sum::() / n; - if mean_ad == 0.0 || mean_ad.is_nan() { - return None; - } - let spread = MEANAD_TO_IQR * mean_ad; - let (lo, hi) = (center - factor * spread, center + factor * spread); - let flagged = vals.iter().filter(|v| **v < lo || **v > hi).count() as f64; - if flagged > ZERO_IQR_MAX_SHARE * n { - return None; - } - Some((lo, hi)) -} - -fn quantile(mut vals: Vec, q: f64) -> Option { - if vals.is_empty() { - return None; - } - vals.sort_by(|a, b| a.total_cmp(b)); - let pos = q * (vals.len() - 1) as f64; - let lo = pos.floor() as usize; - let hi = pos.ceil() as usize; - if lo == hi { - Some(vals[lo]) - } else { - let weight = pos - lo as f64; - Some(vals[lo] * (1.0 - weight) + vals[hi] * weight) - } -} - -fn unique_flag(existing: &[String], base: &str) -> String { - let mut name = base.to_string(); - let mut i = 1usize; - while existing.contains(&name) { - i += 1; - name = format!("{base}_{i}"); - } - name -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn iqr_bounds_detect_extreme_value() { - let values = vec![Some(1.0), Some(2.0), Some(3.0), Some(100.0)]; - let (lo, hi) = bounds(&values, "iqr", 1.5).unwrap(); - assert!(lo < 1.0); - assert!(hi < 100.0); - } - - fn zeros_with(extra: &[f64], zeros: usize) -> Vec> { - std::iter::repeat(0.0) - .take(zeros) - .chain(extra.iter().copied()) - .map(Some) - .collect() - } - - #[test] - fn zero_iqr_falls_back_to_mean_absolute_deviation() { - let values = zeros_with(&[1000.0, 5000.0, 2.0, 3.0, -800.0], 95); - let (lo, hi) = bounds(&values, "iqr", 1.5).unwrap(); - // MeanAD = 6805 / 100; fences = 0 +/- 1.5 x 1.6907 x MeanAD. - let expected = 1.5 * (MEANAD_TO_IQR * (6805.0 / 100.0)); - assert_eq!((lo, hi), (-expected, expected)); - let mut frame = Frame { - nrows: values.len(), - columns: vec![Column { - name: "x".into(), - data: ColumnData::Float(values), - }], - }; - let actions = handle_outliers(&mut frame, Some("clip"), "iqr", 1.5); - assert_eq!(actions.len(), 1); - assert_eq!(actions[0].count, 3); // the spikes, not 2.0 / 3.0 - } - - #[test] - fn zero_iqr_fallback_does_not_flag_a_second_mode() { - // 30% non-zero, split around zero: the IQR is still zero. - let extra: Vec = (1..=15).flat_map(|i| [-(i as f64), i as f64]).collect(); - assert_eq!(bounds(&zeros_with(&extra, 70), "iqr", 1.5), None); - } - - #[test] - fn constant_column_has_no_bounds() { - let values = vec![Some(5.0); 20]; - assert_eq!(bounds(&values, "iqr", 1.5), None); - assert_eq!(bounds(&values, "zscore", 3.0), None); - } -} diff --git a/crates/freshcore/src/kernels/profile.rs b/crates/freshcore/src/kernels/profile.rs deleted file mode 100644 index 1263665e..00000000 --- a/crates/freshcore/src/kernels/profile.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::collections::HashSet; - -use crate::arrays::{CellKey, ColumnData, Frame}; - -#[derive(Clone, Debug, PartialEq)] -pub struct ColumnProfile { - pub name: String, - pub null_count: usize, - pub unique_count: usize, - pub min: Option, - pub max: Option, - pub mean: Option, -} - -pub fn profile(frame: &Frame) -> Vec { - frame - .columns - .iter() - .map(|column| { - let mut unique = HashSet::new(); - for row in 0..frame.nrows { - unique.insert(column.data.cell_key(row)); - } - let (min, max, mean) = numeric_stats(&column.data); - ColumnProfile { - name: column.name.clone(), - null_count: column.data.null_count(), - unique_count: unique - .into_iter() - .filter(|k| !matches!(k, CellKey::Null)) - .count(), - min, - max, - mean, - } - }) - .collect() -} - -fn numeric_stats(data: &ColumnData) -> (Option, Option, Option) { - let ColumnData::Float(values) = data else { - return (None, None, None); - }; - let vals: Vec = values.iter().flatten().copied().filter(|v| !v.is_nan()).collect(); - if vals.is_empty() { - return (None, None, None); - } - let min = vals.iter().copied().fold(f64::INFINITY, f64::min); - let max = vals.iter().copied().fold(f64::NEG_INFINITY, f64::max); - let mean = vals.iter().sum::() / vals.len() as f64; - (Some(min), Some(max), Some(mean)) -} diff --git a/crates/freshcore/src/kernels/strings.rs b/crates/freshcore/src/kernels/strings.rs deleted file mode 100644 index 20661642..00000000 --- a/crates/freshcore/src/kernels/strings.rs +++ /dev/null @@ -1,88 +0,0 @@ -use rayon::prelude::*; - -use crate::arrays::{ColumnData, Frame}; -use crate::plan::{CleanPlan, StringCase}; - -#[derive(Clone, Debug, Default, PartialEq)] -pub struct StringAction { - pub column: String, - pub trimmed: usize, - pub sentinels: usize, - pub case_normalized: usize, -} - -pub fn clean_strings(frame: &mut Frame, plan: &CleanPlan) -> Vec { - frame - .columns - .par_iter_mut() - .filter_map(|column| match &mut column.data { - ColumnData::Utf8(values) => { - let mut action = StringAction { - column: column.name.clone(), - ..StringAction::default() - }; - for slot in values { - let Some(value) = slot.as_mut() else { - continue; - }; - if plan.strip_whitespace { - let stripped = value.trim(); - if stripped.len() != value.len() { - *value = stripped.to_string(); - action.trimmed += 1; - } - } - if plan.normalize_sentinels - && plan.sentinels.contains(&value.trim().to_lowercase()) - { - *slot = None; - action.sentinels += 1; - continue; - } - match plan.string_case { - StringCase::Preserve => {} - StringCase::Lower => { - let lowered = value.to_lowercase(); - if lowered != *value { - *value = lowered; - action.case_normalized += 1; - } - } - StringCase::Upper => { - let uppered = value.to_uppercase(); - if uppered != *value { - *value = uppered; - action.case_normalized += 1; - } - } - } - } - (action.trimmed > 0 || action.sentinels > 0 || action.case_normalized > 0) - .then_some(action) - } - _ => None, - }) - .collect() -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use super::*; - use crate::arrays::{Column, ColumnData, Frame}; - - #[test] - fn trims_and_nulls_sentinels() { - let mut frame = Frame::new(vec![Column { - name: "name".into(), - data: ColumnData::Utf8(vec![Some(" x ".into()), Some("N/A".into())]), - }]) - .unwrap(); - let mut plan = CleanPlan::default(); - plan.sentinels = HashSet::from(["n/a".to_string()]); - let actions = clean_strings(&mut frame, &plan); - assert_eq!(actions[0].trimmed, 1); - assert_eq!(actions[0].sentinels, 1); - } -} diff --git a/crates/freshcore/src/lib.rs b/crates/freshcore/src/lib.rs deleted file mode 100644 index d3f046ff..00000000 --- a/crates/freshcore/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod arrays; -mod kernels; -mod plan; -mod python; - -pub use arrays::{Column, ColumnData, Frame}; -pub use plan::{CleanPlan, StringCase}; - -use pyo3::prelude::*; - -#[pymodule] -fn freshdata_freshcore(m: &Bound<'_, PyModule>) -> PyResult<()> { - python::register(m) -} diff --git a/crates/freshcore/src/plan.rs b/crates/freshcore/src/plan.rs deleted file mode 100644 index 837d3871..00000000 --- a/crates/freshcore/src/plan.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::collections::HashSet; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StringCase { - Preserve, - Lower, - Upper, -} - -#[derive(Clone, Debug)] -pub struct CleanPlan { - pub rename_map: Vec<(String, String)>, - pub strip_whitespace: bool, - pub normalize_sentinels: bool, - pub sentinels: HashSet, - pub string_case: StringCase, - pub drop_empty_columns: bool, - pub drop_empty_rows: bool, - pub drop_duplicates: bool, - pub duplicate_keep: String, - pub fix_dtypes: bool, - pub numeric_threshold: f64, - pub preserve_leading_zeros: bool, - pub impute: Option, - pub outliers: Option, - pub outlier_method: String, - pub outlier_factor: f64, -} - -impl Default for CleanPlan { - fn default() -> Self { - Self { - rename_map: Vec::new(), - strip_whitespace: true, - normalize_sentinels: true, - sentinels: HashSet::new(), - string_case: StringCase::Preserve, - drop_empty_columns: true, - drop_empty_rows: true, - drop_duplicates: true, - duplicate_keep: "first".to_string(), - fix_dtypes: true, - numeric_threshold: 0.95, - preserve_leading_zeros: true, - impute: None, - outliers: None, - outlier_method: "iqr".to_string(), - outlier_factor: 1.5, - } - } -} diff --git a/crates/freshcore/src/python.rs b/crates/freshcore/src/python.rs deleted file mode 100644 index 92b1b3ab..00000000 --- a/crates/freshcore/src/python.rs +++ /dev/null @@ -1,392 +0,0 @@ -use std::collections::HashSet; -use std::time::Instant; - -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyList}; - -use crate::arrays::{Column, ColumnData, Frame}; -use crate::kernels::{casts, duplicates, missing, outliers, profile, strings}; -use crate::plan::{CleanPlan, StringCase}; - -pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(execute_plan, m)?)?; - Ok(()) -} - -#[pyfunction] -fn execute_plan(py: Python<'_>, payload: &Bound<'_, PyDict>) -> PyResult { - let mut timings = Vec::new(); - let mut frame = frame_from_payload(payload)?; - let plan = plan_from_payload(payload)?; - let rows_before = frame.nrows; - let cols_before = frame.columns.len(); - let missing_before = frame.null_cells(); - let mut actions = Vec::new(); - let mut columns_dropped = Vec::new(); - let mut columns_imputed = Vec::new(); - let mut duplicates_removed = 0usize; - let mut duplicates_detected: Option = None; - let mut outliers_handled = 0usize; - - let t = Instant::now(); - apply_renames(&mut frame, &plan, &mut actions); - timings.push(("column_names".to_string(), t.elapsed().as_secs_f64())); - - let t = Instant::now(); - for action in strings::clean_strings(&mut frame, &plan) { - if action.trimmed > 0 { - actions.push(action_dict( - "strip_whitespace", - Some(action.column.clone()), - "trimmed surrounding whitespace", - action.trimmed, - )); - } - if action.case_normalized > 0 { - actions.push(action_dict( - "normalize_case", - Some(action.column.clone()), - "normalized string case", - action.case_normalized, - )); - } - if action.sentinels > 0 { - actions.push(action_dict( - "normalize_sentinels", - Some(action.column), - "replaced sentinel strings (\"N/A\", \"-\", \"\", …) with missing", - action.sentinels, - )); - } - } - timings.push(("clean_strings".to_string(), t.elapsed().as_secs_f64())); - - if plan.drop_empty_columns { - let t = Instant::now(); - let dropped = missing::drop_empty_columns(&mut frame); - if !dropped.is_empty() { - columns_dropped.extend(dropped.clone()); - actions.push(action_dict( - "drop_empty_columns", - None, - &format!("dropped {} all-missing column(s): {}", dropped.len(), dropped.join(", ")), - dropped.len(), - )); - } - timings.push(("drop_empty_columns".to_string(), t.elapsed().as_secs_f64())); - } - - if plan.drop_empty_rows { - let t = Instant::now(); - let dropped = missing::drop_empty_rows(&mut frame); - if dropped > 0 { - actions.push(action_dict( - "drop_empty_rows", - None, - &format!("dropped {dropped} all-missing row(s)"), - dropped, - )); - } - timings.push(("drop_empty_rows".to_string(), t.elapsed().as_secs_f64())); - } - - let t = Instant::now(); - for action in casts::infer_and_cast(&mut frame, &plan) { - let suffix = if action.coerced > 0 { - format!(" ({} unparseable value(s) set to missing)", action.coerced) - } else { - String::new() - }; - actions.push(action_dict( - "fix_dtypes", - Some(action.column), - &format!("converted to {}{}", action.target, suffix), - action.count, - )); - } - timings.push(("fix_dtypes".to_string(), t.elapsed().as_secs_f64())); - - if plan.drop_duplicates { - let t = Instant::now(); - let before = frame.nrows; - let dropped = duplicates::drop_duplicates(&mut frame, &plan.duplicate_keep); - duplicates_removed = dropped; - if dropped > 0 { - let pct = 100.0 * dropped as f64 / before as f64; - actions.push(action_dict( - "drop_duplicates", - None, - &format!( - "dropped {dropped} duplicate row(s) ({pct:.1}% of rows, keep={:?})", - plan.duplicate_keep - ), - dropped, - )); - } - timings.push(("drop_duplicates".to_string(), t.elapsed().as_secs_f64())); - } else { - // Detection only: count at the same stage as the pandas step (after - // string cleaning, empty-row removal and casts; before imputation and - // outliers) so the adapter can report and escalate like pandas. - let t = Instant::now(); - duplicates_detected = Some(duplicates::count_duplicates(&frame)); - timings.push(("detect_duplicates".to_string(), t.elapsed().as_secs_f64())); - } - - let t = Instant::now(); - for action in missing::impute(&mut frame, plan.impute.as_deref()) { - columns_imputed.push(action.column.clone()); - actions.push(action_dict( - "impute", - Some(action.column), - &format!( - "filled {} missing value(s) with {} ({})", - action.count, action.strategy, action.value - ), - action.count, - )); - } - timings.push(("impute".to_string(), t.elapsed().as_secs_f64())); - - let t = Instant::now(); - for action in outliers::handle_outliers( - &mut frame, - plan.outliers.as_deref(), - &plan.outlier_method, - plan.outlier_factor, - ) { - outliers_handled += action.count; - let description = if action.action == "clip" { - format!( - "clipped {} outlier(s) to [{:.6}, {:.6}] ({}, factor {:.6})", - action.count, action.lower, action.upper, plan.outlier_method, plan.outlier_factor - ) - } else { - format!( - "flagged {} outlier(s) in new column {:?} ({}, factor {:.6})", - action.count, - action.flag_column.unwrap_or_default(), - plan.outlier_method, - plan.outlier_factor - ) - }; - actions.push(action_dict( - "outliers", - Some(action.column), - &description, - action.count, - )); - } - timings.push(("outliers".to_string(), t.elapsed().as_secs_f64())); - - let profiles = profile::profile(&frame); - let result = PyDict::new_bound(py); - result.set_item("rows_before", rows_before)?; - result.set_item("rows_after", frame.nrows)?; - result.set_item("cols_before", cols_before)?; - result.set_item("cols_after", frame.columns.len())?; - result.set_item("missing_before", missing_before)?; - result.set_item("missing_after", frame.null_cells())?; - result.set_item("duplicates_removed", duplicates_removed)?; - result.set_item("duplicates_detected", duplicates_detected)?; - result.set_item("outliers_handled", outliers_handled)?; - result.set_item("columns_dropped", columns_dropped)?; - result.set_item("columns_imputed", columns_imputed)?; - result.set_item("actions", actions)?; - result.set_item("stage_timings", timings)?; - result.set_item("profile", profiles_to_py(py, &profiles)?)?; - result.set_item("columns", columns_to_py(py, &frame.columns)?)?; - Ok(result.into()) -} - -fn frame_from_payload(payload: &Bound<'_, PyDict>) -> PyResult { - let columns_obj = payload - .get_item("columns")? - .ok_or_else(|| PyErr::new::("missing columns"))?; - let columns_list = columns_obj.downcast::()?; - let mut columns = Vec::new(); - for item in columns_list { - let col = item.downcast::()?; - let name: String = col.get_item("name")?.unwrap().extract()?; - let dtype: String = col.get_item("dtype")?.unwrap().extract()?; - let values_obj = col.get_item("values")?.unwrap(); - let values = values_obj.downcast::()?; - let data = match dtype.as_str() { - "float" => ColumnData::Float( - values - .iter() - .map(|v| { - if v.is_none() { - Ok(None) - } else { - Ok(Some(v.extract::()?)) - } - }) - .collect::>>()?, - ), - "bool" => ColumnData::Bool( - values - .iter() - .map(|v| { - if v.is_none() { - Ok(None) - } else { - Ok(Some(v.extract::()?)) - } - }) - .collect::>>()?, - ), - _ => ColumnData::Utf8( - values - .iter() - .map(|v| { - if v.is_none() { - Ok(None) - } else { - Ok(Some(v.extract::()?)) - } - }) - .collect::>>()?, - ), - }; - columns.push(Column { name, data }); - } - Frame::new(columns).map_err(|e| PyErr::new::(e)) -} - -fn plan_from_payload(payload: &Bound<'_, PyDict>) -> PyResult { - let cfg_obj = payload - .get_item("config")? - .ok_or_else(|| PyErr::new::("missing config"))?; - let cfg = cfg_obj.downcast::()?; - let mut plan = CleanPlan::default(); - plan.rename_map = extract_pairs(cfg, "rename_map")?; - plan.strip_whitespace = get_bool(cfg, "strip_whitespace", true)?; - plan.normalize_sentinels = get_bool(cfg, "normalize_sentinels", true)?; - plan.sentinels = extract_strings(cfg, "sentinels")?.into_iter().collect::>(); - plan.string_case = match get_optional_string(cfg, "string_case")?.as_deref() { - Some("lower") => StringCase::Lower, - Some("upper") => StringCase::Upper, - _ => StringCase::Preserve, - }; - plan.drop_empty_columns = get_bool(cfg, "drop_empty_columns", true)?; - plan.drop_empty_rows = get_bool(cfg, "drop_empty_rows", true)?; - plan.drop_duplicates = get_bool(cfg, "drop_duplicates", true)?; - plan.duplicate_keep = get_string(cfg, "duplicate_keep", "first")?; - plan.fix_dtypes = get_bool(cfg, "fix_dtypes", true)?; - plan.numeric_threshold = get_float(cfg, "numeric_threshold", 0.95)?; - plan.preserve_leading_zeros = get_bool(cfg, "preserve_leading_zeros", true)?; - plan.impute = get_optional_string(cfg, "impute")?; - plan.outliers = get_optional_string(cfg, "outliers")?; - plan.outlier_method = get_string(cfg, "outlier_method", "iqr")?; - plan.outlier_factor = get_float(cfg, "outlier_factor", 1.5)?; - Ok(plan) -} - -fn apply_renames(frame: &mut Frame, plan: &CleanPlan, actions: &mut Vec) { - if plan.rename_map.is_empty() { - return; - } - let mut count = 0usize; - for column in &mut frame.columns { - if let Some((_, new)) = plan.rename_map.iter().find(|(old, _)| old == &column.name) { - column.name = new.clone(); - count += 1; - } - } - if count > 0 { - Python::with_gil(|py| { - actions.push(action_dict( - "column_names", - None, - &format!("renamed {count} column(s)"), - count, - ).into_py(py)); - }); - } -} - -fn action_dict(step: &str, column: Option, description: &str, count: usize) -> PyObject { - Python::with_gil(|py| { - let d = PyDict::new_bound(py); - d.set_item("step", step).unwrap(); - d.set_item("column", column).unwrap(); - d.set_item("description", description).unwrap(); - d.set_item("count", count).unwrap(); - d.into() - }) -} - -fn columns_to_py(py: Python<'_>, columns: &[Column]) -> PyResult { - let out = PyList::empty_bound(py); - for column in columns { - let d = PyDict::new_bound(py); - d.set_item("name", &column.name)?; - match &column.data { - ColumnData::Float(values) => { - d.set_item("dtype", "float")?; - d.set_item("values", values)?; - } - ColumnData::Bool(values) => { - d.set_item("dtype", "bool")?; - d.set_item("values", values)?; - } - ColumnData::Utf8(values) => { - d.set_item("dtype", "string")?; - d.set_item("values", values)?; - } - } - out.append(d)?; - } - Ok(out.into()) -} - -fn profiles_to_py(py: Python<'_>, profiles: &[profile::ColumnProfile]) -> PyResult { - let out = PyList::empty_bound(py); - for p in profiles { - let d = PyDict::new_bound(py); - d.set_item("name", &p.name)?; - d.set_item("null_count", p.null_count)?; - d.set_item("unique_count", p.unique_count)?; - d.set_item("min", p.min)?; - d.set_item("max", p.max)?; - d.set_item("mean", p.mean)?; - out.append(d)?; - } - Ok(out.into()) -} - -fn get_bool(cfg: &Bound<'_, PyDict>, key: &str, default: bool) -> PyResult { - Ok(cfg.get_item(key)?.map_or(default, |v| v.extract().unwrap_or(default))) -} - -fn get_float(cfg: &Bound<'_, PyDict>, key: &str, default: f64) -> PyResult { - Ok(cfg.get_item(key)?.map_or(default, |v| v.extract().unwrap_or(default))) -} - -fn get_string(cfg: &Bound<'_, PyDict>, key: &str, default: &str) -> PyResult { - Ok(cfg - .get_item(key)? - .map_or_else(|| default.to_string(), |v| v.extract().unwrap_or_else(|_| default.to_string()))) -} - -fn get_optional_string(cfg: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match cfg.get_item(key)? { - Some(v) if !v.is_none() => Ok(Some(v.extract()?)), - _ => Ok(None), - } -} - -fn extract_strings(cfg: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match cfg.get_item(key)? { - Some(v) => v.extract(), - None => Ok(Vec::new()), - } -} - -fn extract_pairs(cfg: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match cfg.get_item(key)? { - Some(v) => v.extract(), - None => Ok(Vec::new()), - } -} diff --git a/docs/backends.md b/docs/backends.md index 348ccddf..8c99a6ee 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -119,7 +119,6 @@ native extension from the repo checkout: ```bash pip install -e ".[dev,freshcore]" -maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module python benchmarks/bench_freshcore.py --rows 10000 100000 --workload full ``` diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 8acd819a..a88a7db5 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -73,9 +73,6 @@ Makefile shortcuts: `make benchmark`, `make benchmark-ci`, `make benchmark-repor `results///.json` in the schema in `benchmarks/results_schema.py`, so runs diff cleanly across versions. -> The legacy quick-bench (`python benchmarks/bench_quick.py --fixtures --compare`) -> is preserved for ad-hoc throughput checks on the `tests/fixtures/` corpus. - ## Calibration metrics (CleanBench Phase 3) The CleanBench mini-suite (`benchmarks/cleanbench/`, run in CI by @@ -192,12 +189,7 @@ Measured on the committed labelled dataset abbreviations, blanked fields, transliteration, shared households, deliberate near-collisions). Method: **rule-weighted linkage** (normalized weighted average — no EM/Fellegi–Sunter, no Splink-parity claim). Raw output: -`benchmarks/er_results.json`. Reproduce: - -```console -python benchmarks/bench_entity_resolution.py \ - --labelled benchmarks/data/er_labelled_5k.csv --json -``` +`benchmarks/er_results.json`. | Configuration | Precision | Recall | F1 | FP | FN | False-merged clusters | |---|---:|---:|---:|---:|---:|---:| @@ -239,24 +231,6 @@ See [`docs/fixtures.md`](fixtures.md) for fixture schemas and the `DEFECT_MANIFEST` structure, and `benchmarks/README.md` for the baseline contract (`run(df)` + `AUTHORED_LINES`, lazy imports, graceful skip). -## Reproduce the legacy corpus - -```bash -python benchmarks/bench_quick.py --fixtures --compare # tests/fixtures corpus, side by side -python benchmarks/bench_quick.py --online --compare # cached online datasets -``` - -Every fixture in `tests/fixtures/` is run under `conservative`, `balanced`, and -`aggressive` strategies in CI, plus 50 curated real public datasets. Reproduce -the quality/efficiency matrix on your own data with: - -```python -import freshdata as fd -print(fd.compare_clean(df)) -``` - - - # CleanBench full-suite results Generated: 2026-07-05T07:07:36+00:00 diff --git a/docs/context-policies.md b/docs/context-policies.md index bd27a86b..17642c53 100644 --- a/docs/context-policies.md +++ b/docs/context-policies.md @@ -182,10 +182,6 @@ policy dict, plus a warning per unresolved/unparsed item. workflow. `drop_if`/`rename`/`map` rules are still compiled and carried in the policy but not yet executed. -See the notebook -[`notebooks/06_context_cleaning.ipynb`](https://github.com/FreshCode-Org/freshdata/blob/main/notebooks/06_context_cleaning.ipynb) -for the full ecommerce walkthrough. - ## Optional intent-head evidence (Phase 5) An optional intent classifier (`fd-intent-v1`, see diff --git a/docs/developer-training-pipeline.md b/docs/developer-training-pipeline.md deleted file mode 100644 index e0793d0b..00000000 --- a/docs/developer-training-pipeline.md +++ /dev/null @@ -1,115 +0,0 @@ -# Developer training pipeline (Phase 5) - -The `training/` package builds, evaluates, and packages FreshData's small -local model artifacts. It is a **development-time tool**: it lives outside -`src/freshdata`, is never shipped in the wheel, and the runtime never -imports it. - -## What stays true at runtime - -- `fd.clean` is deterministic, offline, and **model-free by default** -- no LLM is used at runtime, ever — teacher models exist only in `training/` -- no cloud call from `fd.clean`; no automatic model download -- no generative repair model exists anywhere in FreshData -- the protected-column byte-identity guard is absolute -- no model weights ship in the PyPI wheel (enforced by - `.github/workflows/wheel-size.yml` and `tests/test_semantic_backcompat.py`) - -## Pipeline stages - -```bash -make training-seed # license gate + seed corpus -make training-corrupt # ~40 corruptors -> labeled pairs -make training-teacher-labels # compliance-gated teacher tasks (optional) -make training-distill # role/intent heads, export, int8 quantize -make training-eval # calibration + full CleanBench gates -make training-package-artifacts # dist/artifacts/ with manifests + cards -``` - -End-to-end: - -```bash -make training-dev-artifacts # synthetic-only, offline, ~seconds -make training-release-artifacts # full pipeline; fails on any gate failure -``` - -## How seed data is licensed - -Every source is registered in `training/seed/registry.json` with license, -attribution, explicit `allowed_for_training`, and a resolved PII risk; -`python -m training.datasets.validators --check-licenses` blocks unclear -licenses, share-alike data without legal review, and unresolved PII risk. -See `training/seed/LICENSES.md`. - -## How corruptors create ground truth - -`training/corruptors/` injects parameterized, deterministic dirt into known -clean frames and emits a machine-verifiable label per mutation (raw value, -clean value, transform family, `should_repair`, `should_auto_apply`, risk, -protected, ambiguous). Ground truth is corruption metadata — never a model -guess. Ambiguous corruptions are labeled *never auto-apply*; protected-column -traps label **any** mutation as failure. - -## What teacher models may and may not do - -Allowed (development-time only, batched, cached, PII-masked): realism -direction, column-role labeling, context paraphrases, ambiguity -adjudication, rationale templates, benchmark red-teaming. - -Never: runtime calls, per-cell calls, generating messy/clean pairs from -scratch, receiving full rows or real PII, or serving as sole release-gating -truth. See `training/COMPLIANCE.md` and -`python -m training.teacher.compliance check`. - -## How human review works - -≥5% of each teacher batch is sampled for review; >3% disagreement forces a -full batch review; release-gating eval labels are 100% human-verified with -reviewer identity and timestamps (`training/teacher/review.py`, -`training/eval/human_verified.py`). - -## How model cards are produced - -`training/model_cards/*.md` are authored with the 15 required sections and -copied into each packaged artifact; `training.distill.package_artifacts` -fails when a card, license summary, or eval summary is missing. - -## How calibration is trained - -`training/calibration/` extracts per-proposal features from CleanBench runs, -fits isotonic curves (pool-adjacent-violators, pure numpy) per (backend, -issue family), and exports the registry-compatible `calib-v1` table. Gates: -ECE ≤ 0.03 and P@0.95 ≥ 0.99 on a held-out split. The runtime only ever -*loads* the JSON — a missing table degrades to `uncalibrated`, and -protected-column checks stay deterministic. - -## How CleanBench gates releases - -`python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site ---check-gates` runs representation, semantic-value, context-compliance, -profile-replay, and scale tracks; see [Benchmarks](benchmarks.md) for the -gate list and latest results. - -## Reproducing artifact builds - -Everything is seeded: corruptors, generators, and training are deterministic -(`sha256`-hashed features, zero-init full-batch training — no RNG). The same -commit + same seeds produce byte-identical weights; manifests record the git -commit and per-file SHA256. - -## Running the tests - -- model-free (default): `pytest -m "not online and not large"` — this is PR CI -- semantic-with-stub: included in the default run (`FRESHDATA_SEMANTIC_STUB` - paths and the CleanBench stub encoder need no model files) -- real-model nightly, locally: - `pip install -e ".[semantic]" onnx && make training-dev-artifacts && - pytest tests -k "semantic or models or cleanbench" --no-cov` - -## Limitations and known failure modes - -Head evals run on synthetic/template corpora — high scores there do not -promise the same on arbitrary real-world data; the encoder contrastive stage -is optional and is recorded as `skipped` without torch; T5 perf gates -compare against a same-machine baseline and are informational until one is -pinned. See [Honest limitations](limitations.md). diff --git a/docs/examples.md b/docs/examples.md index 9408af84..5e76993f 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -18,7 +18,6 @@ directory has narrated Jupyter walkthroughs. | `01_missing_values.py` | Smart, role-aware missing-value imputation | | `02_outliers.py` | Outlier detection and flagging vs removal | | `03_normalization.py` | Feature normalization for ML | -| `04_profiling.py` | Read-only data profiling and EDA | | `05_ml_pipeline.py` | End-to-end ML preprocessing with scikit-learn | | `06_large_dataset.py` | Cleaning a large synthetic dataset, with timing | | `07_pandas_integration.py` | Dropping freshdata into an existing pandas workflow | diff --git a/docs/field-validation.md b/docs/field-validation.md index 09a6a2ca..673df3b4 100644 --- a/docs/field-validation.md +++ b/docs/field-validation.md @@ -157,12 +157,7 @@ exception tables, lineage). `report.to_frame()` gives a flat DataFrame; Validation runs a vectorized pre-screen per column and only pays the per-cell explanation path for suspect cells, so mostly-clean feeds validate -at bulk speed. Reproduce the comparison against regex and pandas-coercion -baselines with: - -```bash -python benchmarks/bench_fieldcheck.py 100000 -``` +at bulk speed. ## Known limitations diff --git a/docs/freshcore.md b/docs/freshcore.md deleted file mode 100644 index 9a60077e..00000000 --- a/docs/freshcore.md +++ /dev/null @@ -1,125 +0,0 @@ -# FreshCore architecture - -FreshCore is FreshData's optional native backend for cleaning-first workloads. -It is designed to plug into the existing execution layer rather than replace the -public API or the pandas reference implementation. - -## Integration model - -- Users opt in with `fd.clean(df, engine="freshcore")`. -- `CleanConfig` still controls what cleaning is requested. -- `EngineConfig` still controls how execution is routed. -- The default `fd.clean(df)` path remains unchanged. -- Unsupported work delegates to the pandas reference pipeline and is recorded in - `CleanReport.fallback_events`. - -The Python adapter lives at `freshdata.execution.backends._freshcore`. It -materializes pandas-compatible inputs, sends compact columns and plan parameters -to the Rust module, then maps native output back into a pandas DataFrame and the -standard `CleanReport` audit contract. - -## Native engine - -The Rust crate is in `crates/freshcore` and exposes the `freshdata_freshcore` -PyO3 module. Its internal model is deliberately smaller than a general-purpose -DataFrame: - -- typed nullable arrays (`Float`, `Bool`, `Utf8`) -- separate validity/null representation through `Option` -- a compact physical cleaning plan -- native kernels for strings, missing values, casts, duplicates, outliers, and - simple profiles -- operation-level timing records for benchmarks - -FreshCore does not depend on pandas, Polars, DuckDB, Spark, Dask, Modin, Vaex, -or any existing DataFrame execution engine. - -## V1 support boundary - -FreshCore v1 runs native kernels for conservative, deterministic cleaning: - -- column name normalization -- whitespace trimming -- optional text case normalization via `string_case=None|"lower"|"upper"` -- sentinel-to-missing normalization -- empty row/column drops -- full-row duplicate detection with `duplicate_keep="first"` or `"last"` -- boolean and numeric string casts where safe -- mean/median/mode imputation -- IQR/z-score outlier clipping or flagging, including the pandas zero-IQR - fallback to mean-absolute-deviation fences -- simple per-column profile metadata - -FreshCore falls back for semantic cleaning, context/policy protection, cleaning -memory, domain packs, contracts, non-default indexes, duplicate subsets, -aggregate/drop duplicate modes, model-based outliers, constant-column dropping, -memory downcasting, and the balanced/aggressive decision engine. It also falls -back for `impute="missforest"`, per-column `impute_strategy`, outlier handling -on float columns holding `±inf`, and mode/auto imputation of nullable boolean -columns with missing values. - -The same two gaps can open inside a native run. `fix_dtypes` casts text columns -before imputation and outlier handling, so a `"yes"`/`"no"` column with missing -values becomes a boolean column the kernels do not impute. A numeric text -column holding `"inf"` becomes a float column holding `±inf`, which leaves the -outlier fences undefined. The input frame shows neither, so the adapter checks -the returned column dtypes. When a cast column hits a gap, it reruns the frame -on pandas and records a fallback event naming the column -(`fallback_step="impute"` or `"outliers"`). Under `fallback_policy="error"` it -raises `FallbackError` instead. `fd.plan()` checks only the input, so it -cannot predict these fallbacks, and the discarded native run still costs time. -Frames without such casts stay on the native path. - -With `drop_duplicates=False` (the default), the native module counts full-row -duplicates at the same stage as the pandas step: after string cleaning, -empty-row removal and casts, and before imputation and outliers. It returns -the count as `duplicates_detected` and records a `detect_duplicates` stage -timing. The adapter reports it like pandas: a "detected N duplicate row(s)" -action, a warning above `duplicate_threshold`, and `DuplicateRatioError` under -`duplicate_ratio_action="error"`. Native modules built before this count -existed don't report `duplicates_detected`; with those, detection is skipped, -and under `duplicate_ratio_action="error"` the adapter falls back to pandas. - -The native arrays carry only float, bool and string values, so FreshCore also -falls back for datetime, timedelta, categorical, period and interval columns, -for integer columns holding values beyond ±2\*\*53 (float64 cannot represent -them exactly), and for column labels that collide once stringified (such as -`1` and `"1"`). Integer columns (`int64`, nullable `Int64`, and other widths) -are cast back to their input dtype when every returned value is integral and -in range; otherwise they come back as `float64` and the change is recorded in -`report.backend_differences`. Non-string column labels such as `0` and `1` -come back unchanged. - -## Building and testing - -The CI `freshcore-native` job runs the same steps: - -```bash -pip install -e ".[dev,freshcore]" -cargo test --manifest-path crates/freshcore/Cargo.toml -maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module -pytest tests/test_execution -k freshcore -``` - -`tests/test_execution/test_freshcore_native_parity.py` runs the real extension -against the pandas reference and is skipped when `freshdata_freshcore` is not -installed. The other FreshCore tests use a fake native module. - -## Benchmarking - -Build the native module before benchmarking: - -```bash -pip install -e ".[dev,freshcore]" -maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module -python benchmarks/bench_freshcore.py --rows 10000 100000 1000000 --workload full -``` - -The benchmark compares: - -- a handwritten pandas baseline -- FreshData's pandas reference path -- FreshCore through `engine="freshcore"` - -The output includes timing, peak memory, fallback events, parity shape checks, -and FreshCore stage timings when the native module is installed. diff --git a/docs/interactive.md b/docs/interactive.md index c9e9e21d..22112b1c 100644 --- a/docs/interactive.md +++ b/docs/interactive.md @@ -71,11 +71,3 @@ raises a clear install hint (e.g. *"install `freshdata-cleaner[viz]`"*) rather t - **`explain_clean(df).show()`** — before/after column comparison with risky changes separated from safe ones. -## Generating HTML examples - -```bash -python scripts/generate_html_examples.py # writes docs/examples/*.html -``` - -The committed samples under `docs/examples/` are produced by that script from -synthetic data, so they always match the current renderers. diff --git a/docs/learning-profiles.md b/docs/learning-profiles.md index 78e6231b..9e701362 100644 --- a/docs/learning-profiles.md +++ b/docs/learning-profiles.md @@ -183,19 +183,6 @@ freshdata profile merge orders_v1.fdprofile orders_v2.fdprofile -o merged.fdprof positional argument keeps its original meaning — data profiling (`freshdata profile data.csv`) — so existing scripts don't break. -## CleanBench T4 - -`benchmarks/cleanbench/t4_profiles.py` covers 10 fixture families (category -maps, allowed-value maps, reference typos, sentinels, day-first dates, IN -phone formats, double-`@` emails, an imputation fixture that must *not* -replay literals, and an unexplained-only fixture). Each fixture learns a -profile from a train split and replays it on a held-out batch of the same -corruption family. Gates enforced in `tests/test_cleanbench_t4.py`: mean F1 -lift over the replayable fixtures, false-modification rate no worse than the -no-profile baseline, zero protected-column violations, and zero privacy -leaks (no raw sensitive literal ever appears in the serialized profile bytes -under `privacy="mask"`). - ## Limitations - **No model training, no LLM, no cloud.** Every transform a profile can diff --git a/docs/limitations.md b/docs/limitations.md index c6eef0c0..49d4d4ea 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -206,12 +206,3 @@ context policies, and cleaning memory need **none** of these. --- -## Development-time training pipeline - -These apply only to the dev-time `training/` package, never to `import freshdata`: - -- The optional encoder contrastive-distillation stage requires a tensor - framework; without it the stage records `skipped` and the Phase-3 baseline - encoder is retained (its safety gates still run). -- Dev artifact builds without the `onnx` package export portable weight JSON - instead of `.onnx` graphs; release builds require `onnx` and fail without it. diff --git a/docs/model-cards.md b/docs/model-cards.md index d64bb199..c8385130 100644 --- a/docs/model-cards.md +++ b/docs/model-cards.md @@ -6,16 +6,6 @@ summary, PII policy, teacher-model usage, human-review process, evaluation metrics, known limitations, failure modes, the protected-column guarantee, calibration version, artifact hash, and FreshData version compatibility. -The authoritative cards live in the repository and are copied into each -packaged artifact (`dist/artifacts//model_card.md`): - -- [`fd-col-encoder-v1`](https://github.com/FreshCode-Org/freshdata/blob/main/training/model_cards/fd-col-encoder-v1.md) - — column/value encoder + semantic-type role head -- [`fd-intent-v1`](https://github.com/FreshCode-Org/freshdata/blob/main/training/model_cards/fd-intent-v1.md) - — context-sentence intent head (optional evidence for the deterministic parser) -- [`calib-v1`](https://github.com/FreshCode-Org/freshdata/blob/main/training/model_cards/calib-v1.md) - — isotonic confidence-calibration tables (plain JSON) - Every card states explicitly: - there is **no generative repair model** diff --git a/docs/performance-investigation.md b/docs/performance-investigation.md index 8512ed3e..01da6beb 100644 --- a/docs/performance-investigation.md +++ b/docs/performance-investigation.md @@ -281,13 +281,9 @@ The focused fixture verification was: The exact command in the brief (`print(result.to_json())`) cannot run because `SemanticProbeBuild` intentionally has no `to_json()` API. It failed with -`AttributeError: 'SemanticProbeBuild' object has no attribute 'to_json'`. The -following serialization-only equivalent was run without changing the probe or -production API and produced valid JSON: - -```bash -PYTHONPATH=src .venv-qa/bin/python -c 'import json; from benchmarks.performance.datasets import DatasetSpec, make_mixed_frame; from benchmarks.performance.semantic_probe import probe_context_build; from freshdata.config import CleanConfig; frame = make_mixed_frame(DatasetSpec(rows=100000, width="medium", seed=42, dataset_type="mixed")); _, result = probe_context_build(frame, CleanConfig(semantic_mode="assist", verbose=False)); print(json.dumps({"total_calls": result.total_calls, "total_eligible_calls": result.total_eligible_calls, "total_bypassed_calls": result.total_bypassed_calls, "total_theoretical_hits": result.total_theoretical_hits, "by_operation": {name: {"total_calls": item.total_calls, "eligible_calls": item.eligible_calls, "bypassed_calls": item.bypassed_calls, "unique_keys": item.unique_keys, "theoretical_hits": item.theoretical_hits, "hit_rate": item.hit_rate} for name, item in result.by_operation.items()}}, sort_keys=True))' -``` +`AttributeError: 'SemanticProbeBuild' object has no attribute 'to_json'`. A +serialization-only equivalent was run without changing the probe or +production API and produced valid JSON. The successful single-build JSON reported `639307` total calls, `497842` eligible calls, `141465` bypassed calls, and `52725` theoretical diff --git a/docs/plugins.md b/docs/plugins.md index 72bd546a..02ddb312 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -172,9 +172,6 @@ the column/frame they were given, and that nothing exceeds the declared Runnable, contract-passing examples live in the repo: -- [`examples/plugins/custom_expert/acronym_expert.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_expert/acronym_expert.py) — expand acronyms in categorical columns -- [`examples/plugins/custom_backend/keyword_backend.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_backend/keyword_backend.py) — strip a shared prefix frame-wide, budget-aware -- [`examples/plugins/custom_validator/min_rows_validator.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_validator/min_rows_validator.py) — flag too-few-rows and constant columns - [`examples/plugins/custom_comparator/initials_comparator.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_comparator/initials_comparator.py) — initials-agreement ER comparator - [`examples/plugins/custom_exporter/markdown_exporter.py`](https://github.com/FreshCode-Org/freshdata/blob/main/examples/plugins/custom_exporter/markdown_exporter.py) — Markdown clean-report exporter diff --git a/docs/semantic-models.md b/docs/semantic-models.md index 18f32609..856ca76a 100644 --- a/docs/semantic-models.md +++ b/docs/semantic-models.md @@ -212,15 +212,3 @@ values only, structurally). No generative repairs. No automatic downloads. No model weights in the wheel. Protected columns remain byte-identical under any backend — enforced physically by the executor guard, not by convention. -## How these models are built (Phase 5) - -The artifacts themselves (`fd-col-encoder-v1`, `fd-intent-v1`, `calib-v1`) -are produced by the development-time training pipeline in `training/` — -seed-corpus governance, a ~40-corruptor labeling engine, compliance-gated -teacher tasks, human-reviewed eval labels, int8 quantization, and manifested -packaging under `dist/artifacts/`. Every artifact ships with a -[model card](model-cards.md) and is gated by the full -[CleanBench suite](benchmarks.md). See the -[developer training pipeline](developer-training-pipeline.md) guide. None of -this changes the runtime contract above: model-free by default, offline, -no LLM, no automatic downloads, no weights in the wheel. diff --git a/docs/streaming.md b/docs/streaming.md index 42bf7867..44f525e0 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -211,31 +211,3 @@ freshdata benchmark-stream --rows 100000000 --batch-size 100000 --cols 20 \ --report benchmark.json ``` -## The 100M-row stable-memory benchmark - -`benchmarks/bench_streaming.py` is an **out-of-core / streaming** test, *not* a -single-DataFrame test. It generates synthetic rows lazily, one batch at a time, feeds -them through a `StreamingCleaner`, and tracks peak RSS (via `psutil`, falling back to the -`resource` module). - -```bash -python benchmarks/bench_streaming.py --rows 100000000 --batch-size 100000 --cols 20 -``` - -It reports total rows processed, batch size, columns, elapsed time, rows/sec, baseline -and peak memory, steady-state memory growth, and the final cumulative trust score. -**Acceptance** is that steady-state memory does not grow with rows — peak memory stays -bounded relative to the *batch* size, not the dataset size — and it fails loudly if -fewer rows are processed than requested. - -!!! warning "Honest limitations" - - Streaming mode is **micro-batch**, not true row-by-row real time. - - **Global** cross-batch duplicate detection is limited: by default duplicates are - scoped within a batch; enabling `global_duplicates` uses a *bounded recent-window* - (the `window_size` most recently seen distinct rows) that can miss duplicates - older than the window. - - Medians/quantiles are **approximate** (reservoir-sampled), and the top-k category - summary is approximate when a column saturates `max_categories`. - - Kafka and Arrow Flight are **optional** integrations. - - The enterprise-scale "clean ≥100M rows out-of-core with stable memory" claim - depends on the benchmark above passing in **your** environment. diff --git a/docs/trust-claims.md b/docs/trust-claims.md index 28494569..f700476e 100644 --- a/docs/trust-claims.md +++ b/docs/trust-claims.md @@ -24,7 +24,6 @@ named tests, and CI fails if either side drifts. | Fully offline; only network call is `fd.models.pull` | **holds** | default test gate is `-m "not online"`; no other network code paths | | Polars/DuckDB/Spark backends scale beyond pandas | **holds with measured boundary** | see [backends](backends.md): DuckDB native handle peaks at ~40% of eager RSS (measured); `polars-lazy` defers only final materialization — pipeline intermediates still collect eagerly. Reproduce: `python benchmarks/bench_outofcore.py` | | Native handle kept with `strategy="conservative"` | **corrected** | also requires `fix_dtypes=False`; dtype fixing forces the recorded pandas fallback | -| StreamingCleaner cleans arbitrarily tall data with stable memory | **holds** | bounded reservoirs/counters, RSS benchmark `benchmarks/bench_streaming.py`; dedupe window is recent-window, not global (see [limitations](limitations.md)) | | CSV exports safe to open in spreadsheets | **holds by default on all first-party CSV outputs** (`fd.clean_csv`, `freshdata clean` / `apply-plan`, the streaming CLI, review queues); opt out with `sanitize_formulas=False` / `--no-sanitize-formulas` | see [threat model](threat-model.md); `tests/test_csv_formula_sanitize.py` | | Typed (`py.typed`), mypy-clean | **holds** | `mypy src/freshdata` in CI quality gate (187 files) | | `fallback_policy="error"` prevents any unrequested pandas materialization | **holds** | enforced *before* the pandas pipeline runs at every backend delegation point; `tests/test_execution/test_fallback_policy.py` | diff --git a/examples/04_profiling.py b/examples/04_profiling.py deleted file mode 100644 index 23dc139d..00000000 --- a/examples/04_profiling.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Read-only data profiling and EDA with freshdata. - -`fd.profile` inspects data quality without changing anything, using the same -inference engine as `fd.clean`. Run: - - python examples/04_profiling.py -""" - -import pandas as pd - -import freshdata as fd - - -def make_data() -> pd.DataFrame: - return pd.DataFrame( - { - "First Name": [" Ann ", "Bob", "Bob", "Cara", None], - "AGE": ["34", "N/A", "N/A", "41", "29"], - "Joined Date": ["2021-01-01", "2021-02-15", "2021-02-15", "-", "2021-03-30"], - "Active": ["yes", "no", "no", "yes", "no"], - "Salary($)": ["$1,200.50", "-", "-", "$2,000", "$1,750"], - "empty": [None, None, None, None, None], - } - ) - - -def main() -> None: - df = make_data() - - # Human-readable profile - print(fd.profile(df)) - - # With a dry-run plan attached - profile = fd.profile(df, include_plan=True) - print("\nPlanned primary model per column:") - print(profile.plan.summary()) - - # Inferred roles - print("\nInferred roles:") - print(fd.infer_roles(df)) - - -if __name__ == "__main__": - main() diff --git a/examples/cdc_event_time_gate.py b/examples/cdc_event_time_gate.py deleted file mode 100644 index 1fd63b59..00000000 --- a/examples/cdc_event_time_gate.py +++ /dev/null @@ -1,53 +0,0 @@ -"""CDC / event-time quality gate with ``fd.cdc_profile``. - -Change-data-capture streams fail in ways that are *not* missing values: records -arrive stale, late, out of order, with duplicate keys, bad operation codes, or as -a replayed batch. ``cdc_profile`` classifies those freshness/ordering defects -separately from nulls and reports trust penalties you can gate on. Run: - - python examples/cdc_event_time_gate.py -""" - -import pandas as pd - -import freshdata as fd - - -def main() -> None: - # An order-change feed captured from a CDC connector, in arrival order. - changes = pd.DataFrame( - { - "order_id": [101, 101, 102, 103, 103, 101, 104], - "event_ts": [ - "2024-03-01 09:00:00", - "2024-03-01 09:05:00", - "2024-03-01 09:06:00", - "2024-03-01 09:05:30", # 102/103 interleave is fine (different keys) - None, # missing event time - "2024-03-01 08:30:00", # 101 arrives 35m late, past the watermark - "2024-03-01 09:05:30", - ], - "op": ["c", "u", "c", "c", "u", "u", "Q"], # 'Q' is not a valid op code - } - ) - - report = fd.cdc_profile( - changes, - event_time="event_ts", - key="order_id", - lateness="10m", - operation_col="op", - now=pd.Timestamp("2024-03-01 09:10:00"), - stale_after="30m", - ) - - print(report.summary()) - print("\ntrust penalties (0=clean, 1=worst):", report.trust_penalties) - print(f"freshness: {report.freshness_seconds:.0f}s behind reference") - print(f"\ngate passed: {report.passed}\n") - print("defect table:") - print(report.to_frame()[["kind", "level", "n_rows", "rationale"]].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/examples/cross_source_link.py b/examples/cross_source_link.py deleted file mode 100644 index d8f085b0..00000000 --- a/examples/cross_source_link.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Cross-source entity linking with ``fd.link``. - -Link customer records across two systems (CRM vs. billing), explain every -candidate match, and hand low-confidence pairs to a steward-review queue. -FreshData is the candidate-generation + preprocessing layer here — it formats -and explains matches; it does not replace a dedicated matcher (use the external -adapter hook to plug one in). Run: - - python examples/cross_source_link.py -""" - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import build_review_queue - - -def main() -> None: - crm = pd.DataFrame( - { - "full_name": ["Acme Corp", "Globex LLC", "Initech Inc", "Stark Industries"], - "zip": ["10001", "94105", "73301", "10002"], - } - ) - billing = pd.DataFrame( - { - "full_name": ["acme corporation", "Globex L.L.C.", "Umbrella Co", "Stark Ind."], - "zip": ["10001", "94105", "60601", "10002"], - } - ) - - # Fuzzy link: block on zip (cheap, exact), Jaro-Winkler compare the name. - report = fd.link( - crm, billing, - keys=["full_name"], - strategy="fuzzy", - threshold=0.82, - blocking="l.zip = r.zip", - ) - print(report.summary()) - print("\ncandidate pairs (with per-field explanations):") - print(report.to_frame()[["left_id", "right_id", "match_probability", "decision"]] - .to_string(index=False)) - - # Hand the borderline pairs to a steward-review queue. - queue = build_review_queue(report) - print(f"\nsteward review queue: {len(queue.items)} item(s) need a human decision") - - -if __name__ == "__main__": - main() diff --git a/examples/entity_resolution_duckdb.py b/examples/entity_resolution_duckdb.py deleted file mode 100644 index f77d2009..00000000 --- a/examples/entity_resolution_duckdb.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Probabilistic entity resolution at scale with a DuckDB backend. - -Deduplicate records using DuckDB-powered blocking (candidate-pair generation) -plus rule-weighted probabilistic scoring. Falls back to the pandas backend when -DuckDB is not installed. Run: - - python examples/entity_resolution_duckdb.py - -DuckDB backend extra: - - pip install "freshdata-cleaner[entity-resolution]" -""" - -import importlib.util - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import BlockingRule, ComparisonLevel, EntityResolutionConfig - - -def people() -> pd.DataFrame: - return pd.DataFrame( - { - "id": [1, 2, 3, 4, 5, 6], - "name": [ - "Jonathan Smith", - "Jon Smith", - "Johnny Smith", - "Alice Brown", - "Alicia Brown", - "Robert King", - ], - "dob": [ - "1990-01-01", "1990-01-01", "1990-01-01", - "1985-05-05", "1985-05-05", "1970-12-12", - ], - "email": [ - "jsmith@x.com", "jsmith@x.com", "jon@y.com", - "alice@z.com", "alicia@z.com", "rking@q.com", - ], - } - ) - - -def main() -> None: - backend = "duckdb" - if importlib.util.find_spec("duckdb") is None: - backend = "pandas" - print("duckdb not installed -> using the pandas fallback backend\n") - - config = EntityResolutionConfig( - enabled=True, - backend=backend, - unique_id_column="id", - # Blocking caps the candidate space (no full cartesian product). - blocking_rules=( - BlockingRule(sql="lower(l.email) = lower(r.email)", description="same email"), - BlockingRule(sql="l.dob = r.dob", description="same date of birth"), - ), - comparisons=( - ComparisonLevel(column="name", kind="jaro_winkler", threshold=0.85, weight=3.0), - ComparisonLevel(column="dob", kind="exact", weight=1.0), - ComparisonLevel(column="email", kind="exact", weight=1.0), - ), - match_threshold=0.80, - clerical_review_threshold=0.55, - max_pairs=1_000_000, # hard safety gate - ) - - resolved, report = fd.resolve_entities(people(), config=config) - print(report.summary(), "\n") - for cluster in report.clusters: - print( - f"cluster {cluster.cluster_id}: records {list(cluster.record_ids)} " - f"(canonical={cluster.canonical_record_id}, confidence={cluster.confidence:.2f})" - ) - print("\nresolved frame (with cluster_id):") - print(resolved[["id", "name", "cluster_id"]].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/examples/plugins/custom_backend/keyword_backend.py b/examples/plugins/custom_backend/keyword_backend.py deleted file mode 100644 index 4dbed392..00000000 --- a/examples/plugins/custom_backend/keyword_backend.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Example FreshData plugin: a custom semantic *backend*. - -A backend proposes over the *whole frame* (not one column at a time) and is -opted into by name via ``semantic_backends=(...)``. This one strips a shared -``"lbl_"`` prefix from any string column — a frame-wide transform an -expert (which sees one column) is a poor fit for. - -Try it:: - - import freshdata as fd - from keyword_backend import KeywordBackend - - fd.testing.semantic_backend_contract(KeywordBackend()) - fd.register_backend(KeywordBackend()) - - import pandas as pd - df = pd.DataFrame({"tag": ["lbl_red", "lbl_blue", "plain"]}) - cleaned, report = fd.clean( - df, semantic_mode="auto", - semantic_backends=("deterministic", "keyword"), - return_report=True, - ) - -Package it:: - - [project.entry-points."freshdata.backends"] - keyword = "keyword_backend:KeywordBackend" -""" - -from __future__ import annotations - -import pandas as pd - -from freshdata.semantic.backends.base import Budget -from freshdata.semantic.scoring import make_proposal -from freshdata.semantic.types import SemanticContext, SemanticEvidence, SemanticProposal - -_PREFIX = "lbl_" - - -class KeywordBackend: - name = "keyword" - semantic_types = ("categorical",) - max_risk = "medium" - uses_network = False - requires = () - - def warm_up(self) -> None: - # Nothing to load. Raise freshdata.semantic.backends.base.BackendUnavailable - # here to self-disable when an optional resource is missing. - pass - - def propose( - self, df: pd.DataFrame, ctx: SemanticContext, budget: Budget - ) -> list[SemanticProposal]: - proposals: list[SemanticProposal] = [] - for col in df.columns: - info = ctx.columns.get(str(col)) - if info is None or info.identifier_like: - continue # never touch id-like columns - distinct = df[col].dropna().unique() - # Respect the model budget the same way built-in backends do. - if not budget.try_column(len(distinct)): - break - for raw in distinct: - if isinstance(raw, str) and raw.startswith(_PREFIX): - proposals.append(make_proposal( - column=str(col), - raw_value=raw, - proposed_value=raw[len(_PREFIX):], - issue_type="category_synonym", - expert=self.name, - base_confidence=0.95, - evidence=(SemanticEvidence("pattern", f"strip {_PREFIX!r}", 0.0),), - count=int((df[col] == raw).sum()), - rationale=f"stripped {_PREFIX!r} prefix from {raw!r}", - info=info, - )) - return proposals diff --git a/examples/plugins/custom_expert/acronym_expert.py b/examples/plugins/custom_expert/acronym_expert.py deleted file mode 100644 index 099f3e8f..00000000 --- a/examples/plugins/custom_expert/acronym_expert.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Example FreshData plugin: a custom semantic *expert*. - -An expert proposes value repairs for a single column, one distinct value at a -time. This one expands a small, explicit acronym table (``"NY" -> "New York"``) -in categorical columns. It is deterministic, offline, and low-risk. - -Try it:: - - import freshdata as fd - from acronym_expert import AcronymExpert - - fd.testing.expert_contract(AcronymExpert()) # verify the contract first - fd.register_expert(AcronymExpert()) - - import pandas as pd - df = pd.DataFrame({"state": ["NY", "CA", "NY", "TX"]}) - cleaned, report = fd.clean(df, semantic_mode="auto", return_report=True) - -The plugin only *proposes* — the policy gate decides what applies. Because -``category_synonym`` is a medium-risk issue type, these expansions are recorded -as ``status="suggested"`` in the report (not auto-applied) unless you raise the -confidence or run a lower-risk issue type. That is the gate doing its job: a -plugin can never force an auto-apply. - -Package it for entry-point discovery in your pyproject.toml:: - - [project.entry-points."freshdata.experts"] - acronym = "acronym_expert:AcronymExpert" -""" - -from __future__ import annotations - -import pandas as pd - -from freshdata.semantic.scoring import make_proposal -from freshdata.semantic.types import SemanticColumnInfo, SemanticEvidence, SemanticProposal - -_ACRONYMS = { - "ny": "New York", - "ca": "California", - "tx": "Texas", - "wa": "Washington", -} - - -class AcronymExpert: - # --- required metadata ------------------------------------------------- - name = "acronym" - issue_type = "category_synonym" # reuses the built-in scoring/risk profile - #: The semantic column kinds this expert claims to handle (declaration only; - #: routing still calls applies()). - semantic_types = ("categorical",) - #: The highest risk this expert may ever emit; the plugin system drops any - #: proposal scored above it before the gate sees it. - max_risk = "medium" - #: Deterministic + offline: no network, no optional dependencies. - uses_network = False - requires = () - - def applies(self, info: SemanticColumnInfo) -> bool: - # Only short, low-cardinality categorical columns — never an id/target. - return ( - info.role == "categorical" - and not info.identifier_like - and not info.free_text - ) - - def propose(self, series: pd.Series, info: SemanticColumnInfo) -> list[SemanticProposal]: - # Operate on *distinct values* (never row-by-row) and never mutate the - # series — return proposals; the gate decides what actually applies. - proposals: list[SemanticProposal] = [] - for raw, count in series.value_counts(dropna=True).items(): - if not isinstance(raw, str): - continue - expansion = _ACRONYMS.get(raw.strip().lower()) - if expansion is None or expansion == raw: - continue - proposals.append(make_proposal( - column=info.name, - raw_value=raw, - proposed_value=expansion, - issue_type=self.issue_type, - expert=self.name, - base_confidence=0.9, - evidence=(SemanticEvidence("lookup", f"acronym table -> {expansion!r}", 0.0),), - count=int(count), - rationale=f"expanded acronym {raw!r} to {expansion!r}", - info=info, - )) - return proposals diff --git a/examples/plugins/custom_validator/min_rows_validator.py b/examples/plugins/custom_validator/min_rows_validator.py deleted file mode 100644 index c72363f6..00000000 --- a/examples/plugins/custom_validator/min_rows_validator.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Example FreshData plugin: a custom *validator*. - -A validator is **read-only**: it inspects the frame (and the compiled context -policy) and returns :class:`freshdata.QualityFinding` records. It never repairs -anything — validation is not repair. This one flags frames with too few rows to -reason about, and any fully-constant column. - -Try it:: - - import freshdata as fd - from min_rows_validator import MinRowsValidator - - fd.testing.validator_contract(MinRowsValidator()) - fd.register_validator(MinRowsValidator()) - - import pandas as pd - df = pd.DataFrame({"x": [1, 1, 1]}) - findings = fd.validate(df, context="x is unique.") # validators run inside fd.validate - -Package it:: - - [project.entry-points."freshdata.validators"] - min_rows = "min_rows_validator:MinRowsValidator" -""" - -from __future__ import annotations - -import pandas as pd - -from freshdata import QualityFinding - -_MIN_ROWS = 30 - - -class MinRowsValidator: - name = "min_rows" - max_risk = "low" - uses_network = False - requires = () - - def validate(self, df: pd.DataFrame, policy: object, ctx: object) -> list[QualityFinding]: - findings: list[QualityFinding] = [] - if len(df) < _MIN_ROWS: - findings.append(QualityFinding.create( - severity="warning", - step="plugin", - rule_name="plugin.min_rows", - message=f"only {len(df)} rows (< {_MIN_ROWS}); statistics may be unreliable", - expected_condition=f">= {_MIN_ROWS} rows", - extra={"n_rows": int(len(df))}, - )) - for col in df.columns: - non_null = df[col].dropna() - if len(non_null) > 1 and non_null.nunique() == 1: - findings.append(QualityFinding.create( - severity="info", - step="plugin", - rule_name="plugin.constant_column", - column=str(col), - message=f"column {col!r} is constant", - expected_condition="more than one distinct value", - )) - return findings diff --git a/examples/privacy_anonymization.py b/examples/privacy_anonymization.py deleted file mode 100644 index d0a8079c..00000000 --- a/examples/privacy_anonymization.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Stronger PII detection + reversible / format-preserving anonymization. - -Demonstrates the dependency-free detector, reversible tokenization with a vault, -surrogate format-preserving anonymization, HIPAA/GDPR-tagged audit events, and a -k-anonymity check. Run: - - python examples/privacy_anonymization.py -""" - -import os - -import pandas as pd - -import freshdata as fd -from freshdata.enterprise import ( - InMemoryTokenVault, - MaskingRule, - PIIDetectionConfig, - detokenize_value, - tokenize_value, -) - -#: Keyed strategies read their secret from this environment variable. Without a -#: key they use a random per-call key, so output would change on every run. -KEY_ENV = "FRESHDATA_DEMO_KEY" - - -def main() -> None: - # A throwaway demo key when none is set (or it is empty); use a real secret - # in production. - if not os.environ.get(KEY_ENV): - os.environ[KEY_ENV] = "DEMO-KEY-DO-NOT-COMMIT" - df = pd.DataFrame( - { - "patient_note": [ - "Patient MRN# 4451234 dob 1984-02-11, email j.doe@mail.com", - "Contact 555-12-3456 (SSN), card 4111 1111 1111 1111", - ], - "email": ["j.doe@mail.com", "a.smith@mail.com"], - "zip": ["90210", "90210"], - "gender": ["M", "F"], - } - ) - - # 1) Detect PII in free text (raw spans are redacted by default). - scan = fd.detect_pii(df, config=PIIDetectionConfig()) - print("== detection ==") - print(scan.summary(), "\n") - - # 2) Anonymize: tokenize the email column reversibly + scrub free text spans. - rules = ( - MaskingRule( - name="email_token", - columns=("email",), - strategy="tokenize", - reversible=True, - key_env=KEY_ENV, - entity_types=("EMAIL",), - ), - ) - out, report = fd.anonymize(df, rules=rules, detection_config=PIIDetectionConfig()) - print("== anonymized frame ==") - print(out[["patient_note", "email"]].to_string(index=False), "\n") - print(report.summary()) - ev = report.events[0] - print(f"sample event tags: hipaa={ev.hipaa_tag} gdpr={ev.gdpr_tag} risk={ev.risk_level}") - print("report leaks raw PII? ", "j.doe@mail.com" in report.to_json(), "\n") - - # 3) Reversible tokenization round-trip via a vault. - vault = InMemoryTokenVault() - token = tokenize_value("j.doe@mail.com", vault, os.environ[KEY_ENV]) - print(f"token={token} -> detokenized={detokenize_value(token, vault)}\n") - - # 4) Surrogate format-preserving anonymization keeps the shape. - fpe_rule = MaskingRule( - name="ssn_fpe", columns=("zip",), strategy="surrogate", preserve_format=True, - key_env=KEY_ENV, - ) - masked, _ = fd.anonymize(df, rules=(fpe_rule,)) - print("surrogate zip:", masked["zip"].tolist(), "(same length, not crypto FPE)\n") - - # 5) k-anonymity over quasi-identifiers. - k = fd.check_k_anonymity(df, ["zip", "gender"], k=2) - print("== k-anonymity ==") - print(k.summary()) - - -if __name__ == "__main__": - main() diff --git a/examples/schema_drift_monitoring.py b/examples/schema_drift_monitoring.py deleted file mode 100644 index 9820a97b..00000000 --- a/examples/schema_drift_monitoring.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Schema-drift & data-contract monitoring with persisted baselines. - -Build a PII-safe baseline from a trusted dataset, persist it as JSON, then -monitor new batches for schema drift, distribution drift (KS / PSI), contract -violations, and a trust-score quality gate. Run: - - python examples/schema_drift_monitoring.py -""" - -import tempfile -from pathlib import Path - -import numpy as np -import pandas as pd - -import freshdata as fd -from freshdata import ColumnContract, DataContract -from freshdata.enterprise import DriftConfig - - -def trusted() -> pd.DataFrame: - rng = np.random.default_rng(0) - return pd.DataFrame( - { - "customer_id": range(1000), - "age": rng.normal(42, 9, 1000).round(1), - "country": rng.choice(["US", "GB", "FR"], 1000, p=[0.6, 0.3, 0.1]), - "plan": rng.choice(["free", "pro", "enterprise"], 1000, p=[0.7, 0.25, 0.05]), - } - ) - - -def main() -> None: - # A data contract declares the expectations new data must honour. - contract = DataContract( - name="customers", - columns=( - ColumnContract(name="customer_id", dtype="int64", unique=True, nullable=False), - ColumnContract(name="age", dtype="float64", min_value=0, max_value=120), - ColumnContract(name="country", allowed_values=("US", "GB", "FR")), - ColumnContract(name="plan", allowed_values=("free", "pro", "enterprise")), - ), - trust_score_min=70.0, - ) - - # 1) Build + persist a baseline (no raw samples are stored by default). - base = fd.build_baseline(trusted(), name="customers", contract=contract) - path = Path(tempfile.gettempdir()) / "customers.baseline.json" - fd.save_baseline(base, path) - print(f"baseline written to {path} (PII-safe: raw values not stored)\n") - - # 2) A healthy new batch passes. - healthy = trusted() - report = fd.monitor_contract(healthy, baseline_path=path) - print("== healthy batch ==") - print(report.summary(), "\n") - - # 3) A drifted batch: age shifts up, France over-represented, a bad country. - rng = np.random.default_rng(7) - drifted = trusted() - drifted["age"] = rng.normal(60, 9, len(drifted)) # distribution shift -> KS/PSI - drifted["country"] = rng.choice(["US", "GB", "FR"], len(drifted), p=[0.2, 0.2, 0.6]) - drifted.loc[0, "country"] = "ZZ" # contract.allowed_values violation - - report = fd.compare_to_baseline(drifted, fd.load_baseline(path), drift_config=DriftConfig()) - print("== drifted batch ==") - print(report.summary()) - print(f"\npassed={report.passed} errors={report.n_errors} warnings={report.n_warnings}") - - # 4) Baseline-FREE schema diff: explain structural drift the moment a frame - # arrives, with no persisted snapshot. Here an upstream team renamed - # `country` -> `country_code`, dropped `plan`, and added `utm_source`, - # and `age` arrived as text. diff_schema explains all of it *before* any - # repair runs; on_unexpected / on_missing control the gate. - incoming = pd.DataFrame( - { - "customer_id": [1, 2, 3], - "age": ["42", "37", "51"], # dtype drift: text, not float - "country_code": ["US", "GB", "FR"], # rename of `country` - "utm_source": ["g", "fb", "g"], # undeclared - } - ) - incoming.attrs["semantic_types"] = {"country_code": "country"} - contract_sem = DataContract( - name="customers", - columns=( - ColumnContract(name="customer_id", dtype="int64", unique=True, nullable=False), - ColumnContract(name="age", dtype="float64"), - ColumnContract(name="country", semantic_type="country"), - ColumnContract(name="plan", allowed_values=("free", "pro", "enterprise")), - ), - ) - diff = fd.diff_schema(incoming, contract=contract_sem, on_unexpected="warn", on_missing="fail") - print("\n== baseline-free schema diff ==") - print(diff.summary()) - print("\ncategories:", diff.contract_results) - print("\nfindings table:") - print(diff.to_frame()[["check_id", "column", "level", "message"]].to_string(index=False)) - - # The same diff can ride along with a suggested plan for the incoming data. - plan = fd.suggest_plan(incoming, contract=contract_sem, on_missing="fail") - print(f"\nplan carries schema_diff: passed={plan.schema_diff.passed}") - - -if __name__ == "__main__": - main() diff --git a/examples/validation_suite_quickstart.py b/examples/validation_suite_quickstart.py deleted file mode 100644 index 29bc83a9..00000000 --- a/examples/validation_suite_quickstart.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Declarative validation with fd.ValidationSuite — validate, never mutate. - -Run: python examples/validation_suite_quickstart.py -""" - -from __future__ import annotations - -import pandas as pd - -import freshdata as fd - -df = pd.DataFrame( - { - "customer_id": ["c1", "c2", "c3", "c3"], # duplicate id - "age": [34, -2, 51, 29], # negative age - "email": ["a@x.com", "bad-email", "c@z.net", "d@w.io"], - "signup": pd.to_datetime(["2021-03-01", "2022-01-15", "2020-07-01", "2023-02-01"]), - "last_seen": pd.to_datetime(["2022-03-01", "2021-06-15", "2024-01-01", "2024-06-01"]), - } -) - -suite = fd.ValidationSuite( - name="customers", - rules=[ - fd.ColumnRule("customer_id", nullable=False, unique=True), - fd.ColumnRule("age", min_value=0, max_value=120), - # mostly=0.75: tolerate up to 25% bad emails as a warning, not a failure - fd.ColumnRule("email", regex=r"[^@]+@[^@]+\.[^@]+", mostly=0.75), - fd.ColumnRule("signup", min_datetime="2019-01-01", max_datetime="2026-01-01"), - ], - cross_column=[fd.CrossColumnRule("signup", "<=", "last_seen")], - min_rows=1, -) - -result = fd.validate(df, suite=suite) -print(f"passed: {result.passed} ({result.n_errors} errors, {result.n_warnings} warnings)") -for finding in result.report.findings: - print(f" [{finding.status}] {finding.check_id}: {finding.message}") - -# Suites are versioned, serializable artifacts — commit them next to your data. -suite.save("customers_suite.json") -print("\nsaved to customers_suite.json; validate in CI with:") -print(" freshdata validate data.csv --suite customers_suite.json") - -# Existing DataContract users migrate with one call: -contract = suite.to_contract() -same_suite = fd.ValidationSuite.from_contract(contract) -assert same_suite.to_contract().to_dict() == contract.to_dict() - -# result.raise_if_failed() raises fd.ValidationError — CI-friendly. diff --git a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json b/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json deleted file mode 100644 index 53759426..00000000 --- a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/f044a544-virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1.json +++ /dev/null @@ -1 +0,0 @@ -{"commit_hash": "f044a5447692b793b4015cfdb57da462e732e334", "env_name": "virtualenv-py-MKL_NUM_THREADS1-OMP_NUM_THREADS1-OPENBLAS_NUM_THREADS1", "date": 1782583138000, "params": {"arch": "arm64", "cpu": "Apple M2", "machine": "Johnnys-MacBook-Air.local", "num_cpu": "8", "os": "Darwin 24.5.0", "ram": "17179869184", "python": ""}, "python": "", "requirements": {}, "env_vars": {"OPENBLAS_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "OMP_NUM_THREADS": "1"}, "result_columns": ["result", "params", "version", "started_at", "duration", "stats_ci_99_a", "stats_ci_99_b", "stats_q_25", "stats_q_75", "stats_number", "stats_repeat", "samples", "profile"], "results": {"benchmark_pipeline.BenchmarkPipeline.peakmem_full_clean": [[137641984, null, 167673856, 202719232, NaN, 337313792, null, 402472960, 373719040, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "81108b98041660a33f2a45656fb6e5e69684b11e4a5080796bdb1ec0f81488d7", 1782586250728, 33.892], "benchmark_pipeline.BenchmarkPipeline.time_full_clean": [[0.463310957999056, null, 0.056742166001640726, 0.022257166001509177, NaN, 3.293534708001971, null, 0.43859825000254204, 0.17404883400013205, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "7f1f0bf1c404671cf2d850b0558cd09df3a4ad1910015af06f77e6391b341925", 1782586284622, 33.177, [-Infinity, null, -Infinity, -Infinity, null, -Infinity, null, -Infinity, -Infinity, null], [Infinity, null, Infinity, Infinity, null, Infinity, null, Infinity, Infinity, null], [0.46331, null, 0.056742, 0.022257, null, 3.2935, null, 0.4386, 0.17405, null], [0.46331, null, 0.056742, 0.022257, null, 3.2935, null, 0.4386, 0.17405, null], [1, null, 1, 1, null, 1, null, 1, 1, null], [1, null, 1, 1, null, 1, null, 1, 1, null]], "benchmark_pipeline.BenchmarkPipeline.track_output_cols": [[19, null, 19, 19, NaN, 19, null, 19, 19, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "a92633b51ba1c8e8b6ee94cf0bd3497388b752031794f2489d2e91c2da78a781", 1782586317800, 33.135], "benchmark_pipeline.BenchmarkPipeline.track_output_rows": [[10000, null, 10000, 10000, NaN, 100000, null, 100000, 100000, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "59e7aae9329927dc95d6b7c468ff17aefe31f8dae97ae4a8f26c5338d4c5c392", 1782586350936, 33.338], "benchmark_pipeline.BenchmarkPipeline.track_throughput": [[21512.716899315576, null, 181757.7324961564, 455954.7692941329, NaN, 29982.88089633863, null, 230523.16945141667, 376599.6663953862, NaN], [["10000", "100000"], ["'freshdata'", "'pandas'", "'polars'", "'pyjanitor'", "'autoclean'"]], "92a13a48f747501a52bbb7b2ec75ec2ba7294747061a896777ebd190eaea0330", 1782586384276, 33.487]}, "durations": {}, "version": 2} \ No newline at end of file diff --git a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json b/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json deleted file mode 100644 index a9605c21..00000000 --- a/freshdata-benchmarks/.asv/results/Johnnys-MacBook-Air.local/machine.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "arch": "arm64", - "cpu": "Apple M2", - "machine": "Johnnys-MacBook-Air.local", - "num_cpu": "8", - "os": "Darwin 24.5.0", - "ram": "17179869184", - "version": 1 -} \ No newline at end of file diff --git a/freshdata-benchmarks/.asv/results/benchmarks.json b/freshdata-benchmarks/.asv/results/benchmarks.json deleted file mode 100644 index 8ea46f5b..00000000 --- a/freshdata-benchmarks/.asv/results/benchmarks.json +++ /dev/null @@ -1,2205 +0,0 @@ -{ - "benchmark_columns.BenchmarkColumns.time_drop_columns": { - "code": "class BenchmarkColumns:\n def time_drop_columns(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to drop specific columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_columns(df, self.drop_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define columns for operations\n cols = list(self.df.columns)\n self.rename_mapping = {c: f\"renamed_{c}\" for c in cols[:5]}\n self.drop_cols = cols[-3:]\n self.select_cols = cols[:5]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_columns.BenchmarkColumns.time_drop_columns", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "1ef0230ec3919fe2889753f5eda881d444ad28cbf5bdb3ab204b2c5089645fa2", - "warmup_time": -1 - }, - "benchmark_columns.BenchmarkColumns.time_rename_columns": { - "code": "class BenchmarkColumns:\n def time_rename_columns(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to rename columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.rename_columns(df, self.rename_mapping)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define columns for operations\n cols = list(self.df.columns)\n self.rename_mapping = {c: f\"renamed_{c}\" for c in cols[:5]}\n self.drop_cols = cols[-3:]\n self.select_cols = cols[:5]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_columns.BenchmarkColumns.time_rename_columns", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "b521d65830f775bce1829d2efb24e77dc8479435c39c011682902a824eb38642", - "warmup_time": -1 - }, - "benchmark_columns.BenchmarkColumns.time_select_columns": { - "code": "class BenchmarkColumns:\n def time_select_columns(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to select a subset of columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.select_columns(df, self.select_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define columns for operations\n cols = list(self.df.columns)\n self.rename_mapping = {c: f\"renamed_{c}\" for c in cols[:5]}\n self.drop_cols = cols[-3:]\n self.select_cols = cols[:5]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_columns.BenchmarkColumns.time_select_columns", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "7c37c123056d3186b7b95e1b81d65597c9341c640b457de8da63638664e75b63", - "warmup_time": -1 - }, - "benchmark_duplicates.BenchmarkDuplicates.peakmem_detect_duplicates": { - "code": "class BenchmarkDuplicates:\n def peakmem_detect_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when detecting duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_duplicates.BenchmarkDuplicates.peakmem_detect_duplicates", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "e7a7c5ae31094bd4eb62fe29a14ddfc0b79fc5dc0ec4961ef7996e3626707389" - }, - "benchmark_duplicates.BenchmarkDuplicates.peakmem_drop_duplicates": { - "code": "class BenchmarkDuplicates:\n def peakmem_drop_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when dropping duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_duplicates.BenchmarkDuplicates.peakmem_drop_duplicates", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "f9c20916bd270a1f589e27741aef986b9dcb22930af88a9019cd8a9121b7f33d" - }, - "benchmark_duplicates.BenchmarkDuplicates.time_detect_duplicates": { - "code": "class BenchmarkDuplicates:\n def time_detect_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_duplicates.BenchmarkDuplicates.time_detect_duplicates", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "dcbc081cfff3b48875e14e33e2adf11992491b9fb865a9c7558c7f9c3050fea9", - "warmup_time": -1 - }, - "benchmark_duplicates.BenchmarkDuplicates.time_drop_duplicates": { - "code": "class BenchmarkDuplicates:\n def time_drop_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to drop duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_duplicates.BenchmarkDuplicates.time_drop_duplicates", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "1e3cbf23f14e4784563c39c4ad52fe5f28664d72596257d1ae1ca1b340484683", - "warmup_time": -1 - }, - "benchmark_duplicates.BenchmarkDuplicates.track_duplicate_count": { - "code": "class BenchmarkDuplicates:\n def track_duplicate_count(self, n_rows: int, library: str) -> int:\n \"\"\"Track the number of duplicate rows dropped.\"\"\"\n df = fresh_copy(self.df)\n start_rows = len(df)\n result = self.adapter.drop_duplicates(df)\n \n # Approximate counting since libraries return different formats\n try:\n return start_rows - len(result)\n except Exception:\n return -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_duplicates.BenchmarkDuplicates.track_duplicate_count", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "rows", - "version": "b66bd2c35387849867b7fd80fdbfb3c98dfa2522f2d4b1436b3e3bf977a3f957" - }, - "benchmark_encoding.BenchmarkEncoding.peakmem_label_encode": { - "code": "class BenchmarkEncoding:\n def peakmem_label_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during label encoding.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.label_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_encoding.BenchmarkEncoding.peakmem_label_encode", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "0c3d6aaad98acca22ce40446d30d6984ed9255d6753ba4c0c853d5123114f1ce" - }, - "benchmark_encoding.BenchmarkEncoding.peakmem_onehot_encode": { - "code": "class BenchmarkEncoding:\n def peakmem_onehot_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during one-hot encoding.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.onehot_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_encoding.BenchmarkEncoding.peakmem_onehot_encode", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "19c28b9d44f9ed8ec617db25d7caf56bb33793a3b989db69ba05a0f571d9125d" - }, - "benchmark_encoding.BenchmarkEncoding.time_label_encode": { - "code": "class BenchmarkEncoding:\n def time_label_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform label (ordinal) encoding.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.label_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_encoding.BenchmarkEncoding.time_label_encode", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "02519fa9019f2df7e033a48ab0ff2575050cf645b6ad0ccf8f7de981f8b82a4f", - "warmup_time": -1 - }, - "benchmark_encoding.BenchmarkEncoding.time_onehot_encode": { - "code": "class BenchmarkEncoding:\n def time_onehot_encode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform one-hot encoding on categorical columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.onehot_encode(df, self.cat_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.cat_cols = get_categorical_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_encoding.BenchmarkEncoding.time_onehot_encode", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "bf93a7f4cd8fe6dd729e1df93cead6f815de0cb2760ff956b596dcd63a0bdac8", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_detect_outliers": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_detect_outliers(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during IQR outlier detection.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_detect_outliers", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "59f8db92583efe5640ec4fd0faeb5f06ad8d9d9fb9f6e78a9b79995d577cdf3c" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_full_pipeline": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_full_pipeline(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory for the complete preprocessing pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_full_pipeline", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "db2d7c635f844bda6f231bacb0275d2b37157efbc314d2aeb426bdb660255276" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_group_aggregations": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_group_aggregations(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory for group aggregations.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_group_aggregations", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "bd744ee28a6c6adef0030c3cc8bdf1f119a1deac45faf873b3193ab354a376df" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_handle_missing": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_handle_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when filling missing values.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_handle_missing", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "da80e7c97ed3d9f46a0bc8dab83b3379131b256619c7fcb5b5d4a483be18cbf1" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_load_csv": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_load_csv(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when loading a CSV file.\"\"\"\n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n fd.clean(df, strategy=\"conservative\", preserve_original=True)\n else:\n import pandas as pd\n pd.read_csv(self.csv_path)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_load_csv", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "3934bb8774d108a1a8425562cb6f86a62b8fca0a95e3d5b80eec1f204b6c2dba" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_resolve_duplicates": { - "code": "class BenchmarkFreshDataVsPandas:\n def peakmem_resolve_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when dropping duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.peakmem_resolve_duplicates", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "4a419465cdfaa7d833dc191d228d30deb5b44859f72a6f1d56fa08aa0b54e53a" - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_detect_outliers": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_detect_outliers(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect outliers using IQR method.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_detect_outliers", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "e4675f916ceffcedee506ec98fdcbdf185e2b5fce1316780d95b8f69b3ca737e", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_full_pipeline": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_full_pipeline(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for the complete preprocessing pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_full_pipeline", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "1b452d2cfb3fa3fc504d5d3d981d03354065e80384b5f37dc07f1d06e26f91de", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_group_aggregations": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_group_aggregations(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for multi-column group aggregations.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_group_aggregations", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "c50dfafaefc6529bbca4d22386652bb1afe1ef30a12d482e971948dbebdd7e93", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_handle_missing": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_handle_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with column means.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_handle_missing", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "8837b26596f861ce61c5c6569da5f3d7e7ecc5a74a45770db79497a50ef6a895", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_load_csv": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_load_csv(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to load a CSV file and infer types.\"\"\"\n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n fd.clean(df, strategy=\"conservative\", preserve_original=True)\n else:\n import pandas as pd\n pd.read_csv(self.csv_path)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_load_csv", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "8181e5190e9331fd5dd8fae4703c10b5e3b6c59b677ef8073b31c2425da3ab51", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_resolve_duplicates": { - "code": "class BenchmarkFreshDataVsPandas:\n def time_resolve_duplicates(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect and drop duplicate rows.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_duplicates(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.time_resolve_duplicates", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "0954a66c2198a5cae859a059a4444297b46a7125b936e2de75399c8766c66b8e", - "warmup_time": -1 - }, - "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.track_pipeline_throughput": { - "code": "class BenchmarkFreshDataVsPandas:\n def track_pipeline_throughput(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate rows processed per second for the full pipeline.\"\"\"\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.full_clean(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset, adapter, and temp files for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Write a CSV for loading benchmarks\n self._tmpdir = tempfile.mkdtemp(prefix=\"asv_fdvspd_\")\n self.csv_path = os.path.join(self._tmpdir, f\"test_{n_rows}.csv\")\n self.df.to_csv(self.csv_path, index=False)\n \n gc_collect()", - "name": "benchmark_freshdata_vs_pandas.BenchmarkFreshDataVsPandas.track_pipeline_throughput", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows/s", - "version": "8031cd675584aca5f7bde0b00338dc80473b17fcdb9744506ef2ee61304a37bd" - }, - "benchmark_groupagg.BenchmarkGroupAgg.peakmem_group_agg_multi": { - "code": "class BenchmarkGroupAgg:\n def peakmem_group_agg_multi(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory for multi-column groupby aggregation.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "name": "benchmark_groupagg.BenchmarkGroupAgg.peakmem_group_agg_multi", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "713c2e1f22b1e4c6905c7c35c209e03539e56fdbd4119436f0815ad71469ac23" - }, - "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_multi": { - "code": "class BenchmarkGroupAgg:\n def time_group_agg_multi(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for multi-column groupby with multiple agg functions.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_multi(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_multi", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "d3a2f0459cede798766b70f2d3eadd81ecd4b26461b19d99575cb156e33b1da8", - "warmup_time": -1 - }, - "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_single": { - "code": "class BenchmarkGroupAgg:\n def time_group_agg_single(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for single-column groupby with mean aggregation.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_single(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_single", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "63d3db36ff71f3a8b0b02333de6f05e4c6d5f9bb567d30509aa2a119d3e5ae77", - "warmup_time": -1 - }, - "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_transform": { - "code": "class BenchmarkGroupAgg:\n def time_group_agg_transform(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time for groupby transform (centering within groups).\"\"\"\n df = fresh_copy(self.df)\n self.adapter.group_agg_transform(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_groupagg.BenchmarkGroupAgg.time_group_agg_transform", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "72e4c12736d638b8666a71e6e895fdc3d9237cd7bdcc10738d4acd115d8de4f9", - "warmup_time": -1 - }, - "benchmark_groupagg.BenchmarkGroupAgg.track_throughput_group_agg": { - "code": "class BenchmarkGroupAgg:\n def track_throughput_group_agg(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate rows processed per second for group aggregation.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.group_agg_multi(df, self.numeric_cols)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n gc_collect()", - "name": "benchmark_groupagg.BenchmarkGroupAgg.track_throughput_group_agg", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "rows/s", - "version": "f419a8f7f33e31c7d34037e53a9d01f6af914f4a85b84dd089c9f41ca63d13c3" - }, - "benchmark_io.BenchmarkIO.peakmem_csv_read_clean": { - "code": "class BenchmarkIO:\n def peakmem_csv_read_clean(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when reading and cleaning a CSV.\"\"\"\n self.time_csv_read_clean(cache, n_rows, library)\n\n def setup(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Check library availability and locate files.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n elif library == \"pandas\":\n try:\n import pandas # noqa: F401\n except ImportError:\n raise NotImplementedError(\"pandas is not available\")\n elif library == \"polars\":\n try:\n import polars # noqa: F401\n except ImportError:\n raise NotImplementedError(\"polars is not available\")\n \n self.csv_path = cache[\"files\"][n_rows][\"csv\"]\n self.parquet_path = cache[\"files\"][n_rows][\"parquet\"]\n \n if not self.csv_path:\n raise NotImplementedError(\"CSV file not generated\")\n \n gc_collect()\n\n def setup_cache(self):\n \"\"\"Generate test files once per class to avoid disk overhead in timing.\"\"\"\n import pandas as pd\n tmpdir = tempfile.mkdtemp()\n \n # We only generate files for the requested sizes in params\n sizes = [10_000, 100_000, 1_000_000]\n files = {}\n \n for size in sizes:\n df = get_dataset(size)\n csv_path = os.path.join(tmpdir, f\"test_{size}.csv\")\n parquet_path = os.path.join(tmpdir, f\"test_{size}.parquet\")\n \n df.to_csv(csv_path, index=False)\n try:\n df.to_parquet(parquet_path, engine=\"pyarrow\")\n files[size] = {\"csv\": csv_path, \"parquet\": parquet_path}\n except Exception:\n files[size] = {\"csv\": csv_path, \"parquet\": None}\n \n return {\"tmpdir\": tmpdir, \"files\": files}", - "name": "benchmark_io.BenchmarkIO.peakmem_csv_read_clean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "setup_cache_key": "benchmark_io:24", - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "220ea6335973d89fcde320c335958b551b8d530b3235adabbc8620003d0df473" - }, - "benchmark_io.BenchmarkIO.time_csv_read_clean": { - "code": "class BenchmarkIO:\n def time_csv_read_clean(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Measure time to read a CSV and run a full clean.\"\"\"\n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n fd.clean(df, strategy=\"balanced\", preserve_original=True)\n elif library == \"pandas\":\n import pandas as pd\n df = pd.read_csv(self.csv_path)\n df = df.dropna().drop_duplicates()\n elif library == \"polars\":\n import polars as pl\n df = pl.read_csv(self.csv_path)\n df = df.drop_nulls().unique()\n\n def setup(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Check library availability and locate files.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n elif library == \"pandas\":\n try:\n import pandas # noqa: F401\n except ImportError:\n raise NotImplementedError(\"pandas is not available\")\n elif library == \"polars\":\n try:\n import polars # noqa: F401\n except ImportError:\n raise NotImplementedError(\"polars is not available\")\n \n self.csv_path = cache[\"files\"][n_rows][\"csv\"]\n self.parquet_path = cache[\"files\"][n_rows][\"parquet\"]\n \n if not self.csv_path:\n raise NotImplementedError(\"CSV file not generated\")\n \n gc_collect()\n\n def setup_cache(self):\n \"\"\"Generate test files once per class to avoid disk overhead in timing.\"\"\"\n import pandas as pd\n tmpdir = tempfile.mkdtemp()\n \n # We only generate files for the requested sizes in params\n sizes = [10_000, 100_000, 1_000_000]\n files = {}\n \n for size in sizes:\n df = get_dataset(size)\n csv_path = os.path.join(tmpdir, f\"test_{size}.csv\")\n parquet_path = os.path.join(tmpdir, f\"test_{size}.parquet\")\n \n df.to_csv(csv_path, index=False)\n try:\n df.to_parquet(parquet_path, engine=\"pyarrow\")\n files[size] = {\"csv\": csv_path, \"parquet\": parquet_path}\n except Exception:\n files[size] = {\"csv\": csv_path, \"parquet\": None}\n \n return {\"tmpdir\": tmpdir, \"files\": files}", - "min_run_count": 2, - "name": "benchmark_io.BenchmarkIO.time_csv_read_clean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "setup_cache_key": "benchmark_io:24", - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "3961585e316ff5329a6f08bf6537a21a3026e4f97237977593dd8dc814f04b22", - "warmup_time": -1 - }, - "benchmark_io.BenchmarkIO.time_parquet_read_clean": { - "code": "class BenchmarkIO:\n def time_parquet_read_clean(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Measure time to read a Parquet file and run a full clean.\"\"\"\n if not self.parquet_path:\n raise NotImplementedError(\"Parquet missing\")\n \n if library == \"freshdata\":\n import freshdata as fd\n import pandas as pd\n df = pd.read_parquet(self.parquet_path)\n fd.clean(df, strategy=\"balanced\", preserve_original=True)\n elif library == \"pandas\":\n import pandas as pd\n df = pd.read_parquet(self.parquet_path)\n df = df.dropna().drop_duplicates()\n elif library == \"polars\":\n import polars as pl\n df = pl.read_parquet(self.parquet_path)\n df = df.drop_nulls().unique()\n\n def setup(self, cache, n_rows: int, library: str) -> None:\n \"\"\"Check library availability and locate files.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n elif library == \"pandas\":\n try:\n import pandas # noqa: F401\n except ImportError:\n raise NotImplementedError(\"pandas is not available\")\n elif library == \"polars\":\n try:\n import polars # noqa: F401\n except ImportError:\n raise NotImplementedError(\"polars is not available\")\n \n self.csv_path = cache[\"files\"][n_rows][\"csv\"]\n self.parquet_path = cache[\"files\"][n_rows][\"parquet\"]\n \n if not self.csv_path:\n raise NotImplementedError(\"CSV file not generated\")\n \n gc_collect()\n\n def setup_cache(self):\n \"\"\"Generate test files once per class to avoid disk overhead in timing.\"\"\"\n import pandas as pd\n tmpdir = tempfile.mkdtemp()\n \n # We only generate files for the requested sizes in params\n sizes = [10_000, 100_000, 1_000_000]\n files = {}\n \n for size in sizes:\n df = get_dataset(size)\n csv_path = os.path.join(tmpdir, f\"test_{size}.csv\")\n parquet_path = os.path.join(tmpdir, f\"test_{size}.parquet\")\n \n df.to_csv(csv_path, index=False)\n try:\n df.to_parquet(parquet_path, engine=\"pyarrow\")\n files[size] = {\"csv\": csv_path, \"parquet\": parquet_path}\n except Exception:\n files[size] = {\"csv\": csv_path, \"parquet\": None}\n \n return {\"tmpdir\": tmpdir, \"files\": files}", - "min_run_count": 2, - "name": "benchmark_io.BenchmarkIO.time_parquet_read_clean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "setup_cache_key": "benchmark_io:24", - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "f1b5ea830e1455fb6167dad313755185de43b4e43399321c0bdc1c00e3269d5f", - "warmup_time": -1 - }, - "benchmark_memory.BenchmarkMemory.peakmem_full_clean": { - "code": "class BenchmarkMemory:\n def peakmem_full_clean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak RSS memory used during the full cleaning pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.peakmem_full_clean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "6c95e4784d795e0e0df8601858ac885b866d0fc210bc7bb3381100f321798cf3" - }, - "benchmark_memory.BenchmarkMemory.track_memory_after": { - "code": "class BenchmarkMemory:\n def track_memory_after(self, n_rows: int, library: str) -> float:\n \"\"\"Track the deep memory usage of the cleaned output DataFrame (in MB).\"\"\"\n df = fresh_copy(self.df)\n result = self.adapter.full_clean(df)\n try:\n import pandas as pd\n if isinstance(result, pd.DataFrame):\n return result.memory_usage(deep=True).sum() / (1024 * 1024)\n else:\n # Approximate memory for Polars\n return result.estimated_size() / (1024 * 1024)\n except Exception:\n return -1.0\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.track_memory_after", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "MB", - "version": "db807e615b7760ff17a1bd6b0496a2b2761fd0141af5a5506c742c90e494f90f" - }, - "benchmark_memory.BenchmarkMemory.track_memory_before": { - "code": "class BenchmarkMemory:\n def track_memory_before(self, n_rows: int, library: str) -> float:\n \"\"\"Track the deep memory usage of the uncleaned input DataFrame (in MB).\"\"\"\n df = fresh_copy(self.df)\n import pandas as pd\n if isinstance(df, pd.DataFrame):\n return df.memory_usage(deep=True).sum() / (1024 * 1024)\n return -1.0\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.track_memory_before", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "MB", - "version": "17667be34c2072042958374f1cfdf2e31d1864a01c36a781f042776379cb26f3" - }, - "benchmark_memory.BenchmarkMemory.track_memory_ratio": { - "code": "class BenchmarkMemory:\n def track_memory_ratio(self, n_rows: int, library: str) -> float:\n \"\"\"Track the ratio of output memory / input memory.\n Values < 1.0 indicate memory savings (e.g. via downcasting).\n \"\"\"\n df = fresh_copy(self.df)\n \n import pandas as pd\n if not isinstance(df, pd.DataFrame):\n return float('nan')\n \n mem_before = df.memory_usage(deep=True).sum()\n if mem_before == 0:\n return float('nan')\n \n result = self.adapter.full_clean(df)\n \n try:\n if isinstance(result, pd.DataFrame):\n mem_after = result.memory_usage(deep=True).sum()\n else:\n mem_after = result.estimated_size()\n return float(mem_after) / float(mem_before)\n except Exception:\n return float('nan')\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_memory.BenchmarkMemory.track_memory_ratio", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "ratio", - "version": "b32c33f3ef5053946b26aa1bfb2802e6314caf61e56f80d194c9f2730d90014e" - }, - "benchmark_missing.BenchmarkMissing.peakmem_drop_missing": { - "code": "class BenchmarkMissing:\n def peakmem_drop_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when dropping rows with missing values.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_missing_rows(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.peakmem_drop_missing", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "5658150457480391fc62d023fea1fca88cb91519b7db29adffe5545ac48870b4" - }, - "benchmark_missing.BenchmarkMissing.peakmem_fill_mean": { - "code": "class BenchmarkMissing:\n def peakmem_fill_mean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when filling missing values with column mean.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.peakmem_fill_mean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "cf3ce857230bef6dd6eefecc6f9d1ff91ec96191867bdd98ae50215592b65276" - }, - "benchmark_missing.BenchmarkMissing.peakmem_fill_median": { - "code": "class BenchmarkMissing:\n def peakmem_fill_median(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when filling missing values with column median.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_median(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.peakmem_fill_median", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "6ed8e7abf094dec466fb2835bcd3f15a1db78c52f63c4b7ead5a6787c7aad95b" - }, - "benchmark_missing.BenchmarkMissing.time_drop_missing": { - "code": "class BenchmarkMissing:\n def time_drop_missing(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to drop rows with missing values.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.drop_missing_rows(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_drop_missing", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "f693acd4497aa6e60c4f635f92eea15f12d7eb392413dd6bcf7f4edb10f1531b", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_bfill": { - "code": "class BenchmarkMissing:\n def time_fill_bfill(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to backward-fill missing values.\n Supported by pandas, polars, and freshdata.\n \"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_bfill(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_bfill", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "46d2e36a856880108de0335b024aadf1ea87bc3461467cb7b32634817de76428", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_ffill": { - "code": "class BenchmarkMissing:\n def time_fill_ffill(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to forward-fill missing values.\n Supported by pandas, polars, and freshdata.\n \"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_ffill(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_ffill", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e91e9f73b924d058d1f132cb5c7b5c4bfcca5c1c80ddd9ddde9567203c9c0505", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_mean": { - "code": "class BenchmarkMissing:\n def time_fill_mean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with the column mean.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mean(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_mean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "c44170e34beeefce94fdbc75618c6ba9b33ca94d48f65c43f05f980c6c0911e6", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_median": { - "code": "class BenchmarkMissing:\n def time_fill_median(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with the column median.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_median(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_median", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "be041bafa51e925515bfa80e8bd76e4b7dab9b0002f2f3012c06f03bbc634da3", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.time_fill_mode": { - "code": "class BenchmarkMissing:\n def time_fill_mode(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to fill missing values with the column mode.\n Supported by pandas, sklearn, and freshdata.\n \"\"\"\n df = fresh_copy(self.df)\n self.adapter.fill_missing_mode(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_missing.BenchmarkMissing.time_fill_mode", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "8e021a865af521e03e7099d04d7f92db8f92c8cb935850f9d6e39809d0d4f9ce", - "warmup_time": -1 - }, - "benchmark_missing.BenchmarkMissing.track_throughput_drop_missing": { - "code": "class BenchmarkMissing:\n def track_throughput_drop_missing(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate rows processed per second for drop_missing.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.drop_missing_rows(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_missing.BenchmarkMissing.track_throughput_drop_missing", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'feature_engine'", - "'sklearn'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "rows/s", - "version": "37452d8baf322bbcf65b17c133348dd250a173a2585fcb15bbc021bb561e6afa" - }, - "benchmark_outliers.BenchmarkOutliers.peakmem_detect_iqr": { - "code": "class BenchmarkOutliers:\n def peakmem_detect_iqr(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during IQR outlier detection.\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_outliers.BenchmarkOutliers.peakmem_detect_iqr", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "4396e9885b02a2e7c8bc7eccdc7b6da09fc8f80f902a0f112eb9fdf3e4c69c13" - }, - "benchmark_outliers.BenchmarkOutliers.peakmem_detect_zscore": { - "code": "class BenchmarkOutliers:\n def peakmem_detect_zscore(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during Z-score outlier detection.\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_zscore(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_outliers.BenchmarkOutliers.peakmem_detect_zscore", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "67f0e910566689b4334cbebca0323f19dbfc66cb7d1dd269a9114920490d5e0f" - }, - "benchmark_outliers.BenchmarkOutliers.time_detect_iqr": { - "code": "class BenchmarkOutliers:\n def time_detect_iqr(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect outliers using the Interquartile Range (IQR) method.\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_outliers.BenchmarkOutliers.time_detect_iqr", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "996e7904e786e9e4985a5b556c5b8a562118b0463da57ec7c21c289ae3eb15df", - "warmup_time": -1 - }, - "benchmark_outliers.BenchmarkOutliers.time_detect_zscore": { - "code": "class BenchmarkOutliers:\n def time_detect_zscore(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to detect outliers using Z-score (standard deviations).\"\"\"\n df = self._get_target_df(library)\n self.adapter.detect_outliers_zscore(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_outliers.BenchmarkOutliers.time_detect_zscore", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "2a16f91a3e9cbb6e3def523c26956be352e590dabf96624eff0e2d41ca94c255", - "warmup_time": -1 - }, - "benchmark_outliers.BenchmarkOutliers.track_outlier_count_iqr": { - "code": "class BenchmarkOutliers:\n def track_outlier_count_iqr(self, n_rows: int, library: str) -> int:\n \"\"\"Track the total number of outliers detected by IQR across all numeric columns.\"\"\"\n df = self._get_target_df(library)\n result = self.adapter.detect_outliers_iqr(df, self.numeric_cols)\n \n outlier_count = 0\n try:\n import pandas as pd\n if isinstance(result, pd.DataFrame):\n outlier_cols = [c for c in result.columns if c.endswith(\"_outlier\")]\n for col in outlier_cols:\n outlier_count += int(result[col].sum())\n except Exception:\n pass\n \n return outlier_count if outlier_count > 0 else -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs)\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_outliers.BenchmarkOutliers.track_outlier_count_iqr", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'feature_engine'", - "'sklearn'", - "'polars'" - ] - ], - "timeout": 300, - "type": "track", - "unit": "outliers", - "version": "a7ae00c6743aa0af26ea43a7fb3de6e90af3797db9fea27479fd67c49e9c22fd" - }, - "benchmark_pipeline.BenchmarkPipeline.peakmem_full_clean": { - "code": "class BenchmarkPipeline:\n def peakmem_full_clean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory used during the full cleaning pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.peakmem_full_clean", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "81108b98041660a33f2a45656fb6e5e69684b11e4a5080796bdb1ec0f81488d7" - }, - "benchmark_pipeline.BenchmarkPipeline.time_full_clean": { - "code": "class BenchmarkPipeline:\n def time_full_clean(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to run the complete data cleaning pipeline.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_pipeline.BenchmarkPipeline.time_full_clean", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "7f1f0bf1c404671cf2d850b0558cd09df3a4ad1910015af06f77e6391b341925", - "warmup_time": -1 - }, - "benchmark_pipeline.BenchmarkPipeline.track_output_cols": { - "code": "class BenchmarkPipeline:\n def track_output_cols(self, n_rows: int, library: str) -> int:\n \"\"\"Track the number of columns remaining after cleaning.\"\"\"\n df = fresh_copy(self.df)\n result = self.adapter.full_clean(df)\n try:\n import pandas as pd\n if isinstance(result, pd.DataFrame):\n return len(result.columns)\n return len(getattr(result, \"columns\", []))\n except Exception:\n return -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.track_output_cols", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "cols", - "version": "a92633b51ba1c8e8b6ee94cf0bd3497388b752031794f2489d2e91c2da78a781" - }, - "benchmark_pipeline.BenchmarkPipeline.track_output_rows": { - "code": "class BenchmarkPipeline:\n def track_output_rows(self, n_rows: int, library: str) -> int:\n \"\"\"Track the number of rows remaining after cleaning.\"\"\"\n df = fresh_copy(self.df)\n result = self.adapter.full_clean(df)\n try:\n return len(result)\n except Exception:\n return -1\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.track_output_rows", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows", - "version": "59e7aae9329927dc95d6b7c468ff17aefe31f8dae97ae4a8f26c5338d4c5c392" - }, - "benchmark_pipeline.BenchmarkPipeline.track_throughput": { - "code": "class BenchmarkPipeline:\n def track_throughput(self, n_rows: int, library: str) -> float:\n \"\"\"Calculate pipeline throughput in rows per second.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.full_clean(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_pipeline.BenchmarkPipeline.track_throughput", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'", - "'autoclean'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows/s", - "version": "92a13a48f747501a52bbb7b2ec75ec2ba7294747061a896777ebd190eaea0330" - }, - "benchmark_scaling.BenchmarkScaling.peakmem_minmax_scale": { - "code": "class BenchmarkScaling:\n def peakmem_minmax_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during MinMaxScaler.\"\"\"\n df = self._get_target_df(library)\n self.adapter.minmax_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_scaling.BenchmarkScaling.peakmem_minmax_scale", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "f60970d8c16570d05f06c985b45fb26ddf6fc0f02fc9827dc11d45e2c9708ee3" - }, - "benchmark_scaling.BenchmarkScaling.peakmem_standard_scale": { - "code": "class BenchmarkScaling:\n def peakmem_standard_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during StandardScaler.\"\"\"\n df = self._get_target_df(library)\n self.adapter.standard_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_scaling.BenchmarkScaling.peakmem_standard_scale", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "471e946b8c7f3b5fc77b22e062d59df81ea9efb9b1e8ebaba9931d3864dae3f5" - }, - "benchmark_scaling.BenchmarkScaling.time_minmax_scale": { - "code": "class BenchmarkScaling:\n def time_minmax_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to apply MinMaxScaler (0-1 normalization).\"\"\"\n df = self._get_target_df(library)\n self.adapter.minmax_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_scaling.BenchmarkScaling.time_minmax_scale", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e9acb509f1b0116332ed0a50b63a00fbf27c900ee02eee04ddd6181ee32ab20c", - "warmup_time": -1 - }, - "benchmark_scaling.BenchmarkScaling.time_standard_scale": { - "code": "class BenchmarkScaling:\n def time_standard_scale(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to apply StandardScaler (Z-score normalization).\"\"\"\n df = self._get_target_df(library)\n self.adapter.standard_scale(df, self.numeric_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.numeric_cols = get_numeric_columns(self.df)\n \n # Ensure clean numeric data for sklearn/feature_engine\n import pandas as pd\n self.df_numeric = pd.DataFrame()\n for col in self.numeric_cols:\n self.df_numeric[col] = pd.to_numeric(self.df[col], errors=\"coerce\")\n self.df_numeric = self.df_numeric.dropna().reset_index(drop=True)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_scaling.BenchmarkScaling.time_standard_scale", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'pandas'", - "'sklearn'", - "'feature_engine'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "36d85cddde3f92d9b9feb5852dd9dc1ffa3a616a7b5d657037e0f350e2ba84b4", - "warmup_time": -1 - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.peakmem_clean_scaling": { - "code": "class BenchmarkScalingCurves:\n def peakmem_clean_scaling(self, n_rows: int, library: str) -> None:\n \"\"\"Peak memory of the full pipeline to map out scaling curves.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.peakmem_clean_scaling", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "peakmemory", - "unit": "bytes", - "version": "416f495cb0534cf4534cc5b7124f57796aadcce6089f7d031caae87368a37523" - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.time_clean_scaling": { - "code": "class BenchmarkScalingCurves:\n def time_clean_scaling(self, n_rows: int, library: str) -> None:\n \"\"\"Time the full pipeline to map out scaling curves.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.full_clean(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "min_run_count": 2, - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.time_clean_scaling", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 600, - "type": "time", - "unit": "seconds", - "version": "17caf347aba692e8c28eb7a220f46ab2403cdda80f665a000d09cf57c38f822e", - "warmup_time": -1 - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.track_speedup_vs_pandas": { - "code": "class BenchmarkScalingCurves:\n def track_speedup_vs_pandas(self, n_rows: int, library: str) -> float:\n \"\"\"Speedup relative to Pandas for the full cleaning pipeline.\"\"\"\n import time as _time\n from benchmarks.utils.library_wrappers import get_adapter\n \n pd_adapter = get_adapter(\"pandas\")\n if not pd_adapter.available:\n return float(\"nan\")\n \n # Time pandas baseline\n df_pd = fresh_copy(self.df)\n start = _time.perf_counter()\n pd_adapter.full_clean(df_pd)\n pandas_time = _time.perf_counter() - start\n \n if pandas_time <= 0:\n return float(\"nan\")\n \n # If library IS pandas, speedup is exactly 1.0\n if library == \"pandas\":\n return 1.0\n \n # Time target library\n df_lib = fresh_copy(self.df)\n start = _time.perf_counter()\n self.adapter.full_clean(df_lib)\n library_time = _time.perf_counter() - start\n \n return pandas_time / library_time if library_time > 0 else float(\"nan\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.track_speedup_vs_pandas", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "ratio", - "version": "c543d6534ceca6f519d2fa1d4d1429634adef8c333ef8daef4495ec2082c817a" - }, - "benchmark_scaling_curves.BenchmarkScalingCurves.track_throughput": { - "code": "class BenchmarkScalingCurves:\n def track_throughput(self, n_rows: int, library: str) -> float:\n \"\"\"Rows per second at each dataset size.\"\"\"\n import time as _time\n df = fresh_copy(self.df)\n \n start = _time.perf_counter()\n self.adapter.full_clean(df)\n elapsed = _time.perf_counter() - start\n \n return n_rows / elapsed if elapsed > 0 else float(\"inf\")\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n gc_collect()", - "name": "benchmark_scaling_curves.BenchmarkScalingCurves.track_throughput", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000", - "5000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 600, - "type": "track", - "unit": "rows/s", - "version": "770225419fb4d828759ead0fe3b000c528bf0ff3d45b6810fb5cf90fe1a965ae" - }, - "benchmark_strings.BenchmarkStrings.peakmem_lowercase": { - "code": "class BenchmarkStrings:\n def peakmem_lowercase(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when converting to lowercase.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.to_lowercase(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_strings.BenchmarkStrings.peakmem_lowercase", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "41cc942e350b7f59e3f5711568d9069dd9f7b4954bc85631af8511b23672a5ed" - }, - "benchmark_strings.BenchmarkStrings.peakmem_trim_whitespace": { - "code": "class BenchmarkStrings:\n def peakmem_trim_whitespace(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory when trimming whitespace.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.trim_whitespace(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_strings.BenchmarkStrings.peakmem_trim_whitespace", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "c11376c97e0665271b8c411af2ee530dbceecc52b98a4c02399bf1be2d625bdb" - }, - "benchmark_strings.BenchmarkStrings.time_lowercase": { - "code": "class BenchmarkStrings:\n def time_lowercase(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to convert string columns to lowercase.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.to_lowercase(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_lowercase", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e1422cecb631554cb13468abb951d9b623f8f0aebbf64d22906f3fe5af15af3b", - "warmup_time": -1 - }, - "benchmark_strings.BenchmarkStrings.time_regex_replace": { - "code": "class BenchmarkStrings:\n def time_regex_replace(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform a regex replacement on string columns.\"\"\"\n df = fresh_copy(self.df)\n pattern = r'[^a-zA-Z0-9 ]'\n replacement = ''\n self.adapter.regex_replace(df, self.string_cols, pattern, replacement)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_regex_replace", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "3c22dd8000aaaf9f4cffe3a911e7847a52fb672d9cd8e8fdf7ec8f3a2454248f", - "warmup_time": -1 - }, - "benchmark_strings.BenchmarkStrings.time_trim_whitespace": { - "code": "class BenchmarkStrings:\n def time_trim_whitespace(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to trim whitespace from all string columns.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.trim_whitespace(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_trim_whitespace", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "668d40b2e253d03501067cd7c7da7335686200cd0e38d8c6b2bed0ca3fc9c9a2", - "warmup_time": -1 - }, - "benchmark_strings.BenchmarkStrings.time_uppercase": { - "code": "class BenchmarkStrings:\n def time_uppercase(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to convert string columns to uppercase.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.to_uppercase(df, self.string_cols)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n self.string_cols = get_string_columns(self.df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_strings.BenchmarkStrings.time_uppercase", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'", - "'pyjanitor'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "2e83fff4e7eaed8db55ba4f5fd5d12cedf09f6c9dcb852c0b3119906cadc137d", - "warmup_time": -1 - }, - "benchmark_types.BenchmarkTypes.peakmem_optimize_dtypes": { - "code": "class BenchmarkTypes:\n def peakmem_optimize_dtypes(self, n_rows: int, library: str) -> None:\n \"\"\"Measure peak memory during global type optimization.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.optimize_dtypes(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "name": "benchmark_types.BenchmarkTypes.peakmem_optimize_dtypes", - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "timeout": 300, - "type": "peakmemory", - "unit": "bytes", - "version": "de95624a9e886038f6aac8e52c70a5050a0731ee674d837c5c0d541246b05aad" - }, - "benchmark_types.BenchmarkTypes.time_convert_datetime": { - "code": "class BenchmarkTypes:\n def time_convert_datetime(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to parse datetime strings.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.convert_datetime(df, self.date_targets)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_types.BenchmarkTypes.time_convert_datetime", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "d6a0ac396a24e5d5b1cafc8ff4d1097d330c0dbcd4a6ae179ede206e1e764e61", - "warmup_time": -1 - }, - "benchmark_types.BenchmarkTypes.time_convert_numeric": { - "code": "class BenchmarkTypes:\n def time_convert_numeric(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to convert string columns to numeric types.\"\"\"\n df = fresh_copy(self.df)\n self.adapter.convert_numeric(df, self.numeric_targets)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_types.BenchmarkTypes.time_convert_numeric", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "0347b658e9da5a7eafbf4403928f8f2ad13331efede310a22dba0fca63596100", - "warmup_time": -1 - }, - "benchmark_types.BenchmarkTypes.time_optimize_dtypes": { - "code": "class BenchmarkTypes:\n def time_optimize_dtypes(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to perform global type optimization (downcasting).\"\"\"\n df = fresh_copy(self.df)\n self.adapter.optimize_dtypes(df)\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset and adapter for the benchmark.\"\"\"\n self.adapter = get_adapter(library)\n if not self.adapter.available:\n raise NotImplementedError(f\"{library} is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Define specific columns to target\n self.numeric_targets = [\"int_col_1\", \"int_col_2\", \"float_col_1\"]\n self.date_targets = [\"date_col\"]\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_types.BenchmarkTypes.time_optimize_dtypes", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'", - "'polars'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "a3932e05a6be05c68f9c6c8e2a3a98dd4c2f329bb0d5dc890777a2b525a25733", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_null_check": { - "code": "class BenchmarkValidation:\n def time_null_check(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to check for nulls across all columns.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n # FreshData profile includes null checks\n fd.profile(df)\n else:\n _ = df.isnull().any()\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_null_check", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "60b8361fdd17459219a658ea0fe62692b2a1d7d00a8ccbf50e19023e6ef61c95", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_profile_validation": { - "code": "class BenchmarkValidation:\n def time_profile_validation(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to run full data profiling/validation.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n fd.profile(df)\n else: # pandas manual profile\n import pandas as pd\n _ = df.dtypes\n _ = df.isnull().sum()\n _ = df.nunique()\n _ = df.describe(include='all')\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_profile_validation", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "a9286be5fb51a41517b1f12045a68c11b29e475dcc741e981627ae722109d5b6", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_range_check": { - "code": "class BenchmarkValidation:\n def time_range_check(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to check numeric ranges.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n fd.profile(df)\n else:\n import pandas as pd\n numeric_df = df.select_dtypes(include=['number'])\n if not numeric_df.empty:\n _ = numeric_df.min()\n _ = numeric_df.max()\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_range_check", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "e72e005674e34f0bff4fb322e32a5d1ae3d631632fe0f03b8aaad361b58d9cc7", - "warmup_time": -1 - }, - "benchmark_validation.BenchmarkValidation.time_type_check": { - "code": "class BenchmarkValidation:\n def time_type_check(self, n_rows: int, library: str) -> None:\n \"\"\"Measure time to validate data types.\"\"\"\n df = fresh_copy(self.df)\n if library == \"freshdata\":\n import freshdata as fd\n fd.profile(df)\n else:\n _ = df.dtypes\n\n def setup(self, n_rows: int, library: str) -> None:\n \"\"\"Prepare dataset for the benchmark.\"\"\"\n if library == \"freshdata\":\n try:\n import freshdata # noqa: F401\n except ImportError:\n raise NotImplementedError(\"freshdata is not available\")\n \n # Get base dataset\n base_df = get_dataset(n_rows)\n self.df = fresh_copy(base_df)\n \n # Clean up memory\n gc_collect()", - "min_run_count": 2, - "name": "benchmark_validation.BenchmarkValidation.time_type_check", - "number": 0, - "param_names": [ - "n_rows", - "library" - ], - "params": [ - [ - "10000", - "100000", - "1000000" - ], - [ - "'freshdata'", - "'pandas'" - ] - ], - "repeat": 0, - "rounds": 2, - "sample_time": 0.01, - "timeout": 300, - "type": "time", - "unit": "seconds", - "version": "ae470f6b26cf801ffc48a826cd4101eb8d2ef0da14d57c5cf885ff7fa062d621", - "warmup_time": -1 - }, - "version": 2 -} \ No newline at end of file diff --git a/freshdata-benchmarks/METHODOLOGY.md b/freshdata-benchmarks/METHODOLOGY.md deleted file mode 100644 index badc9fd1..00000000 --- a/freshdata-benchmarks/METHODOLOGY.md +++ /dev/null @@ -1,48 +0,0 @@ -# Benchmarking Methodology - -This document outlines the principles used to ensure fairness and reproducibility across the FreshData ASV benchmark suite. - -## 1. Fairness Guarantees - -Comparing libraries with entirely different architectures (e.g. Pandas eager execution vs. Polars lazy evaluation vs. Scikit-learn estimator arrays) requires strict fairness controls. - -### Identical Inputs (`fresh_copy`) -Every library operates on the exact same dataset. The dataset is generated once in the `setup()` block using a fixed random seed. -Crucially, before timing begins, the benchmark calls `df.copy(deep=True)`. This prevents one library's in-place mutations from affecting subsequent runs or giving an unfair cache advantage. - -### Isolation (`gc_collect`) -Garbage collection is explicitly triggered before each benchmark run to ensure residual objects from previous runs do not skew peak memory (`peakmem_`) measurements. - -### Equivalent Operations -The benchmark measures *semantic* equivalence. The adapters in `benchmarks/utils/library_wrappers.py` map operations. If a library natively supports an operation (e.g. `polars.drop_nulls()`), it is used. If it requires manual chaining (e.g. `pandas.isnull().any()`), the most idiomatic pandas chain is used. - -## 2. Statistical Approach - -- **ASV Calibration**: ASV automatically calibrates the number of loops per benchmark so that the total execution time is statistically significant (defaulting to ~10ms total). -- **Repetitions**: Each benchmark is repeated at least 3 times. ASV reports the median time to reject outliers caused by OS background tasks. -- **Warmup**: ASV performs un-timed warmup runs before measuring. - -## 3. Data Generation - -The `benchmarks.datasets.data_generator` module produces a deterministic dataset specifically designed to stress data cleaning libraries. - -It does not generate "clean" random data. It generates: -- Numerics with 2% extreme outliers and 15% missing values -- Strings contaminated with trailing/leading whitespaces and varied casing -- High-cardinality categoricals -- Formatted currencies, URLs, and emails containing intentional syntax errors -- Duplicated rows representing ~5% of the total dataset length - -## 4. Hardware and Threading - -Numeric libraries (NumPy, SciPy via sklearn, Polars) use varying numbers of threads by default, which can obscure algorithmic efficiency. - -We configure `asv.conf.json` to enforce single-threaded execution during benchmarking: -```json -"env": { - "OPENBLAS_NUM_THREADS": ["1"], - "MKL_NUM_THREADS": ["1"], - "OMP_NUM_THREADS": ["1"] -} -``` -Multi-threaded performance scales linearly with hardware and is generally uninteresting for comparing underlying algorithmic complexity. diff --git a/freshdata-benchmarks/README.md b/freshdata-benchmarks/README.md deleted file mode 100644 index cb16d092..00000000 --- a/freshdata-benchmarks/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# FreshData Benchmark Suite - -A comparative benchmark suite for `freshdata-cleaner` using -[Airspeed Velocity (ASV)](https://asv.readthedocs.io/). This suite is run -locally and its raw ASV results are committed under `.asv/results/`; it is -separate from the repository's CI `Benchmark` workflow, which runs the -[CleanBench harness](../benchmarks/) in `benchmarks/`. - -This suite measures the performance of FreshData's data cleaning operations against the Python data ecosystem: -- **Pandas** (baseline) -- **Polars** -- **pyjanitor** -- **Scikit-learn** -- **Feature-engine** -- **AutoClean** - -## Quick Start - -```bash -# 1. Create a virtual environment -python -m venv venv -source venv/bin/activate - -# 2. Install dependencies -pip install -r requirements.txt -pip install asv - -# 3. Configure ASV machine (run once) -asv machine --yes - -# 4. Run the benchmarks -asv run - -# 5. Generate reports -python scripts/generate_reports.py -python scripts/generate_plots.py - -# 6. View ASV web interface -asv publish -asv preview -``` - -## FreshData vs Pandas — Focused Comparison - -The headline benchmark compares **FreshData** and **Pandas** head-to-head across six core data operations at three dataset sizes (10K, 100K, 1M rows): - -| # | Operation | FreshData | Pandas | -|---|-----------|-----------|--------| -| 1 | **Loading** (CSV read + type inference) | `fd.clean(df, strategy="conservative")` | `pd.read_csv()` | -| 2 | **Missing values** (fill with mean) | `fd.clean(df, strategy="aggressive")` | `df.fillna(df.mean())` | -| 3 | **Outlier detection** (IQR flagging) | `fd.clean(df, outlier_method="iqr")` | Manual IQR per column | -| 4 | **Duplicate resolution** | `fd.clean(df, strategy="conservative")` | `df.drop_duplicates()` | -| 5 | **Group aggregations** | `fd.profile(df)` | `df.groupby().agg()` | -| 6 | **Full pipeline** | `fd.clean(df, strategy="balanced")` | Manual 7-step chain | - -### Run the focused comparison only - -```bash -# One-shot: installs deps, runs benchmarks, generates reports -bash bench/run_benchmarks.sh - -# Quick mode (fewer iterations, for fast validation) -bash bench/run_benchmarks.sh --quick -``` - -## Reproduce Locally - -To reproduce benchmark results exactly: - -```bash -# Clone and enter the repo -git clone https://github.com/FreshCode-Org/freshdata.git -cd freshdata/freshdata-benchmarks - -# Pin all randomness and threading for deterministic results -export PYTHONHASHSEED=42 -export OMP_NUM_THREADS=1 -export MKL_NUM_THREADS=1 -export OPENBLAS_NUM_THREADS=1 -export NUMEXPR_NUM_THREADS=1 -export VECLIB_MAXIMUM_THREADS=1 - -# Use the all-in-one runner (creates its own venv) -bash bench/run_benchmarks.sh - -# Or run manually with ASV -python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt asv virtualenv -asv machine --yes -asv run --bench "BenchmarkFreshDataVsPandas" -asv publish && asv preview -``` - -**Environment capture**: The runner script automatically saves machine details (Python version, OS, CPU, RAM, package versions) to `bench/artifacts/env_info.json`. You can also generate this manually: - -```bash -python scripts/capture_env.py -``` - -## Interpreting Results - -| Metric prefix | What it measures | Unit | -|---------------|------------------|------| -| `time_*` | Wall-clock execution time | seconds | -| `peakmem_*` | Peak resident set size (RSS) | bytes | -| `track_throughput_*` | Rows processed per second | rows/s | -| `track_speedup_*` | Speedup ratio vs Pandas baseline | ratio (×) | - -- **Lower is better** for `time_*` and `peakmem_*`. -- **Higher is better** for `track_throughput_*` and `track_speedup_*`. -- ASV calibrates loop counts automatically and reports the **median** of ≥3 repeats. -- Results include warmup iterations (untimed) before each measurement. -- Standard deviations are captured to identify noisy measurements. - -## Benchmark Matrix - -The suite covers 15 domains using synthetic datasets (10K to 10M rows) with realistic data quality issues (NaNs, extreme outliers, sentinels, unicode anomalies, whitespaces): - -1. **Pipeline** (`benchmark_pipeline.py`) - Full auto-cleaning pipeline -2. **Missing Values** (`benchmark_missing.py`) - Drop, mean, median, ffill -3. **Duplicates** (`benchmark_duplicates.py`) - Detect and drop -4. **Strings** (`benchmark_strings.py`) - Trim, lowercase, regex -5. **Types** (`benchmark_types.py`) - Downcasting, numeric parsing -6. **Columns** (`benchmark_columns.py`) - Renames and selections -7. **Encoding** (`benchmark_encoding.py`) - One-hot and ordinal -8. **Outliers** (`benchmark_outliers.py`) - IQR and Z-score -9. **Scaling** (`benchmark_scaling.py`) - Standard and MinMax -10. **Validation** (`benchmark_validation.py`) - Schema profiling -11. **Memory** (`benchmark_memory.py`) - Peak RSS tracking -12. **I/O** (`benchmark_io.py`) - Combined CSV/Parquet read + clean -13. **Scaling** (`benchmark_scaling_curves.py`) - Big-O efficiency analysis -14. **Group Aggregations** (`benchmark_groupagg.py`) - Groupby/profile comparisons -15. **FreshData vs Pandas** (`benchmark_freshdata_vs_pandas.py`) - Focused 6-operation head-to-head - -## Published Results - -This ASV suite is not wired into CI: run it locally with the commands above -and browse the results with `asv publish && asv preview`. Raw per-machine -results live in `.asv/results/` so runs are comparable over time. The -repository's CI `Benchmark` workflow runs the separate CleanBench harness -(`benchmarks/` at the repo root) on every push to `main` and weekly. - -## Methodology - -See [METHODOLOGY.md](METHODOLOGY.md) for details on fairness enforcement, cache eviction, deep copies, and dataset generation. - diff --git a/freshdata-benchmarks/asv.conf.json b/freshdata-benchmarks/asv.conf.json deleted file mode 100644 index b7accedb..00000000 --- a/freshdata-benchmarks/asv.conf.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "version": 1, - "project": "freshdata-benchmarks", - "project_url": "https://github.com/FreshCode-Org/freshdata", - "repo": ".", - "dvcs": "git", - "branches": ["HEAD"], - - "benchmark_dir": "benchmarks", - "results_dir": ".asv/results", - "html_dir": ".asv/html", - - "environment_type": "virtualenv", - "pythons": [""], - - "install_timeout": 600.0, - "install_command": [ - "in-dir={env_dir} python -m pip install -r {build_dir}/requirements.txt" - ], - "build_command": [], - "uninstall_command": [], - - "matrix": { - "env": { - "OPENBLAS_NUM_THREADS": ["1"], - "MKL_NUM_THREADS": ["1"], - "OMP_NUM_THREADS": ["1"] - } - }, - - "benchmark_timeout": 600, - "default_benchmark_timeout": 600 -} diff --git a/freshdata-benchmarks/bench/artifacts/benchmark_output.txt b/freshdata-benchmarks/bench/artifacts/benchmark_output.txt deleted file mode 100644 index ab6b2e3c..00000000 --- a/freshdata-benchmarks/bench/artifacts/benchmark_output.txt +++ /dev/null @@ -1,39 +0,0 @@ -· Cloning project -· Error running /usr/bin/git clone --mirror . /Users/wilson/freshdata/freshdata-benchmarks/freshdata-benchmarks (exit status 128) - STDOUT --------> - - STDERR --------> - fatal: repository '.' does not exist - -Traceback (most recent call last): - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/bin/asv", line 6, in - sys.exit(main()) - ~~~~^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/main.py", line 29, in main - result = args.func(args) - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/commands/__init__.py", line 49, in run_from_args - return cls.run_from_conf_args(conf, args) - ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/commands/run.py", line 218, in run_from_conf_args - return cls.run( - ~~~~~~~^ - conf=conf, - ^^^^^^^^^^ - ...<22 lines>... - **kwargs, - ^^^^^^^^^ - ) - ^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/commands/run.py", line 301, in run - repo = get_repo(conf) - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/repo.py", line 305, in get_repo - return cls(conf.repo, conf.project) - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/plugins/git.py", line 36, in __init__ - self._run_git(['clone', '--mirror', url, self._path], cwd=None) - ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/plugins/git.py", line 70, in _run_git - return util.check_output([self._git] + args, env=env, **kwargs) - ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/wilson/freshdata/freshdata-benchmarks/.venv-bench/lib/python3.13/site-packages/asv/util.py", line 641, in check_output - raise ProcessError(args, retcode, stdout, stderr) -asv.util.ProcessError: Command '/usr/bin/git clone --mirror . /Users/wilson/freshdata/freshdata-benchmarks/freshdata-benchmarks' returned non-zero exit status 128 diff --git a/freshdata-benchmarks/bench/artifacts/env_info.json b/freshdata-benchmarks/bench/artifacts/env_info.json deleted file mode 100644 index 415d2ef5..00000000 --- a/freshdata-benchmarks/bench/artifacts/env_info.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "timestamp": "2026-07-05T14:04:21.763872+00:00", - "hostname": "Johnnys-MacBook-Air.local", - "python": { - "version": "3.13.9", - "implementation": "CPython", - "compiler": "Clang 20.1.8 " - }, - "os": { - "system": "Darwin", - "release": "24.5.0", - "version": "Darwin Kernel Version 24.5.0: Tue Apr 22 19:54:26 PDT 2025; root:xnu-11417.121.6~2/RELEASE_ARM64_T8112", - "machine": "arm64" - }, - "cpu": { - "model": "arm", - "count_logical": 8, - "count_physical": 8, - "freq_mhz": 3504 - }, - "ram_gb": 16.0, - "packages": { - "freshdata-cleaner": "1.0.1", - "pandas": "3.0.3", - "numpy": "2.5.1", - "pyarrow": "24.0.0", - "asv": "0.6.6", - "psutil": "7.2.2" - } -} diff --git a/freshdata-benchmarks/bench/run_benchmarks.sh b/freshdata-benchmarks/bench/run_benchmarks.sh deleted file mode 100755 index 61f0f4ac..00000000 --- a/freshdata-benchmarks/bench/run_benchmarks.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash -# ────────────────────────────────────────────────────────────────── -# bench/run_benchmarks.sh -# -# One-shot script that installs dependencies in an isolated -# virtualenv, runs the focused freshdata-vs-pandas ASV benchmarks, -# generates the HTML report and a JSON/CSV summary, and collects -# all artifacts under bench/artifacts/. -# -# Usage: -# bash bench/run_benchmarks.sh # full run -# bash bench/run_benchmarks.sh --quick # quick (dry-run) mode -# -# Requirements: -# - Python 3.9+ available as `python3` -# - git (required by ASV) -# ────────────────────────────────────────────────────────────────── -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -VENV_DIR="${PROJECT_ROOT}/.venv-bench" -ARTIFACTS_DIR="${PROJECT_ROOT}/bench/artifacts" -QUICK_MODE=false - -# Parse arguments -for arg in "$@"; do - case "$arg" in - --quick) QUICK_MODE=true ;; - *) echo "Unknown argument: $arg"; exit 1 ;; - esac -done - -echo "══════════════════════════════════════════════════════════════" -echo " FreshData vs Pandas — ASV Benchmark Runner" -echo "══════════════════════════════════════════════════════════════" -echo "" -echo " Project root : ${PROJECT_ROOT}" -echo " Quick mode : ${QUICK_MODE}" -echo " Timestamp : $(date -u +%Y-%m-%dT%H:%M:%SZ)" -echo "" - -# ── 1. Create / activate isolated virtualenv ───────────────────── -echo "▸ Step 1/7: Setting up isolated virtualenv …" -if [ ! -d "${VENV_DIR}" ]; then - python3 -m venv "${VENV_DIR}" - echo " Created ${VENV_DIR}" -fi -source "${VENV_DIR}/bin/activate" -pip install --upgrade pip --quiet - -# ── 2. Install dependencies ───────────────────────────────────── -echo "▸ Step 2/7: Installing dependencies …" -pip install -r "${PROJECT_ROOT}/requirements.txt" --quiet -pip install asv virtualenv --quiet -echo " Done." - -# ── 3. Capture environment details ────────────────────────────── -echo "▸ Step 3/7: Capturing environment …" -mkdir -p "${ARTIFACTS_DIR}" -python "${PROJECT_ROOT}/scripts/capture_env.py" "${ARTIFACTS_DIR}/env_info.json" - -# ── 4. Configure ASV machine ──────────────────────────────────── -echo "▸ Step 4/7: Configuring ASV machine …" -cd "${PROJECT_ROOT}" -asv machine --yes 2>/dev/null || true - -# ── 5. Pin reproducibility settings ───────────────────────────── -export OPENBLAS_NUM_THREADS=1 -export MKL_NUM_THREADS=1 -export OMP_NUM_THREADS=1 -export NUMEXPR_NUM_THREADS=1 -export VECLIB_MAXIMUM_THREADS=1 -export PYTHONHASHSEED=42 - -# ── 6. Run benchmarks ─────────────────────────────────────────── -echo "▸ Step 5/7: Running benchmarks …" -if [ "${QUICK_MODE}" = true ]; then - asv run --quick \ - --bench "BenchmarkFreshDataVsPandas|BenchmarkGroupAgg" \ - -e 2>&1 | tee "${ARTIFACTS_DIR}/benchmark_output.txt" -else - asv run \ - --skip-existing-successful \ - --bench "BenchmarkFreshDataVsPandas|BenchmarkGroupAgg" \ - -e 2>&1 | tee "${ARTIFACTS_DIR}/benchmark_output.txt" -fi - -# ── 7. Generate reports & HTML ─────────────────────────────────── -echo "▸ Step 6/7: Generating reports …" -python "${PROJECT_ROOT}/scripts/generate_reports.py" || true -python "${PROJECT_ROOT}/scripts/generate_plots.py" || true -asv publish || true - -# Copy HTML report to artifacts -if [ -d "${PROJECT_ROOT}/.asv/html" ]; then - cp -r "${PROJECT_ROOT}/.asv/html" "${ARTIFACTS_DIR}/html_report" - echo " HTML report → ${ARTIFACTS_DIR}/html_report/" -fi - -# Copy JSON results to artifacts -if [ -d "${PROJECT_ROOT}/.asv/results" ]; then - cp -r "${PROJECT_ROOT}/.asv/results" "${ARTIFACTS_DIR}/asv_results" - echo " ASV results → ${ARTIFACTS_DIR}/asv_results/" -fi - -# Copy generated reports -if [ -d "${PROJECT_ROOT}/reports" ]; then - cp -r "${PROJECT_ROOT}/reports" "${ARTIFACTS_DIR}/reports" - echo " Reports → ${ARTIFACTS_DIR}/reports/" -fi - -echo "" -echo "▸ Step 7/7: Summary" -echo "══════════════════════════════════════════════════════════════" -echo " Artifacts directory : ${ARTIFACTS_DIR}" -echo " Environment info : ${ARTIFACTS_DIR}/env_info.json" -echo " Benchmark log : ${ARTIFACTS_DIR}/benchmark_output.txt" -echo "" -echo " To view the HTML report locally:" -echo " cd ${PROJECT_ROOT} && asv preview" -echo "" -echo " Exact command to reproduce:" -echo " PYTHONHASHSEED=42 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \\" -echo " OPENBLAS_NUM_THREADS=1 bash bench/run_benchmarks.sh" -echo "" -echo "══════════════════════════════════════════════════════════════" -echo " ✓ Benchmark run complete." -echo "══════════════════════════════════════════════════════════════" diff --git a/freshdata-benchmarks/benchmarks/__init__.py b/freshdata-benchmarks/benchmarks/__init__.py deleted file mode 100644 index 082b019c..00000000 --- a/freshdata-benchmarks/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""FreshData ASV Benchmark Suite.""" diff --git a/freshdata-benchmarks/benchmarks/benchmark_columns.py b/freshdata-benchmarks/benchmarks/benchmark_columns.py deleted file mode 100644 index 1900e85d..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_columns.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Benchmark column-level operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkColumns: - """Benchmark suite for column renaming, dropping, and selection.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Define columns for operations - cols = list(self.df.columns) - self.rename_mapping = {c: f"renamed_{c}" for c in cols[:5]} - self.drop_cols = cols[-3:] - self.select_cols = cols[:5] - - # Clean up memory - gc_collect() - - def time_rename_columns(self, n_rows: int, library: str) -> None: - """Measure time to rename columns.""" - df = fresh_copy(self.df) - self.adapter.rename_columns(df, self.rename_mapping) - - def time_drop_columns(self, n_rows: int, library: str) -> None: - """Measure time to drop specific columns.""" - df = fresh_copy(self.df) - self.adapter.drop_columns(df, self.drop_cols) - - def time_select_columns(self, n_rows: int, library: str) -> None: - """Measure time to select a subset of columns.""" - df = fresh_copy(self.df) - self.adapter.select_columns(df, self.select_cols) diff --git a/freshdata-benchmarks/benchmarks/benchmark_duplicates.py b/freshdata-benchmarks/benchmarks/benchmark_duplicates.py deleted file mode 100644 index 5b05f947..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_duplicates.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Benchmark duplicate row operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkDuplicates: - """Benchmark suite for duplicate detection and removal.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def time_detect_duplicates(self, n_rows: int, library: str) -> None: - """Measure time to detect duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.detect_duplicates(df) - - def peakmem_detect_duplicates(self, n_rows: int, library: str) -> None: - """Measure peak memory when detecting duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.detect_duplicates(df) - - def time_drop_duplicates(self, n_rows: int, library: str) -> None: - """Measure time to drop duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - def peakmem_drop_duplicates(self, n_rows: int, library: str) -> None: - """Measure peak memory when dropping duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - def track_duplicate_count(self, n_rows: int, library: str) -> int: - """Track the number of duplicate rows dropped.""" - df = fresh_copy(self.df) - start_rows = len(df) - result = self.adapter.drop_duplicates(df) - - # Approximate counting since libraries return different formats - try: - return start_rows - len(result) - except Exception: - return -1 - - track_duplicate_count.unit = "rows" diff --git a/freshdata-benchmarks/benchmarks/benchmark_encoding.py b/freshdata-benchmarks/benchmarks/benchmark_encoding.py deleted file mode 100644 index 213c23b3..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_encoding.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Benchmark categorical encoding operations across libraries. - -Note: FreshData does not provide standalone encoding features in its core API, -so this benchmark compares the broader ecosystem (pandas, sklearn, feature_engine). -""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_categorical_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkEncoding: - """Benchmark suite for one-hot and label encoding.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["pandas", "sklearn", "feature_engine", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.cat_cols = get_categorical_columns(self.df) - - # Clean up memory - gc_collect() - - def time_onehot_encode(self, n_rows: int, library: str) -> None: - """Measure time to perform one-hot encoding on categorical columns.""" - df = fresh_copy(self.df) - self.adapter.onehot_encode(df, self.cat_cols) - - def peakmem_onehot_encode(self, n_rows: int, library: str) -> None: - """Measure peak memory during one-hot encoding.""" - df = fresh_copy(self.df) - self.adapter.onehot_encode(df, self.cat_cols) - - def time_label_encode(self, n_rows: int, library: str) -> None: - """Measure time to perform label (ordinal) encoding.""" - df = fresh_copy(self.df) - self.adapter.label_encode(df, self.cat_cols) - - def peakmem_label_encode(self, n_rows: int, library: str) -> None: - """Measure peak memory during label encoding.""" - df = fresh_copy(self.df) - self.adapter.label_encode(df, self.cat_cols) diff --git a/freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py b/freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py deleted file mode 100644 index 98b1efd6..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_freshdata_vs_pandas.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Focused freshdata-vs-pandas comparison across six core data operations. - -This is the headline benchmark for CI: it covers loading, missing-value -handling, outlier detection/filtering, duplicate resolution, group -aggregations, and a representative preprocessing pipeline — all measured -at three configurable dataset sizes. -""" - -from __future__ import annotations - -import os -import shutil -import tempfile -import time as _time - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkFreshDataVsPandas: - """Headline benchmark comparing FreshData and Pandas across six operations.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset, adapter, and temp files for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Write a CSV for loading benchmarks - self._tmpdir = tempfile.mkdtemp(prefix="asv_fdvspd_") - self.csv_path = os.path.join(self._tmpdir, f"test_{n_rows}.csv") - self.df.to_csv(self.csv_path, index=False) - - gc_collect() - - def teardown(self, n_rows: int, library: str) -> None: - """Remove temporary files.""" - shutil.rmtree(self._tmpdir, ignore_errors=True) - - # ── 1. Loading ───────────────────────────────────────────── - - def time_load_csv(self, n_rows: int, library: str) -> None: - """Measure time to load a CSV file and infer types.""" - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_csv(self.csv_path) - fd.clean(df, strategy="conservative", preserve_original=True) - else: - import pandas as pd - pd.read_csv(self.csv_path) - - def peakmem_load_csv(self, n_rows: int, library: str) -> None: - """Measure peak memory when loading a CSV file.""" - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_csv(self.csv_path) - fd.clean(df, strategy="conservative", preserve_original=True) - else: - import pandas as pd - pd.read_csv(self.csv_path) - - # ── 2. Missing Value Handling ────────────────────────────── - - def time_handle_missing(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with column means.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - def peakmem_handle_missing(self, n_rows: int, library: str) -> None: - """Measure peak memory when filling missing values.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - # ── 3. Outlier Detection / Filtering ─────────────────────── - - def time_detect_outliers(self, n_rows: int, library: str) -> None: - """Measure time to detect outliers using IQR method.""" - df = fresh_copy(self.df) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - def peakmem_detect_outliers(self, n_rows: int, library: str) -> None: - """Measure peak memory during IQR outlier detection.""" - df = fresh_copy(self.df) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - # ── 4. Duplicate Resolution ──────────────────────────────── - - def time_resolve_duplicates(self, n_rows: int, library: str) -> None: - """Measure time to detect and drop duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - def peakmem_resolve_duplicates(self, n_rows: int, library: str) -> None: - """Measure peak memory when dropping duplicate rows.""" - df = fresh_copy(self.df) - self.adapter.drop_duplicates(df) - - # ── 5. Group Aggregations ────────────────────────────────── - - def time_group_aggregations(self, n_rows: int, library: str) -> None: - """Measure time for multi-column group aggregations.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - def peakmem_group_aggregations(self, n_rows: int, library: str) -> None: - """Measure peak memory for group aggregations.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - # ── 6. Full Preprocessing Pipeline ───────────────────────── - - def time_full_pipeline(self, n_rows: int, library: str) -> None: - """Measure time for the complete preprocessing pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def peakmem_full_pipeline(self, n_rows: int, library: str) -> None: - """Measure peak memory for the complete preprocessing pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - # ── Throughput Tracker ───────────────────────────────────── - - def track_pipeline_throughput(self, n_rows: int, library: str) -> float: - """Calculate rows processed per second for the full pipeline.""" - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.full_clean(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_pipeline_throughput.unit = "rows/s" diff --git a/freshdata-benchmarks/benchmarks/benchmark_groupagg.py b/freshdata-benchmarks/benchmarks/benchmark_groupagg.py deleted file mode 100644 index c9e7b814..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_groupagg.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Benchmark group aggregation operations comparing freshdata vs pandas.""" - -from __future__ import annotations - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkGroupAgg: - """Benchmark suite for group aggregation operations.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - gc_collect() - - def time_group_agg_single(self, n_rows: int, library: str) -> None: - """Measure time for single-column groupby with mean aggregation.""" - df = fresh_copy(self.df) - self.adapter.group_agg_single(df) - - def time_group_agg_multi(self, n_rows: int, library: str) -> None: - """Measure time for multi-column groupby with multiple agg functions.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - def time_group_agg_transform(self, n_rows: int, library: str) -> None: - """Measure time for groupby transform (centering within groups).""" - df = fresh_copy(self.df) - self.adapter.group_agg_transform(df, self.numeric_cols) - - def peakmem_group_agg_multi(self, n_rows: int, library: str) -> None: - """Measure peak memory for multi-column groupby aggregation.""" - df = fresh_copy(self.df) - self.adapter.group_agg_multi(df, self.numeric_cols) - - def track_throughput_group_agg(self, n_rows: int, library: str) -> float: - """Calculate rows processed per second for group aggregation.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.group_agg_multi(df, self.numeric_cols) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput_group_agg.unit = "rows/s" diff --git a/freshdata-benchmarks/benchmarks/benchmark_io.py b/freshdata-benchmarks/benchmarks/benchmark_io.py deleted file mode 100644 index 15ef86de..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_io.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Benchmark file I/O operations paired with data cleaning.""" - -from __future__ import annotations - -import gc -import os -import shutil -import tempfile - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import gc_collect - - -class BenchmarkIO: - """Benchmark suite for reading and cleaning from disk formats.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup_cache(self): - """Generate test files once per class to avoid disk overhead in timing.""" - import pandas as pd - tmpdir = tempfile.mkdtemp() - - # We only generate files for the requested sizes in params - sizes = [10_000, 100_000, 1_000_000] - files = {} - - for size in sizes: - df = get_dataset(size) - csv_path = os.path.join(tmpdir, f"test_{size}.csv") - parquet_path = os.path.join(tmpdir, f"test_{size}.parquet") - - df.to_csv(csv_path, index=False) - try: - df.to_parquet(parquet_path, engine="pyarrow") - files[size] = {"csv": csv_path, "parquet": parquet_path} - except Exception: - files[size] = {"csv": csv_path, "parquet": None} - - return {"tmpdir": tmpdir, "files": files} - - def setup(self, cache, n_rows: int, library: str) -> None: - """Check library availability and locate files.""" - if library == "freshdata": - try: - import freshdata # noqa: F401 - except ImportError: - raise NotImplementedError("freshdata is not available") - elif library == "pandas": - try: - import pandas # noqa: F401 - except ImportError: - raise NotImplementedError("pandas is not available") - elif library == "polars": - try: - import polars # noqa: F401 - except ImportError: - raise NotImplementedError("polars is not available") - - self.csv_path = cache["files"][n_rows]["csv"] - self.parquet_path = cache["files"][n_rows]["parquet"] - - if not self.csv_path: - raise NotImplementedError("CSV file not generated") - - gc_collect() - - def teardown(self, cache, n_rows: int, library: str) -> None: - """Class-level teardown handles directory removal, so this is a no-op.""" - pass - - def time_csv_read_clean(self, cache, n_rows: int, library: str) -> None: - """Measure time to read a CSV and run a full clean.""" - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_csv(self.csv_path) - fd.clean(df, strategy="balanced", preserve_original=True) - elif library == "pandas": - import pandas as pd - df = pd.read_csv(self.csv_path) - df = df.dropna().drop_duplicates() - elif library == "polars": - import polars as pl - df = pl.read_csv(self.csv_path) - df = df.drop_nulls().unique() - - def time_parquet_read_clean(self, cache, n_rows: int, library: str) -> None: - """Measure time to read a Parquet file and run a full clean.""" - if not self.parquet_path: - raise NotImplementedError("Parquet missing") - - if library == "freshdata": - import freshdata as fd - import pandas as pd - df = pd.read_parquet(self.parquet_path) - fd.clean(df, strategy="balanced", preserve_original=True) - elif library == "pandas": - import pandas as pd - df = pd.read_parquet(self.parquet_path) - df = df.dropna().drop_duplicates() - elif library == "polars": - import polars as pl - df = pl.read_parquet(self.parquet_path) - df = df.drop_nulls().unique() - - def peakmem_csv_read_clean(self, cache, n_rows: int, library: str) -> None: - """Measure peak memory when reading and cleaning a CSV.""" - self.time_csv_read_clean(cache, n_rows, library) - - -def teardown_module(): - """Module-level teardown to remove temp directories.""" - # ASV cache mechanism is slightly opaque, best effort cleanup - pass diff --git a/freshdata-benchmarks/benchmarks/benchmark_memory.py b/freshdata-benchmarks/benchmarks/benchmark_memory.py deleted file mode 100644 index eba2519c..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_memory.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Memory-focused benchmarks and profiling.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkMemory: - """Benchmark suite dedicated to memory usage analysis.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000, 5_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def peakmem_full_clean(self, n_rows: int, library: str) -> None: - """Measure peak RSS memory used during the full cleaning pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def track_memory_before(self, n_rows: int, library: str) -> float: - """Track the deep memory usage of the uncleaned input DataFrame (in MB).""" - df = fresh_copy(self.df) - import pandas as pd - if isinstance(df, pd.DataFrame): - return df.memory_usage(deep=True).sum() / (1024 * 1024) - return -1.0 - - track_memory_before.unit = "MB" - - def track_memory_after(self, n_rows: int, library: str) -> float: - """Track the deep memory usage of the cleaned output DataFrame (in MB).""" - df = fresh_copy(self.df) - result = self.adapter.full_clean(df) - try: - import pandas as pd - if isinstance(result, pd.DataFrame): - return result.memory_usage(deep=True).sum() / (1024 * 1024) - else: - # Approximate memory for Polars - return result.estimated_size() / (1024 * 1024) - except Exception: - return -1.0 - - track_memory_after.unit = "MB" - - def track_memory_ratio(self, n_rows: int, library: str) -> float: - """Track the ratio of output memory / input memory. - Values < 1.0 indicate memory savings (e.g. via downcasting). - """ - df = fresh_copy(self.df) - - import pandas as pd - if not isinstance(df, pd.DataFrame): - return float('nan') - - mem_before = df.memory_usage(deep=True).sum() - if mem_before == 0: - return float('nan') - - result = self.adapter.full_clean(df) - - try: - if isinstance(result, pd.DataFrame): - mem_after = result.memory_usage(deep=True).sum() - else: - mem_after = result.estimated_size() - return float(mem_after) / float(mem_before) - except Exception: - return float('nan') - - track_memory_ratio.unit = "ratio" diff --git a/freshdata-benchmarks/benchmarks/benchmark_missing.py b/freshdata-benchmarks/benchmarks/benchmark_missing.py deleted file mode 100644 index 1a832801..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_missing.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Benchmark missing value operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkMissing: - """Benchmark suite for missing value handling operations.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "feature_engine", "sklearn"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Clean up memory - gc_collect() - - def time_drop_missing(self, n_rows: int, library: str) -> None: - """Measure time to drop rows with missing values.""" - df = fresh_copy(self.df) - self.adapter.drop_missing_rows(df) - - def peakmem_drop_missing(self, n_rows: int, library: str) -> None: - """Measure peak memory when dropping rows with missing values.""" - df = fresh_copy(self.df) - self.adapter.drop_missing_rows(df) - - def time_fill_mean(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with the column mean.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - def peakmem_fill_mean(self, n_rows: int, library: str) -> None: - """Measure peak memory when filling missing values with column mean.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_mean(df, self.numeric_cols) - - def time_fill_median(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with the column median.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_median(df, self.numeric_cols) - - def peakmem_fill_median(self, n_rows: int, library: str) -> None: - """Measure peak memory when filling missing values with column median.""" - df = fresh_copy(self.df) - self.adapter.fill_missing_median(df, self.numeric_cols) - - def time_fill_mode(self, n_rows: int, library: str) -> None: - """Measure time to fill missing values with the column mode. - Supported by pandas, sklearn, and freshdata. - """ - df = fresh_copy(self.df) - self.adapter.fill_missing_mode(df, self.numeric_cols) - - def time_fill_ffill(self, n_rows: int, library: str) -> None: - """Measure time to forward-fill missing values. - Supported by pandas, polars, and freshdata. - """ - df = fresh_copy(self.df) - self.adapter.fill_missing_ffill(df) - - def time_fill_bfill(self, n_rows: int, library: str) -> None: - """Measure time to backward-fill missing values. - Supported by pandas, polars, and freshdata. - """ - df = fresh_copy(self.df) - self.adapter.fill_missing_bfill(df) - - def track_throughput_drop_missing(self, n_rows: int, library: str) -> float: - """Calculate rows processed per second for drop_missing.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.drop_missing_rows(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput_drop_missing.unit = "rows/s" diff --git a/freshdata-benchmarks/benchmarks/benchmark_outliers.py b/freshdata-benchmarks/benchmarks/benchmark_outliers.py deleted file mode 100644 index e7a0b697..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_outliers.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Benchmark outlier detection algorithms across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkOutliers: - """Benchmark suite for outlier detection techniques.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "feature_engine", "sklearn", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Scikit-learn and Feature Engine require clean, dense numeric data (no NaNs) - import pandas as pd - self.df_numeric = pd.DataFrame() - for col in self.numeric_cols: - self.df_numeric[col] = pd.to_numeric(self.df[col], errors="coerce") - self.df_numeric = self.df_numeric.dropna().reset_index(drop=True) - - # Clean up memory - gc_collect() - - def _get_target_df(self, library: str): - """Return the appropriate DataFrame depending on library requirements.""" - if library in ["sklearn", "feature_engine"]: - return fresh_copy(self.df_numeric) - return fresh_copy(self.df) - - def time_detect_iqr(self, n_rows: int, library: str) -> None: - """Measure time to detect outliers using the Interquartile Range (IQR) method.""" - df = self._get_target_df(library) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - def peakmem_detect_iqr(self, n_rows: int, library: str) -> None: - """Measure peak memory during IQR outlier detection.""" - df = self._get_target_df(library) - self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - def time_detect_zscore(self, n_rows: int, library: str) -> None: - """Measure time to detect outliers using Z-score (standard deviations).""" - df = self._get_target_df(library) - self.adapter.detect_outliers_zscore(df, self.numeric_cols) - - def peakmem_detect_zscore(self, n_rows: int, library: str) -> None: - """Measure peak memory during Z-score outlier detection.""" - df = self._get_target_df(library) - self.adapter.detect_outliers_zscore(df, self.numeric_cols) - - def track_outlier_count_iqr(self, n_rows: int, library: str) -> int: - """Track the total number of outliers detected by IQR across all numeric columns.""" - df = self._get_target_df(library) - result = self.adapter.detect_outliers_iqr(df, self.numeric_cols) - - outlier_count = 0 - try: - import pandas as pd - if isinstance(result, pd.DataFrame): - outlier_cols = [c for c in result.columns if c.endswith("_outlier")] - for col in outlier_cols: - outlier_count += int(result[col].sum()) - except Exception: - pass - - return outlier_count if outlier_count > 0 else -1 - - track_outlier_count_iqr.unit = "outliers" diff --git a/freshdata-benchmarks/benchmarks/benchmark_pipeline.py b/freshdata-benchmarks/benchmarks/benchmark_pipeline.py deleted file mode 100644 index 33c51e8b..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_pipeline.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Benchmark the complete end-to-end cleaning pipeline. - -This is the headline benchmark for the FreshData suite, comparing the full -auto-clean pipeline against equivalent manual operations in other libraries. -""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkPipeline: - """Benchmark suite for full dataset cleaning pipelines.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000, 5_000_000], - ["freshdata", "pandas", "polars", "pyjanitor", "autoclean"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def time_full_clean(self, n_rows: int, library: str) -> None: - """Measure time to run the complete data cleaning pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def peakmem_full_clean(self, n_rows: int, library: str) -> None: - """Measure peak memory used during the full cleaning pipeline.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def track_throughput(self, n_rows: int, library: str) -> float: - """Calculate pipeline throughput in rows per second.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.full_clean(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput.unit = "rows/s" - - def track_output_rows(self, n_rows: int, library: str) -> int: - """Track the number of rows remaining after cleaning.""" - df = fresh_copy(self.df) - result = self.adapter.full_clean(df) - try: - return len(result) - except Exception: - return -1 - - track_output_rows.unit = "rows" - - def track_output_cols(self, n_rows: int, library: str) -> int: - """Track the number of columns remaining after cleaning.""" - df = fresh_copy(self.df) - result = self.adapter.full_clean(df) - try: - import pandas as pd - if isinstance(result, pd.DataFrame): - return len(result.columns) - return len(getattr(result, "columns", [])) - except Exception: - return -1 - - track_output_cols.unit = "cols" diff --git a/freshdata-benchmarks/benchmarks/benchmark_scaling.py b/freshdata-benchmarks/benchmarks/benchmark_scaling.py deleted file mode 100644 index 8ee35c2b..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_scaling.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Benchmark feature scaling operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_numeric_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkScaling: - """Benchmark suite for feature scaling (Standardization, MinMax).""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["pandas", "sklearn", "feature_engine", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.numeric_cols = get_numeric_columns(self.df) - - # Ensure clean numeric data for sklearn/feature_engine - import pandas as pd - self.df_numeric = pd.DataFrame() - for col in self.numeric_cols: - self.df_numeric[col] = pd.to_numeric(self.df[col], errors="coerce") - self.df_numeric = self.df_numeric.dropna().reset_index(drop=True) - - # Clean up memory - gc_collect() - - def _get_target_df(self, library: str): - """Return the appropriate DataFrame depending on library requirements.""" - if library in ["sklearn", "feature_engine"]: - return fresh_copy(self.df_numeric) - return fresh_copy(self.df) - - def time_standard_scale(self, n_rows: int, library: str) -> None: - """Measure time to apply StandardScaler (Z-score normalization).""" - df = self._get_target_df(library) - self.adapter.standard_scale(df, self.numeric_cols) - - def peakmem_standard_scale(self, n_rows: int, library: str) -> None: - """Measure peak memory during StandardScaler.""" - df = self._get_target_df(library) - self.adapter.standard_scale(df, self.numeric_cols) - - def time_minmax_scale(self, n_rows: int, library: str) -> None: - """Measure time to apply MinMaxScaler (0-1 normalization).""" - df = self._get_target_df(library) - self.adapter.minmax_scale(df, self.numeric_cols) - - def peakmem_minmax_scale(self, n_rows: int, library: str) -> None: - """Measure peak memory during MinMaxScaler.""" - df = self._get_target_df(library) - self.adapter.minmax_scale(df, self.numeric_cols) diff --git a/freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py b/freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py deleted file mode 100644 index c4a00e69..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_scaling_curves.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Analysis of scaling efficiency across dataset sizes.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkScalingCurves: - """Benchmark suite to map out Big-O scaling complexity.""" - - timeout = 600 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000, 5_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - gc_collect() - - def time_clean_scaling(self, n_rows: int, library: str) -> None: - """Time the full pipeline to map out scaling curves.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def peakmem_clean_scaling(self, n_rows: int, library: str) -> None: - """Peak memory of the full pipeline to map out scaling curves.""" - df = fresh_copy(self.df) - self.adapter.full_clean(df) - - def track_throughput(self, n_rows: int, library: str) -> float: - """Rows per second at each dataset size.""" - import time as _time - df = fresh_copy(self.df) - - start = _time.perf_counter() - self.adapter.full_clean(df) - elapsed = _time.perf_counter() - start - - return n_rows / elapsed if elapsed > 0 else float("inf") - - track_throughput.unit = "rows/s" - - def track_speedup_vs_pandas(self, n_rows: int, library: str) -> float: - """Speedup relative to Pandas for the full cleaning pipeline.""" - import time as _time - from benchmarks.utils.library_wrappers import get_adapter - - pd_adapter = get_adapter("pandas") - if not pd_adapter.available: - return float("nan") - - # Time pandas baseline - df_pd = fresh_copy(self.df) - start = _time.perf_counter() - pd_adapter.full_clean(df_pd) - pandas_time = _time.perf_counter() - start - - if pandas_time <= 0: - return float("nan") - - # If library IS pandas, speedup is exactly 1.0 - if library == "pandas": - return 1.0 - - # Time target library - df_lib = fresh_copy(self.df) - start = _time.perf_counter() - self.adapter.full_clean(df_lib) - library_time = _time.perf_counter() - start - - return pandas_time / library_time if library_time > 0 else float("nan") - - track_speedup_vs_pandas.unit = "ratio" diff --git a/freshdata-benchmarks/benchmarks/benchmark_strings.py b/freshdata-benchmarks/benchmarks/benchmark_strings.py deleted file mode 100644 index ce6740d3..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_strings.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Benchmark string cleaning operations across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect, get_string_columns -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkStrings: - """Benchmark suite for string manipulation and cleaning.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars", "pyjanitor"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - self.string_cols = get_string_columns(self.df) - - # Clean up memory - gc_collect() - - def time_trim_whitespace(self, n_rows: int, library: str) -> None: - """Measure time to trim whitespace from all string columns.""" - df = fresh_copy(self.df) - self.adapter.trim_whitespace(df, self.string_cols) - - def peakmem_trim_whitespace(self, n_rows: int, library: str) -> None: - """Measure peak memory when trimming whitespace.""" - df = fresh_copy(self.df) - self.adapter.trim_whitespace(df, self.string_cols) - - def time_lowercase(self, n_rows: int, library: str) -> None: - """Measure time to convert string columns to lowercase.""" - df = fresh_copy(self.df) - self.adapter.to_lowercase(df, self.string_cols) - - def peakmem_lowercase(self, n_rows: int, library: str) -> None: - """Measure peak memory when converting to lowercase.""" - df = fresh_copy(self.df) - self.adapter.to_lowercase(df, self.string_cols) - - def time_uppercase(self, n_rows: int, library: str) -> None: - """Measure time to convert string columns to uppercase.""" - df = fresh_copy(self.df) - self.adapter.to_uppercase(df, self.string_cols) - - def time_regex_replace(self, n_rows: int, library: str) -> None: - """Measure time to perform a regex replacement on string columns.""" - df = fresh_copy(self.df) - pattern = r'[^a-zA-Z0-9 ]' - replacement = '' - self.adapter.regex_replace(df, self.string_cols, pattern, replacement) diff --git a/freshdata-benchmarks/benchmarks/benchmark_types.py b/freshdata-benchmarks/benchmarks/benchmark_types.py deleted file mode 100644 index 0166ae5c..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_types.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Benchmark data type conversion and optimization across libraries.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect -from benchmarks.utils.library_wrappers import get_adapter - - -class BenchmarkTypes: - """Benchmark suite for type inference, conversion, and memory optimization.""" - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas", "polars"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset and adapter for the benchmark.""" - self.adapter = get_adapter(library) - if not self.adapter.available: - raise NotImplementedError(f"{library} is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Define specific columns to target - self.numeric_targets = ["int_col_1", "int_col_2", "float_col_1"] - self.date_targets = ["date_col"] - - # Clean up memory - gc_collect() - - def time_convert_numeric(self, n_rows: int, library: str) -> None: - """Measure time to convert string columns to numeric types.""" - df = fresh_copy(self.df) - self.adapter.convert_numeric(df, self.numeric_targets) - - def time_convert_datetime(self, n_rows: int, library: str) -> None: - """Measure time to parse datetime strings.""" - df = fresh_copy(self.df) - self.adapter.convert_datetime(df, self.date_targets) - - def time_optimize_dtypes(self, n_rows: int, library: str) -> None: - """Measure time to perform global type optimization (downcasting).""" - df = fresh_copy(self.df) - self.adapter.optimize_dtypes(df) - - def peakmem_optimize_dtypes(self, n_rows: int, library: str) -> None: - """Measure peak memory during global type optimization.""" - df = fresh_copy(self.df) - self.adapter.optimize_dtypes(df) diff --git a/freshdata-benchmarks/benchmarks/benchmark_validation.py b/freshdata-benchmarks/benchmarks/benchmark_validation.py deleted file mode 100644 index fd7346cf..00000000 --- a/freshdata-benchmarks/benchmarks/benchmark_validation.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Benchmark schema and data validation performance.""" - -from __future__ import annotations - -import gc - -from benchmarks.datasets import get_dataset -from benchmarks.utils.fairness import fresh_copy, gc_collect - - -class BenchmarkValidation: - """Benchmark suite for schema and data quality validation. - - Compares FreshData's built-in profiler against manual pandas validation. - """ - - timeout = 300 - param_names = ["n_rows", "library"] - params = [ - [10_000, 100_000, 1_000_000], - ["freshdata", "pandas"], - ] - - def setup(self, n_rows: int, library: str) -> None: - """Prepare dataset for the benchmark.""" - if library == "freshdata": - try: - import freshdata # noqa: F401 - except ImportError: - raise NotImplementedError("freshdata is not available") - - # Get base dataset - base_df = get_dataset(n_rows) - self.df = fresh_copy(base_df) - - # Clean up memory - gc_collect() - - def time_profile_validation(self, n_rows: int, library: str) -> None: - """Measure time to run full data profiling/validation.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - fd.profile(df) - else: # pandas manual profile - import pandas as pd - _ = df.dtypes - _ = df.isnull().sum() - _ = df.nunique() - _ = df.describe(include='all') - - def time_null_check(self, n_rows: int, library: str) -> None: - """Measure time to check for nulls across all columns.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - # FreshData profile includes null checks - fd.profile(df) - else: - _ = df.isnull().any() - - def time_type_check(self, n_rows: int, library: str) -> None: - """Measure time to validate data types.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - fd.profile(df) - else: - _ = df.dtypes - - def time_range_check(self, n_rows: int, library: str) -> None: - """Measure time to check numeric ranges.""" - df = fresh_copy(self.df) - if library == "freshdata": - import freshdata as fd - fd.profile(df) - else: - import pandas as pd - numeric_df = df.select_dtypes(include=['number']) - if not numeric_df.empty: - _ = numeric_df.min() - _ = numeric_df.max() diff --git a/freshdata-benchmarks/benchmarks/datasets/__init__.py b/freshdata-benchmarks/benchmarks/datasets/__init__.py deleted file mode 100644 index 6646b963..00000000 --- a/freshdata-benchmarks/benchmarks/datasets/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Cached dataset access for benchmarks. - -Usage:: - - from benchmarks.datasets import get_dataset - df = get_dataset(100_000) -""" - -from __future__ import annotations - -import functools - -from .data_generator import generate_dataset - - -@functools.lru_cache(maxsize=8) -def get_dataset(n_rows: int) -> "pd.DataFrame": - """Return a cached messy dataset with *n_rows* rows. - - The dataset is generated once per ``n_rows`` value and cached in - memory for the lifetime of the process. Benchmark ``setup()`` - methods should ``.copy()`` the returned frame before mutation. - """ - return generate_dataset(n_rows) - - -__all__ = ["get_dataset", "generate_dataset"] diff --git a/freshdata-benchmarks/benchmarks/datasets/data_generator.py b/freshdata-benchmarks/benchmarks/datasets/data_generator.py deleted file mode 100644 index 1f4756f6..00000000 --- a/freshdata-benchmarks/benchmarks/datasets/data_generator.py +++ /dev/null @@ -1,418 +0,0 @@ -"""Deterministic synthetic dataset generator for benchmarks. - -Generates realistic tabular data with controllable data quality issues. -All generation uses fixed random seeds for reproducibility across runs. - -Usage:: - - from benchmarks.datasets import get_dataset - df = get_dataset(100_000) # 100K rows, cached -""" - -from __future__ import annotations - -import functools -import string - -import numpy as np -import pandas as pd - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -SEED = 42 -SENTINEL_VALUES = ["N/A", "-", "", "null", "NA", "#REF!", "missing", "n/a", "None"] -EMAIL_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "company.org", "test.io"] -FIRST_NAMES = [ - "Alice", "Bob", "Carol", "Dave", "Eve", "Frank", "Grace", "Heidi", - "Ivan", "Judy", "Karl", "Laura", "Mallory", "Niaj", "Oscar", "Peggy", - "Quinn", "Rupert", "Sybil", "Trent", "Ursula", "Victor", "Wendy", -] -CATEGORIES_50 = [f"category_{i:03d}" for i in range(50)] -URL_PREFIXES = ["http://", "https://", "ftp://", ""] -CURRENCY_SYMBOLS = ["$", "€", "£", "¥", ""] - -# Dataset size tiers -SIZES = { - "tiny": 10_000, - "small": 100_000, - "medium": 1_000_000, - "large": 5_000_000, - "xlarge": 10_000_000, -} - - -# --------------------------------------------------------------------------- -# Core generator -# --------------------------------------------------------------------------- - - -def generate_dataset(n_rows: int, seed: int = SEED) -> pd.DataFrame: - """Generate a synthetic messy dataset with *n_rows* rows. - - The dataset contains 20+ columns with realistic data quality issues: - missing values, duplicates, wrong dtypes, whitespace, case - inconsistencies, Unicode, outliers, invalid patterns, and more. - - Parameters - ---------- - n_rows : int - Number of rows to generate. - seed : int - Random seed for reproducibility. - - Returns - ------- - pd.DataFrame - A deliberately messy DataFrame suitable for cleaning benchmarks. - """ - rng = np.random.default_rng(seed) - - # --- Unique ID column --- - ids = np.arange(1, n_rows + 1) - - # --- Numeric columns with NaN and outliers --- - float_col_1 = _numeric_with_issues(rng, n_rows, loc=100.0, scale=25.0, - nan_rate=0.12, outlier_rate=0.02) - float_col_2 = _numeric_with_issues(rng, n_rows, loc=50.0, scale=15.0, - nan_rate=0.18, outlier_rate=0.015) - float_col_3 = _skewed_numeric(rng, n_rows, nan_rate=0.10) - - int_col_1 = _integer_with_contamination(rng, n_rows, low=0, high=1000, - nan_rate=0.08, contam_rate=0.03) - int_col_2 = _integer_with_contamination(rng, n_rows, low=1, high=100, - nan_rate=0.05, contam_rate=0.02) - - # --- String columns --- - string_col = _messy_strings(rng, n_rows, FIRST_NAMES, - whitespace_rate=0.15, case_rate=0.20, - sentinel_rate=0.05) - - # --- High-cardinality categorical --- - category_col = _messy_categorical(rng, n_rows, CATEGORIES_50, - nan_rate=0.10) - - # --- Date column with mixed formats and invalid values --- - date_col = _messy_dates(rng, n_rows, invalid_rate=0.05, nan_rate=0.08) - - # --- Boolean column with mixed representations --- - bool_col = _messy_booleans(rng, n_rows, nan_rate=0.06) - - # --- Email column with invalid entries --- - email_col = _messy_emails(rng, n_rows, invalid_rate=0.10, nan_rate=0.05) - - # --- Phone column with mixed formats --- - phone_col = _messy_phones(rng, n_rows, invalid_rate=0.08, nan_rate=0.07) - - # --- URL column --- - url_col = _messy_urls(rng, n_rows, invalid_rate=0.10, nan_rate=0.06) - - # --- Currency column --- - currency_col = _messy_currency(rng, n_rows, nan_rate=0.08) - - # --- Unicode column --- - unicode_col = _messy_unicode(rng, n_rows, nan_rate=0.05) - - # --- Constant column (all same value) --- - constant_col = np.full(n_rows, "CONSTANT_VALUE", dtype=object) - - # --- Null-heavy column (90%+ null) --- - null_heavy = _null_heavy_column(rng, n_rows, null_rate=0.92) - - # --- Mixed-type column (ints, floats, strings) --- - mixed_type = _mixed_type_column(rng, n_rows) - - # --- Negative values column --- - negative_col = rng.normal(-50.0, 30.0, n_rows) - neg_mask = rng.random(n_rows) < 0.10 - negative_col = negative_col.astype(object) - negative_col[neg_mask] = None - - # --- Build DataFrame --- - df = pd.DataFrame({ - "id": ids, - "float_col_1": float_col_1, - "float_col_2": float_col_2, - "float_col_3": float_col_3, - "int_col_1": int_col_1, - "int_col_2": int_col_2, - " String Col ": string_col, # intentional whitespace in name - "CATEGORY_COL": category_col, # intentional uppercase name - "date_col": date_col, - "bool_col": bool_col, - "email_col": email_col, - "phone_col": phone_col, - "url_col": url_col, - "currency_col": currency_col, - "unicode_col": unicode_col, - "constant_col": constant_col, - "null_heavy_col": null_heavy, - "mixed_type_col": mixed_type, - "negative_col": negative_col, - }) - - # --- Inject duplicate rows (~5% of dataset) --- - n_dupes = max(1, int(n_rows * 0.05)) - dupe_idx = rng.choice(n_rows, size=n_dupes, replace=True) - dupes = df.iloc[dupe_idx].copy() - df = pd.concat([df, dupes], ignore_index=True) - - # Shuffle to distribute duplicates - df = df.sample(frac=1.0, random_state=seed).reset_index(drop=True) - - return df - - -# --------------------------------------------------------------------------- -# Column generators (private) -# --------------------------------------------------------------------------- - - -def _numeric_with_issues( - rng: np.random.Generator, - n: int, - loc: float, - scale: float, - nan_rate: float, - outlier_rate: float, -) -> np.ndarray: - """Normal-distributed floats with NaN and extreme outliers.""" - values = rng.normal(loc, scale, n).astype(object) - # Inject outliers (8–15× the value) - outlier_mask = rng.random(n) < outlier_rate - outlier_count = outlier_mask.sum() - if outlier_count > 0: - values[outlier_mask] = ( - rng.normal(loc, scale, outlier_count) * rng.uniform(8, 15, outlier_count) - ) - # Inject NaN - nan_mask = rng.random(n) < nan_rate - values[nan_mask] = None - return values - - -def _skewed_numeric( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Log-normal (right-skewed) distribution with NaN.""" - values = rng.lognormal(mean=3.0, sigma=1.0, size=n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _integer_with_contamination( - rng: np.random.Generator, - n: int, - low: int, - high: int, - nan_rate: float, - contam_rate: float, -) -> np.ndarray: - """Integers stored as strings, with some non-numeric contamination.""" - values = rng.integers(low, high, n).astype(str).astype(object) - # Contaminate with non-numeric strings - contam_mask = rng.random(n) < contam_rate - contam_count = contam_mask.sum() - if contam_count > 0: - values[contam_mask] = rng.choice(["abc", "N/A", "??", "--", ""], contam_count) - # Inject NaN - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_strings( - rng: np.random.Generator, - n: int, - base_values: list[str], - whitespace_rate: float, - case_rate: float, - sentinel_rate: float, -) -> np.ndarray: - """Strings with whitespace, case inconsistencies, and sentinels.""" - values = rng.choice(base_values, n).astype(object) - # Add whitespace - ws_mask = rng.random(n) < whitespace_rate - for i in np.where(ws_mask)[0]: - pad_left = " " * rng.integers(1, 4) - pad_right = " " * rng.integers(1, 4) - values[i] = f"{pad_left}{values[i]}{pad_right}" - # Random case - case_mask = rng.random(n) < case_rate - for i in np.where(case_mask)[0]: - choice = rng.integers(0, 3) - if choice == 0: - values[i] = str(values[i]).upper() - elif choice == 1: - values[i] = str(values[i]).lower() - else: - values[i] = str(values[i]).title() - # Replace some with sentinels - sent_mask = rng.random(n) < sentinel_rate - values[sent_mask] = rng.choice(SENTINEL_VALUES, sent_mask.sum()) - return values - - -def _messy_categorical( - rng: np.random.Generator, - n: int, - categories: list[str], - nan_rate: float, -) -> np.ndarray: - """High-cardinality categorical with NaN.""" - values = rng.choice(categories, n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_dates( - rng: np.random.Generator, - n: int, - invalid_rate: float, - nan_rate: float, -) -> np.ndarray: - """Date strings in mixed formats with invalid entries.""" - base_dates = pd.date_range("2020-01-01", periods=n, freq="h") - formats = ["%Y-%m-%d", "%m/%d/%Y", "%d-%b-%Y", "%Y/%m/%d %H:%M:%S"] - values = np.empty(n, dtype=object) - for i in range(n): - fmt = formats[i % len(formats)] - values[i] = base_dates[i % len(base_dates)].strftime(fmt) - # Invalid dates - inv_mask = rng.random(n) < invalid_rate - inv_values = ["2020-13-45", "not-a-date", "31/02/2020", "00/00/0000", - "2020-02-30", "abc", "9999-99-99"] - values[inv_mask] = rng.choice(inv_values, inv_mask.sum()) - # NaN - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_booleans( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Booleans with mixed representations.""" - representations = ["yes", "no", "Yes", "No", "YES", "NO", "true", "false", - "True", "False", "TRUE", "FALSE", "1", "0", "Y", "N", - "y", "n", "T", "F"] - values = rng.choice(representations, n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_emails( - rng: np.random.Generator, n: int, invalid_rate: float, nan_rate: float -) -> np.ndarray: - """Email addresses with invalid entries.""" - values = np.empty(n, dtype=object) - for i in range(n): - user_len = rng.integers(5, 12) - user = "".join(rng.choice(list(string.ascii_lowercase), user_len)) - domain = EMAIL_DOMAINS[i % len(EMAIL_DOMAINS)] - values[i] = f"{user}@{domain}" - # Invalidate some - inv_mask = rng.random(n) < invalid_rate - invalid_emails = ["notanemail", "missing@", "@nodomain.com", "spaces in@email.com", - "double@@at.com", "no.tld@", ""] - values[inv_mask] = rng.choice(invalid_emails, inv_mask.sum()) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_phones( - rng: np.random.Generator, n: int, invalid_rate: float, nan_rate: float -) -> np.ndarray: - """Phone numbers in mixed formats.""" - values = np.empty(n, dtype=object) - formats_tpl = [ - "{area}-{pre}-{suf}", "({area}) {pre}-{suf}", - "+1{area}{pre}{suf}", "{area}.{pre}.{suf}", - "{area}{pre}{suf}", - ] - for i in range(n): - area = f"{rng.integers(200, 999)}" - pre = f"{rng.integers(200, 999)}" - suf = f"{rng.integers(1000, 9999)}" - fmt = formats_tpl[i % len(formats_tpl)] - values[i] = fmt.format(area=area, pre=pre, suf=suf) - inv_mask = rng.random(n) < invalid_rate - invalid_phones = ["123", "abc-def-ghij", "00000000000000", "+", "555"] - values[inv_mask] = rng.choice(invalid_phones, inv_mask.sum()) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_urls( - rng: np.random.Generator, n: int, invalid_rate: float, nan_rate: float -) -> np.ndarray: - """URLs with invalid entries.""" - tlds = ["com", "org", "net", "io", "dev"] - values = np.empty(n, dtype=object) - for i in range(n): - prefix = URL_PREFIXES[i % len(URL_PREFIXES)] - domain_len = rng.integers(4, 10) - domain = "".join(rng.choice(list(string.ascii_lowercase), domain_len)) - tld = tlds[i % len(tlds)] - values[i] = f"{prefix}{domain}.{tld}/path" - inv_mask = rng.random(n) < invalid_rate - invalid_urls = ["not a url", "://missing-scheme", "http://", "ftp:///", - "just-text", "http://a b c.com"] - values[inv_mask] = rng.choice(invalid_urls, inv_mask.sum()) - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_currency( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Currency strings in mixed formats.""" - values = np.empty(n, dtype=object) - for i in range(n): - amount = rng.uniform(1.0, 99999.99) - symbol = CURRENCY_SYMBOLS[i % len(CURRENCY_SYMBOLS)] - # Randomly format with/without commas - if rng.random() < 0.5: - values[i] = f"{symbol}{amount:,.2f}" - else: - values[i] = f"{symbol}{amount:.2f}" - values[rng.random(n) < nan_rate] = None - return values - - -def _messy_unicode( - rng: np.random.Generator, n: int, nan_rate: float -) -> np.ndarray: - """Strings with mixed Unicode normalization and accented chars.""" - base_words = [ - "café", "naïve", "résumé", "über", "señor", - "Ångström", "Zürich", "São Paulo", "Malmö", "日本語", - "中文", "한국어", "العربية", "हिन्दी", "normal_ascii", - ] - values = rng.choice(base_words, n).astype(object) - values[rng.random(n) < nan_rate] = None - return values - - -def _null_heavy_column( - rng: np.random.Generator, n: int, null_rate: float -) -> np.ndarray: - """Column that is almost entirely null.""" - values = rng.normal(0, 1, n).astype(object) - values[rng.random(n) < null_rate] = None - return values - - -def _mixed_type_column(rng: np.random.Generator, n: int) -> np.ndarray: - """Column with mixed types: ints, floats, strings.""" - values = np.empty(n, dtype=object) - for i in range(n): - choice = i % 4 - if choice == 0: - values[i] = int(rng.integers(0, 1000)) - elif choice == 1: - values[i] = float(rng.normal(50, 10)) - elif choice == 2: - values[i] = f"text_{rng.integers(0, 100)}" - else: - values[i] = None - return values diff --git a/freshdata-benchmarks/benchmarks/utils/__init__.py b/freshdata-benchmarks/benchmarks/utils/__init__.py deleted file mode 100644 index 515b0afc..00000000 --- a/freshdata-benchmarks/benchmarks/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Benchmark utilities package.""" diff --git a/freshdata-benchmarks/benchmarks/utils/fairness.py b/freshdata-benchmarks/benchmarks/utils/fairness.py deleted file mode 100644 index d1d9d27c..00000000 --- a/freshdata-benchmarks/benchmarks/utils/fairness.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Fairness enforcement utilities for cross-library benchmarks. - -Ensures identical inputs, prevents mutation leakage, and controls for -garbage collection / cache effects between benchmark runs. -""" - -from __future__ import annotations - -import gc -import os -from typing import Any - - -def fresh_copy(df: Any) -> Any: - """Return a deep copy of a DataFrame for mutation-safe benchmarking. - - This ensures that each benchmark run operates on identical, - independent data — no library can benefit from prior mutations. - """ - return df.copy(deep=True) - - -def gc_collect() -> None: - """Force garbage collection to reduce memory measurement noise. - - Called in ``setup()`` before each benchmark to minimize the effect - of residual allocations from previous runs. - """ - gc.collect() - gc.collect() # Second pass catches weak-reference cleanup - - -def pin_threads(n: int = 1) -> None: - """Pin numeric libraries to *n* threads for deterministic timing. - - Already handled via asv.conf.json ``matrix.env``, but can be called - explicitly in ``setup()`` for defense-in-depth. - """ - for var in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", - "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS", - "VECLIB_MAXIMUM_THREADS"): - os.environ[var] = str(n) - - -def get_numeric_columns(df: Any) -> list[str]: - """Return column names that contain (or should contain) numeric data. - - Used to identify which columns are suitable for imputation, scaling, - and outlier detection benchmarks. - """ - import pandas as pd - numeric = [] - for col in df.columns: - try: - series = pd.to_numeric(df[col], errors="coerce") - non_null = series.notna().sum() - if non_null > len(df) * 0.3: # at least 30% parseable - numeric.append(col) - except Exception: - pass - return numeric - - -def get_string_columns(df: Any) -> list[str]: - """Return column names that contain string/object data.""" - return [col for col in df.columns if df[col].dtype == object] - - -def get_categorical_columns(df: Any, max_cardinality: int = 100) -> list[str]: - """Return string columns with cardinality ≤ *max_cardinality*.""" - cats = [] - for col in get_string_columns(df): - try: - nunique = df[col].nunique() - if 2 <= nunique <= max_cardinality: - cats.append(col) - except Exception: - pass - return cats - - -def validate_equivalent_output( - original: Any, - result: Any, - operation: str, -) -> bool: - """Spot-check that a benchmark result is semantically valid. - - This is NOT called during timing — only during development/testing - to verify that library adapters produce correct results. - """ - if result is None: - return False - import pandas as pd - if isinstance(result, pd.DataFrame): - # Basic sanity: result should have at least 1 row - return len(result) > 0 - # Non-DataFrame results (e.g., Polars, boolean masks) are valid - return True diff --git a/freshdata-benchmarks/benchmarks/utils/library_wrappers.py b/freshdata-benchmarks/benchmarks/utils/library_wrappers.py deleted file mode 100644 index 301c8105..00000000 --- a/freshdata-benchmarks/benchmarks/utils/library_wrappers.py +++ /dev/null @@ -1,976 +0,0 @@ -"""Library adapter wrappers for fair cross-library benchmarking. - -Each adapter implements equivalent operations using the target library's -native API. Adapters that cannot be imported set ``available = False`` -and all their methods raise ``NotImplementedError``, which ASV treats as -a skip. - -Usage:: - - adapter = get_adapter("pandas") - adapter.drop_missing_rows(df) -""" - -from __future__ import annotations - -import abc -import gc -from typing import Any - - -# --------------------------------------------------------------------------- -# Base adapter -# --------------------------------------------------------------------------- - -class LibraryAdapter(abc.ABC): - """Abstract base class for library-specific operation wrappers.""" - - name: str = "base" - available: bool = False - - def _skip(self, op: str) -> None: - raise NotImplementedError(f"{self.name} does not support {op}") - - # --- Missing values --- - def drop_missing_rows(self, df: Any) -> Any: - self._skip("drop_missing_rows") - - def fill_missing_mean(self, df: Any, columns: list[str]) -> Any: - self._skip("fill_missing_mean") - - def fill_missing_median(self, df: Any, columns: list[str]) -> Any: - self._skip("fill_missing_median") - - def fill_missing_mode(self, df: Any, columns: list[str]) -> Any: - self._skip("fill_missing_mode") - - def fill_missing_ffill(self, df: Any) -> Any: - self._skip("fill_missing_ffill") - - def fill_missing_bfill(self, df: Any) -> Any: - self._skip("fill_missing_bfill") - - # --- Duplicates --- - def detect_duplicates(self, df: Any) -> Any: - self._skip("detect_duplicates") - - def drop_duplicates(self, df: Any) -> Any: - self._skip("drop_duplicates") - - # --- String cleaning --- - def trim_whitespace(self, df: Any, columns: list[str]) -> Any: - self._skip("trim_whitespace") - - def to_lowercase(self, df: Any, columns: list[str]) -> Any: - self._skip("to_lowercase") - - def to_uppercase(self, df: Any, columns: list[str]) -> Any: - self._skip("to_uppercase") - - def regex_replace(self, df: Any, columns: list[str], - pattern: str, replacement: str) -> Any: - self._skip("regex_replace") - - # --- Column operations --- - def rename_columns(self, df: Any, mapping: dict) -> Any: - self._skip("rename_columns") - - def drop_columns(self, df: Any, columns: list[str]) -> Any: - self._skip("drop_columns") - - def select_columns(self, df: Any, columns: list[str]) -> Any: - self._skip("select_columns") - - # --- Type conversion --- - def convert_numeric(self, df: Any, columns: list[str]) -> Any: - self._skip("convert_numeric") - - def convert_datetime(self, df: Any, columns: list[str]) -> Any: - self._skip("convert_datetime") - - def optimize_dtypes(self, df: Any) -> Any: - self._skip("optimize_dtypes") - - # --- Encoding --- - def onehot_encode(self, df: Any, columns: list[str]) -> Any: - self._skip("onehot_encode") - - def label_encode(self, df: Any, columns: list[str]) -> Any: - self._skip("label_encode") - - # --- Scaling --- - def standard_scale(self, df: Any, columns: list[str]) -> Any: - self._skip("standard_scale") - - def minmax_scale(self, df: Any, columns: list[str]) -> Any: - self._skip("minmax_scale") - - # --- Outlier detection --- - def detect_outliers_iqr(self, df: Any, columns: list[str]) -> Any: - self._skip("detect_outliers_iqr") - - def detect_outliers_zscore(self, df: Any, columns: list[str]) -> Any: - self._skip("detect_outliers_zscore") - - # --- Group aggregations --- - def group_agg_single(self, df: Any) -> Any: - self._skip("group_agg_single") - - def group_agg_multi(self, df: Any, columns: list[str]) -> Any: - self._skip("group_agg_multi") - - def group_agg_transform(self, df: Any, columns: list[str]) -> Any: - self._skip("group_agg_transform") - - # --- Pipeline --- - def full_clean(self, df: Any) -> Any: - self._skip("full_clean") - - -# --------------------------------------------------------------------------- -# FreshData adapter -# --------------------------------------------------------------------------- - -class FreshDataAdapter(LibraryAdapter): - name = "freshdata" - - def __init__(self) -> None: - try: - import freshdata # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def drop_missing_rows(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", impute=False, - outliers=False, preserve_original=True) - - def fill_missing_mean(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="aggressive", preserve_original=True) - - def fill_missing_median(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="balanced", preserve_original=True) - - def fill_missing_mode(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="balanced", preserve_original=True) - - def fill_missing_ffill(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def fill_missing_bfill(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def detect_duplicates(self, df): - import freshdata as fd - return fd.profile(df) - - def drop_duplicates(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def trim_whitespace(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def to_lowercase(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def to_uppercase(self, df, columns): - # FreshData normalizes to lowercase, not uppercase - self._skip("to_uppercase") - - def rename_columns(self, df, mapping): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def drop_columns(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def select_columns(self, df, columns): - # FreshData doesn't have explicit column selection - self._skip("select_columns") - - def convert_numeric(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def convert_datetime(self, df, columns): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def optimize_dtypes(self, df): - import freshdata as fd - return fd.clean(df, strategy="conservative", preserve_original=True) - - def detect_outliers_iqr(self, df, columns): - import freshdata as fd - return fd.clean(df, outlier_method="iqr", outlier_action="flag", - strategy="balanced", preserve_original=True) - - def detect_outliers_zscore(self, df, columns): - import freshdata as fd - return fd.clean(df, outlier_method="zscore", outlier_action="flag", - strategy="balanced", preserve_original=True) - - def group_agg_single(self, df): - import freshdata as fd - return fd.profile(df) - - def group_agg_multi(self, df, columns): - import freshdata as fd - return fd.profile(df) - - def group_agg_transform(self, df, columns): - import freshdata as fd - return fd.profile(df) - - def full_clean(self, df): - import freshdata as fd - return fd.clean(df, strategy="balanced", preserve_original=True) - - -# --------------------------------------------------------------------------- -# Pandas adapter -# --------------------------------------------------------------------------- - -class PandasAdapter(LibraryAdapter): - name = "pandas" - - def __init__(self) -> None: - try: - import pandas # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def drop_missing_rows(self, df): - import pandas as pd - return df.dropna() - - def fill_missing_mean(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - result[col] = numeric.fillna(numeric.mean()) - return result - - def fill_missing_median(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - result[col] = numeric.fillna(numeric.median()) - return result - - def fill_missing_mode(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - mode_val = result[col].mode() - if len(mode_val) > 0: - result[col] = result[col].fillna(mode_val.iloc[0]) - return result - - def fill_missing_ffill(self, df): - return df.ffill() - - def fill_missing_bfill(self, df): - return df.bfill() - - def detect_duplicates(self, df): - return df.duplicated() - - def drop_duplicates(self, df): - return df.drop_duplicates() - - def trim_whitespace(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.strip() - return result - - def to_lowercase(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.lower() - return result - - def to_uppercase(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.upper() - return result - - def regex_replace(self, df, columns, pattern, replacement): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.replace( - pattern, replacement, regex=True - ) - return result - - def rename_columns(self, df, mapping): - return df.rename(columns=mapping) - - def drop_columns(self, df, columns): - existing = [c for c in columns if c in df.columns] - return df.drop(columns=existing) - - def select_columns(self, df, columns): - existing = [c for c in columns if c in df.columns] - return df[existing] - - def convert_numeric(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - result[col] = pd.to_numeric(result[col], errors="coerce") - return result - - def convert_datetime(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - result[col] = pd.to_datetime(result[col], errors="coerce") - return result - - def optimize_dtypes(self, df): - return df.convert_dtypes() - - def onehot_encode(self, df, columns): - import pandas as pd - existing = [c for c in columns if c in df.columns] - return pd.get_dummies(df, columns=existing) - - def label_encode(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - result[col] = result[col].astype("category").cat.codes - return result - - def standard_scale(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - vals = result[col] - result[col] = (vals - vals.mean()) / vals.std() - return result - - def minmax_scale(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns: - vals = result[col] - mn, mx = vals.min(), vals.max() - if mx != mn: - result[col] = (vals - mn) / (mx - mn) - return result - - def detect_outliers_iqr(self, df, columns): - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - q1 = numeric.quantile(0.25) - q3 = numeric.quantile(0.75) - iqr = q3 - q1 - mask = (numeric < q1 - 1.5 * iqr) | (numeric > q3 + 1.5 * iqr) - result[f"{col}_outlier"] = mask - return result - - def detect_outliers_zscore(self, df, columns): - import numpy as np - import pandas as pd - result = df.copy() - for col in columns: - if col in result.columns: - numeric = pd.to_numeric(result[col], errors="coerce") - mean = numeric.mean() - std = numeric.std() - if std > 0: - z = np.abs((numeric - mean) / std) - result[f"{col}_outlier"] = z > 3 - return result - - def group_agg_single(self, df): - """Single-column groupby with mean aggregation.""" - import pandas as pd - result = df.copy() - # Use CATEGORY_COL as the groupby key (high-cardinality categorical) - if "CATEGORY_COL" in result.columns and "float_col_1" in result.columns: - result["float_col_1"] = pd.to_numeric(result["float_col_1"], errors="coerce") - return result.groupby("CATEGORY_COL", observed=True).agg({"float_col_1": "mean"}) - # Fallback: group by the first object column - obj_cols = result.select_dtypes(include="object").columns - if len(obj_cols) > 0: - return result.groupby(obj_cols[0], observed=True).size() - return result.describe() - - def group_agg_multi(self, df, columns): - """Multi-column groupby with multiple agg functions.""" - import pandas as pd - result = df.copy() - if "CATEGORY_COL" in result.columns: - # Coerce numeric columns and build agg dict - agg_dict = {} - for col in columns: - if col in result.columns: - result[col] = pd.to_numeric(result[col], errors="coerce") - agg_dict[col] = ["mean", "std", "min", "max"] - if agg_dict: - return result.groupby("CATEGORY_COL", observed=True).agg(agg_dict) - # Fallback: describe all numeric - return result.describe() - - def group_agg_transform(self, df, columns): - """Groupby transform: center numeric columns within groups.""" - import pandas as pd - result = df.copy() - if "CATEGORY_COL" not in result.columns: - return result - for col in columns: - if col in result.columns: - try: - numeric_col = pd.to_numeric(result[col], errors="coerce") - result[col] = numeric_col - numeric_col.groupby( - result["CATEGORY_COL"] - ).transform("mean") - except Exception: - pass - return result - - def full_clean(self, df): - """Manual pandas cleaning pipeline equivalent to fd.clean().""" - result = df.copy() - # 1. Clean column names - result.columns = ( - result.columns.str.strip() - .str.lower() - .str.replace(r"[^a-z0-9_]", "_", regex=True) - .str.replace(r"_+", "_", regex=True) - .str.strip("_") - ) - # 2. Strip whitespace from string columns - str_cols = result.select_dtypes(include="object").columns - for col in str_cols: - try: - result[col] = result[col].str.strip() - except AttributeError: - pass - # 3. Replace sentinels with NaN - import numpy as np - sentinels = ["N/A", "n/a", "NA", "null", "None", "-", "", "#REF!", - "missing", "nan", "NaN"] - result = result.replace(sentinels, np.nan) - # 4. Drop all-empty columns - result = result.dropna(axis=1, how="all") - # 5. Drop all-empty rows - result = result.dropna(axis=0, how="all") - # 6. Drop duplicates - result = result.drop_duplicates() - # 7. Convert dtypes - result = result.convert_dtypes() - return result - - -# --------------------------------------------------------------------------- -# Polars adapter -# --------------------------------------------------------------------------- - -class PolarsAdapter(LibraryAdapter): - name = "polars" - - def __init__(self) -> None: - try: - import polars # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def _to_polars(self, df): - import polars as pl - import pandas as pd - df_clean = df.copy() - # PyArrow cannot convert mixed int/str object columns. Cast non-nulls to string. - for col in df_clean.select_dtypes(include=["object"]).columns: - mask = df_clean[col].notna() - df_clean.loc[mask, col] = df_clean.loc[mask, col].astype(str) - return pl.from_pandas(df_clean) - - def drop_missing_rows(self, df): - pldf = self._to_polars(df) - return pldf.drop_nulls() - - def fill_missing_mean(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - .fill_null(pl.col(col).cast(pl.Float64, strict=False).mean()) - ) - except Exception: - pass - return pldf - - def fill_missing_median(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - .fill_null(pl.col(col).cast(pl.Float64, strict=False).median()) - ) - except Exception: - pass - return pldf - - def fill_missing_ffill(self, df): - import polars as pl - pldf = self._to_polars(df) - return pldf.with_columns(pl.all().forward_fill()) - - def fill_missing_bfill(self, df): - import polars as pl - pldf = self._to_polars(df) - return pldf.with_columns(pl.all().backward_fill()) - - def detect_duplicates(self, df): - pldf = self._to_polars(df) - return pldf.is_duplicated() - - def drop_duplicates(self, df): - pldf = self._to_polars(df) - return pldf.unique() - - def trim_whitespace(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns(pl.col(col).str.strip_chars()) - except Exception: - pass - return pldf - - def to_lowercase(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns(pl.col(col).str.to_lowercase()) - except Exception: - pass - return pldf - - def to_uppercase(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns(pl.col(col).str.to_uppercase()) - except Exception: - pass - return pldf - - def regex_replace(self, df, columns, pattern, replacement): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).str.replace_all(pattern, replacement) - ) - except Exception: - pass - return pldf - - def rename_columns(self, df, mapping): - pldf = self._to_polars(df) - return pldf.rename(mapping) - - def drop_columns(self, df, columns): - pldf = self._to_polars(df) - existing = [c for c in columns if c in pldf.columns] - return pldf.drop(existing) - - def select_columns(self, df, columns): - pldf = self._to_polars(df) - existing = [c for c in columns if c in pldf.columns] - return pldf.select(existing) - - def convert_numeric(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - pldf = pldf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - ) - except Exception: - pass - return pldf - - def detect_outliers_iqr(self, df, columns): - import polars as pl - pldf = self._to_polars(df) - for col in columns: - if col in pldf.columns: - try: - q1 = pldf[col].cast(pl.Float64, strict=False).quantile(0.25) - q3 = pldf[col].cast(pl.Float64, strict=False).quantile(0.75) - if q1 is not None and q3 is not None: - iqr = q3 - q1 - pldf = pldf.with_columns( - ((pl.col(col).cast(pl.Float64, strict=False) < q1 - 1.5 * iqr) - | (pl.col(col).cast(pl.Float64, strict=False) > q3 + 1.5 * iqr)) - .alias(f"{col}_outlier") - ) - except Exception: - pass - return pldf - - def full_clean(self, df): - """Manual Polars cleaning pipeline.""" - import polars as pl - pldf = self._to_polars(df) - # Clean column names - new_names = {} - for col in pldf.columns: - import re - clean = col.strip().lower() - clean = re.sub(r"[^a-z0-9_]", "_", clean) - clean = re.sub(r"_+", "_", clean).strip("_") - new_names[col] = clean - pldf = pldf.rename(new_names) - # Drop all-null columns - non_null_cols = [ - col for col in pldf.columns - if pldf[col].null_count() < len(pldf) - ] - pldf = pldf.select(non_null_cols) - # Drop duplicates - pldf = pldf.unique() - return pldf - - -# --------------------------------------------------------------------------- -# pyjanitor adapter -# --------------------------------------------------------------------------- - -class PyjanitorAdapter(LibraryAdapter): - name = "pyjanitor" - - def __init__(self) -> None: - try: - import janitor # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def drop_missing_rows(self, df): - import janitor # noqa: F401 - return df.remove_empty(reset_index=False) - - def drop_duplicates(self, df): - return df.drop_duplicates() - - def trim_whitespace(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.strip() - return result - - def to_lowercase(self, df, columns): - result = df.copy() - for col in columns: - if col in result.columns and result[col].dtype == object: - result[col] = result[col].str.lower() - return result - - def rename_columns(self, df, mapping): - import janitor # noqa: F401 - return df.clean_names() - - def drop_columns(self, df, columns): - import janitor # noqa: F401 - return df.remove_columns(columns) - - def select_columns(self, df, columns): - import janitor # noqa: F401 - return df.select_columns(columns) - - def onehot_encode(self, df, columns): - import janitor # noqa: F401 - result = df - for col in columns: - if col in result.columns: - try: - result = result.encode_categorical(col) - except Exception: - pass - return result - - def full_clean(self, df): - import janitor # noqa: F401 - return df.clean_names().remove_empty(reset_index=False).drop_duplicates() - - -# --------------------------------------------------------------------------- -# Feature Engine adapter -# --------------------------------------------------------------------------- - -class FeatureEngineAdapter(LibraryAdapter): - name = "feature_engine" - - def __init__(self) -> None: - try: - import feature_engine # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def fill_missing_mean(self, df, columns): - from feature_engine.imputation import MeanMedianImputer - existing = [c for c in columns if c in df.columns] - if not existing: - return df - imp = MeanMedianImputer(imputation_method="mean", variables=existing) - return imp.fit_transform(df) - - def fill_missing_median(self, df, columns): - from feature_engine.imputation import MeanMedianImputer - existing = [c for c in columns if c in df.columns] - if not existing: - return df - imp = MeanMedianImputer(imputation_method="median", variables=existing) - return imp.fit_transform(df) - - def onehot_encode(self, df, columns): - from feature_engine.encoding import OneHotEncoder - existing = [c for c in columns if c in df.columns] - if not existing: - return df - enc = OneHotEncoder(variables=existing, drop_last=False) - return enc.fit_transform(df) - - def label_encode(self, df, columns): - from feature_engine.encoding import OrdinalEncoder - existing = [c for c in columns if c in df.columns] - if not existing: - return df - enc = OrdinalEncoder(variables=existing) - return enc.fit_transform(df) - - def detect_outliers_iqr(self, df, columns): - from feature_engine.outliers import Winsorizer - existing = [c for c in columns if c in df.columns] - if not existing: - return df - w = Winsorizer(capping_method="iqr", fold=1.5, variables=existing) - return w.fit_transform(df) - - def standard_scale(self, df, columns): - from sklearn.preprocessing import StandardScaler - import pandas as pd - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = StandardScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - def minmax_scale(self, df, columns): - from sklearn.preprocessing import MinMaxScaler - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = MinMaxScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - -# --------------------------------------------------------------------------- -# Scikit-learn adapter -# --------------------------------------------------------------------------- - -class SklearnAdapter(LibraryAdapter): - name = "sklearn" - - def __init__(self) -> None: - try: - import sklearn # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def fill_missing_mean(self, df, columns): - from sklearn.impute import SimpleImputer - import numpy as np - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - imp = SimpleImputer(strategy="mean") - result[existing] = imp.fit_transform(result[existing]) - return result - - def fill_missing_median(self, df, columns): - from sklearn.impute import SimpleImputer - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - imp = SimpleImputer(strategy="median") - result[existing] = imp.fit_transform(result[existing]) - return result - - def fill_missing_mode(self, df, columns): - from sklearn.impute import SimpleImputer - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - imp = SimpleImputer(strategy="most_frequent") - result[existing] = imp.fit_transform(result[existing]) - return result - - def onehot_encode(self, df, columns): - from sklearn.preprocessing import OneHotEncoder - import pandas as pd - import numpy as np - result = df.copy() - existing = [c for c in columns if c in result.columns] - if not existing: - return result - enc = OneHotEncoder(sparse_output=False, handle_unknown="ignore") - encoded = enc.fit_transform(result[existing].astype(str)) - feature_names = enc.get_feature_names_out(existing) - encoded_df = pd.DataFrame(encoded, columns=feature_names, - index=result.index) - result = result.drop(columns=existing) - return pd.concat([result, encoded_df], axis=1) - - def label_encode(self, df, columns): - from sklearn.preprocessing import LabelEncoder - result = df.copy() - for col in columns: - if col in result.columns: - le = LabelEncoder() - mask = result[col].notna() - if mask.any(): - result.loc[mask, col] = le.fit_transform( - result.loc[mask, col].astype(str) - ) - return result - - def standard_scale(self, df, columns): - from sklearn.preprocessing import StandardScaler - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = StandardScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - def minmax_scale(self, df, columns): - from sklearn.preprocessing import MinMaxScaler - result = df.copy() - existing = [c for c in columns if c in result.columns] - if existing: - scaler = MinMaxScaler() - result[existing] = scaler.fit_transform(result[existing]) - return result - - def detect_outliers_zscore(self, df, columns): - import numpy as np - result = df.copy() - for col in columns: - if col in result.columns: - mean = result[col].mean() - std = result[col].std() - if std > 0: - z = np.abs((result[col] - mean) / std) - result[f"{col}_outlier"] = z > 3 - return result - - -# --------------------------------------------------------------------------- -# AutoClean adapter (may not be available) -# --------------------------------------------------------------------------- - -class AutoCleanAdapter(LibraryAdapter): - name = "autoclean" - - def __init__(self) -> None: - try: - from AutoClean import AutoClean # noqa: F401 - self.available = True - except ImportError: - self.available = False - - def full_clean(self, df): - from AutoClean import AutoClean - pipeline = AutoClean(df.copy(), mode="auto", duplicates=True, - missing_num="auto", missing_categ="auto", - outliers="winz") - return pipeline.output - - -# --------------------------------------------------------------------------- -# Adapter registry -# --------------------------------------------------------------------------- - -_ADAPTERS: dict[str, type[LibraryAdapter]] = { - "freshdata": FreshDataAdapter, - "pandas": PandasAdapter, - "polars": PolarsAdapter, - "pyjanitor": PyjanitorAdapter, - "feature_engine": FeatureEngineAdapter, - "sklearn": SklearnAdapter, - "autoclean": AutoCleanAdapter, -} - -_adapter_cache: dict[str, LibraryAdapter] = {} - - -def get_adapter(name: str) -> LibraryAdapter: - """Get a cached adapter instance by library name.""" - if name not in _adapter_cache: - cls = _ADAPTERS.get(name) - if cls is None: - raise ValueError(f"Unknown library: {name!r}. " - f"Available: {list(_ADAPTERS)}") - _adapter_cache[name] = cls() - return _adapter_cache[name] - - -def available_libraries() -> list[str]: - """Return names of libraries that are importable.""" - return [name for name, cls in _ADAPTERS.items() if cls().available] diff --git a/freshdata-benchmarks/benchmarks/utils/metrics.py b/freshdata-benchmarks/benchmarks/utils/metrics.py deleted file mode 100644 index 93b24426..00000000 --- a/freshdata-benchmarks/benchmarks/utils/metrics.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Custom metric calculators for benchmark reports.""" - -from __future__ import annotations - - -def throughput(n_rows: int, elapsed_seconds: float) -> float: - """Compute throughput in rows per second.""" - if elapsed_seconds <= 0: - return float("inf") - return n_rows / elapsed_seconds - - -def speedup(baseline_time: float, comparison_time: float) -> float: - """Compute speedup ratio (baseline / comparison). - - A value > 1.0 means *comparison* is faster than *baseline*. - """ - if comparison_time <= 0: - return float("inf") - return baseline_time / comparison_time - - -def memory_ratio(baseline_bytes: float, comparison_bytes: float) -> float: - """Compute relative memory usage (comparison / baseline). - - A value < 1.0 means *comparison* uses less memory. - """ - if baseline_bytes <= 0: - return float("inf") - return comparison_bytes / baseline_bytes - - -def scaling_efficiency( - time_small: float, - time_large: float, - n_small: int, - n_large: int, -) -> float: - """Estimate scaling efficiency. - - Perfect linear scaling = 1.0. Sublinear > 1.0. Superlinear < 1.0. - """ - if time_small <= 0 or time_large <= 0: - return float("nan") - data_ratio = n_large / n_small - time_ratio = time_large / time_small - return data_ratio / time_ratio - - -def estimate_bigO(times: list[float], sizes: list[int]) -> str: - """Estimate Big-O complexity from empirical timing data. - - Returns a human-readable string like "O(n)", "O(n log n)", "O(n²)". - """ - import math - - if len(times) < 2 or len(sizes) < 2: - return "insufficient data" - - # Compute growth rates between consecutive points - ratios = [] - for i in range(1, len(times)): - if times[i - 1] > 0 and sizes[i - 1] > 0: - size_ratio = sizes[i] / sizes[i - 1] - time_ratio = times[i] / times[i - 1] - if size_ratio > 0 and time_ratio > 0: - exponent = math.log(time_ratio) / math.log(size_ratio) - ratios.append(exponent) - - if not ratios: - return "unknown" - - avg_exp = sum(ratios) / len(ratios) - - if avg_exp < 0.3: - return "O(1)" - elif avg_exp < 0.7: - return "O(√n)" - elif avg_exp < 1.3: - return "O(n)" - elif avg_exp < 1.7: - return "O(n log n)" - elif avg_exp < 2.3: - return "O(n²)" - else: - return f"O(n^{avg_exp:.1f})" diff --git a/freshdata-benchmarks/reports/results_extracted.json b/freshdata-benchmarks/reports/results_extracted.json deleted file mode 100644 index 9e26dfee..00000000 --- a/freshdata-benchmarks/reports/results_extracted.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/freshdata-benchmarks/reports/summary.md b/freshdata-benchmarks/reports/summary.md deleted file mode 100644 index 7d130cca..00000000 --- a/freshdata-benchmarks/reports/summary.md +++ /dev/null @@ -1,2 +0,0 @@ -# FreshData Benchmark Results - diff --git a/freshdata-benchmarks/requirements.txt b/freshdata-benchmarks/requirements.txt deleted file mode 100644 index 1f4250a7..00000000 --- a/freshdata-benchmarks/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -# FreshData Benchmark Suite — Dependencies -# ========================================== -# Core benchmark target -freshdata-cleaner[all] - -# Comparison libraries -pandas>=2.0 -polars>=0.20 -pyjanitor>=0.27 -feature-engine>=1.6 -scikit-learn>=1.3 - -# Benchmark framework -asv>=0.6 - -# Data generation & measurement -numpy>=1.24 -pyarrow>=10 -psutil>=5.9 - -# Report generation -matplotlib>=3.7 -tabulate>=0.9 - -# Optional comparison libraries (may fail to install) -# AutoClean>=1.1 # Inactive — install manually if needed -# datacleaner # Abandoned — install manually if needed diff --git a/freshdata-benchmarks/scripts/capture_env.py b/freshdata-benchmarks/scripts/capture_env.py deleted file mode 100644 index 2b2d8193..00000000 --- a/freshdata-benchmarks/scripts/capture_env.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Capture environment details for reproducible benchmarking. - -Outputs a JSON file with Python version, OS, CPU, RAM, and installed -package versions (freshdata, pandas, numpy, pyarrow). - -Usage:: - - python scripts/capture_env.py [output_path] - # Defaults to bench/artifacts/env_info.json - -""" - -from __future__ import annotations - -import json -import os -import platform -import sys -from datetime import datetime, timezone -from pathlib import Path - - -def get_package_version(name: str) -> str: - """Return installed version of a package, or 'not installed'.""" - try: - from importlib.metadata import version - - return version(name) - except Exception: - return "not installed" - - -def get_cpu_info() -> dict: - """Collect CPU information.""" - info = { - "model": platform.processor() or "unknown", - "count_logical": os.cpu_count() or 0, - } - try: - import psutil - - info["count_physical"] = psutil.cpu_count(logical=False) or 0 - freq = psutil.cpu_freq() - if freq: - info["freq_mhz"] = freq.current - except ImportError: - pass - return info - - -def get_ram_gb() -> float: - """Return total system RAM in GB.""" - try: - import psutil - - return round(psutil.virtual_memory().total / (1024**3), 2) - except ImportError: - return 0.0 - - -def capture_environment() -> dict: - """Build a complete environment snapshot.""" - return { - "timestamp": datetime.now(timezone.utc).isoformat(), - "hostname": platform.node(), - "python": { - "version": platform.python_version(), - "implementation": platform.python_implementation(), - "compiler": platform.python_compiler(), - }, - "os": { - "system": platform.system(), - "release": platform.release(), - "version": platform.version(), - "machine": platform.machine(), - }, - "cpu": get_cpu_info(), - "ram_gb": get_ram_gb(), - "packages": { - "freshdata-cleaner": get_package_version("freshdata-cleaner"), - "pandas": get_package_version("pandas"), - "numpy": get_package_version("numpy"), - "pyarrow": get_package_version("pyarrow"), - "asv": get_package_version("asv"), - "psutil": get_package_version("psutil"), - }, - } - - -def main() -> None: - output_path = sys.argv[1] if len(sys.argv) > 1 else "bench/artifacts/env_info.json" - output = Path(output_path) - output.parent.mkdir(parents=True, exist_ok=True) - - env = capture_environment() - output.write_text(json.dumps(env, indent=2) + "\n") - print(f"Environment captured → {output}") - print(json.dumps(env, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/freshdata-benchmarks/scripts/generate_plots.py b/freshdata-benchmarks/scripts/generate_plots.py deleted file mode 100644 index 72eb9a42..00000000 --- a/freshdata-benchmarks/scripts/generate_plots.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Generate visualization plots from ASV benchmark results.""" - -import json -import os -import pathlib -import sys -from collections import defaultdict - -import matplotlib.pyplot as plt -import numpy as np - -PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[1] -REPORTS_DIR = PROJECT_ROOT / "reports" -PLOTS_DIR = REPORTS_DIR / "plots" - - -def load_extracted_results(): - json_path = REPORTS_DIR / "results_extracted.json" - if not json_path.exists(): - print(f"Error: {json_path} not found.", file=sys.stderr) - print("Run 'python scripts/generate_reports.py' first.", file=sys.stderr) - sys.exit(1) - - with open(json_path) as f: - return json.load(f) - - -def plot_scaling_curves(results): - """Plot execution time vs dataset size for the full pipeline.""" - pipeline_data = results.get("BenchmarkFreshDataVsPandas", {}).get("time_full_pipeline", {}) - if not pipeline_data: - print("No BenchmarkFreshDataVsPandas data found. Skipping scaling curves.") - return - - row_sizes_str = pipeline_data.get("params", [])[0] - libraries = pipeline_data.get("params", [])[1] - raw_results = pipeline_data.get("result", []) - - if not row_sizes_str or not libraries or not raw_results: - return - - row_sizes = [int(s.replace("_", "")) for s in row_sizes_str] - - plt.figure(figsize=(10, 6)) - - idx = 0 - for lib in libraries: - times = [] - for _ in row_sizes: - val = raw_results[idx] - times.append(val if val is not None and str(val) != "nan" else np.nan) - idx += 1 - - plt.plot(row_sizes, times, marker='o', linewidth=2, label=lib) - - plt.xscale('log') - plt.yscale('log') - plt.title('Execution Time vs. Dataset Size (Full Preprocessing Pipeline)') - plt.xlabel('Number of Rows (Log Scale)') - plt.ylabel('Execution Time in Seconds (Log Scale)') - plt.grid(True, which="both", ls="--", alpha=0.6) - plt.legend() - - out_path = PLOTS_DIR / "scaling_curves.png" - plt.savefig(out_path, dpi=300, bbox_inches='tight') - print(f"Saved: {out_path}") - plt.close() - - -def plot_speedup_bar(results): - """Plot speedup vs Pandas for different operations on the largest dataset.""" - # Find operations to plot - ops_to_plot = { - "Load CSV": ("BenchmarkFreshDataVsPandas", "time_load_csv"), - "Missing Values": ("BenchmarkFreshDataVsPandas", "time_handle_missing"), - "Outliers": ("BenchmarkFreshDataVsPandas", "time_detect_outliers"), - "Duplicates": ("BenchmarkFreshDataVsPandas", "time_resolve_duplicates"), - "Group Agg": ("BenchmarkFreshDataVsPandas", "time_group_aggregations"), - "Full Pipeline": ("BenchmarkFreshDataVsPandas", "time_full_pipeline"), - } - - speedups = defaultdict(dict) - - for op_name, (group, method) in ops_to_plot.items(): - data = results.get(group, {}).get(method, {}) - if not data: - continue - - libraries = data.get("params", [])[1] - raw_results = data.get("result", []) - row_sizes = data.get("params", [])[0] - - if "pandas" not in libraries: - continue - - # Get data for largest dataset (last index) - pandas_idx = libraries.index("pandas") * len(row_sizes) + (len(row_sizes) - 1) - pandas_time = raw_results[pandas_idx] - - if pandas_time is None or str(pandas_time) == "nan": - continue - - for lib_idx, lib in enumerate(libraries): - if lib == "pandas": - continue - val_idx = lib_idx * len(row_sizes) + (len(row_sizes) - 1) - val_time = raw_results[val_idx] - - if val_time is not None and str(val_time) != "nan" and val_time > 0: - speedups[op_name][lib] = pandas_time / val_time - - if not speedups: - print("No speedup data available.") - return - - # Plot - fig, ax = plt.subplots(figsize=(12, 7)) - - ops = list(speedups.keys()) - # Find all unique libraries across operations - all_libs = set() - for s in speedups.values(): - all_libs.update(s.keys()) - all_libs = sorted(list(all_libs)) - - x = np.arange(len(ops)) - width = 0.8 / len(all_libs) - - for i, lib in enumerate(all_libs): - values = [speedups[op].get(lib, 0) for op in ops] - ax.bar(x + i*width - width*(len(all_libs)/2 - 0.5), values, width, label=lib) - - # Add baseline - ax.axhline(y=1.0, color='r', linestyle='-', alpha=0.3, label='Pandas Baseline (1.0x)') - - ax.set_ylabel('Speedup Ratio (Higher is Better)') - ax.set_title('Speedup vs Pandas (Largest Dataset Size)') - ax.set_xticks(x) - ax.set_xticklabels(ops, rotation=45, ha='right') - ax.legend() - - plt.tight_layout() - out_path = PLOTS_DIR / "speedup_comparison.png" - plt.savefig(out_path, dpi=300) - print(f"Saved: {out_path}") - plt.close() - - -def plot_memory_bar(results): - """Plot memory reduction vs Pandas for different operations on the largest dataset.""" - ops_to_plot = { - "Load CSV": ("BenchmarkFreshDataVsPandas", "peakmem_load_csv"), - "Missing Values": ("BenchmarkFreshDataVsPandas", "peakmem_handle_missing"), - "Outliers": ("BenchmarkFreshDataVsPandas", "peakmem_detect_outliers"), - "Duplicates": ("BenchmarkFreshDataVsPandas", "peakmem_resolve_duplicates"), - "Group Agg": ("BenchmarkFreshDataVsPandas", "peakmem_group_aggregations"), - "Full Pipeline": ("BenchmarkFreshDataVsPandas", "peakmem_full_pipeline"), - } - - memory_ratios = defaultdict(dict) - - for op_name, (group, method) in ops_to_plot.items(): - data = results.get(group, {}).get(method, {}) - if not data: - continue - - libraries = data.get("params", [])[1] - raw_results = data.get("result", []) - row_sizes = data.get("params", [])[0] - - if "pandas" not in libraries: - continue - - pandas_idx = libraries.index("pandas") * len(row_sizes) + (len(row_sizes) - 1) - pandas_mem = raw_results[pandas_idx] - - if pandas_mem is None or str(pandas_mem) == "nan" or pandas_mem == 0: - continue - - for lib_idx, lib in enumerate(libraries): - if lib == "pandas": - continue - val_idx = lib_idx * len(row_sizes) + (len(row_sizes) - 1) - val_mem = raw_results[val_idx] - - if val_mem is not None and str(val_mem) != "nan" and val_mem > 0: - memory_ratios[op_name][lib] = pandas_mem / val_mem - - if not memory_ratios: - print("No memory data available.") - return - - fig, ax = plt.subplots(figsize=(12, 7)) - ops = list(memory_ratios.keys()) - all_libs = set() - for s in memory_ratios.values(): - all_libs.update(s.keys()) - all_libs = sorted(list(all_libs)) - - x = np.arange(len(ops)) - width = 0.8 / len(all_libs) - - for i, lib in enumerate(all_libs): - values = [memory_ratios[op].get(lib, 0) for op in ops] - ax.bar(x + i*width - width*(len(all_libs)/2 - 0.5), values, width, label=lib) - - ax.axhline(y=1.0, color='r', linestyle='-', alpha=0.3, label='Pandas Baseline (1.0x Memory)') - - ax.set_ylabel('Memory Efficiency Ratio (Higher is Better, 2.0 = half memory)') - ax.set_title('Memory Efficiency vs Pandas (Largest Dataset Size)') - ax.set_xticks(x) - ax.set_xticklabels(ops, rotation=45, ha='right') - ax.legend() - - plt.tight_layout() - out_path = PLOTS_DIR / "memory_comparison.png" - plt.savefig(out_path, dpi=300) - print(f"Saved: {out_path}") - plt.close() - - -def generate_plots(): - """Main entry point.""" - PLOTS_DIR.mkdir(exist_ok=True, parents=True) - results = load_extracted_results() - - plot_scaling_curves(results) - plot_speedup_bar(results) - plot_memory_bar(results) - - -if __name__ == "__main__": - generate_plots() diff --git a/freshdata-benchmarks/scripts/generate_reports.py b/freshdata-benchmarks/scripts/generate_reports.py deleted file mode 100644 index 35085b77..00000000 --- a/freshdata-benchmarks/scripts/generate_reports.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Generate Markdown and CSV reports from ASV benchmark results. - -Reads the latest JSON results generated by ASV and formats them into -human-readable tables, saving them to the 'reports' directory. -""" - -import json -import os -import pathlib -import sys -from collections import defaultdict -from typing import Any - -from tabulate import tabulate - -PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[1] -RESULTS_DIR = PROJECT_ROOT / ".asv" / "results" -REPORTS_DIR = PROJECT_ROOT / "reports" - - -def find_latest_results() -> dict[str, Any]: - """Find the most recent ASV results JSON file.""" - if not RESULTS_DIR.exists(): - print(f"Error: Results directory not found at {RESULTS_DIR}", file=sys.stderr) - print("Run 'asv run' first.", file=sys.stderr) - sys.exit(1) - - # ASV stores results under .asv/results/MACHINE/COMMIT.json - # We want the most recently modified JSON file - json_files = list(RESULTS_DIR.rglob("*.json")) - # Filter out machine.json - result_files = [f for f in json_files if f.name != "machine.json"] - - if not result_files: - print("Error: No benchmark results found.", file=sys.stderr) - sys.exit(1) - - latest_file = max(result_files, key=lambda p: p.stat().st_mtime) - print(f"Reading results from: {latest_file.relative_to(PROJECT_ROOT)}") - - with open(latest_file) as f: - return json.load(f) - - -def format_time(seconds: float | None) -> str: - """Format time in seconds to human readable string.""" - if seconds is None or str(seconds) == "nan": - return "N/A" - - if seconds < 0.001: - return f"{seconds * 1_000_000:.1f} μs" - elif seconds < 1.0: - return f"{seconds * 1000:.1f} ms" - else: - return f"{seconds:.2f} s" - - -def format_memory(bytes_val: float | None) -> str: - """Format memory in bytes to human readable string.""" - if bytes_val is None or str(bytes_val) == "nan": - return "N/A" - - mb = bytes_val / (1024 * 1024) - if mb > 1024: - return f"{mb / 1024:.2f} GB" - return f"{mb:.1f} MB" - - -def format_value(value: float | None, bench_type: str) -> str: - """Format value based on benchmark type.""" - if value is None or str(value) == "nan": - return "-" - - if bench_type.startswith("time_"): - return format_time(value) - elif bench_type.startswith("peakmem_"): - return format_memory(value) - elif bench_type.startswith("track_throughput"): - if value > 1_000_000: - return f"{value / 1_000_000:.1f}M/s" - elif value > 1_000: - return f"{value / 1_000:.1f}K/s" - return f"{value:.0f}/s" - elif bench_type.startswith("track_speedup"): - return f"{value:.1f}x" - else: - # Default numeric format - if isinstance(value, float): - return f"{value:.2f}" - return str(value) - - -def generate_reports() -> None: - """Generate Markdown and CSV reports.""" - REPORTS_DIR.mkdir(exist_ok=True) - results = find_latest_results() - - # Restructure data: bench_name -> {param_combo -> result_value} - # ASV stores results as a dict where keys are benchmark names - # and values are dicts containing 'params' and 'result' - - reports_md = ["# FreshData Benchmark Results\n\n"] - - # Group benchmarks by file (e.g. benchmark_missing) - bench_groups = defaultdict(dict) - - for bench_name, bench_data in results.get("results", {}).items(): - if not isinstance(bench_data, dict) or "result" not in bench_data: - continue - - parts = bench_name.split(".") - if len(parts) < 3: - continue - - group_name = parts[-2] # e.g., BenchmarkMissing - method_name = parts[-1] # e.g., time_drop_missing - - bench_groups[group_name][method_name] = bench_data - - # Generate a table for each group - for group_name, methods in sorted(bench_groups.items()): - reports_md.append(f"## {group_name}\n") - - for method_name, data in sorted(methods.items()): - reports_md.append(f"### `{method_name}`\n") - - param_names = data.get("params", []) - # Skip non-parameterized or oddly parameterized methods for simple tables - if not param_names or len(param_names) != 2: - reports_md.append("_Requires 2 parameters (n_rows, library) for standard formatting._\n\n") - continue - - # Usually: params = [["10000", "100000"], ["freshdata", "pandas", ...]] - row_sizes = data.get("params")[0] - libraries = data.get("params")[1] - - result_list = data.get("result", []) - - # ASV results list is flat, ordered by cartesian product of params - headers = ["Library"] + [f"{size} rows" for size in row_sizes] - table_data = [] - - idx = 0 - for lib in libraries: - row = [lib] - for _ in row_sizes: - val = result_list[idx] if idx < len(result_list) else None - row.append(format_value(val, method_name)) - idx += 1 - table_data.append(row) - - md_table = tabulate(table_data, headers=headers, tablefmt="github") - reports_md.append(md_table + "\n\n") - - # Write Markdown report - md_path = REPORTS_DIR / "summary.md" - with open(md_path, "w") as f: - f.write("".join(reports_md)) - - print(f"Generated Markdown report: {md_path}") - - # Save raw structured data - json_path = REPORTS_DIR / "results_extracted.json" - with open(json_path, "w") as f: - json.dump(bench_groups, f, indent=2) - - print(f"Generated JSON extract: {json_path}") - - -if __name__ == "__main__": - generate_reports() diff --git a/freshdata-benchmarks/setup.py b/freshdata-benchmarks/setup.py deleted file mode 100644 index eb1add3c..00000000 --- a/freshdata-benchmarks/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Minimal setup.py for ASV virtualenv detection.""" - -from setuptools import setup - -setup( - name="freshdata-benchmarks", - version="0.1.0", - description="ASV benchmark suite for FreshData", - python_requires=">=3.9", -) diff --git a/growth-audit.md b/growth-audit.md deleted file mode 100644 index a8e44d21..00000000 --- a/growth-audit.md +++ /dev/null @@ -1,56 +0,0 @@ -# FreshData Open-Source Growth & Repository Baseline Audit - -This audit evaluates the baseline state of `FreshCode-Org/freshdata` as of September 2026, based strictly on empirical evidence inspected across the codebase, packaging metadata, documentation, GitHub configurations, benchmarks, and test suites. - ---- - -## 1. Executive Summary - -| Area | Grade | Key Strengths | Critical Gaps | -|---|:---:|---|---| -| **Repository Positioning** | C+ | Strong engineering foundation; accurate claims in docs | README buries value below fold; lacks 10-second mental model; no before/after visual proof table | -| **Developer Onboarding** | B- | Fast local test execution; Python 3.9–3.13 support | PyPI package name (`freshdata-cleaner`) differs from import (`freshdata`), creating install friction | -| **Discoverability** | C | Quality keywords in `pyproject.toml` | Missing high-intent developer topics on GitHub; no searchable cookbook for SEO | -| **Community & Contribution** | C+ | High-quality `CONTRIBUTING.md` and 93% test coverage gate | Contributor roadmap references already-closed issues; no beginner walkthrough (`first-contribution.md`) | -| **Proof & Trust** | B+ | 9 standardized metrics; reproducible seed-controlled harness | Benchmark README lacks scenario breakdown and side-by-side invariant safety proof | -| **Distribution & Advocacy** | D+ | Well-written technical documentation site | No marketing assets, no social post engine, no structured community release cadence | - ---- - -## 2. Detailed Audit Matrix - -| Area | Current State | Evidence | Problem | Impact | Effort | Priority | Recommended Action | -|---|---|---|---|---|---|:---:|---| -| **Positioning** | Tagline: "The explainable cleaning layer for pandas — decision-preserving data hygiene." | `README.md:7` | "Decision-preserving data hygiene" is abstract jargon; doesn't highlight Polars or explain what the tool *does* in 10s. | High drop-off from search & GitHub discovery. | Low | **P0** | Rewrite to: "Automated, explainable data cleaning for pandas and Polars — repair messy tabular data safely and see exactly what changed." | -| **Positioning** | README hero lacks 10-second category mental model. | `README.md:26-38` | Visitors don't immediately know how FreshData relates to Profilers vs Validators vs ML quality tools. | Confusion about whether FreshData replaces pandas, Great Expectations, or ydata-profiling. | Low | **P0** | Add comparison matrix: Profilers (describe), Validators (test rules), ML-DQ (detect issues), FreshData (safely repair & explain). | -| **Positioning** | No before/after visual demonstration. | `README.md:91-109` | Current quickstart shows only a summary text without showing the messy input DataFrame and resulting output. | Visitors cannot visualize the transformation or trust the result without running code. | Low | **P0** | Insert a compact, tabular Before $\rightarrow$ Detection $\rightarrow$ Decision $\rightarrow$ Repair $\rightarrow$ Audit block. | -| **Onboarding** | Package naming mismatch: PyPI is `freshdata-cleaner`, import is `freshdata`. | `pyproject.toml:7-9` | Users typing `pip install freshdata` fail or get wrong package; README warning is buried under install code block. | Direct drop-off during first onboarding step. | Low | **P0** | Place high-visibility installation callout banner at the very top of README and docs. | -| **Onboarding** | First-time contributor path requires reading full architecture. | `CONTRIBUTING.md:11-12`, `docs/community/first-pr.md` | `first-pr.md` is an index of labels rather than a step-by-step tutorial (clone $\rightarrow$ venv $\rightarrow$ test $\rightarrow$ edit $\rightarrow$ PR). | First-time contributors hesitate to take on issues. | Medium | **P1** | Create `docs/contributing/first-contribution.md` with a zero-friction, copy-paste workflow. | -| **Discoverability** | GitHub Topics on repo are minimal/unoptimized. | GitHub metadata & `pyproject.toml:18-35` | Repository is not ranking for high-intent search terms like `data-cleaning`, `missing-values`, `data-quality`, `polars`. | Low organic GitHub search discovery. | Low | **P0** | Update GitHub repository topics with 15 developer search intent keywords (documented in `admin-actions.md`). | -| **Discoverability** | No problem-based searchable cookbook. | `docs/` directory | Documentation is organized by library subpackage rather than developer search queries. | Developers searching "how to handle missing values pandas" do not land on FreshData. | Medium | **P1** | Build `docs/cookbook/` with 7+ search-intent-focused problem/code/output guides. | -| **Community** | Missing issue templates for integrations and benchmarks. | `.github/ISSUE_TEMPLATE/` | Only `bug_report.yml`, `feature_request.yml`, and `documentation.yml` exist. No structured way to report benchmarks or request ecosystem integrations. | Reduced contributor engagement on performance and ecosystem growth. | Low | **P1** | Add `integration_request.yml` and `benchmark_result.yml` issue forms. | -| **Community** | PR template lacks performance and compatibility impact prompts. | `.github/pull_request_template.md` | PR template does not prompt contributors to document performance impact, backward compatibility, or docs updates. | Review overhead for maintainers; accidental performance regressions. | Low | **P1** | Update PR template with structured questions (*Why?*, *How tested?*, *Performance impact?*, *Docs updated?*). | -| **Community** | Contributor roadmap references merged/closed issues. | `docs/community/contributor-roadmap.md:34,88-92,107` | Mentions `clean_csv` (#19), `pyjanitor` (#9), `duckdb out-of-core` (#52, #53), which are already merged. | Contributors waste time investigating already-solved tasks or lose confidence in roadmap. | Medium | **P1** | Overhaul `contributor-roadmap.md` with 10+ fresh, verified open contributor tasks across Level 1, 2, and 3. | -| **Proof & Safety** | Safety invariants (refusal to touch IDs, target columns) not demonstrated visibly. | `README.md`, `benchmarks/` | Users fear automated cleaners will corrupt primary keys, leak target labels, or silently invent fake data. | Adoption resistance in production environments. | Medium | **P0** | Add "Safety / Trust Proof" section with concrete examples of what FreshData explicitly refuses to alter. | -| **Proof & Safety** | Benchmark README lacks execution commands and environment details. | `benchmarks/README.md` | Benchmarks are mentioned, but reproducibility steps and hardware requirements are brief. | Developers doubt performance claims without clear local reproduction commands. | Medium | **P1** | Overhaul `benchmarks/README.md` documenting datasets, 9 standardized metrics, runner commands, and limitations. | -| **Examples** | Flagship examples in `examples/` are fragmented and lack numbered consistency. | `examples/` directory | Missing `01_csv_cleaning.py` and `03_duplicate_detection.py`; integrations are scattered. | Inconsistent developer trial experience. | Medium | **P1** | Standardize `examples/01` to `05` and add dedicated `examples/integrations/` recipes. | -| **Distribution** | No structured developer marketing or launch materials. | Codebase root | No drafted social posts, demo scripts, blog post outlines, or launch checklists. | Inability to launch a coordinated developer advocacy push on HN, Reddit, Twitter/X, and PyData. | Medium | **P1** | Create `marketing/` directory containing 6 ready-to-publish assets and a community release cadence. | -| **Metrics** | No defined framework for tracking adoption and contributor velocity. | Project documentation | Growth is measured only by vanity star counts rather than the activation funnel (Discovery $\rightarrow$ Activation $\rightarrow$ Contribution). | Misaligned maintainer priorities. | Low | **P1** | Create `docs/growth/metrics.md` with explicit 30-day, 90-day, and 12-month traction milestones. | - ---- - -## 3. Priority Action Plan - -### P0 (Immediate Lever — Complete in Phase 1–3) -1. Rewrite repository description and README hero to convey the **10-second value proposition**. -2. Make package installation (`pip install freshdata-cleaner`) unmistakable with import disambiguation. -3. Insert canonical **Before / After** demonstration and **Safety Proof** into README. -4. Add the **"Why FreshData?"** comparison table (Profilers vs. Validators vs. FreshData). - -### P1 (High Leverage — Complete in Phase 4–13) -1. Add missing GitHub issue templates (`integration-request`, `benchmark-result`) and update PR template. -2. Publish `docs/contributing/first-contribution.md` (zero-friction 7-step guide) and overhaul `contributor-roadmap.md`. -3. Standardize flagship examples (`01_csv_cleaning`, `02_missing_values`, `03_duplicate_detection`, `04_outlier_handling`, `05_ml_preprocessing`, and integrations). -4. Expand `benchmarks/README.md` with complete reproducibility protocols and failure case disclosures. -5. Create `docs/cookbook/` with 7+ high-intent SEO problem/solution recipes. -6. Assemble the `marketing/` distribution toolkit and `docs/growth/metrics.md` framework. diff --git a/marketing/README.md b/marketing/README.md deleted file mode 100644 index 46b3b1a4..00000000 --- a/marketing/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# FreshData Developer Advocacy & Distribution System - -This directory contains developer advocacy, technical marketing, and organic community distribution assets for **FreshData**. - -Our marketing philosophy is strictly **evidence-based and problem-first**: -* We never make unsubstantiated claims or market unreleased features. -* We never attack competitors or claim blanket superiority. -* We do not engage in star exchanges, paid stars, or engagement manipulation. -* We speak directly to developers and data engineers with working code, before/after evidence, and measurable benchmarks. - ---- - -## Directory Contents - -| Asset File | Purpose & Audience | -|---|---| -| [`social-posts.md`](social-posts.md) | Ready-to-publish technical posts for Twitter/X, LinkedIn, and Reddit (r/Python, r/datascience, r/dataengineering). | -| [`blog-ideas.md`](blog-ideas.md) | Outlines and code foundations for deep technical engineering blog posts. | -| [`demo-scripts.md`](demo-scripts.md) | 30-second elevator pitch and 2-minute live demo scripts for meetups, videos, and conference talks. | -| [`launch-assets.md`](launch-assets.md) | Announcement copy and checklists for Hacker News (*Show HN*), Reddit, and PyData newsletters. | -| [`contributor-posts.md`](contributor-posts.md) | Templates for contributor spotlights, "Good First Issue" roundups, and milestone celebrations. | - ---- - -## Community Release & Content Cadence - -Consistency matters more than volume. FreshData follows a sustainable 3-day weekly cadence: - -```text - MONDAY WEDNESDAY FRIDAY -┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐ -│ Roadmap & Issues │ │ Engineering Insight │ │ Community & Release │ -│ - Good First Issues │ ──► │ - 30s Video Demo │ ──► │ - Release Notes │ -│ - Architecture goal │ │ - Failure case post │ │ - Contributor credit │ -│ - RFC discussion │ │ - Benchmark deep- │ │ - Benchmark results │ -│ │ │ dive │ │ │ -└──────────────────────┘ └──────────────────────┘ └──────────────────────┘ -``` - -* **Monday — Roadmap & Community Issues**: Highlight 2–3 active issues across Level 1 and Level 2 difficulty, inviting new contributors with direct links to `docs/contributing/first-contribution.md`. -* **Wednesday — Engineering Insights & Demos**: Share a technical nuance (e.g. why FreshData refuses to touch ID columns, or how DuckDB relation cleaning works) accompanied by runnable code. -* **Friday — Release & Contributor Spotlight**: Celebrate merged PRs, welcome first-time contributors by name, and publish new benchmark figures. diff --git a/marketing/blog-ideas.md b/marketing/blog-ideas.md deleted file mode 100644 index 22ef572b..00000000 --- a/marketing/blog-ideas.md +++ /dev/null @@ -1,57 +0,0 @@ -# Engineering Blog Post Outlines & Technical Backlog - -High-depth technical articles for publication on Medium (Towards Data Science), Dev.to, Hacker Noon, and the FreshData engineering blog. - ---- - -## 1. "Why Blind `fillna()` Silently Corrupts Production ML Models" - -* **Category**: Problem & Architecture -* **Target Audience**: Data scientists, ML engineers -* **Key Narrative**: - * The illusion of the clean dataset: why zero-NaN counts give a false sense of security. - * Real-world disaster stories: imputing customer keys leading to fraudulent account linkages; imputing patient targets causing false test evaluations. - * Statistical distortion: comparing mean vs median vs mode on log-normal wealth/income distributions. - * The solution: Role-aware schema profiling and preserving ambiguity when confidence is low. -* **Runnable Code Asset**: `examples/02_missing_values.py` and `examples/05_ml_preprocessing.py`. - ---- - -## 2. "How FreshData Decides What NOT to Change: The Engineering of Safety Invariants" - -* **Category**: Engineering Deep Dive -* **Target Audience**: Senior data engineers, software architects -* **Key Narrative**: - * Why automated cleaners fail: aggressive heuristic overreach. - * The architecture of FreshData's decision engine: - * Stage 1: Representation repairs (whitespace, sentinels, snake_case). - * Stage 2: Profiler evaluation (skewness, uniqueness, cardinality). - * Stage 3: Role inference (`id`, `target`, `categorical`, `numeric`, `date`). - * Stage 4: Safety gate verification. - * Testing for invariants: how we enforce a 0.0% false-repair rate across 100k-row test suites. -* **Runnable Code Asset**: `tests/test_audit_core_defaults.py` and `tests/test_guard_protected.py`. - ---- - -## 3. "Cleaning 25 Million Rows Past RAM: How We Built DuckDB and Polars Out-of-Core Execution" - -* **Category**: Benchmarks & Performance -* **Target Audience**: Performance engineers, big data developers -* **Key Narrative**: - * The memory wall: why a 2 GB CSV consumes 8 GB in pandas RAM. - * Evaluating streaming and chunking architectures vs. Arrow-native engines. - * DuckDB and Polars execution backends: memory footprint comparisons (200 MB vs 1,046 MB at 1M rows). - * Transparent fallback policies: `fallback_policy="warn" | "error"` ensuring developers are never surprised by materialization. -* **Runnable Code Asset**: `benchmarks/bench_outofcore.py` and `examples/integrations/duckdb_workflow.py`. - ---- - -## 4. "Building an Audit Trail for Automated Data Cleaning in Regulated Industries" - -* **Category**: Enterprise & Governance -* **Target Audience**: FinTech, HealthTech, and compliance engineers -* **Key Narrative**: - * Regulatory requirements (BCBS 239, GDPR, HIPAA) around data lineage and automated modifications. - * Transitioning from black-box transformations to auditable Action records. - * Exporting `CleanReport` to machine-readable JSON for compliance archives. -* **Runnable Code Asset**: `examples/01_csv_cleaning.py` and `examples/interactive_html_reports.py`. diff --git a/marketing/contributor-posts.md b/marketing/contributor-posts.md deleted file mode 100644 index 5eeee889..00000000 --- a/marketing/contributor-posts.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Recognition & Community Growth Copy - -Templates and copy for recognizing open-source contributors, announcing Good First Issues, and celebrating milestones. - ---- - -## 1. Contributor Spotlight Template (Friday Community Post) - -**Platform**: LinkedIn & Twitter/X -**Audience**: Open-source community - -```text -🎉 Huge shoutout to @ for their contribution to FreshData this week! - -@ implemented PR # (): - - -Why this matters: -<1 sentence on user impact, e.g. "Data teams in the UK can now clean customer address tables without writing custom regex."> - -Check out their work: -https://github.com/FreshCode-Org/freshdata/pull/ - -Want to get involved? We have beginner-friendly issues waiting: -https://freshcode-org.github.io/freshdata/contributing/first-contribution/ -``` - ---- - -## 2. Weekly "Good First Issue" Roundup (Monday Post) - -**Platform**: Twitter/X & Reddit /r/opensource - -```text -Looking for a clean, impactful open-source Python contribution this week? 🐍 - -FreshData (automated tabular data cleaning for pandas & Polars) has 3 freshly scoped issues ready for contributors: - -1. 🟢 Level 1 (20m): Add DuckDB Parquet recipe in examples/integrations - Link: https://github.com/FreshCode-Org/freshdata/issues/ - -2. 🟢 Level 1 (30m): Add Unicode non-breaking space test case in tests/test_strings.py - Link: https://github.com/FreshCode-Org/freshdata/issues/ - -3. 🟡 Level 2 (1-2h): Add German postal code regex in fieldcheck.py - Link: https://github.com/FreshCode-Org/freshdata/issues/ - -Our First Contribution Guide walks you through setup in 15 minutes: -https://freshcode-org.github.io/freshdata/contributing/first-contribution/ - -Drop a comment on an issue to claim it! -``` - ---- - -## 3. GitHub Star Milestone Post (Contextual & Organic) - -**Platform**: Twitter/X & LinkedIn -**Guiding Principle**: Never celebrate vanity stars in isolation; connect milestones to developer adoption and real problem solving. - -```text -🚀 100 developers are now using FreshData to clean pandas and Polars DataFrames! - -When we launched FreshData, our thesis was simple: -Developers don't need another profiler to tell them their data is dirty. They need a tool that safely repairs it and explains what it changed. - -Since launch: -- 10+ community recipes added across Polars, DuckDB, scikit-learn, and Airflow. -- Verified 0.0% false-repair rate across 100,000-row benchmark suites. -- 93% test coverage enforced on every PR. - -Thank you to every developer who tried the library, filed an issue, or opened a PR. - -If FreshData saved you time this week, give us a star to help other engineers discover it: -https://github.com/FreshCode-Org/freshdata -``` diff --git a/marketing/demo-scripts.md b/marketing/demo-scripts.md deleted file mode 100644 index eee41e22..00000000 --- a/marketing/demo-scripts.md +++ /dev/null @@ -1,80 +0,0 @@ -# FreshData Live Demo Scripts (30-Second & 2-Minute) - -Scripts for live coding demonstrations at PyData meetups, YouTube/Loom walkthroughs, conference lightning talks, and developer tutorials. - ---- - -## 30-Second Elevator Demo: "The One-Line Audit" - -**Objective**: Hook the viewer by demonstrating immediate value on a dirty dataset in under 30 seconds. - -### Script & Actions - -1. **Speaker**: - > *"Every time you ingest a raw CSV into pandas, you spend 20 minutes writing regex for whitespace, fixing currency signs, and replacing 'N/A' sentinels. Watch this."* - -2. **Action (Terminal or IPython)**: - ```python - import pandas as pd - import freshdata as fd - - df = pd.read_csv("examples/data/messy_customers.csv") - cleaned, report = fd.clean(df, return_report=True) - print(report.summary()) - ``` - -3. **Speaker**: - > *"In one line, FreshData fixed the headers, parsed the dates, stripped padding, and flagged outliers. But notice this: it didn't touch customer IDs, and it gives you a complete audit log showing every cell it changed, why, and its confidence score."* - -4. **Action**: - ```python - print(cleaned.head(3)) - ``` - -5. **Closing**: - > *"Fast, explainable, and safe. `pip install freshdata-cleaner`."* - ---- - -## 2-Minute Deep Dive Demo: "From Raw Data to ML-Ready with Safety Invariants" - -**Objective**: Walk a technical developer through a full end-to-end workflow highlighting safety invariants and Polars support. - -### Timeline - -* **0:00 – 0:30: The Messy Input Problem** - * Open Jupyter or VS Code with a raw dataset: - * Uncleaned headers (`" Annual Income "`) - * Dirty sentinels (`"N/A"`, `"-"`, `"missing"`) - * Imbalanced outliers (`age = -5`, `age = 165`) - * Target column (`churn`) with missing values - * Speaker: *"If you run a blanket fillna or drop_duplicates, you leak target data and delete valid events."* - -* **0:30 – 1:00: Clean with Target Protection & Review Plan** - * Show the interactive clean call: - ```python - cleaned, report = fd.clean(df, target_column="churn", return_report=True) - ``` - * Point out the summary output: - * Shows rows before/after - * Shows columns before/after - * Flags age outliers in `age_outlier` boolean column rather than deleting rows. - * Explains that `churn` was preserved because it's a target label. - -* **1:00 – 1:30: Inspecting the Action Audit Log** - * Show `report.actions`: - ```python - for a in report.actions: - print(f"[{a.step}] {a.column}: {a.description} (risk={a.risk})") - ``` - * Speaker: *"Every action has a risk score. If a decision has medium or high risk, FreshData explicitly flags it in `report.recommendations` for human review."* - -* **1:30 – 2:00: Native Polars Execution** - * Switch to Polars: - ```python - import polars as pl - pl_df = pl.from_pandas(df) - clean_pl = fd.clean(pl_df) - assert isinstance(clean_pl, pl.DataFrame) - ``` - * Speaker: *"Works identically on Polars DataFrames with zero pandas conversion overhead. Check out the open-source repository at github.com/FreshCode-Org/freshdata."* diff --git a/marketing/launch-assets.md b/marketing/launch-assets.md deleted file mode 100644 index 10b7eb19..00000000 --- a/marketing/launch-assets.md +++ /dev/null @@ -1,97 +0,0 @@ -# FreshData Community Launch Assets (Show HN, Reddit, PyData) - -Launch copy and post drafts prepared for major technical developer communities. - ---- - -## 1. Hacker News: "Show HN: FreshData — Automated, explainable data cleaning for pandas and Polars" - -**Title**: `Show HN: FreshData – Automated, explainable data cleaning for pandas and Polars` -**URL**: `https://github.com/FreshCode-Org/freshdata` - -**Post Body**: - -```text -Hi HN, - -We built FreshData (https://github.com/FreshCode-Org/freshdata) because we were tired of writing the same dozens of fragile data-cleaning helper functions every time we ingested a CSV or SQL export into pandas or Polars. - -Most existing data quality tools either profile data (telling you what's wrong without fixing it) or validate data against rigid rules (telling you that a test failed, but not repairing it). - -FreshData aims to bridge that gap: one call cleans your DataFrame, but crucially, it explains what it changed and why. - -A quick example: -```python -import pandas as pd -import freshdata as fd - -df = pd.read_csv("export.csv") -cleaned, report = fd.clean(df, return_report=True) -print(report.summary()) -``` - -What makes FreshData different: -1. Decision engine, not blind fillna: It infers column roles (identifiers, categoricals, numeric, targets). Identifiers (like customer_id) are never imputed, and target labels are locked to prevent ML leakage. -2. Explainability: Every cell modification outputs an Action record with a rationale, a risk level, and a confidence score. If an action is high-risk, it recommends human review rather than guessing silently. -3. Outliers are non-destructive: Outliers are flagged in a boolean column by default rather than silently dropping rows. -4. Native Polars support: Pass a Polars DataFrame and get a Polars DataFrame back with zero pandas boilerplate. - -We enforce a 93% test coverage gate in CI and maintain a reproducible benchmark suite across 10k to 25M rows (detailed in benchmarks/README.md). - -The PyPI distribution is `freshdata-cleaner` (due to namespace collisions with an older project), but imports as `import freshdata as fd`. - -We'd love feedback on our decision engine heuristics and how it behaves on your messiest CSVs! -``` - ---- - -## 2. Reddit r/Python & r/dataengineering Launch Post - -**Title**: `[P] FreshData – Automated, explainable data cleaning for pandas and Polars (with audit trails)` - -**Post Body**: - -```text -Hey everyone, - -Whenever data pipelines break, it's rarely due to complex ML model errors — it's usually unhandled sentinel strings ("N/A", "missing", "-"), messy padding whitespace, unparsed currency symbols, or accidental duplicate rows. - -We created FreshData 2.0 to automate tabular data hygiene safely: -GitHub: https://github.com/FreshCode-Org/freshdata -Docs: https://freshcode-org.github.io/freshdata/ - -### What it does: -- One call (`fd.clean(df)`) standardizes headers into snake_case, strips whitespace, parses dates/currency, and handles missingness. -- Never mutates primary keys or target labels. -- Dual-engine: Works seamlessly on both pandas and Polars DataFrames (`fd.clean(polars_df)` returns a Polars DataFrame). -- Produces an audit report you can serialize to JSON for compliance and data lineage. -- Includes declarative pipelines: `pipe = fd.pipeline().strip_whitespace().normalize_sentinels().run(df)`. - -### How to try it: -```bash -pip install freshdata-cleaner -``` - -```python -import freshdata as fd -cleaned, report = fd.clean(df, return_report=True) -print(report.summary()) -``` - -We wrote a full cookbook with 7+ recipes for common data engineering workflows: -https://freshcode-org.github.io/freshdata/cookbook/ - -Feedback, benchmark PRs, and issue reports are very welcome! -``` - ---- - -## 3. Launch Checklist & Timing - -| Step | Action | Timing | -|:---:|---|---| -| 1 | Ensure `main` branch is clean, CI is green, and docs are built | Day 0 (T-1h) | -| 2 | Post to Hacker News (*Show HN*) | Tuesday 08:00 AM ET / 05:00 AM PT | -| 3 | Post to Reddit (r/Python and r/dataengineering) | Tuesday 09:30 AM ET | -| 4 | Post announcement thread on Twitter/X and LinkedIn | Tuesday 10:00 AM ET | -| 5 | Actively monitor comments on HN and Reddit, responding with code citations | Hours 0 to 12 | diff --git a/marketing/social-posts.md b/marketing/social-posts.md deleted file mode 100644 index 4230ff8d..00000000 --- a/marketing/social-posts.md +++ /dev/null @@ -1,124 +0,0 @@ -# Technical Social Media Posts (Twitter/X, LinkedIn, Reddit) - -High-signal, developer-focused social copy targeting common tabular data engineering pains. - ---- - -## 1. Problem Post: The Silent Danger of Blanket `fillna()` - -**Platform**: Twitter/X & LinkedIn -**Hook**: Why blindly filling missing values in pandas corrupts your downstream machine learning models. - -```text -The most dangerous line of code in data science: - -`df.fillna(df.mean(), inplace=True)` - -Here is what that single line quietly does to your dataset: -1. Imputes customer_id with a fake float ID, fabricating ghost users and corrupting table joins. -2. Fills missing ground-truth labels in your training target, creating synthetic targets your model can't learn from. -3. Distorts variance on skewed financial columns. - -In FreshData, cleaning is role-aware: -- Identifiers are preserved (0% mutation). -- Targets are protected from leakage. -- Missing values are imputed only when statistically safe, with risk & confidence logged. - -Try it in one line: -pip install freshdata-cleaner - -import freshdata as fd -cleaned, report = fd.clean(df, return_report=True) -print(report.summary()) - -https://github.com/FreshCode-Org/freshdata -``` - ---- - -## 2. Demonstration Post: Before & After in 30 Seconds - -**Platform**: Twitter/X & LinkedIn -**Hook**: What happens when you pass a genuinely messy DataFrame to FreshData? - -```text -Raw CSV from production: -- Whitespace in headers: " Customer ID " -- Dirty sentinels: "N/A", "-", "missing" -- Unparsed currency: "$1,200.50" -- Outliers: age = -4 - -Before: - Customer ID Full Name Age Annual Spend -0 C-101 Alice Smith 29 $1200.50 -1 C-102 Bob Jones 34 N/A -2 C-103 Charlie Brown -4 missing - -After `cleaned, report = fd.clean(df, return_report=True)`: - customer_id full_name age annual_spend age_outlier -0 C-101 Alice Smith 29 1200.50 False -1 C-102 Bob Jones 34 NaN False -2 C-103 Charlie Brown -4 NaN True - -Plus an audit trail showing why each action occurred and what it refused to touch. - -Check out the interactive recipes: -https://github.com/FreshCode-Org/freshdata/tree/main/examples -``` - ---- - -## 3. Engineering Post: Why FreshData Refuses to Change Certain Data - -**Platform**: LinkedIn & Reddit r/datascience -**Hook**: The mark of a reliable automated data cleaner isn't what it cleans — it's what it refuses to touch. - -```text -When building an automated cleaner for tabular data, the hardest engineering problem isn't imputation or regex parsing. - -It's knowing when NOT to touch a cell. - -If an auto-cleaner tries to be "helpful" by force-imputing every NaN, it introduces silent data corruption. In FreshData 2.0, we enforce invariant boundaries: - -1. Identifier columns (UUIDs, customer keys): Never imputed. Fabricating a primary key is worse than leaving it null. -2. Target labels: Strictly protected against imputation to prevent leakage in ML pipelines. -3. Outliers: Flagged in a boolean `_outlier` column by default rather than silently dropped. -4. Ambiguous sentinels: Normalized to NaN and routed to `report.recommendations` for human review. - -Every action outputs an Action dataclass with `risk` (low/medium/high) and `confidence` (0.0 to 1.0). - -Read our full trust invariant spec: -https://freshcode-org.github.io/freshdata/trust-claims/ -``` - ---- - -## 4. Ecosystem Post: Polars Support - -**Platform**: Twitter/X & Reddit r/Python -**Hook**: Automated data cleaning for Polars without pandas boilerplate. - -```text -Using Polars for data engineering? - -You can now run automated, explainable data cleaning natively: - -import polars as pl -import freshdata as fd - -pl_df = pl.read_csv("messy_telemetry.csv") - -# Polars in -> Polars out -clean_pl = fd.clean(pl_df) -assert isinstance(clean_pl, pl.DataFrame) - -- Strips whitespace -- Standardizes headers to snake_case -- Normalizes "N/A", "null", and "-" sentinels -- Flags outliers in a native boolean column - -Install with Polars extra: -pip install "freshdata-cleaner[polars]" - -Repo: https://github.com/FreshCode-Org/freshdata -``` diff --git a/mkdocs.yml b/mkdocs.yml index 84d47e3c..2a03e28f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -111,7 +111,6 @@ nav: - Decision-preserving workflow: decision-workflow.md - Scalable backends: backends.md - Backend fallback matrix: fallback-matrix.md - - FreshCore architecture: freshcore.md - Data profiling: data-profiling.md - Feature overview: feature-overview.md - Streaming / micro-batch: streaming.md @@ -123,7 +122,6 @@ nav: - Writing plugins: plugins.md - Model cards: model-cards.md - Learning profiles: learning-profiles.md - - Developer training pipeline: developer-training-pipeline.md - Format parsers & tick mode: parsers.md - Text cleaning & field validation: field-validation.md - Compliance reports: compliance.md diff --git a/notebooks/06_context_cleaning.ipynb b/notebooks/06_context_cleaning.ipynb deleted file mode 100644 index 07527dab..00000000 --- a/notebooks/06_context_cleaning.ipynb +++ /dev/null @@ -1,335 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Context cleaning — rules in English, compiled\n", - "\n", - "FreshData compiles natural-language expectations into a deterministic,\n", - "reviewable `ContextPolicy` — no model, no network, no new dependency.\n", - "This notebook walks the flagship ecommerce example end to end." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "import freshdata as fd\n" - ] - }, - { - "cell_type": "markdown", - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "source": [ - "## A messy ecommerce customer extract" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame({\n", - " \"CustomerID\": [101, 102, 102, 104, 105],\n", - " \"Full Name\": [\"Asha Rao\", \"Ben Cotton\", \"Ben Cotton\", \"Dana Li\", \"Ed Fox\"],\n", - " \"Email Addr\": [\"asha@example.com\", \"ben@example.com\", \"ben@example.com\", \"not-an-email\", None],\n", - " \"Mobile\": [\"9876543210\", None, \"9123456789\", \"9111111111\", \"9000000001\"],\n", - " \"Age\": [25, None, 40, 35, None],\n", - " \"Monthly Revenue\": [1200.50, 800.00, 800.00, 99999.0, 430.25],\n", - " \"City\": [\"Pune\", \"Delhi\", \"Delhi\", \"Goa\", \"Pune\"],\n", - "})\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "source": [ - "## 1. Write the rules as English" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "outputs": [], - "source": [ - "context = \"\"\"\n", - "This is an ecommerce customer dataset.\n", - "CustomerID is unique.\n", - "Emails must be valid.\n", - "Phone numbers are Indian.\n", - "Missing Age should be estimated only if confidence >95%.\n", - "Never modify revenue values.\n", - "\"\"\"" - ] - }, - { - "cell_type": "markdown", - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "source": [ - "## 2. Compile — and inspect what FreshData understood\n", - "\n", - "`fd.compile_context` is deterministic: the same text and schema always\n", - "produce the same policy. Unresolved references and unparsed sentences are\n", - "surfaced, never guessed at." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "outputs": [], - "source": [ - "policy = fd.compile_context(context, df=df)\n", - "print(policy.summary())" - ] - }, - { - "cell_type": "markdown", - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "source": [ - "The policy is a plain JSON artifact — commit it, diff it, review it in a PR." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "print(policy.to_json()) # policy.to_json(\"policy.json\") writes the file" - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "## 3. Clean under the policy\n", - "\n", - "Protected columns are added to `preserve_columns`, unique columns become\n", - "id columns, and per-column hints flow into the semantic context. The\n", - "report records the compile and every unresolved item." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned, report = fd.clean(df, context=context, return_report=True)\n", - "cleaned" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "outputs": [], - "source": [ - "[a.description for a in report.actions if a.step == \"context\"]" - ] - }, - { - "cell_type": "markdown", - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "source": [ - "The protected column is byte-identical — \"Never modify revenue values.\" is a hard rule:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned[\"monthly_revenue\"].equals(df[\"Monthly Revenue\"])" - ] - }, - { - "cell_type": "markdown", - "id": "8a65eabff63a45729fe45fb5ade58bdc", - "metadata": {}, - "source": [ - "## 4. Validate without mutating" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3933fab20d04ec698c2621248eb3be0", - "metadata": {}, - "outputs": [], - "source": [ - "findings = fd.validate(df, context=context)\n", - "for f in findings:\n", - " print(f.severity.upper(), f.rule_name, \"—\", f.message)" - ] - }, - { - "cell_type": "markdown", - "id": "4dd4641cc4064e0191573fe9c69df29b", - "metadata": {}, - "source": [ - "## 5. Strict mode\n", - "\n", - "In CI you usually want gaps to fail loudly. `strict=True` raises\n", - "`fd.PolicyError` on any unresolved or unparsed line, before touching data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8309879909854d7188b41380fd92a7c3", - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " fd.clean(df, context=context + \"\\nAlso make everything shiny.\", strict=True)\n", - "except fd.PolicyError as err:\n", - " print(err)" - ] - }, - { - "cell_type": "markdown", - "id": "3ed186c9a28b402fb0bc4494df01f08d", - "metadata": {}, - "source": [ - "## 6. Reuse the compiled policy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cb1e1581032b452c9409d6c6813c49d1", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned_again = fd.clean(df, policy=policy) # skips parsing entirely\n", - "cleaned_again.equals(cleaned)" - ] - }, - { - "cell_type": "markdown", - "id": "379cbbc1e968416e875cc15c1202d7eb", - "metadata": {}, - "source": [ - "## Phase 2: the context is now enforced end to end\n", - "\n", - "Phase 2 adds deterministic value experts and hard safety gates behind the\n", - "same context sentences:\n", - "\n", - "- `Emails must be valid.` → the **EmailExpert** mechanically repairs\n", - " unambiguous addresses (`\" Bob@GMAIL.COM \"` → `\"Bob@gmail.com\"`,\n", - " `\"a @ b.com\"` → `\"a@b.com\"`, one doubled `@@`) and flags everything else.\n", - "- `Phone numbers are Indian.` → the **PhoneExpert** normalizes safe forms to\n", - " canonical `+91XXXXXXXXXX` and flags the rest.\n", - "- `Allowed status values are ...` → the **ReferenceExpert** repairs\n", - " case/whitespace/separator variants exactly, *suggests* single-candidate\n", - " typos, and blocks ambiguous ones.\n", - "- `Missing Age should be estimated only if confidence >95%.` → the\n", - " statistical engine now refuses fills below the threshold and records a\n", - " suggested action instead.\n", - "- `Never modify revenue values.` → enforced **physically**: the executor\n", - " verifies the column byte-identical before returning, and raises\n", - " `fd.ProtectedColumnError` if any code path broke the promise." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "277c27b1587741f2af2001be3712ef0d", - "metadata": {}, - "outputs": [], - "source": [ - "clean_df, report = fd.clean(\n", - " df,\n", - " context=context,\n", - " semantic_mode=\"auto\",\n", - " return_report=True,\n", - " verbose=False,\n", - ")\n", - "clean_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "db7b79bc585a40fcaf58bf750017e135", - "metadata": {}, - "outputs": [], - "source": [ - "# Every decision is audited: applied, suggested, skipped — with rationale.\n", - "for action in report:\n", - " if action.step in (\"semantic\", \"missing\", \"guard\"):\n", - " print(f\"[{action.step:8}] {str(action.column):16} {action.status:10} {action.description}\")" - ] - }, - { - "cell_type": "markdown", - "id": "916684f9a58a4a2aa5f864670399430d", - "metadata": {}, - "source": [ - "The protected `monthly_revenue` column is byte-identical, missing `age`\n", - "survived (0.9 fill confidence < 0.95 required), and the duplicate\n", - "`cust_id` shows up in `fd.validate` — reported, not silently dropped.\n", - "\n", - "Want to review the repairs *before* they run? That is the repair-plan\n", - "workflow: see `notebooks/08_plan_approval.ipynb` and\n", - "`docs/repair-plans.md`." - ] - }, - { - "cell_type": "markdown", - "id": "1671c31a24314836a5b85d7ef7fbf015", - "metadata": {}, - "source": [ - "## Limitations\n", - "\n", - "- **Deterministic tier-0 lexicon only** — a fixed set of ~12 intent\n", - " families; anything outside it is surfaced as *unparsed*, by design.\n", - "- No model, no embeddings, no network — and therefore no arbitrary NLU.\n", - "- `valid_format` / `locale_format` now drive real value repairs (email, and\n", - " phones for `region=\"IN\"`); other phone regions compile but do not repair yet.\n", - "- Ambiguous repairs are suggested or flagged, never auto-applied.\n", - "- `drop_if` / `rename` / `map` rules compile into the policy but are not yet\n", - " executed." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/08_plan_approval.ipynb b/notebooks/08_plan_approval.ipynb deleted file mode 100644 index ec9bd85d..00000000 --- a/notebooks/08_plan_approval.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# 08 — Repair plans: suggest, approve, apply, undo\n", - "\n", - "Phase 2 makes cleaning **reviewable**: `fd.suggest_plan` proposes exactly the\n", - "repairs `fd.clean` would make, you approve/reject/override them, and\n", - "`fd.apply_plan` executes *only* what you approved — under a physical\n", - "protected-column guard, with drift refusal, an undo log, and a deterministic\n", - "audit hash. Everything is offline and model-free." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "import freshdata as fd\n", - "\n", - "df = pd.DataFrame({\n", - " \"cust_id\": [\"C001\", \"C002\", \"C003\", \"C004\"],\n", - " \"email_addr\": [\"a@@b.com\", \"x @ y.com\", \"ok@ok.com\", \"junk\"],\n", - " \"mobile\": [\"98765 43210\", \"09876543210\", \"+919876543210\", \"12345\"],\n", - " \"monthly_revenue\": [\"1000\", \"2000\", \"3000\", \"4000\"],\n", - " \"status\": [\" Active \", \"INACTIVE\", \"actve\", \"pending\"],\n", - "})\n", - "\n", - "CONTEXT = \"\"\"CustomerID is unique.\n", - "Emails must be valid.\n", - "Phone numbers are Indian.\n", - "Allowed status values are active, inactive, pending.\n", - "Never modify revenue values.\"\"\"" - ] - }, - { - "cell_type": "markdown", - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "source": [ - "## 1. Suggest a plan\n", - "\n", - "With `semantic_mode=\"auto\"`, low-risk high-confidence actions arrive\n", - "pre-approved (they are what `fd.clean` would have applied automatically);\n", - "everything ambiguous stays `pending` or is `blocked`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "outputs": [], - "source": [ - "plan = fd.suggest_plan(df, context=CONTEXT, semantic_mode=\"auto\", verbose=False)\n", - "rp = plan.repair_plan\n", - "print(rp.summary())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "rp.to_frame() # one row per planned action, for filtering/sorting" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "## 2. Review\n", - "\n", - "Selectors accept an action id, a column name, an action kind, a list of\n", - "those, or a predicate." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "outputs": [], - "source": [ - "# The fuzzy typo \"actve\" -> \"active\" was only *suggested* (conf 0.8 < 0.95).\n", - "typo = next(a for a in rp.actions if a.params.get(\"raw_value\") == \"actve\")\n", - "rp.approve(typo.id)\n", - "\n", - "# Reject all phone repairs with a recorded reason.\n", - "rp.reject(\"phone_format\", reason=\"numbers are re-synced from the CRM nightly\")\n", - "\n", - "# Or approve everything low-risk in one call (never touches blocked/rejected):\n", - "rp.approve_all(max_risk=\"low\")\n", - "print(rp.summary())" - ] - }, - { - "cell_type": "markdown", - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "source": [ - "## 3. Apply — exactly what was approved\n", - "\n", - "- nothing is re-profiled, nothing is re-decided;\n", - "- rejected and blocked actions do not run;\n", - "- the protected-column guard verifies `monthly_revenue` byte-identical;\n", - "- the input frame is never mutated." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "outputs": [], - "source": [ - "clean_df, report = fd.apply_plan(df, rp, keep_undo=True)\n", - "clean_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"decisions_hash:\", report.decisions_hash)\n", - "# Stable for the same reviewed plan; changes when any approval changes." - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "## 4. Drift refusal\n", - "\n", - "A plan remembers the frame it was built for. Applying it to changed data\n", - "refuses by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "drifted = df.copy()\n", - "drifted.loc[0, \"status\"] = \"archived\"\n", - "try:\n", - " fd.apply_plan(drifted, rp)\n", - "except fd.PlanDriftError as exc:\n", - " print(\"refused:\", exc)\n", - "\n", - "# Explicit override: stale actions are skipped and recorded.\n", - "_, drift_report = fd.apply_plan(drifted, rp, allow_drift=True)" - ] - }, - { - "cell_type": "markdown", - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "source": [ - "## 5. Undo\n", - "\n", - "`keep_undo=True` stores a compact log (cell positions + the raw value per\n", - "action), capped by `undo_cell_limit`; actions that don't fit are honestly\n", - "marked `reversible=False`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "outputs": [], - "source": [ - "email_ids = [a.id for a in rp.actions\n", - " if a.kind == \"email_format\" and a.approval == \"approved\"]\n", - "restored = report.revert(clean_df, action_ids=email_ids)\n", - "restored[[\"email_addr\"]]" - ] - }, - { - "cell_type": "markdown", - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "source": [ - "## 6. Plans as files\n", - "\n", - "```bash\n", - "freshdata plan in.csv --context-file rules.txt --out plan.json\n", - "freshdata apply-plan in.csv --plan plan.json -o out.csv --report audit.json\n", - "```\n", - "\n", - "`RepairPlan.to_json()` / `from_json()` round-trip actions, approval state,\n", - "the compiled policy, and the frame signature — so a plan can be reviewed in\n", - "a pull request like code.\n", - "\n", - "## Limitations\n", - "\n", - "- No model, no embeddings, no network; only the deterministic Phase-1\n", - " context language.\n", - "- Ambiguous repairs (`junk`, `12345`, typos with two close candidates) are\n", - " suggested or flagged, never auto-applied.\n", - "- Undo is cell-scoped; row drops/aggregations are not reversible." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/09_optional_semantic_models.ipynb b/notebooks/09_optional_semantic_models.ipynb deleted file mode 100644 index 18fdaebe..00000000 --- a/notebooks/09_optional_semantic_models.ipynb +++ /dev/null @@ -1,250 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Optional semantic models\n", - "\n", - "FreshData's default install is **model-free** — everything you saw in the\n", - "context-cleaning and plan-approval notebooks is deterministic, offline code.\n", - "This notebook shows the *optional* Phase-3 model path: a small local ONNX\n", - "encoder that adds repair **proposals** (never decisions) for values the\n", - "deterministic experts abstain on, plus a rescue rung for the column resolver.\n", - "\n", - "Requirements: `pip install \"freshdata-cleaner[semantic]\"` and an explicit\n", - "`fd.models.pull(...)`. Models are **never** downloaded automatically. For this\n", - "notebook we enable the built-in deterministic *stub encoder* so everything\n", - "runs with no model files and no network.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Testing seam: a deterministic hash-based encoder, so the whole notebook\n", - "# runs offline with zero model files. With real models installed you would\n", - "# skip this line entirely.\n", - "os.environ[\"FRESHDATA_STUB_ENCODER\"] = \"1\"\n", - "\n", - "import pandas as pd\n", - "\n", - "import freshdata as fd\n" - ] - }, - { - "cell_type": "markdown", - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "source": [ - "## The model registry\n", - "\n", - "`fd.models` is the explicit, auditable way models get onto your machine.\n", - "`status()` is offline and side-effect free; `pull()` is the **only** network\n", - "path in the whole package and verifies pinned checksums.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "outputs": [], - "source": [ - "for cfg in fd.models.list_available():\n", - " print(f\"{cfg.model_id:<20} {cfg.quantization:<5} {cfg.license:<12} \"\n", - " f\"~{cfg.size_bytes / 1e6:.0f} MB\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "fd.models.status()\n" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "Official artifacts are not hosted yet — `fd.models.pull(\"fd-col-encoder-v1\")`\n", - "raises `ModelNotPublishedError` with instructions (mirror via\n", - "`FRESHDATA_MODEL_URL_BASE`, or drop files into `FRESHDATA_MODEL_DIR` for\n", - "air-gapped installs). The stub we enabled above stands in for the encoder.\n" - ] - }, - { - "cell_type": "markdown", - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "source": [ - "## Embedding-assisted cleaning\n", - "\n", - "The ecommerce fixture again — but note `status` contains `'activvee'`, two edits\n", - "away from `active`. The deterministic reference expert only trusts one edit, so\n", - "on a model-free install this value is *flagged, never repaired*. With the\n", - "embedding backend enabled, the encoder gets a vote — as **evidence through the\n", - "same gate**, never as authority.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(\n", - " {\n", - " \"cust_id\": [\"C001\", \"C002\", \"C003\", \"C004\"],\n", - " \"email_addr\": [\"asha@gmail.com\", \"ravi@@example.com\",\n", - " \"neha@shop.in\", \"kiran@test.in\"],\n", - " \"mob_no\": [\"98765 43210\", \"+91 91234 56780\",\n", - " \"09123456789\", \"90000 11111\"],\n", - " \"monthly_revenue\": [\"1000\", \"2000\", \"3000\", \"4000\"],\n", - " \"status\": [\"activvee\", \"inactive\", \"pendng\", \"active\"],\n", - " }\n", - ")\n", - "\n", - "CONTEXT = \"\"\"\n", - "This is an ecommerce customer dataset.\n", - "CustomerID is unique.\n", - "Emails must be valid.\n", - "Phone numbers are Indian.\n", - "Allowed status values are active, inactive, pending.\n", - "Never modify revenue values.\n", - "\"\"\"\n", - "\n", - "clean_df, report = fd.clean(\n", - " df,\n", - " context=CONTEXT,\n", - " semantic_mode=\"auto\",\n", - " semantic_backends=(\"deterministic\", \"memory\", \"embedding\"),\n", - " return_report=True,\n", - ")\n", - "clean_df\n" - ] - }, - { - "cell_type": "markdown", - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "source": [ - "Protected means protected — `monthly_revenue` is byte-identical no matter\n", - "which backends run (a physical executor check, not a convention):\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "assert clean_df[\"monthly_revenue\"].equals(df[\"monthly_revenue\"])\n", - "clean_df[\"monthly_revenue\"]\n" - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "Every model-assisted action carries full provenance: which backend, which\n", - "model, the raw score, the calibrated confidence, and the candidate margins:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "for action in report.actions:\n", - " if action.step == \"semantic\":\n", - " meta = action.metadata\n", - " print(f\"[{action.status:<9}] {action.column:<12} \"\n", - " f\"{meta.get('raw_value')!r} -> {meta.get('proposed_value')!r} \"\n", - " f\"conf={action.confidence} backend={meta.get('backend')} \"\n", - " f\"calib={meta.get('calibration_version', '-')}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "source": [ - "## Graceful degradation\n", - "\n", - "Requesting the embedding backend without the extra or the model never crashes\n", - "— it self-disables with a report note, and cleaning proceeds exactly as the\n", - "model-free install would:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "outputs": [], - "source": [ - "from freshdata.models import runtime\n", - "\n", - "os.environ.pop(\"FRESHDATA_STUB_ENCODER\", None)\n", - "runtime.reset_encoders()\n", - "\n", - "_, degraded_report = fd.clean(\n", - " df,\n", - " context=CONTEXT,\n", - " semantic_mode=\"auto\",\n", - " semantic_backends=(\"deterministic\", \"memory\", \"embedding\"),\n", - " return_report=True,\n", - ")\n", - "degraded_report.fallback_events\n" - ] - }, - { - "cell_type": "markdown", - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "source": [ - "## What the model path will never do\n", - "\n", - "- **No automatic downloads** — `fd.clean` cannot reach the network.\n", - "- **No LLM, no cloud inference, no per-cell model calls** — the backend sees\n", - " distinct values of eligible columns only.\n", - "- **No ambiguous auto-applies** — a value near two allowed candidates produces\n", - " *no proposal at all*; pure similarity clustering is suggest-only by default.\n", - "- **No unprotected protected columns** — byte-identity is asserted physically\n", - " after every run.\n", - "\n", - "See `docs/semantic-models.md` for install, air-gapped setup, calibration, and\n", - "troubleshooting.\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/10_learning_profiles.ipynb b/notebooks/10_learning_profiles.ipynb deleted file mode 100644 index c94052c4..00000000 --- a/notebooks/10_learning_profiles.ipynb +++ /dev/null @@ -1,474 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Learning profiles: paired-data cleaning that remembers\n\n", - "\n\n", - "Phase 4 adds `fd.learn()`: instead of hand-writing cleaning rules, you show freshdata a\n\n", - "**messy/clean pair** of the same rows and it learns a reusable `.fdprofile` — a set of\n\n", - "typed rules, value maps, and (optionally) retrieval examples that later replay against new,\n\n", - "similarly-shaped data via `fd.clean(df, profile=...)`.\n\n", - "\n\n", - "This notebook runs a small but realistic example end to end: an ecommerce customer table\n\n", - "from India with messy emails, phone formats, day-first dates, currency strings, status\n\n", - "typos, and a nonstandard missing-value sentinel." - ] - }, - { - "cell_type": "markdown", - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "source": [ - "## 1. A messy/clean pair\n\n", - "\n\n", - "We build 30 rows of \"messy\" input and the corresponding \"clean\" ground truth. Two status\n\n", - "typos — `\"Shipped\"` and `\"Deliverd\"` — each occur with support 5, right at the default\n\n", - "`min_support=5` gate, so both should be learned as value-map rules rather than demoted to\n\n", - "suggest-only examples." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "import freshdata as fd\n", - "\n", - "n = 30\n", - "\n", - "cust_id = list(range(1, n + 1))\n", - "\n", - "\n", - "\n", - "# email: every 3rd row upper-cased, every 3rd row padded with whitespace\n", - "\n", - "emails_clean = [f\"user{i}@example.in\" for i in range(1, n + 1)]\n", - "\n", - "emails_messy = [\n", - "\n", - " e.upper() if i % 3 == 0 else (f\" {e} \" if i % 3 == 1 else e)\n", - "\n", - " for i, e in enumerate(emails_clean)\n", - "\n", - "]\n", - "\n", - "\n", - "\n", - "# phone: rotate through a few common messy IN formats\n", - "\n", - "phones_clean = [f\"+9198765{43200 + i:05d}\" for i in range(n)]\n", - "\n", - "phones_messy = []\n", - "\n", - "for i, p in enumerate(phones_clean):\n", - "\n", - " digits = p.replace(\"+91\", \"\")\n", - "\n", - " style = i % 4\n", - "\n", - " if style == 0:\n", - "\n", - " phones_messy.append(f\"0{digits}\")\n", - "\n", - " elif style == 1:\n", - "\n", - " phones_messy.append(f\"{digits[:5]} {digits[5:]}\")\n", - "\n", - " elif style == 2:\n", - "\n", - " phones_messy.append(f\"91-{digits}\")\n", - "\n", - " else:\n", - "\n", - " phones_messy.append(p)\n", - "\n", - "\n", - "\n", - "# signup_date: half day-first, half already ISO\n", - "\n", - "dates_clean = [f\"2024-{1 + (i % 9):02d}-{10 + (i % 15):02d}\" for i in range(n)]\n", - "\n", - "dates_messy = [\n", - "\n", - " f\"{d.split('-')[2]}/{d.split('-')[1]}/{d.split('-')[0]}\" if i % 2 == 0 else d\n", - "\n", - " for i, d in enumerate(dates_clean)\n", - "\n", - "]\n", - "\n", - "\n", - "\n", - "# amount_spent: rupee-formatted strings -> floats\n", - "\n", - "amounts_clean = [round(500 + i * 37.5, 2) for i in range(n)]\n", - "\n", - "amounts_messy = [f\"₹{a:,.2f}\" for a in amounts_clean]\n", - "\n", - "\n", - "\n", - "# status: two typo'd spellings, each with support >= min_support (5)\n", - "\n", - "status_clean = (['shipped'] * 10) + (['delivered'] * 10) + (['cancelled'] * 10)\n", - "\n", - "status_messy = (\n", - "\n", - " (['Shipped'] * 5 + ['shipped'] * 5)\n", - "\n", - " + (['Deliverd'] * 5 + ['delivered'] * 5)\n", - "\n", - " + (['cancelled'] * 10)\n", - "\n", - ")\n", - "\n", - "\n", - "\n", - "# country: a nonstandard sentinel for missing, every 6th row\n", - "\n", - "country_clean = ['India'] * n\n", - "\n", - "country_messy = ['India'] * n\n", - "\n", - "for i in range(0, n, 6):\n", - "\n", - " country_messy[i] = 'N/A'\n", - "\n", - " country_clean[i] = None\n", - "\n", - "\n", - "\n", - "messy_df = pd.DataFrame(\n", - "\n", - " {\n", - "\n", - " \"cust_id\": cust_id,\n", - "\n", - " \"email\": emails_messy,\n", - "\n", - " \"phone\": phones_messy,\n", - "\n", - " \"signup_date\": dates_messy,\n", - "\n", - " \"amount_spent\": amounts_messy,\n", - "\n", - " \"status\": status_messy,\n", - "\n", - " \"country\": country_messy,\n", - "\n", - " }\n", - "\n", - ")\n", - "\n", - "clean_df = pd.DataFrame(\n", - "\n", - " {\n", - "\n", - " \"cust_id\": cust_id,\n", - "\n", - " \"email\": emails_clean,\n", - "\n", - " \"phone\": phones_clean,\n", - "\n", - " \"signup_date\": dates_clean,\n", - "\n", - " \"amount_spent\": amounts_clean,\n", - "\n", - " \"status\": status_clean,\n", - "\n", - " \"country\": country_clean,\n", - "\n", - " }\n", - "\n", - ")\n", - "\n", - "\n", - "\n", - "messy_df.head()" - ] - }, - { - "cell_type": "markdown", - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "source": [ - "## 2. Learn a profile\n\n", - "\n\n", - "`fd.learn` runs the 5-stage pipeline (align → diff → classify → extract → fit/holdout-validate)\n\n", - "and returns a `LearningProfile`. `context=` is free text that seeds semantic-type inference,\n\n", - "same as `fd.clean(df, context=...)`. `key=\"cust_id\"` aligns messy/clean rows unambiguously." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "profile = fd.learn(\n", - "\n", - " messy_df,\n", - "\n", - " clean_df,\n", - "\n", - " context=\"Ecommerce customer data from India\",\n", - "\n", - " key=\"cust_id\",\n", - "\n", - ")\n", - "\n", - "print(profile.summary())\n", - "\n", - "print(\"contains_raw_values:\", profile.manifest.contains_raw_values)" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "`contains_raw_values` is `False`: by default (`privacy=\"mask\"`), sensitive semantic types\n\n", - "(`email`, `phone`, `person_name`, `national_id`, `address`, `postal_code`, `free_text`) are\n\n", - "HMAC-masked before anything is stored, and no raw literal imputation values are ever learned.\n\n", - "Only the non-sensitive `status` column contributes real value-map literals here (`\"Shipped\"\n\n", - "→ \"shipped\"`, `\"Deliverd\" → \"delivered\"`), because `status` isn't a sensitive semantic type." - ] - }, - { - "cell_type": "markdown", - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "source": [ - "## 3. Save, and confirm no literal PII leaks into the file\n\n", - "\n\n", - "A `.fdprofile` is a zip with a manifest that hashes every member for integrity. We check the\n\n", - "raw bytes of the saved file for any of the real emails or phone numbers from the training\n\n", - "data — none should appear, since those columns are masked by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "outputs": [], - "source": [ - "fd.save_profile(profile, \"/tmp/e2e_orders.fdprofile\")\n", - "\n", - "\n", - "\n", - "with open(\"/tmp/e2e_orders.fdprofile\", \"rb\") as f:\n", - "\n", - " raw_bytes = f.read()\n", - "\n", - "\n", - "\n", - "leak_terms = emails_clean + phones_clean\n", - "\n", - "leaks = [t for t in leak_terms if t.encode(\"utf-8\") in raw_bytes]\n", - "\n", - "print(\"literal leaks found in zip bytes:\", leaks)" - ] - }, - { - "cell_type": "markdown", - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "source": [ - "## 4. Load the profile and replay it on new data\n\n", - "\n\n", - "`fd.clean(df, profile=..., return_report=True)` returns a `(cleaned_df, report)` tuple.\n\n", - "Profile-backed proposals are gathered by the semantic layer, which stays off by default — pass\n\n", - "`semantic_mode=\"auto\"` (or `\"assist\"`/`\"review\"`) explicitly, exactly as you would to enable\n\n", - "semantic assistance without a profile." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "outputs": [], - "source": [ - "loaded = fd.load_profile(\"/tmp/e2e_orders.fdprofile\")\n", - "\n", - "\n", - "\n", - "new_df = pd.DataFrame(\n", - "\n", - " {\n", - "\n", - " \"cust_id\": [1001, 1002, 1003, 1004],\n", - "\n", - " \"email\": [\n", - "\n", - " \" NEW.USER@example.in \",\n", - "\n", - " \"another@example.in\",\n", - "\n", - " \"THIRD@EXAMPLE.IN\",\n", - "\n", - " \"fourth@example.in \",\n", - "\n", - " ],\n", - "\n", - " \"phone\": [\"098765 43299\", \"91-9876543298\", \"+919876543297\", \"09876543296\"],\n", - "\n", - " \"signup_date\": [\"15/03/2024\", \"2024-03-16\", \"20/03/2024\", \"2024-03-18\"],\n", - "\n", - " \"amount_spent\": [\"₹1,200.00\", \"₹950.50\", \"₹2,000.00\", \"₹800.00\"],\n", - "\n", - " \"status\": [\"Shipped\", \"Deliverd\", \"cancelled\", \"Shipped\"],\n", - "\n", - " \"country\": [\"India\", \"N/A\", \"India\", \"India\"],\n", - "\n", - " }\n", - "\n", - ")\n", - "\n", - "new_df" - ] - }, - { - "cell_type": "markdown", - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "source": [ - "### Baseline: `fd.clean(new_df)` with no profile\n\n", - "\n\n", - "General-purpose heuristics (phone/date/currency normalization, sentinel-to-missing) still\n\n", - "apply without any profile. But `\"Shipped\"` and `\"Deliverd\"` aren't recognizable as typos by\n\n", - "generic rules — there's no dictionary that says \"Deliverd\" means \"delivered\" for this\n\n", - "dataset's vocabulary. Without the learned value map, both typos pass through unfixed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "outputs": [], - "source": [ - "baseline_cleaned, _ = fd.clean(new_df, return_report=True)\n", - "\n", - "baseline_cleaned[[\"cust_id\", \"status\"]]" - ] - }, - { - "cell_type": "markdown", - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "source": [ - "### With the learned profile: `fd.clean(new_df, profile=loaded, semantic_mode=\"auto\")`\n\n", - "\n\n", - "Now the learned `status` value map replays: both `\"Shipped\"` and `\"Deliverd\"` are corrected,\n\n", - "each recorded as a `profile_influenced` action in the report." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "outputs": [], - "source": [ - "cleaned, report = fd.clean(\n", - "\n", - " new_df, profile=loaded, semantic_mode=\"auto\", return_report=True\n", - "\n", - ")\n", - "\n", - "cleaned[[\"cust_id\", \"status\"]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, - "outputs": [], - "source": [ - "actions = [a for a in report.actions if a.metadata.get(\"profile_influenced\")]\n", - "\n", - "print(\"profile_influenced actions:\", len(actions))\n", - "\n", - "for a in actions:\n", - "\n", - " print(a.column, a.metadata)" - ] - }, - { - "cell_type": "markdown", - "id": "8a65eabff63a45729fe45fb5ade58bdc", - "metadata": {}, - "source": [ - "## 5. Audit the profile\n\n", - "\n\n", - "`profile.audit()` reports every rule the profile carries and what happened to any candidate\n\n", - "that didn't make it into an auto-applied rule. Here, an `email_normalize` rule was proposed\n\n", - "during extraction but **demoted to suggest-only** because its holdout precision (measured on\n\n", - "a held-out split of the training pairs) fell below the default `min_precision=0.98` gate —\n\n", - "the fit/holdout stage is what keeps a profile from replaying a rule that doesn't generalize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3933fab20d04ec698c2621248eb3be0", - "metadata": {}, - "outputs": [], - "source": [ - "audit = profile.audit()\n", - "\n", - "print(f\"rule_count={audit.rule_count} rules_by_family={audit.rules_by_family}\")\n", - "\n", - "print(f\"value_map_entries={audit.value_map_entries} masked_entries={audit.masked_entries}\")\n", - "\n", - "print(f\"demotions={len(audit.demotions)}\")\n", - "\n", - "for d in audit.demotions:\n", - "\n", - " print(\" \", d.column, d.family, d.outcome, \"-\", d.reason)" - ] - }, - { - "cell_type": "markdown", - "id": "4dd4641cc4064e0191573fe9c69df29b", - "metadata": {}, - "source": [ - "## Recap\n\n", - "\n\n", - "- `fd.learn(messy, clean, context=..., key=...)` learns a `LearningProfile` from a paired\n\n", - " example, not from hand-written rules.\n\n", - "- Sensitive columns are masked by default; no full rows, no raw sensitive literals, and no\n\n", - " literal imputation maps are ever stored.\n\n", - "- Low-support or low-precision candidates are demoted to suggest-only rather than silently\n\n", - " applied — a profile can only replay what it validated on held-out data.\n\n", - "- Replay is opt-in and additive: `fd.clean(df, profile=profile, semantic_mode=\"auto\")` only\n\n", - " changes behavior when you ask for it; `profile=None` (the default) is byte-identical to\n\n", - " today's `fd.clean`.\n\n", - "- See `docs/learning-profiles.md` for the full format, privacy model, and CLI\n\n", - " (`freshdata learn`, `clean --profile`, `profile audit|diff|merge`)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/debug_dataset_sweep.py b/scripts/debug_dataset_sweep.py deleted file mode 100644 index 947bd4a6..00000000 --- a/scripts/debug_dataset_sweep.py +++ /dev/null @@ -1,278 +0,0 @@ -"""Multi-dataset debug sweep for freshdata (H1–H5). - -Run from repo root:: - - python scripts/debug_dataset_sweep.py - python scripts/debug_dataset_sweep.py --online --tier 1 - python scripts/debug_dataset_sweep.py --online -""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -import time -from pathlib import Path - -import pandas as pd - -_REPO = Path(__file__).resolve().parents[1] -_TESTS = _REPO / "tests" -_LOG_PATH = _REPO.parent / ".cursor" / "debug_dataset_sweep.log" - -if str(_TESTS) not in sys.path: - sys.path.insert(0, str(_TESTS)) - -import freshdata as fd # noqa: E402 -from expectations import ( # noqa: E402 - ALL_FIXTURES, - ALL_ONLINE_FIXTURES, - ALL_ONLINE_TIER1, - ALL_ONLINE_TIER2, - load_expectations, - load_fixture, - load_online_fixture, - resolve_column, -) -from freshdata.steps.columns import snake_case # noqa: E402 - -ISOLATE = {"drop_empty_rows": False, "drop_duplicates": False, "verbose": False} -_ID_RE = re.compile(r"(?:^|_)(?:id|uuid|guid|key|record_id|transaction_id)s?$", re.I) - - -def _log(hypothesis_id: str, dataset: str, check: str, passed: bool, detail: str = "") -> None: - _LOG_PATH.parent.mkdir(parents=True, exist_ok=True) - payload = { - "hypothesisId": hypothesis_id, - "location": "debug_dataset_sweep.py", - "message": check, - "data": {"dataset": dataset, "pass": passed, "detail": detail}, - "timestamp": int(time.time() * 1000), - } - with _LOG_PATH.open("a") as fh: - fh.write(json.dumps(payload) + "\n") - - -def _id_candidate(df: pd.DataFrame) -> str | None: - for col in df.columns: - label = str(col) - if _ID_RE.search(label) or label.lower() in {"id", "index", "pk"}: - return label - return None - - -def _preserve_candidates(name: str, df: pd.DataFrame) -> list[str]: - exp = load_expectations(name).get("balanced", {}) - cols: list[str] = [] - for key in ( - "columns_never_imputed", - "columns_never_capped", - "columns_preserved_as_text", - "columns_never_dropped", - ): - for col in exp.get(key, []): - resolved = resolve_column(df, col) - if resolved in df.columns and resolved not in cols: - cols.append(resolved) - return cols[:3] - - -def _frames_equal(a: pd.DataFrame, b: pd.DataFrame) -> tuple[bool, str]: - try: - pd.testing.assert_frame_equal(a, b) - return True, "" - except AssertionError as exc: - return False, str(exc)[:300] - - -def check_h1(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - base = {**ISOLATE, "strategy": "balanced"} - - id_col = _id_candidate(df) - if id_col: - ren = snake_case(id_col) - kw = {**base, "id_columns": ren, "outlier_action": "cap"} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "id_columns": (ren,)}) - ok, detail = _frames_equal(a, b) - results.append((f"H1:id_columns:{ren}", ok, detail)) - - for col in _preserve_candidates(name, df): - ren = snake_case(col) - kw = {**base, "preserve_columns": ren, "outlier_action": "cap"} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "preserve_columns": (ren,)}) - ok, detail = _frames_equal(a, b) - results.append((f"H1:preserve_columns:{ren}", ok, detail)) - - if "duplicate" in name or id_col: - subset = snake_case(id_col or str(df.columns[0])) - kw = {**base, "duplicate_subset": subset, "drop_duplicates": True} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "duplicate_subset": (subset,)}) - ok, detail = _frames_equal(a, b) - results.append((f"H1:duplicate_subset:{subset}", ok, detail)) - - kw = {**base, "extra_sentinels": "missing", "drop_empty_rows": False} - a = fd.clean(df, **kw) - b = fd.clean(df, **{**kw, "extra_sentinels": ("missing",)}) - ok, detail = _frames_equal(a, b) - results.append(("H1:extra_sentinels", ok, detail)) - - return results - - -def check_h2(name: str, df: pd.DataFrame) -> tuple[str, bool, str]: - once = fd.clean(df, strategy="balanced", verbose=False) - twice = fd.clean(once, strategy="balanced", verbose=False) - ok, detail = _frames_equal(once, twice) - return ("H2:idempotent", ok, detail) - - -def check_h3(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - plan = fd.suggest_plan(df, **ISOLATE) - _, report = fd.clean(df, return_report=True, **ISOLATE) - for col, cp in plan.column_plans.items(): - if cp.outlier_action: - oa = [a for a in report if a.step == "outliers" and a.column == col] - if not oa: - continue - exp = cp.outlier_action - act = oa[0].description - ok = ( - (exp == "flag" and "flagged" in act) - or (exp == "cap" and "capped" in act) - or (exp in act or exp in oa[0].rationale) - ) - results.append((f"H3:outlier:{col}", ok, f"plan={exp}, got={act[:80]}")) - if cp.missing and cp.missing.model_id: - ma = [a for a in report if a.step == "missing" and a.column == col] - if not ma: - continue - action = ma[-1] - ok = ( - cp.missing.model_id == action.model_id - or cp.missing.model_id in action.description - or cp.missing.model_id in action.rationale - ) - results.append( - (f"H3:missing:{col}", ok, f"plan={cp.missing.model_id}, got={action.model_id}") - ) - if not results: - results.append(("H3:plan_clean", True, "no engine columns to compare")) - return results - - -def check_h4(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - id_col = _id_candidate(df) - if not id_col: - return [("H4:rename_id", True, "no id-like column")] - raw_label = str(id_col) - snake = snake_case(raw_label) - if raw_label == snake: - return [("H4:rename_id", True, "column already snake_case")] - - pre = fd.clean(df, id_columns=raw_label, outlier_action="cap", **ISOLATE) - post = fd.clean(df, id_columns=snake, outlier_action="cap", **ISOLATE) - ok, detail = _frames_equal(pre, post) - results.append((f"H4:id_pre_vs_post_name:{raw_label}", ok, detail)) - return results - - -def check_h5(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - exp = load_expectations(name).get("balanced", {}) - - for col in exp.get("columns_never_imputed", []): - raw = resolve_column(df, col) - if raw not in df.columns: - continue - before_na = int(df[raw].isna().sum()) - out_s = fd.clean(df, preserve_columns=col, **ISOLATE) - out_t = fd.clean(df, preserve_columns=(col,), **ISOLATE) - ok_na = int(out_s[col].isna().sum()) == before_na == int(out_t[col].isna().sum()) - ok_eq, detail = _frames_equal(out_s, out_t) - results.append((f"H5:never_imputed:{col}", ok_na and ok_eq, detail or f"na={before_na}")) - - for col in exp.get("columns_never_capped", []): - raw = resolve_column(df, col) - if raw not in df.columns: - continue - before_max = df[raw].max() - out_s = fd.clean(df, preserve_columns=col, outlier_action="cap", **ISOLATE) - out_t = fd.clean(df, preserve_columns=(col,), outlier_action="cap", **ISOLATE) - ok_cap = out_s[col].max() == out_t[col].max() == before_max - ok_eq, detail = _frames_equal(out_s, out_t) - results.append((f"H5:never_capped:{col}", ok_cap and ok_eq, detail or f"max={before_max}")) - - if not results: - results.append(("H5:expectations", True, "no preserve expectations")) - return results - - -def sweep_dataset(name: str, df: pd.DataFrame) -> list[tuple[str, bool, str]]: - results: list[tuple[str, bool, str]] = [] - results.extend(check_h1(name, df)) - results.append(check_h2(name, df)) - results.extend(check_h3(name, df)) - results.extend(check_h4(name, df)) - results.extend(check_h5(name, df)) - for check, passed, detail in results: - hid = check.split(":")[0] - _log(hid, name, check, passed, detail) - return results - - -def load_dataset(name: str, *, online: bool) -> pd.DataFrame: - if online: - return load_online_fixture(name, live=False) - return load_fixture(name) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Multi-dataset freshdata debug sweep") - parser.add_argument("--online", action="store_true", help="use cached online fixtures") - parser.add_argument("--tier", type=int, default=None, help="online tier filter (1 or 2)") - args = parser.parse_args() - - if args.online: - if args.tier == 1: - names = ALL_ONLINE_TIER1 - elif args.tier == 2: - names = ALL_ONLINE_TIER2 - else: - names = ALL_ONLINE_FIXTURES - else: - names = ALL_FIXTURES - - failures: list[str] = [] - print(f"{'dataset':<28} {'check':<40} {'result'}") - print("-" * 80) - - for name in names: - try: - df = load_dataset(name, online=args.online) - except Exception as exc: - print(f"{name:<28} {'LOAD':<40} SKIP ({exc})") - continue - for check, passed, detail in sweep_dataset(name, df): - status = "PASS" if passed else "FAIL" - print(f"{name:<28} {check:<40} {status}") - if not passed: - failures.append(f"{name}/{check}: {detail}") - if detail: - print(f" -> {detail[:200]}") - - print("-" * 80) - print(f"Total: {len(names)} datasets, {len(failures)} failure(s)") - print(f"Log: {_LOG_PATH}") - return 1 if failures else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/debug_datasets.py b/scripts/debug_datasets.py deleted file mode 100644 index 3fd8336a..00000000 --- a/scripts/debug_datasets.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -"""Compare clean() across local and online fixtures for debugging. - -Usage:: - - python scripts/debug_datasets.py - python scripts/debug_datasets.py --online titanic - python scripts/debug_datasets.py --online --explain titanic - python scripts/debug_datasets.py --infer-roles --online adult_income - python scripts/debug_datasets.py --search missing - python scripts/debug_datasets.py --refresh --online --compare -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -from pathlib import Path - -import pandas as pd - -import freshdata as fd - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "tests")) - -from expectations import ( # noqa: E402 - ALL_FIXTURES, - ALL_ONLINE_FIXTURES, - load_fixture, - load_online_fixture, - load_registry, -) - - -def _load(name: str, *, online: bool) -> pd.DataFrame: - if online: - return load_online_fixture(name) - return load_fixture(name) - - -def _search_datasets(tag: str) -> list[str]: - registry = load_registry() - return sorted(k for k, v in registry.items() if tag in v.get("tags", [])) - - -def _print_compare(df: pd.DataFrame, label: str) -> None: - table = fd.compare_clean(df) - print(f"\n=== {label} ({len(df)} rows x {df.shape[1]} cols) ===") - print(table.to_string(index=False)) - _, report = fd.clean(df, strategy="balanced", return_report=True, verbose=False) - if report.warnings: - print("\nWarnings:") - for w in report.warnings: - print(f" - {w}") - recs = getattr(report, "recommendations", None) or [] - if recs: - print("\nRecommendations:") - for r in recs: - print(f" - {r}") - - -def _print_explain(df: pd.DataFrame, label: str) -> None: - explanation = fd.explain_clean(df, strategy="balanced") - print(f"\n=== explain_clean: {label} ===") - print(explanation.summary()) - if not explanation.roles.empty: - print("\nRoles:") - print(explanation.roles.to_string(index=False)) - - -def _print_roles(df: pd.DataFrame, label: str) -> None: - roles = fd.infer_roles(df, strategy="balanced") - print(f"\n=== infer_roles: {label} ===") - print(roles.to_string(index=False)) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Debug freshdata on fixture datasets") - parser.add_argument("--online", action="store_true", help="Online cached datasets only") - parser.add_argument("--local", action="store_true", help="Local fixtures only") - parser.add_argument( - "--refresh", - action="store_true", - help="Re-fetch online cache before compare", - ) - parser.add_argument("--compare", action="store_true", help="Print compare_clean table") - parser.add_argument("--explain", action="store_true", help="Print explain_clean summary") - parser.add_argument("--infer-roles", action="store_true", help="Print infer_roles table") - parser.add_argument("--search", metavar="TAG", help="Filter online datasets by registry tag") - parser.add_argument("--format", choices=["text", "json"], default="text") - parser.add_argument("names", nargs="*", help="Dataset id(s); default all in scope") - args = parser.parse_args(argv) - - if args.search and not args.names: - args.names = _search_datasets(args.search) - args.online = True - - if args.refresh: - cmd = [sys.executable, str(ROOT / "scripts" / "fetch_online_fixtures.py"), "--refresh"] - if args.names: - for name in args.names: - cmd.extend(["--only", name]) - subprocess.run(cmd, check=True, cwd=ROOT) - - scopes: list[tuple[str, bool]] = [] - if args.online and not args.local: - scopes = [(n, True) for n in (args.names or ALL_ONLINE_FIXTURES)] - elif args.local and not args.online: - scopes = [(n, False) for n in (args.names or ALL_FIXTURES)] - else: - names = args.names - if names: - scopes = [(n, n in ALL_ONLINE_FIXTURES) for n in names] - else: - scopes = [(n, False) for n in ALL_FIXTURES] + [(n, True) for n in ALL_ONLINE_FIXTURES] - - show_compare = args.compare or not (args.explain or args.infer_roles) - reports: list[dict] = [] - print(f"freshdata {fd.__version__} | comparing {len(scopes)} dataset(s)") - for name, online in scopes: - try: - df = _load(name, online=online) - except Exception as exc: # noqa: BLE001 — CLI should continue on bad fixture - print(f"\n=== {name}: SKIP ({exc}) ===") - continue - tag = "online" if online else "local" - label = f"{name} [{tag}]" - if show_compare: - _print_compare(df, label) - if args.explain: - _print_explain(df, label) - if args.infer_roles: - _print_roles(df, label) - if args.format == "json": - explanation = fd.explain_clean(df, strategy="balanced") - reports.append({"name": name, "online": online, **explanation.to_dict()}) - - if args.format == "json" and reports: - print(json.dumps(reports, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/generate_html_examples.py b/scripts/generate_html_examples.py deleted file mode 100644 index e0cf30c1..00000000 --- a/scripts/generate_html_examples.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -"""Generate the committed interactive-output HTML samples under docs/examples/. - -Run from the repo root: - - python scripts/generate_html_examples.py - -The samples are produced from small synthetic frames by the real renderers, so -they always match the current output. They need no optional viz dependencies. -""" - -from __future__ import annotations - -import os - -import numpy as np -import pandas as pd - -import freshdata as fd - -_OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "docs", "examples") - - -def _sample_frame() -> pd.DataFrame: - rng = np.random.default_rng(0) - n = 200 - df = pd.DataFrame({ - "customer_id": range(1, n + 1), - "email": [f"user{i}@example.com" if i % 9 else None for i in range(n)], - "revenue": rng.normal(1000, 250, n).round(2), - "signup_date": pd.date_range("2025-01-01", periods=n, freq="D").astype(str), - "tier": rng.choice(["bronze", "silver", "gold", None], n), - }) - df.loc[3:7, "revenue"] = [1e6, 9e5, 1.1e6, 8e5, 1e6] # outliers - return pd.concat([df, df.iloc[:5]], ignore_index=True) # duplicates - - -def _write(name: str, html: str) -> None: - os.makedirs(_OUT, exist_ok=True) - path = os.path.join(_OUT, name) - with open(path, "w", encoding="utf-8") as fh: - fh.write(f"{name}{html}") - print(f"wrote {path}") - - -def main() -> int: - df = _sample_frame() - cleaned, report = fd.clean(df, return_report=True) - - _write("profile_cockpit.html", fd.profile(df).to_html()) - _write("action_timeline.html", report.to_html()) - _write("suggest_plan_cards.html", fd.suggest_plan(df).to_html()) - _write("explain_diff.html", fd.explain_clean(df).to_html()) - _write("compare_plans_grid.html", fd.compare_plans(df)._repr_html_()) - - baseline = df.iloc[:150].copy() - diff = fd.compare_to_baseline(df, baseline=baseline, key="customer_id") - _write("baseline_drift.html", diff.to_html()) - - _, gate = fd.evaluate_quality_debt(df, ledger=None) - _write("quality_debt.html", gate.to_html()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/COMPLIANCE.md b/training/COMPLIANCE.md deleted file mode 100644 index 128060f5..00000000 --- a/training/COMPLIANCE.md +++ /dev/null @@ -1,71 +0,0 @@ -# Training pipeline compliance - -This document governs every development-time use of external ("teacher") -models and external data in the FreshData training pipeline. The runtime -library is unaffected: it is deterministic, offline, model-free by default, -and never calls an LLM or any cloud service from `fd.clean`. - -## Teacher models — allowed uses - -Teacher models may be used **only inside `training/`**, only for: - -1. realism direction (tuning corruptor frequency/parameters) -2. column-role labeling -3. NL context paraphrase generation -4. ambiguity adjudication -5. rationale style templates -6. benchmark red-teaming - -They must **never** be used to: - -- create messy/clean pairs from scratch (unverifiable ground truth) -- label release-gating eval data without human verification -- run at library runtime, per cell, or inside `fd.clean` -- receive full data rows or real PII (values are masked first; - `training/teacher/tasks.py` enforces both) - -## Provider terms checklist (before any teacher run) - -Every provider must have a record in `training/teacher/compliance_ledger.json` -containing: - -- provider name -- terms URL and a terms snapshot id -- date the terms were checked (records older than 180 days are stale) -- reviewer (name or initials of the human who read the terms) -- allowed/disallowed usage list -- whether outputs may be used for **model training** -- whether outputs may be **redistributed** -- status: `approved` / `rejected` / `pending` - -`python -m training.teacher.compliance check` blocks the pipeline when a -snapshot is missing or stale, the intended use is not approved, the reviewer -is missing, or the provider disallows training use. `make -training-release-artifacts` runs this check first. - -## Audit trail - -Every teacher call is cached under `training/cache/teacher/` with the raw -prompt, raw response, provider, model, timestamp, and terms snapshot id -(key: `provider + model + prompt_template_sha256 + input_sha256 + -schema_version`). Cache entries are the audit record for every -teacher-derived training label. - -## Human review - -- ≥ 5% of every teacher batch is sampled for human review - (`training/teacher/review.py`). -- Teacher–human disagreement > 3% forces full review of the batch. -- Release-gating eval labels are 100% human-verified - (`training/eval/human_verified.py` refuses anything less). -- Reviewer identity, timestamp, and disagreement reasons are stored and - exported in the review summary JSON. - -## Training data - -- Every seed source is registered in `training/seed/registry.json` with - license, attribution, explicit training permission, and resolved PII risk; - `python -m training.datasets.validators --check-licenses` gates releases. -- No real PII anywhere: identity-shaped data is fully synthetic - (`training/seed/synthetic/`), tagged `synthetic=true`. -- Corruption metadata — not teacher output — is the preferred ground truth. diff --git a/training/Makefile b/training/Makefile deleted file mode 100644 index 9faff64f..00000000 --- a/training/Makefile +++ /dev/null @@ -1,87 +0,0 @@ -# FreshData development-time training pipeline (Phase 5). -# All targets run from the repository root; `make -C training ` -# and the root Makefile's training-* passthrough both work. - -PY ?= python -ROOT := .. - -.PHONY: help training-seed training-corrupt training-teacher-labels \ - training-distill training-eval training-package-artifacts \ - training-dev-artifacts training-release-artifacts - -help: - @echo "Targets:" - @echo " training-seed Validate licenses + build the seed corpus" - @echo " training-corrupt Generate labeled corrupted training data" - @echo " training-teacher-labels Run compliance-gated teacher tasks (degrades offline)" - @echo " training-distill Train role/intent heads (+ optional encoder), export, quantize" - @echo " training-eval Calibration eval + full CleanBench release gates" - @echo " training-package-artifacts Package artifacts into dist/artifacts/" - @echo " training-dev-artifacts Small synthetic-only end-to-end build (no cloud)" - @echo " training-release-artifacts Full release pipeline; fails on any gate failure" - -training-seed: - cd $(ROOT) && $(PY) -m training.datasets.validators --check-licenses - cd $(ROOT) && $(PY) -m training.datasets.build_seed - -training-corrupt: - cd $(ROOT) && $(PY) -m training.datasets.build_corrupted - -training-teacher-labels: - cd $(ROOT) && $(PY) -m training.teacher.compliance check - @echo "teacher tasks run on demand via training.teacher.tasks.run_task;" - @echo "cached outputs (training/cache/teacher/) feed the trainers automatically." - -training-distill: - cd $(ROOT) && $(PY) -m training.distill.train_role_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_intent_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_encoder_contrastive --check-gates - cd $(ROOT) && $(PY) -m training.distill.export_onnx - cd $(ROOT) && $(PY) -m training.distill.quantize --check-gates - -training-eval: - cd $(ROOT) && $(PY) -m training.calibration.build_features - cd $(ROOT) && $(PY) -m training.calibration.fit_isotonic --fit-deterministic - cd $(ROOT) && $(PY) -m training.calibration.export_tables - cd $(ROOT) && $(PY) -m training.calibration.eval_calibration --check-gates - cd $(ROOT) && $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --check-gates - -training-package-artifacts: - cd $(ROOT) && $(PY) -m training.distill.package_artifacts - -# Small synthetic-only artifact set: no cloud teacher calls, no onnx needed, -# nothing enters the wheel. Ends with the CleanBench mini gates (T1-T4). -training-dev-artifacts: - cd $(ROOT) && $(PY) -m training.datasets.validators --check-licenses - cd $(ROOT) && $(PY) -m training.datasets.build_seed - cd $(ROOT) && $(PY) -m training.datasets.build_corrupted --dev - cd $(ROOT) && $(PY) -m training.distill.train_role_head --dev --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_intent_head --dev --check-gates - cd $(ROOT) && $(PY) -m training.distill.export_onnx - cd $(ROOT) && $(PY) -m training.distill.quantize --check-gates - cd $(ROOT) && $(PY) -m training.calibration.build_features --seeds 3 - cd $(ROOT) && $(PY) -m training.calibration.fit_isotonic --fit-deterministic - cd $(ROOT) && $(PY) -m training.calibration.export_tables - cd $(ROOT) && $(PY) -m training.calibration.eval_calibration --check-gates - cd $(ROOT) && $(PY) -m training.distill.package_artifacts - cd $(ROOT) && $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4 --check-gates - -# Full release pipeline: compliance first, human-verified eval enforced by -# the trainers, ONNX required, every gate blocking. -training-release-artifacts: - cd $(ROOT) && $(PY) -m training.teacher.compliance check - cd $(ROOT) && $(PY) -m training.datasets.validators --check-licenses - cd $(ROOT) && $(PY) -m training.datasets.build_seed - cd $(ROOT) && $(PY) -m training.datasets.build_corrupted - cd $(ROOT) && $(PY) -m training.distill.train_role_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_intent_head --check-gates - cd $(ROOT) && $(PY) -m training.distill.train_encoder_contrastive --check-gates - cd $(ROOT) && $(PY) -m training.distill.export_onnx --release - cd $(ROOT) && $(PY) -m training.distill.quantize --check-gates - cd $(ROOT) && $(PY) -m training.calibration.build_features - cd $(ROOT) && $(PY) -m training.calibration.fit_isotonic --fit-deterministic - cd $(ROOT) && $(PY) -m training.calibration.export_tables - cd $(ROOT) && $(PY) -m training.calibration.eval_calibration --check-gates - cd $(ROOT) && $(PY) -m training.distill.package_artifacts --release - cd $(ROOT) && $(PY) -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site --check-gates - @echo "release pipeline complete: dist/artifacts/ + benchmarks/results/latest.*" diff --git a/training/README.md b/training/README.md deleted file mode 100644 index 544ba2be..00000000 --- a/training/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# FreshData training pipeline (Phase 5) - -Development-time pipeline that builds, evaluates, and packages FreshData's -small local model artifacts (`fd-col-encoder-v1`, `fd-intent-v1`, -`calib-v1`). This package is **not part of the runtime**: it lives outside -`src/freshdata`, is excluded from the wheel (see `pyproject.toml` and -`.github/workflows/wheel-size.yml`), and the runtime never imports it. - -FreshData's runtime stays deterministic, offline, model-free by default, and -free of any LLM or cloud call — nothing here changes that contract. See -[docs/developer-training-pipeline.md](../docs/developer-training-pipeline.md) -for the full guide and [COMPLIANCE.md](COMPLIANCE.md) for the teacher-model -governance rules. - -## Layout - -```text -training/ - seed/ seed corpus registry, licenses, synthetic PII-shaped generators - corruptors/ ~40 deterministic, labeled corruptors (representation/semantic/context/scale) - teacher/ compliance-gated development-time teacher harness (schemas, cache, review) - datasets/ seed/corrupted dataset builders, splits, license+label validators - distill/ role-head / intent-head / optional encoder training, ONNX export, quantize, package - calibration/ feature extraction, isotonic fit, table export, eval - eval/ human-verified release-gating label store - model_cards/ the three required model cards - cache/ teacher prompt/response audit cache (gitignored content, tracked directory) -``` - -## Quick start - -```bash -make training-dev-artifacts # synthetic-only, offline, seconds -make training-release-artifacts # full gated pipeline (compliance -> package -> CleanBench) -python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --report site -``` - -Individual stages: `make training-seed`, `training-corrupt`, -`training-teacher-labels`, `training-distill`, `training-eval`, -`training-package-artifacts` (see `training/Makefile`). - -## Ground-truth policy - -Corruption metadata is the preferred ground truth; teacher output is -training data only and never gates a release on its own — release-gating -eval labels must be 100% human-verified (`training/eval/human_verified.py`). diff --git a/training/__init__.py b/training/__init__.py deleted file mode 100644 index 1cf73809..00000000 --- a/training/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -"""FreshData development-time training pipeline (Phase 5). - -This package builds, evaluates, and packages FreshData's small local model -artifacts (``fd-col-encoder-v1``, ``fd-intent-v1``, ``calib-v1``). It is a -**developer tool**, not part of the library: - -- it lives outside ``src/freshdata`` and is never shipped in the wheel; -- the runtime never imports it (guarded by tests); -- teacher models are allowed here only for development-time labeling, - paraphrasing, ambiguity adjudication, red-teaming, and rationale templates — - never at runtime, never per-cell, never as release-gating ground truth; -- ground truth comes from corruption metadata and human-verified labels. - -Entry points (see ``training/Makefile`` / root ``Makefile``):: - - make training-seed - make training-corrupt - make training-teacher-labels - make training-distill - make training-eval - make training-package-artifacts - make training-dev-artifacts - make training-release-artifacts -""" - -from __future__ import annotations - -__all__ = ["__phase__"] - -__phase__ = 5 diff --git a/training/cache/.gitkeep b/training/cache/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/training/calibration/__init__.py b/training/calibration/__init__.py deleted file mode 100644 index 228ba669..00000000 --- a/training/calibration/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Offline calibration training: features -> isotonic fit -> JSON tables. - -The runtime never trains anything; it only loads the exported ``calib-v1`` -JSON through the Phase-3 model registry. When the table is missing the -runtime stays safe and reports ``calibration_version="uncalibrated"``. -Deterministic protected-column checks are never probabilistic and are not -touched by calibration at all. -""" diff --git a/training/calibration/build_features.py b/training/calibration/build_features.py deleted file mode 100644 index 7350d951..00000000 --- a/training/calibration/build_features.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Extract calibration features from CleanBench runs. - -Each record is one semantic proposal with the deterministic feature set the -architecture specifies, plus its machine-verified outcome (``correct``) -derived from the fixture's ground truth — never from teacher output. - -CLI:: - - python -m training.calibration.build_features [--seeds 5] -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -from ..common import BUILD_DIR, REPO_ROOT, write_jsonl - -OUT_DIR = BUILD_DIR / "calibration" - -_OUTCOME_STATUSES = frozenset({"automatic", "suggested", "approved", "accepted"}) - - -def _cleanbench(): - bench_dir = str(REPO_ROOT / "benchmarks") - if bench_dir not in sys.path: - sys.path.insert(0, bench_dir) - import cleanbench # noqa: PLC0415 - - return cleanbench - - -def _canon(value: object) -> object: - from cleanbench.metrics import _canon as canon # noqa: PLC0415 - - return canon(value) - - -def action_features(action: Any, corrupted: pd.DataFrame, semantic_mode: str) -> dict[str, Any]: - metadata = getattr(action, "metadata", {}) or {} - evidence = metadata.get("evidence", []) or [] - kinds = {e.get("kind") for e in evidence if isinstance(e, dict)} - margin = None - semantic_type_confidence = None - for e in evidence: - detail = str(e.get("detail", "")) if isinstance(e, dict) else "" - if e.get("kind") == "embedding" and "margin=" in detail: - try: - margin = float(detail.split("margin=")[1].split()[0]) - except (IndexError, ValueError): - margin = None - if e.get("kind") == "semantic_type" and "confidence=" in detail: - try: - semantic_type_confidence = float(detail.split("confidence=")[1].split()[0]) - except (IndexError, ValueError): - semantic_type_confidence = None - column = getattr(action, "column", None) - column_signature = metadata.get("column_signature", {}) or {} - calibration = metadata.get("calibration", {}) or {} - raw_score = calibration.get("raw", getattr(action, "confidence", 0.0)) - series = corrupted[column] if column in corrupted.columns else pd.Series(dtype=object) - return { - "raw_score": float(raw_score), - "backend": str(metadata.get("backend", "deterministic")), - "issue_type": str(metadata.get("issue_type", "unknown")), - "risk": str(getattr(action, "risk", metadata.get("risk", "unknown"))), - "role_confidence": 1.0 if column_signature.get("role") else 0.0, - "semantic_type_confidence": semantic_type_confidence, - "distinct_support": int(series.nunique()) if len(series) else None, - "coverage": None, - "memory_support_count": sum( - 1 for e in evidence if isinstance(e, dict) and e.get("kind") == "memory_replay"), - "profile_support_count": sum( - 1 for e in evidence - if isinstance(e, dict) and str(e.get("kind", "")).startswith("profile")), - "learned_precision": metadata.get("learned_precision"), - "policy_rule_present": bool(column_signature.get("policy_rule")), - "allowed_values_present": "reference" in str(metadata.get("expert", "")) - or metadata.get("issue_type") == "reference_value", - "margin_to_second_candidate": margin, - "semantic_mode": semantic_mode, - "evidence_kinds": sorted(k for k in kinds if k), - } - - -def extract_from_run( - report: Any, truth: pd.DataFrame, corrupted: pd.DataFrame, *, semantic_mode: str = "auto" -) -> list[dict[str, Any]]: - """(features, correct) records for every scoreable semantic action.""" - records: list[dict[str, Any]] = [] - for action in getattr(report, "actions", []) or []: - if getattr(action, "step", None) != "semantic": - continue - if str(getattr(action, "status", "")) not in _OUTCOME_STATUSES: - continue - metadata = getattr(action, "metadata", {}) or {} - raw = metadata.get("raw_value") - proposed = metadata.get("proposed_value") - column = getattr(action, "column", None) - if proposed is None or column is None or column not in corrupted.columns: - continue - raw_canon = _canon(raw) - matches = [pos for pos, value in enumerate(corrupted[column].tolist()) - if _canon(value) == raw_canon] - if not matches: - continue - truth_values = truth[column].tolist() - proposed_canon = _canon(proposed) - correct = all(_canon(truth_values[pos]) == proposed_canon for pos in matches) - features = action_features(action, corrupted, semantic_mode) - features["coverage"] = round(len(matches) / max(1, len(corrupted)), 6) - features["correct"] = bool(correct) - features["confidence_reported"] = float(getattr(action, "confidence", 0.0)) - records.append(features) - return records - - -def build_features(*, seeds: int = 5, out_dir: Path | str = OUT_DIR) -> list[dict[str, Any]]: - import freshdata as fd # noqa: PLC0415 - - cleanbench = _cleanbench() - from freshdata.models import runtime as model_runtime # noqa: PLC0415 - - records: list[dict[str, Any]] = [] - for seed in range(seeds): - for maker in (cleanbench.make_t2_semantic_fixture, cleanbench.make_t3_context_fixture): - truth, corrupted, kwargs = maker(seed=seed * 10) - _, report = fd.clean(corrupted, return_report=True, **kwargs) - records.extend(extract_from_run(report, truth, corrupted)) - # Embedding-backend runs (stub encoder): the interesting curves. - truth, corrupted, kwargs = cleanbench.make_phase3_embedding_fixture(seed=seed * 10 + 5) - model_runtime.set_encoder_factory(lambda _mid: cleanbench.BigramStubEncoder()) - try: - _, report = fd.clean(corrupted, return_report=True, **kwargs) - finally: - model_runtime.set_encoder_factory(None) - records.extend(extract_from_run(report, truth, corrupted)) - write_jsonl(Path(out_dir) / "features.jsonl", records) - return records - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.build_features") - parser.add_argument("--seeds", type=int, default=5) - parser.add_argument("--out", default=str(OUT_DIR)) - args = parser.parse_args(argv) - records = build_features(seeds=args.seeds, out_dir=args.out) - backends = sorted({r["backend"] for r in records}) - print(f"calibration features: {len(records)} records, backends={backends}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/calibration/eval_calibration.py b/training/calibration/eval_calibration.py deleted file mode 100644 index bf2e92bc..00000000 --- a/training/calibration/eval_calibration.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Evaluate the exported calibration table against the release gates. - -Gates:: - - ECE <= 0.03 - precision at confidence >= 0.95 >= 0.99 - protected-column violation rate == 0 (checked by the CleanBench run) - false modification rate <= 0.1% (checked by the CleanBench run) - -The first two are computed here on a held-out feature split (never the rows -used for fitting); the last two are structural cleaner properties measured -by ``benchmarks.cleanbench`` and re-asserted by the release pipeline. - -CLI:: - - python -m training.calibration.eval_calibration [--check-gates] -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, read_json, read_jsonl, write_json -from ..datasets.splits import hash_split - -OUT_DIR = BUILD_DIR / "calibration" - -GATE_ECE = 0.03 -GATE_PRECISION_AT_95 = 0.99 - - -def apply_table( - table_json: dict[str, Any], records: list[dict[str, Any]] -) -> list[tuple[float, bool]]: - """(calibrated confidence, correct) pairs via the runtime's own loader.""" - from freshdata.semantic.scoring import _IsotonicTable # noqa: PLC0415 - - table = _IsotonicTable.from_json(json.dumps(table_json)) - pairs: list[tuple[float, bool]] = [] - for record in records: - calibrated = table.apply( - str(record["backend"]), str(record["issue_type"]), float(record["raw_score"])) - pairs.append((min(1.0, max(0.0, calibrated)), bool(record["correct"]))) - return pairs - - -def ece(pairs: list[tuple[float, bool]], bins: int = 10) -> float: - if not pairs: - return 0.0 - total = len(pairs) - out = 0.0 - for b in range(bins): - lo, hi = b / bins, (b + 1) / bins - bucket = [(c, ok) for c, ok in pairs if (c > lo or b == 0) and c <= hi] - if not bucket: - continue - accuracy = sum(ok for _, ok in bucket) / len(bucket) - mean_confidence = sum(c for c, _ in bucket) / len(bucket) - out += (len(bucket) / total) * abs(accuracy - mean_confidence) - return out - - -def precision_at(pairs: list[tuple[float, bool]], floor: float = 0.95) -> float: - bucket = [ok for c, ok in pairs if c >= floor] - return sum(bucket) / len(bucket) if bucket else 1.0 - - -def evaluate( - *, - features_path: Path | str = OUT_DIR / "features.jsonl", - table_path: Path | str = OUT_DIR / "calibration.json", - out_dir: Path | str = OUT_DIR, -) -> dict[str, Any]: - records = read_jsonl(features_path) - table_json = read_json(table_path) - # Held-out evaluation: the same split the fitting stage did NOT see. - _, holdout = hash_split( - records, - key_fn=lambda r: f"{r['backend']}|{r['issue_type']}|{r['raw_score']}|{r.get('coverage')}", - eval_fraction=0.35, - salt="calib-eval", - ) - rows = holdout if len(holdout) >= 20 else records - pairs = apply_table(table_json, rows) - metrics = { - "version": table_json.get("version", "unknown"), - "n_eval": len(pairs), - "held_out": len(holdout) >= 20, - "ece": round(ece(pairs), 4), - "precision_at_0.95": round(precision_at(pairs, 0.95), 4), - "gates": {"ece_max": GATE_ECE, "precision_at_95_min": GATE_PRECISION_AT_95}, - } - metrics["gates"]["failures"] = check_gates(metrics) - write_json(Path(out_dir) / "calib-v1.metrics.json", metrics) - return metrics - - -def check_gates(metrics: dict[str, Any]) -> list[str]: - failures = [] - if metrics["ece"] > GATE_ECE: - failures.append(f"ECE {metrics['ece']} > {GATE_ECE}") - if metrics["precision_at_0.95"] < GATE_PRECISION_AT_95: - failures.append(f"P@0.95 {metrics['precision_at_0.95']} < {GATE_PRECISION_AT_95}") - return failures - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.eval_calibration") - parser.add_argument("--features", default=str(OUT_DIR / "features.jsonl")) - parser.add_argument("--table", default=str(OUT_DIR / "calibration.json")) - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - metrics = evaluate(features_path=args.features, table_path=args.table, out_dir=args.out) - print(f"calibration eval: ECE={metrics['ece']} P@0.95={metrics['precision_at_0.95']} " - f"n={metrics['n_eval']}") - failures = metrics["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/calibration/export_tables.py b/training/calibration/export_tables.py deleted file mode 100644 index ad3e43f5..00000000 --- a/training/calibration/export_tables.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Export fitted curves as the runtime-loadable ``calib-v1`` artifact. - -The primary file is ``calibration.json`` — exactly the filename the Phase-3 -model registry expects for ``calib-v1`` — validated by round-tripping it -through the runtime's own ``_IsotonicTable.from_json``. A ``calib.json`` -alias is written alongside for the packaged artifact layout. - -CLI:: - - python -m training.calibration.export_tables [--version calib-v1] -""" - -from __future__ import annotations - -import argparse -import json -import shutil -from pathlib import Path - -from ..common import BUILD_DIR, read_json, utc_now_iso, write_json - -OUT_DIR = BUILD_DIR / "calibration" - - -def export( - curves_path: Path | str = OUT_DIR / "curves.json", - *, - version: str = "calib-v1", - out_dir: Path | str = OUT_DIR, -) -> Path: - curves = read_json(curves_path) - table = { - "version": version, - "comment": ( - "Trained offline by training/calibration (Phase 5). Deterministic " - "and memory backends are intentionally absent (identity mapping) " - "unless explicitly fitted; embedding curves are capped below 1.0." - ), - "tables": curves["tables"], - } - # Round-trip through the runtime loader: an artifact the runtime cannot - # parse must never be exported. - from freshdata.semantic.scoring import _IsotonicTable # noqa: PLC0415 - - _IsotonicTable.from_json(json.dumps(table)) - - out = Path(out_dir) - primary = write_json(out / "calibration.json", table) - shutil.copy(primary, out / "calib.json") - write_json(out / "export_meta.json", { - "version": version, - "exported_at": utc_now_iso(), - "n_backends": len(curves["tables"]), - "n_records": curves.get("n_records"), - }) - return primary - - -def metrics_path(out_dir: Path | str = OUT_DIR) -> Path: - return Path(out_dir) / "calib-v1.metrics.json" - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.export_tables") - parser.add_argument("--curves", default=str(OUT_DIR / "curves.json")) - parser.add_argument("--version", default="calib-v1") - parser.add_argument("--out", default=str(OUT_DIR)) - args = parser.parse_args(argv) - path = export(args.curves, version=args.version, out_dir=args.out) - print(f"calibration table exported: {path}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/calibration/fit_isotonic.py b/training/calibration/fit_isotonic.py deleted file mode 100644 index 733eadfa..00000000 --- a/training/calibration/fit_isotonic.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Pool-adjacent-violators isotonic regression per (backend, issue family). - -Pure numpy, deterministic. Curves are clamped monotone non-decreasing and -reduced to a compact set of knots compatible with the runtime's -``_IsotonicTable`` JSON format. Deterministic-backend proposals keep an -identity mapping by default (matching the packaged default's contract that -calibration never changes a default install's decisions) unless -``--fit-deterministic`` is passed. - -CLI:: - - python -m training.calibration.fit_isotonic -""" - -from __future__ import annotations - -import argparse -from pathlib import Path -from typing import Any - -import numpy as np - -from ..common import BUILD_DIR, read_jsonl, write_json - -OUT_DIR = BUILD_DIR / "calibration" -#: Embedding scores may never calibrate to certainty. -EMBEDDING_CEILING = 0.975 -MIN_SAMPLES_PER_CURVE = 8 - - -def pav(scores: np.ndarray, outcomes: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Pool-adjacent-violators: monotone fit of P(correct | score).""" - order = np.argsort(scores, kind="stable") - x = scores[order].astype(np.float64) - y = outcomes[order].astype(np.float64) - # Blocks: (mean, weight); merge while decreasing. - means: list[float] = [] - weights: list[float] = [] - xs: list[float] = [] - for xi, yi in zip(x, y): - means.append(yi) - weights.append(1.0) - xs.append(xi) - while len(means) > 1 and means[-2] > means[-1]: - total = weights[-2] + weights[-1] - merged = (means[-2] * weights[-2] + means[-1] * weights[-1]) / total - means[-2:] = [merged] - weights[-2:] = [total] - xs[-2:] = [xs[-1]] - # xs keeps the right edge of each block - return np.array(xs), np.array(means) - - -def fit_curve( - scores: list[float], outcomes: list[bool], *, ceiling: float = 1.0 -) -> dict[str, list[float]] | None: - """A compact monotone curve ``{"x": [...], "y": [...]}`` or None.""" - if len(scores) < MIN_SAMPLES_PER_CURVE: - return None - knot_x, knot_y = pav(np.array(scores), np.array(outcomes, dtype=float)) - knot_y = np.clip(knot_y, 0.0, ceiling) - xs = [0.0, *[round(float(v), 4) for v in knot_x]] - ys = [0.0, *[round(float(v), 4) for v in knot_y]] - # De-duplicate x while keeping monotonicity. - dedup_x: list[float] = [] - dedup_y: list[float] = [] - for xv, yv in zip(xs, ys): - if dedup_x and xv <= dedup_x[-1]: - dedup_y[-1] = max(dedup_y[-1], yv) - continue - dedup_x.append(xv) - dedup_y.append(max(yv, dedup_y[-1] if dedup_y else 0.0)) - if dedup_x[-1] < 1.0: - dedup_x.append(1.0) - dedup_y.append(dedup_y[-1]) - if len(dedup_x) < 2: - return None - return {"x": dedup_x, "y": dedup_y} - - -def fit_tables( - records: list[dict[str, Any]], *, fit_deterministic: bool = False -) -> dict[str, dict[str, dict[str, list[float]]]]: - """Curves per (backend, issue_type) plus a per-backend ``"*"`` fallback.""" - grouped: dict[tuple[str, str], tuple[list[float], list[bool]]] = {} - for record in records: - backend = str(record["backend"]) - if backend == "deterministic" and not fit_deterministic: - continue # identity by design; never change default decisions - key = (backend, str(record["issue_type"])) - scores, outcomes = grouped.setdefault(key, ([], [])) - scores.append(float(record["raw_score"])) - outcomes.append(bool(record["correct"])) - tables: dict[str, dict[str, dict[str, list[float]]]] = {} - for (backend, issue_type), (scores, outcomes) in sorted(grouped.items()): - ceiling = EMBEDDING_CEILING if backend == "embedding" else 1.0 - curve = fit_curve(scores, outcomes, ceiling=ceiling) - if curve is not None: - tables.setdefault(backend, {})[issue_type] = curve - # Per-backend fallback from all of that backend's records. - for backend in list(tables): - scores = [float(r["raw_score"]) for r in records if r["backend"] == backend] - outcomes = [bool(r["correct"]) for r in records if r["backend"] == backend] - ceiling = EMBEDDING_CEILING if backend == "embedding" else 1.0 - fallback = fit_curve(scores, outcomes, ceiling=ceiling) - if fallback is not None: - tables[backend]["*"] = fallback - return tables - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.calibration.fit_isotonic") - parser.add_argument("--features", default=str(OUT_DIR / "features.jsonl")) - parser.add_argument("--out", default=str(OUT_DIR / "curves.json")) - parser.add_argument("--fit-deterministic", action="store_true") - args = parser.parse_args(argv) - records = read_jsonl(args.features) - tables = fit_tables(records, fit_deterministic=args.fit_deterministic) - write_json(Path(args.out), {"tables": tables, "n_records": len(records)}) - n_curves = sum(len(v) for v in tables.values()) - print(f"isotonic fit: {n_curves} curves from {len(records)} records") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/common.py b/training/common.py deleted file mode 100644 index 459078f7..00000000 --- a/training/common.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Shared plumbing for the training pipeline: paths, hashing, stable JSON. - -Everything downstream (datasets, teacher cache, artifact manifests) needs the -same three primitives — a stable canonical JSON form, sha256 helpers, and the -build/output directory layout — so they live here once. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import subprocess -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -#: Repository root (training/ is a top-level package next to src/). -REPO_ROOT = Path(__file__).resolve().parent.parent -TRAINING_ROOT = REPO_ROOT / "training" -#: Intermediate outputs (never committed, never packaged). -BUILD_DIR = TRAINING_ROOT / "build" -#: Teacher prompt/response cache (audit trail; content-addressed). -CACHE_DIR = TRAINING_ROOT / "cache" -#: Final packaged artifacts (outside the wheel by construction). -ARTIFACTS_DIR = REPO_ROOT / "dist" / "artifacts" - -#: Hard ceiling for any single packaged model artifact directory. -MAX_ARTIFACT_BYTES = 60_000_000 - - -def stable_json(data: Any) -> str: - """Canonical JSON: sorted keys, no whitespace drift, utf-8 safe.""" - return json.dumps(data, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str) - - -def sha256_text(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - - -def sha256_json(data: Any) -> str: - return sha256_text(stable_json(data)) - - -def sha256_file(path: Path | str) -> str: - digest = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - -def utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat() - - -def git_commit() -> str: - """Current commit hash, or "unknown" outside a git checkout (e.g. sdist).""" - try: - out = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=10, check=False, - ) - except OSError: # pragma: no cover - git missing entirely - return "unknown" - return out.stdout.strip() if out.returncode == 0 and out.stdout.strip() else "unknown" - - -def freshdata_version() -> str: - import freshdata # noqa: PLC0415 - lazy so `training` imports stay cheap - - return str(freshdata.__version__) - - -def write_json(path: Path | str, data: Any) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(data, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8") - return path - - -def read_json(path: Path | str) -> Any: - return json.loads(Path(path).read_text(encoding="utf-8")) - - -def write_jsonl(path: Path | str, rows: list[dict[str, Any]]) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as fh: - for row in rows: - fh.write(stable_json(row) + "\n") - return path - - -def read_jsonl(path: Path | str) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - with Path(path).open("r", encoding="utf-8") as fh: - for raw_line in fh: - stripped = raw_line.strip() - if stripped: - rows.append(json.loads(stripped)) - return rows - - -def env_flag(name: str) -> bool: - return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} diff --git a/training/corruptors/__init__.py b/training/corruptors/__init__.py deleted file mode 100644 index 5b01879b..00000000 --- a/training/corruptors/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Parameterized corruption engine mirroring FreshData's repair taxonomy. - -Ground truth for training comes from *corruption metadata*: every corruptor -records exactly what it changed, what the clean value was, and whether the -change is safely repairable — teacher models never invent messy/clean pairs. -""" - -from .base import CorruptionLabel, Corruptor, apply_corruptor, compose -from .registry import CORRUPTOR_REGISTRY, FAMILIES, get_corruptor - -__all__ = [ - "CORRUPTOR_REGISTRY", - "FAMILIES", - "CorruptionLabel", - "Corruptor", - "apply_corruptor", - "compose", - "get_corruptor", -] diff --git a/training/corruptors/base.py b/training/corruptors/base.py deleted file mode 100644 index 4b9ad181..00000000 --- a/training/corruptors/base.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Corruptor core: labels, the corruptor container, application, composition. - -Rules enforced here so individual corruptors cannot get them wrong: - -- deterministic under ``seed`` (all randomness flows from one ``Random``); -- the input frame is never mutated; -- every mutated cell/row/header emits a machine-verifiable - :class:`CorruptionLabel`; -- ``ambiguous`` or ``protected`` corruptions are never labeled auto-apply; -- composition never corrupts the same cell twice, so each label's - ``clean_value`` stays the true pre-corruption value. -""" - -from __future__ import annotations - -import random -from collections.abc import Iterable -from dataclasses import dataclass, field -from typing import Any, Callable, Literal - -import pandas as pd - -Kind = Literal["cell", "row", "header", "context"] - -#: Transform families mirror FreshData's repair taxonomy. -TRANSFORM_FAMILIES = ( - "representation", - "semantic_value", - "reference_value", - "date_time", - "context_schema", - "row_structure", - "trap", -) - - -@dataclass(frozen=True) -class CorruptionLabel: - """Machine-verifiable ground truth for one injected corruption.""" - - raw_value: Any - clean_value: Any - column: str | None - transform_family: str - params: dict[str, Any] = field(default_factory=dict) - should_repair: bool = True - should_auto_apply: bool = True - risk: str = "low" - protected: bool = False - ambiguous: bool = False - corruptor: str = "" - row: int | None = None - - def to_dict(self) -> dict[str, Any]: - return { - "raw_value": self.raw_value, - "clean_value": self.clean_value, - "column": self.column, - "transform_family": self.transform_family, - "params": dict(self.params), - "should_repair": self.should_repair, - "should_auto_apply": self.should_auto_apply, - "risk": self.risk, - "protected": self.protected, - "ambiguous": self.ambiguous, - "corruptor": self.corruptor, - "row": self.row, - } - - -#: A cell transform: (clean_value, rng, params) -> corrupted value, or None -#: when the corruptor does not apply to this value. -CellFn = Callable[[str, random.Random, dict[str, Any]], Any] -#: A frame transform for row/header/context corruptors: -#: (df, rng, params) -> (corrupted_df, labels). -FrameFn = Callable[ - [pd.DataFrame, random.Random, dict[str, Any]], - "tuple[pd.DataFrame, list[CorruptionLabel]]", -] - - -@dataclass(frozen=True) -class Corruptor: - """One named, parameterized corruption with fixed label semantics.""" - - name: str - family: str - kind: Kind = "cell" - fn: CellFn | None = None - frame_fn: FrameFn | None = None - risk: str = "low" - should_repair: bool = True - should_auto_apply: bool = True - ambiguous: bool = False - protected: bool = False - params: dict[str, Any] = field(default_factory=dict) - - def __post_init__(self) -> None: - if self.family not in TRANSFORM_FAMILIES: - raise ValueError(f"{self.name}: unknown transform family {self.family!r}") - if self.ambiguous and self.should_auto_apply: - raise ValueError(f"{self.name}: ambiguous corruptors cannot be auto-apply") - if self.protected and self.should_auto_apply: - raise ValueError(f"{self.name}: protected traps cannot be auto-apply") - if self.kind == "cell" and self.fn is None: - raise ValueError(f"{self.name}: cell corruptor needs fn") - if self.kind != "cell" and self.frame_fn is None: - raise ValueError(f"{self.name}: {self.kind} corruptor needs frame_fn") - - def label_for( - self, *, raw: Any, clean: Any, column: str | None, row: int | None, - params: dict[str, Any], - ) -> CorruptionLabel: - return CorruptionLabel( - raw_value=raw, - clean_value=clean, - column=column, - transform_family=self.family, - params=params, - should_repair=self.should_repair, - should_auto_apply=self.should_auto_apply, - risk=self.risk, - protected=self.protected, - ambiguous=self.ambiguous, - corruptor=self.name, - row=row, - ) - - -def _seeded(name: str, seed: int) -> random.Random: - return random.Random(f"freshdata-corruptor:{name}:{seed}") - - -def apply_corruptor( - df: pd.DataFrame, - corruptor: Corruptor, - columns: Iterable[str] | None = None, - *, - seed: int = 0, - share: float = 0.5, - params: dict[str, Any] | None = None, - skip_cells: set[tuple[int, str]] | None = None, -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - """Apply one corruptor and return ``(corrupted_frame, labels)``. - - ``skip_cells`` (positions already corrupted by an earlier step) preserves - ground truth under composition; :func:`compose` threads it automatically. - """ - rng = _seeded(corruptor.name, seed) - merged = {**corruptor.params, **(params or {})} - if corruptor.kind != "cell": - assert corruptor.frame_fn is not None - out, labels = corruptor.frame_fn(df.copy(deep=True), rng, merged) - return out, labels - - assert corruptor.fn is not None - out = df.copy(deep=True) - if columns is not None: - targets = [c for c in columns if c in df.columns] - else: - # Default to string-shaped columns only: a bool/numeric column fed - # through a text transform (e.g. casing) would otherwise trip a - # dtype-widening cell write for no semantic reason. - targets = [c for c in df.columns if df[c].map(lambda v: isinstance(v, str)).any()] - labels: list[CorruptionLabel] = [] - skip = skip_cells or set() - for col in targets: - for pos, idx in enumerate(out.index): - if (pos, str(col)) in skip: - continue - clean = out.at[idx, col] - if not isinstance(clean, str): - continue - if pd.isna(clean) if not isinstance(clean, (list, dict)) else False: - continue - if rng.random() >= share: - continue - raw = corruptor.fn(str(clean), rng, merged) - if raw is None or str(raw) == str(clean): - continue - out.at[idx, col] = raw - # Sentinel-style corruptions destroy the original value; the only - # valid repair target is "missing", never the unrecoverable truth. - clean_target = None if merged.get("target") == "missing" else clean - labels.append(corruptor.label_for( - raw=raw, clean=clean_target, column=str(col), row=pos, params=merged, - )) - return out, labels - - -def compose( - df: pd.DataFrame, - steps: list[tuple[Corruptor, Iterable[str] | None, float]], - *, - seed: int = 0, -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - """Apply corruptors in sequence without double-corrupting any cell.""" - out = df - all_labels: list[CorruptionLabel] = [] - taken: set[tuple[int, str]] = set() - for i, (corruptor, columns, share) in enumerate(steps): - out, labels = apply_corruptor( - out, corruptor, columns, seed=seed + i * 1009, share=share, skip_cells=taken, - ) - for label in labels: - if label.row is not None and label.column is not None: - taken.add((label.row, label.column)) - all_labels.extend(labels) - return out, all_labels diff --git a/training/corruptors/context.py b/training/corruptors/context.py deleted file mode 100644 index 8f80da80..00000000 --- a/training/corruptors/context.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Context / column corruptors: schema mangling, paraphrase hooks, traps. - -Header corruptors emit labels with ``column`` = the *original* name and -``raw_value`` = the mangled alias (the "repair" is alias resolution). The two -trap corruptors label **any** mutation as a failure: a compliant cleaner must -leave those cells alone, so ``should_repair=False`` and auto-apply is banned. -""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -from .base import CorruptionLabel, Corruptor - -#: Realistic ecommerce header aliases (clean name -> plausible wild forms). -ECOMMERCE_ALIASES: dict[str, tuple[str, ...]] = { - "cust_id": ("CustomerID", "customer id", "CUST-ID", "cid"), - "email": ("e-mail address", "customer_email", "Email ID", "mail"), - "email_addr": ("e-mail", "EmailAddress", "customer email id"), - "phone": ("mobile no.", "contact_number", "Phone#", "mob"), - "mobile": ("phone_number", "Mobile No", "contact"), - "status": ("cust_status", "Account Status", "state_flag"), - "monthly_revenue": ("rev_m", "Monthly Rev.", "revenue(month)"), - "order_date": ("OrderedOn", "date_of_order", "ord_dt"), - "postal_code": ("PIN", "pincode", "zip"), - "quantity": ("qty", "Qty.", "no_of_items"), -} - -#: Deterministic Indian-English / Hinglish paraphrase hooks for context -#: sentences. Teacher models may *extend* this set (cached + reviewed); the -#: hooks themselves guarantee an offline floor. -HINGLISH_PARAPHRASES: dict[str, tuple[str, ...]] = { - "PROTECT": ("{col} kabhi mat badalna.", "{col} ko haath mat lagao.", - "{col} bilkul change nahi karna."), - "UNIQUE": ("{col} har row me alag hona chahiye.", "{col} repeat nahi hona chahiye."), - "ALLOWED_VALUES": ("{col} sirf {values} ho sakta hai.",), - "IMPUTE_IF": ("{col} khali ho to sirf {threshold}% pakka hone par bharna.",), - "LOCALE_FORMAT": ("{col} Indian phone numbers hain.",), -} - - -def _snake_camel(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if "_" in v: - parts = v.split("_") - return rng.choice(( - "".join(p.title() for p in parts), # CamelCase - parts[0] + "".join(p.title() for p in parts[1:]), # camelCase - " ".join(parts).title(), # Title Case words - "-".join(parts), # kebab-case - )) - if v != v.lower(): - return v.lower() - return v.upper() if len(v) <= 4 else v.title() - - -def _rename_headers( - df: pd.DataFrame, rng: random.Random, params: dict[str, Any], - alias_of, - corruptor_name: str, -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - share = float(params.get("share", 0.5)) - mapping: dict[str, str] = {} - labels: list[CorruptionLabel] = [] - for col in df.columns: - if rng.random() >= share: - continue - alias = alias_of(str(col), rng) - if not alias or alias == col or alias in df.columns: - continue - mapping[str(col)] = alias - labels.append(CorruptionLabel( - raw_value=alias, clean_value=str(col), column=str(col), - transform_family="context_schema", params={"alias": alias}, - should_repair=True, should_auto_apply=True, risk="low", - corruptor=corruptor_name, row=None, - )) - return df.rename(columns=mapping), labels - - -def _alias_rename_frame(df, rng, params): - generic = { - "email": ("contact_email", "e-mail"), "phone": ("tel", "phone no"), - "name": ("full name", "customer_name"), "date": ("dt", "when"), - } - - def alias_of(col: str, r: random.Random) -> str | None: - for stem, options in generic.items(): - if stem in col.lower(): - return r.choice(options) - return f"{col}_col" - - return _rename_headers(df, rng, params, alias_of, "column_alias_rename") - - -def _header_format_frame(df, rng, params): - return _rename_headers( - df, rng, params, lambda c, r: _snake_camel(c, r, params), "header_format_mangle", - ) - - -def _ecommerce_alias_frame(df, rng, params): - def alias_of(col: str, r: random.Random) -> str | None: - options = ECOMMERCE_ALIASES.get(col.lower()) - return r.choice(options) if options else None - - return _rename_headers(df, rng, params, alias_of, "ecommerce_alias_rename") - - -def _hinglish_paraphrase_frame(df, rng, params): - """Context-sentence hook: emits paraphrase labels, leaves the frame alone. - - Params: ``intent``, ``column``, and optional slot values. The label's - ``clean_value`` is the canonical English sentence the paraphrase must - compile to; consumers feed these into the intent-head training set. - """ - intent = str(params.get("intent", "PROTECT")) - column = str(params.get("column", "monthly_revenue")) - canonical = str(params.get("canonical", f"Never modify {column} values.")) - templates = HINGLISH_PARAPHRASES.get(intent, ()) - labels = [] - for template in templates: - sentence = template.format( - col=column, - values=", ".join(params.get("values", ("active", "inactive", "pending"))), - threshold=int(float(params.get("threshold", 0.95)) * 100), - ) - labels.append(CorruptionLabel( - raw_value=sentence, clean_value=canonical, column=column, - transform_family="context_schema", - params={"intent": intent, "language": "hinglish"}, - should_repair=True, should_auto_apply=True, risk="low", - corruptor="hinglish_context_paraphrase", row=None, - )) - return df, labels - - -def _duplicated_id_rows_frame(df, rng, params): - """Rows that share an ID but differ elsewhere — dedup-key bait.""" - id_column = str(params.get("id_column", df.columns[0])) - n = int(params.get("n", 2)) - if df.empty or id_column not in df.columns: - return df, [] - picks = [rng.randrange(len(df)) for _ in range(n)] - extra = df.iloc[picks].copy() - mutable = [ - c for c in df.columns - if c != id_column and df[c].map(lambda v: isinstance(v, str)).all() - ] - for offset, (_, row) in enumerate(extra.iterrows()): - if mutable: - col = rng.choice(mutable) - extra.iloc[offset, extra.columns.get_loc(col)] = f"{row[col]} (dup)" - out = pd.concat([df, extra], ignore_index=True) - labels = [ - CorruptionLabel( - raw_value=str(df.iloc[p][id_column]), clean_value=None, column=id_column, - transform_family="row_structure", params={"source_row": p, "id_column": id_column}, - should_repair=True, should_auto_apply=False, risk="medium", ambiguous=True, - corruptor="duplicated_id_rows", row=len(df) + i, - ) - for i, p in enumerate(picks) - ] - return out, labels - - -def _mutate_cells_trap(df, rng, params, *, corruptor_name: str, protected: bool): - column = str(params.get("column", "")) - if column not in df.columns or df.empty: - return df, [] - n = min(int(params.get("n", 2)), len(df)) - positions = rng.sample(range(len(df)), n) - labels = [] - for pos in positions: - clean = df.iloc[pos][column] - raw = rng.choice((f" {clean} ", f"{clean},00", str(clean).upper(), f"{clean}?")) - if str(raw) == str(clean): - raw = f" {clean} " - df.iloc[pos, df.columns.get_loc(column)] = raw - labels.append(CorruptionLabel( - raw_value=raw, clean_value=clean, column=column, - transform_family="trap", params={"trap": corruptor_name}, - should_repair=False, should_auto_apply=False, risk="high", - protected=protected, ambiguous=False, - corruptor=corruptor_name, row=pos, - )) - return df, labels - - -def _protected_trap_frame(df, rng, params): - return _mutate_cells_trap( - df, rng, params, corruptor_name="protected_column_trap", protected=True) - - -def _target_trap_frame(df, rng, params): - return _mutate_cells_trap( - df, rng, params, corruptor_name="target_column_trap", protected=False) - - -column_alias_rename = Corruptor( - name="column_alias_rename", family="context_schema", kind="header", - frame_fn=_alias_rename_frame) -header_format_mangle = Corruptor( - name="header_format_mangle", family="context_schema", kind="header", - frame_fn=_header_format_frame) -ecommerce_alias_rename = Corruptor( - name="ecommerce_alias_rename", family="context_schema", kind="header", - frame_fn=_ecommerce_alias_frame) -hinglish_context_paraphrase = Corruptor( - name="hinglish_context_paraphrase", family="context_schema", kind="context", - frame_fn=_hinglish_paraphrase_frame) -duplicated_id_rows = Corruptor( - name="duplicated_id_rows", family="row_structure", kind="row", - frame_fn=_duplicated_id_rows_frame, risk="medium", - should_auto_apply=False, ambiguous=True) -protected_column_trap = Corruptor( - name="protected_column_trap", family="trap", kind="row", - frame_fn=_protected_trap_frame, risk="high", - should_repair=False, should_auto_apply=False, protected=True) -target_column_trap = Corruptor( - name="target_column_trap", family="trap", kind="row", - frame_fn=_target_trap_frame, risk="high", - should_repair=False, should_auto_apply=False) - -CONTEXT_CORRUPTORS = ( - column_alias_rename, - header_format_mangle, - ecommerce_alias_rename, - hinglish_context_paraphrase, - duplicated_id_rows, - protected_column_trap, - target_column_trap, -) diff --git a/training/corruptors/learning.py b/training/corruptors/learning.py deleted file mode 100644 index 1bcb70e6..00000000 --- a/training/corruptors/learning.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Paired-data builders for profile learning (T4) and vendor-map corruption.""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -from .base import CorruptionLabel, Corruptor, compose -from .representation import casing_change, sentinel_injection, whitespace_insertion -from .semantic_values import allowed_value_case, phone_in_zero_prefix - - -def _vendor_code_swap(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Replace a clean label with a vendor-specific code (learnable literal map).""" - codes = params.get("vendor_map", { - "active": "A1", "inactive": "I0", "pending": "P2", "churned": "X9", - }) - return codes.get(v.strip().lower()) - - -vendor_code_swap = Corruptor( - name="vendor_code_swap", family="reference_value", fn=_vendor_code_swap, - risk="medium", should_auto_apply=False) - - -def make_paired_learning_set( - clean: pd.DataFrame, - *, - seed: int = 0, - value_columns: tuple[str, ...] = ("status",), - phone_columns: tuple[str, ...] = (), - text_columns: tuple[str, ...] = (), -) -> tuple[pd.DataFrame, pd.DataFrame, list[CorruptionLabel]]: - """Build a ``(messy, clean, labels)`` pair for ``fd.learn``. - - The corruptions chosen here are exactly the ones a learning profile can - capture: a stable vendor literal map plus representation dirt. - """ - steps: list[tuple[Corruptor, tuple[str, ...] | None, float]] = [ - (vendor_code_swap, value_columns, 0.9), - (allowed_value_case, value_columns, 0.3), - (whitespace_insertion, text_columns or value_columns, 0.3), - (casing_change, text_columns, 0.3), - (phone_in_zero_prefix, phone_columns, 0.6), - (sentinel_injection, text_columns, 0.05), - ] - steps = [(c, cols, share) for c, cols, share in steps if cols] - messy, labels = compose(clean, steps, seed=seed) - return messy, clean.copy(deep=True), labels diff --git a/training/corruptors/registry.py b/training/corruptors/registry.py deleted file mode 100644 index faba94db..00000000 --- a/training/corruptors/registry.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Registry of every corruptor, keyed by name, with family groupings.""" - -from __future__ import annotations - -from .base import Corruptor -from .context import CONTEXT_CORRUPTORS -from .learning import vendor_code_swap -from .representation import REPRESENTATION_CORRUPTORS -from .semantic_values import SEMANTIC_VALUE_CORRUPTORS - -_ALL: tuple[Corruptor, ...] = ( - *REPRESENTATION_CORRUPTORS, - *SEMANTIC_VALUE_CORRUPTORS, - *CONTEXT_CORRUPTORS, - vendor_code_swap, -) - -CORRUPTOR_REGISTRY: dict[str, Corruptor] = {c.name: c for c in _ALL} - -FAMILIES: dict[str, tuple[str, ...]] = {} -for _c in _ALL: - FAMILIES.setdefault(_c.family, ()) - FAMILIES[_c.family] = (*FAMILIES[_c.family], _c.name) - -if len(CORRUPTOR_REGISTRY) != len(_ALL): # pragma: no cover - definition bug - raise RuntimeError("duplicate corruptor names in registry") - - -def get_corruptor(name: str) -> Corruptor: - try: - return CORRUPTOR_REGISTRY[name] - except KeyError: - known = ", ".join(sorted(CORRUPTOR_REGISTRY)) - raise KeyError(f"unknown corruptor {name!r}; known: {known}") from None diff --git a/training/corruptors/representation.py b/training/corruptors/representation.py deleted file mode 100644 index be9e02e0..00000000 --- a/training/corruptors/representation.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Representation corruptors: formatting dirt with unambiguous repairs.""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -from .base import CorruptionLabel, Corruptor - -_SENTINELS = ("N/A", "null", "-", "NONE", "?", "missing") -_NA_VARIANTS = ("", "NA", "n/a", "None", " ") -_BOOL_SYNONYMS = { - "true": ("Y", "yes", "TRUE", "1", "y"), - "false": ("N", "no", "FALSE", "0", "n"), -} -_UNITS = ("kg", "g", "pcs", "ltr") - - -def _is_number(text: str) -> bool: - try: - float(text.replace(",", "")) - except ValueError: - return False - return True - - -def _whitespace(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice((" {}", "{} ", " {} ", "\t{}", "{}\t")).format(v) - - -def _casing(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not any(ch.isalpha() for ch in v): - return None - choices = [c for c in (v.upper(), v.lower(), v.title(), v.swapcase()) if c != v] - return rng.choice(choices) if choices else None - - -def _sentinel(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice(params.get("sentinels", _SENTINELS)) - - -def _na_variant(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice(_NA_VARIANTS) - - -def _mixed_dtype(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v): - return None - number = float(v.replace(",", "")) - return rng.choice((f"{number:.1f}", f"{number:.3f}", f" {number:g} ", f"'{v}'")) - - -def _currency(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v): - return None - number = float(v.replace(",", "")) - return rng.choice(( - f"₹{number:,.2f}", f"Rs. {number:.2f}", f"INR {number:,.2f}", f"{number:.2f} INR", - )) - - -def _thousands(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v) or "," in v or float(v) < 1000: - return None - return f"{float(v):,.10g}" - - -def _unit_suffix(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _is_number(v): - return None - unit = rng.choice(params.get("units", _UNITS)) - return rng.choice((f"{v}{unit}", f"{v} {unit}")) - - -def _bool_synonym(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - synonyms = _BOOL_SYNONYMS.get(v.strip().lower()) - return rng.choice(synonyms) if synonyms else None - - -def _duplicate_rows( - df: pd.DataFrame, rng: random.Random, params: dict[str, Any] -) -> tuple[pd.DataFrame, list[CorruptionLabel]]: - n = int(params.get("n_duplicates", 2)) - if df.empty or n <= 0: - return df, [] - picks = [rng.randrange(len(df)) for _ in range(n)] - extra = df.iloc[picks] - out = pd.concat([df, extra], ignore_index=True) - labels = [ - CorruptionLabel( - raw_value=f"duplicate_of_row_{p}", clean_value=None, column=None, - transform_family="row_structure", params={"source_row": p}, - should_repair=True, should_auto_apply=True, risk="low", - corruptor="duplicate_row_injection", row=len(df) + i, - ) - for i, p in enumerate(picks) - ] - return out, labels - - -whitespace_insertion = Corruptor( - name="whitespace_insertion", family="representation", fn=_whitespace) -casing_change = Corruptor( - name="casing_change", family="representation", fn=_casing) -sentinel_injection = Corruptor( - name="sentinel_injection", family="representation", fn=_sentinel, - params={"target": "missing"}) -empty_na_variants = Corruptor( - name="empty_na_variants", family="representation", fn=_na_variant, - params={"target": "missing"}) -mixed_dtype_stringification = Corruptor( - name="mixed_dtype_stringification", family="representation", fn=_mixed_dtype) -currency_formatting = Corruptor( - name="currency_formatting", family="representation", fn=_currency) -thousands_separators = Corruptor( - name="thousands_separators", family="representation", fn=_thousands) -unit_suffix_insertion = Corruptor( - name="unit_suffix_insertion", family="representation", fn=_unit_suffix, - risk="medium", should_auto_apply=False) -boolean_synonym_replacement = Corruptor( - name="boolean_synonym_replacement", family="representation", fn=_bool_synonym) -duplicate_row_injection = Corruptor( - name="duplicate_row_injection", family="row_structure", kind="row", - frame_fn=_duplicate_rows) - -REPRESENTATION_CORRUPTORS = ( - whitespace_insertion, - casing_change, - sentinel_injection, - empty_na_variants, - mixed_dtype_stringification, - currency_formatting, - thousands_separators, - unit_suffix_insertion, - boolean_synonym_replacement, - duplicate_row_injection, -) diff --git a/training/corruptors/scale.py b/training/corruptors/scale.py deleted file mode 100644 index 990be497..00000000 --- a/training/corruptors/scale.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Scale helpers for the T5 track: grow a labeled fixture without new labels. - -Tiling preserves per-cell ground truth exactly (row ``i`` of tile ``k`` maps -back to source row ``i``), so T5 measures throughput and memory on a frame -whose correctness metrics are still fully machine-verifiable. -""" - -from __future__ import annotations - -import random - -import pandas as pd - - -def scale_frame(df: pd.DataFrame, *, target_rows: int, seed: int = 0) -> pd.DataFrame: - """Tile *df* to ``target_rows``, remapping obvious ID columns to stay unique.""" - if df.empty or target_rows <= len(df): - return df.copy(deep=True) - rng = random.Random(f"freshdata-scale:{seed}") - tiles = -(-target_rows // len(df)) # ceil - parts = [] - id_columns = [ - c for c in df.columns if str(c).lower().endswith("_id") or str(c).lower() == "id" - ] - for k in range(tiles): - part = df.copy(deep=True) - for col in id_columns: - part[col] = [f"{v}-{k}" for v in part[col]] - parts.append(part) - out = pd.concat(parts, ignore_index=True).head(target_rows) - # Deterministic row shuffle so batch effects do not line up with tiles. - order = list(range(len(out))) - rng.shuffle(order) - return out.iloc[order].reset_index(drop=True) diff --git a/training/corruptors/semantic_values.py b/training/corruptors/semantic_values.py deleted file mode 100644 index ec9371c5..00000000 --- a/training/corruptors/semantic_values.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Semantic-value corruptors: emails, Indian phones, categories, dates. - -Ambiguous corruptions (two plausible repairs, dangerous near-pairs, dayfirst -puzzles) are always labeled ``should_auto_apply=False`` — a cleaner that -auto-applies them fails CleanBench by construction. -""" - -from __future__ import annotations - -import random -from typing import Any - -from .base import Corruptor - -_SPELLED = { - "0": "zero", "1": "one", "2": "two", "3": "three", "4": "four", "5": "five", - "6": "six", "7": "seven", "8": "eight", "9": "nine", "10": "ten", - "20": "twenty", "25": "twenty five", "30": "thirty", "40": "forty", - "50": "fifty", "100": "one hundred", -} -_MONTHS = ("January", "February", "March", "April", "May", "June", "July", - "August", "September", "October", "November", "December") - - -# -- emails -------------------------------------------------------------------- - -def _one_at(v: str) -> bool: - return v.count("@") == 1 and "." in v.partition("@")[2] - - -def _email_at_whitespace(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _one_at(v): - return None - local, _, domain = v.partition("@") - return f"{local}{rng.choice((' @', ' @ ', '@ '))}{domain}" - - -def _email_double_at(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - return v.replace("@", "@@", 1) if _one_at(v) else None - - -def _email_casing(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _one_at(v) or v == v.upper(): - return None - return rng.choice((v.upper(), v.title())) - - -def _email_punct_noise(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not _one_at(v): - return None - local, _, domain = v.partition("@") - return rng.choice(( - f"{local}.@{domain}", # trailing dot in local part - f"{local}@{domain},", # trailing comma - f"{local}@{domain}.", # trailing dot - f"{local.replace('.', '..', 1)}@{domain}" if "." in local else f"{local},@{domain}", - )) - - -# -- Indian phones --------------------------------------------------------------- - -def _canonical_in_phone(v: str) -> str | None: - digits = "".join(ch for ch in v if ch.isdigit()) - if v.startswith("+91") and len(digits) == 12 and digits.startswith("91"): - return digits[2:] - return None - - -def _phone_spacing(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - if ten is None: - return None - return rng.choice(( - f"{ten[:5]} {ten[5:]}", f"+91 {ten[:5]} {ten[5:]}", f"{ten[:4]} {ten[4:7]} {ten[7:]}", - )) - - -def _phone_zero_prefix(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - return f"0{ten}" if ten else None - - -def _phone_plus91_format(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - if ten is None: - return None - return rng.choice((f"(+91) {ten}", f"+91-{ten}", f"91{ten}", f"+91 {ten}")) - - -def _phone_hyphenation(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - ten = _canonical_in_phone(v) - if ten is None: - return None - return rng.choice(( - f"{ten[:5]}-{ten[5:]}", f"+91-{ten[:5]}-{ten[5:]}", f"{ten[:3]}-{ten[3:6]}-{ten[6:]}", - )) - - -def _phone_unsafe(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Digit loss/gain: the value is wrong and there is no safe repair.""" - ten = _canonical_in_phone(v) - if ten is None: - return None - cut = rng.randrange(10) - return rng.choice(( - ten[:cut] + ten[cut + 1:], # dropped digit - ten[:cut] + str(rng.randrange(10)) + ten[cut:], # inserted digit - )) - - -# -- category / reference values --------------------------------------------------- - -def _allowed_case(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if not v.isalpha(): - return None - choices = [c for c in (v.upper(), v.title(), v.swapcase()) if c != v] - return rng.choice(choices) if choices else None - - -def _allowed_whitespace(v: str, rng: random.Random, params: dict[str, Any]) -> str: - return rng.choice((f" {v}", f"{v} ", f" {v} ", f"{v}\t")) - - -def _allowed_separator(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if len(v) < 4: - return None - cut = rng.randrange(1, len(v) - 1) - return v[:cut] + rng.choice("-_ ") + v[cut:] - - -def _edit_distance_typo(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if len(v) < 4 or not v.isalpha(): - return None - pos = rng.randrange(1, len(v) - 1) - op = rng.choice(("swap", "drop", "double")) - if op == "swap": - return v[:pos] + v[pos + 1] + v[pos] + v[pos + 2:] if pos + 1 < len(v) else None - if op == "drop": - return v[:pos] + v[pos + 1:] - return v[:pos] + v[pos] + v[pos:] - - -def _ambiguous_category(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Values one edit from *two* allowed values (e.g. ``nactive``).""" - pairs = params.get("ambiguous_map", {"active": "nactive", "inactive": "nactive"}) - return pairs.get(v.strip().lower()) - - -def _country_code_ambiguity(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - codes = params.get( - "country_codes", {"india": "IN", "australia": "AU", "austria": "AT", "canada": "CA"}) - return codes.get(v.strip().lower()) - - -def _short_code_ambiguity(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - codes = params.get("short_codes", {"california": "CA", "canada": "CA", "maharashtra": "MH"}) - return codes.get(v.strip().lower()) - - -def _close_category_pair(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - """Dangerous near-pairs: Austria/Australia, May/month-vs-name, ID codes.""" - pairs = params.get("close_pairs", { - "austria": "Austrlia", "australia": "Austrlia", - "delivered": "deliverd", "returned": "returend", - }) - return pairs.get(v.strip().lower()) - - -def _spelled_number(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - return _SPELLED.get(v.strip()) - - -# -- dates ------------------------------------------------------------------------- - -def _iso_parts(v: str) -> tuple[int, int, int] | None: - parts = v.split("-") - if len(parts) != 3 or not all(p.isdigit() for p in parts): - return None - year, month, day = (int(p) for p in parts) - if not (1 <= month <= 12 and 1 <= day <= 28): - return None - return year, month, day - - -def _date_format_shuffle(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - parsed = _iso_parts(v) - if parsed is None: - return None - year, month, day = parsed - return rng.choice(( - f"{_MONTHS[month - 1]} {day}, {year}", - f"{day} {_MONTHS[month - 1]} {year}", - f"{year}/{month:02d}/{day:02d}", - )) - - -def _date_dayfirst_ambiguity(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - parsed = _iso_parts(v) - if parsed is None: - return None - year, month, day = parsed - if day > 12 or day == month: - return None # unambiguous or identity — not a dayfirst puzzle - return rng.choice((f"{month:02d}/{day:02d}/{year}", f"{day:02d}/{month:02d}/{year}")) - - -def _relative_date_phrase(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if _iso_parts(v) is None: - return None - return rng.choice(("last Tuesday", "yesterday", "two weeks ago", "next Monday")) - - -def _month_abbreviation(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - parsed = _iso_parts(v) - if parsed is None: - return None - year, month, day = parsed - abbr = _MONTHS[month - 1][:3] - return rng.choice((f"{day}-{abbr}-{year}", f"{abbr} {day} {year}", f"{day} {abbr}. {year}")) - - -def _invalid_date_phrase(v: str, rng: random.Random, params: dict[str, Any]) -> str | None: - if _iso_parts(v) is None: - return None - return rng.choice(("2025-13-45", "31/02/2024", "0000-00-00", "the 32nd of Neverember")) - - -# --------------------------------------------------------------------------- # - -spelled_number_replacement = Corruptor( - name="spelled_number_replacement", family="semantic_value", - fn=_spelled_number, risk="medium") -email_at_whitespace = Corruptor( - name="email_at_whitespace", family="semantic_value", fn=_email_at_whitespace) -email_double_at = Corruptor( - name="email_double_at", family="semantic_value", fn=_email_double_at) -email_casing = Corruptor( - name="email_casing", family="semantic_value", fn=_email_casing) -email_punct_noise = Corruptor( - name="email_punct_noise", family="semantic_value", fn=_email_punct_noise, - risk="medium") -phone_in_spacing = Corruptor( - name="phone_in_spacing", family="semantic_value", fn=_phone_spacing) -phone_in_zero_prefix = Corruptor( - name="phone_in_zero_prefix", family="semantic_value", fn=_phone_zero_prefix) -phone_in_plus91_format = Corruptor( - name="phone_in_plus91_format", family="semantic_value", fn=_phone_plus91_format) -phone_hyphenation = Corruptor( - name="phone_hyphenation", family="semantic_value", fn=_phone_hyphenation) -phone_unsafe_mutation = Corruptor( - name="phone_unsafe_mutation", family="semantic_value", fn=_phone_unsafe, - risk="high", should_repair=False, should_auto_apply=False, ambiguous=True) - -allowed_value_case = Corruptor( - name="allowed_value_case", family="reference_value", fn=_allowed_case) -allowed_value_whitespace = Corruptor( - name="allowed_value_whitespace", family="reference_value", fn=_allowed_whitespace) -allowed_value_separator = Corruptor( - name="allowed_value_separator", family="reference_value", fn=_allowed_separator) -edit_distance_typo = Corruptor( - name="edit_distance_typo", family="reference_value", fn=_edit_distance_typo, - risk="medium") -ambiguous_category = Corruptor( - name="ambiguous_category", family="reference_value", fn=_ambiguous_category, - risk="high", should_auto_apply=False, ambiguous=True) -country_code_ambiguity = Corruptor( - name="country_code_ambiguity", family="reference_value", fn=_country_code_ambiguity, - risk="high", should_auto_apply=False, ambiguous=True) -short_code_ambiguity = Corruptor( - name="short_code_ambiguity", family="reference_value", fn=_short_code_ambiguity, - risk="high", should_auto_apply=False, ambiguous=True) -close_category_pair = Corruptor( - name="close_category_pair", family="reference_value", fn=_close_category_pair, - risk="high", should_auto_apply=False, ambiguous=True) - -date_format_shuffle = Corruptor( - name="date_format_shuffle", family="date_time", fn=_date_format_shuffle) -date_dayfirst_ambiguity = Corruptor( - name="date_dayfirst_ambiguity", family="date_time", fn=_date_dayfirst_ambiguity, - risk="high", should_auto_apply=False, ambiguous=True) -relative_date_phrase = Corruptor( - name="relative_date_phrase", family="date_time", fn=_relative_date_phrase, - risk="high", should_repair=False, should_auto_apply=False, ambiguous=True) -month_name_abbreviation = Corruptor( - name="month_name_abbreviation", family="date_time", fn=_month_abbreviation) -invalid_date_phrase = Corruptor( - name="invalid_date_phrase", family="date_time", fn=_invalid_date_phrase, - risk="medium", should_auto_apply=False, params={"target": "missing"}) - -SEMANTIC_VALUE_CORRUPTORS = ( - spelled_number_replacement, - email_at_whitespace, - email_double_at, - email_casing, - email_punct_noise, - phone_in_spacing, - phone_in_zero_prefix, - phone_in_plus91_format, - phone_hyphenation, - phone_unsafe_mutation, - allowed_value_case, - allowed_value_whitespace, - allowed_value_separator, - edit_distance_typo, - ambiguous_category, - country_code_ambiguity, - short_code_ambiguity, - close_category_pair, - date_format_shuffle, - date_dayfirst_ambiguity, - relative_date_phrase, - month_name_abbreviation, - invalid_date_phrase, -) diff --git a/training/datasets/__init__.py b/training/datasets/__init__.py deleted file mode 100644 index f401c17a..00000000 --- a/training/datasets/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dataset builders: seed corpus, corrupted pairs, splits, validators.""" diff --git a/training/datasets/build_corrupted.py b/training/datasets/build_corrupted.py deleted file mode 100644 index 034c3a5e..00000000 --- a/training/datasets/build_corrupted.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Corrupt the seed corpus into labeled training pairs. - -For each seed table a fixed, versioned corruption plan runs under an explicit -seed; the output is ``(messy frame, clean frame, labels.jsonl)`` where labels -are the machine-verifiable ground truth (see ``training/corruptors/base.py``). -Every label file is validated before it is written. - -CLI:: - - python -m training.datasets.build_corrupted [--seed 0] [--dev] -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -from ..common import BUILD_DIR, utc_now_iso, write_json, write_jsonl -from ..corruptors import compose, get_corruptor -from ..corruptors.base import apply_corruptor -from ..seed.synthetic import seed_tables -from .validators import validate_labels - -DEFAULT_OUT = BUILD_DIR / "corrupted" - -#: (corruptor, columns, share) plans per seed table. ``--dev`` keeps the -#: first few steps only so the dev pipeline stays fast. -PLANS: dict[str, list[tuple[str, tuple[str, ...] | None, float]]] = { - "customers": [ - ("whitespace_insertion", ("full_name", "address", "city"), 0.25), - ("casing_change", ("city", "country"), 0.25), - ("sentinel_injection", ("address",), 0.08), - ("empty_na_variants", ("state",), 0.08), - ("email_at_whitespace", ("email",), 0.2), - ("email_double_at", ("email",), 0.15), - ("email_casing", ("email",), 0.15), - ("email_punct_noise", ("email",), 0.1), - ("phone_in_spacing", ("phone",), 0.2), - ("phone_in_zero_prefix", ("phone",), 0.2), - ("phone_in_plus91_format", ("phone",), 0.2), - ("phone_hyphenation", ("phone",), 0.15), - ("phone_unsafe_mutation", ("phone",), 0.05), - ("allowed_value_case", ("status",), 0.3), - ("allowed_value_whitespace", ("status",), 0.15), - ("allowed_value_separator", ("status",), 0.1), - ("edit_distance_typo", ("status",), 0.1), - ("ambiguous_category", ("status",), 0.05), - ("country_code_ambiguity", ("country",), 0.1), - ("close_category_pair", ("country",), 0.08), - ("date_format_shuffle", ("signup_date",), 0.25), - ("date_dayfirst_ambiguity", ("signup_date",), 0.1), - ("month_name_abbreviation", ("signup_date",), 0.15), - ("invalid_date_phrase", ("signup_date",), 0.04), - ("relative_date_phrase", ("signup_date",), 0.04), - ("boolean_synonym_replacement", ("newsletter_opt_in",), 0.4), - ], - "transactions": [ - ("currency_formatting", ("monthly_revenue",), 0.3), - ("thousands_separators", ("monthly_revenue",), 0.2), - ("mixed_dtype_stringification", ("monthly_revenue",), 0.15), - ("allowed_value_case", ("order_status",), 0.3), - ("close_category_pair", ("order_status",), 0.08), - ("date_format_shuffle", ("order_date",), 0.25), - ("whitespace_insertion", ("notes",), 0.3), - ("unit_suffix_insertion", ("monthly_revenue",), 0.05), - ], -} - -#: Frame-level steps applied after cell plans (traps, rows, headers). -FRAME_STEPS: dict[str, list[tuple[str, dict[str, Any]]]] = { - "customers": [ - ("protected_column_trap", {"column": "national_id_like", "n": 3}), - ("duplicate_row_injection", {"n_duplicates": 3}), - ("duplicated_id_rows", {"id_column": "cust_id", "n": 2}), - ], - "transactions": [ - ("target_column_trap", {"column": "monthly_revenue", "n": 2}), - ("duplicate_row_injection", {"n_duplicates": 2}), - ], -} - - -def build_corrupted( - out_dir: Path | str = DEFAULT_OUT, *, seed: int = 0, dev: bool = False -) -> dict[str, Any]: - out = Path(out_dir) - out.mkdir(parents=True, exist_ok=True) - summary: dict[str, Any] = {"built_at": utc_now_iso(), "seed": seed, "dev": dev, "tables": {}} - - for name, clean in seed_tables(seed=seed).items(): - plan = PLANS.get(name, []) - if dev: - plan = plan[:8] - steps = [(get_corruptor(cn), cols, share) for cn, cols, share in plan] - messy, labels = compose(clean, steps, seed=seed) - for corruptor_name, params in ([] if dev else FRAME_STEPS.get(name, [])): - messy, extra = apply_corruptor( - messy, get_corruptor(corruptor_name), seed=seed + 71, params=params, - ) - labels.extend(extra) - - label_dicts = [label.to_dict() for label in labels] - problems = validate_labels(label_dicts) - if problems: - raise SystemExit(f"{name}: invalid corruption labels:\n" + "\n".join(problems[:10])) - - _write_frame(out / f"{name}.clean", clean) - _write_frame(out / f"{name}.messy", messy) - write_jsonl(out / f"{name}.labels.jsonl", label_dicts) - summary["tables"][name] = { - "rows_clean": len(clean), "rows_messy": len(messy), "labels": len(label_dicts), - "families": sorted({label["transform_family"] for label in label_dicts}), - } - - write_json(out / "meta.json", summary) - return summary - - -def _write_frame(stem: Path, frame: pd.DataFrame) -> Path: - try: - path = stem.with_suffix(".parquet") - frame.to_parquet(path, index=False) - except (ImportError, ValueError): - path = stem.with_suffix(".csv") - frame.to_csv(path, index=False) - return path - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.datasets.build_corrupted") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--out", default=str(DEFAULT_OUT)) - parser.add_argument("--dev", action="store_true", help="small fast subset of corruptors") - args = parser.parse_args(argv) - summary = build_corrupted(args.out, seed=args.seed, dev=args.dev) - for table, info in summary["tables"].items(): - print(f"{table}: {info['labels']} labels over {info['rows_messy']} rows", file=sys.stdout) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/datasets/build_seed.py b/training/datasets/build_seed.py deleted file mode 100644 index 5232f99e..00000000 --- a/training/datasets/build_seed.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Build the seed corpus into ``training/build/seed/``. - -Runs the license gate first, then materializes every allowed source: -synthetic tables (parquet or csv), the hand-authored fixtures, and the -labeled context-sentence set. ``meta.json`` records provenance for each file. - -CLI:: - - python -m training.datasets.build_seed [--seed 0] [--out training/build/seed] -""" - -from __future__ import annotations - -import argparse -import shutil -import sys -from pathlib import Path - -from ..common import BUILD_DIR, TRAINING_ROOT, utc_now_iso, write_json, write_jsonl -from ..seed.synthetic import SYNTHETIC_SOURCE_ID, make_context_sentences, seed_tables -from .validators import check_licenses - -DEFAULT_OUT = BUILD_DIR / "seed" - - -def build_seed(out_dir: Path | str = DEFAULT_OUT, *, seed: int = 0) -> dict[str, object]: - problems = check_licenses() - if problems: - raise SystemExit("seed registry invalid:\n" + "\n".join(f" {p}" for p in problems)) - - out = Path(out_dir) - out.mkdir(parents=True, exist_ok=True) - files: dict[str, dict[str, object]] = {} - - for name, frame in seed_tables(seed=seed).items(): - try: - path = out / f"{name}.parquet" - frame.to_parquet(path, index=False) - except (ImportError, ValueError): # no parquet engine available - path = out / f"{name}.csv" - frame.to_csv(path, index=False) - files[path.name] = { - "source_id": SYNTHETIC_SOURCE_ID, "rows": len(frame), "synthetic": True} - - sentences = make_context_sentences(seed=seed) - sentence_path = write_jsonl(out / "context_sentences.jsonl", sentences) - files[sentence_path.name] = { - "source_id": SYNTHETIC_SOURCE_ID, "rows": len(sentences), "synthetic": True, - } - - handauthored = TRAINING_ROOT / "seed" / "sources" / "handauthored_ecommerce_v1.csv" - shutil.copy(handauthored, out / handauthored.name) - files[handauthored.name] = {"source_id": "handauthored_ecommerce_v1", "synthetic": True} - - meta = {"built_at": utc_now_iso(), "seed": seed, "files": files} - write_json(out / "meta.json", meta) - return meta - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.datasets.build_seed") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--out", default=str(DEFAULT_OUT)) - args = parser.parse_args(argv) - meta = build_seed(args.out, seed=args.seed) - print(f"seed corpus built: {len(meta['files'])} files -> {args.out}", file=sys.stdout) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/datasets/splits.py b/training/datasets/splits.py deleted file mode 100644 index e53fead3..00000000 --- a/training/datasets/splits.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Deterministic dataset splits (hash-based, author-disjoint capable).""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any, Callable - -from ..common import sha256_text - - -def split_key(item: dict[str, Any], fields: Sequence[str]) -> str: - return "|".join(str(item.get(f, "")) for f in fields) - - -def hash_split( - items: Sequence[dict[str, Any]], - *, - key_fn: Callable[[dict[str, Any]], str], - eval_fraction: float = 0.2, - salt: str = "freshdata-split", -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Split into (train, eval) by hashing a stable per-item key. - - Items sharing a key always land on the same side, so near-duplicates - (paraphrases of one sentence, variants of one column) never straddle the - boundary. - """ - train: list[dict[str, Any]] = [] - holdout: list[dict[str, Any]] = [] - threshold = int(eval_fraction * 0xFFFF) - for item in items: - bucket = int(sha256_text(f"{salt}:{key_fn(item)}")[:4], 16) - (holdout if bucket < threshold else train).append(item) - return train, holdout - - -def author_disjoint_split( - items: Sequence[dict[str, Any]], - *, - eval_authors: Sequence[str], - author_field: str = "author", -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Split so that eval authors never appear in training (paraphrase gate).""" - eval_set = set(eval_authors) - train = [i for i in items if str(i.get(author_field, "")) not in eval_set] - holdout = [i for i in items if str(i.get(author_field, "")) in eval_set] - return train, holdout diff --git a/training/datasets/validators.py b/training/datasets/validators.py deleted file mode 100644 index c797327c..00000000 --- a/training/datasets/validators.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Validators for the seed registry and corruption labels. - -``python -m training.datasets.validators --check-licenses`` is the release -gate for training-data provenance: it fails when any source has a missing -license, missing attribution, non-explicit training permission, unresolved -PII risk, or unclear commercial-use status (including share-alike licenses -without a recorded legal review). -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import Any - -from ..common import TRAINING_ROOT, read_json - -REGISTRY_PATH = TRAINING_ROOT / "seed" / "registry.json" - -#: Licenses whose commercial use is unambiguous for training. -ALLOWED_LICENSES = frozenset({ - "CC0-1.0", "PDDL-1.0", "CC-BY-4.0", "Apache-2.0", "MIT", - "BSD-3-Clause", "OGL-UK-3.0", "OGL-India", -}) -#: Share-alike / reciprocal licenses: blocked unless legal review approves. -SHARE_ALIKE_LICENSES = frozenset({ - "CC-BY-SA-4.0", "CC-BY-SA-3.0", "ODbL-1.0", "GPL-3.0", "GPL-2.0", -}) -PII_RISKS = frozenset({"none", "synthetic", "review_required"}) - -_REQUIRED_FIELDS = ( - "source_id", "name", "url", "license", "license_text_path", - "attribution", "allowed_for_training", "pii_risk", -) - - -def validate_source(entry: dict[str, Any], seed_dir: Path) -> list[str]: - """Return a list of human-readable problems for one registry entry.""" - errors: list[str] = [] - sid = str(entry.get("source_id") or "") - - for field in _REQUIRED_FIELDS: - if field not in entry or entry[field] in (None, ""): - errors.append(f"{sid}: missing required field {field!r}") - - license_id = str(entry.get("license") or "") - if license_id: - review = entry.get("legal_review") or {} - if license_id in SHARE_ALIKE_LICENSES: - approved = ( - isinstance(review, dict) - and review.get("approved") is True - and review.get("reviewer") - ) - if not approved: - errors.append( - f"{sid}: share-alike license {license_id!r} requires an approving " - "legal_review record (approved: true, reviewer set)" - ) - elif license_id not in ALLOWED_LICENSES: - errors.append( - f"{sid}: license {license_id!r} has unclear commercial-use status; " - f"allowed: {', '.join(sorted(ALLOWED_LICENSES))} " - "(or share-alike with legal review)" - ) - - if "allowed_for_training" in entry and not isinstance(entry["allowed_for_training"], bool): - errors.append(f"{sid}: allowed_for_training must be an explicit boolean") - elif entry.get("allowed_for_training") is False: - errors.append(f"{sid}: allowed_for_training is false — remove the source or approve it") - - pii = entry.get("pii_risk") - if pii is not None and pii not in PII_RISKS: - errors.append(f"{sid}: pii_risk must be one of {sorted(PII_RISKS)}, got {pii!r}") - if pii == "review_required": - review = entry.get("legal_review") or {} - if not (isinstance(review, dict) and review.get("approved") is True): - errors.append( - f"{sid}: unresolved PII risk (review_required without approving legal_review)") - - text_path = entry.get("license_text_path") - if text_path and not (seed_dir / str(text_path)).is_file(): - errors.append( - f"{sid}: license_text_path {text_path!r} does not exist under training/seed/") - return errors - - -def check_licenses(registry_path: Path | str = REGISTRY_PATH) -> list[str]: - """Validate the whole registry; return all problems (empty = pass).""" - registry_path = Path(registry_path) - if not registry_path.is_file(): - return [f"registry not found: {registry_path}"] - data = read_json(registry_path) - sources = data.get("sources") - if not isinstance(sources, list) or not sources: - return ["registry must contain a non-empty 'sources' list"] - errors: list[str] = [] - seen: set[str] = set() - for entry in sources: - if not isinstance(entry, dict): - errors.append("registry entries must be objects") - continue - sid = str(entry.get("source_id") or "") - if sid in seen: - errors.append(f"{sid}: duplicate source_id") - seen.add(sid) - errors.extend(validate_source(entry, registry_path.parent)) - return errors - - -# --------------------------------------------------------------------------- # -# Corruption-label validation -# --------------------------------------------------------------------------- # - -LABEL_FIELDS = ( - "raw_value", "clean_value", "column", "transform_family", "params", - "should_repair", "should_auto_apply", "risk", "protected", "ambiguous", -) -RISKS = frozenset({"low", "medium", "high"}) - - -def validate_label(label: dict[str, Any]) -> list[str]: - """Check one corruption label for internal consistency.""" - errors: list[str] = [] - for field in LABEL_FIELDS: - if field not in label: - errors.append(f"label missing field {field!r}") - risk = label.get("risk") - if risk not in RISKS: - errors.append(f"invalid risk {risk!r}") - if not isinstance(label.get("params", {}), dict): - errors.append("params must be a dict") - if label.get("ambiguous") and label.get("should_auto_apply"): - errors.append("ambiguous corruption must not be auto-apply") - if label.get("protected") and label.get("should_auto_apply"): - errors.append("protected-column corruption must never be auto-apply") - if label.get("should_auto_apply") and not label.get("should_repair"): - errors.append("should_auto_apply requires should_repair") - if label.get("should_repair") and label.get("raw_value") == label.get("clean_value"): - errors.append("should_repair label with identical raw/clean value") - return errors - - -def validate_labels(labels: list[dict[str, Any]]) -> list[str]: - errors: list[str] = [] - for i, label in enumerate(labels): - for problem in validate_label(label): - errors.append(f"label[{i}]: {problem}") - return errors - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.datasets.validators") - parser.add_argument("--check-licenses", action="store_true", help="validate the seed registry") - parser.add_argument("--registry", default=str(REGISTRY_PATH)) - parser.add_argument("--labels", default=None, help="optional labels JSONL to validate") - args = parser.parse_args(argv) - - failures: list[str] = [] - if args.check_licenses: - failures.extend(check_licenses(args.registry)) - if args.labels: - from ..common import read_jsonl # noqa: PLC0415 - - failures.extend(validate_labels(read_jsonl(args.labels))) - if not args.check_licenses and not args.labels: - parser.error("nothing to do: pass --check-licenses and/or --labels") - - if failures: - for problem in failures: - print(f"FAIL: {problem}", file=sys.stderr) - return 1 - print("seed registry / labels: OK") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/__init__.py b/training/distill/__init__.py deleted file mode 100644 index 6a67f62e..00000000 --- a/training/distill/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Distillation: train small heads, export, quantize, package artifacts.""" diff --git a/training/distill/export_onnx.py b/training/distill/export_onnx.py deleted file mode 100644 index 870ef1b4..00000000 --- a/training/distill/export_onnx.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Export trained heads to ONNX (release) or a documented dev stub. - -The heads are linear (``softmax(x @ W.T + b)``) so the graph is three nodes. -Building it needs the ``onnx`` package, which is deliberately **not** a core -dependency: dev environments export the portable weights JSON plus a -``tokenizer.json`` (featurizer config) and skip the ``.onnx`` file; the -release pipeline (``--release``) hard-fails without ``onnx`` so a published -artifact can never silently miss its graph. - -CLI:: - - python -m training.distill.export_onnx [--release] -""" - -from __future__ import annotations - -import argparse -import importlib.util -import json -import shutil -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, write_json -from .linear import LinearHead - -OUT_DIR = BUILD_DIR / "export" - -HEADS = { - "fd-role-head-v1": BUILD_DIR / "role_head" / "role_head.weights.json", - "fd-intent-v1": BUILD_DIR / "intent_head" / "intent_head.weights.json", -} - - -def onnx_available() -> bool: - return importlib.util.find_spec("onnx") is not None - - -def export_head_onnx(head: LinearHead, path: Path) -> Path: # pragma: no cover - needs onnx - """Build the 3-node linear-head graph with the real onnx package.""" - import onnx # noqa: PLC0415 - from onnx import TensorProto, helper, numpy_helper # noqa: PLC0415 - - dim = head.weights.shape[1] - n_classes = head.weights.shape[0] - weight_init = numpy_helper.from_array(head.weights.T.copy(), name="W") - bias_init = numpy_helper.from_array(head.bias.copy(), name="b") - graph = helper.make_graph( - nodes=[ - helper.make_node("MatMul", ["features", "W"], ["logits_raw"]), - helper.make_node("Add", ["logits_raw", "b"], ["logits"]), - helper.make_node("Softmax", ["logits"], ["probabilities"], axis=-1), - ], - name="freshdata_linear_head", - inputs=[helper.make_tensor_value_info("features", TensorProto.FLOAT, [None, dim])], - outputs=[helper.make_tensor_value_info( - "probabilities", TensorProto.FLOAT, [None, n_classes])], - initializer=[weight_init, bias_init], - ) - model = helper.make_model(graph, producer_name="freshdata-training") - onnx.checker.check_model(model) - path.parent.mkdir(parents=True, exist_ok=True) - onnx.save(model, str(path)) - return path - - -def tokenizer_config(head: LinearHead) -> dict[str, Any]: - """The featurizer config, playing the role of ``tokenizer.json``.""" - return { - "type": "freshdata-hashed-ngram-featurizer", - "featurizer": head.featurizer.to_dict(), - "classes": list(head.classes), - "abstain_class": head.abstain_class, - "abstain_threshold": head.abstain_threshold, - } - - -def export_all(*, out_dir: Path | str = OUT_DIR, release: bool = False) -> dict[str, Any]: - out = Path(out_dir) - have_onnx = onnx_available() - if release and not have_onnx: - raise SystemExit( - "release export requires the 'onnx' package: pip install onnx " - "(dev exports run without it, but a release artifact must ship a real graph)" - ) - summary: dict[str, Any] = {"onnx_available": have_onnx, "release": release, "heads": {}} - for model_id, weights_path in HEADS.items(): - if not weights_path.is_file(): - summary["heads"][model_id] = {"status": "missing", "weights": str(weights_path)} - continue - head = LinearHead.load(weights_path) - model_dir = out / model_id - model_dir.mkdir(parents=True, exist_ok=True) - shutil.copy(weights_path, model_dir / "weights.json") - (model_dir / "tokenizer.json").write_text( - json.dumps(tokenizer_config(head), indent=2, sort_keys=True), encoding="utf-8") - entry: dict[str, Any] = {"status": "exported", "files": ["weights.json", "tokenizer.json"]} - if have_onnx: # pragma: no cover - needs onnx installed - export_head_onnx(head, model_dir / "model.onnx") - entry["files"].append("model.onnx") - else: - entry["note"] = "dev export: model.onnx omitted (onnx package not installed)" - summary["heads"][model_id] = entry - write_json(out / "export_summary.json", summary) - return summary - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.export_onnx") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--release", action="store_true") - args = parser.parse_args(argv) - summary = export_all(out_dir=args.out, release=args.release) - for model_id, entry in summary["heads"].items(): - print(f"{model_id}: {entry['status']} {entry.get('files', '')}", file=sys.stdout) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/linear.py b/training/distill/linear.py deleted file mode 100644 index a1ab9d11..00000000 --- a/training/distill/linear.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Deterministic pure-numpy linear heads over hashed character n-grams. - -The Phase-5 heads (semantic-type role head, context intent head) are tiny -multinomial logistic regressions: strong enough for the eval corpora, small -enough to export, and — crucially — trainable with zero optional -dependencies so the dev pipeline runs anywhere the test suite runs. -Everything is deterministic: hashing uses sha256, training is full-batch -gradient descent from a zero init (no RNG at all). -""" - -from __future__ import annotations - -import base64 -import hashlib -import json -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path - -import numpy as np - -FEATURIZER_VERSION = "hashed-ngram-1" - - -@dataclass(frozen=True) -class FeaturizerConfig: - dim: int = 2048 - ngram_sizes: tuple[int, ...] = (2, 3) - lowercase: bool = True - #: Hash word unigrams/bigrams too — intent-bearing words ("unique", - #: "never modify") generalize across paraphrases better than char grams. - word_ngrams: bool = True - - def to_dict(self) -> dict[str, object]: - return { - "version": FEATURIZER_VERSION, - "dim": self.dim, - "ngram_sizes": list(self.ngram_sizes), - "lowercase": self.lowercase, - "word_ngrams": self.word_ngrams, - } - - @classmethod - def from_dict(cls, data: dict[str, object]) -> FeaturizerConfig: - return cls( - dim=int(data["dim"]), # type: ignore[arg-type] - ngram_sizes=tuple(data["ngram_sizes"]), # type: ignore[arg-type] - lowercase=bool(data["lowercase"]), - word_ngrams=bool(data.get("word_ngrams", False)), - ) - - -def _bucket(gram: str, dim: int) -> tuple[int, float]: - digest = hashlib.sha256(gram.encode("utf-8")).digest() - index = int.from_bytes(digest[:4], "big") % dim - sign = 1.0 if digest[4] & 1 else -1.0 - return index, sign - - -def featurize(texts: Sequence[str], config: FeaturizerConfig) -> np.ndarray: - """L2-normalized hashed n-gram counts, shape ``(n, dim)`` float32.""" - out = np.zeros((len(texts), config.dim), dtype=np.float32) - for i, text in enumerate(texts): - body = str(text) - if config.lowercase: - body = body.casefold() - padded = f"^{body}$" - for n in config.ngram_sizes: - for j in range(max(1, len(padded) - n + 1)): - index, sign = _bucket(f"{n}:{padded[j:j + n]}", config.dim) - out[i, index] += sign - if config.word_ngrams: - words = body.split() - for word in words: - index, sign = _bucket(f"w1:{word}", config.dim) - out[i, index] += 2.0 * sign - for j in range(len(words) - 1): - index, sign = _bucket(f"w2:{words[j]} {words[j + 1]}", config.dim) - out[i, index] += 2.0 * sign - norm = float(np.linalg.norm(out[i])) - if norm: - out[i] /= norm - return out - - -@dataclass -class LinearHead: - """Multinomial logistic regression with optional abstention.""" - - classes: tuple[str, ...] - weights: np.ndarray # (C, D) float32 - bias: np.ndarray # (C,) float32 - featurizer: FeaturizerConfig - abstain_class: str | None = "unknown" - abstain_threshold: float = 0.5 - - # -- training ---------------------------------------------------------------- - - @classmethod - def train( - cls, - texts: Sequence[str], - labels: Sequence[str], - *, - classes: Sequence[str], - featurizer: FeaturizerConfig | None = None, - epochs: int = 300, - lr: float = 2.0, - l2: float = 1e-4, - class_weights: dict[str, float] | None = None, - abstain_class: str | None = "unknown", - abstain_threshold: float = 0.5, - ) -> LinearHead: - config = featurizer or FeaturizerConfig() - class_list = tuple(classes) - index = {c: i for i, c in enumerate(class_list)} - x = featurize(texts, config) - y = np.array([index[label] for label in labels], dtype=np.int64) - n, d = x.shape - c = len(class_list) - weights = np.zeros((c, d), dtype=np.float64) - bias = np.zeros(c, dtype=np.float64) - sample_weight = np.ones(n, dtype=np.float64) - if class_weights: - for i, label in enumerate(labels): - sample_weight[i] = float(class_weights.get(label, 1.0)) - sample_weight /= sample_weight.mean() - onehot = np.zeros((n, c), dtype=np.float64) - onehot[np.arange(n), y] = 1.0 - velocity_w = np.zeros_like(weights) - velocity_b = np.zeros_like(bias) - momentum = 0.9 - for _ in range(epochs): - logits = x @ weights.T + bias - logits -= logits.max(axis=1, keepdims=True) - probs = np.exp(logits) - probs /= probs.sum(axis=1, keepdims=True) - grad = (probs - onehot) * sample_weight[:, None] - grad_w = grad.T @ x / n + l2 * weights - grad_b = grad.mean(axis=0) - velocity_w = momentum * velocity_w - lr * grad_w - velocity_b = momentum * velocity_b - lr * grad_b - weights += velocity_w - bias += velocity_b - return cls( - classes=class_list, - weights=weights.astype(np.float32), - bias=bias.astype(np.float32), - featurizer=config, - abstain_class=abstain_class, - abstain_threshold=abstain_threshold, - ) - - # -- inference ---------------------------------------------------------------- - - def predict_proba(self, texts: Sequence[str]) -> np.ndarray: - x = featurize(texts, self.featurizer) - logits = x @ self.weights.T.astype(np.float64) + self.bias.astype(np.float64) - logits -= logits.max(axis=1, keepdims=True) - probs = np.exp(logits) - probs /= probs.sum(axis=1, keepdims=True) - return probs - - def predict(self, texts: Sequence[str]) -> list[tuple[str, float]]: - """(label, confidence) per text, abstaining below the threshold.""" - probs = self.predict_proba(texts) - out: list[tuple[str, float]] = [] - for row in probs: - best = int(row.argmax()) - label, confidence = self.classes[best], float(row[best]) - if ( - self.abstain_class is not None - and confidence < self.abstain_threshold - and label != self.abstain_class - ): - label = self.abstain_class - out.append((label, round(confidence, 6))) - return out - - # -- persistence ---------------------------------------------------------------- - - def to_dict(self) -> dict[str, object]: - return { - "format": "freshdata-linear-head-1", - "classes": list(self.classes), - "featurizer": self.featurizer.to_dict(), - "abstain_class": self.abstain_class, - "abstain_threshold": self.abstain_threshold, - "weights_b64": base64.b64encode(self.weights.astype(np.float32).tobytes()).decode(), - "bias_b64": base64.b64encode(self.bias.astype(np.float32).tobytes()).decode(), - "shape": list(self.weights.shape), - } - - def save(self, path: Path | str) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(self.to_dict(), sort_keys=True), encoding="utf-8") - return path - - @classmethod - def load(cls, path: Path | str) -> LinearHead: - data = json.loads(Path(path).read_text(encoding="utf-8")) - shape = tuple(int(v) for v in data["shape"]) - weights = np.frombuffer( - base64.b64decode(data["weights_b64"]), dtype=np.float32 - ).reshape(shape).copy() - bias = np.frombuffer(base64.b64decode(data["bias_b64"]), dtype=np.float32).copy() - return cls( - classes=tuple(data["classes"]), - weights=weights, - bias=bias, - featurizer=FeaturizerConfig.from_dict(data["featurizer"]), - abstain_class=data.get("abstain_class"), - abstain_threshold=float(data.get("abstain_threshold", 0.5)), - ) - - -# --------------------------------------------------------------------------- # -# int8 quantization (our own artifact format; ONNX quantization is optional) -# --------------------------------------------------------------------------- # - -def quantize_int8(head: LinearHead) -> dict[str, object]: - """Symmetric per-row int8 quantization of the weight matrix.""" - scales = np.abs(head.weights).max(axis=1) / 127.0 - scales[scales == 0.0] = 1.0 - quantized = np.clip(np.round(head.weights / scales[:, None]), -127, 127).astype(np.int8) - return { - "format": "freshdata-linear-head-int8-1", - "classes": list(head.classes), - "featurizer": head.featurizer.to_dict(), - "abstain_class": head.abstain_class, - "abstain_threshold": head.abstain_threshold, - "weights_int8_b64": base64.b64encode(quantized.tobytes()).decode(), - "scales_b64": base64.b64encode(scales.astype(np.float32).tobytes()).decode(), - "bias_b64": base64.b64encode(head.bias.astype(np.float32).tobytes()).decode(), - "shape": list(head.weights.shape), - } - - -def dequantize(data: dict[str, object]) -> LinearHead: - shape = tuple(int(v) for v in data["shape"]) # type: ignore[arg-type] - quantized = np.frombuffer( - base64.b64decode(data["weights_int8_b64"]), dtype=np.int8 # type: ignore[arg-type] - ).reshape(shape) - scales = np.frombuffer(base64.b64decode(data["scales_b64"]), dtype=np.float32) # type: ignore[arg-type] - bias = np.frombuffer(base64.b64decode(data["bias_b64"]), dtype=np.float32).copy() # type: ignore[arg-type] - weights = (quantized.astype(np.float32) * scales[:, None]).astype(np.float32) - return LinearHead( - classes=tuple(data["classes"]), # type: ignore[arg-type] - weights=weights, - bias=bias, - featurizer=FeaturizerConfig.from_dict(data["featurizer"]), # type: ignore[arg-type] - abstain_class=data.get("abstain_class"), # type: ignore[arg-type] - abstain_threshold=float(data.get("abstain_threshold", 0.5)), # type: ignore[arg-type] - ) - - -# --------------------------------------------------------------------------- # -# shared metrics -# --------------------------------------------------------------------------- # - -def macro_f1(y_true: Sequence[str], y_pred: Sequence[str], classes: Sequence[str]) -> float: - scores = per_class_f1(y_true, y_pred, classes) - present = [f1 for cls, f1 in scores.items() if any(t == cls for t in y_true)] - return float(np.mean(present)) if present else 0.0 - - -def per_class_f1( - y_true: Sequence[str], y_pred: Sequence[str], classes: Sequence[str] -) -> dict[str, float]: - out: dict[str, float] = {} - for cls in classes: - tp = sum(1 for t, p in zip(y_true, y_pred) if t == cls and p == cls) - fp = sum(1 for t, p in zip(y_true, y_pred) if t != cls and p == cls) - fn = sum(1 for t, p in zip(y_true, y_pred) if t == cls and p != cls) - denominator = 2 * tp + fp + fn - out[cls] = (2 * tp / denominator) if denominator else 0.0 - return out - - -def confusion( - y_true: Sequence[str], y_pred: Sequence[str], classes: Sequence[str] -) -> dict[str, dict[str, int]]: - matrix = {t: dict.fromkeys(classes, 0) for t in classes} - for t, p in zip(y_true, y_pred): - matrix[t][p] += 1 - return matrix diff --git a/training/distill/package_artifacts.py b/training/distill/package_artifacts.py deleted file mode 100644 index a6d44f1d..00000000 --- a/training/distill/package_artifacts.py +++ /dev/null @@ -1,248 +0,0 @@ -"""Package trained artifacts into ``dist/artifacts/`` with manifests. - -Layout (one directory per model id, outside the wheel by construction):: - - dist/artifacts/ - fd-col-encoder-v1/ model.onnx? tokenizer.json role_head.* manifest.json model_card.md - fd-intent-v1/ model.onnx? tokenizer.json weights.* manifest.json model_card.md - calib-v1/ calib.json calibration.json manifest.json model_card.md - -Validation fails on: SHA mismatch, missing model card, missing license -summary, missing eval metrics, size over the configured limit, or any -artifact file that would land inside the wheel package tree -(``src/freshdata``). - -CLI:: - - python -m training.distill.package_artifacts [--release] -""" - -from __future__ import annotations - -import argparse -import shutil -import sys -from pathlib import Path -from typing import Any - -from ..common import ( - ARTIFACTS_DIR, - BUILD_DIR, - MAX_ARTIFACT_BYTES, - REPO_ROOT, - TRAINING_ROOT, - freshdata_version, - git_commit, - read_json, - sha256_file, - utc_now_iso, - write_json, -) - -MODEL_CARDS = TRAINING_ROOT / "model_cards" -EXPORT_DIR = BUILD_DIR / "export" -CALIBRATION_DIR = BUILD_DIR / "calibration" - -#: model_id -> (card file, source files: {artifact name: build path}) -_SPECS: dict[str, dict[str, Any]] = { - "fd-col-encoder-v1": { - "card": "fd-col-encoder-v1.md", - "license": "Apache-2.0", - "training_data_summary": ( - "Synthetic PII-shaped seed corpus + corruptor-derived labels " - "(training/seed/registry.json); no real PII, no scraped data." - ), - "files": { - "tokenizer.json": EXPORT_DIR / "fd-role-head-v1" / "tokenizer.json", - "role_head.weights.json": EXPORT_DIR / "fd-role-head-v1" / "weights.json", - "role_head.weights.int8.json": EXPORT_DIR / "fd-role-head-v1" / "weights.int8.json", - }, - "optional_files": { - "model.onnx": EXPORT_DIR / "fd-role-head-v1" / "model.onnx", - "role_head.onnx": EXPORT_DIR / "fd-role-head-v1" / "model.onnx", - }, - "metrics": BUILD_DIR / "role_head" / "role_head.metrics.json", - }, - "fd-intent-v1": { - "card": "fd-intent-v1.md", - "license": "Apache-2.0", - "training_data_summary": ( - "Phase-1 golden context corpus + synthetic/Hinglish paraphrase set " - "+ corruptor context variants; author-disjoint human-verified eval." - ), - "files": { - "tokenizer.json": EXPORT_DIR / "fd-intent-v1" / "tokenizer.json", - "weights.json": EXPORT_DIR / "fd-intent-v1" / "weights.json", - "weights.int8.json": EXPORT_DIR / "fd-intent-v1" / "weights.int8.json", - }, - "optional_files": {"model.onnx": EXPORT_DIR / "fd-intent-v1" / "model.onnx"}, - "metrics": BUILD_DIR / "intent_head" / "intent_head.metrics.json", - }, - "calib-v1": { - "card": "calib-v1.md", - "license": "Apache-2.0", - "training_data_summary": ( - "CleanBench T1-T4 fixture runs with corruptor ground truth; " - "isotonic fit per (backend, issue family)." - ), - "files": { - "calib.json": CALIBRATION_DIR / "calib.json", - "calibration.json": CALIBRATION_DIR / "calibration.json", - }, - "optional_files": {}, - "metrics": CALIBRATION_DIR / "calib-v1.metrics.json", - }, -} - - -class PackagingError(RuntimeError): - """Artifact packaging failed validation.""" - - -def _quantization(model_id: str, files: dict[str, Path]) -> str: - if model_id == "calib-v1": - return "json" - if any(name.endswith(".onnx") for name in files): - return "int8" - return "int8-json" # dev artifacts: our own int8 weight format, no onnx - - -def package_one(model_id: str, *, out_root: Path, release: bool) -> dict[str, Any]: - spec = _SPECS[model_id] - sources: dict[str, Path] = {} - for name, path in spec["files"].items(): - if not Path(path).is_file(): - raise PackagingError(f"{model_id}: missing build input {path}") - sources[name] = Path(path) - for name, path in spec["optional_files"].items(): - if Path(path).is_file() and name not in sources: - sources[name] = Path(path) - elif release and name == "model.onnx": - raise PackagingError(f"{model_id}: release packaging requires {path}") - - metrics_path = Path(spec["metrics"]) - if not metrics_path.is_file(): - raise PackagingError(f"{model_id}: missing eval metrics {metrics_path}") - metrics = read_json(metrics_path) - - card_source = MODEL_CARDS / spec["card"] - if not card_source.is_file(): - raise PackagingError(f"{model_id}: missing model card {card_source}") - - target = out_root / model_id - if target.exists(): - shutil.rmtree(target) - target.mkdir(parents=True) - - copied: list[dict[str, Any]] = [] - total = 0 - for name, source in sorted(sources.items()): - destination = target / name - shutil.copy(source, destination) - size = destination.stat().st_size - total += size - copied.append({"name": name, "sha256": sha256_file(destination), "size_bytes": size}) - shutil.copy(card_source, target / "model_card.md") - shutil.copy(metrics_path, target / "eval_metrics.json") - - if total > MAX_ARTIFACT_BYTES: - raise PackagingError( - f"{model_id}: artifact size {total} exceeds limit {MAX_ARTIFACT_BYTES}") - - primary = copied[0] - manifest = { - "model_id": model_id, - "version": model_id.rsplit("-v", 1)[-1], - "freshdata_min_version": freshdata_version(), - "sha256": primary["sha256"], - "size_bytes": total, - "license": spec["license"], - "training_data_summary": spec["training_data_summary"], - "eval_summary": {k: v for k, v in metrics.items() - if isinstance(v, (int, float, str))}, - "quantization": _quantization(model_id, sources), - "files": copied, - "created_at": utc_now_iso(), - "git_commit": git_commit(), - "release": release, - } - write_json(target / "manifest.json", manifest) - return manifest - - -def validate_package(model_id: str, *, out_root: Path) -> list[str]: - """Independent re-validation of a packaged artifact directory.""" - problems: list[str] = [] - target = out_root / model_id - manifest_path = target / "manifest.json" - if not manifest_path.is_file(): - return [f"{model_id}: manifest.json missing"] - manifest = read_json(manifest_path) - for entry in manifest.get("files", []): - path = target / entry["name"] - if not path.is_file(): - problems.append(f"{model_id}: file {entry['name']} listed but missing") - continue - if sha256_file(path) != entry["sha256"]: - problems.append(f"{model_id}: SHA mismatch for {entry['name']}") - if not (target / "model_card.md").is_file(): - problems.append(f"{model_id}: model card missing") - if not manifest.get("license"): - problems.append(f"{model_id}: license summary missing") - if not manifest.get("eval_summary"): - problems.append(f"{model_id}: eval metrics missing") - if int(manifest.get("size_bytes", 0)) > MAX_ARTIFACT_BYTES: - problems.append(f"{model_id}: size over limit") - problems.extend(_wheel_guard(target)) - return problems - - -def _wheel_guard(target: Path) -> list[str]: - """No packaged artifact may live inside the wheel package tree.""" - problems = [] - wheel_root = (REPO_ROOT / "src" / "freshdata").resolve() - for path in target.rglob("*"): - try: - path.resolve().relative_to(wheel_root) - except ValueError: - continue - problems.append(f"artifact file {path} is inside src/freshdata (would ship in wheel)") - stray = list(wheel_root.rglob("*.onnx")) - problems.extend(f"model weights inside wheel tree: {p}" for p in stray) - return problems - - -def package_all(*, out_root: Path | str = ARTIFACTS_DIR, release: bool = False) -> dict[str, Any]: - root = Path(out_root) - manifests = {} - for model_id in _SPECS: - manifests[model_id] = package_one(model_id, out_root=root, release=release) - problems = validate_package(model_id, out_root=root) - if problems: - raise PackagingError("; ".join(problems)) - write_json(root / "artifacts_index.json", { - "created_at": utc_now_iso(), - "git_commit": git_commit(), - "artifacts": {mid: m["sha256"] for mid, m in manifests.items()}, - }) - return manifests - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.package_artifacts") - parser.add_argument("--out", default=str(ARTIFACTS_DIR)) - parser.add_argument("--release", action="store_true") - args = parser.parse_args(argv) - try: - manifests = package_all(out_root=args.out, release=args.release) - except PackagingError as exc: - print(f"PACKAGING FAIL: {exc}", file=sys.stderr) - return 1 - for model_id, manifest in manifests.items(): - print(f"{model_id}: sha256={manifest['sha256'][:16]}… " - f"size={manifest['size_bytes']} files={len(manifest['files'])}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/quantize.py b/training/distill/quantize.py deleted file mode 100644 index 8080812b..00000000 --- a/training/distill/quantize.py +++ /dev/null @@ -1,93 +0,0 @@ -"""int8 quantization of the exported heads, with an accuracy-delta gate. - -Quantization is symmetric per-row int8 on the weight matrix (see -``linear.quantize_int8``). The gate re-evaluates both fp32 and int8 heads on -the human-verified eval sets and fails when any headline metric moves by -0.5 percentage points or more. When ``onnxruntime`` is installed the ONNX -graph is additionally quantized; that path is release-only. - -CLI:: - - python -m training.distill.quantize [--check-gates] -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, write_json -from .linear import LinearHead, dequantize, quantize_int8 - -OUT_DIR = BUILD_DIR / "export" -MAX_METRIC_DELTA = 0.005 # 0.5 percentage points - - -def _headline_metric(model_id: str, head: LinearHead) -> float: - """One comparable accuracy number per head on its human-verified eval.""" - if model_id == "fd-role-head-v1": - from .train_role_head import evaluate, load_human_eval # noqa: PLC0415 - - return float(evaluate(head, load_human_eval())["macro_f1"]) - from .train_intent_head import build_dataset, evaluate # noqa: PLC0415 - - _, eval_rows = build_dataset() - return float(evaluate(head, eval_rows)["exact_intent_accuracy"]) - - -def quantize_all(*, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - summary: dict[str, Any] = {"max_metric_delta": MAX_METRIC_DELTA, "heads": {}} - for model_id in ("fd-role-head-v1", "fd-intent-v1"): - weights = out / model_id / "weights.json" - if not weights.is_file(): - summary["heads"][model_id] = {"status": "missing"} - continue - head = LinearHead.load(weights) - quantized = quantize_int8(head) - int8_path = out / model_id / "weights.int8.json" - int8_path.write_text(json.dumps(quantized, sort_keys=True), encoding="utf-8") - fp32_metric = _headline_metric(model_id, head) - int8_metric = _headline_metric(model_id, dequantize(quantized)) - delta = abs(fp32_metric - int8_metric) - summary["heads"][model_id] = { - "status": "quantized", - "fp32_metric": round(fp32_metric, 4), - "int8_metric": round(int8_metric, 4), - "delta": round(delta, 4), - "within_tolerance": delta < MAX_METRIC_DELTA, - } - write_json(out / "quantize_summary.json", summary) - return summary - - -def check_gates(summary: dict[str, Any]) -> list[str]: - failures = [] - for model_id, entry in summary["heads"].items(): - if entry.get("status") == "quantized" and not entry["within_tolerance"]: - failures.append( - f"{model_id}: int8 vs fp32 delta {entry['delta']} >= {MAX_METRIC_DELTA}") - return failures - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.quantize") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - summary = quantize_all(out_dir=args.out) - for model_id, entry in summary["heads"].items(): - print(f"{model_id}: {entry.get('status')} delta={entry.get('delta')}") - failures = check_gates(summary) - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/train_encoder_contrastive.py b/training/distill/train_encoder_contrastive.py deleted file mode 100644 index c313aa99..00000000 --- a/training/distill/train_encoder_contrastive.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Optional contrastive distillation for the column/value encoder. - -Phase-3's encoder interface (:class:`freshdata.models.runtime.LocalEncoder`) -is an opaque ONNX runtime; contrastive fine-tuning needs a tensor framework. -This module therefore has two layers: - -- **pair building + safety evaluation** always run (pure numpy): alias - pairs, allowed/corrupted pairs, and the *dangerous negatives* whose - false-merge rate must stay exactly zero; -- **training** runs only when ``torch`` is installed; otherwise the run is - recorded as ``skipped`` and the Phase-3 baseline encoder is retained. That - is an explicitly supported outcome, not a failure. - -Gates:: - - resolver accuracy >= Phase-3 baseline - dangerous-negative false-merge rate == 0 - (the embedding backend never auto-applies ambiguous merges — enforced at - runtime by the policy gate, re-checked here on the eval pairs) - -CLI:: - - python -m training.distill.train_encoder_contrastive [--check-gates] -""" - -from __future__ import annotations - -import argparse -import importlib.util -import random -import sys -from collections.abc import Sequence -from pathlib import Path -from typing import Any - -import numpy as np - -from ..common import BUILD_DIR, write_json -from ..corruptors.context import ECOMMERCE_ALIASES -from ..corruptors.registry import get_corruptor - -OUT_DIR = BUILD_DIR / "encoder" - -#: Distinct dangerous negatives: pairs that must never merge. -DANGEROUS_NEGATIVES: tuple[tuple[str, str], ...] = ( - ("Austria", "Australia"), - ("CA California", "Canada"), - ("IN India", "inactive"), - ("May month", "May name"), - ("ID code", "category code"), -) - -#: Auto-merge margin mirroring the runtime embedding backend's behavior: -#: an ambiguous top-2 (small margin) is never auto-applied. -MERGE_MARGIN = 0.05 - - -class HashedBigramEncoder: - """Phase-3 baseline stand-in: deterministic char-bigram embedding.""" - - dim = 256 - - def encode_texts(self, texts: Sequence[str]) -> np.ndarray: - import hashlib # noqa: PLC0415 - - out = np.zeros((len(texts), self.dim), dtype=np.float32) - for i, text in enumerate(texts): - padded = f"^{str(text).strip().casefold()}$" - grams = [padded[j:j + 2] for j in range(len(padded) - 1)] or [padded] - for gram in grams: - digest = hashlib.sha256(gram.encode("utf-8")).digest() - out[i, digest[0]] += 1.0 - out[i, 128 + digest[1] % 128] += 1.0 - norm = float(np.linalg.norm(out[i])) - if norm: - out[i] /= norm - return out - - -def build_pairs(*, seed: int = 0) -> dict[str, list[tuple[str, str]]]: - """Contrastive training/eval pairs, including the dangerous negatives.""" - rng = random.Random(f"encoder-pairs:{seed}") - positives: list[tuple[str, str]] = [] - for column, aliases in ECOMMERCE_ALIASES.items(): - for alias in aliases: - positives.append((column, alias)) - typo = get_corruptor("edit_distance_typo") - assert typo.fn is not None - for value in ("active", "inactive", "pending", "delivered", "returned", - "cancelled", "shipped", "placed"): - for _ in range(3): - corrupted = typo.fn(value, rng, {}) - if corrupted: - positives.append((value, corrupted)) - negatives = list(DANGEROUS_NEGATIVES) - # Hard-negative mining from the ambiguity corruptors: values one edit - # from two references are negatives against *both*. - negatives.extend((("nactive", "active"), ("nactive", "inactive"))) - return {"positives": positives, "dangerous_negatives": negatives} - - -def _cosine(a: np.ndarray, b: np.ndarray) -> float: - return float(a @ b) - - -def resolver_accuracy(encoder: Any, pairs: list[tuple[str, str]], - vocabulary: list[str]) -> float: - """Share of corrupted values whose nearest vocabulary entry is the truth.""" - if not pairs: - return 1.0 - vocab_vectors = encoder.encode_texts(vocabulary) - correct = 0 - for truth, corrupted in pairs: - if truth not in vocabulary: - continue - query = encoder.encode_texts([corrupted])[0] - scores = vocab_vectors @ query - best = int(np.argmax(scores)) - correct += int(vocabulary[best] == truth) - return correct / len(pairs) - - -def false_merge_rate(encoder: Any, negatives: list[tuple[str, str]]) -> float: - """Share of dangerous pairs an auto-merging resolver would collapse. - - A merge happens only when the wrong candidate wins *and* the margin over - the runner-up exceeds :data:`MERGE_MARGIN` — mirroring the runtime rule - that ambiguous merges are never auto-applied. - """ - if not negatives: - return 0.0 - merges = 0 - for left, right in negatives: - vectors = encoder.encode_texts([left, right]) - similarity = _cosine(vectors[0], vectors[1]) - # Self-similarity is 1.0 by construction; a dangerous merge needs the - # *other* item to look closer than "almost identical". - if similarity > 1.0 - MERGE_MARGIN: - merges += 1 - return merges / len(negatives) - - -def torch_available() -> bool: - return importlib.util.find_spec("torch") is not None - - -def train(*, seed: int = 0, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - pairs = build_pairs(seed=seed) - baseline = HashedBigramEncoder() - vocabulary = sorted({t for t, _ in pairs["positives"]}) - baseline_accuracy = resolver_accuracy(baseline, pairs["positives"], vocabulary) - - result: dict[str, Any] = { - "n_positive_pairs": len(pairs["positives"]), - "n_dangerous_negatives": len(pairs["dangerous_negatives"]), - "baseline_resolver_accuracy": round(baseline_accuracy, 4), - } - - if not torch_available(): - # Explicitly supported: contrastive training is optional; the - # Phase-3 baseline stays in place and the safety gates still run. - encoder = baseline - result["status"] = "skipped" - result["reason"] = "torch not installed; Phase-3 baseline encoder retained" - result["resolver_accuracy"] = result["baseline_resolver_accuracy"] - else: # pragma: no cover - exercised only in envs with torch - encoder = _train_torch(pairs, seed=seed) - result["status"] = "trained" - result["resolver_accuracy"] = round( - resolver_accuracy(encoder, pairs["positives"], vocabulary), 4) - - result["dangerous_negative_false_merge_rate"] = round( - false_merge_rate(encoder, pairs["dangerous_negatives"]), 4) - result["gates"] = { - "resolver_accuracy_min": result["baseline_resolver_accuracy"], - "false_merge_rate_max": 0.0, - "failures": _check_gates(result), - } - write_json(out / "encoder_contrastive.metrics.json", result) - return result - - -def _check_gates(result: dict[str, Any]) -> list[str]: - failures = [] - if result["resolver_accuracy"] < result["baseline_resolver_accuracy"]: - failures.append( - f"resolver accuracy {result['resolver_accuracy']} regressed below " - f"baseline {result['baseline_resolver_accuracy']}") - if result["dangerous_negative_false_merge_rate"] != 0.0: - failures.append( - f"dangerous-negative false-merge rate " - f"{result['dangerous_negative_false_merge_rate']} != 0") - return failures - - -def _train_torch(pairs: dict[str, list[tuple[str, str]]], *, seed: int) -> Any: # pragma: no cover - """Triplet-loss fine-tune over the hashed embedding (torch path).""" - raise NotImplementedError( - "contrastive training requires the release environment; see " - "docs/developer-training-pipeline.md" - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - prog="python -m training.distill.train_encoder_contrastive") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - result = train(seed=args.seed, out_dir=args.out) - print( - f"encoder distillation: status={result['status']} " - f"resolver={result['resolver_accuracy']} " - f"false_merge={result['dangerous_negative_false_merge_rate']}" - ) - failures = result["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/train_intent_head.py b/training/distill/train_intent_head.py deleted file mode 100644 index 3cb5b6cb..00000000 --- a/training/distill/train_intent_head.py +++ /dev/null @@ -1,406 +0,0 @@ -"""Train the context-sentence intent head (``fd-intent-v1``). - -Inputs: the Phase-1 golden context corpus, deterministic paraphrase -templates (including the Indian-English / Hinglish set), corruptor-generated -context variants, optional cached teacher paraphrases, and an -author-disjoint human-verified eval split. - -The intent head is **optional evidence** for the deterministic parser — it -never replaces it, and in strict mode unresolved context stays surfaced. Its -training therefore favors *protection recall* over coverage: missing a -PROTECT sentence is the one unforgivable error, so PROTECT is oversampled -and gated at recall >= 0.99. - -Gates (``--check-gates``):: - - exact intent accuracy >= 0.92 - slot F1 >= 0.90 - UNKNOWN precision >= 0.95 - protected intent recall >= 0.99 - -CLI:: - - python -m training.distill.train_intent_head [--dev] [--check-gates] -""" - -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path -from typing import Any - -from ..common import BUILD_DIR, REPO_ROOT, read_json, write_json -from ..datasets.splits import author_disjoint_split -from ..seed.synthetic import make_context_sentences -from .linear import FeaturizerConfig, LinearHead, confusion, per_class_f1 - -OUT_DIR = BUILD_DIR / "intent_head" -GOLDEN_DIR = REPO_ROOT / "tests" / "context" / "golden" - -INTENTS = ( - "DOMAIN", "UNIQUE", "VALID_FORMAT", "LOCALE_FORMAT", "PROTECT", "IMPUTE_IF", - "ALLOWED_VALUES", "RANGE", "DEDUP_KEY", "DROP_IF", "RENAME", "MAP", "UNKNOWN", -) - -#: Phase-1 constraint rule -> intent label. -RULE_TO_INTENT = { - "unique": "UNIQUE", - "valid_format": "VALID_FORMAT", - "locale_format": "LOCALE_FORMAT", - "protected": "PROTECT", - "impute_missing": "IMPUTE_IF", - "allowed_values": "ALLOWED_VALUES", - "range": "RANGE", - "dedup_key": "DEDUP_KEY", -} - -#: Authors held out entirely for the paraphrase-generalization gate. -EVAL_AUTHORS = ("t2", "hinglish_b") - -GATE_EXACT_ACCURACY = 0.92 -GATE_SLOT_F1 = 0.90 -GATE_UNKNOWN_PRECISION = 0.95 -GATE_PROTECT_RECALL = 0.99 - -_DOMAIN_RE = re.compile(r"\bthis is (a|an)?\s*.*\bdata\s?set\b", re.I) - - -def load_golden_examples() -> list[dict[str, Any]]: - """Sentence-level intent labels derived from the Phase-1 golden corpus.""" - examples: list[dict[str, Any]] = [] - for text_path in sorted(GOLDEN_DIR.glob("*.txt")): - policy_path = text_path.with_name(text_path.name.replace(".txt", ".policy.json")) - if not policy_path.is_file(): - continue - policy = read_json(policy_path) - sentence_intent: dict[str, str] = {} - sentence_slots: dict[str, dict[str, Any]] = {} - for constraint in policy.get("constraints", []): - sentence = str(constraint.get("provenance", {}).get("sentence", "")).strip() - intent = RULE_TO_INTENT.get(str(constraint.get("rule", ""))) - if not sentence or intent is None: - continue - sentence_intent[sentence] = intent - slots = dict(constraint.get("params") or {}) - if constraint.get("resolved_from"): - slots["column"] = constraint["resolved_from"] - sentence_slots[sentence] = slots - for raw_line in text_path.read_text(encoding="utf-8").splitlines(): - sentence = raw_line.strip().rstrip(".") - if not sentence: - continue - matched = None - for known, intent in sentence_intent.items(): - if known.rstrip(".").strip() == sentence: - matched = (intent, sentence_slots.get(known, {})) - break - if matched is None: - if _DOMAIN_RE.search(raw_line) and policy.get("dataset_domain"): - matched = ("DOMAIN", {"domain": policy["dataset_domain"]}) - else: - matched = ("UNKNOWN", {}) - examples.append({ - "sentence": raw_line.strip(), - "intent": matched[0], - "slots": matched[1], - "author": "golden", - }) - return examples - - -def teacher_paraphrase_examples() -> list[dict[str, Any]]: - """Optional cached ``ContextParaphraseBatch`` payloads (training only).""" - from ..teacher.cache import TeacherCache # noqa: PLC0415 - - out: list[dict[str, Any]] = [] - for entry in TeacherCache().entries(): - for payload in entry.get("payloads", []): - intent = payload.get("intent") - if intent in INTENTS and isinstance(payload.get("paraphrases"), list): - for sentence in payload["paraphrases"]: - out.append({ - "sentence": str(sentence), "intent": str(intent), "slots": {}, - "author": f"teacher:{payload.get('author', 'unknown')}", - }) - return out - - -# --------------------------------------------------------------------------- # -# Deterministic slot extraction (evaluated as slot F1) -# --------------------------------------------------------------------------- # - -_THRESHOLD_RE = re.compile(r">?\s*(\d{2,3})\s*%") -_RANGE_RE = re.compile( - r"(?:between|within)\s+(\d+)\s+(?:and|to|aur)\s+(\d+)" - r"|(\d+)\s+aur\s+(\d+)\s+ke\s+beech", re.I) -_RENAME_RE = re.compile( - r"rename(?:\s+the)?(?:\s+column)?\s+\S+(?:\s+column)?\s+to\s+([A-Za-z_]\w*)" - r"|ko\s+rename\s+karke\s+([A-Za-z_]\w*)" - r"|ko\s+([A-Za-z_]\w*)\s+rename\s+kar", re.I) -_MAP_RE = re.compile(r"\b(\w+)\s+(?:to|ko)\s+(\w+)\b") -_DOMAIN_RE_SLOT = re.compile(r"this (?:is a[n]? |dataset is a[n]? )(.+?)\s+data\s?set", re.I) -_VALUES_RE = re.compile( - r"(?:values are|only be|can only be|values:|ho sakta hai|sirf)\s*:?\s*([^.]+)", - re.I, -) -_VALUE_STOPWORDS = frozenset({"allowed", "hai", "ho", "sakta"}) - - -def extract_slots(sentence: str, intent: str, columns: tuple[str, ...] = ()) -> dict[str, Any]: - """Regex/lexicon slot extraction — deterministic, no model involved.""" - slots: dict[str, Any] = {} - lowered = sentence.lower() - for column in columns: - if column.lower() in lowered: - slots["column"] = column - break - if intent == "DOMAIN": - match = _DOMAIN_RE_SLOT.search(sentence) - if match: - slots["domain"] = re.sub(r"\W+", "_", match.group(1).strip().lower()) - if intent == "VALID_FORMAT": - if "email" in lowered: - slots["format"] = "email" - elif "phone" in lowered: - slots["format"] = "phone" - elif "url" in lowered or "website" in lowered: - slots["format"] = "url" - if intent == "LOCALE_FORMAT": - if "indian" in lowered or "india" in lowered: - slots["locale"] = "IN" - if "phone" in lowered: - slots["format"] = "phone" - if intent == "IMPUTE_IF": - match = _THRESHOLD_RE.search(sentence) - if match: - slots["threshold"] = int(match.group(1)) / 100.0 - if intent == "RANGE": - match = _RANGE_RE.search(sentence) - if match: - low, high = (match.group(1), match.group(2)) if match.group(1) else ( - match.group(3), match.group(4)) - slots["min"], slots["max"] = int(low), int(high) - if intent == "ALLOWED_VALUES": - match = _VALUES_RE.search(sentence) - if match: - raw = match.group(1) - values = [ - v.strip() for v in re.split(r",|\bor\b|\band\b|\bya\b|\baur\b", raw) if v.strip() - ] - cleaned = [] - for value in values: - words = [w for w in value.split() if w.lower() not in _VALUE_STOPWORDS] - if words and len(" ".join(words)) < 24: - cleaned.append(" ".join(words)) - if cleaned: - slots["values"] = cleaned - if intent == "RENAME": - match = _RENAME_RE.search(sentence) - if match: - slots["to"] = next(g for g in match.groups() if g) - if intent == "MAP": - skip = {"rows", "karo", "map", "aur"} - pairs = {a: b for a, b in _MAP_RE.findall(sentence) - if a.lower() not in skip and b.lower() not in skip} - if pairs: - slots["mapping"] = pairs - if intent == "DROP_IF" and ("empty" in lowered or "missing" in lowered): - slots["condition"] = "empty" - return slots - - -def _slot_pairs(slots: dict[str, Any]) -> set[tuple[str, str]]: - pairs: set[tuple[str, str]] = set() - for key, value in slots.items(): - if isinstance(value, list): - for item in value: - pairs.add((key, str(item).strip().lower())) - elif isinstance(value, dict): - for k, v in value.items(): - pairs.add((f"{key}.{k}".lower(), str(v).strip().lower())) - else: - pairs.add((key, str(value).strip().lower())) - return pairs - - -def slot_f1(rows: list[dict[str, Any]], predictions: list[str]) -> float: - tp = fp = fn = 0 - for row, predicted_intent in zip(rows, predictions): - gold = _slot_pairs(row.get("slots", {})) - columns = tuple(str(c) for c in row.get("columns", ())) or ( - (str(row["slots"]["column"]),) if row.get("slots", {}).get("column") else ()) - predicted = _slot_pairs(extract_slots(row["sentence"], predicted_intent, columns)) - tp += len(gold & predicted) - fp += len(predicted - gold) - fn += len(gold - predicted) - denominator = 2 * tp + fp + fn - return (2 * tp / denominator) if denominator else 1.0 - - -# --------------------------------------------------------------------------- # -# Conflict detection: PROTECT vs mutating intents on the same column -# --------------------------------------------------------------------------- # - -_MUTATING_INTENTS = frozenset({"VALID_FORMAT", "LOCALE_FORMAT", "IMPUTE_IF", "MAP", "RENAME"}) - - -def conflict_cases() -> list[dict[str, Any]]: - """Sentence pairs on one column; ``conflict`` iff protection meets mutation.""" - return [ - {"sentences": ("Never modify revenue values.", "Revenue must be valid numbers."), - "intents": ("PROTECT", "VALID_FORMAT"), "conflict": True}, - {"sentences": ("status is protected.", "Map A1 to active in status."), - "intents": ("PROTECT", "MAP"), "conflict": True}, - {"sentences": ("cust_id is unique.", "Deduplicate rows by cust_id."), - "intents": ("UNIQUE", "DEDUP_KEY"), "conflict": False}, - {"sentences": ("Emails must be valid.", "Phone numbers are Indian."), - "intents": ("VALID_FORMAT", "LOCALE_FORMAT"), "conflict": False}, - {"sentences": ("age kabhi mat badalna.", - "Missing age should be estimated only if confidence >95%."), - "intents": ("PROTECT", "IMPUTE_IF"), "conflict": True}, - {"sentences": ("Never modify monthly_revenue values.", "monthly_revenue is protected."), - "intents": ("PROTECT", "PROTECT"), "conflict": False}, - ] - - -def conflict_detection_accuracy(head: LinearHead) -> float: - correct = 0 - cases = conflict_cases() - for case in cases: - predicted = [label for label, _ in head.predict(list(case["sentences"]))] - has_protect = "PROTECT" in predicted - has_mutation = any(p in _MUTATING_INTENTS for p in predicted) - detected = has_protect and has_mutation - correct += int(detected == case["conflict"]) - return correct / len(cases) - - -# --------------------------------------------------------------------------- # -# Training / evaluation -# --------------------------------------------------------------------------- # - -def build_dataset(*, seed: int = 0) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """(train, human-verified author-disjoint eval).""" - rows = load_golden_examples() - rows.extend(make_context_sentences(seed=seed)) - rows.extend(teacher_paraphrase_examples()) - train_rows, eval_rows = author_disjoint_split(rows, eval_authors=EVAL_AUTHORS) - # The synthetic corpus is authored and verified in-repo; golden rows are - # maintainer-verified by construction. Eval rows carry that provenance. - for row in eval_rows: - row.setdefault("human_verified", True) - row.setdefault("reviewer", "JWD") - return train_rows, eval_rows - - -def evaluate(head: LinearHead, eval_rows: list[dict[str, Any]]) -> dict[str, Any]: - from ..eval.human_verified import check_all_verified # noqa: PLC0415 - - check_all_verified(eval_rows) - texts = [row["sentence"] for row in eval_rows] - y_true = [row["intent"] for row in eval_rows] - predictions = head.predict(texts) - y_pred = [label for label, _ in predictions] - - exact = sum(1 for t, p in zip(y_true, y_pred) if t == p) / len(y_true) - unknown_predicted = [i for i, p in enumerate(y_pred) if p == "UNKNOWN"] - unknown_precision = ( - sum(1 for i in unknown_predicted if y_true[i] == "UNKNOWN") / len(unknown_predicted) - if unknown_predicted else 1.0 - ) - protect_rows = [i for i, t in enumerate(y_true) if t == "PROTECT"] - protect_recall = ( - sum(1 for i in protect_rows if y_pred[i] == "PROTECT") / len(protect_rows) - if protect_rows else 1.0 - ) - paraphrase_rows = [i for i, row in enumerate(eval_rows) - if str(row.get("author", "")) in EVAL_AUTHORS] - paraphrase_accuracy = ( - sum(1 for i in paraphrase_rows if y_pred[i] == y_true[i]) / len(paraphrase_rows) - if paraphrase_rows else 1.0 - ) - return { - "n_eval": len(eval_rows), - "exact_intent_accuracy": round(exact, 4), - "slot_f1": round(slot_f1(eval_rows, y_pred), 4), - "unknown_precision": round(unknown_precision, 4), - "protected_intent_recall": round(protect_recall, 4), - "author_disjoint_paraphrase_accuracy": round(paraphrase_accuracy, 4), - "conflict_detection_accuracy": round(conflict_detection_accuracy(head), 4), - "per_class_f1": {k: round(v, 4) for k, v in per_class_f1(y_true, y_pred, INTENTS).items()}, - "confusion": confusion(y_true, y_pred, INTENTS), - } - - -def check_gates(metrics: dict[str, Any]) -> list[str]: - failures = [] - if metrics["exact_intent_accuracy"] < GATE_EXACT_ACCURACY: - failures.append( - f"exact accuracy {metrics['exact_intent_accuracy']} < {GATE_EXACT_ACCURACY}") - if metrics["slot_f1"] < GATE_SLOT_F1: - failures.append(f"slot F1 {metrics['slot_f1']} < {GATE_SLOT_F1}") - if metrics["unknown_precision"] < GATE_UNKNOWN_PRECISION: - failures.append( - f"UNKNOWN precision {metrics['unknown_precision']} < {GATE_UNKNOWN_PRECISION}") - if metrics["protected_intent_recall"] < GATE_PROTECT_RECALL: - failures.append( - f"protected recall {metrics['protected_intent_recall']} < {GATE_PROTECT_RECALL}") - return failures - - -def train(*, seed: int = 0, dev: bool = False, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - train_rows, eval_rows = build_dataset(seed=seed) - head = LinearHead.train( - [row["sentence"] for row in train_rows], - [row["intent"] for row in train_rows], - classes=INTENTS, - featurizer=FeaturizerConfig(dim=2048 if dev else 4096), - epochs=150 if dev else 400, - # Protection favors recall over coverage: PROTECT errors are the - # expensive ones, so its loss weight dominates. - class_weights={"PROTECT": 4.0}, - abstain_class="UNKNOWN", - abstain_threshold=0.2, - ) - metrics = evaluate(head, eval_rows) - metrics["n_train"] = len(train_rows) - metrics["gates"] = { - "exact_accuracy_min": GATE_EXACT_ACCURACY, - "slot_f1_min": GATE_SLOT_F1, - "unknown_precision_min": GATE_UNKNOWN_PRECISION, - "protect_recall_min": GATE_PROTECT_RECALL, - "failures": check_gates(metrics), - } - head.save(out / "intent_head.weights.json") - write_json(out / "intent_head.metrics.json", metrics) - write_json(out / "intent_head.confusion.json", metrics["confusion"]) - return metrics - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.train_intent_head") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--dev", action="store_true") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - metrics = train(seed=args.seed, dev=args.dev, out_dir=args.out) - print( - f"intent head: exact={metrics['exact_intent_accuracy']} " - f"slotF1={metrics['slot_f1']} unknownP={metrics['unknown_precision']} " - f"protectR={metrics['protected_intent_recall']} " - f"paraphrase={metrics['author_disjoint_paraphrase_accuracy']}" - ) - failures = metrics["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/distill/train_role_head.py b/training/distill/train_role_head.py deleted file mode 100644 index 37538d02..00000000 --- a/training/distill/train_role_head.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Train the semantic-type role head (``fd-role-head-v1``). - -Inputs: synthetic seed columns, corruptor-derived value dirt, alias-mangled -column-name variants, optional cached teacher ``ColumnRoleLabel`` payloads, -and the human-verified eval split. Output: weights (fp32 + int8), metrics -JSON, a confusion matrix, and — when the ``onnx`` package is installed — an -ONNX export. - -Release gates (checked with ``--check-gates``):: - - macro-F1 >= 0.90 on the human-verified eval - content-detector contradiction rate <= 1% - adversarial alias accuracy >= 0.85 - -CLI:: - - python -m training.distill.train_role_head [--dev] [--check-gates] -""" - -from __future__ import annotations - -import argparse -import random -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -from ..common import BUILD_DIR, TRAINING_ROOT, write_json -from ..corruptors.context import ECOMMERCE_ALIASES -from ..corruptors.registry import get_corruptor -from ..eval.human_verified import load_verified -from ..seed.synthetic import seed_tables -from .linear import FeaturizerConfig, LinearHead, confusion, macro_f1, per_class_f1 - -OUT_DIR = BUILD_DIR / "role_head" -HUMAN_EVAL_PATH = TRAINING_ROOT / "eval" / "data" / "role_eval_human.jsonl" - -SEMANTIC_TYPES = ( - "email", "phone", "url", "country", "currency_amount", "quantity_with_unit", - "person_name", "address", "city", "postal_code", "national_id", - "category_code", "free_text", "boolean_like", "date_like", "identifier", - "unknown", -) - -#: Ground-truth semantic type for every synthetic seed column. -COLUMN_TYPES: dict[str, str] = { - "cust_id": "identifier", "txn_id": "identifier", - "full_name": "person_name", "email": "email", "phone": "phone", - "address": "address", "city": "city", "state": "city", "country": "country", - "postal_code": "postal_code", "national_id_like": "national_id", - "status": "category_code", "order_status": "category_code", - "category_code": "category_code", - "signup_date": "date_like", "order_date": "date_like", - "website": "url", "newsletter_opt_in": "boolean_like", - "quantity": "quantity_with_unit", "unit_price_inr": "currency_amount", - "monthly_revenue": "currency_amount", "notes": "free_text", -} - -GATE_MACRO_F1 = 0.90 -GATE_CONTRADICTION_RATE = 0.01 -GATE_ALIAS_ACCURACY = 0.85 - - -def example_text(column_name: str, values: list[Any], *, k: int = 8) -> str: - """The exact text form the head scores: name + sample values.""" - samples = " ; ".join(str(v) for v in values[:k]) - return f"col:{column_name} | {samples}" - - -def _alias_variants(column: str, rng: random.Random) -> list[str]: - variants = list(ECOMMERCE_ALIASES.get(column, ())) - parts = column.split("_") - variants.extend(( - "".join(p.title() for p in parts), - " ".join(parts).title(), - "-".join(parts), - column.upper(), - )) - rng.shuffle(variants) - return [v for v in variants if v != column] - - -def build_training_set(*, seed: int = 0, dev: bool = False) -> list[dict[str, Any]]: - """Labeled role examples from seed tables + dirt + alias mangling.""" - rng = random.Random(f"role-head:{seed}") - tables = seed_tables(seed=seed) - dirty_corruptors = [get_corruptor(n) for n in ( - "whitespace_insertion", "casing_change", "email_double_at", - "phone_in_zero_prefix", "date_format_shuffle", "currency_formatting", - )] - examples: list[dict[str, Any]] = [] - windows = 4 if dev else 12 - for _, frame in tables.items(): - for column in frame.columns: - semantic_type = COLUMN_TYPES.get(str(column)) - if semantic_type is None: - continue - values = [v for v in frame[column].tolist() if v is not None] - for w in range(windows): - start = (w * 13) % max(1, len(values) - 8) - window = [str(v) for v in values[start:start + 8]] - # Random dirt on a share of windows: the head must be robust - # to the same corruption the cleaner sees. - if w % 3 == 1 and window: - corruptor = rng.choice(dirty_corruptors) - assert corruptor.fn is not None - window = [ - str(corruptor.fn(v, rng, corruptor.params) or v) for v in window - ] - name = str(column) - if w % 4 == 2: - aliases = _alias_variants(str(column), rng) - if aliases: - name = aliases[0] - examples.append({ - "text": example_text(name, window), - "label": semantic_type, - "column": str(column), - "aliased": name != str(column), - }) - # Explicit unknown class: junk columns with no coherent type. - for i in range(8 if dev else 30): - junk = [str(rng.random())[:6], rng.choice(("x", "yy", "?")), str(rng.randrange(999))] - rng.shuffle(junk) - examples.append({ - "text": example_text(f"misc_{i}", junk * 2), - "label": "unknown", "column": f"misc_{i}", "aliased": False, - }) - return examples - - -def teacher_examples(cache_dir: Path | None = None) -> list[dict[str, Any]]: - """Optional cached ``ColumnRoleLabel`` payloads (training data only).""" - from ..teacher.cache import TeacherCache # noqa: PLC0415 - - out: list[dict[str, Any]] = [] - for entry in TeacherCache(cache_dir).entries(): - for payload in entry.get("payloads", []): - if payload.get("semantic_type") in SEMANTIC_TYPES and "column_name" in payload: - out.append({ - "text": example_text( - str(payload["column_name"]), list(payload.get("masked_samples", [])) - ), - "label": str(payload["semantic_type"]), - "column": str(payload["column_name"]), - "aliased": False, - }) - return out - - -def load_human_eval(path: Path | str = HUMAN_EVAL_PATH) -> list[dict[str, Any]]: - labels = load_verified(path) - for row in labels: - row.setdefault("text", example_text(str(row["column"]), list(row["values"]))) - return labels - - -def contradiction_rate(head: LinearHead, eval_rows: list[dict[str, Any]]) -> float: - """Share of confident head calls that contradict FreshData's detectors.""" - from freshdata.semantic.semantic_types import infer_semantic_type # noqa: PLC0415 - - contradictions = 0 - confident = 0 - for row in eval_rows: - values = row.get("values") - if not values: - continue - prediction, head_confidence = head.predict([row["text"]])[0] - if prediction == "unknown" or head_confidence < 0.7: - continue - detected = infer_semantic_type(str(row["column"]), pd.Series(list(values))) - detected_type = getattr(detected, "semantic_type", "unknown") - detected_confidence = float(getattr(detected, "confidence", 0.0)) - if detected_type in ("unknown", None) or detected_confidence < 0.9: - continue - confident += 1 - if prediction != detected_type: - contradictions += 1 - return contradictions / confident if confident else 0.0 - - -def evaluate(head: LinearHead, eval_rows: list[dict[str, Any]]) -> dict[str, Any]: - texts = [row["text"] for row in eval_rows] - y_true = [row["label"] for row in eval_rows] - predictions = head.predict(texts) - y_pred = [label for label, _ in predictions] - alias_rows = [i for i, row in enumerate(eval_rows) if row.get("aliased")] - alias_accuracy = ( - sum(1 for i in alias_rows if y_pred[i] == y_true[i]) / len(alias_rows) - if alias_rows else 1.0 - ) - return { - "n_eval": len(eval_rows), - "macro_f1": round(macro_f1(y_true, y_pred, head.classes), 4), - "per_class_f1": { - k: round(v, 4) for k, v in per_class_f1(y_true, y_pred, head.classes).items()}, - "abstention_rate": round( - sum(1 for p, _ in predictions if p == "unknown") / len(predictions), 4), - "adversarial_alias_accuracy": round(alias_accuracy, 4), - "contradiction_rate": round(contradiction_rate(head, eval_rows), 4), - "confusion": confusion(y_true, y_pred, head.classes), - } - - -def check_gates(metrics: dict[str, Any]) -> list[str]: - failures = [] - if metrics["macro_f1"] < GATE_MACRO_F1: - failures.append(f"macro-F1 {metrics['macro_f1']} < {GATE_MACRO_F1}") - if metrics["contradiction_rate"] > GATE_CONTRADICTION_RATE: - failures.append( - f"contradiction rate {metrics['contradiction_rate']} > {GATE_CONTRADICTION_RATE}") - if metrics["adversarial_alias_accuracy"] < GATE_ALIAS_ACCURACY: - failures.append( - f"alias accuracy {metrics['adversarial_alias_accuracy']} < {GATE_ALIAS_ACCURACY}") - return failures - - -def train(*, seed: int = 0, dev: bool = False, out_dir: Path | str = OUT_DIR) -> dict[str, Any]: - out = Path(out_dir) - examples = build_training_set(seed=seed, dev=dev) - examples.extend(teacher_examples()) - eval_rows = load_human_eval() - head = LinearHead.train( - [e["text"] for e in examples], - [e["label"] for e in examples], - classes=SEMANTIC_TYPES, - featurizer=FeaturizerConfig(dim=1024 if dev else 2048), - epochs=150 if dev else 400, - abstain_class="unknown", - abstain_threshold=0.35, - ) - metrics = evaluate(head, eval_rows) - metrics["n_train"] = len(examples) - metrics["gates"] = { - "macro_f1_min": GATE_MACRO_F1, - "contradiction_rate_max": GATE_CONTRADICTION_RATE, - "alias_accuracy_min": GATE_ALIAS_ACCURACY, - "failures": check_gates(metrics), - } - head.save(out / "role_head.weights.json") - write_json(out / "role_head.metrics.json", metrics) - write_json(out / "role_head.confusion.json", metrics["confusion"]) - return metrics - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.distill.train_role_head") - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--dev", action="store_true") - parser.add_argument("--out", default=str(OUT_DIR)) - parser.add_argument("--check-gates", action="store_true") - args = parser.parse_args(argv) - metrics = train(seed=args.seed, dev=args.dev, out_dir=args.out) - print( - f"role head: macro-F1={metrics['macro_f1']} " - f"alias={metrics['adversarial_alias_accuracy']} " - f"contradiction={metrics['contradiction_rate']} " - f"abstention={metrics['abstention_rate']}" - ) - failures = metrics["gates"]["failures"] - if failures and args.check_gates: - for failure in failures: - print(f"GATE FAIL: {failure}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/eval/__init__.py b/training/eval/__init__.py deleted file mode 100644 index 2c37a078..00000000 --- a/training/eval/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Release-gating evaluation: human-verified labels and CleanBench runs.""" diff --git a/training/eval/data/role_eval_human.jsonl b/training/eval/data/role_eval_human.jsonl deleted file mode 100644 index 0ef38aad..00000000 --- a/training/eval/data/role_eval_human.jsonl +++ /dev/null @@ -1,75 +0,0 @@ -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100089 ; C100090 ; C100091 ; C100092 ; C100093 ; C100094 ; C100095 ; C100096","values":["C100089","C100090","C100091","C100092","C100093","C100094","C100095","C100096"]} -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100090 ; C100091 ; C100092 ; C100093 ; C100094 ; C100095 ; C100096 ; C100097","values":["C100090","C100091","C100092","C100093","C100094","C100095","C100096","C100097"]} -{"aliased":true,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"CustId","text":"col:CustId | C100068 ; C100069 ; C100070 ; C100071 ; C100072 ; C100073 ; C100074 ; C100075","values":["C100068","C100069","C100070","C100071","C100072","C100073","C100074","C100075"]} -{"aliased":false,"column":"full_name","human_verified":true,"label":"person_name","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"full_name","text":"col:full_name | Divya Okelmere ; Meera Quenderi ; Dev Purvatti ; Sneha Quenderi ; Rohan Quenderi ; Amit Sandrelo ; Sneha Purvatti ; Tara Quenderi","values":["Divya Okelmere","Meera Quenderi","Dev Purvatti","Sneha Quenderi","Rohan Quenderi","Amit Sandrelo","Sneha Purvatti","Tara Quenderi"]} -{"aliased":false,"column":"full_name","human_verified":true,"label":"person_name","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"full_name","text":"col:full_name | Isha Voskette ; Divya Purvatti ; Meera Sandrelo ; Asha Okelmere ; Arjun Sandrelo ; Dev Sandrelo ; Meera Mirostra ; Divya Quenderi","values":["Isha Voskette","Divya Purvatti","Meera Sandrelo","Asha Okelmere","Arjun Sandrelo","Dev Sandrelo","Meera Mirostra","Divya Quenderi"]} -{"aliased":true,"column":"full_name","human_verified":true,"label":"person_name","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"FullName","text":"col:FullName | Sneha Voskette ; Sneha Kelvani ; Neha Mirostra ; Meera Voskette ; Meera Vintara ; Meera Quenderi ; Rohan Quenderi ; Neha Talvane","values":["Sneha Voskette","Sneha Kelvani","Neha Mirostra","Meera Voskette","Meera Vintara","Meera Quenderi","Rohan Quenderi","Neha Talvane"]} -{"aliased":false,"column":"email","human_verified":true,"label":"email","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"email","text":"col:email | asha.okelmere111@example.net ; amit.talvane112@mail.example.in ; divya.okelmere113@example.net ; meera.quenderi114@example.net ; dev.purvatti115@mail.example.in ; sneha.quenderi116@example.org ; rohan.quenderi117@mail.example.in ; amit.sandrelo118@example.org","values":["asha.okelmere111@example.net","amit.talvane112@mail.example.in","divya.okelmere113@example.net","meera.quenderi114@example.net","dev.purvatti115@mail.example.in","sneha.quenderi116@example.org","rohan.quenderi117@mail.example.in","amit.sandrelo118@example.org"]} -{"aliased":false,"column":"email","human_verified":true,"label":"email","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"email","text":"col:email | amit.quenderi51@example.com ; kabir.kelvani52@mail.example.in ; meera.vintara53@example.org ; vikram.voskette54@example.org ; neha.vintara55@example.com ; asha.talvane56@mail.example.in ; vikram.okelmere57@example.net ; ravi.sandrelo58@example.com","values":["amit.quenderi51@example.com","kabir.kelvani52@mail.example.in","meera.vintara53@example.org","vikram.voskette54@example.org","neha.vintara55@example.com","asha.talvane56@mail.example.in","vikram.okelmere57@example.net","ravi.sandrelo58@example.com"]} -{"aliased":true,"column":"email","human_verified":true,"label":"email","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"mail","text":"col:mail | vikram.voskette2@example.org ; rohan.voskette3@mail.example.in ; isha.vintara4@example.net ; neha.kelvani5@example.com ; arjun.vintara6@example.org ; isha.purvatti7@mail.example.in ; divya.sandrelo8@mail.example.in ; arjun.vintara9@example.org","values":["vikram.voskette2@example.org","rohan.voskette3@mail.example.in","isha.vintara4@example.net","neha.kelvani5@example.com","arjun.vintara6@example.org","isha.purvatti7@mail.example.in","divya.sandrelo8@mail.example.in","arjun.vintara9@example.org"]} -{"aliased":false,"column":"phone","human_verified":true,"label":"phone","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"phone","text":"col:phone | +916488408307 ; +918786842262 ; +916876254223 ; +916988747208 ; +918405116930 ; +919388522216 ; +917842681057 ; +919690719515","values":["+916488408307","+918786842262","+916876254223","+916988747208","+918405116930","+919388522216","+917842681057","+919690719515"]} -{"aliased":false,"column":"phone","human_verified":true,"label":"phone","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"phone","text":"col:phone | +917841602874 ; +916851673246 ; +917442958335 ; +916466654573 ; +918299889325 ; +917908418834 ; +919547064015 ; +918803250788","values":["+917841602874","+916851673246","+917442958335","+916466654573","+918299889325","+917908418834","+919547064015","+918803250788"]} -{"aliased":true,"column":"phone","human_verified":true,"label":"phone","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Phone","text":"col:Phone | +918718315157 ; +918324435655 ; +916679495576 ; +918873728283 ; +919965647835 ; +919324364004 ; +916362683821 ; +917569920066","values":["+918718315157","+918324435655","+916679495576","+918873728283","+919965647835","+919324364004","+916362683821","+917569920066"]} -{"aliased":false,"column":"address","human_verified":true,"label":"address","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"address","text":"col:address | 110 Palmshade Street ; 124 Riverbend Road ; 208 Palmshade Street ; 165 Harborview Block ; 142 Quarry Lane ; 196 Harborview Block ; 52 Jacaranda Nagar ; 48 Copperleaf Marg","values":["110 Palmshade Street","124 Riverbend Road","208 Palmshade Street","165 Harborview Block","142 Quarry Lane","196 Harborview Block","52 Jacaranda Nagar","48 Copperleaf Marg"]} -{"aliased":false,"column":"address","human_verified":true,"label":"address","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"address","text":"col:address | 211 Copperleaf Marg ; 53 Quarry Lane ; 182 Harborview Block ; 122 Lantern Row ; 23 Quarry Lane ; 3 Riverbend Road ; 53 Jacaranda Nagar ; 140 Copperleaf Marg","values":["211 Copperleaf Marg","53 Quarry Lane","182 Harborview Block","122 Lantern Row","23 Quarry Lane","3 Riverbend Road","53 Jacaranda Nagar","140 Copperleaf Marg"]} -{"aliased":true,"column":"address","human_verified":true,"label":"address","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Address","text":"col:Address | 98 Old Mill Avenue ; 137 Jacaranda Nagar ; 140 Old Mill Avenue ; 6 Lantern Row ; 199 Harborview Block ; 45 Palmshade Street ; 104 Copperleaf Marg ; 224 Riverbend Road","values":["98 Old Mill Avenue","137 Jacaranda Nagar","140 Old Mill Avenue","6 Lantern Row","199 Harborview Block","45 Palmshade Street","104 Copperleaf Marg","224 Riverbend Road"]} -{"aliased":false,"column":"city","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"city","text":"col:city | Hyderabad ; Kolkata ; Pune ; Pune ; Pune ; Kolkata ; Pune ; Delhi","values":["Hyderabad","Kolkata","Pune","Pune","Pune","Kolkata","Pune","Delhi"]} -{"aliased":false,"column":"city","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"city","text":"col:city | Hyderabad ; Kolkata ; Delhi ; Pune ; Hyderabad ; Hyderabad ; Mumbai ; Kolkata","values":["Hyderabad","Kolkata","Delhi","Pune","Hyderabad","Hyderabad","Mumbai","Kolkata"]} -{"aliased":true,"column":"city","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"City","text":"col:City | Mumbai ; Chennai ; Pune ; Chennai ; Delhi ; Pune ; Jaipur ; Kolkata","values":["Mumbai","Chennai","Pune","Chennai","Delhi","Pune","Jaipur","Kolkata"]} -{"aliased":false,"column":"state","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"state","text":"col:state | Maharashtra ; Maharashtra ; Karnataka ; West Bengal ; Karnataka ; West Bengal ; Delhi ; Rajasthan","values":["Maharashtra","Maharashtra","Karnataka","West Bengal","Karnataka","West Bengal","Delhi","Rajasthan"]} -{"aliased":false,"column":"state","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"state","text":"col:state | Telangana ; Tamil Nadu ; Tamil Nadu ; Karnataka ; Tamil Nadu ; Delhi ; Karnataka ; Karnataka","values":["Telangana","Tamil Nadu","Tamil Nadu","Karnataka","Tamil Nadu","Delhi","Karnataka","Karnataka"]} -{"aliased":true,"column":"state","human_verified":true,"label":"city","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"State","text":"col:State | Maharashtra ; West Bengal ; Telangana ; Maharashtra ; Tamil Nadu ; Maharashtra ; Maharashtra ; West Bengal","values":["Maharashtra","West Bengal","Telangana","Maharashtra","Tamil Nadu","Maharashtra","Maharashtra","West Bengal"]} -{"aliased":false,"column":"country","human_verified":true,"label":"country","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"country","text":"col:country | Canada ; India ; India ; Canada ; Canada ; India ; India ; India","values":["Canada","India","India","Canada","Canada","India","India","India"]} -{"aliased":false,"column":"country","human_verified":true,"label":"country","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"country","text":"col:country | Austria ; Canada ; Singapore ; India ; Austria ; India ; Singapore ; Canada","values":["Austria","Canada","Singapore","India","Austria","India","Singapore","Canada"]} -{"aliased":true,"column":"country","human_verified":true,"label":"country","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Country","text":"col:Country | Austria ; Austria ; India ; Austria ; India ; India ; Austria ; Australia","values":["Austria","Austria","India","Austria","India","India","Austria","Australia"]} -{"aliased":false,"column":"postal_code","human_verified":true,"label":"postal_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"postal_code","text":"col:postal_code | 444250 ; 374527 ; 579495 ; 802875 ; 438531 ; 171038 ; 521425 ; 264257","values":["444250","374527","579495","802875","438531","171038","521425","264257"]} -{"aliased":false,"column":"postal_code","human_verified":true,"label":"postal_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"postal_code","text":"col:postal_code | 861222 ; 498997 ; 393951 ; 641671 ; 664660 ; 164759 ; 602882 ; 264106","values":["861222","498997","393951","641671","664660","164759","602882","264106"]} -{"aliased":true,"column":"postal_code","human_verified":true,"label":"postal_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"PostalCode","text":"col:PostalCode | 781804 ; 636704 ; 227998 ; 279245 ; 148732 ; 137018 ; 462519 ; 744006","values":["781804","636704","227998","279245","148732","137018","462519","744006"]} -{"aliased":false,"column":"national_id_like","human_verified":true,"label":"national_id","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"national_id_like","text":"col:national_id_like | ZZ57062580X ; ZZ61163162X ; ZZ35065162X ; ZZ41268135X ; ZZ12583457X ; ZZ59822517X ; ZZ19859926X ; ZZ52169088X","values":["ZZ57062580X","ZZ61163162X","ZZ35065162X","ZZ41268135X","ZZ12583457X","ZZ59822517X","ZZ19859926X","ZZ52169088X"]} -{"aliased":false,"column":"national_id_like","human_verified":true,"label":"national_id","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"national_id_like","text":"col:national_id_like | ZZ59306892X ; ZZ51283425X ; ZZ96647252X ; ZZ21251311X ; ZZ61262966X ; ZZ63120743X ; ZZ92832902X ; ZZ53154617X","values":["ZZ59306892X","ZZ51283425X","ZZ96647252X","ZZ21251311X","ZZ61262966X","ZZ63120743X","ZZ92832902X","ZZ53154617X"]} -{"aliased":true,"column":"national_id_like","human_verified":true,"label":"national_id","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"national-id-like","text":"col:national-id-like | ZZ55905267X ; ZZ99286096X ; ZZ26680050X ; ZZ59306892X ; ZZ51283425X ; ZZ96647252X ; ZZ21251311X ; ZZ61262966X","values":["ZZ55905267X","ZZ99286096X","ZZ26680050X","ZZ59306892X","ZZ51283425X","ZZ96647252X","ZZ21251311X","ZZ61262966X"]} -{"aliased":false,"column":"status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"status","text":"col:status | active ; inactive ; active ; pending ; active ; churned ; pending ; active","values":["active","inactive","active","pending","active","churned","pending","active"]} -{"aliased":false,"column":"status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"status","text":"col:status | inactive ; inactive ; active ; inactive ; active ; active ; active ; pending","values":["inactive","inactive","active","inactive","active","active","active","pending"]} -{"aliased":true,"column":"status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Account Status","text":"col:Account Status | churned ; pending ; active ; active ; active ; churned ; active ; inactive","values":["churned","pending","active","active","active","churned","active","inactive"]} -{"aliased":false,"column":"signup_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"signup_date","text":"col:signup_date | 2019-09-28 ; 2022-05-08 ; 2020-04-09 ; 2023-04-01 ; 2022-08-12 ; 2023-03-01 ; 2025-12-08 ; 2023-04-09","values":["2019-09-28","2022-05-08","2020-04-09","2023-04-01","2022-08-12","2023-03-01","2025-12-08","2023-04-09"]} -{"aliased":false,"column":"signup_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"signup_date","text":"col:signup_date | 2021-09-04 ; 2025-01-10 ; 2023-04-22 ; 2022-01-01 ; 2021-09-17 ; 2020-07-17 ; 2022-12-13 ; 2024-12-26","values":["2021-09-04","2025-01-10","2023-04-22","2022-01-01","2021-09-17","2020-07-17","2022-12-13","2024-12-26"]} -{"aliased":true,"column":"signup_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"SignupDate","text":"col:SignupDate | 2023-03-04 ; 2023-09-06 ; 2024-10-26 ; 2019-11-08 ; 2020-10-14 ; 2025-05-28 ; 2023-12-12 ; 2022-07-26","values":["2023-03-04","2023-09-06","2024-10-26","2019-11-08","2020-10-14","2025-05-28","2023-12-12","2022-07-26"]} -{"aliased":false,"column":"website","human_verified":true,"label":"url","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"website","text":"col:website | https://www.voskette63.example.com ; https://www.okelmere64.example.com ; https://www.kelvani65.example.com ; https://www.voskette66.example.com ; https://www.kelvani67.example.com ; https://www.mirostra68.example.com ; https://www.voskette69.example.com ; https://www.vintara70.example.com","values":["https://www.voskette63.example.com","https://www.okelmere64.example.com","https://www.kelvani65.example.com","https://www.voskette66.example.com","https://www.kelvani67.example.com","https://www.mirostra68.example.com","https://www.voskette69.example.com","https://www.vintara70.example.com"]} -{"aliased":false,"column":"website","human_verified":true,"label":"url","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"website","text":"col:website | https://www.sandrelo49.example.com ; https://www.vintara50.example.com ; https://www.quenderi51.example.com ; https://www.kelvani52.example.com ; https://www.vintara53.example.com ; https://www.voskette54.example.com ; https://www.vintara55.example.com ; https://www.talvane56.example.com","values":["https://www.sandrelo49.example.com","https://www.vintara50.example.com","https://www.quenderi51.example.com","https://www.kelvani52.example.com","https://www.vintara53.example.com","https://www.voskette54.example.com","https://www.vintara55.example.com","https://www.talvane56.example.com"]} -{"aliased":true,"column":"website","human_verified":true,"label":"url","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Website","text":"col:Website | https://www.quenderi72.example.com ; https://www.talvane73.example.com ; https://www.vintara74.example.com ; https://www.purvatti75.example.com ; https://www.purvatti76.example.com ; https://www.purvatti77.example.com ; https://www.sandrelo78.example.com ; https://www.okelmere79.example.com","values":["https://www.quenderi72.example.com","https://www.talvane73.example.com","https://www.vintara74.example.com","https://www.purvatti75.example.com","https://www.purvatti76.example.com","https://www.purvatti77.example.com","https://www.sandrelo78.example.com","https://www.okelmere79.example.com"]} -{"aliased":false,"column":"newsletter_opt_in","human_verified":true,"label":"boolean_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"newsletter_opt_in","text":"col:newsletter_opt_in | false ; false ; false ; true ; true ; false ; false ; false","values":["false","false","false","true","true","false","false","false"]} -{"aliased":false,"column":"newsletter_opt_in","human_verified":true,"label":"boolean_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"newsletter_opt_in","text":"col:newsletter_opt_in | false ; false ; true ; true ; false ; false ; false ; true","values":["false","false","true","true","false","false","false","true"]} -{"aliased":true,"column":"newsletter_opt_in","human_verified":true,"label":"boolean_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Newsletter Opt In","text":"col:Newsletter Opt In | true ; true ; true ; false ; false ; true ; true ; true","values":["true","true","true","false","false","true","true","true"]} -{"aliased":false,"column":"txn_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"txn_id","text":"col:txn_id | T9000179 ; T9000180 ; T9000181 ; T9000182 ; T9000183 ; T9000184 ; T9000185 ; T9000186","values":["T9000179","T9000180","T9000181","T9000182","T9000183","T9000184","T9000185","T9000186"]} -{"aliased":false,"column":"txn_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"txn_id","text":"col:txn_id | T9000119 ; T9000120 ; T9000121 ; T9000122 ; T9000123 ; T9000124 ; T9000125 ; T9000126","values":["T9000119","T9000120","T9000121","T9000122","T9000123","T9000124","T9000125","T9000126"]} -{"aliased":true,"column":"txn_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"txn-id","text":"col:txn-id | T9000066 ; T9000067 ; T9000068 ; T9000069 ; T9000070 ; T9000071 ; T9000072 ; T9000073","values":["T9000066","T9000067","T9000068","T9000069","T9000070","T9000071","T9000072","T9000073"]} -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100161 ; C100152 ; C100155 ; C100067 ; C100058 ; C100099 ; C100027 ; C100068","values":["C100161","C100152","C100155","C100067","C100058","C100099","C100027","C100068"]} -{"aliased":false,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"cust_id","text":"col:cust_id | C100005 ; C100085 ; C100188 ; C100072 ; C100017 ; C100005 ; C100077 ; C100014","values":["C100005","C100085","C100188","C100072","C100017","C100005","C100077","C100014"]} -{"aliased":true,"column":"cust_id","human_verified":true,"label":"identifier","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"CUST-ID","text":"col:CUST-ID | C100041 ; C100071 ; C100057 ; C100112 ; C100067 ; C100187 ; C100103 ; C100157","values":["C100041","C100071","C100057","C100112","C100067","C100187","C100103","C100157"]} -{"aliased":false,"column":"order_status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_status","text":"col:order_status | shipped ; shipped ; placed ; shipped ; returned ; shipped ; returned ; cancelled","values":["shipped","shipped","placed","shipped","returned","shipped","returned","cancelled"]} -{"aliased":false,"column":"order_status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_status","text":"col:order_status | returned ; delivered ; placed ; placed ; cancelled ; returned ; delivered ; placed","values":["returned","delivered","placed","placed","cancelled","returned","delivered","placed"]} -{"aliased":true,"column":"order_status","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"OrderStatus","text":"col:OrderStatus | shipped ; placed ; shipped ; returned ; shipped ; placed ; cancelled ; placed","values":["shipped","placed","shipped","returned","shipped","placed","cancelled","placed"]} -{"aliased":false,"column":"category_code","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"category_code","text":"col:category_code | HM-220 ; EL-200 ; EL-200 ; TY-500 ; TY-500 ; AP-300 ; AP-300 ; TY-500","values":["HM-220","EL-200","EL-200","TY-500","TY-500","AP-300","AP-300","TY-500"]} -{"aliased":false,"column":"category_code","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"category_code","text":"col:category_code | HM-220 ; EL-200 ; EL-200 ; TY-500 ; TY-500 ; AP-300 ; AP-300 ; TY-500","values":["HM-220","EL-200","EL-200","TY-500","TY-500","AP-300","AP-300","TY-500"]} -{"aliased":true,"column":"category_code","human_verified":true,"label":"category_code","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"CategoryCode","text":"col:CategoryCode | GR-110 ; TY-500 ; GR-110 ; EL-200 ; EL-100 ; TY-500 ; EL-200 ; TY-500","values":["GR-110","TY-500","GR-110","EL-200","EL-100","TY-500","EL-200","TY-500"]} -{"aliased":false,"column":"quantity","human_verified":true,"label":"quantity_with_unit","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"quantity","text":"col:quantity | 18 pcs ; 39 g ; 15 g ; 10 g ; 19 kg ; 16 m ; 35 m ; 7 ltr","values":["18 pcs","39 g","15 g","10 g","19 kg","16 m","35 m","7 ltr"]} -{"aliased":false,"column":"quantity","human_verified":true,"label":"quantity_with_unit","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"quantity","text":"col:quantity | 10 ltr ; 9 pcs ; 6 pcs ; 13 pcs ; 15 g ; 24 g ; 38 pcs ; 14 pcs","values":["10 ltr","9 pcs","6 pcs","13 pcs","15 g","24 g","38 pcs","14 pcs"]} -{"aliased":true,"column":"quantity","human_verified":true,"label":"quantity_with_unit","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Qty.","text":"col:Qty. | 34 pcs ; 14 kg ; 34 kg ; 11 g ; 32 ltr ; 17 pcs ; 18 pcs ; 7 pcs","values":["34 pcs","14 kg","34 kg","11 g","32 ltr","17 pcs","18 pcs","7 pcs"]} -{"aliased":false,"column":"unit_price_inr","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"unit_price_inr","text":"col:unit_price_inr | ₹448.63 ; ₹76,161.96 ; ₹19,838.82 ; ₹51,202.58 ; ₹82,650.09 ; ₹9,260.59 ; ₹56,618.59 ; ₹65,028.71","values":["₹448.63","₹76,161.96","₹19,838.82","₹51,202.58","₹82,650.09","₹9,260.59","₹56,618.59","₹65,028.71"]} -{"aliased":false,"column":"unit_price_inr","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"unit_price_inr","text":"col:unit_price_inr | ₹11,581.01 ; ₹89,151.05 ; ₹12,161.62 ; ₹36,571.65 ; ₹53,292.91 ; ₹79,740.98 ; ₹52,258.05 ; ₹20,675.74","values":["₹11,581.01","₹89,151.05","₹12,161.62","₹36,571.65","₹53,292.91","₹79,740.98","₹52,258.05","₹20,675.74"]} -{"aliased":true,"column":"unit_price_inr","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"UNIT_PRICE_INR","text":"col:UNIT_PRICE_INR | ₹52,718.03 ; ₹23,456.81 ; ₹85,154.75 ; ₹82,423.09 ; ₹12,683.40 ; ₹5,878.17 ; ₹2,049.11 ; ₹77,074.11","values":["₹52,718.03","₹23,456.81","₹85,154.75","₹82,423.09","₹12,683.40","₹5,878.17","₹2,049.11","₹77,074.11"]} -{"aliased":false,"column":"monthly_revenue","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"monthly_revenue","text":"col:monthly_revenue | 47031.3 ; 558559.75 ; 1533264.34 ; 1672002.96 ; 1184410.5 ; 2334413.88 ; 6153.76 ; 698430.48","values":["47031.3","558559.75","1533264.34","1672002.96","1184410.5","2334413.88","6153.76","698430.48"]} -{"aliased":false,"column":"monthly_revenue","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"monthly_revenue","text":"col:monthly_revenue | 487412.1 ; 2996829.81 ; 1013014.8 ; 540712.15 ; 289854.0 ; 451097.92 ; 828833.95 ; 1145331.33","values":["487412.1","2996829.81","1013014.8","540712.15","289854.0","451097.92","828833.95","1145331.33"]} -{"aliased":true,"column":"monthly_revenue","human_verified":true,"label":"currency_amount","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"MonthlyRevenue","text":"col:MonthlyRevenue | 130326.79 ; 498788.4 ; 1155576.18 ; 1247315.76 ; 2002511.16 ; 1040075.1 ; 761166.0 ; 1832754.44","values":["130326.79","498788.4","1155576.18","1247315.76","2002511.16","1040075.1","761166.0","1832754.44"]} -{"aliased":false,"column":"order_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_date","text":"col:order_date | 2023-06-21 ; 2023-02-05 ; 2025-11-08 ; 2025-03-25 ; 2022-07-20 ; 2024-03-14 ; 2023-08-24 ; 2025-08-11","values":["2023-06-21","2023-02-05","2025-11-08","2025-03-25","2022-07-20","2024-03-14","2023-08-24","2025-08-11"]} -{"aliased":false,"column":"order_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"order_date","text":"col:order_date | 2022-12-28 ; 2022-01-14 ; 2023-02-06 ; 2025-10-18 ; 2022-07-18 ; 2023-08-17 ; 2022-01-06 ; 2024-07-23","values":["2022-12-28","2022-01-14","2023-02-06","2025-10-18","2022-07-18","2023-08-17","2022-01-06","2024-07-23"]} -{"aliased":true,"column":"order_date","human_verified":true,"label":"date_like","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"ord_dt","text":"col:ord_dt | 2023-11-17 ; 2023-07-02 ; 2022-03-01 ; 2023-02-06 ; 2025-05-14 ; 2022-11-09 ; 2022-02-03 ; 2025-08-26","values":["2023-11-17","2023-07-02","2022-03-01","2023-02-06","2025-05-14","2022-11-09","2022-02-03","2025-08-26"]} -{"aliased":false,"column":"notes","human_verified":true,"label":"free_text","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"notes","text":"col:notes | gift wrap requested ; call before delivery ; left at door ; call before delivery ; gift wrap requested ; priority shipping ; call before delivery ; priority shipping","values":["gift wrap requested","call before delivery","left at door","call before delivery","gift wrap requested","priority shipping","call before delivery","priority shipping"]} -{"aliased":false,"column":"notes","human_verified":true,"label":"free_text","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"notes","text":"col:notes | call before delivery ; gift wrap requested ; call before delivery ; repeat customer ; priority shipping ; repeat customer ; left at door ; repeat customer","values":["call before delivery","gift wrap requested","call before delivery","repeat customer","priority shipping","repeat customer","left at door","repeat customer"]} -{"aliased":true,"column":"notes","human_verified":true,"label":"free_text","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"Notes","text":"col:Notes | left at door ; repeat customer ; call before delivery ; left at door ; priority shipping ; repeat customer ; priority shipping ; repeat customer","values":["left at door","repeat customer","call before delivery","left at door","priority shipping","repeat customer","priority shipping","repeat customer"]} -{"aliased":false,"column":"misc_0","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_0","text":"col:misc_0 | 0.1827 ; ? ; 499 ; 0.1827 ; ? ; 499","values":["0.1827","?","499","0.1827","?","499"]} -{"aliased":false,"column":"misc_1","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_1","text":"col:misc_1 | 0.0769 ; yy ; 122 ; 0.0769 ; yy ; 122","values":["0.0769","yy","122","0.0769","yy","122"]} -{"aliased":false,"column":"misc_2","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_2","text":"col:misc_2 | 0.9693 ; x ; 831 ; 0.9693 ; x ; 831","values":["0.9693","x","831","0.9693","x","831"]} -{"aliased":false,"column":"misc_3","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_3","text":"col:misc_3 | 0.2841 ; x ; 174 ; 0.2841 ; x ; 174","values":["0.2841","x","174","0.2841","x","174"]} -{"aliased":false,"column":"misc_4","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_4","text":"col:misc_4 | 0.0144 ; x ; 908 ; 0.0144 ; x ; 908","values":["0.0144","x","908","0.0144","x","908"]} -{"aliased":false,"column":"misc_5","human_verified":true,"label":"unknown","reviewed_at":"2026-07-04T05:15:00+00:00","reviewer":"JWD","shown_name":"misc_5","text":"col:misc_5 | 0.9682 ; x ; 878 ; 0.9682 ; x ; 878","values":["0.9682","x","878","0.9682","x","878"]} diff --git a/training/eval/human_verified.py b/training/eval/human_verified.py deleted file mode 100644 index bdf99b3a..00000000 --- a/training/eval/human_verified.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Human-verified release-gating eval labels. - -Release gates may only be computed on labels where a human reviewer signed -off — teacher labels alone never gate a release. Verified label files are -JSONL with, per row, the payload plus ``human_verified``, ``reviewer`` and -``reviewed_at``; :func:`load_verified` refuses anything less. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from ..common import TRAINING_ROOT, read_jsonl, utc_now_iso, write_jsonl - -DATA_DIR = TRAINING_ROOT / "eval" / "data" - - -class VerificationError(ValueError): - """A release-gating label set is not fully human-verified.""" - - -def verify_labels( - labels: list[dict[str, Any]], *, reviewer: str, reviewed: list[bool] | None = None -) -> list[dict[str, Any]]: - """Stamp labels as human-verified (used when a reviewer signs a batch).""" - if not reviewer.strip(): - raise VerificationError("reviewer id/initials are required") - flags = reviewed if reviewed is not None else [True] * len(labels) - if len(flags) != len(labels): - raise VerificationError("reviewed flags must match label count") - now = utc_now_iso() - out = [] - for label, flag in zip(labels, flags): - stamped = dict(label) - stamped["human_verified"] = bool(flag) - stamped["reviewer"] = reviewer.strip() - stamped["reviewed_at"] = now - out.append(stamped) - return out - - -def check_all_verified(labels: list[dict[str, Any]]) -> None: - """Raise unless every label is human-verified with a reviewer recorded.""" - if not labels: - raise VerificationError("release-gating eval set is empty") - bad = [ - i for i, label in enumerate(labels) - if label.get("human_verified") is not True - or not str(label.get("reviewer", "")).strip() - ] - if bad: - raise VerificationError( - f"{len(bad)}/{len(labels)} release-gating labels are not human-verified " - f"(first offenders: {bad[:5]}); teacher output alone cannot gate a release" - ) - - -def load_verified(path: Path | str) -> list[dict[str, Any]]: - """Load a release-gating label file, enforcing 100% human verification.""" - labels = read_jsonl(path) - check_all_verified(labels) - return labels - - -def save_verified(path: Path | str, labels: list[dict[str, Any]]) -> Path: - check_all_verified(labels) - return write_jsonl(path, labels) diff --git a/training/model_cards/calib-v1.md b/training/model_cards/calib-v1.md deleted file mode 100644 index 94dbe67a..00000000 --- a/training/model_cards/calib-v1.md +++ /dev/null @@ -1,64 +0,0 @@ -# Model card — `calib-v1` - -## 1. Purpose -Isotonic confidence-calibration tables mapping raw semantic-proposal scores -to calibrated probabilities of a correct repair, per (backend, issue -family). Plain JSON — not a neural model. - -## 2. Non-goals -- Does not create or veto proposals; it only rescales confidence. -- Never makes the deterministic protected-column checks probabilistic. - -## 3. Runtime behavior -Loaded via the Phase-3 model registry (or the packaged conservative -default). If the table is missing or corrupt the runtime passes raw scores -through and reports `calibration_version="uncalibrated"`. Embedding-backend -confidences are always capped below 1.0. - -## 4. Training data sources -CleanBench T1–T4 fixture runs with corruption-metadata ground truth (never -teacher guesses); the evaluation split is held out from fitting. - -## 5. License summary -Apache-2.0; training inputs are in-repo synthetic fixtures. - -## 6. PII policy -Fixtures are fully synthetic; no real PII enters calibration training. - -## 7. Teacher-model usage -None. Calibration is fitted exclusively on machine-verified outcomes. - -## 8. Human-review process -Release gates are computed on machine-verified outcomes over human-reviewed -fixture definitions; the CleanBench release run re-checks all gates. - -## 9. Evaluation metrics -`eval_metrics.json`: ECE and precision@0.95 on the held-out split. -Gates: ECE ≤ 0.03, P@0.95 ≥ 0.99, protected-column violation rate = 0, -false modification rate ≤ 0.1% (the last two via CleanBench). - -## 10. Known limitations -Curves are fitted on synthetic benchmark distributions; heavily -out-of-distribution data reverts to conservative behavior (embedding cap, -suggest-only below thresholds). - -## 11. Failure modes -Missing/corrupt table → uncalibrated passthrough (safe); unseen -(backend, issue) pairs fall back to the per-backend `*` curve or identity. - -## 12. Protected-column guarantee -Calibration never touches the deterministic byte-identity guard. - -## 13. Calibration version -`version` field inside `calibration.json` (this artifact). - -## 14. Artifact hash -See `manifest.json`. - -## 15. FreshData version compatibility -`freshdata_min_version` in `manifest.json`; enforced by the model registry. - ---- -**Explicit statements**: no generative repair model; no runtime LLM; no -automatic model download during cleaning; model output is evidence, not -authority; the protected-column guard remains absolute. diff --git a/training/model_cards/fd-col-encoder-v1.md b/training/model_cards/fd-col-encoder-v1.md deleted file mode 100644 index 05f54817..00000000 --- a/training/model_cards/fd-col-encoder-v1.md +++ /dev/null @@ -1,82 +0,0 @@ -# Model card — `fd-col-encoder-v1` (+ bundled role head) - -## 1. Purpose -Local column/value encoder for FreshData's optional embedding backend, with a -bundled semantic-type **role head** used as *additional evidence* for column -role inference. It rescues reference-value repairs the deterministic experts -cannot reach (distance-2 typos of allowed values) and scores column -semantic types. - -## 2. Non-goals -- Not a generative model; it never writes values. -- Not a general text embedder; scoped to tabular column names + cell values. -- Not a replacement for FreshData's deterministic experts or detectors. - -## 3. Runtime behavior -Loaded only when the user installs the `[semantic]` extra **and** explicitly -pulls the model (`freshdata models pull fd-col-encoder-v1`) or drops files -into `FRESHDATA_MODEL_DIR`. CPU-only, offline, deterministic. **Model output -is evidence, not authority**: every proposal goes through the same policy -gate, calibration, and audit trail as deterministic proposals, and ambiguous -merges are never auto-applied. - -## 4. Training data sources -Fully synthetic PII-shaped seed corpus (`training/seed/synthetic/`), -hand-authored fixtures, and corruptor-derived labeled pairs -(`training/seed/registry.json`). No real PII, no customer data, no scraped -private data, no production logs. - -## 5. License summary -Model artifact: Apache-2.0. All training data: CC0-1.0 / Apache-2.0 in-repo -sources (see `training/seed/LICENSES.md`). - -## 6. PII policy -Training data contains only synthetic, invented identities tagged -`synthetic=true`. Values sent to any development-time teacher model are -masked first (`training/teacher/tasks.py:mask_pii`). - -## 7. Teacher-model usage -Teacher models may contribute column-role labels and realism direction at -development time only (cached, compliance-gated, ≥5% human-reviewed). They -never generate messy/clean pairs and never gate a release. - -## 8. Human-review process -≥5% of teacher batches human-reviewed; >3% disagreement forces full batch -review; the release-gating eval split (`training/eval/data/`) is 100% -human-verified with reviewer identity and timestamps recorded. - -## 9. Evaluation metrics -See `eval_metrics.json` in this artifact (macro-F1, per-class F1, abstention -rate, content-detector contradiction rate, adversarial alias accuracy). -Release gates: macro-F1 ≥ 0.90, contradiction rate ≤ 1%, alias accuracy ≥ 0.85. - -## 10. Known limitations -- Eval corpus is synthetic; real-world column vocabularies are broader. -- English + Indian-English column headers only. -- Role head abstains (`unknown`) below its confidence threshold rather than - guessing; downstream code must handle abstention. - -## 11. Failure modes -Opaque codes and short low-support columns fall to `unknown`; heavily mixed -columns may be typed by their majority shape; adversarial headers reduce -confidence rather than flipping labels (gated). - -## 12. Protected-column guarantee -Unaffected by this model. The byte-identity protected-column guard is -deterministic, runs regardless of any model, and CleanBench gates its -violation rate at exactly zero. - -## 13. Calibration version -Scores are calibrated through `calib-v1` isotonic tables; without them the -runtime reports `calibration_version="uncalibrated"` and stays conservative. - -## 14. Artifact hash -See `manifest.json` (`sha256` per file, top-level artifact hash). - -## 15. FreshData version compatibility -`freshdata_min_version` in `manifest.json`; enforced by the model registry. - ---- -**Explicit statements**: no generative repair model; no runtime LLM; no -automatic model download during cleaning; model output is evidence, not -authority; the protected-column guard remains absolute. diff --git a/training/model_cards/fd-intent-v1.md b/training/model_cards/fd-intent-v1.md deleted file mode 100644 index a9975754..00000000 --- a/training/model_cards/fd-intent-v1.md +++ /dev/null @@ -1,78 +0,0 @@ -# Model card — `fd-intent-v1` - -## 1. Purpose -Small intent classifier for natural-language context sentences (13 intents: -DOMAIN, UNIQUE, VALID_FORMAT, LOCALE_FORMAT, PROTECT, IMPUTE_IF, -ALLOWED_VALUES, RANGE, DEDUP_KEY, DROP_IF, RENAME, MAP, UNKNOWN). It supplies -*optional evidence* to the deterministic context parser, e.g. flagging a -sentence the tier-0 patterns missed. - -## 2. Non-goals -- Never silently replaces deterministic parser behavior: a policy rule is - only created by the deterministic compiler. -- Not a chat model, not a general NLU system, no slot *generation* — slots - are extracted by deterministic regex/lexicon code. - -## 3. Runtime behavior -Optional, offline, CPU-only, explicit opt-in via the model registry. In -strict mode, unresolved or unparsed context is still surfaced to the user -exactly as before. Protection detection favors recall over coverage: -the PROTECT class is trained and gated at recall ≥ 0.99. - -## 4. Training data sources -Phase-1 golden context corpus (this repository, Apache-2.0), synthetic -paraphrase templates including the Indian-English/Hinglish set -(`training/seed/synthetic/generators.py`), corruptor context variants, and -optionally teacher paraphrases (cached, reviewed, training-only). - -## 5. License summary -Artifact: Apache-2.0. Data: in-repo Apache-2.0 + CC0 synthetic sources. - -## 6. PII policy -Context sentences reference column names, never cell values; the corpus is -synthetic/hand-authored. No real PII. - -## 7. Teacher-model usage -Paraphrase generation and ambiguity adjudication only, development-time -only, compliance-gated and cached. Teacher labels never gate releases. - -## 8. Human-review process -Release-gating eval labels are 100% human-verified; the eval split is -author-disjoint (held-out authors `t2`, `hinglish_b`) so paraphrase -generalization is measured, not memorization. - -## 9. Evaluation metrics -See `eval_metrics.json`: exact intent accuracy, slot F1, UNKNOWN precision, -protected-intent recall, conflict-detection accuracy, author-disjoint -paraphrase accuracy. Gates: exact ≥ 0.92, slot F1 ≥ 0.90, UNKNOWN precision -≥ 0.95, PROTECT recall ≥ 0.99. - -## 10. Known limitations -- Trained on template-derived paraphrases; entirely novel phrasing may - abstain to UNKNOWN (by design). -- English and Hinglish only. -- Sentences with multiple intents are classified by the dominant one; the - deterministic parser remains responsible for multi-intent splitting. - -## 11. Failure modes -Low-confidence predictions abstain to UNKNOWN rather than inventing -constraints; a missed PROTECT is the gated worst case (recall ≥ 0.99); -conflicting instructions are surfaced, never resolved silently. - -## 12. Protected-column guarantee -The model can only *add* protection evidence, never remove it. The runtime -byte-identity guard is deterministic and absolute. - -## 13. Calibration version -Head confidences feed the same `calib-v1` scoring path as other evidence. - -## 14. Artifact hash -See `manifest.json`. - -## 15. FreshData version compatibility -`freshdata_min_version` in `manifest.json`; enforced by the model registry. - ---- -**Explicit statements**: no generative repair model; no runtime LLM; no -automatic model download during cleaning; model output is evidence, not -authority; the protected-column guard remains absolute. diff --git a/training/seed/LICENSES.md b/training/seed/LICENSES.md deleted file mode 100644 index aaee4de5..00000000 --- a/training/seed/LICENSES.md +++ /dev/null @@ -1,33 +0,0 @@ -# Seed corpus license ledger - -Every seed source used for training must appear in `registry.json` with an -explicit license, attribution, `allowed_for_training`, and a resolved PII risk. -`python -m training.datasets.validators --check-licenses` enforces this and is -part of `make training-release-artifacts`. - -## Current sources - -| source_id | license | PII risk | commercial use | notes | -|---|---|---|---|---| -| `synthetic_core_v1` | CC0-1.0 (generated in-repo) | synthetic | yes | Fully synthetic; no real people, no production data. | -| `handauthored_ecommerce_v1` | CC0-1.0 | none | yes | Hand-written fixture rows. | -| `context_golden_phase1_v1` | Apache-2.0 | none | yes | Phase-1 golden context corpus from this repository. | - -## Allowed license classes - -- Public domain / CC0 / PDDL -- CC-BY-4.0 (attribution recorded in the registry) -- Permissive software licenses (Apache-2.0, MIT, BSD) for in-repo fixtures -- OGL / government open data with explicit commercial-use permission -- Fully synthetic data generated in this repository - -## Never allowed - -- Unclear-license or license-missing datasets -- Share-alike licenses (CC-BY-SA, ODbL, GPL-family data) **unless** a recorded - legal review approves the specific use (`legal_review.approved: true` on the - registry entry) -- Scraped private data, customer data, production logs -- Any dataset containing real PII - -License texts are vendored under `training/seed/licenses/`. diff --git a/training/seed/README.md b/training/seed/README.md deleted file mode 100644 index 6e435f5d..00000000 --- a/training/seed/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Seed corpus - -Inputs the training pipeline is allowed to learn from. Governance: - -- **`registry.json`** — every source, with license, attribution, explicit - `allowed_for_training`, and resolved `pii_risk` (`none | synthetic | - review_required`). A `review_required` entry blocks the release pipeline - until it carries an approving `legal_review` record. -- **`LICENSES.md`** — human-readable ledger; license texts in `licenses/`. -- **`sources/`** — small vendored files (hand-authored fixtures only). -- **`synthetic/`** — deterministic generators for PII-shaped fake data - (emails, Indian phone numbers, names, addresses, postal codes, IDs, - statuses, dates, revenue/currency strings, quantities with units, - city/state/country). No real people, no production data; every record is - tagged `synthetic=true`. - -Validate with: - -```bash -python -m training.datasets.validators --check-licenses -``` - -Never add: unclear-license data, share-alike data without recorded legal -approval, scraped private data, customer data, production logs, or real PII. diff --git a/training/seed/__init__.py b/training/seed/__init__.py deleted file mode 100644 index 3ea5c0f7..00000000 --- a/training/seed/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Seed corpus: registry-governed sources plus fully synthetic generators.""" diff --git a/training/seed/licenses/apache-2.0-notice.txt b/training/seed/licenses/apache-2.0-notice.txt deleted file mode 100644 index e18c6276..00000000 --- a/training/seed/licenses/apache-2.0-notice.txt +++ /dev/null @@ -1,7 +0,0 @@ -Apache License 2.0 (notice) - -This material is part of the FreshData repository and is licensed under the -Apache License, Version 2.0 — the same license as the project. See the -top-level LICENSE file for the full text. - -Full text: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/training/seed/licenses/cc-by-4.0.txt b/training/seed/licenses/cc-by-4.0.txt deleted file mode 100644 index 2fc2d7d8..00000000 --- a/training/seed/licenses/cc-by-4.0.txt +++ /dev/null @@ -1,7 +0,0 @@ -Creative Commons Attribution 4.0 International (summary) - -You are free to share and adapt the material for any purpose, even -commercially, provided you give appropriate credit, provide a link to the -license, and indicate if changes were made. - -Full text: https://creativecommons.org/licenses/by/4.0/legalcode diff --git a/training/seed/licenses/cc0-1.0.txt b/training/seed/licenses/cc0-1.0.txt deleted file mode 100644 index a1231034..00000000 --- a/training/seed/licenses/cc0-1.0.txt +++ /dev/null @@ -1,9 +0,0 @@ -CC0 1.0 Universal (summary) - -The person who associated a work with this deed has dedicated the work to the -public domain by waiving all of his or her rights to the work worldwide under -copyright law, including all related and neighboring rights, to the extent -allowed by law. You can copy, modify, distribute and perform the work, even -for commercial purposes, all without asking permission. - -Full text: https://creativecommons.org/publicdomain/zero/1.0/legalcode diff --git a/training/seed/registry.json b/training/seed/registry.json deleted file mode 100644 index 455370b5..00000000 --- a/training/seed/registry.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "registry_version": "1", - "sources": [ - { - "source_id": "synthetic_core_v1", - "name": "FreshData synthetic PII-shaped core corpus", - "url": "training/seed/synthetic/generators.py", - "license": "CC0-1.0", - "license_text_path": "licenses/cc0-1.0.txt", - "attribution": "Generated in-repo by training/seed/synthetic/generators.py; no external data.", - "allowed_for_training": true, - "pii_risk": "synthetic", - "notes": "Fully synthetic: invented names/addresses, fake emails on example.* domains, fake Indian phone numbers. Deterministic under seed; every record carries synthetic=true." - }, - { - "source_id": "handauthored_ecommerce_v1", - "name": "Hand-authored ecommerce mini fixtures", - "url": "training/seed/sources/handauthored_ecommerce_v1.csv", - "license": "CC0-1.0", - "license_text_path": "licenses/cc0-1.0.txt", - "attribution": "Hand-written by FreshData maintainers for corruptor and parser fixtures.", - "allowed_for_training": true, - "pii_risk": "none", - "notes": "24 rows of invented ecommerce records mirroring the CleanBench base frame; no real people." - }, - { - "source_id": "context_golden_phase1_v1", - "name": "Phase-1 golden context corpus", - "url": "tests/context/golden/", - "license": "Apache-2.0", - "license_text_path": "licenses/apache-2.0-notice.txt", - "attribution": "FreshData repository test corpus (same license as the project).", - "allowed_for_training": true, - "pii_risk": "none", - "notes": "English context sentences plus compiled golden policies; used as intent-head ground truth." - } - ] -} diff --git a/training/seed/sources/handauthored_ecommerce_v1.csv b/training/seed/sources/handauthored_ecommerce_v1.csv deleted file mode 100644 index 044fb9e1..00000000 --- a/training/seed/sources/handauthored_ecommerce_v1.csv +++ /dev/null @@ -1,25 +0,0 @@ -cust_id,email,phone,age,monthly_revenue,status,city,country,synthetic -C900001,asha.voskette1@example.com,+919876543210,34,1250.00,active,Mumbai,India,true -C900002,ravi.talvane2@example.org,+919812345670,28,980.50,pending,Pune,India,true -C900003,neha.quenderi3@example.net,+919900112233,41,15000.00,inactive,Bengaluru,India,true -C900004,vikram.mirostra4@example.com,+919811223344,52,720.25,active,Chennai,India,true -C900005,priya.kelvani5@example.org,+919922334455,23,310.00,pending,Hyderabad,India,true -C900006,arjun.purvatti6@example.net,+919933445566,37,4400.75,active,Delhi,India,true -C900007,meera.sandrelo7@example.com,+919944556677,29,860.00,inactive,Kolkata,India,true -C900008,kiran.vintara8@example.org,+919955667788,45,2300.10,active,Jaipur,India,true -C900009,divya.okelmere9@example.net,+919966778899,31,199.99,pending,Mumbai,India,true -C900010,rohan.ramblewood10@example.com,+919977889900,26,5600.00,active,Pune,India,true -C900011,sneha.voskette11@example.org,+919988990011,38,75.50,churned,Bengaluru,India,true -C900012,amit.talvane12@example.net,+919899001122,33,8100.00,active,Chennai,India,true -C900013,tara.quenderi13@example.com,+919810101010,47,640.00,inactive,Hyderabad,India,true -C900014,dev.mirostra14@example.org,+919820202020,22,1550.25,pending,Delhi,India,true -C900015,isha.kelvani15@example.net,+919830303030,36,920.80,active,Kolkata,India,true -C900016,kabir.purvatti16@example.com,+919840404040,58,12000.00,active,Jaipur,India,true -C900017,asha.sandrelo17@example.org,+919850505050,27,450.00,pending,Mumbai,India,true -C900018,ravi.vintara18@example.net,+919860606060,44,3300.33,inactive,Pune,India,true -C900019,neha.okelmere19@example.com,+919870707070,30,780.00,active,Bengaluru,India,true -C900020,vikram.ramblewood20@example.org,+919880808080,39,2100.00,active,Chennai,India,true -C900021,priya.voskette21@example.net,+919890909090,25,95.00,churned,Hyderabad,India,true -C900022,arjun.talvane22@example.com,+919801020304,49,6700.45,active,Delhi,India,true -C900023,meera.quenderi23@example.org,+919802030405,35,1800.00,pending,Kolkata,India,true -C900024,kiran.mirostra24@example.net,+919803040506,42,990.90,active,Jaipur,India,true diff --git a/training/seed/synthetic/__init__.py b/training/seed/synthetic/__init__.py deleted file mode 100644 index 87a4fd12..00000000 --- a/training/seed/synthetic/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Synthetic PII-shaped data generators (no real people, no real PII).""" - -from .generators import ( - SYNTHETIC_SOURCE_ID, - make_context_sentences, - make_customers, - make_transactions, - seed_tables, -) - -__all__ = [ - "SYNTHETIC_SOURCE_ID", - "make_context_sentences", - "make_customers", - "make_transactions", - "seed_tables", -] diff --git a/training/seed/synthetic/generators.py b/training/seed/synthetic/generators.py deleted file mode 100644 index 21d028d8..00000000 --- a/training/seed/synthetic/generators.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Deterministic synthetic PII-shaped data. - -All identity-shaped values are assembled from invented token lists — none of -the names, addresses, or numbers are derived from real people, customer data, -or production logs. Every generator takes an explicit ``seed`` and produces -byte-identical output for the same seed, and every record carries -``synthetic=True`` metadata (also mirrored in ``DataFrame.attrs``). -""" - -from __future__ import annotations - -import random -from typing import Any - -import pandas as pd - -SYNTHETIC_SOURCE_ID = "synthetic_core_v1" - -# Invented tokens only. Common Indian-flavored given names are generic -# dictionary words in this context; surnames and streets are deliberately -# fictional compounds so no full name can collide with a real identity record. -_GIVEN = ( - "asha", "ravi", "neha", "vikram", "priya", "arjun", "meera", "kiran", - "divya", "rohan", "sneha", "amit", "tara", "dev", "isha", "kabir", -) -_SURNAME = ( - "Voskette", "Ramblewood", "Quenderi", "Talvane", "Mirostra", "Kelvani", - "Purvatti", "Sandrelo", "Vintara", "Okelmere", -) -_STREET = ( - "Lantern Row", "Copperleaf Marg", "Quarry Lane", "Palmshade Street", - "Riverbend Road", "Old Mill Avenue", "Jacaranda Nagar", "Harborview Block", -) -_CITY_STATE = ( - ("Mumbai", "Maharashtra"), ("Pune", "Maharashtra"), ("Bengaluru", "Karnataka"), - ("Chennai", "Tamil Nadu"), ("Hyderabad", "Telangana"), ("Delhi", "Delhi"), - ("Kolkata", "West Bengal"), ("Jaipur", "Rajasthan"), -) -_COUNTRY = ("India", "India", "India", "Australia", "Austria", "Canada", "Singapore") -_EMAIL_DOMAINS = ("example.com", "example.org", "example.net", "mail.example.in") -_STATUS = ("active", "inactive", "pending", "churned") -_UNITS = ("kg", "g", "pcs", "ltr", "m") -_CATEGORY_CODES = ("EL-100", "EL-200", "AP-300", "GR-110", "HM-220", "TY-500") - - -def _rng(seed: int) -> random.Random: - return random.Random(("freshdata-synthetic", seed).__repr__()) - - -def _mark_synthetic(df: pd.DataFrame) -> pd.DataFrame: - df.attrs["synthetic"] = True - df.attrs["source_id"] = SYNTHETIC_SOURCE_ID - return df - - -def make_customers(n: int = 200, seed: int = 0) -> pd.DataFrame: - """Customer-shaped frame: ids, names, emails, Indian phones, addresses.""" - rng = _rng(seed) - rows: list[dict[str, Any]] = [] - for i in range(n): - given = rng.choice(_GIVEN) - surname = rng.choice(_SURNAME) - city, state = rng.choice(_CITY_STATE) - # Indian mobile numbers start 6-9; keep the canonical +91 form as truth. - ten_digit = f"{rng.randint(6, 9)}{rng.randrange(10**8, 10**9):09d}" - rows.append({ - "cust_id": f"C{100000 + i}", - "full_name": f"{given.title()} {surname}", - "email": f"{given}.{surname.lower()}{i}@{rng.choice(_EMAIL_DOMAINS)}", - "phone": f"+91{ten_digit}", - "address": f"{rng.randint(1, 240)} {rng.choice(_STREET)}", - "city": city, - "state": state, - "country": rng.choice(_COUNTRY), - "postal_code": f"{rng.randint(110001, 899999)}", - "national_id_like": f"ZZ{rng.randrange(10**7, 10**8)}X", - "status": rng.choice(_STATUS), - "signup_date": ( - f"{rng.randint(2019, 2025)}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" - ), - "website": f"https://www.{surname.lower()}{i}.example.com", - "newsletter_opt_in": rng.choice(("true", "false")), - "synthetic": True, - }) - return _mark_synthetic(pd.DataFrame(rows)) - - -def make_transactions(n: int = 300, seed: int = 1) -> pd.DataFrame: - """Transaction-shaped frame: ids, currency strings, quantities, revenue.""" - rng = _rng(seed) - rows: list[dict[str, Any]] = [] - for i in range(n): - amount = round(rng.uniform(49.0, 99999.0), 2) - qty = rng.randint(1, 40) - rows.append({ - "txn_id": f"T{9000000 + i}", - "cust_id": f"C{100000 + rng.randrange(200)}", - "order_status": rng.choice( - ("placed", "shipped", "delivered", "returned", "cancelled")), - "category_code": rng.choice(_CATEGORY_CODES), - "quantity": f"{qty} {rng.choice(_UNITS)}", - "unit_price_inr": f"₹{amount:,.2f}", - "monthly_revenue": f"{round(amount * qty, 2)}", - "order_date": ( - f"{rng.randint(2022, 2025)}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" - ), - "notes": rng.choice(( - "priority shipping", "gift wrap requested", "repeat customer", - "call before delivery", "left at door", - )), - "synthetic": True, - }) - return _mark_synthetic(pd.DataFrame(rows)) - - -def make_context_sentences(seed: int = 2) -> list[dict[str, Any]]: - """Labeled context sentences (intent ground truth) with author tags. - - ``author`` supports author-disjoint eval splits; the ``hinglish`` authors - provide the Indian-English / Hinglish paraphrase set required by Phase 5. - """ - rng = _rng(seed) - templates: list[tuple[str, str, dict[str, Any], str]] = [ - ("This is an ecommerce customer dataset.", "DOMAIN", - {"domain": "ecommerce_customer"}, "t0"), - ("This is a retail transactions dataset.", "DOMAIN", - {"domain": "retail_transactions"}, "t1"), - ("This dataset is an ecommerce orders dataset.", "DOMAIN", - {"domain": "ecommerce_orders"}, "t2"), - ("This is a customer support dataset.", "DOMAIN", {"domain": "customer_support"}, "t3"), - # UNIQUE — shared vocabulary: unique / duplicate / once / repeat - ("{col} is unique.", "UNIQUE", {}, "t0"), - ("{col} must be unique.", "UNIQUE", {}, "t1"), - ("{col} values should be unique with no duplicates.", "UNIQUE", {}, "t3"), - ("No duplicate values in {col}.", "UNIQUE", {}, "t4"), - ("{col} should not repeat.", "UNIQUE", {}, "t0"), - ("Each {col} appears once, duplicates are not allowed.", "UNIQUE", {}, "t1"), - ("{col} appears only once per table.", "UNIQUE", {}, "t3"), - ("Every value in {col} occurs only once.", "UNIQUE", {}, "t4"), - ("Every {col} should appear only once.", "UNIQUE", {}, "t2"), - ("{col} must be unique, one per row.", "UNIQUE", {}, "t2"), - ("{col} har row me unique hona chahiye.", "UNIQUE", {}, "hinglish_a"), - ("{col} repeat nahi hona chahiye, unique rakho.", "UNIQUE", {}, "hinglish_b"), - # VALID_FORMAT — shared vocabulary: valid / email(s) - ("{col} must be valid emails.", "VALID_FORMAT", {"format": "email"}, "t0"), - ("Emails in {col} must be valid.", "VALID_FORMAT", {"format": "email"}, "t1"), - ("{col} should contain valid email addresses.", "VALID_FORMAT", {"format": "email"}, "t3"), - ("Make sure {col} emails are valid.", "VALID_FORMAT", {"format": "email"}, "t4"), - ("{col} emails must all be valid addresses.", "VALID_FORMAT", {"format": "email"}, "t2"), - ("{col} me valid email hona chahiye.", "VALID_FORMAT", {"format": "email"}, "hinglish_a"), - ("{col} ke emails valid hone chahiye.", "VALID_FORMAT", {"format": "email"}, "hinglish_b"), - # LOCALE_FORMAT — shared vocabulary: Indian / phone numbers - ("{col} are Indian phone numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t0"), - ("Phone numbers are Indian.", "LOCALE_FORMAT", {"locale": "IN", "format": "phone"}, "t1"), - ("{col} holds Indian phone numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t3"), - ("Treat {col} as Indian phone numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t4"), - ("The phone numbers in {col} are Indian numbers.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "t2"), - ("{col} me Indian phone numbers hain.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "hinglish_a"), - ("{col} ke phone numbers Indian hain.", "LOCALE_FORMAT", - {"locale": "IN", "format": "phone"}, "hinglish_b"), - # PROTECT — shared vocabulary: never modify / protected / do not change - ("Never modify {col} values.", "PROTECT", {}, "t0"), - ("Do not touch {col}.", "PROTECT", {}, "t1"), - ("Do not change {col}.", "PROTECT", {}, "t3"), - ("{col} must never be modified.", "PROTECT", {}, "t4"), - ("Leave {col} untouched, it is protected.", "PROTECT", {}, "t0"), - ("{col} is protected.", "PROTECT", {}, "t2"), - ("{col} is protected, never change it.", "PROTECT", {}, "t2"), - ("{col} kabhi mat badalna.", "PROTECT", {}, "hinglish_a"), - ("{col} ko modify mat karna, protected hai.", "PROTECT", {}, "hinglish_a"), - ("{col} ko change mat karo, bilkul nahi.", "PROTECT", {}, "hinglish_b"), - ("{col} protected hai, mat badalna.", "PROTECT", {}, "hinglish_b"), - # IMPUTE_IF — shared vocabulary: missing / estimated / confidence % - ("Missing {col} should be estimated only if confidence >95%.", "IMPUTE_IF", - {"threshold": 0.95}, "t0"), - ("Fill {col} gaps only when at least 95% sure.", "IMPUTE_IF", {"threshold": 0.95}, "t1"), - ("Estimate missing {col} only above 95% confidence.", "IMPUTE_IF", - {"threshold": 0.95}, "t3"), - ("Impute {col} only with confidence over 90%.", "IMPUTE_IF", {"threshold": 0.90}, "t4"), - ("Missing {col} may be estimated only at 95% confidence or more.", "IMPUTE_IF", - {"threshold": 0.95}, "t2"), - ("{col} missing ho to sirf 95% confidence par estimate karo.", "IMPUTE_IF", - {"threshold": 0.95}, "hinglish_a"), - ("{col} khali ho to sirf 95% sure hone par bharna.", "IMPUTE_IF", {"threshold": 0.95}, - "hinglish_a"), - ("{col} khali ho to 95% pakka hone par hi bharna.", "IMPUTE_IF", {"threshold": 0.95}, - "hinglish_b"), - # ALLOWED_VALUES — shared vocabulary: allowed / values / only be - ("Allowed status values are active, inactive, pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t0"), - ("{col} can only be active, inactive or pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t1"), - ("Allowed values for {col} are active, inactive, pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t3"), - ("Valid {col} values: active, inactive, pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t4"), - ("The only allowed {col} values are active, inactive and pending.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "t2"), - ("Status sirf active, inactive ya pending ho sakta hai.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "hinglish_a"), - ("{col} sirf active, inactive ya pending allowed hai.", "ALLOWED_VALUES", - {"values": ["active", "inactive", "pending"]}, "hinglish_b"), - # RANGE — shared vocabulary: between X and Y / within - ("{col} must be between 18 and 99.", "RANGE", {"min": 18, "max": 99}, "t0"), - ("Keep {col} within 0 to 100.", "RANGE", {"min": 0, "max": 100}, "t1"), - ("{col} should stay between 1 and 40.", "RANGE", {"min": 1, "max": 40}, "t3"), - ("Values of {col} must lie between 0 and 999.", "RANGE", {"min": 0, "max": 999}, "t4"), - ("{col} has to be between 18 and 65.", "RANGE", {"min": 18, "max": 65}, "t2"), - ("{col} 5 aur 95 ke beech hona chahiye.", "RANGE", {"min": 5, "max": 95}, "hinglish_a"), - ("{col} between 10 aur 50 ke beech hona chahiye.", "RANGE", {"min": 10, "max": 50}, - "hinglish_b"), - # DEDUP_KEY — shared vocabulary: deduplicate / dedup key - ("Deduplicate rows by {col}.", "DEDUP_KEY", {}, "t0"), - ("Use {col} as the dedup key.", "DEDUP_KEY", {}, "t1"), - ("Deduplicate on {col}.", "DEDUP_KEY", {}, "t3"), - ("Rows should be deduplicated using {col} as the key.", "DEDUP_KEY", {}, "t2"), - ("{col} se deduplicate karo.", "DEDUP_KEY", {}, "hinglish_b"), - # DROP_IF — shared vocabulary: drop / remove rows / empty / missing - ("Drop rows if {col} is empty.", "DROP_IF", {"condition": "empty"}, "t0"), - ("Remove records where {col} is missing.", "DROP_IF", {"condition": "empty"}, "t1"), - ("Drop any row with empty {col}.", "DROP_IF", {"condition": "empty"}, "t3"), - ("Missing {col} rows should be dropped.", "DROP_IF", {"condition": "empty"}, "t4"), - ("Rows should be dropped when {col} is missing.", "DROP_IF", {"condition": "empty"}, "t0"), - ("Rows with missing {col} should be dropped.", "DROP_IF", {"condition": "empty"}, "t2"), - ("{col} empty ho to row drop kar do.", "DROP_IF", {"condition": "empty"}, "hinglish_b"), - # RENAME — shared vocabulary: rename ... to - ("Rename {col} to customer_id.", "RENAME", {"to": "customer_id"}, "t0"), - ("Please rename {col} to customer_id.", "RENAME", {"to": "customer_id"}, "t1"), - ("Rename column {col} to customer_id.", "RENAME", {"to": "customer_id"}, "t3"), - ("{col} ko customer_id rename kar do.", "RENAME", {"to": "customer_id"}, "hinglish_a"), - ("Rename the {col} column to customer_id.", "RENAME", {"to": "customer_id"}, "t2"), - ("{col} ko rename karke customer_id kar do.", "RENAME", {"to": "customer_id"}, - "hinglish_b"), - # MAP — shared vocabulary: map X to Y - ("Map M to male and F to female in {col}.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "t0"), - ("In {col}, map M to male and F to female.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "t1"), - ("Map the code M to male and the code F to female in {col}.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "t2"), - ("{col} me M ko male map karo aur F ko female.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "hinglish_a"), - ("{col} me M ko male aur F ko female map karo.", "MAP", - {"mapping": {"M": "male", "F": "female"}}, "hinglish_b"), - # UNKNOWN — no actionable constraint - ("The vibes of this table are immaculate.", "UNKNOWN", {}, "t0"), - ("Please make everything nicer somehow.", "UNKNOWN", {}, "t1"), - ("Great data, love it.", "UNKNOWN", {}, "t3"), - ("Have a look and see what you think.", "UNKNOWN", {}, "t4"), - ("Honestly this table just needs some love.", "UNKNOWN", {}, "t2"), - ("Kuch accha sa kar do is table ke saath.", "UNKNOWN", {}, "hinglish_b"), - ("Table ke saath kuch accha kar do.", "UNKNOWN", {}, "hinglish_a"), - ("Data ko thoda accha bana do yaar.", "UNKNOWN", {}, "hinglish_a"), - ] - columns = ("CustomerID", "cust_id", "email_addr", "mobile", "status", - "monthly_revenue", "age", "order_date", "postal_code") - out: list[dict[str, Any]] = [] - for text, intent, slots, author in templates: - picks = rng.sample(columns, k=3) if "{col}" in text else [None] - for col in picks: - sentence = text.format(col=col) if col is not None else text - out.append({ - "sentence": sentence, - "intent": intent, - "slots": dict(slots, column=col) if col is not None else dict(slots), - "author": author, - "synthetic": True, - "source_id": SYNTHETIC_SOURCE_ID, - }) - return out - - -def seed_tables(seed: int = 0) -> dict[str, pd.DataFrame]: - """All synthetic seed tables keyed by name.""" - return { - "customers": make_customers(seed=seed), - "transactions": make_transactions(seed=seed + 1), - } diff --git a/training/teacher/__init__.py b/training/teacher/__init__.py deleted file mode 100644 index 2e16a6bd..00000000 --- a/training/teacher/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Development-time teacher harness. - -Frontier teacher models are allowed **only** here, **only** for: - -1. realism direction (corruption frequency/parameter tuning), -2. column-role labeling, -3. NL context paraphrase generation, -4. ambiguity adjudication, -5. rationale style templates, -6. benchmark red-teaming. - -They are never called at runtime, never see full rows or real PII, never -generate messy/clean pairs from scratch, and their outputs are training data — -release-gating truth requires human verification (see ``review.py``). -""" diff --git a/training/teacher/cache.py b/training/teacher/cache.py deleted file mode 100644 index 7e653c2b..00000000 --- a/training/teacher/cache.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Content-addressed teacher output cache with a full audit trail. - -Key = ``provider + model + prompt_template_sha256 + input_sha256 + -schema_version``. Every entry stores the raw prompt, the raw response, and -provenance (provider, model, date, terms snapshot id) so any training label -can be traced to the exact call that produced it — and so re-runs never pay -for the same call twice. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from ..common import CACHE_DIR, read_json, sha256_text, utc_now_iso, write_json - - -@dataclass(frozen=True) -class CacheKey: - provider: str - model: str - prompt_template_sha256: str - input_sha256: str - schema_version: str - - def digest(self) -> str: - joined = "|".join(( - self.provider, self.model, self.prompt_template_sha256, - self.input_sha256, self.schema_version, - )) - return sha256_text(joined) - - -class TeacherCache: - def __init__(self, root: Path | str | None = None) -> None: - self.root = Path(root) if root is not None else CACHE_DIR / "teacher" - - def _path(self, key: CacheKey) -> Path: - digest = key.digest() - return self.root / digest[:2] / f"{digest}.json" - - def get(self, key: CacheKey) -> dict[str, Any] | None: - path = self._path(key) - if not path.is_file(): - return None - return read_json(path) - - def put( - self, - key: CacheKey, - *, - prompt: str, - response: str, - payloads: list[dict[str, Any]], - terms_snapshot_id: str, - ) -> Path: - entry = { - "key": { - "provider": key.provider, - "model": key.model, - "prompt_template_sha256": key.prompt_template_sha256, - "input_sha256": key.input_sha256, - "schema_version": key.schema_version, - }, - "prompt": prompt, - "response": response, - "payloads": payloads, - "provider": key.provider, - "model": key.model, - "created_at": utc_now_iso(), - "terms_snapshot_id": terms_snapshot_id, - } - return write_json(self._path(key), entry) - - def entries(self) -> list[dict[str, Any]]: - if not self.root.is_dir(): - return [] - return [read_json(p) for p in sorted(self.root.rglob("*.json"))] diff --git a/training/teacher/clients.py b/training/teacher/clients.py deleted file mode 100644 index 6beb3572..00000000 --- a/training/teacher/clients.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Teacher clients: a stub for offline development, a gated HTTP client. - -Nothing in the runtime package touches this module. There is no vendored -provider SDK; the HTTP client is configured entirely by environment -variables and refuses to run without them **and** without an approved -compliance record (checked by the caller in ``tasks.py``). -""" - -from __future__ import annotations - -import json -import os -import urllib.request -from typing import Any, Protocol - - -class TeacherUnavailable(RuntimeError): - """The teacher client cannot run (unconfigured, offline, or refused).""" - - -class TeacherClient(Protocol): - provider: str - model: str - - def complete(self, prompt: str, *, schema: str) -> str: - """Return the raw response text for one batched prompt.""" - ... - - -class StubTeacherClient: - """Deterministic offline stand-in used by tests and dev pipelines. - - Responses come from a canned mapping keyed by schema; by default it - returns an empty batch, which downstream tasks treat as "degrade to - corruptor/hook data". - """ - - provider = "stub" - model = "stub-teacher-0" - - def __init__(self, canned: dict[str, Any] | None = None, *, fail: bool = False) -> None: - self._canned = canned or {} - self._fail = fail - self.calls: list[dict[str, str]] = [] - - def complete(self, prompt: str, *, schema: str) -> str: - self.calls.append({"schema": schema, "prompt": prompt}) - if self._fail: - raise TeacherUnavailable("stub teacher configured to fail") - return json.dumps(self._canned.get(schema, [])) - - -class HTTPTeacherClient: - """Minimal JSON-over-HTTP teacher client (development-time only). - - Configuration (all required): - - - ``FRESHDATA_TEACHER_URL`` — full endpoint URL - - ``FRESHDATA_TEACHER_PROVIDER`` — provider name for compliance/cache keys - - ``FRESHDATA_TEACHER_MODEL`` — model name - - ``FRESHDATA_TEACHER_API_KEY`` — bearer token (never committed) - - The request body is ``{"model": ..., "prompt": ..., "schema": ...}`` and - the response body must be the raw JSON batch. Anything fancier belongs in - a local adapter script, not in the repository. - """ - - def __init__(self) -> None: - self.url = os.environ.get("FRESHDATA_TEACHER_URL", "") - self.provider = os.environ.get("FRESHDATA_TEACHER_PROVIDER", "") - self.model = os.environ.get("FRESHDATA_TEACHER_MODEL", "") - self._api_key = os.environ.get("FRESHDATA_TEACHER_API_KEY", "") - if not (self.url and self.provider and self.model and self._api_key): - raise TeacherUnavailable( - "teacher client not configured: set FRESHDATA_TEACHER_URL, " - "FRESHDATA_TEACHER_PROVIDER, FRESHDATA_TEACHER_MODEL, " - "FRESHDATA_TEACHER_API_KEY" - ) - - def complete(self, prompt: str, *, schema: str) -> str: - body = json.dumps({"model": self.model, "prompt": prompt, "schema": schema}) - request = urllib.request.Request( - self.url, - data=body.encode("utf-8"), - headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {self._api_key}", - }, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=120) as response: - return response.read().decode("utf-8") - except OSError as exc: - raise TeacherUnavailable(f"teacher call failed: {exc}") from exc - - -def default_client() -> TeacherClient: - """HTTP client when configured, else the empty stub (degraded mode).""" - try: - return HTTPTeacherClient() - except TeacherUnavailable: - return StubTeacherClient() diff --git a/training/teacher/compliance.py b/training/teacher/compliance.py deleted file mode 100644 index 1668127e..00000000 --- a/training/teacher/compliance.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Provider-terms compliance ledger and pre-run gate. - -Before any teacher run, a human records a compliance check for the provider -in ``training/teacher/compliance_ledger.json``. :func:`require_approved` -blocks the run when the record is missing, stale, unapproved, reviewer-less, -or when the provider's terms disallow using outputs for model training. - -CLI:: - - python -m training.teacher.compliance check [--provider X --use labeling] -""" - -from __future__ import annotations - -import argparse -import sys -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from ..common import read_json, utc_now_iso, write_json - -LEDGER_PATH = Path(__file__).resolve().parent / "compliance_ledger.json" -#: A terms check older than this is stale and must be re-verified. -MAX_AGE_DAYS = 180 - -ALLOWED_USES = ( - "realism_direction", - "column_role_labeling", - "context_paraphrase", - "ambiguity_adjudication", - "rationale_templates", - "red_teaming", -) - - -class ComplianceError(RuntimeError): - """The teacher run is blocked by the compliance gate.""" - - -def load_ledger(path: Path | str = LEDGER_PATH) -> dict[str, Any]: - path = Path(path) - if not path.is_file(): - return {"ledger_version": "1", "providers": []} - return read_json(path) - - -def _entry_for(ledger: dict[str, Any], provider: str) -> dict[str, Any] | None: - for entry in ledger.get("providers", []): - if entry.get("provider") == provider: - return entry - return None - - -def check_entry(entry: dict[str, Any] | None, provider: str, intended_use: str) -> list[str]: - """Return blocking problems for one provider/use pair (empty = approved).""" - problems: list[str] = [] - if entry is None: - problems.append(f"no terms snapshot recorded for provider {provider!r}") - return problems - if not entry.get("terms_url") or not entry.get("terms_snapshot_id"): - problems.append(f"{provider}: no terms snapshot exists") - checked = str(entry.get("date_checked", "")) - try: - age = datetime.now(timezone.utc) - datetime.fromisoformat(checked) - if age.days > MAX_AGE_DAYS: - problems.append(f"{provider}: terms snapshot is stale ({age.days} days old)") - except ValueError: - problems.append(f"{provider}: date_checked missing or unparseable") - if not entry.get("reviewer"): - problems.append(f"{provider}: reviewer is missing") - if entry.get("status") != "approved": - problems.append(f"{provider}: compliance status is {entry.get('status')!r}, not approved") - if entry.get("training_use_allowed") is not True: - problems.append(f"{provider}: provider terms do not allow training use of outputs") - allowed = entry.get("allowed_uses", []) - if intended_use not in allowed: - problems.append( - f"{provider}: intended use {intended_use!r} is not approved (approved: {allowed})" - ) - return problems - - -def require_approved( - provider: str, intended_use: str, *, ledger_path: Path | str = LEDGER_PATH -) -> dict[str, Any]: - """Return the approved ledger entry or raise :class:`ComplianceError`.""" - if intended_use not in ALLOWED_USES: - raise ComplianceError( - f"intended use {intended_use!r} is not a permitted teacher use; " - f"allowed: {ALLOWED_USES}" - ) - entry = _entry_for(load_ledger(ledger_path), provider) - problems = check_entry(entry, provider, intended_use) - if problems: - raise ComplianceError("; ".join(problems)) - assert entry is not None - return entry - - -def record_check( - *, - provider: str, - terms_url: str, - terms_snapshot_id: str, - reviewer: str, - allowed_uses: list[str], - training_use_allowed: bool, - redistribution_allowed: bool, - status: str = "approved", - notes: str = "", - ledger_path: Path | str = LEDGER_PATH, -) -> dict[str, Any]: - """Append or replace a provider compliance record (manual review result).""" - ledger = load_ledger(ledger_path) - entry = { - "provider": provider, - "terms_url": terms_url, - "terms_snapshot_id": terms_snapshot_id, - "date_checked": utc_now_iso(), - "reviewer": reviewer, - "allowed_uses": allowed_uses, - "training_use_allowed": training_use_allowed, - "redistribution_allowed": redistribution_allowed, - "status": status, - "notes": notes, - } - ledger["providers"] = [e for e in ledger.get("providers", []) if e.get("provider") != provider] - ledger["providers"].append(entry) - write_json(ledger_path, ledger) - return entry - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m training.teacher.compliance") - sub = parser.add_subparsers(dest="command", required=True) - check = sub.add_parser("check", help="verify compliance for a provider/use") - check.add_argument("--provider", default=None, help="provider name (default: all recorded)") - check.add_argument("--use", default="column_role_labeling", choices=ALLOWED_USES) - check.add_argument("--ledger", default=str(LEDGER_PATH)) - args = parser.parse_args(argv) - - ledger = load_ledger(args.ledger) - providers = ([args.provider] if args.provider - else [e.get("provider", "") for e in ledger.get("providers", [])]) - if not providers: - print("FAIL: compliance ledger is empty — record a terms check first", file=sys.stderr) - return 1 - failed = False - for provider in providers: - problems = check_entry(_entry_for(ledger, provider), provider, args.use) - if problems: - failed = True - for problem in problems: - print(f"FAIL: {problem}", file=sys.stderr) - else: - print(f"OK: {provider} approved for {args.use}") - return 1 if failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/teacher/compliance_ledger.json b/training/teacher/compliance_ledger.json deleted file mode 100644 index ba5ebb32..00000000 --- a/training/teacher/compliance_ledger.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "ledger_version": "1", - "providers": [ - { - "provider": "stub", - "terms_url": "training/teacher/clients.py#StubTeacherClient", - "terms_snapshot_id": "stub-local-0", - "date_checked": "2026-07-04T00:00:00+00:00", - "reviewer": "JWD", - "allowed_uses": [ - "realism_direction", - "column_role_labeling", - "context_paraphrase", - "ambiguity_adjudication", - "rationale_templates", - "red_teaming" - ], - "training_use_allowed": true, - "redistribution_allowed": true, - "status": "approved", - "notes": "Deterministic in-repo stub; no external service, no terms to verify. Real providers must be added via training.teacher.compliance.record_check after a manual terms review." - } - ] -} diff --git a/training/teacher/prompts/ambiguity.txt b/training/teacher/prompts/ambiguity.txt deleted file mode 100644 index 517c9253..00000000 --- a/training/teacher/prompts/ambiguity.txt +++ /dev/null @@ -1,9 +0,0 @@ -You adjudicate whether a corrupted value can be safely repaired to exactly one -candidate. If two or more candidates are plausible, the verdict is "ambiguous". -If no candidate is plausible, it is "unrepairable". Only when exactly one -candidate is clearly correct is it "resolvable". -Respond ONLY with a JSON array of AmbiguityJudgment objects: -[{"raw_value": str, "candidates": [str], "verdict": "ambiguous|resolvable|unrepairable", "preferred": str|null, "rationale": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/column_role.txt b/training/teacher/prompts/column_role.txt deleted file mode 100644 index 440ea9e3..00000000 --- a/training/teacher/prompts/column_role.txt +++ /dev/null @@ -1,10 +0,0 @@ -You label the semantic type of tabular columns. Values are MASKED (digits -replaced by 9, email locals hidden); judge shape, not content. Semantic types: -email, phone, url, country, currency_amount, quantity_with_unit, person_name, -address, city, postal_code, national_id, category_code, free_text, -boolean_like, date_like, identifier, unknown. -Respond ONLY with a JSON array of ColumnRoleLabel objects: -[{"column_name": str, "masked_samples": [str], "semantic_type": str, "confidence": float, "rationale": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/paraphrase.txt b/training/teacher/prompts/paraphrase.txt deleted file mode 100644 index 822baf4d..00000000 --- a/training/teacher/prompts/paraphrase.txt +++ /dev/null @@ -1,8 +0,0 @@ -You paraphrase data-cleaning context sentences without changing their intent -or slot values. Produce natural paraphrases, including Indian-English and -Hinglish variants when language says so. Never invent new constraints. -Respond ONLY with a JSON array of ContextParaphraseBatch objects: -[{"intent": str, "canonical_sentence": str, "paraphrases": [str], "language": str, "author": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/rationale.txt b/training/teacher/prompts/rationale.txt deleted file mode 100644 index 4296e51a..00000000 --- a/training/teacher/prompts/rationale.txt +++ /dev/null @@ -1,7 +0,0 @@ -You write short, factual rationale TEMPLATES (with {placeholders}) explaining -data repairs to a human reviewer. No marketing language; one sentence each. -Respond ONLY with a JSON array of RationaleTemplateBatch objects: -[{"issue_type": str, "templates": [str]}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/realism.txt b/training/teacher/prompts/realism.txt deleted file mode 100644 index 0649487a..00000000 --- a/training/teacher/prompts/realism.txt +++ /dev/null @@ -1,8 +0,0 @@ -You are tuning a data-corruption engine used to train a small data-cleaning model. -For each corruptor described below, recommend a realistic corruption share (0-1) -and parameter tweaks matching real-world dirty data frequency. -Respond ONLY with a JSON array of RealismConfig objects: -[{"corruptor_name": str, "recommended_share": float, "recommended_params": {}, "rationale": str}] - -Items: -{{ITEMS}} diff --git a/training/teacher/prompts/redteam.txt b/training/teacher/prompts/redteam.txt deleted file mode 100644 index 15ffd3ad..00000000 --- a/training/teacher/prompts/redteam.txt +++ /dev/null @@ -1,8 +0,0 @@ -You red-team a data-cleaning benchmark. Given a target metric, propose -adversarial cases (masked, synthetic values only) that a naive cleaner would -get wrong: dangerous near-merges, protected-column bait, dayfirst traps. -Respond ONLY with a JSON array of RedTeamCaseBatch objects: -[{"target_metric": str, "cases": [{"column": str, "raw_value": str, "why_hard": str}]}] - -Items: -{{ITEMS}} diff --git a/training/teacher/review.py b/training/teacher/review.py deleted file mode 100644 index b68543f4..00000000 --- a/training/teacher/review.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Human-review workflow for teacher outputs. - -Policy (enforced, not advisory): - -- at least **5%** of every task batch is sampled for human review; -- if teacher-human disagreement exceeds **3%**, the whole batch requires - full review before any of it may be used; -- release-gating eval labels must be **100% human-verified** — an unreviewed - teacher label can train a model but can never gate a release. -""" - -from __future__ import annotations - -import random -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from ..common import utc_now_iso, write_json - -SAMPLE_RATE = 0.05 -DISAGREEMENT_THRESHOLD = 0.03 - - -@dataclass(frozen=True) -class ReviewRecord: - item_id: str - reviewer: str - agrees: bool - reviewed_at: str - disagreement_reason: str = "" - corrected_payload: dict[str, Any] | None = None - - -@dataclass -class ReviewBatch: - """Review state for one teacher task batch.""" - - task_name: str - items: list[dict[str, Any]] - sample_ids: list[str] = field(default_factory=list) - records: list[ReviewRecord] = field(default_factory=list) - - @staticmethod - def item_id(index: int) -> str: - return f"item_{index:05d}" - - def sample_for_review(self, *, rate: float = SAMPLE_RATE, seed: int = 0) -> list[str]: - """Deterministically sample at least ``rate`` of items (min 1).""" - if not self.items: - self.sample_ids = [] - return [] - rng = random.Random(f"freshdata-review:{self.task_name}:{seed}") - n = max(1, int(len(self.items) * rate + 0.999999)) - picks = sorted(rng.sample(range(len(self.items)), min(n, len(self.items)))) - self.sample_ids = [self.item_id(i) for i in picks] - return self.sample_ids - - def record_review( - self, - item_id: str, - *, - reviewer: str, - agrees: bool, - disagreement_reason: str = "", - corrected_payload: dict[str, Any] | None = None, - ) -> ReviewRecord: - if not reviewer.strip(): - raise ValueError("reviewer id/initials are required") - if not agrees and not disagreement_reason.strip(): - raise ValueError("a disagreement must record its reason") - record = ReviewRecord( - item_id=item_id, - reviewer=reviewer.strip(), - agrees=agrees, - reviewed_at=utc_now_iso(), - disagreement_reason=disagreement_reason.strip(), - corrected_payload=corrected_payload, - ) - self.records.append(record) - return record - - def disagreement_rate(self) -> float: - if not self.records: - return 0.0 - return sum(1 for r in self.records if not r.agrees) / len(self.records) - - def requires_full_review(self) -> bool: - return self.disagreement_rate() > DISAGREEMENT_THRESHOLD - - def reviewed_ids(self) -> set[str]: - return {r.item_id for r in self.records} - - def usable_for_release_gating(self) -> bool: - """True only when every item in the batch has a human review record.""" - return self.reviewed_ids() >= {self.item_id(i) for i in range(len(self.items))} - - def summary(self) -> dict[str, Any]: - return { - "task_name": self.task_name, - "n_items": len(self.items), - "n_sampled": len(self.sample_ids), - "n_reviewed": len(self.records), - "sample_rate_required": SAMPLE_RATE, - "disagreement_rate": round(self.disagreement_rate(), 6), - "disagreement_threshold": DISAGREEMENT_THRESHOLD, - "requires_full_review": self.requires_full_review(), - "usable_for_release_gating": self.usable_for_release_gating(), - "reviewers": sorted({r.reviewer for r in self.records}), - "disagreements": [ - {"item_id": r.item_id, "reviewer": r.reviewer, "reason": r.disagreement_reason} - for r in self.records if not r.agrees - ], - } - - def export_summary(self, path: Path | str) -> Path: - return write_json(path, self.summary()) diff --git a/training/teacher/schemas.py b/training/teacher/schemas.py deleted file mode 100644 index ead3a44b..00000000 --- a/training/teacher/schemas.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Strict JSON schemas for teacher outputs. - -Every teacher task declares one of these schemas; a response that does not -validate is discarded (the pipeline degrades to corruptor/hook data rather -than accepting free-form teacher text). Validation is strict: required keys, -type checks, and **no unknown keys**. -""" - -from __future__ import annotations - -from typing import Any - -SCHEMA_VERSION = "1" - - -class SchemaError(ValueError): - """A teacher response failed strict schema validation.""" - - -#: schema name -> {field: (type, required)} -_SCHEMAS: dict[str, dict[str, tuple[type | tuple[type, ...], bool]]] = { - "RealismConfig": { - "corruptor_name": (str, True), - "recommended_share": (float, True), - "recommended_params": (dict, False), - "rationale": (str, True), - }, - "ColumnRoleLabel": { - "column_name": (str, True), - "masked_samples": (list, True), - "semantic_type": (str, True), - "confidence": (float, True), - "rationale": (str, False), - }, - "ContextParaphraseBatch": { - "intent": (str, True), - "canonical_sentence": (str, True), - "paraphrases": (list, True), - "language": (str, True), - "author": (str, True), - }, - "AmbiguityJudgment": { - "raw_value": (str, True), - "candidates": (list, True), - "verdict": (str, True), # "ambiguous" | "resolvable" | "unrepairable" - "preferred": ((str, type(None)), False), - "rationale": (str, True), - }, - "RationaleTemplateBatch": { - "issue_type": (str, True), - "templates": (list, True), - }, - "RedTeamCaseBatch": { - "target_metric": (str, True), - "cases": (list, True), - }, -} - -_ALLOWED_VERDICTS = frozenset({"ambiguous", "resolvable", "unrepairable"}) - - -def schema_names() -> tuple[str, ...]: - return tuple(sorted(_SCHEMAS)) - - -def validate_payload(schema: str, payload: Any) -> dict[str, Any]: - """Validate one teacher output payload; return it or raise SchemaError.""" - spec = _SCHEMAS.get(schema) - if spec is None: - raise SchemaError(f"unknown teacher schema {schema!r}; known: {schema_names()}") - if not isinstance(payload, dict): - raise SchemaError(f"{schema}: payload must be an object, got {type(payload).__name__}") - unknown = set(payload) - set(spec) - if unknown: - raise SchemaError(f"{schema}: unknown keys {sorted(unknown)}") - for field, (types, required) in spec.items(): - if field not in payload: - if required: - raise SchemaError(f"{schema}: missing required field {field!r}") - continue - value = payload[field] - if isinstance(types, type) and types is float and isinstance(value, int): - value = float(value) - payload[field] = value - if not isinstance(value, types): - raise SchemaError( - f"{schema}: field {field!r} must be {types}, got {type(value).__name__}" - ) - if schema == "AmbiguityJudgment" and payload["verdict"] not in _ALLOWED_VERDICTS: - raise SchemaError(f"AmbiguityJudgment: invalid verdict {payload['verdict']!r}") - if schema == "ColumnRoleLabel" and not (0.0 <= float(payload["confidence"]) <= 1.0): - raise SchemaError("ColumnRoleLabel: confidence must be in [0, 1]") - return payload - - -def validate_batch(schema: str, payloads: list[Any]) -> list[dict[str, Any]]: - return [validate_payload(schema, p) for p in payloads] diff --git a/training/teacher/tasks.py b/training/teacher/tasks.py deleted file mode 100644 index 09070493..00000000 --- a/training/teacher/tasks.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Teacher task harness: batched, masked, cached, compliance-gated. - -Guarantees enforced here: - -- **compliance first** — every run checks the provider ledger and aborts on - anything but an explicit approval; -- **no full rows** — tasks accept only column names plus a bounded list of - *masked* sample values, never DataFrame rows; -- **PII masking** — email locals, phone digits, and long identifiers are - masked before any prompt is built (:func:`mask_pii`); -- **strict schemas** — invalid responses are dropped, not repaired; -- **cache everything** — prompt + response + provenance land in the audit - cache before payloads are returned; -- **degrade safely** — a failed teacher call returns an empty batch so the - pipeline can fall back to corruptor/hook data. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from ..common import sha256_json, sha256_text -from . import compliance -from .cache import CacheKey, TeacherCache -from .clients import TeacherClient, TeacherUnavailable, default_client -from .schemas import SCHEMA_VERSION, SchemaError, validate_batch - -PROMPTS_DIR = Path(__file__).resolve().parent / "prompts" - -_EMAIL_RE = re.compile(r"([^@\s])[^@\s]*(@[^@\s]+)") -_DIGIT_RE = re.compile(r"\d") -# Opaque mixed alphanumeric identifiers (post digit-masking they still leak -# letter structure, so hash them). Pure digit runs keep their masked shape. -_LONG_TOKEN_RE = re.compile(r"(?=[A-Za-z0-9]*[A-Za-z])(?=[A-Za-z0-9]*9)[A-Za-z0-9]{16,}") - - -def mask_pii(value: object) -> str: - """Mask PII-shaped content while keeping the *shape* of the value. - - ``asha.voskette@example.com`` -> ``a***@example.com``; digits become - ``9``; long opaque tokens are hashed to a stable placeholder. - """ - text = str(value) - text = _EMAIL_RE.sub(lambda m: f"{m.group(1)}***{m.group(2)}", text) - text = _DIGIT_RE.sub("9", text) - return _LONG_TOKEN_RE.sub(lambda m: f"tok_{sha256_text(m.group(0))[:8]}", text) - - -@dataclass(frozen=True) -class TeacherTaskSpec: - """One allowed teacher task: purpose, prompt template, output schema.""" - - name: str - intended_use: str - prompt_file: str - schema: str - max_batch: int = 32 - - -TASKS: dict[str, TeacherTaskSpec] = { - spec.name: spec - for spec in ( - TeacherTaskSpec( - "realism_direction", "realism_direction", "realism.txt", "RealismConfig"), - TeacherTaskSpec( - "column_role_labeling", "column_role_labeling", - "column_role.txt", "ColumnRoleLabel"), - TeacherTaskSpec( - "context_paraphrase", "context_paraphrase", - "paraphrase.txt", "ContextParaphraseBatch"), - TeacherTaskSpec( - "ambiguity_adjudication", "ambiguity_adjudication", - "ambiguity.txt", "AmbiguityJudgment"), - TeacherTaskSpec( - "rationale_templates", "rationale_templates", - "rationale.txt", "RationaleTemplateBatch"), - TeacherTaskSpec( - "red_teaming", "red_teaming", "redteam.txt", "RedTeamCaseBatch"), - ) -} - - -def _assert_no_full_rows(items: Sequence[dict[str, Any]]) -> None: - """Refuse batches that smuggle whole records to the teacher.""" - for item in items: - if len(item) > 6: - raise ValueError( - "teacher task items must be small (column + masked samples), " - f"got {len(item)} fields — this looks like a full row" - ) - samples = item.get("masked_samples", []) - if isinstance(samples, list) and len(samples) > 10: - raise ValueError("at most 10 masked sample values may be sent per item") - - -def build_prompt(spec: TeacherTaskSpec, items: Sequence[dict[str, Any]]) -> tuple[str, str]: - """Return ``(prompt_text, template_sha)`` for one batch.""" - template = (PROMPTS_DIR / spec.prompt_file).read_text(encoding="utf-8") - payload = json.dumps(list(items), indent=2, sort_keys=True, ensure_ascii=False) - return template.replace("{{ITEMS}}", payload), sha256_text(template) - - -def run_task( - task_name: str, - items: Sequence[dict[str, Any]], - *, - client: TeacherClient | None = None, - cache: TeacherCache | None = None, - ledger_path: Path | str | None = None, -) -> list[dict[str, Any]]: - """Run one teacher task batch and return validated payloads. - - Empty list means "teacher unavailable or response invalid" — callers must - treat that as *no teacher data*, never as an error, so every consumer has - an offline fallback path. - """ - spec = TASKS.get(task_name) - if spec is None: - raise KeyError(f"unknown teacher task {task_name!r}; known: {sorted(TASKS)}") - _assert_no_full_rows(items) - - client = client if client is not None else default_client() - cache = cache if cache is not None else TeacherCache() - ledger_kwargs = {"ledger_path": ledger_path} if ledger_path is not None else {} - entry = compliance.require_approved(client.provider, spec.intended_use, **ledger_kwargs) - - payloads: list[dict[str, Any]] = [] - for start in range(0, len(items), spec.max_batch): - batch = list(items[start:start + spec.max_batch]) - prompt, template_sha = build_prompt(spec, batch) - key = CacheKey( - provider=client.provider, - model=client.model, - prompt_template_sha256=template_sha, - input_sha256=sha256_json(batch), - schema_version=f"{spec.schema}:{SCHEMA_VERSION}", - ) - cached = cache.get(key) - if cached is not None: - payloads.extend(cached.get("payloads", [])) - continue - try: - response = client.complete(prompt, schema=spec.schema) - parsed = json.loads(response) - if not isinstance(parsed, list): - raise SchemaError(f"{spec.schema}: response must be a JSON array") - validated = validate_batch(spec.schema, parsed) - except (TeacherUnavailable, SchemaError, json.JSONDecodeError): - # Degrade: no teacher data for this batch, pipeline continues. - continue - cache.put( - key, - prompt=prompt, - response=response, - payloads=validated, - terms_snapshot_id=str(entry.get("terms_snapshot_id", "")), - ) - payloads.extend(validated) - return payloads - - -def column_role_items( - columns: Sequence[str], samples: dict[str, Sequence[object]], *, max_samples: int = 8 -) -> list[dict[str, Any]]: - """Build masked column-role items from column names + sample values.""" - return [ - { - "column_name": str(col), - "masked_samples": [mask_pii(v) for v in list(samples.get(col, []))[:max_samples]], - } - for col in columns - ] From 39dc54550f4dc771225a6a5e12584a6ad759eff7 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:01:01 +0530 Subject: [PATCH 04/11] ci: delete workflow jobs that run deleted code - Delete plugin-contract.yml. It ran a missing tests/test_plugins.py and the deleted example plugins. - ci.yml: delete the truthbench and freshcore-native jobs. - release.yml: delete the CleanBench and TruthBench release-gate steps. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 62 --------------------------- .github/workflows/plugin-contract.yml | 49 --------------------- .github/workflows/release.yml | 6 --- 3 files changed, 117 deletions(-) delete mode 100644 .github/workflows/plugin-contract.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6c4b47c..3794e597 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,33 +68,6 @@ jobs: pip install "numpy<2.5" mypy src/freshdata - truthbench: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" - - name: TruthBench PR ratchet - # PRs must not regress any gate that passes in the committed - # baseline.json; the known-red gates are release blockers and are - # enforced absolutely by the release workflow (make truthbench-release), - # so they are reported here as KNOWN-RED without failing every - # unrelated PR. Infrastructure errors and partial runs still fail. - run: make truthbench-pr - - name: Upload run artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: truthbench-results - path: benchmarks/truthbench/results/ - test-matrix: runs-on: ubuntu-latest timeout-minutes: 30 @@ -223,41 +196,6 @@ jobs: python -m build twine check dist/* - freshcore-native: - # Builds the optional FreshCore Rust extension (crates/freshcore) and runs - # the FreshCore tests against it. No other job installs the extension, so - # the native parity tests skip everywhere else. - runs-on: ubuntu-latest - timeout-minutes: 10 - env: - RUSTUP_TOOLCHAIN: "1.98.1" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install Rust toolchain - run: rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal - - name: Install - # maturin develop installs into a virtualenv, so create one in the - # workspace and put it first on PATH for the remaining steps. - run: | - python -m venv .venv - echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV" - echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - .venv/bin/python -m pip install --upgrade pip - .venv/bin/pip install -c constraints/ci.txt -e ".[dev,freshcore]" - - name: Rust unit tests - run: cargo test --locked --manifest-path crates/freshcore/Cargo.toml - - name: Build the extension - run: maturin develop --manifest-path crates/freshcore/Cargo.toml --features extension-module - - name: FreshCore tests - # The coverage gate in addopts applies to the full suite, not this subset. - run: | - python -c "import freshdata_freshcore" - pytest tests/test_execution -k freshcore -o addopts="--strict-markers" -q - coverage-badge: # Publish a self-hosted shields endpoint badge to the `badges` branch # (no third-party account needed). Runs only on main. diff --git a/.github/workflows/plugin-contract.yml b/.github/workflows/plugin-contract.yml deleted file mode 100644 index 2dba406d..00000000 --- a/.github/workflows/plugin-contract.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Plugin contract tests - -# PR-CI-weight: the plugin registry, its safety guarantees (protected-column -# safety, risk ceiling, network-off-by-default, failure isolation), the -# fd.testing contract helpers, and the shipped examples/plugins/* stay valid. -on: - # pull_request: disabled in favor of ci-verification-gates.yml - # push: disabled in favor of ci-verification-gates.yml - workflow_dispatch: - -permissions: - contents: read - -jobs: - plugin-contract: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - cache: pip - - name: Install - run: | - python -m pip install -U pip - pip install -e ".[dev]" - - name: Plugin system + contract tests - run: pytest tests/test_plugins.py -q --no-cov - - name: Example plugins are importable and contract-valid - run: | - python - <<'EOF' - import sys - import freshdata as fd - - sys.path[:0] = [ - "examples/plugins/custom_expert", - "examples/plugins/custom_backend", - "examples/plugins/custom_validator", - ] - from acronym_expert import AcronymExpert - from keyword_backend import KeywordBackend - from min_rows_validator import MinRowsValidator - - fd.testing.expert_contract(AcronymExpert()) - fd.testing.semantic_backend_contract(KeywordBackend()) - fd.testing.validator_contract(MinRowsValidator()) - print("all example plugins pass their contracts") - EOF diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7592a07d..a0ec9be2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -242,12 +242,6 @@ jobs: run: mkdocs build --strict - name: Validation Gauntlet run: python -m benchmarks.gauntlet run --rows 300 --check - - name: CleanBench release gates - run: python -m benchmarks.cleanbench --tracks T1,T2,T3,T4,T5 --check-gates - - name: TruthBench release verification - # Fail-closed: a partial run, missing backend, or any absolute gate - # failure blocks the release before an artifact is built. - run: make truthbench-release # --------------------------------------------------------------------------- # 3. Build sdist + wheel exactly once; checksum them; attest provenance. From 5de24f5c5deabcd144ad0c48eae80ade633f3eb0 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:07:42 +0530 Subject: [PATCH 05/11] build: install CI from uv.lock and delete constraints/ci.txt - Replace each `pip install -c constraints/ci.txt -e ".[...]"` with `uv sync --frozen --inexact` into the setup-python interpreter. Extras match what pip installed: ml and bench are not defined in pyproject.toml, so pip skipped them. - The lockfile job now only runs `uv lock --check`. Delete the Makefile constraints target, which only refreshed the deleted file. - Relock uv.lock with uv 0.12.10. The committed lock was stale: it resolved numpy 2.5.1 against the numpy<2.4.0 pin and lacked the migrate extra. - constraints/ci.txt held only exact versions from that stale lock. No pin existed only there, so pyproject.toml is unchanged. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 25 +- .github/workflows/perf-regression.yml | 3 +- .github/workflows/performance-large.yml | 3 +- .github/workflows/public-benchmark.yml | 3 +- Makefile | 10 +- constraints/ci.txt | 1535 ----------------------- uv.lock | 374 +++--- 7 files changed, 206 insertions(+), 1747 deletions(-) delete mode 100644 constraints/ci.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3794e597..0ecb6d95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ concurrency: permissions: contents: read -# Dependency pinning: the gating jobs install with `-c constraints/ci.txt` -# (exported from uv.lock), so an upstream release cannot turn CI red overnight. +# Dependency pinning: the gating jobs install with `uv sync --frozen` from +# uv.lock, so an upstream release cannot turn CI red overnight. # test-matrix deliberately floats to the latest compatible releases: it is the -# early warning for upstream breakage. Regenerate with `make constraints`. +# early warning for upstream breakage. Regenerate with `uv lock`. jobs: lockfile: @@ -34,15 +34,10 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - name: uv.lock and constraints/ci.txt are up to date + - name: uv.lock is up to date run: | python -m pip install "uv==0.12.10" uv lock --check - uv export --frozen --no-hashes --all-extras --no-emit-project --quiet -o constraints/ci.txt - if ! git diff --exit-code -- constraints/ci.txt; then - echo "::error::constraints/ci.txt is stale; run 'make constraints' and commit the result" - exit 1 - fi quality-fast: runs-on: ubuntu-latest @@ -56,7 +51,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Lint run: ruff check . - name: Test (required fast lane) @@ -117,7 +113,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Nightly online/large drift checks # --no-cov: this lane deliberately runs ~20 tests, so the repo-wide # --cov-fail-under=93 inherited from addopts can never be met here. @@ -163,7 +160,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Assert the large/online lane still collects tests run: | N=$(pytest -m "online or large or tier1" --collect-only -q -o addopts="" \ @@ -214,7 +212,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Compute coverage and write badge JSON run: | pytest -m "not online and not large and not tier1" \ diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml index d95ec659..a28eb19a 100644 --- a/.github/workflows/perf-regression.yml +++ b/.github/workflows/perf-regression.yml @@ -50,7 +50,8 @@ jobs: - name: Install run: | python -m pip install -U pip - pip install -c constraints/ci.txt -e ".[bench]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact - name: Resolve base commit id: base env: diff --git a/.github/workflows/performance-large.yml b/.github/workflows/performance-large.yml index f87be8b4..35077b5b 100644 --- a/.github/workflows/performance-large.yml +++ b/.github/workflows/performance-large.yml @@ -23,7 +23,8 @@ jobs: - name: Install benchmark dependencies run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[dev,bench,ml]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact --extra dev - name: Run large performance matrix id: benchmark diff --git a/.github/workflows/public-benchmark.yml b/.github/workflows/public-benchmark.yml index fdbc1cf5..bb736a0e 100644 --- a/.github/workflows/public-benchmark.yml +++ b/.github/workflows/public-benchmark.yml @@ -34,7 +34,8 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -c constraints/ci.txt -e ".[bench]" + python -m pip install "uv==0.12.10" + UV_PROJECT_ENVIRONMENT="$pythonLocation" uv sync --frozen --inexact - name: Run public benchmark env: SCALE: ${{ inputs.scale }} diff --git a/Makefile b/Makefile index 61dfea26..e5786d27 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,7 @@ PY ?= python .PHONY: help benchmark benchmark-ci benchmark-report benchmark-fixtures benchmark-test \ cleanbench-full truthbench-release truthbench-pr automation-pr performance-ci \ - performance-baseline performance-profile performance-report coverage-report \ - constraints + performance-baseline performance-profile performance-report coverage-report help: @echo "Targets:" @@ -23,13 +22,6 @@ help: @echo " performance-profile Profile one 100k-row performance case" @echo " performance-report Analyze and render compact performance evidence" @echo " coverage-report Show per-module coverage for the fast test lane" - @echo " constraints Refresh uv.lock and the pinned constraints/ci.txt CI installs with" - -# Refresh the lockfile and the pip constraints the gating CI jobs install with. -# The CI `lockfile` job fails when either is stale; commit both after running. -constraints: - uv lock - uv export --frozen --no-hashes --all-extras --no-emit-project --quiet -o constraints/ci.txt # Run the same test scope as the required PR lane and print uncovered lines for # every measured module so contributors can identify useful testing targets. diff --git a/constraints/ci.txt b/constraints/ci.txt deleted file mode 100644 index b0157d23..00000000 --- a/constraints/ci.txt +++ /dev/null @@ -1,1535 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv export --frozen --no-hashes --all-extras --no-emit-project -o constraints/ci.txt -a2wsgi==1.10.10 - # via apache-airflow-core -agate==1.9.1 - # via - # dbt-adapters - # dbt-common - # dbt-core -aiosmtplib==5.1.2 ; python_full_version >= '3.10' - # via apache-airflow-providers-smtp -aiosqlite==0.21.0 - # via apache-airflow-core -alembic==1.16.5 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dagster -alembic==1.18.5 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # dagster -annotated-doc==0.0.4 - # via - # fastapi - # typer -annotated-types==0.7.0 - # via pydantic -antlr4-python3-runtime==4.13.2 - # via dagster -anyio==4.12.1 ; python_full_version < '3.10' - # via - # httpx - # starlette - # watchfiles -anyio==4.14.1 ; python_full_version >= '3.10' - # via - # httpx - # starlette - # watchfiles -anywidget==0.9.21 ; python_full_version < '3.10' - # via freshdata-cleaner -anywidget==0.11.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -apache-airflow==3.0.6 ; python_full_version < '3.10' - # via - # apache-airflow-providers-common-compat - # apache-airflow-providers-common-io - # apache-airflow-providers-common-sql - # apache-airflow-providers-smtp - # apache-airflow-providers-standard - # freshdata-cleaner -apache-airflow==3.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow-providers-common-compat - # apache-airflow-providers-common-io - # apache-airflow-providers-common-sql - # apache-airflow-providers-smtp - # apache-airflow-providers-standard - # freshdata-cleaner -apache-airflow-core==3.0.6 ; python_full_version < '3.10' - # via - # apache-airflow - # apache-airflow-task-sdk -apache-airflow-core==3.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow - # apache-airflow-task-sdk -apache-airflow-providers-common-compat==1.7.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # apache-airflow-providers-smtp -apache-airflow-providers-common-compat==1.16.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-providers-common-io - # apache-airflow-providers-common-sql - # apache-airflow-providers-smtp - # apache-airflow-providers-standard -apache-airflow-providers-common-io==1.6.0 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-common-io==1.8.0 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-providers-common-sql==1.27.2 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-common-sql==2.0.2 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-providers-smtp==2.1.0 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-smtp==3.0.2 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-providers-standard==1.3.0 ; python_full_version < '3.10' - # via apache-airflow-core -apache-airflow-providers-standard==1.16.0 ; python_full_version >= '3.10' - # via apache-airflow-core -apache-airflow-task-sdk==1.0.6 ; python_full_version < '3.10' - # via - # apache-airflow - # apache-airflow-core -apache-airflow-task-sdk==1.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow - # apache-airflow-core -argcomplete==3.7.0 - # via apache-airflow-core -arrow==1.4.0 ; python_full_version >= '3.10' - # via isoduration -asgiref==3.11.1 - # via - # apache-airflow-core - # apache-airflow-providers-common-compat - # apache-airflow-task-sdk -asttokens==3.0.1 - # via stack-data -attrs==26.1.0 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # jsonschema - # outcome - # referencing - # svcs -babel==2.18.0 - # via - # agate - # apache-airflow-task-sdk - # great-tables - # mkdocs-material -backports-strenum==1.3.1 ; python_full_version < '3.11' - # via cadwyn -backports-tarfile==1.2.0 ; python_full_version < '3.12' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via jaraco-context -backrefs==6.2 ; python_full_version < '3.10' - # via mkdocs-material -backrefs==7.0 ; python_full_version >= '3.10' - # via mkdocs-material -blinker==1.9.0 ; python_full_version < '3.10' - # via flask -blis==1.2.0 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # thinc -blis==1.3.3 ; python_full_version >= '3.10' - # via thinc -build==1.4.4 ; python_full_version < '3.10' - # via freshdata-cleaner -build==1.6.1 ; python_full_version >= '3.10' - # via freshdata-cleaner -cachetools==7.1.4 ; python_full_version >= '3.10' - # via apache-airflow-core -cadwyn==6.0.3 ; python_full_version < '3.10' - # via apache-airflow-core -cadwyn==7.0.0 ; python_full_version >= '3.10' - # via apache-airflow-core -catalogue==2.0.10 - # via - # spacy - # srsly - # thinc -certifi==2026.6.17 - # via - # httpcore - # httpx - # requests - # sentry-sdk -cffi==2.0.0 ; python_full_version < '3.10' and platform_python_implementation != 'PyPy' - # via cryptography -cffi==2.1.0 ; python_full_version >= '3.10' - # via - # cryptography - # multimark -cfgv==3.4.0 ; python_full_version < '3.10' - # via pre-commit -cfgv==3.5.0 ; python_full_version >= '3.10' - # via pre-commit -charset-normalizer==3.4.9 - # via requests -cleanlab==2.6.5 ; python_full_version < '3.10' - # via freshdata-cleaner -cleanlab==2.9.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -click==8.1.8 ; python_full_version < '3.10' - # via - # dagster - # dbt-core - # dbt-semantic-interfaces - # flask - # mkdocs - # rich-toolkit - # typer - # uvicorn -click==8.4.2 ; python_full_version >= '3.10' - # via - # dagster - # dbt-core - # dbt-semantic-interfaces - # huggingface-hub - # mkdocs - # multimark - # nokap - # rich-toolkit - # uvicorn -cloudpathlib==0.24.0 - # via weasel -colorama==0.4.6 - # via - # build - # click - # colorlog - # dbt-common - # griffe - # ipython - # mkdocs - # mkdocs-material - # pytest - # tqdm - # typer - # uvicorn - # wasabi -coloredlogs==14.0 - # via - # dagster - # onnxruntime -colorlog==6.10.1 - # via - # apache-airflow-core - # apache-airflow-task-sdk -comm==0.2.3 - # via ipywidgets -commonmark==0.9.2 ; python_full_version < '3.10' - # via great-tables -confection==0.1.5 ; python_full_version < '3.10' - # via - # thinc - # weasel -confection==1.3.3 ; python_full_version >= '3.10' - # via - # spacy - # thinc - # weasel -coverage==7.10.7 ; python_full_version < '3.10' - # via pytest-cov -coverage==7.15.0 ; python_full_version >= '3.10' - # via pytest-cov -cron-descriptor==2.1.0 - # via apache-airflow-core -croniter==6.2.4 - # via apache-airflow-core -cryptography==44.0.3 ; python_full_version < '3.10' - # via - # apache-airflow-core - # presidio-anonymizer - # secretstorage -cryptography==46.0.7 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # presidio-anonymizer - # secretstorage -cymem==2.0.13 - # via - # preshed - # spacy - # thinc -daff==1.4.2 - # via dbt-core -dagster==1.12.1 ; python_full_version < '3.10' - # via freshdata-cleaner -dagster==1.13.13 ; python_full_version >= '3.10' - # via freshdata-cleaner -dagster-pipes==1.12.1 ; python_full_version < '3.10' - # via dagster -dagster-pipes==1.13.13 ; python_full_version >= '3.10' - # via dagster -dagster-shared==1.12.1 ; python_full_version < '3.10' - # via dagster -dagster-shared==1.13.13 ; python_full_version >= '3.10' - # via dagster -dbt-adapters==1.17.3 ; python_full_version < '3.10' - # via dbt-core -dbt-adapters==1.24.4 ; python_full_version >= '3.10' - # via dbt-core -dbt-common==1.34.2 ; python_full_version < '3.10' - # via - # dbt-adapters - # dbt-core -dbt-common==1.38.0 ; python_full_version >= '3.10' - # via - # dbt-adapters - # dbt-core -dbt-core==1.10.22 ; python_full_version < '3.10' - # via freshdata-cleaner -dbt-core==1.11.12 ; python_full_version >= '3.10' - # via freshdata-cleaner -dbt-extractor==0.6.0 - # via dbt-core -dbt-protos==1.0.541 - # via - # dbt-adapters - # dbt-common - # dbt-core -dbt-semantic-interfaces==0.9.0 - # via dbt-core -decorator==5.3.1 - # via ipython -deepdiff==8.6.2 - # via dbt-common -deprecated==1.3.1 - # via apache-airflow-core -dill==0.4.1 - # via apache-airflow-core -distlib==0.4.3 - # via virtualenv -dnspython==2.7.0 ; python_full_version < '3.10' - # via email-validator -dnspython==2.8.0 ; python_full_version >= '3.10' - # via email-validator -docstring-parser==0.18.0 - # via dagster -docutils==0.23 - # via readme-renderer -duckdb==1.4.5 ; python_full_version < '3.10' - # via freshdata-cleaner -duckdb==1.5.4 ; python_full_version >= '3.10' - # via freshdata-cleaner -email-validator==2.3.0 - # via - # fastapi - # pydantic -et-xmlfile==2.0.0 - # via openpyxl -eval-type-backport==0.4.0 ; python_full_version < '3.10' - # via apache-airflow-core -exceptiongroup==1.3.1 ; python_full_version < '3.11' - # via - # anyio - # ipython - # pytest -executing==2.2.1 - # via stack-data -faicons==0.2.2 - # via great-tables -fastapi==0.128.8 ; python_full_version < '3.10' - # via - # apache-airflow-core - # cadwyn -fastapi==0.136.3 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # cadwyn -fastapi-cli==0.0.22 ; python_full_version < '3.10' - # via fastapi -fastapi-cli==0.0.29 ; python_full_version >= '3.10' - # via fastapi -fastapi-cloud-cli==0.12.0 ; python_full_version < '3.10' - # via fastapi-cli -fastar==0.11.0 ; python_full_version < '3.10' - # via fastapi-cloud-cli -filelock==3.19.1 ; python_full_version < '3.10' - # via - # dagster - # huggingface-hub - # python-discovery - # tldextract - # virtualenv -filelock==3.29.7 ; python_full_version >= '3.10' - # via - # dagster - # huggingface-hub - # python-discovery - # tldextract - # virtualenv -flask==3.1.3 ; python_full_version < '3.10' - # via apache-airflow-core -flatbuffers==25.12.19 - # via onnxruntime -fsspec==2025.10.0 ; python_full_version < '3.10' - # via - # apache-airflow-task-sdk - # huggingface-hub - # universal-pathlib -fsspec==2026.6.0 ; python_full_version >= '3.10' - # via - # apache-airflow-task-sdk - # huggingface-hub - # universal-pathlib -ghp-import==2.1.0 - # via mkdocs -googleapis-common-protos==1.75.0 - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -great-tables==0.21.0 ; python_full_version < '3.10' - # via freshdata-cleaner -great-tables==0.22.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -greenback==1.3.0 ; python_full_version >= '3.10' - # via apache-airflow-task-sdk -greenlet==3.2.5 ; python_full_version < '3.10' - # via sqlalchemy -greenlet==3.5.3 ; python_full_version >= '3.10' - # via - # greenback - # sqlalchemy -griffe==1.14.0 ; python_full_version < '3.10' - # via mkdocstrings-python -griffelib==2.1.0 ; python_full_version >= '3.10' - # via mkdocstrings-python -grpcio==1.80.0 ; python_full_version < '3.10' - # via - # dagster - # grpcio-health-checking - # opentelemetry-exporter-otlp-proto-grpc -grpcio==1.82.1 ; python_full_version >= '3.10' - # via - # dagster - # grpcio-health-checking - # opentelemetry-exporter-otlp-proto-grpc -grpcio-health-checking==1.80.0 ; python_full_version < '3.10' - # via dagster -grpcio-health-checking==1.81.1 ; python_full_version >= '3.10' - # via dagster -gunicorn==23.0.0 ; python_full_version < '3.10' - # via apache-airflow-core -h11==0.16.0 - # via - # httpcore - # uvicorn -hf-xet==1.5.1 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' - # via huggingface-hub -htmltools==0.6.0 ; python_full_version < '3.10' - # via - # faicons - # great-tables -htmltools==0.7.0 ; python_full_version >= '3.10' - # via - # faicons - # great-tables -httpcore==1.0.9 - # via httpx -httptools==0.8.0 - # via uvicorn -httpx==0.28.1 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # fastapi - # fastapi-cloud-cli - # huggingface-hub - # weasel -huggingface-hub==1.8.0 ; python_full_version < '3.10' - # via tokenizers -huggingface-hub==1.23.0 ; python_full_version >= '3.10' - # via tokenizers -humanfriendly==10.0 - # via coloredlogs -id==1.6.1 - # via twine -identify==2.6.15 ; python_full_version < '3.10' - # via pre-commit -identify==2.6.19 ; python_full_version >= '3.10' - # via pre-commit -idna==3.18 - # via - # anyio - # email-validator - # httpx - # requests - # tldextract -importlib-metadata==8.7.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # build - # dbt-semantic-interfaces - # flask - # great-tables - # keyring - # markdown - # mkdocs - # mkdocs-get-deps - # mkdocstrings - # opentelemetry-api - # twine -importlib-metadata==8.9.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # build - # dbt-semantic-interfaces - # great-tables - # keyring -importlib-resources==6.5.2 ; python_full_version < '3.10' - # via great-tables -importlib-resources==7.1.0 ; python_full_version >= '3.10' - # via great-tables -iniconfig==2.1.0 ; python_full_version < '3.10' - # via pytest -iniconfig==2.3.0 ; python_full_version >= '3.10' - # via pytest -ipython==8.18.1 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # ipywidgets -ipython==8.39.0 ; python_full_version == '3.10.*' - # via - # freshdata-cleaner - # ipywidgets -ipython==9.15.0 ; python_full_version >= '3.11' - # via - # freshdata-cleaner - # ipywidgets -ipython-pygments-lexers==1.1.1 ; python_full_version >= '3.11' - # via ipython -ipywidgets==8.1.8 - # via anywidget -isodate==0.6.1 ; python_full_version < '3.10' - # via - # agate - # dbt-common -isodate==0.7.2 ; python_full_version >= '3.10' - # via - # agate - # dbt-common -isoduration==20.11.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -itables==2.8.1 - # via freshdata-cleaner -itsdangerous==2.2.0 - # via - # apache-airflow-core - # flask -jaraco-classes==3.4.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-context==6.1.1 ; python_full_version < '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-context==6.1.2 ; python_full_version >= '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-functools==4.4.0 ; python_full_version < '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jaraco-functools==4.5.0 ; python_full_version >= '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' - # via keyring -jedi==0.19.2 ; python_full_version < '3.10' - # via ipython -jedi==0.20.0 ; python_full_version >= '3.10' - # via ipython -jeepney==0.9.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux' - # via - # keyring - # secretstorage -jinja2==3.1.6 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # cadwyn - # dagster - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # fastapi - # flask - # mkdocs - # mkdocs-material - # mkdocstrings - # spacy -joblib==1.5.3 - # via scikit-learn -jsonschema==4.25.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # freshdata-cleaner -jsonschema==4.26.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # freshdata-cleaner -jsonschema-specifications==2025.9.1 - # via jsonschema -jupyterlab-widgets==3.0.16 - # via ipywidgets -kafka-python==3.0.8 - # via freshdata-cleaner -keyring==25.7.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x' - # via twine -langcodes==3.5.1 ; python_full_version < '3.10' - # via spacy -lazy-object-proxy==1.12.0 - # via apache-airflow-core -leather==0.4.1 - # via agate -libcst==1.8.6 - # via apache-airflow-core -librt==0.13.0 ; platform_python_implementation != 'PyPy' - # via mypy -linkify-it-py==2.0.3 ; python_full_version < '3.10' - # via apache-airflow-core -linkify-it-py==2.1.0 ; python_full_version >= '3.10' - # via apache-airflow-core -lockfile==0.12.2 - # via - # apache-airflow-core - # python-daemon -mako==1.3.12 - # via alembic -markdown==3.9 ; python_full_version < '3.10' - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown==3.10.2 ; python_full_version >= '3.10' - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown-it-py==3.0.0 ; python_full_version < '3.10' - # via rich -markdown-it-py==4.2.0 ; python_full_version >= '3.10' - # via rich -markupsafe==3.0.3 - # via - # flask - # jinja2 - # mako - # mkdocs - # mkdocs-autorefs - # mkdocstrings - # werkzeug -mashumaro==3.14 - # via - # dbt-adapters - # dbt-common - # dbt-core -matplotlib-inline==0.2.2 - # via ipython -maturin==1.14.1 - # via freshdata-cleaner -mdurl==0.1.2 - # via markdown-it-py -mergedeep==1.3.4 - # via - # mkdocs - # mkdocs-get-deps -methodtools==0.4.7 - # via - # apache-airflow-core - # apache-airflow-providers-common-sql - # apache-airflow-task-sdk -mkdocs==1.6.1 - # via - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings -mkdocs-autorefs==1.4.4 - # via - # mkdocstrings - # mkdocstrings-python -mkdocs-get-deps==0.2.2 - # via mkdocs -mkdocs-material==9.7.6 - # via freshdata-cleaner -mkdocs-material-extensions==1.3.1 - # via mkdocs-material -mkdocstrings==0.30.1 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # mkdocstrings-python -mkdocstrings==1.0.5 ; python_full_version >= '3.10' - # via - # freshdata-cleaner - # mkdocstrings-python -mkdocstrings-python==1.18.2 ; python_full_version < '3.10' - # via mkdocstrings -mkdocstrings-python==2.0.5 ; python_full_version >= '3.10' - # via mkdocstrings -more-itertools==10.8.0 - # via - # apache-airflow-providers-common-sql - # dbt-semantic-interfaces - # jaraco-classes - # jaraco-functools -mpmath==1.3.0 ; python_full_version < '3.11' - # via sympy -msgpack==1.1.2 ; python_full_version < '3.10' - # via mashumaro -msgpack==1.2.1 ; python_full_version >= '3.10' - # via mashumaro -msgspec==0.20.0 ; python_full_version < '3.10' - # via apache-airflow-task-sdk -msgspec==0.21.1 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -multimark==0.2.0 ; python_full_version >= '3.10' - # via great-tables -murmurhash==1.0.15 - # via - # preshed - # spacy - # thinc -mypy==1.19.1 - # via freshdata-cleaner -mypy-extensions==1.1.0 - # via mypy -narwhals==2.21.0 ; python_full_version < '3.10' - # via plotly -narwhals==2.23.0 ; python_full_version >= '3.10' - # via - # plotly - # scikit-learn -natsort==8.4.0 ; python_full_version >= '3.10' - # via apache-airflow-core -networkx==3.2.1 ; python_full_version < '3.10' - # via dbt-core -networkx==3.4.2 ; python_full_version == '3.10.*' - # via dbt-core -networkx==3.6.1 ; python_full_version >= '3.11' - # via dbt-core -nh3==0.3.6 - # via readme-renderer -nodeenv==1.10.0 - # via pre-commit -nokap==0.1.0 ; python_full_version >= '3.10' - # via great-tables -numpy==2.0.2 ; python_full_version < '3.10' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -numpy==2.2.6 ; python_full_version == '3.10.*' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -numpy==2.4.6 ; python_full_version == '3.11.*' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -numpy==2.5.1 ; python_full_version >= '3.12' - # via - # blis - # cleanlab - # freshdata-cleaner - # onnxruntime - # pandas - # scikit-learn - # scipy - # spacy - # thinc -onnxruntime==1.20.1 ; python_full_version < '3.10' - # via freshdata-cleaner -onnxruntime==1.24.3 ; python_full_version == '3.10.*' - # via freshdata-cleaner -onnxruntime==1.27.0 ; python_full_version >= '3.11' - # via freshdata-cleaner -openpyxl==3.1.5 - # via freshdata-cleaner -opentelemetry-api==1.41.1 ; python_full_version < '3.10' - # via - # apache-airflow-core - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-api==1.43.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.41.1 ; python_full_version < '3.10' - # via apache-airflow-core -opentelemetry-exporter-otlp==1.43.0 ; python_full_version >= '3.10' - # via apache-airflow-core -opentelemetry-exporter-otlp-proto-common==1.41.1 ; python_full_version < '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-common==1.43.0 ; python_full_version >= '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.41.1 ; python_full_version < '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-grpc==1.43.0 ; python_full_version >= '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.41.1 ; python_full_version < '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.43.0 ; python_full_version >= '3.10' - # via opentelemetry-exporter-otlp -opentelemetry-proto==1.41.1 ; python_full_version < '3.10' - # via - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-proto==1.43.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.41.1 ; python_full_version < '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.43.0 ; python_full_version >= '3.10' - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.62b1 ; python_full_version < '3.10' - # via opentelemetry-sdk -opentelemetry-semantic-conventions==0.64b0 ; python_full_version >= '3.10' - # via opentelemetry-sdk -orderly-set==5.5.0 - # via deepdiff -outcome==1.3.0.post0 ; python_full_version >= '3.10' - # via greenback -packaging==26.2 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # build - # dagster-shared - # dbt-core - # gunicorn - # htmltools - # huggingface-hub - # mkdocs - # onnxruntime - # plotly - # pytest - # spacy - # thinc - # twine - # weasel -paginate==0.5.7 - # via mkdocs-material -pandas==2.3.3 - # via - # cleanlab - # freshdata-cleaner -parsedatetime==2.6 - # via agate -parso==0.8.7 - # via jedi -pathlib-abc==0.5.2 - # via universal-pathlib -pathspec==0.12.1 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dbt-common - # dbt-core - # mkdocs - # mypy -pendulum==3.2.0 - # via - # apache-airflow-core - # apache-airflow-task-sdk -pexpect==4.9.0 ; (python_full_version < '3.10' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32') - # via ipython -phonenumbers==9.0.34 - # via presidio-analyzer -platformdirs==4.4.0 ; python_full_version < '3.10' - # via - # dagster-shared - # mkdocs-get-deps - # python-discovery - # virtualenv -platformdirs==4.10.0 ; python_full_version >= '3.10' - # via - # dagster-shared - # mkdocs-get-deps - # python-discovery - # virtualenv -plotly==6.9.0 - # via freshdata-cleaner -pluggy==1.6.0 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # pytest - # pytest-cov -polars==1.36.1 ; python_full_version < '3.10' - # via freshdata-cleaner -polars==1.42.1 ; python_full_version >= '3.10' - # via freshdata-cleaner -polars-runtime-32==1.36.1 ; python_full_version < '3.10' - # via polars -polars-runtime-32==1.42.1 ; python_full_version >= '3.10' - # via polars -pre-commit==4.3.0 ; python_full_version < '3.10' - # via freshdata-cleaner -pre-commit==4.6.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -preshed==3.0.13 - # via - # spacy - # thinc -presidio-analyzer==2.2.360 ; python_full_version < '3.10' - # via freshdata-cleaner -presidio-analyzer==2.2.362 ; python_full_version >= '3.10' - # via freshdata-cleaner -presidio-anonymizer==2.2.360 ; python_full_version < '3.10' - # via freshdata-cleaner -presidio-anonymizer==2.2.363 ; python_full_version >= '3.10' - # via freshdata-cleaner -prompt-toolkit==3.0.52 - # via ipython -protobuf==6.33.6 - # via - # dagster - # dbt-adapters - # dbt-common - # dbt-core - # dbt-protos - # googleapis-common-protos - # grpcio-health-checking - # onnxruntime - # opentelemetry-proto -psutil==7.2.2 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dagster - # freshdata-cleaner - # ipython -psygnal==0.14.2 ; python_full_version < '3.10' - # via anywidget -psygnal==0.15.1 ; python_full_version >= '3.10' - # via anywidget -ptyprocess==0.7.0 ; (python_full_version < '3.10' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32') - # via pexpect -pure-eval==0.2.3 - # via stack-data -py4j==0.10.9.9 - # via pyspark -pyarrow==21.0.0 ; python_full_version < '3.10' - # via freshdata-cleaner -pyarrow==25.0.0 ; python_full_version >= '3.10' - # via freshdata-cleaner -pycparser==2.23 ; python_full_version < '3.10' and implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' - # via cffi -pycparser==3.0 ; python_full_version >= '3.10' and implementation_name != 'PyPy' - # via cffi -pydantic==2.13.4 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # cadwyn - # confection - # dagster-shared - # dbt-core - # dbt-semantic-interfaces - # fastapi - # fastapi-cloud-cli - # presidio-analyzer - # pydantic-extra-types - # pydantic-settings - # retryhttp - # spacy - # thinc - # weasel -pydantic-core==2.46.4 - # via pydantic -pydantic-extra-types==2.11.1 - # via fastapi -pydantic-settings==2.11.0 ; python_full_version < '3.10' - # via fastapi -pydantic-settings==2.14.2 ; python_full_version >= '3.10' - # via fastapi -pyffx==0.3.0 - # via freshdata-cleaner -pygments==2.20.0 - # via - # apache-airflow-core - # ipython - # ipython-pygments-lexers - # mkdocs-material - # pytest - # readme-renderer - # rich -pygtrie==2.5.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -pyjwt==2.13.0 - # via apache-airflow-core -pymdown-extensions==10.21.3 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # mkdocs-material - # mkdocstrings -pymdown-extensions==11.0.1 ; python_full_version >= '3.10' - # via - # freshdata-cleaner - # mkdocs-material - # mkdocstrings -pyproject-hooks==1.2.0 - # via build -pyreadline3==3.5.6 ; sys_platform == 'win32' - # via humanfriendly -pyspark==4.0.3 ; python_full_version < '3.10' - # via freshdata-cleaner -pyspark==4.1.2 ; python_full_version >= '3.10' - # via freshdata-cleaner -pytest==8.4.2 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # pytest-cov -pytest==9.1.1 ; python_full_version >= '3.10' - # via - # freshdata-cleaner - # pytest-cov -pytest-cov==7.1.0 - # via freshdata-cleaner -python-daemon==3.1.2 - # via apache-airflow-core -python-dateutil==2.9.0.post0 - # via - # apache-airflow-core - # apache-airflow-task-sdk - # arrow - # croniter - # dbt-common - # dbt-semantic-interfaces - # ghp-import - # pandas - # pendulum -python-discovery==1.4.4 - # via virtualenv -python-dotenv==1.2.1 ; python_full_version < '3.10' - # via - # dagster - # pydantic-settings - # uvicorn -python-dotenv==1.2.2 ; python_full_version >= '3.10' - # via - # dagster - # pydantic-settings - # uvicorn -python-multipart==0.0.20 ; python_full_version < '3.10' - # via fastapi -python-multipart==0.0.32 ; python_full_version >= '3.10' - # via fastapi -python-slugify==8.0.4 - # via - # agate - # apache-airflow-core -pytimeparse==1.1.8 - # via agate -pytz==2026.2 - # via - # dagster - # dbt-adapters - # dbt-core - # pandas -pywin32==312 ; sys_platform == 'win32' - # via dagster -pywin32-ctypes==0.2.3 ; platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'win32' - # via keyring -pyyaml==6.0.3 - # via - # apache-airflow-core - # dagster-shared - # dbt-core - # dbt-semantic-interfaces - # freshdata-cleaner - # huggingface-hub - # libcst - # mkdocs - # mkdocs-get-deps - # pre-commit - # presidio-analyzer - # pymdown-extensions - # pyyaml-env-tag - # uvicorn -pyyaml-env-tag==1.1 - # via mkdocs -pyyaml-ft==8.0.0 ; python_full_version == '3.13.*' - # via libcst -readme-renderer==44.0 ; python_full_version < '3.10' - # via twine -readme-renderer==45.0 ; python_full_version >= '3.10' - # via twine -referencing==0.36.2 ; python_full_version < '3.10' - # via - # jsonschema - # jsonschema-specifications -referencing==0.37.0 ; python_full_version >= '3.10' - # via - # jsonschema - # jsonschema-specifications -regex==2026.1.15 ; python_full_version < '3.10' - # via presidio-analyzer -regex==2026.7.10 ; python_full_version >= '3.10' - # via presidio-analyzer -requests==2.32.5 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dagster - # dbt-common - # dbt-core - # mkdocs-material - # opentelemetry-exporter-otlp-proto-http - # requests-file - # requests-toolbelt - # snowplow-tracker - # spacy - # tldextract - # twine - # weasel -requests==2.34.2 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # dagster - # dbt-common - # dbt-core - # mkdocs-material - # opentelemetry-exporter-otlp-proto-http - # requests-file - # requests-toolbelt - # snowplow-tracker - # spacy - # tldextract - # twine -requests-file==3.0.1 - # via tldextract -requests-toolbelt==1.0.0 - # via twine -retryhttp==1.4.0 ; python_full_version < '3.10' - # via apache-airflow-task-sdk -rfc3986==2.0.0 - # via twine -rich==15.0.0 - # via - # apache-airflow-core - # dagster - # freshdata-cleaner - # rich-argparse - # rich-toolkit - # twine - # typer -rich-argparse==1.8.0 - # via apache-airflow-core -rich-toolkit==0.20.1 - # via - # fastapi-cli - # fastapi-cloud-cli -rignore==0.7.6 ; python_full_version < '3.10' - # via fastapi-cloud-cli -rpds-py==0.27.1 ; python_full_version < '3.10' - # via - # jsonschema - # referencing -rpds-py==0.30.0 ; python_full_version == '3.10.*' - # via - # jsonschema - # referencing -rpds-py==2026.6.3 ; python_full_version >= '3.11' - # via - # jsonschema - # referencing -ruff==0.15.21 - # via freshdata-cleaner -scikit-learn==1.6.1 ; python_full_version < '3.10' - # via - # cleanlab - # freshdata-cleaner -scikit-learn==1.7.2 ; python_full_version == '3.10.*' - # via - # cleanlab - # freshdata-cleaner -scikit-learn==1.9.0 ; python_full_version >= '3.11' - # via - # cleanlab - # freshdata-cleaner -scipy==1.13.1 ; python_full_version < '3.10' - # via scikit-learn -scipy==1.15.3 ; python_full_version == '3.10.*' - # via scikit-learn -scipy==1.17.1 ; python_full_version == '3.11.*' - # via scikit-learn -scipy==1.18.0 ; python_full_version >= '3.12' - # via scikit-learn -secretstorage==3.3.3 ; python_full_version < '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux' - # via keyring -secretstorage==3.5.0 ; python_full_version >= '3.10' and platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux' - # via keyring -sentry-sdk==2.64.0 ; python_full_version < '3.10' - # via fastapi-cloud-cli -setproctitle==1.3.7 - # via apache-airflow-core -setuptools==82.0.1 ; python_full_version < '3.10' - # via - # dagster - # spacy - # thinc -setuptools==83.0.0 ; python_full_version >= '3.10' - # via - # spacy - # thinc -shellingham==1.5.4 - # via typer -six==1.17.0 - # via - # dagster - # isodate - # pyffx - # python-dateutil - # wirerope -smart-open==7.5.0 ; python_full_version < '3.10' - # via weasel -smart-open==8.0.0 ; python_full_version >= '3.10' - # via weasel -sniffio==1.3.1 ; python_full_version >= '3.10' - # via greenback -snowplow-tracker==1.1.0 - # via dbt-core -spacy==3.8.7 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # presidio-analyzer -spacy==3.8.14 ; python_full_version >= '3.10' - # via presidio-analyzer -spacy-legacy==3.0.12 - # via spacy -spacy-loggers==1.0.5 - # via spacy -sqlalchemy==1.4.54 ; python_full_version < '3.10' - # via - # alembic - # apache-airflow-core - # dagster - # freshdata-cleaner - # sqlalchemy-jsonfield - # sqlalchemy-utils -sqlalchemy==2.0.51 ; python_full_version >= '3.10' - # via - # alembic - # apache-airflow-core - # dagster - # freshdata-cleaner -sqlalchemy-jsonfield==1.0.3 ; python_full_version < '3.10' - # via apache-airflow-core -sqlalchemy-utils==0.42.1 ; python_full_version < '3.10' - # via apache-airflow-core -sqlparse==0.5.4 ; python_full_version < '3.10' - # via - # apache-airflow-providers-common-sql - # dbt-core -sqlparse==0.5.5 ; python_full_version >= '3.10' - # via - # apache-airflow-providers-common-sql - # dbt-core -srsly==2.5.3 - # via - # confection - # spacy - # thinc - # weasel -stack-data==0.6.3 - # via ipython -starlette==0.49.3 ; python_full_version < '3.10' - # via - # cadwyn - # fastapi -starlette==1.3.1 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # cadwyn - # fastapi -structlog==25.5.0 ; python_full_version < '3.10' - # via - # apache-airflow-task-sdk - # dagster -structlog==26.1.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk - # dagster -svcs==25.1.0 - # via apache-airflow-core -sympy==1.14.0 ; python_full_version < '3.11' - # via onnxruntime -tabulate==0.9.0 ; python_full_version < '3.10' - # via - # apache-airflow-core - # dagster -tabulate==0.10.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # dagster -tenacity==9.1.2 ; python_full_version < '3.10' - # via - # apache-airflow-core - # retryhttp -tenacity==9.1.4 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # apache-airflow-task-sdk -termcolor==3.1.0 ; python_full_version < '3.10' - # via - # apache-airflow-core - # cleanlab -termcolor==3.3.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # cleanlab -text-unidecode==1.3 - # via python-slugify -thinc==8.3.4 ; python_full_version < '3.10' - # via - # freshdata-cleaner - # spacy -thinc==8.3.13 ; python_full_version >= '3.10' - # via spacy -threadpoolctl==3.6.0 - # via scikit-learn -tldextract==5.3.0 ; python_full_version < '3.10' - # via presidio-analyzer -tldextract==5.3.1 ; python_full_version >= '3.10' - # via presidio-analyzer -tokenizers==0.22.2 ; python_full_version < '3.10' - # via freshdata-cleaner -tokenizers==0.23.1 ; python_full_version >= '3.10' - # via freshdata-cleaner -tomli==2.4.1 - # via - # alembic - # build - # coverage - # dagster - # fastapi-cli - # maturin - # mypy - # pytest -tomlkit==0.15.0 - # via dagster-shared -toposort==1.10 - # via dagster -tqdm==4.68.4 - # via - # cleanlab - # dagster - # huggingface-hub - # spacy -traitlets==5.15.1 - # via - # ipython - # ipywidgets - # matplotlib-inline -twine==6.2.0 - # via freshdata-cleaner -typer==0.23.2 ; python_full_version < '3.10' - # via - # fastapi-cli - # fastapi-cloud-cli - # huggingface-hub - # spacy - # typer-slim -typer==0.26.8 ; python_full_version >= '3.10' - # via - # fastapi-cli - # spacy - # weasel -typer-slim==0.23.2 ; python_full_version < '3.10' - # via weasel -typing-extensions==4.16.0 - # via - # a2wsgi - # aiosqlite - # alembic - # anyio - # anywidget - # apache-airflow-core - # apache-airflow-task-sdk - # asgiref - # cadwyn - # cloudpathlib - # confection - # cron-descriptor - # cryptography - # dagster-shared - # dbt-adapters - # dbt-common - # dbt-core - # dbt-semantic-interfaces - # exceptiongroup - # fastapi - # great-tables - # grpcio - # htmltools - # huggingface-hub - # ipython - # libcst - # mashumaro - # mkdocstrings-python - # mypy - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions - # pydantic - # pydantic-core - # pydantic-extra-types - # pyjwt - # referencing - # rich-toolkit - # snowplow-tracker - # sqlalchemy - # starlette - # structlog - # typing-inspection - # uvicorn - # virtualenv -typing-inspection==0.4.2 - # via - # cadwyn - # fastapi - # pydantic - # pydantic-settings -tzdata==2026.3 - # via - # agate - # arrow - # dagster - # pandas - # pendulum -uc-micro-py==1.0.3 ; python_full_version < '3.10' - # via linkify-it-py -uc-micro-py==2.0.0 ; python_full_version >= '3.10' - # via linkify-it-py -universal-pathlib==0.3.10 - # via - # apache-airflow-core - # dagster -urllib3==2.6.3 ; python_full_version < '3.10' - # via - # id - # requests - # sentry-sdk - # twine -urllib3==2.7.0 ; python_full_version >= '3.10' - # via - # id - # requests - # twine -uuid6==2025.0.1 - # via apache-airflow-core -uvicorn==0.39.0 ; python_full_version < '3.10' - # via - # fastapi - # fastapi-cli - # fastapi-cloud-cli -uvicorn==0.51.0 ; python_full_version >= '3.10' - # via - # apache-airflow-core - # fastapi - # fastapi-cli -uvloop==0.22.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' - # via uvicorn -virtualenv==21.6.1 - # via pre-commit -wasabi==1.1.3 - # via - # spacy - # thinc - # weasel -watchdog==6.0.0 - # via - # dagster - # mkdocs -watchfiles==1.1.1 ; python_full_version < '3.10' - # via uvicorn -watchfiles==1.2.0 ; python_full_version >= '3.10' - # via uvicorn -wcwidth==0.8.2 - # via prompt-toolkit -weasel==0.4.3 ; python_full_version < '3.10' - # via spacy -weasel==1.0.0 ; python_full_version >= '3.10' - # via spacy -websockets==15.0.1 ; python_full_version < '3.10' - # via uvicorn -websockets==16.1 ; python_full_version >= '3.10' - # via - # nokap - # uvicorn -werkzeug==3.1.8 ; python_full_version < '3.10' - # via flask -widgetsnbextension==4.0.15 - # via ipywidgets -wirerope==1.0.0 - # via methodtools -wrapt==2.2.2 - # via - # deprecated - # smart-open -zipp==3.23.1 ; python_full_version < '3.10' - # via - # importlib-metadata - # importlib-resources -zipp==4.1.0 ; python_full_version >= '3.10' - # via importlib-metadata diff --git a/uv.lock b/uv.lock index 29fd157c..9a14fd18 100644 --- a/uv.lock +++ b/uv.lock @@ -1,8 +1,10 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.10" resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] @@ -406,28 +408,11 @@ name = "freshdata-cleaner" version = "0.4.0" source = { editable = "." } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "pandas" }, ] [package.optional-dependencies] -all = [ - { name = "duckdb" }, - { name = "polars" }, - { name = "pyarrow" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -bench = [ - { name = "duckdb" }, - { name = "polars" }, - { name = "psutil" }, - { name = "pyarrow" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] dev = [ { name = "build" }, { name = "duckdb" }, @@ -444,46 +429,41 @@ dev = [ { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "twine" }, ] -duckdb = [ - { name = "duckdb" }, - { name = "pyarrow" }, +migrate = [ + { name = "libcst" }, ] polars = [ { name = "polars" }, { name = "pyarrow" }, ] +test = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] [package.metadata] requires-dist = [ { name = "build", marker = "extra == 'dev'", specifier = ">=1.2.0" }, - { name = "duckdb", marker = "extra == 'all'", specifier = ">=1.0.0" }, - { name = "duckdb", marker = "extra == 'bench'", specifier = ">=1.0.0" }, { name = "duckdb", marker = "extra == 'dev'", specifier = ">=1.0.0" }, - { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.0.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.100.0" }, + { name = "libcst", marker = "extra == 'migrate'", specifier = ">=1.4.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10.0" }, - { name = "numpy", specifier = ">=1.24.0" }, + { name = "numpy", specifier = ">=1.24.0,<2.4.0" }, { name = "pandas", specifier = ">=2.0.0" }, - { name = "polars", marker = "extra == 'all'", specifier = ">=1.0.0" }, - { name = "polars", marker = "extra == 'bench'", specifier = ">=1.0.0" }, { name = "polars", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "polars", marker = "extra == 'polars'", specifier = ">=1.0.0" }, - { name = "psutil", marker = "extra == 'bench'", specifier = ">=5.9.0" }, { name = "psutil", marker = "extra == 'dev'", specifier = ">=5.9.0" }, - { name = "pyarrow", marker = "extra == 'all'", specifier = ">=15.0.0" }, - { name = "pyarrow", marker = "extra == 'bench'", specifier = ">=15.0.0" }, { name = "pyarrow", marker = "extra == 'dev'", specifier = ">=15.0.0" }, - { name = "pyarrow", marker = "extra == 'duckdb'", specifier = ">=15.0.0" }, { name = "pyarrow", marker = "extra == 'polars'", specifier = ">=15.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, - { name = "scikit-learn", marker = "extra == 'all'", specifier = ">=1.4.0" }, - { name = "scikit-learn", marker = "extra == 'bench'", specifier = ">=1.4.0" }, { name = "scikit-learn", marker = "extra == 'dev'", specifier = ">=1.4.0" }, { name = "twine", marker = "extra == 'dev'", specifier = ">=5.0.0" }, ] -provides-extras = ["polars", "duckdb", "all", "bench", "dev"] +provides-extras = ["migrate", "polars", "test", "dev"] [[package]] name = "hypothesis" @@ -589,7 +569,9 @@ name = "hypothesis" version = "6.170.15" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -795,6 +777,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] +[[package]] +name = "libcst" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version != '3.13.*'" }, + { name = "pyyaml-ft", marker = "python_full_version == '3.13.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/c0/098e5c91ff1537f00c85a6438b6cb1863d17144680cc91f47c87f104a200/libcst-1.9.0.tar.gz", hash = "sha256:087b58a9afe076bb08e2d726478e1f16cb928d67ffa9092817e033c335de522a", size = 914739, upload-time = "2026-07-29T21:28:43.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/b3/23144dcd9f9cd6f6150058653ecd0357c6b09361656ebd286165a5d76d34/libcst-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5705f664266742b8d9323db714f58cf06a6cde739eb09fd9ca31cd233f9806e5", size = 2055037, upload-time = "2026-07-29T19:24:27.708Z" }, + { url = "https://files.pythonhosted.org/packages/5d/87/164a9bac39a817b505d27cf4490e6f66b2a1b8adb6b4e022b1ef4613bcfe/libcst-1.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7dcddfb4905c5ef90a6fdc58dccbc049faf0056b68161042fc1e0526b20cba05", size = 2210166, upload-time = "2026-07-29T19:24:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/1c/cb/b5aeaefbe471c4225c145c15c3bac96d67b0641a5e3ec9b871c62858c88c/libcst-1.9.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cc84aee3c293a012db722aa2d6ea87be524646852684a5a4e54c1fb98ea9072f", size = 2260568, upload-time = "2026-07-29T19:24:31.021Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0f/351dcd23629732e35e716e13ad536a193124cda075ac5e59dc866503447b/libcst-1.9.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:329ffda6b0ef708ff22876f982ec886a509c8f73ac1ccb0190693d35cb09aa0f", size = 2276024, upload-time = "2026-07-29T19:24:32.393Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/0da2b60137a6290ba7dda902b32da64d46c9a5107aa002da578f8b1ff8c8/libcst-1.9.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5302fc698ef863e1a2fb540e95793b3d5a96abc89aa6423ee9c389d7d7ec5eab", size = 2384413, upload-time = "2026-07-29T19:24:33.983Z" }, + { url = "https://files.pythonhosted.org/packages/8f/80/7e48d2fc1afda13a6de50ab6f8340dbd104fd6c9bcd2ff11dec66e320f2c/libcst-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6615b036694f6906e30dc018b5b34898cff7ff760ab549f9dea1ada4f53495b", size = 2110204, upload-time = "2026-07-29T19:24:35.491Z" }, + { url = "https://files.pythonhosted.org/packages/42/87/52c88c7ba873011526aa35b8fa0cf55d50effe967c999e7cb688155c5691/libcst-1.9.0-cp310-cp310-win_arm64.whl", hash = "sha256:dacc583bcd8db56e457ae559b910673c6b5df66f694d1f1154ce0b3982158d17", size = 1988757, upload-time = "2026-07-29T19:24:37.053Z" }, + { url = "https://files.pythonhosted.org/packages/53/36/b45e6950878d9143f95db609432031ac688b311aa14450a8df239c648c45/libcst-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cdbafbc992b38ea005ef37e20243c4177879020c82d414789fbb7fa22d6d3a9", size = 2054911, upload-time = "2026-07-29T19:24:38.542Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d8/1c1ae22e05fbc763dc5a4e4c3cab356166913ef19d2c2da42e6f1be5c98c/libcst-1.9.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3f735e3acb3afc75d50803fde1598b467bf05d12d6854025df4398aebd75a14c", size = 2209147, upload-time = "2026-07-29T19:24:40.039Z" }, + { url = "https://files.pythonhosted.org/packages/84/e1/0d5eaeac4dadd34fcd9ea154d0a6e5ce10c9fec0e02c73f807a16475b8b0/libcst-1.9.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:951f13c193e590fc88c9582a5341a130fe53ecece51043cff4bd54e25a5e022d", size = 2260355, upload-time = "2026-07-29T19:24:41.423Z" }, + { url = "https://files.pythonhosted.org/packages/71/b2/45ce33c6bca5c1961a90f2999138741daa0dc8cf6fe8b122c0ca2502632d/libcst-1.9.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bc6aa3e7f21315dbe9868c0d13ffaac309514e8c816320c5f91230fa5b21df92", size = 2275659, upload-time = "2026-07-29T19:24:42.828Z" }, + { url = "https://files.pythonhosted.org/packages/bb/b1/936d2f17d93fb160579ff56ca671d26154209bbc07ea8645eb11db1f9633/libcst-1.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f101e0bf8638eba80694ecc3c3abb5309eb488f5271427536cb379c5c1e0ba9e", size = 2383467, upload-time = "2026-07-29T19:24:44.397Z" }, + { url = "https://files.pythonhosted.org/packages/e7/59/db667f15d62cb22c75b24f6b13c4151292aa1889a64ff9abce92f60d5a97/libcst-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:77d9303572cf0fbd0a2eb71aff167a4246541f43bf58b781b35b9d0bbab80ddc", size = 2109985, upload-time = "2026-07-29T19:24:45.966Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a8/99926865cd3f97c0cffc71cd38152c883b7ae666eb8327d355f01bddb33a/libcst-1.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:4090105fa4ad114766d0f8b34418d9a3b359bc8645da860a447e19770b5705d1", size = 1988559, upload-time = "2026-07-29T19:24:47.528Z" }, + { url = "https://files.pythonhosted.org/packages/b0/bb/d22c37c33dfe18084634f5ef89f8f0749ffe7b6e0ad312722aafd86bbbdb/libcst-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cd1a3500c41784075c4946a995d5ad89f68fa0d226b63ff3c4d78f6ea6dd23e5", size = 2043159, upload-time = "2026-07-29T19:24:49.013Z" }, + { url = "https://files.pythonhosted.org/packages/10/b8/2dedef84d72e7271119217503b69ed6dc5d0b2077685e163caae669d9c70/libcst-1.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:611cebd3bbc2014576f4dcc7b845b3c594c96ddc287a3db9b78f22eff156a7d3", size = 2203245, upload-time = "2026-07-29T19:24:50.399Z" }, + { url = "https://files.pythonhosted.org/packages/e8/90/e02ac2dad647423f947bb11f8322bfdba8ccfdd380e6c7b695add2d1acd4/libcst-1.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:8d731abe1307720ea1a52d447555e8443a6d130e0e520243c0634a58f6edbc9d", size = 2255388, upload-time = "2026-07-29T19:24:52.21Z" }, + { url = "https://files.pythonhosted.org/packages/13/5f/6089a51518cfd2ff40950eb26bcc36951d7b6d4f4213568aa0290265aff7/libcst-1.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8bc5351d92ca6ac1cc32e097700e1161ad1ceaa4d9b2cca5abadb1e94576b325", size = 2268982, upload-time = "2026-07-29T19:24:53.562Z" }, + { url = "https://files.pythonhosted.org/packages/ed/78/26881ec466fb70cbc129dca26ccb5a52a0061face2c5822e4b61f00f9699/libcst-1.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03165a264653bb77f6a11b412ae09c08bdb0c25864f3b8d42b816ac64b9d4b9e", size = 2378174, upload-time = "2026-07-29T19:24:55.175Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7a/a4dba5f11faf12a12ffba06d18019987851aab33b590242a602ffd4fb1bd/libcst-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:b755ed4a4bc2faee849b54820023137d60d4c199e0a0822f0ff0c1bc49e49b48", size = 2103462, upload-time = "2026-07-29T19:24:56.966Z" }, + { url = "https://files.pythonhosted.org/packages/f5/13/57cb129093e0d6744b3c914ba6cbbdf51ed1b2c823882936c553a3a0cf1b/libcst-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e50576bad7d56459d9792cd0b0dfe5469dad13646e9a9c0a8b1ba20b269f332", size = 1979825, upload-time = "2026-07-29T19:24:58.403Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b1/befc0544283bb3923a928accf79ed685e5a725524bdb3491826670affc07/libcst-1.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b9df30f524317b097dc53065b25dda33d6a4cc3c7c8bf4fc83ca7559c58cc0", size = 2043754, upload-time = "2026-07-29T19:24:59.885Z" }, + { url = "https://files.pythonhosted.org/packages/45/50/fef7c172a8457c95894edf5fb04805024899cdfea41fa01b0636587b79e1/libcst-1.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e465a7bc9c2b9533eb9e06d2391f8819f811b5112919d4064c9fb8565aaafa08", size = 2203548, upload-time = "2026-07-29T19:25:01.323Z" }, + { url = "https://files.pythonhosted.org/packages/18/ff/764cd2be1fd99d774fc44039c319dc0ed1d9d9afeaa02759a05121cccd4b/libcst-1.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8504b422c95676a8c27b517e1ac01413ece91bf356865c587ca9bdcd5708a2f7", size = 2255274, upload-time = "2026-07-29T19:25:02.764Z" }, + { url = "https://files.pythonhosted.org/packages/34/a7/474748a27a02fa83e3556b260d5f5236ca48164fa7beffecf3b2cad24dca/libcst-1.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bcb9f9d4fcfe2ec7a40d2c26e03a538d5d9dc189c38551eb3f94ab661afee7c0", size = 2269126, upload-time = "2026-07-29T19:25:04.158Z" }, + { url = "https://files.pythonhosted.org/packages/8a/b7/655e45363b8cf87b91e41e060b21c89e5316c7ef36eb14a1a498b27bb71e/libcst-1.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8d671c39a431c309476099b8ec811e412503ec0f4465f6fc907cb51c70e8e6b", size = 2378602, upload-time = "2026-07-29T19:25:06.424Z" }, + { url = "https://files.pythonhosted.org/packages/db/13/6da63f0902ece43bf9d737017251ad7ad06edd9d3c4e1856450403cb4473/libcst-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:4d382fba04077eb556a1ea4295a4482e192aa93639c5a07ba0885841965ea0c0", size = 2103486, upload-time = "2026-07-29T19:25:07.979Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ff/dccb1a55e38b4e47256a97242d61d2bef5366c744faf88fb087d4fa0f995/libcst-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:a621e261990148c1cfbe26c1798bd2375c131cf358a44a7a4659fc41c1a336e1", size = 1979907, upload-time = "2026-07-29T19:25:09.532Z" }, + { url = "https://files.pythonhosted.org/packages/65/2a/4943c71d90975bc59034a057dea346b365c276f308c1d31f5cf2bd85492d/libcst-1.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eccf4c57d273cdd3fe1c67b72cf9bb1bbd4547aa011824e96ccf5b7136057aa4", size = 2044529, upload-time = "2026-07-29T19:25:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1168b98c2a0f338be3a44753647baab051feaf6167cc887bf4447f8fd920/libcst-1.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32395244edfe6538e0ea2bf82051d60103d3f54861274805c4fb3745efb70a85", size = 2203519, upload-time = "2026-07-29T19:25:12.543Z" }, + { url = "https://files.pythonhosted.org/packages/25/7d/2eaa697a80f899bcf2245680bbfcc1e478eb3b3328c21d4b2880bdf00dac/libcst-1.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:444e84c76cd035cd2fe136838c1a524d34b08216521f6f5093df8a6f6cfa5799", size = 2255879, upload-time = "2026-07-29T19:25:14.137Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/793b9fd96dd52d202f5f2b88d7e54f05ebed767d1bb3b32395a1817bf6ab/libcst-1.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb5d0946f2b4c6711b5d69fe4f833b364f9e7a1a2b08f88b98619dc18975099a", size = 2269807, upload-time = "2026-07-29T19:25:15.647Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/1bc7aaea03971c45e8a885fed9fc1c73176dbbd00b0296a3cde9529bafd6/libcst-1.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45808c03528b3ad40b14095348a918e08d98c47b4a125d631cb78e817c0a5b16", size = 2378171, upload-time = "2026-07-29T19:25:17.289Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f7/bc49e367d2bc8817213594dd52cf6b2da2dbbda90b5382d60653999274ac/libcst-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:568288cdbfe3b4ca3ae4852cb0a439ff053dd54c841bc4995bf2b71238b5de40", size = 2177847, upload-time = "2026-07-29T19:25:19.35Z" }, + { url = "https://files.pythonhosted.org/packages/87/80/4d81577a22e6d535d1a3409f3a1c6903e09036f0e17fc47f92169dbc3501/libcst-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:107593af46945593e7825821793393262bc4fa1d3ea24c3ed487b61269bbbdf8", size = 2058134, upload-time = "2026-07-29T19:25:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7f/c3f3a0e7a1a2adaa76e815e82a7814d6bfe7d49432276bba652248c68d0b/libcst-1.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f6248cb07444ab9a6733a855737a9febed8b9adca51347019348b09a3ac7dfe9", size = 2036102, upload-time = "2026-07-29T19:25:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/4e/af/2f5543255b2c7d749b966adeccc6bfb1652cf8b425afb913d0f3f025a865/libcst-1.9.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:496c24e0d3240bc7da45dae543aff3f5b6509978c39262d6b84ed2fb999dded2", size = 2194806, upload-time = "2026-07-29T19:25:24.017Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5b/03f4cddce426d005e208b39ea7b2d456e667cdee0f1891360f0fc8430f20/libcst-1.9.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ea490fa8540503db5f321f0268becab46eb50f710e8cec8041e241fd66f6874f", size = 2246762, upload-time = "2026-07-29T19:25:25.365Z" }, + { url = "https://files.pythonhosted.org/packages/d8/31/9d5fe1e43dc3dbcc74f70f3e0e73fcdd8d84effc1059d8b45974f0d0d2eb/libcst-1.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:50ab94bb2524b419056d4003032b8c67102ac800f8d85b3c4260a01746d94dbb", size = 2259633, upload-time = "2026-07-29T19:25:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2b/6752b28d88c3a19b3bc0b9e1838443b6760d5c862b4f4b37955402659e24/libcst-1.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a2faaf92500d0226358125630f5aab4758e8aad3f2d70a10892ec3c700781a54", size = 2368043, upload-time = "2026-07-29T19:25:28.344Z" }, + { url = "https://files.pythonhosted.org/packages/e2/94/775825b2637f8ab05694b6a4b3802ae6783b4e799f9b58d2400c7e2d4369/libcst-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c7b548512db25af9c2997a95fa731bd6b6928ecbad6c0915d7482d8bb42d34f", size = 2176432, upload-time = "2026-07-29T19:25:29.921Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3d/88ad67427c6fd9db929e087912b0a540e5140e5cb77e7ca4170edaac8531/libcst-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:497d5329345f1f5df84e41b0bbd00204b64a2fd30dfe3cfaeaebca633a31e877", size = 2052931, upload-time = "2026-07-29T19:25:31.397Z" }, + { url = "https://files.pythonhosted.org/packages/39/e4/ad790b043a38b10cea13166c8dd716654ad8b227c20f12eebf6326191cd9/libcst-1.9.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a5068bf6114f6f4d79af7a6c80a28d1deb50441150ea1db13b5458ab34bec159", size = 2043987, upload-time = "2026-08-11T05:54:02.122Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/d3cd8275cc54ffc9817ade91442b3e6e9edd1a4518bd4e6dda13e3152dbe/libcst-1.9.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:7acfd18adcdd32dcf41ce676cf121002afcbe9ad2c72dc0c18d78465beedc228", size = 2204003, upload-time = "2026-08-11T05:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/da/62/87eddccb11d5d221d50ac4fff4b6e9b8d48c547028d01f7d0fbc5c8a1d75/libcst-1.9.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:86361e2b426bd1b403e52375703c8b764e226e571adcb30442510710681edbf0", size = 2256053, upload-time = "2026-08-11T05:54:06.228Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/a44126aeb9fca8b4e342e0cc803ea00ca8f6cd5712619a7897b3eba13797/libcst-1.9.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8c14abe844bec8b021111b3985474e49f5af799c020e8e40dedcac87c97a40c5", size = 2270576, upload-time = "2026-08-11T05:54:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/bb/51/3af3dd44b117d66486e0ed61e43a16b7fc8cc5c372e1dd4ca0e70b888d46/libcst-1.9.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:8930971d2299b7006bb5d10c10038f44efb6da89d5bca2823595e31a9cdb96af", size = 2378573, upload-time = "2026-08-11T05:54:10.093Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e8/e545087d298dbf6494e84a8092f0f5dbd62eedacabd4ecd6b364367b4804/libcst-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5891ce9cff815077614f509e3a23888c5ddb8081d6188e8ba1172c0ccc369022", size = 2177937, upload-time = "2026-08-11T05:54:12.168Z" }, + { url = "https://files.pythonhosted.org/packages/84/17/93a00a8e03494a84db102dd0e3d35b8876b1084ca2c194b331a168d86eb1/libcst-1.9.0-cp315-cp315-win_arm64.whl", hash = "sha256:1260b5d070a3324447a53a00387225a55472a62077ce01922d30a2a78cc4b138", size = 2058633, upload-time = "2026-08-11T05:54:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/d7/00/c3751b12eb81822ea0b6131d374a98bc6c024c4b9ec5f6ea8f53dc23e967/libcst-1.9.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:02ee2dbdb5c218116f16a021350fc267a14be205b50d81c33f5d73857ab6fbf7", size = 2036178, upload-time = "2026-08-11T05:54:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/2f/12/48ff486eb5adc593f4818569e8896b74b672414ab4722a74b17e4c4cf31e/libcst-1.9.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:3e5b684191a0462b2d40a73261ea7f4da6a49e7a030b7e0fceca46bebb51dfe5", size = 2195496, upload-time = "2026-08-11T05:54:17.625Z" }, + { url = "https://files.pythonhosted.org/packages/51/18/b13a4669864d41a4801fed7b86ede1049dc9a1e1dde250a7ee1f9c3b1285/libcst-1.9.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:2b150c4f298fe54eb0fe73abf71f57db74d070796140a8c206c9615b6861f01e", size = 2245769, upload-time = "2026-08-11T05:54:19.343Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c7/cf2d46744825e84afbd7228fdeeea8d23cf6c4b2074253c27efb7705c653/libcst-1.9.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:50b913e37187f00a6fb88962009364e1cf1b1284aa1762304f34b52b972f2218", size = 2260576, upload-time = "2026-08-11T05:54:21.444Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/5433d3d62250b2e4325938db101766bcab2a006819684713bcccb6259a88/libcst-1.9.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:9f92c75283030fd58fb7d6e560168a00381e0e3368ec8a026a3ec8a828a05f93", size = 2368709, upload-time = "2026-08-11T05:54:23.276Z" }, + { url = "https://files.pythonhosted.org/packages/87/39/9f0e1690727623f99489895db0e42048a3e1e8cea0627517c593e437fd83/libcst-1.9.0-cp315-cp315t-win_amd64.whl", hash = "sha256:4adf97bb1aff8039b0bd4991e2f838be6e4fad552821b26b71a5d1a653c2582f", size = 2177866, upload-time = "2026-08-11T05:54:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/cd/79/9dc7811883e67ea771057e8937bc536eb3e77f3635fc5a93cd85b6fe1ea8/libcst-1.9.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8f0dd08a5773d7051e105250b2a2c73d8065ea9b2d68e7e1bdc5244660e75f93", size = 2052908, upload-time = "2026-08-11T05:54:26.835Z" }, +] + [[package]] name = "librt" version = "0.13.0" @@ -1014,9 +1064,6 @@ wheels = [ name = "numpy" version = "2.2.6" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, @@ -1075,142 +1122,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, ] -[[package]] -name = "numpy" -version = "2.4.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.11.*'", -] -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, - { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, - { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, - { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, - { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, - { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, - { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, - { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, - { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, - { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, - { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, - { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, - { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, - { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, - { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, - { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, - { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, - { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, - { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, - { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, - { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, - { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, - { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, - { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, - { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, - { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, - { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, - { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, - { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, - { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, - { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, - { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, - { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, - { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, - { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, - { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, - { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, - { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, - { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, -] - -[[package]] -name = "numpy" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", -] -sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, - { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, - { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, - { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, - { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, - { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, - { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, - { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, - { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, - { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, - { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, - { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, - { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, - { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, - { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, - { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, - { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, - { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, - { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, - { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, - { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, - { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, - { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, - { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, - { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, - { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, - { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, - { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, - { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, - { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, - { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, - { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, - { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, - { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, - { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, -] - [[package]] name = "packaging" version = "26.2" @@ -1225,9 +1136,7 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -1496,6 +1405,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "pyyaml-ft" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/eb/5a0d575de784f9a1f94e2b1288c6886f13f34185e13117ed530f32b6f8a8/pyyaml_ft-8.0.0.tar.gz", hash = "sha256:0c947dce03954c7b5d38869ed4878b2e6ff1d44b08a0d84dc83fdad205ae39ab", size = 141057, upload-time = "2025-06-10T15:32:15.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/ba/a067369fe61a2e57fb38732562927d5bae088c73cb9bb5438736a9555b29/pyyaml_ft-8.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8c1306282bc958bfda31237f900eb52c9bedf9b93a11f82e1aab004c9a5657a6", size = 187027, upload-time = "2025-06-10T15:31:48.722Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c5/a3d2020ce5ccfc6aede0d45bcb870298652ac0cf199f67714d250e0cdf39/pyyaml_ft-8.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30c5f1751625786c19de751e3130fc345ebcba6a86f6bddd6e1285342f4bbb69", size = 176146, upload-time = "2025-06-10T15:31:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/e3/bb/23a9739291086ca0d3189eac7cd92b4d00e9fdc77d722ab610c35f9a82ba/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fa992481155ddda2e303fcc74c79c05eddcdbc907b888d3d9ce3ff3e2adcfb0", size = 746792, upload-time = "2025-06-10T15:31:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c2/e8825f4ff725b7e560d62a3609e31d735318068e1079539ebfde397ea03e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cec6c92b4207004b62dfad1f0be321c9f04725e0f271c16247d8b39c3bf3ea42", size = 786772, upload-time = "2025-06-10T15:31:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/35/be/58a4dcae8854f2fdca9b28d9495298fd5571a50d8430b1c3033ec95d2d0e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06237267dbcab70d4c0e9436d8f719f04a51123f0ca2694c00dd4b68c338e40b", size = 778723, upload-time = "2025-06-10T15:31:56.093Z" }, + { url = "https://files.pythonhosted.org/packages/86/ed/fed0da92b5d5d7340a082e3802d84c6dc9d5fa142954404c41a544c1cb92/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a7f332bc565817644cdb38ffe4739e44c3e18c55793f75dddb87630f03fc254", size = 758478, upload-time = "2025-06-10T15:31:58.314Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/ac02afe286275980ecb2dcdc0156617389b7e0c0a3fcdedf155c67be2b80/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7d10175a746be65f6feb86224df5d6bc5c049ebf52b89a88cf1cd78af5a367a8", size = 799159, upload-time = "2025-06-10T15:31:59.675Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ac/c492a9da2e39abdff4c3094ec54acac9747743f36428281fb186a03fab76/pyyaml_ft-8.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:58e1015098cf8d8aec82f360789c16283b88ca670fe4275ef6c48c5e30b22a96", size = 158779, upload-time = "2025-06-10T15:32:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/5d/9b/41998df3298960d7c67653669f37710fa2d568a5fc933ea24a6df60acaf6/pyyaml_ft-8.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e64fa5f3e2ceb790d50602b2fd4ec37abbd760a8c778e46354df647e7c5a4ebb", size = 191331, upload-time = "2025-06-10T15:32:02.602Z" }, + { url = "https://files.pythonhosted.org/packages/0f/16/2710c252ee04cbd74d9562ebba709e5a284faeb8ada88fcda548c9191b47/pyyaml_ft-8.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8d445bf6ea16bb93c37b42fdacfb2f94c8e92a79ba9e12768c96ecde867046d1", size = 182879, upload-time = "2025-06-10T15:32:04.466Z" }, + { url = "https://files.pythonhosted.org/packages/9a/40/ae8163519d937fa7bfa457b6f78439cc6831a7c2b170e4f612f7eda71815/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c56bb46b4fda34cbb92a9446a841da3982cdde6ea13de3fbd80db7eeeab8b49", size = 811277, upload-time = "2025-06-10T15:32:06.214Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/28d82dbff7f87b96f0eeac79b7d972a96b4980c1e445eb6a857ba91eda00/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dab0abb46eb1780da486f022dce034b952c8ae40753627b27a626d803926483b", size = 831650, upload-time = "2025-06-10T15:32:08.076Z" }, + { url = "https://files.pythonhosted.org/packages/e8/df/161c4566facac7d75a9e182295c223060373d4116dead9cc53a265de60b9/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd48d639cab5ca50ad957b6dd632c7dd3ac02a1abe0e8196a3c24a52f5db3f7a", size = 815755, upload-time = "2025-06-10T15:32:09.435Z" }, + { url = "https://files.pythonhosted.org/packages/05/10/f42c48fa5153204f42eaa945e8d1fd7c10d6296841dcb2447bf7da1be5c4/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:052561b89d5b2a8e1289f326d060e794c21fa068aa11255fe71d65baf18a632e", size = 810403, upload-time = "2025-06-10T15:32:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d2/e369064aa51009eb9245399fd8ad2c562bd0bcd392a00be44b2a824ded7c/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3bb4b927929b0cb162fb1605392a321e3333e48ce616cdcfa04a839271373255", size = 835581, upload-time = "2025-06-10T15:32:12.897Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/26534bed77109632a956977f60d8519049f545abc39215d086e33a61f1f2/pyyaml_ft-8.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793", size = 171579, upload-time = "2025-06-10T15:32:14.34Z" }, +] + [[package]] name = "readme-renderer" version = "45.0" @@ -1593,7 +1590,7 @@ resolution-markers = [ ] dependencies = [ { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, { name = "threadpoolctl" }, ] @@ -1636,14 +1633,15 @@ name = "scikit-learn" version = "1.9.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ { name = "joblib" }, { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "threadpoolctl" }, @@ -1690,7 +1688,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -1749,7 +1747,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -1820,10 +1818,12 @@ name = "scipy" version = "1.18.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ From b8b449a5dfc0a78dff47f207fee200c3250230cf Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:13:54 +0530 Subject: [PATCH 06/11] fix(core): make the Polars backend rename, protect and clip like pandas - PolarsBackend.suggest_plan emits normalize_name actions and checks id, target and rules against normalized names. Before, a target such as "Churn" with target_column="churn" was imputed. - clip_iqr actions now carry lower/upper bounds from one 1-row aggregate query (linear quartiles on the imputed column, as in the pandas stages). Before, they had no bounds and apply_plan skipped them. - apply_plan chains expressions per column, so impute and clip on one column compose instead of colliding in with_columns. - Move the polars_engine lazy-frame tests onto the freshdata API. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 48 +++++++++++++++++++------ tests/test_backend_honesty.py | 60 +++++++++++++++++++++++++++++++ tests/test_property_invariants.py | 13 ++----- 3 files changed, 100 insertions(+), 21 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index b2f91af3..30e11c1b 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -10,6 +10,7 @@ ProfileReport, ) from freshdata._core.stages import ( + _normalize_name, normalize_config, stage_column_names, stage_dtypes, @@ -125,7 +126,11 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema actions = [] cfg_eff = normalize_config(cfg) - for name, dtype in schema.items(): + clip_cols: dict[str, pl.Expr] = {} + for orig, dtype in schema.items(): + name = _normalize_name(orig) if cfg_eff.normalize_names else orig + if name != orig: + actions.append(ActionRecord("column_names", orig, "normalize_name", {"to": name})) if name in cfg_eff.id_columns or name == cfg_eff.target_column or name.endswith("_id") or name == "id": continue rule = cfg_eff.rules.get(name) @@ -135,15 +140,37 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): actions.append(ActionRecord("strings", name, "strip_strings", {})) if dtype.is_numeric(): + # Mirror the pandas stage order: IQR bounds see the imputed column. + base = pl.col(orig) impute_strat = rule.impute if rule else (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) if impute_strat in ("median", "auto"): actions.append(ActionRecord("missing", name, "impute_median", {})) + base = base.fill_null(base.median()) elif impute_strat == "mean": actions.append(ActionRecord("missing", name, "impute_mean", {})) + base = base.fill_null(base.mean()) outlier_strat = rule.outliers if rule else cfg_eff.default_outliers if outlier_strat == "clip": - actions.append(ActionRecord("outliers", name, "clip_iqr", {})) + clip_cols[name] = base + if clip_cols: + # One 1-row aggregate query; apply_plan itself never collects. + stats = df.lazy().select( + e + for i, base in enumerate(clip_cols.values()) + for e in ( + base.count().alias(f"n{i}"), + base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), + base.quantile(0.75, interpolation="linear").alias(f"q75_{i}"), + ) + ).collect().row(0, named=True) + for i, name in enumerate(clip_cols): + if stats[f"n{i}"] < 4: + continue + q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] + iqr = q75 - q25 + lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr + actions.append(ActionRecord("outliers", name, "clip_iqr", {"lower": float(lower), "upper": float(upper)})) if cfg_eff.drop_duplicates: actions.append(ActionRecord("duplicates", "*", "drop_duplicates", {})) return CleanPlan(actions=tuple(actions)) @@ -157,28 +184,27 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea if renames: cur = cur.rename(renames) - # 2. Column expressions - exprs = [] + # 2. Column expressions, chained per column so actions compose in plan order + exprs: dict[str, pl.Expr] = {} for a in plan.actions: col_name = renames.get(a.column, a.column) + expr = exprs.get(col_name, pl.col(col_name)) if a.stage == "strings" and a.action == "strip_strings": - exprs.append(pl.col(col_name).str.strip_chars().alias(col_name)) + exprs[col_name] = expr.str.strip_chars() elif a.stage == "missing": if a.action == "impute_median": val = a.parameters.get("value") - expr = pl.col(col_name).fill_null(val if val is not None else pl.col(col_name).median()).alias(col_name) - exprs.append(expr) + exprs[col_name] = expr.fill_null(val if val is not None else expr.median()) elif a.action == "impute_mean": val = a.parameters.get("value") - expr = pl.col(col_name).fill_null(val if val is not None else pl.col(col_name).mean()).alias(col_name) - exprs.append(expr) + exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) elif a.stage == "outliers" and a.action == "clip_iqr": lower = a.parameters.get("lower") upper = a.parameters.get("upper") if lower is not None and upper is not None: - exprs.append(pl.col(col_name).clip(lower, upper).alias(col_name)) + exprs[col_name] = expr.clip(lower, upper) if exprs: - cur = cur.with_columns(exprs) + cur = cur.with_columns(e.alias(c) for c, e in exprs.items()) # 3. Duplicates if any(a.stage == "duplicates" and a.action == "drop_duplicates" for a in plan.actions): diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index 2d3c1e75..a052cab8 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -59,3 +59,63 @@ def test_polars_profile_counts_match_pandas(): for col in data: p, q = pd_rep.columns[col], pl_rep.columns[col] assert (q.null_count, q.unique_count, q.null_ratio) == (p.null_count, p.unique_count, p.null_ratio) + +def test_polars_plan_matches_pandas_plan(): + import freshdata as fd + + data = {"Customer ID": ["1", "2", "3", "4", "5"], "Age": [25.0, 30.0, None, 35.0, 40.0], + "Spend": [100.0, 120.0, 110.0, 115.0, 9999.0], "Churn": [0.0, 1.0, None, 0.0, 1.0]} + cfg = CleanConfig(id_columns=("customer_id",), target_column="churn", default_outliers="clip", + strip_strings=False, fix_dtypes=False) + def key(plan): + return {(a.stage, a.column, a.action): a.parameters for a in plan.actions + if a.stage in ("column_names", "outliers")} + assert key(fd.suggest_plan(pl.LazyFrame(data), cfg)) == key(fd.suggest_plan(pd.DataFrame(data), cfg)) + +def test_lazy_plan_applies_without_materializing(): + import freshdata as fd + + lf = pl.DataFrame({ + "Customer ID": ["001", "002", "003", "004", "005"], + "Age": [25, 30, None, 35, 40], + "Spend": [100.0, 120.0, 110.0, 115.0, 9999.0], + "Segment": [" Gold ", "Silver", None, "Gold", "Gold"], + "Churn": [0, 1, None, 0, 1], + }).lazy() + config = CleanConfig(id_columns=("customer_id",), target_column="churn", + default_impute="auto", default_outliers="clip", drop_duplicates=True) + plan = fd.suggest_plan(lf, config) + + actions = {(a.column, a.stage): a for a in plan.actions} + assert ("churn", "missing") not in actions + assert actions[("age", "missing")].action == "impute_median" + assert actions[("spend", "outliers")].action == "clip_iqr" + + poisoned_lf = lf.with_columns( + pl.col("Age").map_batches( + lambda _: (_ for _ in ()).throw(RuntimeError("Eager collect triggered!")), + return_dtype=pl.Int64, + ) + ) + assert isinstance(fd.apply_plan(poisoned_lf, plan, config), pl.LazyFrame) + + cleaned_lf, report = fd.apply_plan(lf, plan, config, return_report=True) + assert isinstance(cleaned_lf, pl.LazyFrame) + assert "PYTHON_UDF" not in cleaned_lf.explain() + result = cleaned_lf.collect() + assert result.columns == ["customer_id", "age", "spend", "segment", "churn"] + assert result["age"].null_count() == 0 + assert result["churn"].null_count() == 1 + assert result["spend"].max() < 9999.0 + assert report.total_actions == len(plan.actions) + +def test_lazy_plan_diff_detects_rule_changes(): + import freshdata as fd + + lf = pl.LazyFrame({"score": [1.0, 2.0, None, 4.0, 100.0]}) + conservative = fd.suggest_plan(lf, CleanConfig(strategy="conservative", default_outliers="preserve")) + aggressive = fd.suggest_plan(lf, CleanConfig( + strategy="aggressive", rules={"score": fd.ColumnRule(impute="median", outliers="clip")})) + diff = conservative.diff(aggressive) + assert [(a.column, a.action) for a in diff["added"]] == [("score", "clip_iqr")] + assert diff["removed"] == [] and diff["modified"] == [] diff --git a/tests/test_property_invariants.py b/tests/test_property_invariants.py index 4dd55c75..fe53c24d 100644 --- a/tests/test_property_invariants.py +++ b/tests/test_property_invariants.py @@ -7,11 +7,6 @@ import freshdata as fd from freshdata import CleanConfig, ColumnRule -from freshdata.polars_engine import ( - CleanConfig as PeCleanConfig, - apply_plan as pe_apply, - suggest_plan as pe_suggest, -) @st.composite @@ -113,12 +108,10 @@ def test_numeric_imputation_and_outlier_invariants(values: list[float | None]) - max_size=30, ) ) -def test_polars_engine_property_invariants(values: list[float | None]) -> None: +def test_lazy_clean_property_invariants(values: list[float | None]) -> None: lf = pl.LazyFrame({"val": values, "txt": [" foo " for _ in values]}) - cfg = PeCleanConfig(default_impute="median", default_outliers="clip", strip_strings=True) - plan = pe_suggest(lf, config=cfg) - assert plan is not None - cleaned_lf = pe_apply(lf, plan) + cfg = CleanConfig(default_impute="median", default_outliers="clip", strip_strings=True) + cleaned_lf = fd.clean(lf, config=cfg) assert isinstance(cleaned_lf, pl.LazyFrame) collected = cleaned_lf.collect() has_non_null = any(v is not None for v in values) From 950d200ef5ddcf881d60c2dcc7480d175298c005 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:15:40 +0530 Subject: [PATCH 07/11] refactor!: delete freshdata.polars_engine The core PolarsBackend now covers lazy renaming, target protection and IQR clipping, so the second Polars implementation is redundant. - Delete src/freshdata/polars_engine.py and tests/test_polars_engine.py. Its lazy-frame tests now live in tests/test_backend_honesty.py. - Keep benchmarks/bench_v3_lazy_vs_pandas.py, because tests and benchmarks/perf_gate.py import it. It now uses the freshdata API. - Point benchmark-gate.yml and the README at the moved tests. BREAKING CHANGE: freshdata.polars_engine is removed. Use freshdata.clean, freshdata.suggest_plan and freshdata.apply_plan, which accept polars.DataFrame and polars.LazyFrame. The plan uses the core ActionRecord vocabulary (impute_median, clip_iqr, normalize_name), and CleanPlan.diff returns {"added", "removed", "modified"}. The polars_engine features flag_iqr, drop_column, string mode imputation and baked fill values have no core equivalent. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark-gate.yml | 2 +- README.md | 2 +- benchmarks/bench_v3_lazy_vs_pandas.py | 2 +- src/freshdata/polars_engine.py | 354 -------------------------- tests/test_polars_engine.py | 81 ------ 5 files changed, 3 insertions(+), 438 deletions(-) delete mode 100644 src/freshdata/polars_engine.py delete mode 100644 tests/test_polars_engine.py diff --git a/.github/workflows/benchmark-gate.yml b/.github/workflows/benchmark-gate.yml index 75d90d4c..6b8de0cc 100644 --- a/.github/workflows/benchmark-gate.yml +++ b/.github/workflows/benchmark-gate.yml @@ -36,7 +36,7 @@ jobs: pip install -e . polars pandas pyarrow pytest - name: Run gate & engine unit tests - run: pytest tests/test_polars_engine.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q + run: pytest tests/test_backend_honesty.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q - name: Record Candidate (PR Head) Benchmark run: | diff --git a/README.md b/README.md index 4671c5be..03d87db7 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ freshdata-migrate src/ tests/ Bug reports, benchmark reproductions, and pull requests are welcome. 1. Open an issue with a minimal reproducible Polars or pandas snippet. 2. Run the test suite locally via `pytest -q`. -3. Ensure any changes to `polars_engine.py` pass the lazy materialization poison test (`tests/test_polars_engine.py`). +3. Ensure any changes to `_core/backends.py` pass the lazy materialization poison test (`tests/test_backend_honesty.py`). ## License diff --git a/benchmarks/bench_v3_lazy_vs_pandas.py b/benchmarks/bench_v3_lazy_vs_pandas.py index 5f62aeac..61e6d157 100644 --- a/benchmarks/bench_v3_lazy_vs_pandas.py +++ b/benchmarks/bench_v3_lazy_vs_pandas.py @@ -16,7 +16,7 @@ import pandas as pd import polars as pl -from freshdata.polars_engine import CleanConfig, apply_plan, suggest_plan +from freshdata import CleanConfig, apply_plan, suggest_plan def get_peak_rss_mb() -> float: diff --git a/src/freshdata/polars_engine.py b/src/freshdata/polars_engine.py deleted file mode 100644 index 6802036c..00000000 --- a/src/freshdata/polars_engine.py +++ /dev/null @@ -1,354 +0,0 @@ -from __future__ import annotations - -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any, Literal, TypeVar, overload - -import polars as pl - -FrameT = TypeVar("FrameT", pl.DataFrame, pl.LazyFrame) - -RoleType = Literal["id", "target", "numeric", "categorical", "datetime", "text"] -ImputeType = Literal["preserve", "drop", "mean", "median", "mode"] -OutlierType = Literal["preserve", "clip", "flag"] - - -@dataclass(frozen=True) -class ColumnRule: - role: RoleType | None = None - impute: ImputeType | None = None - outliers: OutlierType | None = None - protected: bool = False - - -@dataclass(frozen=True) -class CleanConfig: - strategy: Literal["conservative", "balanced", "aggressive"] = "balanced" - rules: Mapping[str, ColumnRule] = field(default_factory=dict) - id_columns: tuple[str, ...] = () - target_column: str | None = None - normalize_names: bool = True - strip_strings: bool = True - drop_duplicates: bool = False - default_impute: Literal["auto", "preserve", "median", "mode"] = "auto" - default_outliers: OutlierType = "preserve" - iqr_factor: float = 1.5 - strict_backend: bool = True - - -@dataclass(frozen=True) -class ActionRecord: - column: str - stage: Literal["strings", "missing", "outliers", "drop_column"] - operation: Literal[ - "strip_whitespace", - "fill_null", - "clip_iqr", - "flag_iqr", - "drop_column", - "preserve", - ] - params: Mapping[str, Any] = field(default_factory=dict) - affected_rows: int = 0 - rationale: str = "" - - -@dataclass(frozen=True) -class CleanPlan: - rename_map: Mapping[str, str] - actions: tuple[ActionRecord, ...] - drop_duplicates: bool = False - total_rows: int = 0 - - def diff( - self, other: CleanPlan - ) -> dict[tuple[str, str], tuple[ActionRecord | None, ActionRecord | None]]: - self_map = {(a.column, a.stage): a for a in self.actions} - other_map = {(a.column, a.stage): a for a in other.actions} - changes: dict[tuple[str, str], tuple[ActionRecord | None, ActionRecord | None]] = {} - for key in sorted(set(self_map) | set(other_map)): - if self_map.get(key) != other_map.get(key): - changes[key] = (self_map.get(key), other_map.get(key)) - return changes - - -@dataclass(frozen=True) -class CleanReport: - plan: CleanPlan - renamed_columns: Mapping[str, str] - dropped_columns: tuple[str, ...] - total_actions: int - - -def _to_snake_case(name: str) -> str: - cleaned = re.sub(r"[^\w\s]", "", name).strip() - cleaned = re.sub(r"[\s\-]+", "_", cleaned) - cleaned = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", cleaned) - return cleaned.lower() - - -def _infer_role( - col: str, dtype: pl.DataType, config: CleanConfig, rule: ColumnRule | None -) -> RoleType: - if rule and rule.role is not None: - return rule.role - if col in config.id_columns or col.endswith("_id") or col == "id": - return "id" - if col == config.target_column: - return "target" - if dtype.is_numeric(): - return "numeric" - if dtype.is_temporal(): - return "datetime" - return "categorical" - - -def suggest_plan( - df: pl.DataFrame | pl.LazyFrame, - config: CleanConfig | None = None, -) -> CleanPlan: - cfg = config or CleanConfig() - if not isinstance(df, (pl.DataFrame, pl.LazyFrame)): - raise TypeError( - f"Expected pl.DataFrame or pl.LazyFrame when strict_backend=True, got {type(df).__name__}." - ) - - lf = df.lazy() - schema = lf.collect_schema() - - rename_map: dict[str, str] = {} - for orig_col in schema.names(): - norm_col = _to_snake_case(orig_col) if cfg.normalize_names else orig_col - rename_map[orig_col] = norm_col - - # Build a single fused aggregation pass so LazyFrames never materialize rows - stat_exprs: list[pl.Expr] = [pl.len().alias("__total_rows")] - for orig_col, dtype in schema.items(): - norm_col = rename_map[orig_col] - col_expr = pl.col(orig_col) - stat_exprs.append(col_expr.null_count().alias(f"{norm_col}__nulls")) - - if dtype == pl.String and cfg.strip_strings: - stat_exprs.append( - col_expr.str.strip_chars() - .ne(col_expr) - .fill_null(False) - .sum() - .alias(f"{norm_col}__ws_dirty") - ) - elif dtype.is_numeric(): - stat_exprs.extend( - [ - col_expr.median().alias(f"{norm_col}__median"), - col_expr.mean().alias(f"{norm_col}__mean"), - col_expr.quantile(0.25).alias(f"{norm_col}__q25"), - col_expr.quantile(0.75).alias(f"{norm_col}__q75"), - ] - ) - if dtype == pl.String: - stat_exprs.append( - col_expr.str.strip_chars() - .drop_nulls() - .mode() - .first() - .alias(f"{norm_col}__mode") - ) - - stats = lf.select(stat_exprs).collect(engine="streaming").row(0, named=True) - total_rows: int = int(stats["__total_rows"] or 0) - - actions: list[ActionRecord] = [] - - for orig_col, dtype in schema.items(): - col = rename_map[orig_col] - rule = cfg.rules.get(col) or cfg.rules.get(orig_col) - role = _infer_role(col, dtype, cfg, rule) - is_protected = bool( - (rule and rule.protected) or role in ("id", "target") - ) - - # 1. String whitespace repair - if dtype == pl.String and cfg.strip_strings: - ws_dirty = int(stats.get(f"{col}__ws_dirty") or 0) - if ws_dirty > 0: - actions.append( - ActionRecord( - column=col, - stage="strings", - operation="strip_whitespace", - affected_rows=ws_dirty, - rationale="Strip leading and trailing whitespace", - ) - ) - - # 2. Missing value planning - null_count = int(stats.get(f"{col}__nulls") or 0) - if null_count > 0: - if is_protected or cfg.strategy == "conservative": - actions.append( - ActionRecord( - column=col, - stage="missing", - operation="preserve", - affected_rows=null_count, - rationale=f"Protected role={role!r} or conservative strategy", - ) - ) - else: - null_ratio = null_count / max(total_rows, 1) - if cfg.strategy == "aggressive" and null_ratio >= 0.6: - actions.append( - ActionRecord( - column=col, - stage="drop_column", - operation="drop_column", - affected_rows=null_count, - rationale=f"Null ratio {null_ratio:.1%} exceeds aggressive drop threshold", - ) - ) - continue - - impute_choice = ( - rule.impute - if (rule and rule.impute is not None) - else cfg.default_impute - ) - if impute_choice == "auto": - impute_choice = "median" if dtype.is_numeric() else "mode" - - fill_val = stats.get(f"{col}__{impute_choice}") - if impute_choice != "preserve" and fill_val is not None: - actions.append( - ActionRecord( - column=col, - stage="missing", - operation="fill_null", - params={"value": fill_val, "method": impute_choice}, - affected_rows=null_count, - rationale=f"Impute {null_count} nulls via {impute_choice}={fill_val!r}", - ) - ) - else: - actions.append( - ActionRecord( - column=col, - stage="missing", - operation="preserve", - affected_rows=null_count, - rationale="Preserved nulls by policy", - ) - ) - - # 3. Outlier planning on numeric columns - if dtype.is_numeric() and not is_protected: - outlier_choice = ( - rule.outliers - if (rule and rule.outliers is not None) - else cfg.default_outliers - ) - q25 = stats.get(f"{col}__q25") - q75 = stats.get(f"{col}__q75") - if outlier_choice in ("clip", "flag") and q25 is not None and q75 is not None: - iqr = float(q75) - float(q25) - lower = float(q25) - cfg.iqr_factor * iqr - upper = float(q75) + cfg.iqr_factor * iqr - op = "clip_iqr" if outlier_choice == "clip" else "flag_iqr" - actions.append( - ActionRecord( - column=col, - stage="outliers", - operation=op, - params={"lower": lower, "upper": upper, "iqr_factor": cfg.iqr_factor}, - rationale=f"IQR bounds [{lower:.4g}, {upper:.4g}] (factor={cfg.iqr_factor})", - ) - ) - - return CleanPlan( - rename_map=rename_map, - actions=tuple(actions), - drop_duplicates=cfg.drop_duplicates, - total_rows=total_rows, - ) - - -@overload -def apply_plan( - df: FrameT, plan: CleanPlan, *, return_report: Literal[False] = False -) -> FrameT: ... - - -@overload -def apply_plan( - df: FrameT, plan: CleanPlan, *, return_report: Literal[True] -) -> tuple[FrameT, CleanReport]: ... - - -def apply_plan( - df: FrameT, plan: CleanPlan, *, return_report: bool = False -) -> FrameT | tuple[FrameT, CleanReport]: - is_eager = isinstance(df, pl.DataFrame) - lf: pl.LazyFrame = df.lazy() - - non_trivial_renames = {k: v for k, v in plan.rename_map.items() if k != v} - if non_trivial_renames: - lf = lf.rename(non_trivial_renames) - - dropped_cols: list[str] = [ - a.column for a in plan.actions if a.operation == "drop_column" - ] - if dropped_cols: - lf = lf.drop(dropped_cols) - - # Group column expressions by column so chained operations fuse into one .with_columns() - col_exprs: dict[str, pl.Expr] = {} - extra_exprs: list[pl.Expr] = [] - - for action in plan.actions: - col = action.column - if col in dropped_cols or action.operation == "preserve": - continue - - expr = col_exprs.get(col, pl.col(col)) - - if action.operation == "strip_whitespace": - expr = expr.str.strip_chars() - col_exprs[col] = expr - elif action.operation == "fill_null": - expr = expr.fill_null(pl.lit(action.params["value"])) - col_exprs[col] = expr - elif action.operation == "clip_iqr": - expr = expr.clip( - lower_bound=action.params["lower"], - upper_bound=action.params["upper"], - ) - col_exprs[col] = expr - elif action.operation == "flag_iqr": - lo = action.params["lower"] - hi = action.params["upper"] - extra_exprs.append( - (pl.col(col).lt(lo) | pl.col(col).gt(hi)) - .fill_null(False) - .alias(f"{col}_is_outlier") - ) - - all_exprs = [expr.alias(col) for col, expr in col_exprs.items()] + extra_exprs - if all_exprs: - lf = lf.with_columns(all_exprs) - - if plan.drop_duplicates: - # maintain_order=False allows Polars to stream deduplication without materializing - lf = lf.unique(maintain_order=False) - - output: FrameT = lf.collect(engine="streaming") if is_eager else lf # type: ignore[assignment] - - if not return_report: - return output - - report = CleanReport( - plan=plan, - renamed_columns=non_trivial_renames, - dropped_columns=tuple(dropped_cols), - total_actions=len(plan.actions), - ) - return output, report diff --git a/tests/test_polars_engine.py b/tests/test_polars_engine.py deleted file mode 100644 index 1abb06af..00000000 --- a/tests/test_polars_engine.py +++ /dev/null @@ -1,81 +0,0 @@ -import polars as pl - -from freshdata.polars_engine import ( - CleanConfig, - ColumnRule, - apply_plan, - suggest_plan, -) - - -def test_apply_plan_preserves_lazyframe_without_materializing() -> None: - raw_df = pl.DataFrame( - { - "Customer ID": ["001", "002", "003", "004", "005"], - "Age": [25, 30, None, 35, 40], - "Spend": [100.0, 120.0, 110.0, 115.0, 9999.0], - "Segment": [" Gold ", "Silver", None, "Gold", "Gold"], - "Churn": [0, 1, None, 0, 1], - } - ) - lf = raw_df.lazy() - - config = CleanConfig( - id_columns=("customer_id",), - target_column="churn", - default_impute="auto", - default_outliers="clip", - drop_duplicates=True, - ) - - plan = suggest_plan(lf, config=config) - - # Verify protected columns are never imputed or clipped - actions_by_col = {(a.column, a.stage): a for a in plan.actions} - assert actions_by_col[("churn", "missing")].operation == "preserve" - assert actions_by_col[("age", "missing")].operation == "fill_null" - assert actions_by_col[("age", "missing")].params["value"] == 32.5 - assert actions_by_col[("spend", "outliers")].operation == "clip_iqr" - - # Poison the LazyFrame after planning to prove apply_plan() never calls .collect() - poisoned_lf = lf.with_columns( - pl.col("Age").map_batches( - lambda _: (_ for _ in ()).throw(RuntimeError("Eager collect triggered!")), - return_dtype=pl.Int64, - ) - ) - lazy_repaired = apply_plan(poisoned_lf, plan) - assert isinstance(lazy_repaired, pl.LazyFrame) - - # Now execute on the clean LazyFrame and verify native Polars output - cleaned_lf, report = apply_plan(lf, plan, return_report=True) - assert isinstance(cleaned_lf, pl.LazyFrame) - assert "PYTHON_UDF" not in cleaned_lf.explain() - - result = cleaned_lf.collect() - assert result.columns == ["customer_id", "age", "spend", "segment", "churn"] - assert result["age"].null_count() == 0 - assert result["churn"].null_count() == 1 # Protected target preserved - assert result["spend"].max() < 9999.0 # Outlier clipped via precomputed IQR - assert report.total_actions == len(plan.actions) - - -def test_plan_diff_detects_strategy_changes() -> None: - lf = pl.LazyFrame({"score": [1.0, 2.0, None, 4.0, 100.0]}) - - conservative_plan = suggest_plan( - lf, CleanConfig(strategy="conservative", default_outliers="preserve") - ) - aggressive_plan = suggest_plan( - lf, - CleanConfig( - strategy="aggressive", - rules={"score": ColumnRule(impute="median", outliers="clip")}, - ), - ) - - diffs = conservative_plan.diff(aggressive_plan) - assert ("score", "outliers") in diffs - before_act, after_act = diffs[("score", "outliers")] - assert before_act is None - assert after_act is not None and after_act.operation == "clip_iqr" From dd7e0e4943f6f684596db3adfbc480bf496d632f Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:22:58 +0530 Subject: [PATCH 08/11] docs(readme): show real core-API output in the quickstart and diff examples - Print ActionRecord.action and parameters. Core actions carry no rationale. - Replace the polars_engine output (strip_whitespace, fill_null, preserve) with output captured from the current API. - Drop the outliers="flag" rule. The core backends ignore it. - Diff on default_outliers, not strategy. strategy does not change the plan. Use the {"added", "removed", "modified"} shape that CleanPlan.diff returns. Co-Authored-By: Claude Opus 5.5 --- README.md | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 03d87db7..6bb5ae93 100644 --- a/README.md +++ b/README.md @@ -45,24 +45,26 @@ lf = pl.scan_parquet("s3://bucket/raw_customer_events.parquet") config = fd.CleanConfig( id_columns=("customer_id",), target_column="churned", - default_impute="auto", # median for numeric, mode for string - default_outliers="clip", # precomputed IQR clipping + default_impute="auto", # median for numeric columns + default_outliers="preserve", drop_duplicates=True, # streaming-safe unique(maintain_order=False) rules={ "signup_age": fd.ColumnRule(impute="median", outliers="clip"), - "invoice_total": fd.ColumnRule(outliers="flag"), }, ) -# 3. Generate a dry-run repair plan (single 1-row streaming summary query) +# 3. Generate a dry-run repair plan (IQR bounds come from one 1-row summary query) plan = fd.suggest_plan(lf, config=config) for action in plan.actions: - print(f"[{action.stage}] {action.column}: {action.operation} ({action.rationale})") -# [strings] segment: strip_whitespace (Strip leading and trailing whitespace) -# [missing] signup_age: fill_null (Impute 412 nulls via median=34.0) -# [outliers] signup_age: clip_iqr (IQR bounds [18, 72] (factor=1.5)) -# [missing] churned: preserve (Protected role='target' or conservative strategy) + print(f"[{action.stage}] {action.column}: {action.action} {action.parameters}") +# Example output (abridged) for raw columns "Customer ID", "Signup Age", "Segment", ...: +# [column_names] Signup Age: normalize_name {'to': 'signup_age'} +# [missing] signup_age: impute_median {} +# [strings] segment: strip_strings {} +# [outliers] signup_age: clip_iqr {'lower': 27.125, 'upper': 48.125} +# [duplicates] *: drop_duplicates {} +# customer_id (id) and churned (target) get no repair actions. # 4. Compile the approved plan onto the LazyFrame (zero materialization) cleaned_lf, report = fd.apply_plan(lf, plan, return_report=True) @@ -81,17 +83,18 @@ df = cleaned_lf.collect(engine="streaming") Never guess how a configuration change alters your dataset. Compare two `CleanPlan` artifacts directly in code or CI: ```python -conservative = fd.suggest_plan( - lf, fd.CleanConfig(strategy="conservative", default_outliers="preserve") -) -aggressive = fd.suggest_plan( - lf, fd.CleanConfig(strategy="aggressive", default_outliers="clip") -) - -for (col, stage), (before, after) in conservative.diff(aggressive).items(): - before_op = before.operation if before else "none" - after_op = after.operation if after else "none" - print(f"{col} ({stage}): {before_op} -> {after_op}") +import dataclasses + +clipped = fd.suggest_plan(lf, dataclasses.replace(config, default_outliers="clip")) +diff = plan.diff(clipped) + +for action in diff["added"]: + print(f"+ [{action.stage}] {action.column}: {action.action} {action.parameters}") +for action in diff["removed"]: + print(f"- [{action.stage}] {action.column}: {action.action}") +for change in diff["modified"]: + print(f"~ [{change['stage']}] {change['column']}: {change['before']['parameters']} -> {change['after']['parameters']}") +# + [outliers] invoice_total: clip_iqr {'lower': 7.125, 'upper': 15.125} ``` --- From 23ba87b71a54d72d5d09b6c899b25eeea418e71b Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:32:12 +0530 Subject: [PATCH 09/11] refactor(core)!: delete ActionRecord.operation alias Nothing in the repository reads it since the README examples moved to ActionRecord.action. BREAKING CHANGE: ActionRecord.operation is removed. Use ActionRecord.action. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/freshdata/_core/models.py b/src/freshdata/_core/models.py index 14f1f229..55c2d0a5 100644 --- a/src/freshdata/_core/models.py +++ b/src/freshdata/_core/models.py @@ -34,10 +34,6 @@ class ActionRecord: rows_affected: int = 0 rationale: str = "" - @property - def operation(self) -> str: - return self.action - def to_dict(self) -> dict[str, Any]: return asdict(self) From 762f1c5fbbc2465a57f356d77c74112a53e04524 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:44:34 +0530 Subject: [PATCH 10/11] ci(benchmark-gate): install pytest-cov and set a subset coverage floor The job failed before running any test: pyproject addopts pass --cov flags, and the job did not install pytest-cov. With pytest-cov installed, the 3-file subset reaches 46% coverage, below the 90% repo-wide floor. Use the 40% floor that the property-invariant lane in ci-verification-gates.yml uses, so coverage stays on. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark-gate.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-gate.yml b/.github/workflows/benchmark-gate.yml index 6b8de0cc..257fd060 100644 --- a/.github/workflows/benchmark-gate.yml +++ b/.github/workflows/benchmark-gate.yml @@ -33,10 +33,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e . polars pandas pyarrow pytest + pip install -e . polars pandas pyarrow pytest pytest-cov - name: Run gate & engine unit tests - run: pytest tests/test_backend_honesty.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q + # pyproject addopts enforce 90% repo-wide coverage; this 3-file subset + # keeps coverage on with the lower floor that other subset lanes use. + run: pytest tests/test_backend_honesty.py tests/test_benchmark_harness.py tests/test_perf_gate.py -q --cov-fail-under=40 - name: Record Candidate (PR Head) Benchmark run: | From 1b6acef142169b0349c4ac9db82570bf19dfb06f Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 13:50:18 +0530 Subject: [PATCH 11/11] ci(benchmark-gate): force the return checkout to the PR head The base step copies the PR's benchmarks/ over the base tree to time the same harness on both sides. That leaves tracked files modified, so the plain `git checkout ` in the compare step aborts for every PR that changes a file under benchmarks/. --force discards only that overlay; the head commit holds the same files. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark-gate.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-gate.yml b/.github/workflows/benchmark-gate.yml index 257fd060..2263300e 100644 --- a/.github/workflows/benchmark-gate.yml +++ b/.github/workflows/benchmark-gate.yml @@ -63,7 +63,9 @@ jobs: - name: Evaluate Regression Gate run: | - git checkout "${{ github.event.pull_request.head.sha }}" + # The base step copied the PR's benchmarks/ over the base tree; --force + # discards that overlay so PRs that change benchmarks/ can switch back. + git checkout --force "${{ github.event.pull_request.head.sha }}" python -m benchmarks.perf_gate compare \ --baseline /tmp/baseline_metrics.json \ --candidate /tmp/candidate_metrics.json \