From 8b222ea36d43cb5d85e1de38f6676316e2171a7c Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:30:14 +0530 Subject: [PATCH 01/17] refactor(core): share the protection check and speed up string cleaning - Add stages._is_protected so pandas and Polars use one ID/target/protected predicate instead of three copies. - Move the per-cell string cleaner to module level, check for str first, and replace re.sub with str.split/join. Output is identical (checked against every Unicode whitespace character and 1,500 fuzzed configs). - Rewrite ARCHITECTURE.md to describe the v3 code that exists. pandas clean() is about 33% faster on 300k rows x 11 columns. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 191 ++++++++++++++++---------------- src/freshdata/_core/backends.py | 15 ++- src/freshdata/_core/stages.py | 46 +++++--- tests/test_stages.py | 26 +++++ 4 files changed, 158 insertions(+), 120 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7dc5b98e..3227096a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,111 +1,110 @@ # Architecture -This is a map for contributors: what lives where in `src/freshdata`, how a -`clean()` call flows through the code, and where to add new behavior. It is -deliberately high-level — read it once before your first code change, then let -the module docstrings and tests fill in the detail. +A map for contributors: what lives where in `src/freshdata`, how a call flows +through the code, and where to add behavior. Read it once before your first +change. Module docstrings and tests hold the detail. -For *user*-facing explanations of features, see the -[documentation site](https://freshcode-org.github.io/freshdata/). This file is -about the *code*. +For user-facing docs, see the +[documentation site](https://freshcode-org.github.io/freshdata/). ## The one-sentence version -`fd.clean(df)` profiles every column, asks a rule-based **decision engine** which -action each column needs, applies those actions as small vectorized **steps**, -and returns the cleaned frame plus a **report** that explains every change. +`fd.suggest_plan(df)` builds an immutable, diffable `CleanPlan` of +`ActionRecord`s. `fd.apply_plan(df, plan)` executes that plan with the backend +that matches the frame type. `fd.clean(df)` runs both in sequence. -## Core flow +## Layout ``` -fd.clean(df) # api.py — the public entry point - │ - ▼ - Cleaner # cleaner.py — orchestrates the pipeline - │ - ├─ profile the frame # profile.py — per-column stats & inferred role - │ - ├─ decision engine # engine/ — pick an action per column - │ ├─ missing.py # imputation strategy - │ ├─ outliers.py # outlier handling - │ ├─ context.py # column-role / context inference - │ └─ model_select.py # when to use model-based imputation - │ - ├─ apply cleaning steps # steps/ — pure vectorized transforms - │ ├─ missing.py dtypes.py duplicates.py - │ ├─ outliers.py strings.py columns.py - │ └─ prune.py memory.py - │ - └─ build the report # report.py — every Action, with rationale, - # risk, confidence, affected-cell count +src/freshdata/ +├── __init__.py public API re-exports +├── _core/ +│ ├── models.py frozen dataclasses: ColumnRule, CleanConfig, ActionRecord, +│ │ CleanPlan (+ diff), ColumnProfile, ProfileReport, CleanReport +│ ├── engine.py public functions + Cleaner; config defaults; report assembly +│ ├── backends.py PandasBackend, PolarsBackend, resolve_backend() +│ └── stages.py pandas planning stages + shared helpers +│ (_normalize_name, normalize_config, _is_protected) +└── cli/migrate.py freshdata-migrate: libcst codemod for 2.x -> 3.x scripts ``` -Two invariants hold everywhere in this path and are enforced in review (see -[CONTRIBUTING.md](CONTRIBUTING.md)): +Dependency direction is strictly one way: +`engine -> backends -> stages -> models`. Inside `_core`, no module imports `engine`. -1. **Everything is reported.** A transformation that changes data must record an - `Action` (see `report.py`) with an affected-cell count. -2. **Vectorized only.** No row-wise `apply` or Python loops over rows in the - cleaning path. +## Data flow -## Top-level modules +``` +fd.clean(df, cfg) + ├─ suggest_plan(df, cfg) + │ └─ resolve_backend(df) dispatch on frame type + │ ├─ PandasBackend.suggest_plan + │ │ normalize_config(cfg) + │ │ for stage in (column_names, strings, dtypes, + │ │ duplicates, missing, outliers): + │ │ df, actions = stage(df, cfg) eager; each stage sees + │ │ the previous stage's output + │ └─ PolarsBackend.suggest_plan + │ schema only + at most ONE 1-row aggregate query + │ (IQR quantiles for clip columns) + │ -> CleanPlan(actions) + └─ apply_plan(df, plan, cfg) + ├─ PandasBackend.apply_plan renames, then actions in plan order (eager) + └─ PolarsBackend.apply_plan renames -> one fused with_columns(...) -> + optional unique(); never collects +``` -| Module | Responsibility | -|---|---| -| `api.py` | The public convenience functions: `clean`, `clean_csv`, `profile`, `plan`, `learn`, `mask_dataframe`, `link`. Start here to trace any public call. | -| `cleaner.py` | The `Cleaner` front-end and the pipeline that wires profiling → engine → steps → report. | -| `profile.py` | Read-only profiling: what is in a frame and what `clean()` *would* do, with no mutation. | -| `plan.py` / `repairplan.py` | Dry-run plans (`plan.py`) and executable, reviewable repair plans (`repairplan.py`). | -| `report.py`, `result.py`, `_reportframe.py` | The report objects returned to users — `Action` records, summaries, `to_dict()`. | -| `provenance.py` | Provenance-aware cleaning for OCR / PDF / document-derived tables. | -| `guard.py` | The hard safety gate that physically protects columns from modification. | -| `findings.py`, `fieldcheck.py`, `textclean.py`, `textlint.py` | Per-cell validation (`fieldcheck`), field-aware text cleaning (`textclean`), and the normalized finding model (`findings`). | -| `insight.py`, `stakeholder.py`, `explain.py` | Presentation layers: action-oriented insight output, business-language stakeholder summaries, and per-decision explanations. | -| `config.py`, `_util.py`, `_sentinels.py` | Configuration and shared internals. | - -## Subpackages - -Each subpackage has a one-line docstring at the top of its `__init__.py` — -these summaries come from that code. - -| Package | What it does | +## Invariants + +1. **Plans are data.** `CleanPlan` and `ActionRecord` are frozen. A plan can be + serialized (`to_dict`), diffed (`diff`), and edited before it is applied. +2. **Lazy stays lazy.** `PolarsBackend.apply_plan` must not call `.collect()`, + `.to_pandas()`, or Python UDFs. `tests/test_backend_honesty.py` enforces this. +3. **Protection lives in one place.** `stages._is_protected` decides which + columns the missing and outlier logic must skip. Do not copy the predicate. +4. **pandas is required. Polars is optional.** `resolve_backend` imports Polars + only for Polars frames. + +## Where to add behavior + +| Change | Edit | |---|---| -| `engine/` | The rule-based decision engine behind `strategy=` — picks an action per column. | -| `steps/` | Individual cleaning steps, each a pure vectorized function. | -| `imputation/` | Internal imputation engines (mean/median/mode → KNN/model-based with the `ml` extra). | -| `domains/` | Domain-specific validator packs (finance, healthcare, energy, retail, transport, education, agriculture, media) plus reference data. **The most self-contained place to start** — see [CONTRIBUTING_DOMAINS.md](CONTRIBUTING_DOMAINS.md). | -| `semantic/` | The offline Semantic Cleaning Layer — experts, scoring, routing. Off by default. | -| `context/` | The deterministic natural-language context compiler (plain-English rules → policy). | -| `enterprise/` | Clustering / entity resolution, trust scoring, lineage, and PII masking. | -| `compliance/` | Maps a `CleanReport` to named control frameworks (21 CFR Part 11, GDPR, HIPAA, SOX, ALCOA+). | -| `execution/` | Pluggable out-of-core backends (Polars / DuckDB / Spark) behind a shared logical plan. | -| `streaming/` | Micro-batch and time-series-aware cleaning in bounded memory. | -| `parsers/` | Structural format readers (HL7 v2, GPX, SDMX, EDIFACT, FHIR). | -| `render/` | Interactive HTML rendering of report objects (the `viz` extra). | -| `models/` | Optional local model registry and runtime (`fd.models`). | -| `learning/` | Paired-data learning: `fd.learn` and reusable `.fdprofile` profiles. | -| `integrations/` | Run `clean` + a trust gate inside orchestrators (Dagster / Airflow / dbt). | -| `adapters/` | Optional framework adapters (Polars, etc.). | -| `experimental/` | Features whose APIs may change between minor releases (e.g. the AI Copilot). | -| `plugins.py` + `testing.py` | The entry-point plugin system and `fd.testing` contract helpers for third-party experts/backends/validators. | - -## Where do I add…? - -- **A new cleaning behavior** → a step in `steps/`, wired through the engine in - `engine/`, with an `Action` recorded in the report. Add the "does not fire - when it shouldn't" test. -- **A new domain validator** → a pack in `domains/`. Follow - [CONTRIBUTING_DOMAINS.md](CONTRIBUTING_DOMAINS.md) — this path is designed to - be self-contained and is the friendliest first code contribution. -- **A new imputation method** → `imputation/`, selected from `engine/model_select.py`. -- **A new execution backend** → `execution/` + a plugin registration; see - [`docs/plugins.md`](https://freshcode-org.github.io/freshdata/plugins/). -- **A new output/report surface** → `render/`, `insight.py`, or `stakeholder.py`. -- **A format parser** → `parsers/`. - -## What to read next - -- [CONTRIBUTING.md](CONTRIBUTING.md) — setup, the checks CI runs, and the PR workflow. -- [CONTRIBUTING_DOMAINS.md](CONTRIBUTING_DOMAINS.md) — the deep dive for domain packs. -- [Contributor roadmap](docs/community/contributor-roadmap.md) — tasks grouped by difficulty. -- The module docstring and the tests next to any file you plan to change. +| New pandas planning rule | a `stage_*` function in `stages.py`, then add it to the tuple in `PandasBackend.suggest_plan` | +| Execute a new action | both `PandasBackend.apply_plan` and `PolarsBackend.apply_plan` | +| New config option | `models.py`, then every backend that must honor it | +| New frame type | a backend class with `profile` / `suggest_plan` / `apply_plan`, plus a branch in `resolve_backend` | + +## Known issues + +These are real defects in the current behavior. They are recorded here, not +fixed, because each fix changes output. + +- **pandas: plan and apply disagree on strings.** `stage_strings` collapses + inner whitespace and nulls `"nan"`/`"none"`/`"null"` tokens. The + `strip_strings` executor in `apply_plan` only strips. Plan statistics + (mode values, medians, `rows_affected`) come from data that `apply_plan` + does not produce. +- **pandas: `apply_plan` nulls non-string cells in object columns.** A mixed + column `[1, " a "]` becomes `[None, "a"]`. +- **pandas: protection is partial.** `stage_strings` and `stage_dtypes` modify + ID, target, and `protected=True` columns. Polars skips them. +- **Backends resolve rules differently.** pandas falls back to the defaults + per field (`rule.impute or default_impute`). Polars falls back only when the + column has no rule at all. `ColumnRule(outliers="clip")` therefore disables + default imputation on Polars only. +- **Polars ignores some actions without warning.** `drop_nulls`, + `drop_outliers`, `parse_currency`, and `impute_mode` are not executed. +- **Accepted but not implemented:** `ColumnRule.role`, `impute="ffill"`, + `outliers="flag"`, `CleanConfig.strategy`, `CleanConfig.strict_backend`. +- **Polars `LazyFrame` profiles report zeros** for row, null, and unique + counts instead of "unknown". +- **`rows_affected` overcounts** for `normalize_name`, `strip_strings`, + `parse_currency`, and `clip_iqr` (it reports the row count, not the cells + changed). Polars reports 0 for every action. + +## Performance notes + +- The pandas path is eager. `clean()` transforms the frame twice (once to plan, + once to apply), and every stage copies the frame. For large data, use a + Polars `LazyFrame`. +- The pandas string stages run one Python call per cell. `_clean_str` is the + hottest function in `clean()`. Keep it allocation-light. diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 30e11c1b..1a5af471 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -10,6 +10,7 @@ ProfileReport, ) from freshdata._core.stages import ( + _is_protected, _normalize_name, normalize_config, stage_column_names, @@ -23,6 +24,12 @@ if TYPE_CHECKING: import polars as pl +def _strip_or_none(x: object) -> str | None: + # Non-string cells become None. This matches the original lambda; see ARCHITECTURE.md "Known issues". + if isinstance(x, str): + return x.strip() or None + return None + class PandasBackend: def profile(self, df: pd.DataFrame) -> ProfileReport: cols = {} @@ -71,9 +78,7 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. if a.stage == "strings" and a.action == "strip_strings": if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): - out[col] = out[col].astype(object).apply( - lambda x: x.strip() if isinstance(x, str) and x.strip() else None - ) + out[col] = out[col].astype(object).apply(_strip_or_none) elif a.stage == "dtypes" and a.action == "parse_currency": cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") @@ -131,11 +136,9 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle name = _normalize_name(orig) if cfg_eff.normalize_names else orig if name != orig: actions.append(ActionRecord("column_names", orig, "normalize_name", {"to": name})) - if name in cfg_eff.id_columns or name == cfg_eff.target_column or name.endswith("_id") or name == "id": + if _is_protected(name, cfg_eff): continue rule = cfg_eff.rules.get(name) - if rule and rule.protected: - continue if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): actions.append(ActionRecord("strings", name, "strip_strings", {})) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 7a638bbb..9aa9af6c 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -8,11 +8,35 @@ from freshdata._core.models import ActionRecord, CleanConfig, ColumnRule CURRENCY_REGEX = re.compile(r"^\s*[-+]?\s*[\$€£¥]?\s*[\d,]+(?:\.\d+)?\s*[\$€£¥]?\s*$") +_NULL_TOKENS = frozenset(("nan", "none", "null")) def _normalize_name(name: str) -> str: s = re.sub(r"[^\w\s]", "", name).strip().lower() return re.sub(r"[\s\-]+", "_", s) +def _is_protected(col: object, cfg: CleanConfig) -> bool: + """Single source of truth for columns the missing/outlier stages must not touch. + + ``cfg`` must already be normalized (see ``normalize_config``). + """ + name = str(col) + if col in cfg.id_columns or col == cfg.target_column or name.endswith("_id") or name == "id": + return True + rule = cfg.rules.get(col) + return rule is not None and rule.protected + +def _clean_str(x: object) -> object: + # Hot path: runs once per cell. str.split() and re's \s share the same + # Unicode whitespace set, so this equals re.sub(r"\s+", " ", x.strip()). + if isinstance(x, str): + val = " ".join(x.split()) + if not val or val.lower() in _NULL_TOKENS: + return None + return val + if pd.isna(x): + return None + return x + def normalize_config(cfg: CleanConfig) -> CleanConfig: if not cfg.normalize_names: return cfg @@ -41,18 +65,8 @@ def stage_strings(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis out = df.copy() for col in out.columns: if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): - s = out[col].astype(object) - def _clean_str(x): - if pd.isna(x) or x is None: - return None - if isinstance(x, str): - val = re.sub(r"\s+", " ", x.strip()) - if not val or val.lower() in ("nan", "none", "null"): - return None - return val - return x - - out[col] = s.apply(_clean_str) + # Series.apply (not .str) keeps pandas' result-dtype inference unchanged. + out[col] = out[col].astype(object).apply(_clean_str) actions.append(ActionRecord("strings", str(col), "strip_strings", {}, len(out))) return out, actions @@ -89,11 +103,9 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis out = df.copy() cfg_eff = normalize_config(cfg) for col in out.columns: - if col in cfg_eff.id_columns or col == cfg_eff.target_column or str(col).endswith("_id") or str(col) == "id": + if _is_protected(col, cfg_eff): continue rule = cfg_eff.rules.get(col, ColumnRule()) - if rule.protected: - continue impute_strategy = rule.impute or (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) null_count = int(out[col].isna().sum()) if impute_strategy == "drop" and (null_count > 0 or len(out) == 0): @@ -127,11 +139,9 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li out = df.copy() cfg_eff = normalize_config(cfg) for col in out.columns: - if col in cfg_eff.id_columns or col == cfg_eff.target_column or str(col).endswith("_id") or str(col) == "id": + if _is_protected(col, cfg_eff): continue rule = cfg_eff.rules.get(col, ColumnRule()) - if rule.protected: - continue strategy = rule.outliers or cfg_eff.default_outliers if strategy not in ("clip", "drop") or not pd.api.types.is_numeric_dtype(out[col]): continue diff --git a/tests/test_stages.py b/tests/test_stages.py index 05fb3bd2..c2d6a4d0 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -123,3 +123,29 @@ def test_stage_outliers_strategies(strategy, values, expected, actions): out, acts = stage_outliers(df, CleanConfig(rules={"x": ColumnRule(outliers=strategy)})) assert out["x"].tolist() == pytest.approx(expected, nan_ok=True) assert [a.action for a in acts] == actions + + +def test_clean_str_matches_regex_whitespace_semantics(): + import re + import sys + + import numpy as np + + from freshdata._core.stages import _clean_str + + # The fast path uses str.split(); it must equal the original re.sub(r"\s+", " ", x.strip()). + for ws in (chr(c) for c in range(sys.maxunicode + 1) if re.match(r"\s", chr(c))): + raw = f"{ws}a{ws}{ws}b{ws}" + assert _clean_str(raw) == re.sub(r"\s+", " ", raw.strip()) + assert [_clean_str(x) for x in (" ", "NULL", " none ", None, np.nan, pd.NA, 3, b"b")] == [ + None, None, None, None, None, None, 3, b"b", + ] + + +def test_is_protected_covers_ids_target_suffix_and_rules(): + from freshdata._core.stages import _is_protected + + cfg = CleanConfig(id_columns=("key",), target_column="y", rules={"p": ColumnRule(protected=True)}) + assert [_is_protected(c, cfg) for c in ("key", "y", "user_id", "id", "p", "x", 7)] == [ + True, True, True, True, True, False, False, + ] From 50de81f20a4f2ae65531b3678da60751a5194935 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:30:47 +0530 Subject: [PATCH 02/17] fix(pandas): use one strip_strings executor for planning and apply suggest_plan collapsed inner whitespace and turned "nan"/"none"/"null" text into missing values. apply_plan only stripped. Plan statistics (mode values, medians, rows_affected) therefore described data that apply_plan never produced. apply_plan also replaced every non-string cell in an object column with None. Both paths now call stages._strip_str: strip surrounding whitespace, turn empty strings into None, keep non-string cells. This matches the README (".str.strip_chars()") and the Polars executor. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 9 ++------- src/freshdata/_core/stages.py | 21 +++++++++------------ tests/test_stages.py | 29 +++++++++++++++++------------ 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 1a5af471..879ae5e3 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -11,6 +11,7 @@ ) from freshdata._core.stages import ( _is_protected, + _strip_str, _normalize_name, normalize_config, stage_column_names, @@ -24,12 +25,6 @@ if TYPE_CHECKING: import polars as pl -def _strip_or_none(x: object) -> str | None: - # Non-string cells become None. This matches the original lambda; see ARCHITECTURE.md "Known issues". - if isinstance(x, str): - return x.strip() or None - return None - class PandasBackend: def profile(self, df: pd.DataFrame) -> ProfileReport: cols = {} @@ -78,7 +73,7 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. if a.stage == "strings" and a.action == "strip_strings": if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): - out[col] = out[col].astype(object).apply(_strip_or_none) + out[col] = out[col].astype(object).apply(_strip_str) elif a.stage == "dtypes" and a.action == "parse_currency": cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 9aa9af6c..5ec082d7 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -8,7 +8,6 @@ from freshdata._core.models import ActionRecord, CleanConfig, ColumnRule CURRENCY_REGEX = re.compile(r"^\s*[-+]?\s*[\$€£¥]?\s*[\d,]+(?:\.\d+)?\s*[\$€£¥]?\s*$") -_NULL_TOKENS = frozenset(("nan", "none", "null")) def _normalize_name(name: str) -> str: s = re.sub(r"[^\w\s]", "", name).strip().lower() @@ -25,17 +24,15 @@ def _is_protected(col: object, cfg: CleanConfig) -> bool: rule = cfg.rules.get(col) return rule is not None and rule.protected -def _clean_str(x: object) -> object: - # Hot path: runs once per cell. str.split() and re's \s share the same - # Unicode whitespace set, so this equals re.sub(r"\s+", " ", x.strip()). +def _strip_str(x: object) -> object: + """The one strip_strings executor, shared by planning and pandas apply_plan. + + Strips surrounding whitespace and turns empty strings into None. Missing + values become None. Other non-string cells are returned unchanged. + """ if isinstance(x, str): - val = " ".join(x.split()) - if not val or val.lower() in _NULL_TOKENS: - return None - return val - if pd.isna(x): - return None - return x + return x.strip() or None + return None if pd.isna(x) else x def normalize_config(cfg: CleanConfig) -> CleanConfig: if not cfg.normalize_names: @@ -66,7 +63,7 @@ def stage_strings(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis for col in out.columns: if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): # Series.apply (not .str) keeps pandas' result-dtype inference unchanged. - out[col] = out[col].astype(object).apply(_clean_str) + out[col] = out[col].astype(object).apply(_strip_str) actions.append(ActionRecord("strings", str(col), "strip_strings", {}, len(out))) return out, actions diff --git a/tests/test_stages.py b/tests/test_stages.py index c2d6a4d0..494996c6 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -125,21 +125,26 @@ def test_stage_outliers_strategies(strategy, values, expected, actions): assert [a.action for a in acts] == actions -def test_clean_str_matches_regex_whitespace_semantics(): - import re - import sys - +def test_strip_str_strips_and_keeps_non_strings(): import numpy as np - from freshdata._core.stages import _clean_str + from freshdata._core.stages import _strip_str - # The fast path uses str.split(); it must equal the original re.sub(r"\s+", " ", x.strip()). - for ws in (chr(c) for c in range(sys.maxunicode + 1) if re.match(r"\s", chr(c))): - raw = f"{ws}a{ws}{ws}b{ws}" - assert _clean_str(raw) == re.sub(r"\s+", " ", raw.strip()) - assert [_clean_str(x) for x in (" ", "NULL", " none ", None, np.nan, pd.NA, 3, b"b")] == [ - None, None, None, None, None, None, 3, b"b", - ] + cells = (" a b ", " ", "null", None, np.nan, pd.NA, 3, 2.5, b"b") + assert [_strip_str(x) for x in cells] == ["a b", None, "null", None, None, None, 3, 2.5, b"b"] + + +def test_plan_statistics_match_applied_strings(): + import freshdata as fd + + df = pd.DataFrame({"s": [" x ", "x", "null", " ", None], "mixed": pd.Series([1, " a ", 2.5, None, "b"], dtype=object)}) + cfg = CleanConfig(rules={"s": ColumnRule(impute="mode")}) + plan = fd.suggest_plan(df, cfg) + out = fd.apply_plan(df, plan, cfg) + mode = next(a for a in plan.actions if a.action == "impute_mode") + assert mode.rows_affected == 2 # " " and None, the cells apply_plan leaves missing + assert out["s"].tolist() == ["x", "x", "null", "x", "x"] + assert out["mixed"].tolist()[:3] == [1, "a", 2.5] def test_is_protected_covers_ids_target_suffix_and_rules(): From bfe0bc76acd360b1ff14c1ebaf3baf9765a00bf6 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:31:01 +0530 Subject: [PATCH 03/17] fix(pandas): keep the mode value's type in impute_mode plans The plan stored str(mode). apply_plan then wrote "1.0" into a float column and turned it into an object column. Store the plain Python scalar instead. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/stages.py | 3 ++- tests/test_stages.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 5ec082d7..cb03a30a 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -127,8 +127,9 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis mode_vals = out[col].mode() if not mode_vals.empty and not pd.isna(mode_vals.iloc[0]): val = mode_vals.iloc[0] + val = val.item() if isinstance(val, np.generic) else val # plain Python scalar, original type out[col] = out[col].fillna(val) - actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": str(val)}, null_count)) + actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": val}, null_count)) return out, actions def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: diff --git a/tests/test_stages.py b/tests/test_stages.py index 494996c6..76beea6d 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -154,3 +154,14 @@ def test_is_protected_covers_ids_target_suffix_and_rules(): assert [_is_protected(c, cfg) for c in ("key", "y", "user_id", "id", "p", "x", 7)] == [ True, True, True, True, True, False, False, ] + + +def test_mode_imputation_keeps_value_type(): + import freshdata as fd + + df = pd.DataFrame({"v": [1.0, 1.0, None, 2.0]}) + cfg = CleanConfig(rules={"v": ColumnRule(impute="mode")}) + plan = fd.suggest_plan(df, cfg) + assert plan.actions[-1].parameters == {"value": 1.0} + out = fd.apply_plan(df, plan, cfg) + assert out["v"].dtype == "float64" and out["v"].tolist() == [1.0, 1.0, 1.0, 2.0] From c13e96a7c4753f157fadbd982b63c94227a391dd Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:31:22 +0530 Subject: [PATCH 04/17] fix(pandas): skip protected columns in the strings and dtypes stages ID columns, the target column, *_id columns and protected=True rules were stripped and currency-parsed on pandas. The Polars planner already skipped them. Both stages now use stages._is_protected. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/stages.py | 6 ++++++ tests/test_stages.py | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index cb03a30a..3a913409 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -60,7 +60,10 @@ def stage_strings(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis return df, [] actions: List[ActionRecord] = [] out = df.copy() + cfg_eff = normalize_config(cfg) for col in out.columns: + if _is_protected(col, cfg_eff): + continue if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): # Series.apply (not .str) keeps pandas' result-dtype inference unchanged. out[col] = out[col].astype(object).apply(_strip_str) @@ -72,7 +75,10 @@ def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List return df, [] actions: List[ActionRecord] = [] out = df.copy() + cfg_eff = normalize_config(cfg) for col in out.columns: + if _is_protected(col, cfg_eff): + continue if pd.api.types.is_object_dtype(out[col]) or pd.api.types.is_string_dtype(out[col]): sample = out[col].dropna().head(20).astype(str) if sample.empty: diff --git a/tests/test_stages.py b/tests/test_stages.py index 76beea6d..8927da93 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -165,3 +165,13 @@ def test_mode_imputation_keeps_value_type(): assert plan.actions[-1].parameters == {"value": 1.0} out = fd.apply_plan(df, plan, cfg) assert out["v"].dtype == "float64" and out["v"].tolist() == [1.0, 1.0, 1.0, 2.0] + + +def test_protected_id_and_target_columns_are_not_stripped_or_parsed(): + import freshdata as fd + + df = pd.DataFrame({"p": [" $1,000 ", "$2", "$3"], "user_id": [" 7 ", "8", "9"], "free": [" $1 ", "$2", "$3"]}) + out = fd.clean(df, CleanConfig(rules={"p": ColumnRule(protected=True)})) + assert out["p"].tolist() == [" $1,000 ", "$2", "$3"] + assert out["user_id"].tolist() == [" 7 ", "8", "9"] + assert out["free"].tolist() == [1, 2, 3] From f443da723ed305470fe4d13d4266447f9fb0f9fd Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:31:48 +0530 Subject: [PATCH 05/17] fix(polars): fall back to config defaults per rule field, like pandas Polars used a column's rule for both impute and outliers whenever the rule existed, so ColumnRule(outliers="clip") silently turned off default imputation (and the reverse). Both backends now resolve each field through stages._impute_strategy and stages._outlier_strategy. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 7 ++++--- src/freshdata/_core/stages.py | 19 ++++++++++++++----- tests/test_backend_honesty.py | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 879ae5e3..45f2f5cb 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -10,7 +10,9 @@ ProfileReport, ) from freshdata._core.stages import ( + _impute_strategy, _is_protected, + _outlier_strategy, _strip_str, _normalize_name, normalize_config, @@ -133,14 +135,13 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle actions.append(ActionRecord("column_names", orig, "normalize_name", {"to": name})) if _is_protected(name, cfg_eff): continue - rule = cfg_eff.rules.get(name) if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): actions.append(ActionRecord("strings", name, "strip_strings", {})) if dtype.is_numeric(): # Mirror the pandas stage order: IQR bounds see the imputed column. base = pl.col(orig) - impute_strat = rule.impute if rule else (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) + impute_strat = _impute_strategy(name, cfg_eff) if impute_strat in ("median", "auto"): actions.append(ActionRecord("missing", name, "impute_median", {})) base = base.fill_null(base.median()) @@ -148,7 +149,7 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle actions.append(ActionRecord("missing", name, "impute_mean", {})) base = base.fill_null(base.mean()) - outlier_strat = rule.outliers if rule else cfg_eff.default_outliers + outlier_strat = _outlier_strategy(name, cfg_eff) if outlier_strat == "clip": clip_cols[name] = base if clip_cols: diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 3a913409..b345844c 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -5,7 +5,7 @@ from typing import Tuple, List import numpy as np import pandas as pd -from freshdata._core.models import ActionRecord, CleanConfig, ColumnRule +from freshdata._core.models import ActionRecord, CleanConfig CURRENCY_REGEX = re.compile(r"^\s*[-+]?\s*[\$€£¥]?\s*[\d,]+(?:\.\d+)?\s*[\$€£¥]?\s*$") @@ -24,6 +24,17 @@ def _is_protected(col: object, cfg: CleanConfig) -> bool: rule = cfg.rules.get(col) return rule is not None and rule.protected +def _impute_strategy(col: object, cfg: CleanConfig) -> str | None: + """Per-field fallback: a rule that leaves ``impute`` unset uses the default.""" + rule = cfg.rules.get(col) + default = None if cfg.default_impute == "preserve" else cfg.default_impute + return (rule.impute if rule else None) or default + +def _outlier_strategy(col: object, cfg: CleanConfig) -> str: + """Per-field fallback: a rule that leaves ``outliers`` unset uses the default.""" + rule = cfg.rules.get(col) + return (rule.outliers if rule else None) or cfg.default_outliers + def _strip_str(x: object) -> object: """The one strip_strings executor, shared by planning and pandas apply_plan. @@ -108,8 +119,7 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis for col in out.columns: if _is_protected(col, cfg_eff): continue - rule = cfg_eff.rules.get(col, ColumnRule()) - impute_strategy = rule.impute or (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) + impute_strategy = _impute_strategy(col, cfg_eff) null_count = int(out[col].isna().sum()) if impute_strategy == "drop" and (null_count > 0 or len(out) == 0): orig_len = len(out) @@ -145,8 +155,7 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li for col in out.columns: if _is_protected(col, cfg_eff): continue - rule = cfg_eff.rules.get(col, ColumnRule()) - strategy = rule.outliers or cfg_eff.default_outliers + strategy = _outlier_strategy(col, cfg_eff) if strategy not in ("clip", "drop") or not pd.api.types.is_numeric_dtype(out[col]): continue s = out[col].dropna() diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index a052cab8..e29ecf1f 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -119,3 +119,17 @@ def test_lazy_plan_diff_detects_rule_changes(): diff = conservative.diff(aggressive) assert [(a.column, a.action) for a in diff["added"]] == [("score", "clip_iqr")] assert diff["removed"] == [] and diff["modified"] == [] + + +@pytest.mark.parametrize("rule, default", [ + ({"outliers": "clip"}, {"default_impute": "median"}), + ({"impute": "mean"}, {"default_outliers": "clip"}), +]) +def test_rules_fall_back_to_defaults_per_field_on_both_backends(rule, default): + import freshdata as fd + + data = {"v": [1.0, None, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(rules={"v": fd.ColumnRule(**rule)}, **default) + pd_actions = {a.action for a in fd.suggest_plan(pd.DataFrame(data), cfg).actions} + pl_actions = {a.action for a in fd.suggest_plan(pl.DataFrame(data), cfg).actions} + assert pd_actions == pl_actions and len(pl_actions) == 2 From 07b49820bfcc62acf9ace9cb2b9fc36f5e3bfb0b Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:34:13 +0530 Subject: [PATCH 06/17] fix: run every plan action on both backends and enforce strict_backend Polars apply_plan ignored drop_nulls, drop_outliers, parse_currency and impute_mode, and both backends skipped unknown actions and malformed parameters without a word. pandas also skipped impute actions that carry no precomputed value, which is every impute action a Polars plan contains. - Polars: native executors for the four missing actions. Row filters run after the fused with_columns. No collect and no Python UDFs. - pandas: compute median/mean/mode when the plan has no value. - Polars planner: honor impute="mode", impute="drop" and outliers="drop". - Both: an action that cannot run raises ValueError when strict_backend=True (the default, as the README promises). With strict_backend=False it is skipped with a UserWarning. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 179 +++++++++++++++++++++----------- tests/test_backend_honesty.py | 56 ++++++++++ 2 files changed, 176 insertions(+), 59 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 45f2f5cb..ce1b8bfa 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -1,5 +1,7 @@ # src/freshdata/_core/backends.py from __future__ import annotations +import warnings +from numbers import Real from typing import TYPE_CHECKING, Any import pandas as pd from freshdata._core.models import ( @@ -27,6 +29,29 @@ if TYPE_CHECKING: import polars as pl +def _reject(a: ActionRecord, cfg: CleanConfig, backend: str, reason: str = "unknown action") -> None: + """Refuse an action the backend cannot run: raise when strict, else warn and skip.""" + msg = f"{backend} backend cannot run {a.stage}/{a.action} on column {a.column!r}: {reason}" + if cfg.strict_backend: + raise ValueError(f"{msg}. Fix the plan, or set CleanConfig(strict_backend=False) to skip it with a warning.") + warnings.warn(f"{msg}. Skipped.", UserWarning, stacklevel=4) + +def _bounds(a: ActionRecord) -> tuple[float, float] | None: + lower, upper = a.parameters.get("lower"), a.parameters.get("upper") + if isinstance(lower, Real) and isinstance(upper, Real): + return float(lower), float(upper) + return None + +def _first_mode(s: pd.Series) -> Any: + modes = s.mode() + return None if modes.empty else modes.iloc[0] + +_PANDAS_FILLS = { + ("missing", "impute_median"): lambda s: s.median(), + ("missing", "impute_mean"): lambda s: s.mean(), + ("missing", "impute_mode"): _first_mode, +} + class PandasBackend: def profile(self, df: pd.DataFrame) -> ProfileReport: cols = {} @@ -64,43 +89,53 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. out = out.rename(columns=renames) for a in plan.actions: - if a.stage == "duplicates" and a.action == "drop_duplicates": + key = (a.stage, a.action) + if key == ("column_names", "normalize_name"): + continue + if key == ("duplicates", "drop_duplicates"): mask = ~out.duplicated().to_numpy() out = out[mask].reset_index(drop=True) continue col = renames.get(a.column, a.column) if col not in out.columns: + _reject(a, cfg, "pandas", f"column {col!r} is not in the frame") continue + s = out[col] - if a.stage == "strings" and a.action == "strip_strings": - if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): - out[col] = out[col].astype(object).apply(_strip_str) - elif a.stage == "dtypes" and a.action == "parse_currency": - cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) + if key == ("strings", "strip_strings"): + if pd.api.types.is_string_dtype(s) or pd.api.types.is_object_dtype(s): + out[col] = s.astype(object).apply(_strip_str) + elif key == ("dtypes", "parse_currency"): + cleaned = s.astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") - elif a.stage == "missing": - if a.action.startswith("impute_"): - val = a.parameters.get("value") - if val is not None: - out[col] = out[col].fillna(val) - elif a.action == "drop_nulls": - mask = ~out[col].isna().to_numpy() + elif key in _PANDAS_FILLS: + val = a.parameters.get("value") + if val is None: # plans from the Polars planner carry no value + val = _PANDAS_FILLS[key](s) + if val is not None and not pd.isna(val): + out[col] = s.fillna(val) + elif key == ("missing", "drop_nulls"): + out = out[s.notna().to_numpy()].reset_index(drop=True) + elif key in (("outliers", "clip_iqr"), ("outliers", "drop_outliers")): + bounds = _bounds(a) + if bounds is None: + _reject(a, cfg, "pandas", "parameters need numeric 'lower' and 'upper'") + elif a.action == "clip_iqr": + out[col] = s.clip(lower=bounds[0], upper=bounds[1]) + else: + mask = (s.between(*bounds) | s.isna()).to_numpy() out = out[mask].reset_index(drop=True) - elif a.stage == "outliers": - if a.action == "clip_iqr": - lower = a.parameters.get("lower") - upper = a.parameters.get("upper") - if lower is not None and upper is not None: - out[col] = out[col].clip(lower=lower, upper=upper) - elif a.action == "drop_outliers": - lower = a.parameters.get("lower") - upper = a.parameters.get("upper") - if lower is not None and upper is not None: - mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() - out = out[mask].reset_index(drop=True) + else: + _reject(a, cfg, "pandas") return out +_OUTLIER_ACTIONS = {"clip": "clip_iqr", "drop": "drop_outliers"} + +def _pl_mode(expr: pl.Expr) -> pl.Expr: + # Smallest of the most frequent non-null values: the same tie-break as pandas Series.mode(). + return expr.drop_nulls().mode().sort().first() + class PolarsBackend: def profile(self, df: pl.DataFrame | pl.LazyFrame) -> ProfileReport: import polars as pl @@ -128,7 +163,7 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema actions = [] cfg_eff = normalize_config(cfg) - clip_cols: dict[str, pl.Expr] = {} + bounded: dict[str, tuple[pl.Expr, str]] = {} for orig, dtype in schema.items(): name = _normalize_name(orig) if cfg_eff.normalize_names else orig if name != orig: @@ -138,38 +173,42 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): actions.append(ActionRecord("strings", name, "strip_strings", {})) - if dtype.is_numeric(): - # Mirror the pandas stage order: IQR bounds see the imputed column. - base = pl.col(orig) - impute_strat = _impute_strategy(name, cfg_eff) - if impute_strat in ("median", "auto"): - actions.append(ActionRecord("missing", name, "impute_median", {})) - base = base.fill_null(base.median()) - elif impute_strat == "mean": - actions.append(ActionRecord("missing", name, "impute_mean", {})) - base = base.fill_null(base.mean()) - - outlier_strat = _outlier_strategy(name, cfg_eff) - if outlier_strat == "clip": - clip_cols[name] = base - if clip_cols: + # Mirror the pandas stage order: IQR bounds see the imputed column. + base = pl.col(orig) + impute_strat = _impute_strategy(name, cfg_eff) + if impute_strat == "drop": + actions.append(ActionRecord("missing", name, "drop_nulls", {})) + elif impute_strat == "mode": + actions.append(ActionRecord("missing", name, "impute_mode", {})) + base = base.fill_null(_pl_mode(base)) + elif dtype.is_numeric() and impute_strat in ("median", "auto"): + actions.append(ActionRecord("missing", name, "impute_median", {})) + base = base.fill_null(base.median()) + elif dtype.is_numeric() and impute_strat == "mean": + actions.append(ActionRecord("missing", name, "impute_mean", {})) + base = base.fill_null(base.mean()) + + outlier_strat = _outlier_strategy(name, cfg_eff) + if dtype.is_numeric() and outlier_strat in _OUTLIER_ACTIONS: + bounded[name] = (base, _OUTLIER_ACTIONS[outlier_strat]) + if bounded: # One 1-row aggregate query; apply_plan itself never collects. stats = df.lazy().select( e - for i, base in enumerate(clip_cols.values()) + for i, (base, _) in enumerate(bounded.values()) for e in ( base.count().alias(f"n{i}"), base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), base.quantile(0.75, interpolation="linear").alias(f"q75_{i}"), ) ).collect().row(0, named=True) - for i, name in enumerate(clip_cols): + for i, (name, (_, action)) in enumerate(bounded.items()): if stats[f"n{i}"] < 4: continue q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] iqr = q75 - q25 lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr - actions.append(ActionRecord("outliers", name, "clip_iqr", {"lower": float(lower), "upper": float(upper)})) + actions.append(ActionRecord("outliers", name, action, {"lower": float(lower), "upper": float(upper)})) if cfg_eff.drop_duplicates: actions.append(ActionRecord("duplicates", "*", "drop_duplicates", {})) return CleanPlan(actions=tuple(actions)) @@ -183,30 +222,52 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea if renames: cur = cur.rename(renames) - # 2. Column expressions, chained per column so actions compose in plan order + # 2. Column expressions, chained per column so actions compose in plan order. + # Row filters run after the fused with_columns, on the transformed columns. exprs: dict[str, pl.Expr] = {} + filters: list[pl.Expr] = [] + dedup = False for a in plan.actions: + key = (a.stage, a.action) + if key == ("column_names", "normalize_name"): + continue + if key == ("duplicates", "drop_duplicates"): + dedup = True + continue col_name = renames.get(a.column, a.column) expr = exprs.get(col_name, pl.col(col_name)) - if a.stage == "strings" and a.action == "strip_strings": + val = a.parameters.get("value") + if key == ("strings", "strip_strings"): exprs[col_name] = expr.str.strip_chars() - elif a.stage == "missing": - if a.action == "impute_median": - val = a.parameters.get("value") - exprs[col_name] = expr.fill_null(val if val is not None else expr.median()) - elif a.action == "impute_mean": - val = a.parameters.get("value") - exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) - elif a.stage == "outliers" and a.action == "clip_iqr": - lower = a.parameters.get("lower") - upper = a.parameters.get("upper") - if lower is not None and upper is not None: - exprs[col_name] = expr.clip(lower, upper) + elif key == ("dtypes", "parse_currency"): + exprs[col_name] = ( + expr.cast(pl.String).str.replace_all(r"[^\d.-]", "").cast(pl.Float64, strict=False) + ) + elif key == ("missing", "impute_median"): + exprs[col_name] = expr.fill_null(val if val is not None else expr.median()) + elif key == ("missing", "impute_mean"): + exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) + elif key == ("missing", "impute_mode"): + exprs[col_name] = expr.fill_null(val if val is not None else _pl_mode(expr)) + elif key == ("missing", "drop_nulls"): + filters.append(pl.col(col_name).is_not_null()) + elif key in (("outliers", "clip_iqr"), ("outliers", "drop_outliers")): + bounds = _bounds(a) + if bounds is None: + _reject(a, cfg, "polars", "parameters need numeric 'lower' and 'upper'") + elif a.action == "clip_iqr": + exprs[col_name] = expr.clip(*bounds) + else: + filters.append(pl.col(col_name).is_between(*bounds) | pl.col(col_name).is_null()) + else: + _reject(a, cfg, "polars") if exprs: cur = cur.with_columns(e.alias(c) for c, e in exprs.items()) + if filters: + cur = cur.filter(*filters) # 3. Duplicates - if any(a.stage == "duplicates" and a.action == "drop_duplicates" for a in plan.actions): + if dedup: if isinstance(cur, pl.LazyFrame): cur = cur.unique(maintain_order=False) else: diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index e29ecf1f..7ec59216 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -133,3 +133,59 @@ def test_rules_fall_back_to_defaults_per_field_on_both_backends(rule, default): pd_actions = {a.action for a in fd.suggest_plan(pd.DataFrame(data), cfg).actions} pl_actions = {a.action for a in fd.suggest_plan(pl.DataFrame(data), cfg).actions} assert pd_actions == pl_actions and len(pl_actions) == 2 + + +def _rows(frame): + if isinstance(frame, pl.LazyFrame): + frame = frame.collect() + if isinstance(frame, pl.DataFrame): + frame = frame.to_pandas() + return sorted(map(tuple, frame.astype(object).where(frame.notna(), None).values.tolist()), key=repr) + + +def test_every_pandas_action_runs_identically_on_polars(): + import freshdata as fd + + data = {"price": ["$1", "$2", "$3", None, "$5", "$6"], "tag": ["a", "a", None, "b", "b", "a"], + "v": [1.0, None, 2.0, 3.0, 2.0, 2.5], "w": [1.0, 2.0, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(rules={"tag": fd.ColumnRule(impute="mode"), "v": fd.ColumnRule(impute="drop"), + "w": fd.ColumnRule(outliers="drop")}, strip_strings=False) + plan = fd.suggest_plan(pd.DataFrame(data), cfg) + assert {a.action for a in plan.actions} >= {"parse_currency", "impute_mode", "drop_nulls", "drop_outliers"} + expected = _rows(fd.apply_plan(pd.DataFrame(data), plan, cfg)) + assert _rows(fd.apply_plan(pl.DataFrame(data), plan, cfg)) == expected + lazy = fd.apply_plan(pl.LazyFrame(data), plan, cfg) + assert "PYTHON_UDF" not in lazy.explain() + assert _rows(lazy) == expected + + +def test_polars_planner_honors_mode_drop_and_outlier_drop_rules(): + import freshdata as fd + + data = {"tag": ["a", "a", None, "b"], "v": [1.0, None, 2.0, 3.0], "w": [1.0, 2.0, 2.0, 3.0]} + cfg = CleanConfig(rules={"tag": fd.ColumnRule(impute="mode"), "v": fd.ColumnRule(impute="drop"), + "w": fd.ColumnRule(outliers="drop")}) + actions = {(a.column, a.action) for a in fd.suggest_plan(pl.LazyFrame(data), cfg).actions} + assert {("tag", "impute_mode"), ("v", "drop_nulls"), ("w", "drop_outliers")} <= actions + + +def test_polars_plan_without_values_imputes_on_pandas(): + import freshdata as fd + + data = {"v": [1.0, None, 3.0], "t": ["x", None, "x"]} + cfg = CleanConfig(rules={"t": fd.ColumnRule(impute="mode")}) + plan = fd.suggest_plan(pl.DataFrame(data), cfg) + out = fd.apply_plan(pd.DataFrame(data), plan, cfg) + assert out["v"].tolist() == [1.0, 2.0, 3.0] and out["t"].tolist() == ["x", "x", "x"] + + +@pytest.mark.parametrize("frame", [pd.DataFrame({"v": [1.0]}), pl.DataFrame({"v": [1.0]})]) +def test_strict_backend_rejects_actions_it_cannot_run(frame): + import freshdata as fd + + for bad in (ActionRecord("missing", "v", "impute_magic"), ActionRecord("outliers", "v", "clip_iqr", {})): + with pytest.raises(ValueError, match="cannot run"): + fd.apply_plan(frame, CleanPlan((bad,))) + with pytest.warns(UserWarning, match="Skipped"): + out = fd.apply_plan(frame, CleanPlan((bad,)), CleanConfig(strict_backend=False)) + assert _rows(out) == _rows(frame) From 354b90faf9ba824602aeaf1616e7c18d5b4e5a56 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:35:24 +0530 Subject: [PATCH 07/17] feat: implement impute="ffill" and outliers="flag" on both backends Both options were accepted by ColumnRule/CleanConfig and did nothing. - impute="ffill": forward-fill the column (pandas ffill, Polars forward_fill). - outliers="flag": add a boolean _outlier column marking values outside the IQR fences, the meaning the v2 docs gave it. Values are not changed. If _outlier already exists, planning and apply raise ValueError instead of overwriting it. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 28 +++++++++++++++++++++++----- src/freshdata/_core/stages.py | 21 +++++++++++++++++++-- tests/test_backend_honesty.py | 23 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index ce1b8bfa..afec1d20 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -16,6 +16,7 @@ _is_protected, _outlier_strategy, _strip_str, + flag_column, _normalize_name, normalize_config, stage_column_names, @@ -115,14 +116,18 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. val = _PANDAS_FILLS[key](s) if val is not None and not pd.isna(val): out[col] = s.fillna(val) + elif key == ("missing", "impute_ffill"): + out[col] = s.ffill() elif key == ("missing", "drop_nulls"): out = out[s.notna().to_numpy()].reset_index(drop=True) - elif key in (("outliers", "clip_iqr"), ("outliers", "drop_outliers")): + elif key in _BOUNDED_ACTIONS: bounds = _bounds(a) if bounds is None: _reject(a, cfg, "pandas", "parameters need numeric 'lower' and 'upper'") elif a.action == "clip_iqr": out[col] = s.clip(lower=bounds[0], upper=bounds[1]) + elif a.action == "flag_iqr": + out[flag_column(col, out.columns)] = (s < bounds[0]) | (s > bounds[1]) else: mask = (s.between(*bounds) | s.isna()).to_numpy() out = out[mask].reset_index(drop=True) @@ -130,7 +135,8 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. _reject(a, cfg, "pandas") return out -_OUTLIER_ACTIONS = {"clip": "clip_iqr", "drop": "drop_outliers"} +_OUTLIER_ACTIONS = {"clip": "clip_iqr", "drop": "drop_outliers", "flag": "flag_iqr"} +_BOUNDED_ACTIONS = {("outliers", v) for v in _OUTLIER_ACTIONS.values()} def _pl_mode(expr: pl.Expr) -> pl.Expr: # Smallest of the most frequent non-null values: the same tie-break as pandas Series.mode(). @@ -181,6 +187,9 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle elif impute_strat == "mode": actions.append(ActionRecord("missing", name, "impute_mode", {})) base = base.fill_null(_pl_mode(base)) + elif impute_strat == "ffill": + actions.append(ActionRecord("missing", name, "impute_ffill", {})) + base = base.forward_fill() elif dtype.is_numeric() and impute_strat in ("median", "auto"): actions.append(ActionRecord("missing", name, "impute_median", {})) base = base.fill_null(base.median()) @@ -202,9 +211,12 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle base.quantile(0.75, interpolation="linear").alias(f"q75_{i}"), ) ).collect().row(0, named=True) + out_names = {a.parameters["to"] for a in actions if a.action == "normalize_name"} | set(schema) for i, (name, (_, action)) in enumerate(bounded.items()): if stats[f"n{i}"] < 4: continue + if action == "flag_iqr": + flag_column(name, out_names) q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] iqr = q75 - q25 lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr @@ -226,6 +238,7 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea # Row filters run after the fused with_columns, on the transformed columns. exprs: dict[str, pl.Expr] = {} filters: list[pl.Expr] = [] + flags: dict[str, pl.Expr] = {} dedup = False for a in plan.actions: key = (a.stage, a.action) @@ -249,20 +262,25 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) elif key == ("missing", "impute_mode"): exprs[col_name] = expr.fill_null(val if val is not None else _pl_mode(expr)) + elif key == ("missing", "impute_ffill"): + exprs[col_name] = expr.forward_fill() elif key == ("missing", "drop_nulls"): filters.append(pl.col(col_name).is_not_null()) - elif key in (("outliers", "clip_iqr"), ("outliers", "drop_outliers")): + elif key in _BOUNDED_ACTIONS: bounds = _bounds(a) if bounds is None: _reject(a, cfg, "polars", "parameters need numeric 'lower' and 'upper'") elif a.action == "clip_iqr": exprs[col_name] = expr.clip(*bounds) + elif a.action == "flag_iqr": + names = cur.collect_schema().names() # schema only, no data + flags[flag_column(col_name, names)] = ((expr < bounds[0]) | (expr > bounds[1])).fill_null(False) else: filters.append(pl.col(col_name).is_between(*bounds) | pl.col(col_name).is_null()) else: _reject(a, cfg, "polars") - if exprs: - cur = cur.with_columns(e.alias(c) for c, e in exprs.items()) + if exprs or flags: + cur = cur.with_columns(e.alias(c) for c, e in {**exprs, **flags}.items()) if filters: cur = cur.filter(*filters) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index b345844c..f23021f0 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -35,6 +35,13 @@ def _outlier_strategy(col: object, cfg: CleanConfig) -> str: rule = cfg.rules.get(col) return (rule.outliers if rule else None) or cfg.default_outliers +def flag_column(col: object, existing: object) -> str: + """Name of the boolean column that outliers="flag" adds. Never overwrites a column.""" + name = f"{col}_outlier" + if name in existing: + raise ValueError(f"outliers='flag' on {col!r} would overwrite the existing column {name!r}") + return name + def _strip_str(x: object) -> object: """The one strip_strings executor, shared by planning and pandas apply_plan. @@ -146,17 +153,23 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis val = val.item() if isinstance(val, np.generic) else val # plain Python scalar, original type out[col] = out[col].fillna(val) actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": val}, null_count)) + elif impute_strategy == "ffill": + filled = out[col].ffill() + filled_count = null_count - int(filled.isna().sum()) + if filled_count: + out[col] = filled + actions.append(ActionRecord("missing", str(col), "impute_ffill", {}, filled_count)) return out, actions def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: actions: List[ActionRecord] = [] out = df.copy() cfg_eff = normalize_config(cfg) - for col in out.columns: + for col in list(out.columns): if _is_protected(col, cfg_eff): continue strategy = _outlier_strategy(col, cfg_eff) - if strategy not in ("clip", "drop") or not pd.api.types.is_numeric_dtype(out[col]): + if strategy not in ("clip", "drop", "flag") or not pd.api.types.is_numeric_dtype(out[col]): continue s = out[col].dropna() if len(s) < 4: @@ -167,6 +180,10 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li if strategy == "clip": out[col] = out[col].clip(lower=lower, upper=upper) actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, len(out))) + elif strategy == "flag": + flag_col = flag_column(col, out.columns) + out[flag_col] = (out[col] < lower) | (out[col] > upper) + actions.append(ActionRecord("outliers", str(col), "flag_iqr", {"lower": float(lower), "upper": float(upper)}, int(out[flag_col].sum()))) else: mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() dropped = int((~mask).sum()) diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index 7ec59216..362a3487 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -189,3 +189,26 @@ def test_strict_backend_rejects_actions_it_cannot_run(frame): with pytest.warns(UserWarning, match="Skipped"): out = fd.apply_plan(frame, CleanPlan((bad,)), CleanConfig(strict_backend=False)) assert _rows(out) == _rows(frame) + + +def test_ffill_and_flag_run_on_every_backend(): + import freshdata as fd + + data = {"f": [1.0, None, None, 4.0, 5.0, 6.0], "w": [1.0, 2.0, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(rules={"f": fd.ColumnRule(impute="ffill"), "w": fd.ColumnRule(outliers="flag")}) + pd_out = fd.apply_plan(pd.DataFrame(data), fd.suggest_plan(pd.DataFrame(data), cfg), cfg) + assert pd_out["f"].tolist() == [1.0, 1.0, 1.0, 4.0, 5.0, 6.0] + assert pd_out["w_outlier"].tolist() == [False] * 5 + [True] + assert pd_out["w"].tolist() == data["w"] # flag never rewrites the values + for frame in (pl.DataFrame(data), pl.LazyFrame(data)): + out = fd.apply_plan(frame, fd.suggest_plan(frame, cfg), cfg) + assert _rows(out) == _rows(pd_out) + + +@pytest.mark.parametrize("make", [pd.DataFrame, pl.DataFrame]) +def test_flag_never_overwrites_an_existing_column(make): + import freshdata as fd + + frame = make({"w": [1.0, 2.0, 2.0, 3.0, 100.0], "w_outlier": ["keep"] * 5}) + with pytest.raises(ValueError, match="overwrite"): + fd.clean(frame, CleanConfig(default_outliers="flag", strip_strings=False)) From 57208300fc1536828ed5504b1a9c71afbd7fc907 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:36:44 +0530 Subject: [PATCH 08/17] fix(config): validate options and stop accepting settings that do nothing ColumnRule and CleanConfig accepted any value, so a typo such as impute="medain" silently did nothing, and id_columns="customer_id" matched column names by substring. - Reject unknown role/impute/outliers/strategy/default values, a bare string for id_columns, non-ColumnRule rules, and a negative iqr_factor. - Widen the Literal hints to every value the stages already handle (default_outliers="drop" is what freshdata-migrate emits). - ColumnRule(role="id"|"target") now protects the column, like id_columns and target_column. - Warn when an option has no effect: strategy other than "balanced", other roles, and default_outliers="auto". Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/models.py | 41 +++++++++++++++++++++++++++++-- src/freshdata/_core/stages.py | 4 +-- tests/test_backend_honesty.py | 4 +-- tests/test_models.py | 40 ++++++++++++++++++++++++++++++ tests/test_property_invariants.py | 1 - 5 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/freshdata/_core/models.py b/src/freshdata/_core/models.py index 55c2d0a5..fafffca4 100644 --- a/src/freshdata/_core/models.py +++ b/src/freshdata/_core/models.py @@ -1,8 +1,21 @@ # src/freshdata/_core/models.py from __future__ import annotations +import warnings from dataclasses import asdict, dataclass, field +from numbers import Real from typing import Any, Literal, Mapping +_ROLES = ("id", "target", "numeric", "categorical", "datetime", "text") +_IMPUTES = ("preserve", "drop", "mean", "median", "mode", "ffill") +_OUTLIERS = ("preserve", "clip", "drop", "flag") + +def _check_choice(name: str, value: object, allowed: tuple[str, ...]) -> None: + if value not in allowed: + raise ValueError(f"{name}={value!r} is not valid. Use one of: {', '.join(allowed)}.") + +def _warn_no_effect(message: str) -> None: + warnings.warn(message, UserWarning, stacklevel=4) + @dataclass(frozen=True) class ColumnRule: role: Literal["id", "target", "numeric", "categorical", "datetime", "text"] | None = None @@ -10,6 +23,13 @@ class ColumnRule: outliers: Literal["preserve", "clip", "drop", "flag"] | None = None protected: bool = False + def __post_init__(self) -> None: + for name, allowed in (("role", _ROLES), ("impute", _IMPUTES), ("outliers", _OUTLIERS)): + if getattr(self, name) is not None: + _check_choice(name, getattr(self, name), allowed) + if self.role not in (None, "id", "target"): + _warn_no_effect(f"ColumnRule(role={self.role!r}) has no effect. Only 'id' and 'target' (protect the column) do.") + @dataclass(frozen=True) class CleanConfig: strategy: Literal["conservative", "balanced", "aggressive"] = "balanced" @@ -20,11 +40,28 @@ class CleanConfig: strip_strings: bool = True fix_dtypes: bool = True drop_duplicates: bool = False - default_impute: Literal["auto", "preserve", "median", "mode"] = "auto" - default_outliers: Literal["auto", "preserve", "clip", "flag"] = "preserve" + default_impute: Literal["auto", "preserve", "drop", "mean", "median", "mode", "ffill"] = "auto" + default_outliers: Literal["auto", "preserve", "clip", "drop", "flag"] = "preserve" iqr_factor: float = 1.5 strict_backend: bool = True + def __post_init__(self) -> None: + _check_choice("strategy", self.strategy, ("conservative", "balanced", "aggressive")) + _check_choice("default_impute", self.default_impute, ("auto", *_IMPUTES)) + _check_choice("default_outliers", self.default_outliers, ("auto", *_OUTLIERS)) + if isinstance(self.id_columns, str): + raise ValueError(f"id_columns must be a tuple of column names, not the string {self.id_columns!r}.") + object.__setattr__(self, "id_columns", tuple(self.id_columns)) + for col, rule in self.rules.items(): + if not isinstance(rule, ColumnRule): + raise TypeError(f"rules[{col!r}] must be a ColumnRule, not {type(rule).__name__}.") + if not isinstance(self.iqr_factor, Real) or isinstance(self.iqr_factor, bool) or self.iqr_factor < 0: + raise ValueError(f"iqr_factor must be a number >= 0, not {self.iqr_factor!r}.") + if self.strategy != "balanced": + _warn_no_effect(f"CleanConfig(strategy={self.strategy!r}) has no effect in freshdata 3.x.") + if self.default_outliers == "auto": + _warn_no_effect("CleanConfig(default_outliers='auto') leaves outliers unchanged, like 'preserve'.") + @dataclass(frozen=True) class ActionRecord: stage: str diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index f23021f0..e628c1ad 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -14,7 +14,7 @@ def _normalize_name(name: str) -> str: return re.sub(r"[\s\-]+", "_", s) def _is_protected(col: object, cfg: CleanConfig) -> bool: - """Single source of truth for columns the missing/outlier stages must not touch. + """Single source of truth for columns that cleaning stages must not touch. ``cfg`` must already be normalized (see ``normalize_config``). """ @@ -22,7 +22,7 @@ def _is_protected(col: object, cfg: CleanConfig) -> bool: if col in cfg.id_columns or col == cfg.target_column or name.endswith("_id") or name == "id": return True rule = cfg.rules.get(col) - return rule is not None and rule.protected + return rule is not None and (rule.protected or rule.role in ("id", "target")) def _impute_strategy(col: object, cfg: CleanConfig) -> str | None: """Per-field fallback: a rule that leaves ``impute`` unset uses the default.""" diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index 362a3487..83ab1c7a 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -113,9 +113,9 @@ def test_lazy_plan_diff_detects_rule_changes(): import freshdata as fd lf = pl.LazyFrame({"score": [1.0, 2.0, None, 4.0, 100.0]}) - conservative = fd.suggest_plan(lf, CleanConfig(strategy="conservative", default_outliers="preserve")) + conservative = fd.suggest_plan(lf, CleanConfig(default_outliers="preserve")) aggressive = fd.suggest_plan(lf, CleanConfig( - strategy="aggressive", rules={"score": fd.ColumnRule(impute="median", outliers="clip")})) + rules={"score": fd.ColumnRule(impute="median", outliers="clip")})) diff = conservative.diff(aggressive) assert [(a.column, a.action) for a in diff["added"]] == [("score", "clip_iqr")] assert diff["removed"] == [] and diff["modified"] == [] diff --git a/tests/test_models.py b/tests/test_models.py index cb52db1a..fe9c454b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -62,3 +62,43 @@ def test_clean_plan_json_roundtrip(): assert len(loaded["actions"]) == 1 assert loaded["actions"][0]["stage"] == "missing" + + +@pytest.mark.parametrize("make", [ + lambda: ColumnRule(impute="medain"), + lambda: ColumnRule(outliers="cap"), + lambda: ColumnRule(role="key"), + lambda: CleanConfig(default_impute="knn"), + lambda: CleanConfig(strategy="fast"), + lambda: CleanConfig(id_columns="customer_id"), + lambda: CleanConfig(iqr_factor=-1), +]) +def test_invalid_options_are_rejected(make): + with pytest.raises(ValueError): + make() + + +def test_rules_must_be_column_rules_and_id_columns_become_a_tuple(): + with pytest.raises(TypeError): + CleanConfig(rules={"a": {"impute": "mean"}}) + assert CleanConfig(id_columns=["a", "b"]).id_columns == ("a", "b") + + +def test_options_without_effect_warn(): + with pytest.warns(UserWarning, match="no effect"): + CleanConfig(strategy="aggressive") + with pytest.warns(UserWarning, match="no effect"): + ColumnRule(role="numeric") + with pytest.warns(UserWarning, match="like 'preserve'"): + CleanConfig(default_outliers="auto") + + +def test_role_id_and_target_protect_the_column(): + import pandas as pd + + import freshdata as fd + + df = pd.DataFrame({"k": [" a ", None, " a "], "v": [1.0, None, 3.0]}) + cfg = CleanConfig(rules={"k": ColumnRule(role="id"), "v": ColumnRule(role="target")}, default_impute="mode") + out = fd.clean(df, cfg) + assert out["k"].tolist()[0] == " a " and out["v"].isna().sum() == 1 diff --git a/tests/test_property_invariants.py b/tests/test_property_invariants.py index fe53c24d..6633efb7 100644 --- a/tests/test_property_invariants.py +++ b/tests/test_property_invariants.py @@ -35,7 +35,6 @@ def dirty_dataframe_strategy(draw: st.DrawFn) -> pd.DataFrame: @given(df=dirty_dataframe_strategy()) def test_five_hard_invariants(df: pd.DataFrame) -> None: cfg = CleanConfig( - strategy="conservative", fix_dtypes=False, drop_duplicates=True, default_impute="preserve", From 03ff405c5cb8430aff129dbd21e4bdb1a9a1826d Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:37:04 +0530 Subject: [PATCH 09/17] fix(api): let clean() and Cleaner() take CleanConfig fields as keywords freshdata-migrate rewrites v2 calls into fd.clean(df, default_impute=...) and fd.Cleaner(id_columns=...), but both raised TypeError on v3. Keyword arguments now override fields of the given (or default) CleanConfig and go through the same validation. Unknown keywords still raise TypeError. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/engine.py | 13 ++++++++++--- tests/test_core_api.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/freshdata/_core/engine.py b/src/freshdata/_core/engine.py index e0ed95fc..f9b27bda 100644 --- a/src/freshdata/_core/engine.py +++ b/src/freshdata/_core/engine.py @@ -1,5 +1,6 @@ # src/freshdata/_core/engine.py from __future__ import annotations +import dataclasses from typing import Any from freshdata._core.backends import resolve_backend from freshdata._core.models import ( @@ -12,6 +13,10 @@ ProfileReport, ) +def _with_overrides(config: CleanConfig | None, overrides: dict[str, Any]) -> CleanConfig: + cfg = config or CleanConfig() + return dataclasses.replace(cfg, **overrides) if overrides else cfg + def profile(df: Any) -> ProfileReport: return resolve_backend(df).profile(df) @@ -40,14 +45,16 @@ def clean( df: Any, config: CleanConfig | None = None, return_report: bool = False, + **overrides: Any, ) -> Any: - cfg = config or CleanConfig() + """Plan and apply in one call. Keyword overrides replace CleanConfig fields.""" + cfg = _with_overrides(config, overrides) plan = suggest_plan(df, cfg) return apply_plan(df, plan, cfg, return_report=return_report) class Cleaner: - def __init__(self, config: CleanConfig | None = None): - self.config = config or CleanConfig() + def __init__(self, config: CleanConfig | None = None, **overrides: Any): + self.config = _with_overrides(config, overrides) def profile(self, df: Any) -> ProfileReport: return profile(df) diff --git a/tests/test_core_api.py b/tests/test_core_api.py index b903b55b..754f4de9 100644 --- a/tests/test_core_api.py +++ b/tests/test_core_api.py @@ -73,3 +73,18 @@ def test_stages_disabled_flags(): cleaned = fd.clean(df, config=cfg) assert " Messy Name " in cleaned.columns assert cleaned[" Messy Name "].iloc[0] == " val " + + +def test_clean_and_cleaner_accept_config_overrides_like_migrate_emits(): + import pandas as pd + import pytest + + import freshdata as fd + + df = pd.DataFrame({"v": [1.0, None, 5.0]}) + assert fd.clean(df, default_impute="mean")["v"].tolist() == [1.0, 3.0, 5.0] + cfg = fd.CleanConfig(default_impute="preserve") + assert fd.clean(df, cfg, default_impute="median")["v"].tolist() == [1.0, 3.0, 5.0] + assert fd.Cleaner(id_columns=["v"]).config.id_columns == ("v",) + with pytest.raises(TypeError): + fd.clean(df, not_an_option=True) From f3efa2c708074cfb8531a06995fb0dd172dadda2 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:38:11 +0530 Subject: [PATCH 10/17] fix(polars): profile LazyFrames with real counts in one aggregate query profile() returned 0 rows, 0 nulls and 0 unique values for every LazyFrame column, which reads as real data. Both DataFrame and LazyFrame input now run one select of len/null_count/n_unique, which Polars evaluates in parallel. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 21 +++++++++++++-------- tests/test_backend_honesty.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index afec1d20..e638fe10 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -146,19 +146,24 @@ class PolarsBackend: def profile(self, df: pl.DataFrame | pl.LazyFrame) -> ProfileReport: import polars as pl - frame = None if isinstance(df, pl.LazyFrame) else df - schema = df.collect_schema() if frame is None else frame.schema - row_count = 0 if frame is None else len(frame) + lf = df.lazy() + schema = lf.collect_schema() + # One aggregate query (row 0 holds every statistic), the same for DataFrame and LazyFrame input. + row = lf.select( + pl.len().alias("rows"), + *(pl.col(c).null_count().alias(f"n{i}") for i, c in enumerate(schema)), + *(pl.col(c).drop_nulls().n_unique().alias(f"u{i}") for i, c in enumerate(schema)), + ).collect().row(0, named=True) + row_count = row["rows"] cols = {} - for name, dtype in schema.items(): - role = "numeric" if dtype.is_numeric() else "categorical" - nulls = 0 if frame is None else frame[name].null_count() + for i, (name, dtype) in enumerate(schema.items()): + nulls = row[f"n{i}"] cols[name] = ColumnProfile( name=name, - inferred_role=role, + inferred_role="numeric" if dtype.is_numeric() else "categorical", null_count=nulls, null_ratio=nulls / max(row_count, 1), - unique_count=0 if frame is None else frame[name].drop_nulls().n_unique(), + unique_count=row[f"u{i}"], sample_values=(), ) return ProfileReport(row_count=row_count, column_count=len(schema), columns=cols) diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index 83ab1c7a..836ac07f 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -212,3 +212,13 @@ def test_flag_never_overwrites_an_existing_column(make): frame = make({"w": [1.0, 2.0, 2.0, 3.0, 100.0], "w_outlier": ["keep"] * 5}) with pytest.raises(ValueError, match="overwrite"): fd.clean(frame, CleanConfig(default_outliers="flag", strip_strings=False)) + + +def test_lazyframe_profile_reports_real_counts(): + data = {"a": [1, 2, None, 2], "b": ["x", None, None, "y"]} + lazy = resolve_backend(pl.LazyFrame(data)).profile(pl.LazyFrame(data)) + eager = resolve_backend(pd.DataFrame(data)).profile(pd.DataFrame(data)) + assert lazy.row_count == 4 + for col in data: + p, q = eager.columns[col], lazy.columns[col] + assert (q.null_count, q.unique_count, q.null_ratio) == (p.null_count, p.unique_count, p.null_ratio) From 397a4939cc97ebd1de85bd9b2c53b35aed284c74 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:39:31 +0530 Subject: [PATCH 11/17] fix: make rows_affected count the cells an action changes pandas reported the frame's row count for normalize_name, strip_strings, parse_currency and clip_iqr. Polars reported 0 for every action. - pandas: renames change 0 cells. strip_strings counts changed cells, parse_currency counts parsed cells, clip_iqr counts clipped values. - Polars: the planner computes the same counts in its one aggregate query (null counts, strip changes, values outside the IQR fences). drop_duplicates stays uncounted on Polars, and its rationale says why: counting needs a full-row hash, which would break streaming-scale planning. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 84 +++++++++++++++++++++------------ src/freshdata/_core/stages.py | 15 ++++-- tests/test_backend_honesty.py | 11 +++++ 3 files changed, 75 insertions(+), 35 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index e638fe10..06ae18f6 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -1,5 +1,6 @@ # src/freshdata/_core/backends.py from __future__ import annotations +import dataclasses import warnings from numbers import Real from typing import TYPE_CHECKING, Any @@ -175,59 +176,82 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle actions = [] cfg_eff = normalize_config(cfg) bounded: dict[str, tuple[pl.Expr, str]] = {} + counts: dict[int, pl.Expr] = {} # action index -> rows_affected, filled by the one query below + + def add(action: ActionRecord, count: pl.Expr | None = None) -> None: + if count is not None: + counts[len(actions)] = count + actions.append(action) + for orig, dtype in schema.items(): name = _normalize_name(orig) if cfg_eff.normalize_names else orig if name != orig: - actions.append(ActionRecord("column_names", orig, "normalize_name", {"to": name})) + add(ActionRecord("column_names", orig, "normalize_name", {"to": name})) if _is_protected(name, cfg_eff): continue + col = pl.col(orig) if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): - actions.append(ActionRecord("strings", name, "strip_strings", {})) + add(ActionRecord("strings", name, "strip_strings", {}), (col != col.str.strip_chars()).sum()) # Mirror the pandas stage order: IQR bounds see the imputed column. - base = pl.col(orig) + base = col impute_strat = _impute_strategy(name, cfg_eff) if impute_strat == "drop": - actions.append(ActionRecord("missing", name, "drop_nulls", {})) + add(ActionRecord("missing", name, "drop_nulls", {}), col.null_count()) elif impute_strat == "mode": - actions.append(ActionRecord("missing", name, "impute_mode", {})) + add(ActionRecord("missing", name, "impute_mode", {}), col.null_count()) base = base.fill_null(_pl_mode(base)) elif impute_strat == "ffill": - actions.append(ActionRecord("missing", name, "impute_ffill", {})) base = base.forward_fill() + add(ActionRecord("missing", name, "impute_ffill", {}), col.null_count() - base.null_count()) elif dtype.is_numeric() and impute_strat in ("median", "auto"): - actions.append(ActionRecord("missing", name, "impute_median", {})) + add(ActionRecord("missing", name, "impute_median", {}), col.null_count()) base = base.fill_null(base.median()) elif dtype.is_numeric() and impute_strat == "mean": - actions.append(ActionRecord("missing", name, "impute_mean", {})) + add(ActionRecord("missing", name, "impute_mean", {}), col.null_count()) base = base.fill_null(base.mean()) outlier_strat = _outlier_strategy(name, cfg_eff) if dtype.is_numeric() and outlier_strat in _OUTLIER_ACTIONS: bounded[name] = (base, _OUTLIER_ACTIONS[outlier_strat]) - if bounded: - # One 1-row aggregate query; apply_plan itself never collects. - stats = df.lazy().select( - e - for i, (base, _) in enumerate(bounded.values()) - for e in ( - base.count().alias(f"n{i}"), - base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), - base.quantile(0.75, interpolation="linear").alias(f"q75_{i}"), - ) - ).collect().row(0, named=True) - out_names = {a.parameters["to"] for a in actions if a.action == "normalize_name"} | set(schema) - for i, (name, (_, action)) in enumerate(bounded.items()): - if stats[f"n{i}"] < 4: - continue - if action == "flag_iqr": - flag_column(name, out_names) - q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] - iqr = q75 - q25 - lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr - actions.append(ActionRecord("outliers", name, action, {"lower": float(lower), "upper": float(upper)})) if cfg_eff.drop_duplicates: - actions.append(ActionRecord("duplicates", "*", "drop_duplicates", {})) + # ponytail: not counted; a full-row hash at plan time would break streaming-scale planning. + add(ActionRecord("duplicates", "*", "drop_duplicates", {}, + rationale="rows_affected not computed on Polars: counting needs a full-row hash")) + + k = cfg_eff.iqr_factor + stat_exprs = [e.alias(f"c{i}") for i, e in counts.items()] + for i, (base, _) in enumerate(bounded.values()): + q25 = base.quantile(0.25, interpolation="linear") + q75 = base.quantile(0.75, interpolation="linear") + lo, hi = q25 - k * (q75 - q25), q75 + k * (q75 - q25) + stat_exprs += [base.count().alias(f"n{i}"), q25.alias(f"q25_{i}"), q75.alias(f"q75_{i}"), + ((base < lo) | (base > hi)).sum().alias(f"out{i}")] + if not stat_exprs: + return CleanPlan(actions=tuple(actions)) + # One 1-row aggregate query; apply_plan itself never collects. + stats = df.lazy().select(stat_exprs).collect().row(0, named=True) + actions = [ + dataclasses.replace(a, rows_affected=int(stats[f"c{i}"] or 0)) if i in counts else a + for i, a in enumerate(actions) + ] + out_names = {a.parameters["to"] for a in actions if a.action == "normalize_name"} | set(schema) + bounded_actions = [] + for i, (name, (_, action)) in enumerate(bounded.items()): + if stats[f"n{i}"] < 4: + continue + if action == "flag_iqr": + flag_column(name, out_names) + q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] + iqr = q75 - q25 + lower, upper = q25 - k * iqr, q75 + k * iqr + bounded_actions.append(ActionRecord("outliers", name, action, {"lower": float(lower), "upper": float(upper)}, + int(stats[f"out{i}"]))) + # Keep the established order: outlier actions come before drop_duplicates. + if cfg_eff.drop_duplicates: + actions[-1:-1] = bounded_actions + else: + actions += bounded_actions return CleanPlan(actions=tuple(actions)) def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: CleanConfig) -> pl.DataFrame | pl.LazyFrame: diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index e628c1ad..d67184a5 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -69,7 +69,7 @@ def stage_column_names(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame norm = _normalize_name(str(col)) if norm != col: rename_map[col] = norm - actions.append(ActionRecord("column_names", str(col), "normalize_name", {"to": norm}, len(df))) + actions.append(ActionRecord("column_names", str(col), "normalize_name", {"to": norm}, 0)) out = df.rename(columns=rename_map) if rename_map else df return out, actions @@ -84,8 +84,11 @@ def stage_strings(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis continue if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): # Series.apply (not .str) keeps pandas' result-dtype inference unchanged. - out[col] = out[col].astype(object).apply(_strip_str) - actions.append(ActionRecord("strings", str(col), "strip_strings", {}, len(out))) + before = out[col].astype(object) + after = before.apply(_strip_str) + changed = (before.astype(str) != after.astype(str)) & ~(before.isna() & after.isna()) + out[col] = after + actions.append(ActionRecord("strings", str(col), "strip_strings", {}, int(changed.sum()))) return out, actions def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: @@ -104,9 +107,10 @@ def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List has_symbol = sample.str.contains(r"[\$€£¥]").any() matches_pattern = sample.apply(lambda v: bool(CURRENCY_REGEX.match(v))) if has_symbol and (matches_pattern.sum() / len(sample)) >= 0.7: + parsed = int(out[col].notna().sum()) cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") - actions.append(ActionRecord("dtypes", str(col), "parse_currency", {"target_dtype": "float64"}, len(out))) + actions.append(ActionRecord("dtypes", str(col), "parse_currency", {"target_dtype": "float64"}, parsed)) return out, actions def stage_duplicates(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: @@ -178,8 +182,9 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li iqr = q75 - q25 lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr if strategy == "clip": + clipped = int(((out[col] < lower) | (out[col] > upper)).sum()) out[col] = out[col].clip(lower=lower, upper=upper) - actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, len(out))) + actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, clipped)) elif strategy == "flag": flag_col = flag_column(col, out.columns) out[flag_col] = (out[col] < lower) | (out[col] > upper) diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index 836ac07f..a4b0d881 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -222,3 +222,14 @@ def test_lazyframe_profile_reports_real_counts(): for col in data: p, q = eager.columns[col], lazy.columns[col] assert (q.null_count, q.unique_count, q.null_ratio) == (p.null_count, p.unique_count, p.null_ratio) + + +def test_rows_affected_counts_changed_cells_on_both_backends(): + import freshdata as fd + + data = {"Name X": [" a", "b", "c ", None, "d", "e"], "v": [1.0, None, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(default_outliers="clip") + want = {("normalize_name", 0), ("strip_strings", 2), ("impute_median", 1), ("clip_iqr", 1)} + for frame in (pd.DataFrame(data), pl.DataFrame(data), pl.LazyFrame(data)): + got = {(a.action, a.rows_affected) for a in fd.suggest_plan(frame, cfg).actions} + assert got == want, type(frame) From 416871fd831392db7a368c500f4fe9ae9e7f6942 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:43:51 +0530 Subject: [PATCH 12/17] perf(polars): compute each IQR quantile once when counting outliers Inlining the quantiles in the outside-fence count made Polars evaluate each one three times (79ms vs 26ms for 6 columns x 300k rows). The planner now adds the quantiles with with_columns and counts against those columns. It is still one query. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/backends.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 06ae18f6..5d98ddab 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -173,7 +173,7 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle import polars as pl schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema - actions = [] + actions: list[ActionRecord] = [] cfg_eff = normalize_config(cfg) bounded: dict[str, tuple[pl.Expr, str]] = {} counts: dict[int, pl.Expr] = {} # action index -> rows_affected, filled by the one query below @@ -221,16 +221,20 @@ def add(action: ActionRecord, count: pl.Expr | None = None) -> None: k = cfg_eff.iqr_factor stat_exprs = [e.alias(f"c{i}") for i, e in counts.items()] + quantiles: list[pl.Expr] = [] for i, (base, _) in enumerate(bounded.values()): - q25 = base.quantile(0.25, interpolation="linear") - q75 = base.quantile(0.75, interpolation="linear") + # Each quantile is computed once in with_columns; inlining it in the count costs about 3x. + q25, q75 = pl.col(f"\0fd_q25_{i}"), pl.col(f"\0fd_q75_{i}") + quantiles += [base.quantile(0.25, interpolation="linear").alias(f"\0fd_q25_{i}"), + base.quantile(0.75, interpolation="linear").alias(f"\0fd_q75_{i}")] lo, hi = q25 - k * (q75 - q25), q75 + k * (q75 - q25) - stat_exprs += [base.count().alias(f"n{i}"), q25.alias(f"q25_{i}"), q75.alias(f"q75_{i}"), + stat_exprs += [base.count().alias(f"n{i}"), q25.first().alias(f"q25_{i}"), q75.first().alias(f"q75_{i}"), ((base < lo) | (base > hi)).sum().alias(f"out{i}")] if not stat_exprs: return CleanPlan(actions=tuple(actions)) # One 1-row aggregate query; apply_plan itself never collects. - stats = df.lazy().select(stat_exprs).collect().row(0, named=True) + lf = df.lazy().with_columns(quantiles) if quantiles else df.lazy() + stats = lf.select(stat_exprs).collect().row(0, named=True) actions = [ dataclasses.replace(a, rows_affected=int(stats[f"c{i}"] or 0)) if i in counts else a for i, a in enumerate(actions) From def290824a79108d69d26b42eb33607f971115b5 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:43:51 +0530 Subject: [PATCH 13/17] fix(api): count LazyFrame report columns from the schema apply_plan(..., return_report=True) called len(lf.columns), which makes Polars emit a PerformanceWarning in the README quickstart. Use collect_schema() for Polars frames. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/engine.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/freshdata/_core/engine.py b/src/freshdata/_core/engine.py index f9b27bda..510f8d8e 100644 --- a/src/freshdata/_core/engine.py +++ b/src/freshdata/_core/engine.py @@ -17,6 +17,11 @@ def _with_overrides(config: CleanConfig | None, overrides: dict[str, Any]) -> Cl cfg = config or CleanConfig() return dataclasses.replace(cfg, **overrides) if overrides else cfg +def _column_count(frame: Any) -> int: + if hasattr(frame, "collect_schema"): # Polars: schema only, no data and no PerformanceWarning + return len(frame.collect_schema()) + return len(frame.columns) if hasattr(frame, "columns") else 0 + def profile(df: Any) -> ProfileReport: return resolve_backend(df).profile(df) @@ -35,8 +40,8 @@ def apply_plan( if return_report: orig_r = 0 if type(df).__name__ == "LazyFrame" else len(df) clean_r = 0 if type(cleaned).__name__ == "LazyFrame" else len(cleaned) - orig_c = len(df.columns) if hasattr(df, "columns") else 0 - clean_c = len(cleaned.columns) if hasattr(cleaned, "columns") else 0 + orig_c = _column_count(df) + clean_c = _column_count(cleaned) rep = CleanReport(orig_r, clean_r, orig_c, clean_c, plan.actions) return cleaned, rep return cleaned From 0305b5299b924265aaa3004f96dc202ba38a1c80 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:43:51 +0530 Subject: [PATCH 14/17] docs: record the backend contract and the remaining differences ARCHITECTURE.md now states the shared-helper and strict-execution rules and replaces the fixed "Known issues" with the backend differences that remain. CHANGELOG.md lists the user-visible changes. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 66 ++++++++++++++++++++++++------------------------- CHANGELOG.md | 33 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 33 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3227096a..c2c9f135 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -59,9 +59,14 @@ fd.clean(df, cfg) serialized (`to_dict`), diffed (`diff`), and edited before it is applied. 2. **Lazy stays lazy.** `PolarsBackend.apply_plan` must not call `.collect()`, `.to_pandas()`, or Python UDFs. `tests/test_backend_honesty.py` enforces this. -3. **Protection lives in one place.** `stages._is_protected` decides which - columns the missing and outlier logic must skip. Do not copy the predicate. -4. **pandas is required. Polars is optional.** `resolve_backend` imports Polars +3. **One definition per decision.** `stages._is_protected` decides which + columns no stage may touch. `stages._impute_strategy` and + `stages._outlier_strategy` resolve rules against defaults. `stages._strip_str` + is the only pandas string executor. Do not copy these. +4. **Every action runs everywhere or fails loudly.** Both executors handle the + same action set. An action a backend cannot run raises `ValueError` when + `strict_backend=True` (the default), and warns and skips otherwise. +5. **pandas is required. Polars is optional.** `resolve_backend` imports Polars only for Polars frames. ## Where to add behavior @@ -69,42 +74,37 @@ fd.clean(df, cfg) | Change | Edit | |---|---| | New pandas planning rule | a `stage_*` function in `stages.py`, then add it to the tuple in `PandasBackend.suggest_plan` | -| Execute a new action | both `PandasBackend.apply_plan` and `PolarsBackend.apply_plan` | -| New config option | `models.py`, then every backend that must honor it | +| Execute a new action | both `PandasBackend.apply_plan` and `PolarsBackend.apply_plan`, plus a case in `test_every_pandas_action_runs_identically_on_polars` | +| New config option | `models.py` (including `__post_init__` validation), then every backend that must honor it | | New frame type | a backend class with `profile` / `suggest_plan` / `apply_plan`, plus a branch in `resolve_backend` | -## Known issues - -These are real defects in the current behavior. They are recorded here, not -fixed, because each fix changes output. - -- **pandas: plan and apply disagree on strings.** `stage_strings` collapses - inner whitespace and nulls `"nan"`/`"none"`/`"null"` tokens. The - `strip_strings` executor in `apply_plan` only strips. Plan statistics - (mode values, medians, `rows_affected`) come from data that `apply_plan` - does not produce. -- **pandas: `apply_plan` nulls non-string cells in object columns.** A mixed - column `[1, " a "]` becomes `[None, "a"]`. -- **pandas: protection is partial.** `stage_strings` and `stage_dtypes` modify - ID, target, and `protected=True` columns. Polars skips them. -- **Backends resolve rules differently.** pandas falls back to the defaults - per field (`rule.impute or default_impute`). Polars falls back only when the - column has no rule at all. `ColumnRule(outliers="clip")` therefore disables - default imputation on Polars only. -- **Polars ignores some actions without warning.** `drop_nulls`, - `drop_outliers`, `parse_currency`, and `impute_mode` are not executed. -- **Accepted but not implemented:** `ColumnRule.role`, `impute="ffill"`, - `outliers="flag"`, `CleanConfig.strategy`, `CleanConfig.strict_backend`. -- **Polars `LazyFrame` profiles report zeros** for row, null, and unique - counts instead of "unknown". -- **`rows_affected` overcounts** for `normalize_name`, `strip_strings`, - `parse_currency`, and `clip_iqr` (it reports the row count, not the cells - changed). Polars reports 0 for every action. +## Known backend differences + +Both backends run the same actions. These differences remain by design or +because closing them costs streaming-scale performance. + +- **Order of row operations.** pandas runs actions in plan order, so it + deduplicates before it imputes, and a row dropped for one column changes the + statistics of the next. Polars computes each column's statistics on the input + frame, runs row filters after the fused `with_columns`, and deduplicates last. +- **Statistics in plans.** pandas plans carry imputation values. Polars plans + compute them at apply time, inside the lazy query. +- **Empty strings.** pandas `strip_strings` turns `""` into a missing value. + Polars keeps `""`. +- **Currency detection.** Only the pandas planner detects currency columns + (`fix_dtypes`). Both executors run a `parse_currency` action. Polars always + returns `Float64`, and pandas may return `int64`. +- **Counts.** Polars does not count `drop_duplicates` in `rows_affected`, because + that needs a full-row hash at plan time. `CleanReport` row counts are 0 for a + `LazyFrame`, because `apply_plan` never collects. +- **Not detected:** two raw column names that normalize to the same name. ## Performance notes - The pandas path is eager. `clean()` transforms the frame twice (once to plan, once to apply), and every stage copies the frame. For large data, use a Polars `LazyFrame`. -- The pandas string stages run one Python call per cell. `_clean_str` is the +- The pandas string stages run one Python call per cell. `_strip_str` is the hottest function in `clean()`. Keep it allocation-light. +- The Polars planner runs at most one 1-row aggregate query. `profile()` also + runs one query, so Polars evaluates all columns in parallel. diff --git a/CHANGELOG.md b/CHANGELOG.md index ca33bfc8..e8d08bfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,41 @@ adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- `impute="ffill"` forward-fills a column, and `outliers="flag"` adds a boolean + `_outlier` column. Both were accepted before but did nothing. +- `fd.clean()` and `fd.Cleaner()` take `CleanConfig` fields as keyword + arguments, so the code that `freshdata-migrate` writes now runs. + +### Changed + +- `strip_strings` now means one thing everywhere: strip surrounding whitespace + and turn empty strings into missing values. pandas planning no longer + collapses inner whitespace or treats `"nan"`/`"none"`/`"null"` text as missing. +- `strict_backend=True` (the default) now raises `ValueError` for any plan + action a backend cannot run, as documented. `strict_backend=False` warns and + skips it. +- `CleanConfig` and `ColumnRule` reject unknown option values, a bare string for + `id_columns`, and a negative `iqr_factor`. Options with no effect + (`strategy`, most `role` values, `default_outliers="auto"`) warn. +- pandas `clean()` is about 33% faster on string-heavy frames. + ### Fixed +- pandas `apply_plan` no longer replaces non-string cells in object columns + with `None`. +- pandas no longer strips or currency-parses ID, target, or protected columns. + `ColumnRule(role="id"|"target")` now protects a column. +- pandas `impute_mode` keeps the value's type instead of writing a string into + numeric columns. +- Polars falls back to the config defaults per rule field, like pandas. +- Polars runs `drop_nulls`, `drop_outliers`, `parse_currency` and `impute_mode` + actions, and its planner honors `mode`, `drop` and outlier `drop` rules. + pandas runs value-less impute actions from Polars plans. +- Polars `profile()` reports real row, null and unique counts for `LazyFrame`. +- `rows_affected` counts the cells an action changes on both backends. + - Polars and DuckDB now disclose a pandas fallback for categorical and Arrow dictionary-backed pandas inputs. This preserves pandas text-cleaning and category metadata behavior rather than silently skipping or changing it. From f44a037e21b9033348427219f41cc99d08b1d65a Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:43:57 +0530 Subject: [PATCH 15/17] chore(types): keep mypy clean for the new validation and helpers Type runtime-checked config values as object, take pandas column labels as Any, import Container from collections.abc, and drop a bool check on iqr_factor that mypy proves unreachable. Co-Authored-By: Claude Opus 5.5 --- src/freshdata/_core/models.py | 10 ++++++---- src/freshdata/_core/stages.py | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/freshdata/_core/models.py b/src/freshdata/_core/models.py index fafffca4..1c3dc7f1 100644 --- a/src/freshdata/_core/models.py +++ b/src/freshdata/_core/models.py @@ -49,14 +49,16 @@ def __post_init__(self) -> None: _check_choice("strategy", self.strategy, ("conservative", "balanced", "aggressive")) _check_choice("default_impute", self.default_impute, ("auto", *_IMPUTES)) _check_choice("default_outliers", self.default_outliers, ("auto", *_OUTLIERS)) - if isinstance(self.id_columns, str): - raise ValueError(f"id_columns must be a tuple of column names, not the string {self.id_columns!r}.") + id_columns: object = self.id_columns # runtime input may not match the annotation + if isinstance(id_columns, str): + raise ValueError(f"id_columns must be a tuple of column names, not the string {id_columns!r}.") object.__setattr__(self, "id_columns", tuple(self.id_columns)) for col, rule in self.rules.items(): if not isinstance(rule, ColumnRule): raise TypeError(f"rules[{col!r}] must be a ColumnRule, not {type(rule).__name__}.") - if not isinstance(self.iqr_factor, Real) or isinstance(self.iqr_factor, bool) or self.iqr_factor < 0: - raise ValueError(f"iqr_factor must be a number >= 0, not {self.iqr_factor!r}.") + factor: object = self.iqr_factor + if not isinstance(factor, Real) or factor < 0: + raise ValueError(f"iqr_factor must be a number >= 0, not {factor!r}.") if self.strategy != "balanced": _warn_no_effect(f"CleanConfig(strategy={self.strategy!r}) has no effect in freshdata 3.x.") if self.default_outliers == "auto": diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index d67184a5..b661cabb 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -2,7 +2,8 @@ from __future__ import annotations import dataclasses import re -from typing import Tuple, List +from collections.abc import Container +from typing import Any, List, Tuple import numpy as np import pandas as pd from freshdata._core.models import ActionRecord, CleanConfig @@ -13,7 +14,7 @@ def _normalize_name(name: str) -> str: s = re.sub(r"[^\w\s]", "", name).strip().lower() return re.sub(r"[\s\-]+", "_", s) -def _is_protected(col: object, cfg: CleanConfig) -> bool: +def _is_protected(col: Any, cfg: CleanConfig) -> bool: """Single source of truth for columns that cleaning stages must not touch. ``cfg`` must already be normalized (see ``normalize_config``). @@ -24,18 +25,18 @@ def _is_protected(col: object, cfg: CleanConfig) -> bool: rule = cfg.rules.get(col) return rule is not None and (rule.protected or rule.role in ("id", "target")) -def _impute_strategy(col: object, cfg: CleanConfig) -> str | None: +def _impute_strategy(col: Any, cfg: CleanConfig) -> str | None: """Per-field fallback: a rule that leaves ``impute`` unset uses the default.""" rule = cfg.rules.get(col) default = None if cfg.default_impute == "preserve" else cfg.default_impute return (rule.impute if rule else None) or default -def _outlier_strategy(col: object, cfg: CleanConfig) -> str: +def _outlier_strategy(col: Any, cfg: CleanConfig) -> str: """Per-field fallback: a rule that leaves ``outliers`` unset uses the default.""" rule = cfg.rules.get(col) return (rule.outliers if rule else None) or cfg.default_outliers -def flag_column(col: object, existing: object) -> str: +def flag_column(col: object, existing: Container[Any]) -> str: """Name of the boolean column that outliers="flag" adds. Never overwrites a column.""" name = f"{col}_outlier" if name in existing: From 8f04493b65989f5fe5b3c3df0de76b7e570929e1 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 17:56:24 +0530 Subject: [PATCH 16/17] test: drop a duplicate pytest import Co-Authored-By: Claude Opus 5.5 --- tests/test_core_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_core_api.py b/tests/test_core_api.py index 754f4de9..a3eb5298 100644 --- a/tests/test_core_api.py +++ b/tests/test_core_api.py @@ -77,7 +77,6 @@ def test_stages_disabled_flags(): def test_clean_and_cleaner_accept_config_overrides_like_migrate_emits(): import pandas as pd - import pytest import freshdata as fd From 55ee1d5036d820b4042098de665e71917f5b5c8c Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Wed, 18 Nov 2026 18:04:02 +0530 Subject: [PATCH 17/17] perf(polars): keep plan-time counting to what the planning query gets free The Polars LazyFrame perf gate failed: total_ms +17.1% (limit +15%), driven by plan_ms 111 -> 174 ms. Counting strip_strings changes needs a per-cell string pass, and counting values outside the IQR fences needs the quantiles broadcast as columns. Together they added about 60% to planning on the 2M-row gate workload. The planner keeps the free counts (null counts for imputation, drop_nulls and ffill) and goes back to main's single-query shape. strip_strings and outlier actions keep rows_affected=0, with a rationale that says why, like drop_duplicates. pandas still counts every action. Local gate reproduction (5 iterations, 2M rows) against origin/main: plan_ms -1.4%, total_ms -15.1%. The previous head gave plan_ms +155%. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 7 +++++-- CHANGELOG.md | 4 +++- src/freshdata/_core/backends.py | 27 ++++++++++++--------------- tests/test_backend_honesty.py | 11 +++++++---- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c2c9f135..46993544 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -94,8 +94,11 @@ because closing them costs streaming-scale performance. - **Currency detection.** Only the pandas planner detects currency columns (`fix_dtypes`). Both executors run a `parse_currency` action. Polars always returns `Float64`, and pandas may return `int64`. -- **Counts.** Polars does not count `drop_duplicates` in `rows_affected`, because - that needs a full-row hash at plan time. `CleanReport` row counts are 0 for a +- **Counts.** Polars plans count only what the planning query gets for free: + null counts for imputation, `drop_nulls`, and `ffill`. `strip_strings`, + outlier actions, and `drop_duplicates` keep `rows_affected=0`, and their + `rationale` says why. Counting them added about 60% to plan time on the CI + performance gate. pandas counts every action. `CleanReport` row counts are 0 for a `LazyFrame`, because `apply_plan` never collects. - **Not detected:** two raw column names that normalize to the same name. diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d08bfe..1e3484d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,9 @@ adheres to [Semantic Versioning](https://semver.org/). actions, and its planner honors `mode`, `drop` and outlier `drop` rules. pandas runs value-less impute actions from Polars plans. - Polars `profile()` reports real row, null and unique counts for `LazyFrame`. -- `rows_affected` counts the cells an action changes on both backends. +- pandas `rows_affected` counts the cells an action changes. Polars counts the + nulls that imputation fills or `drop_nulls` removes. Where Polars does not + count (to keep planning to one cheap query), `rationale` says so. - Polars and DuckDB now disclose a pandas fallback for categorical and Arrow dictionary-backed pandas inputs. This preserves pandas text-cleaning and diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 5d98ddab..d654a037 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -136,6 +136,11 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. _reject(a, cfg, "pandas") return out +def _uncounted(cost: str) -> str: + # ponytail: Polars plans count only what the planning query gets for free (null counts). + # Strip and IQR-fence counts added ~60% to plan time on the CI perf gate. + return f"rows_affected not computed on Polars: counting needs {cost}" + _OUTLIER_ACTIONS = {"clip": "clip_iqr", "drop": "drop_outliers", "flag": "flag_iqr"} _BOUNDED_ACTIONS = {("outliers", v) for v in _OUTLIER_ACTIONS.values()} @@ -192,7 +197,7 @@ def add(action: ActionRecord, count: pl.Expr | None = None) -> None: col = pl.col(orig) if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): - add(ActionRecord("strings", name, "strip_strings", {}), (col != col.str.strip_chars()).sum()) + add(ActionRecord("strings", name, "strip_strings", {}, rationale=_uncounted("a per-cell string pass"))) # Mirror the pandas stage order: IQR bounds see the imputed column. base = col impute_strat = _impute_strategy(name, cfg_eff) @@ -215,26 +220,18 @@ def add(action: ActionRecord, count: pl.Expr | None = None) -> None: if dtype.is_numeric() and outlier_strat in _OUTLIER_ACTIONS: bounded[name] = (base, _OUTLIER_ACTIONS[outlier_strat]) if cfg_eff.drop_duplicates: - # ponytail: not counted; a full-row hash at plan time would break streaming-scale planning. - add(ActionRecord("duplicates", "*", "drop_duplicates", {}, - rationale="rows_affected not computed on Polars: counting needs a full-row hash")) + add(ActionRecord("duplicates", "*", "drop_duplicates", {}, rationale=_uncounted("a full-row hash"))) k = cfg_eff.iqr_factor stat_exprs = [e.alias(f"c{i}") for i, e in counts.items()] - quantiles: list[pl.Expr] = [] for i, (base, _) in enumerate(bounded.values()): - # Each quantile is computed once in with_columns; inlining it in the count costs about 3x. - q25, q75 = pl.col(f"\0fd_q25_{i}"), pl.col(f"\0fd_q75_{i}") - quantiles += [base.quantile(0.25, interpolation="linear").alias(f"\0fd_q25_{i}"), - base.quantile(0.75, interpolation="linear").alias(f"\0fd_q75_{i}")] - lo, hi = q25 - k * (q75 - q25), q75 + k * (q75 - q25) - stat_exprs += [base.count().alias(f"n{i}"), q25.first().alias(f"q25_{i}"), q75.first().alias(f"q75_{i}"), - ((base < lo) | (base > hi)).sum().alias(f"out{i}")] + stat_exprs += [base.count().alias(f"n{i}"), + base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), + base.quantile(0.75, interpolation="linear").alias(f"q75_{i}")] if not stat_exprs: return CleanPlan(actions=tuple(actions)) # One 1-row aggregate query; apply_plan itself never collects. - lf = df.lazy().with_columns(quantiles) if quantiles else df.lazy() - stats = lf.select(stat_exprs).collect().row(0, named=True) + stats = df.lazy().select(stat_exprs).collect().row(0, named=True) actions = [ dataclasses.replace(a, rows_affected=int(stats[f"c{i}"] or 0)) if i in counts else a for i, a in enumerate(actions) @@ -250,7 +247,7 @@ def add(action: ActionRecord, count: pl.Expr | None = None) -> None: iqr = q75 - q25 lower, upper = q25 - k * iqr, q75 + k * iqr bounded_actions.append(ActionRecord("outliers", name, action, {"lower": float(lower), "upper": float(upper)}, - int(stats[f"out{i}"]))) + rationale=_uncounted("a second pass against the IQR fences"))) # Keep the established order: outlier actions come before drop_duplicates. if cfg_eff.drop_duplicates: actions[-1:-1] = bounded_actions diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index a4b0d881..c800be53 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -229,7 +229,10 @@ def test_rows_affected_counts_changed_cells_on_both_backends(): data = {"Name X": [" a", "b", "c ", None, "d", "e"], "v": [1.0, None, 2.0, 3.0, 2.0, 100.0]} cfg = CleanConfig(default_outliers="clip") - want = {("normalize_name", 0), ("strip_strings", 2), ("impute_median", 1), ("clip_iqr", 1)} - for frame in (pd.DataFrame(data), pl.DataFrame(data), pl.LazyFrame(data)): - got = {(a.action, a.rows_affected) for a in fd.suggest_plan(frame, cfg).actions} - assert got == want, type(frame) + exact = {("normalize_name", 0), ("strip_strings", 2), ("impute_median", 1), ("clip_iqr", 1)} + assert {(a.action, a.rows_affected) for a in fd.suggest_plan(pd.DataFrame(data), cfg).actions} == exact + # Polars counts only what its planning query gets for free, and says so for the rest. + for frame in (pl.DataFrame(data), pl.LazyFrame(data)): + plan = fd.suggest_plan(frame, cfg).actions + assert {(a.action, a.rows_affected) for a in plan if not a.rationale} == {("normalize_name", 0), ("impute_median", 1)} + assert {a.action for a in plan if "not computed" in a.rationale} == {"strip_strings", "clip_iqr"}