diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7dc5b98e..46993544 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,111 +1,113 @@ # Architecture -This is a map for contributors: what lives where in `src/freshdata`, how a -`clean()` call flows through the code, and where to add new behavior. It is -deliberately high-level — read it once before your first code change, then let -the module docstrings and tests fill in the detail. +A map for contributors: what lives where in `src/freshdata`, how a call flows +through the code, and where to add behavior. Read it once before your first +change. Module docstrings and tests hold the detail. -For *user*-facing explanations of features, see the -[documentation site](https://freshcode-org.github.io/freshdata/). This file is -about the *code*. +For user-facing docs, see the +[documentation site](https://freshcode-org.github.io/freshdata/). ## The one-sentence version -`fd.clean(df)` profiles every column, asks a rule-based **decision engine** which -action each column needs, applies those actions as small vectorized **steps**, -and returns the cleaned frame plus a **report** that explains every change. +`fd.suggest_plan(df)` builds an immutable, diffable `CleanPlan` of +`ActionRecord`s. `fd.apply_plan(df, plan)` executes that plan with the backend +that matches the frame type. `fd.clean(df)` runs both in sequence. -## Core flow +## Layout ``` -fd.clean(df) # api.py — the public entry point - │ - ▼ - Cleaner # cleaner.py — orchestrates the pipeline - │ - ├─ profile the frame # profile.py — per-column stats & inferred role - │ - ├─ decision engine # engine/ — pick an action per column - │ ├─ missing.py # imputation strategy - │ ├─ outliers.py # outlier handling - │ ├─ context.py # column-role / context inference - │ └─ model_select.py # when to use model-based imputation - │ - ├─ apply cleaning steps # steps/ — pure vectorized transforms - │ ├─ missing.py dtypes.py duplicates.py - │ ├─ outliers.py strings.py columns.py - │ └─ prune.py memory.py - │ - └─ build the report # report.py — every Action, with rationale, - # risk, confidence, affected-cell count +src/freshdata/ +├── __init__.py public API re-exports +├── _core/ +│ ├── models.py frozen dataclasses: ColumnRule, CleanConfig, ActionRecord, +│ │ CleanPlan (+ diff), ColumnProfile, ProfileReport, CleanReport +│ ├── engine.py public functions + Cleaner; config defaults; report assembly +│ ├── backends.py PandasBackend, PolarsBackend, resolve_backend() +│ └── stages.py pandas planning stages + shared helpers +│ (_normalize_name, normalize_config, _is_protected) +└── cli/migrate.py freshdata-migrate: libcst codemod for 2.x -> 3.x scripts ``` -Two invariants hold everywhere in this path and are enforced in review (see -[CONTRIBUTING.md](CONTRIBUTING.md)): +Dependency direction is strictly one way: +`engine -> backends -> stages -> models`. Inside `_core`, no module imports `engine`. -1. **Everything is reported.** A transformation that changes data must record an - `Action` (see `report.py`) with an affected-cell count. -2. **Vectorized only.** No row-wise `apply` or Python loops over rows in the - cleaning path. +## Data flow -## Top-level modules +``` +fd.clean(df, cfg) + ├─ suggest_plan(df, cfg) + │ └─ resolve_backend(df) dispatch on frame type + │ ├─ PandasBackend.suggest_plan + │ │ normalize_config(cfg) + │ │ for stage in (column_names, strings, dtypes, + │ │ duplicates, missing, outliers): + │ │ df, actions = stage(df, cfg) eager; each stage sees + │ │ the previous stage's output + │ └─ PolarsBackend.suggest_plan + │ schema only + at most ONE 1-row aggregate query + │ (IQR quantiles for clip columns) + │ -> CleanPlan(actions) + └─ apply_plan(df, plan, cfg) + ├─ PandasBackend.apply_plan renames, then actions in plan order (eager) + └─ PolarsBackend.apply_plan renames -> one fused with_columns(...) -> + optional unique(); never collects +``` -| Module | Responsibility | -|---|---| -| `api.py` | The public convenience functions: `clean`, `clean_csv`, `profile`, `plan`, `learn`, `mask_dataframe`, `link`. Start here to trace any public call. | -| `cleaner.py` | The `Cleaner` front-end and the pipeline that wires profiling → engine → steps → report. | -| `profile.py` | Read-only profiling: what is in a frame and what `clean()` *would* do, with no mutation. | -| `plan.py` / `repairplan.py` | Dry-run plans (`plan.py`) and executable, reviewable repair plans (`repairplan.py`). | -| `report.py`, `result.py`, `_reportframe.py` | The report objects returned to users — `Action` records, summaries, `to_dict()`. | -| `provenance.py` | Provenance-aware cleaning for OCR / PDF / document-derived tables. | -| `guard.py` | The hard safety gate that physically protects columns from modification. | -| `findings.py`, `fieldcheck.py`, `textclean.py`, `textlint.py` | Per-cell validation (`fieldcheck`), field-aware text cleaning (`textclean`), and the normalized finding model (`findings`). | -| `insight.py`, `stakeholder.py`, `explain.py` | Presentation layers: action-oriented insight output, business-language stakeholder summaries, and per-decision explanations. | -| `config.py`, `_util.py`, `_sentinels.py` | Configuration and shared internals. | - -## Subpackages - -Each subpackage has a one-line docstring at the top of its `__init__.py` — -these summaries come from that code. - -| Package | What it does | +## Invariants + +1. **Plans are data.** `CleanPlan` and `ActionRecord` are frozen. A plan can be + serialized (`to_dict`), diffed (`diff`), and edited before it is applied. +2. **Lazy stays lazy.** `PolarsBackend.apply_plan` must not call `.collect()`, + `.to_pandas()`, or Python UDFs. `tests/test_backend_honesty.py` enforces this. +3. **One definition per decision.** `stages._is_protected` decides which + columns no stage may touch. `stages._impute_strategy` and + `stages._outlier_strategy` resolve rules against defaults. `stages._strip_str` + is the only pandas string executor. Do not copy these. +4. **Every action runs everywhere or fails loudly.** Both executors handle the + same action set. An action a backend cannot run raises `ValueError` when + `strict_backend=True` (the default), and warns and skips otherwise. +5. **pandas is required. Polars is optional.** `resolve_backend` imports Polars + only for Polars frames. + +## Where to add behavior + +| Change | Edit | |---|---| -| `engine/` | The rule-based decision engine behind `strategy=` — picks an action per column. | -| `steps/` | Individual cleaning steps, each a pure vectorized function. | -| `imputation/` | Internal imputation engines (mean/median/mode → KNN/model-based with the `ml` extra). | -| `domains/` | Domain-specific validator packs (finance, healthcare, energy, retail, transport, education, agriculture, media) plus reference data. **The most self-contained place to start** — see [CONTRIBUTING_DOMAINS.md](CONTRIBUTING_DOMAINS.md). | -| `semantic/` | The offline Semantic Cleaning Layer — experts, scoring, routing. Off by default. | -| `context/` | The deterministic natural-language context compiler (plain-English rules → policy). | -| `enterprise/` | Clustering / entity resolution, trust scoring, lineage, and PII masking. | -| `compliance/` | Maps a `CleanReport` to named control frameworks (21 CFR Part 11, GDPR, HIPAA, SOX, ALCOA+). | -| `execution/` | Pluggable out-of-core backends (Polars / DuckDB / Spark) behind a shared logical plan. | -| `streaming/` | Micro-batch and time-series-aware cleaning in bounded memory. | -| `parsers/` | Structural format readers (HL7 v2, GPX, SDMX, EDIFACT, FHIR). | -| `render/` | Interactive HTML rendering of report objects (the `viz` extra). | -| `models/` | Optional local model registry and runtime (`fd.models`). | -| `learning/` | Paired-data learning: `fd.learn` and reusable `.fdprofile` profiles. | -| `integrations/` | Run `clean` + a trust gate inside orchestrators (Dagster / Airflow / dbt). | -| `adapters/` | Optional framework adapters (Polars, etc.). | -| `experimental/` | Features whose APIs may change between minor releases (e.g. the AI Copilot). | -| `plugins.py` + `testing.py` | The entry-point plugin system and `fd.testing` contract helpers for third-party experts/backends/validators. | - -## Where do I add…? - -- **A new cleaning behavior** → a step in `steps/`, wired through the engine in - `engine/`, with an `Action` recorded in the report. Add the "does not fire - when it shouldn't" test. -- **A new domain validator** → a pack in `domains/`. Follow - [CONTRIBUTING_DOMAINS.md](CONTRIBUTING_DOMAINS.md) — this path is designed to - be self-contained and is the friendliest first code contribution. -- **A new imputation method** → `imputation/`, selected from `engine/model_select.py`. -- **A new execution backend** → `execution/` + a plugin registration; see - [`docs/plugins.md`](https://freshcode-org.github.io/freshdata/plugins/). -- **A new output/report surface** → `render/`, `insight.py`, or `stakeholder.py`. -- **A format parser** → `parsers/`. - -## What to read next - -- [CONTRIBUTING.md](CONTRIBUTING.md) — setup, the checks CI runs, and the PR workflow. -- [CONTRIBUTING_DOMAINS.md](CONTRIBUTING_DOMAINS.md) — the deep dive for domain packs. -- [Contributor roadmap](docs/community/contributor-roadmap.md) — tasks grouped by difficulty. -- The module docstring and the tests next to any file you plan to change. +| New pandas planning rule | a `stage_*` function in `stages.py`, then add it to the tuple in `PandasBackend.suggest_plan` | +| Execute a new action | both `PandasBackend.apply_plan` and `PolarsBackend.apply_plan`, plus a case in `test_every_pandas_action_runs_identically_on_polars` | +| New config option | `models.py` (including `__post_init__` validation), then every backend that must honor it | +| New frame type | a backend class with `profile` / `suggest_plan` / `apply_plan`, plus a branch in `resolve_backend` | + +## Known backend differences + +Both backends run the same actions. These differences remain by design or +because closing them costs streaming-scale performance. + +- **Order of row operations.** pandas runs actions in plan order, so it + deduplicates before it imputes, and a row dropped for one column changes the + statistics of the next. Polars computes each column's statistics on the input + frame, runs row filters after the fused `with_columns`, and deduplicates last. +- **Statistics in plans.** pandas plans carry imputation values. Polars plans + compute them at apply time, inside the lazy query. +- **Empty strings.** pandas `strip_strings` turns `""` into a missing value. + Polars keeps `""`. +- **Currency detection.** Only the pandas planner detects currency columns + (`fix_dtypes`). Both executors run a `parse_currency` action. Polars always + returns `Float64`, and pandas may return `int64`. +- **Counts.** Polars plans count only what the planning query gets for free: + null counts for imputation, `drop_nulls`, and `ffill`. `strip_strings`, + outlier actions, and `drop_duplicates` keep `rows_affected=0`, and their + `rationale` says why. Counting them added about 60% to plan time on the CI + performance gate. pandas counts every action. `CleanReport` row counts are 0 for a + `LazyFrame`, because `apply_plan` never collects. +- **Not detected:** two raw column names that normalize to the same name. + +## Performance notes + +- The pandas path is eager. `clean()` transforms the frame twice (once to plan, + once to apply), and every stage copies the frame. For large data, use a + Polars `LazyFrame`. +- The pandas string stages run one Python call per cell. `_strip_str` is the + hottest function in `clean()`. Keep it allocation-light. +- The Polars planner runs at most one 1-row aggregate query. `profile()` also + runs one query, so Polars evaluates all columns in parallel. diff --git a/CHANGELOG.md b/CHANGELOG.md index ca33bfc8..1e3484d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,43 @@ adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- `impute="ffill"` forward-fills a column, and `outliers="flag"` adds a boolean + `_outlier` column. Both were accepted before but did nothing. +- `fd.clean()` and `fd.Cleaner()` take `CleanConfig` fields as keyword + arguments, so the code that `freshdata-migrate` writes now runs. + +### Changed + +- `strip_strings` now means one thing everywhere: strip surrounding whitespace + and turn empty strings into missing values. pandas planning no longer + collapses inner whitespace or treats `"nan"`/`"none"`/`"null"` text as missing. +- `strict_backend=True` (the default) now raises `ValueError` for any plan + action a backend cannot run, as documented. `strict_backend=False` warns and + skips it. +- `CleanConfig` and `ColumnRule` reject unknown option values, a bare string for + `id_columns`, and a negative `iqr_factor`. Options with no effect + (`strategy`, most `role` values, `default_outliers="auto"`) warn. +- pandas `clean()` is about 33% faster on string-heavy frames. + ### Fixed +- pandas `apply_plan` no longer replaces non-string cells in object columns + with `None`. +- pandas no longer strips or currency-parses ID, target, or protected columns. + `ColumnRule(role="id"|"target")` now protects a column. +- pandas `impute_mode` keeps the value's type instead of writing a string into + numeric columns. +- Polars falls back to the config defaults per rule field, like pandas. +- Polars runs `drop_nulls`, `drop_outliers`, `parse_currency` and `impute_mode` + actions, and its planner honors `mode`, `drop` and outlier `drop` rules. + pandas runs value-less impute actions from Polars plans. +- Polars `profile()` reports real row, null and unique counts for `LazyFrame`. +- pandas `rows_affected` counts the cells an action changes. Polars counts the + nulls that imputation fills or `drop_nulls` removes. Where Polars does not + count (to keep planning to one cheap query), `rationale` says so. + - Polars and DuckDB now disclose a pandas fallback for categorical and Arrow dictionary-backed pandas inputs. This preserves pandas text-cleaning and category metadata behavior rather than silently skipping or changing it. diff --git a/src/freshdata/_core/backends.py b/src/freshdata/_core/backends.py index 30e11c1b..d654a037 100644 --- a/src/freshdata/_core/backends.py +++ b/src/freshdata/_core/backends.py @@ -1,5 +1,8 @@ # src/freshdata/_core/backends.py from __future__ import annotations +import dataclasses +import warnings +from numbers import Real from typing import TYPE_CHECKING, Any import pandas as pd from freshdata._core.models import ( @@ -10,6 +13,11 @@ ProfileReport, ) from freshdata._core.stages import ( + _impute_strategy, + _is_protected, + _outlier_strategy, + _strip_str, + flag_column, _normalize_name, normalize_config, stage_column_names, @@ -23,6 +31,29 @@ if TYPE_CHECKING: import polars as pl +def _reject(a: ActionRecord, cfg: CleanConfig, backend: str, reason: str = "unknown action") -> None: + """Refuse an action the backend cannot run: raise when strict, else warn and skip.""" + msg = f"{backend} backend cannot run {a.stage}/{a.action} on column {a.column!r}: {reason}" + if cfg.strict_backend: + raise ValueError(f"{msg}. Fix the plan, or set CleanConfig(strict_backend=False) to skip it with a warning.") + warnings.warn(f"{msg}. Skipped.", UserWarning, stacklevel=4) + +def _bounds(a: ActionRecord) -> tuple[float, float] | None: + lower, upper = a.parameters.get("lower"), a.parameters.get("upper") + if isinstance(lower, Real) and isinstance(upper, Real): + return float(lower), float(upper) + return None + +def _first_mode(s: pd.Series) -> Any: + modes = s.mode() + return None if modes.empty else modes.iloc[0] + +_PANDAS_FILLS = { + ("missing", "impute_median"): lambda s: s.median(), + ("missing", "impute_mean"): lambda s: s.mean(), + ("missing", "impute_mode"): _first_mode, +} + class PandasBackend: def profile(self, df: pd.DataFrame) -> ProfileReport: cols = {} @@ -60,62 +91,85 @@ def apply_plan(self, df: pd.DataFrame, plan: CleanPlan, cfg: CleanConfig) -> pd. out = out.rename(columns=renames) for a in plan.actions: - if a.stage == "duplicates" and a.action == "drop_duplicates": + key = (a.stage, a.action) + if key == ("column_names", "normalize_name"): + continue + if key == ("duplicates", "drop_duplicates"): mask = ~out.duplicated().to_numpy() out = out[mask].reset_index(drop=True) continue col = renames.get(a.column, a.column) if col not in out.columns: + _reject(a, cfg, "pandas", f"column {col!r} is not in the frame") continue + s = out[col] - if a.stage == "strings" and a.action == "strip_strings": - if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): - out[col] = out[col].astype(object).apply( - lambda x: x.strip() if isinstance(x, str) and x.strip() else None - ) - elif a.stage == "dtypes" and a.action == "parse_currency": - cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) + if key == ("strings", "strip_strings"): + if pd.api.types.is_string_dtype(s) or pd.api.types.is_object_dtype(s): + out[col] = s.astype(object).apply(_strip_str) + elif key == ("dtypes", "parse_currency"): + cleaned = s.astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") - elif a.stage == "missing": - if a.action.startswith("impute_"): - val = a.parameters.get("value") - if val is not None: - out[col] = out[col].fillna(val) - elif a.action == "drop_nulls": - mask = ~out[col].isna().to_numpy() + elif key in _PANDAS_FILLS: + val = a.parameters.get("value") + if val is None: # plans from the Polars planner carry no value + val = _PANDAS_FILLS[key](s) + if val is not None and not pd.isna(val): + out[col] = s.fillna(val) + elif key == ("missing", "impute_ffill"): + out[col] = s.ffill() + elif key == ("missing", "drop_nulls"): + out = out[s.notna().to_numpy()].reset_index(drop=True) + elif key in _BOUNDED_ACTIONS: + bounds = _bounds(a) + if bounds is None: + _reject(a, cfg, "pandas", "parameters need numeric 'lower' and 'upper'") + elif a.action == "clip_iqr": + out[col] = s.clip(lower=bounds[0], upper=bounds[1]) + elif a.action == "flag_iqr": + out[flag_column(col, out.columns)] = (s < bounds[0]) | (s > bounds[1]) + else: + mask = (s.between(*bounds) | s.isna()).to_numpy() out = out[mask].reset_index(drop=True) - elif a.stage == "outliers": - if a.action == "clip_iqr": - lower = a.parameters.get("lower") - upper = a.parameters.get("upper") - if lower is not None and upper is not None: - out[col] = out[col].clip(lower=lower, upper=upper) - elif a.action == "drop_outliers": - lower = a.parameters.get("lower") - upper = a.parameters.get("upper") - if lower is not None and upper is not None: - mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() - out = out[mask].reset_index(drop=True) + else: + _reject(a, cfg, "pandas") return out +def _uncounted(cost: str) -> str: + # ponytail: Polars plans count only what the planning query gets for free (null counts). + # Strip and IQR-fence counts added ~60% to plan time on the CI perf gate. + return f"rows_affected not computed on Polars: counting needs {cost}" + +_OUTLIER_ACTIONS = {"clip": "clip_iqr", "drop": "drop_outliers", "flag": "flag_iqr"} +_BOUNDED_ACTIONS = {("outliers", v) for v in _OUTLIER_ACTIONS.values()} + +def _pl_mode(expr: pl.Expr) -> pl.Expr: + # Smallest of the most frequent non-null values: the same tie-break as pandas Series.mode(). + return expr.drop_nulls().mode().sort().first() + class PolarsBackend: def profile(self, df: pl.DataFrame | pl.LazyFrame) -> ProfileReport: import polars as pl - frame = None if isinstance(df, pl.LazyFrame) else df - schema = df.collect_schema() if frame is None else frame.schema - row_count = 0 if frame is None else len(frame) + lf = df.lazy() + schema = lf.collect_schema() + # One aggregate query (row 0 holds every statistic), the same for DataFrame and LazyFrame input. + row = lf.select( + pl.len().alias("rows"), + *(pl.col(c).null_count().alias(f"n{i}") for i, c in enumerate(schema)), + *(pl.col(c).drop_nulls().n_unique().alias(f"u{i}") for i, c in enumerate(schema)), + ).collect().row(0, named=True) + row_count = row["rows"] cols = {} - for name, dtype in schema.items(): - role = "numeric" if dtype.is_numeric() else "categorical" - nulls = 0 if frame is None else frame[name].null_count() + for i, (name, dtype) in enumerate(schema.items()): + nulls = row[f"n{i}"] cols[name] = ColumnProfile( name=name, - inferred_role=role, + inferred_role="numeric" if dtype.is_numeric() else "categorical", null_count=nulls, null_ratio=nulls / max(row_count, 1), - unique_count=0 if frame is None else frame[name].drop_nulls().n_unique(), + unique_count=row[f"u{i}"], sample_values=(), ) return ProfileReport(row_count=row_count, column_count=len(schema), columns=cols) @@ -124,55 +178,81 @@ def suggest_plan(self, df: pl.DataFrame | pl.LazyFrame, cfg: CleanConfig) -> Cle import polars as pl schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema - actions = [] + actions: list[ActionRecord] = [] cfg_eff = normalize_config(cfg) - clip_cols: dict[str, pl.Expr] = {} + bounded: dict[str, tuple[pl.Expr, str]] = {} + counts: dict[int, pl.Expr] = {} # action index -> rows_affected, filled by the one query below + + def add(action: ActionRecord, count: pl.Expr | None = None) -> None: + if count is not None: + counts[len(actions)] = count + actions.append(action) + for orig, dtype in schema.items(): name = _normalize_name(orig) if cfg_eff.normalize_names else orig if name != orig: - actions.append(ActionRecord("column_names", orig, "normalize_name", {"to": name})) - if name in cfg_eff.id_columns or name == cfg_eff.target_column or name.endswith("_id") or name == "id": - continue - rule = cfg_eff.rules.get(name) - if rule and rule.protected: + add(ActionRecord("column_names", orig, "normalize_name", {"to": name})) + if _is_protected(name, cfg_eff): continue + col = pl.col(orig) if cfg_eff.strip_strings and (dtype == pl.Utf8 or dtype == pl.String): - actions.append(ActionRecord("strings", name, "strip_strings", {})) - if dtype.is_numeric(): - # Mirror the pandas stage order: IQR bounds see the imputed column. - base = pl.col(orig) - impute_strat = rule.impute if rule else (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) - if impute_strat in ("median", "auto"): - actions.append(ActionRecord("missing", name, "impute_median", {})) - base = base.fill_null(base.median()) - elif impute_strat == "mean": - actions.append(ActionRecord("missing", name, "impute_mean", {})) - base = base.fill_null(base.mean()) - - outlier_strat = rule.outliers if rule else cfg_eff.default_outliers - if outlier_strat == "clip": - clip_cols[name] = base - if clip_cols: - # One 1-row aggregate query; apply_plan itself never collects. - stats = df.lazy().select( - e - for i, base in enumerate(clip_cols.values()) - for e in ( - base.count().alias(f"n{i}"), - base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), - base.quantile(0.75, interpolation="linear").alias(f"q75_{i}"), - ) - ).collect().row(0, named=True) - for i, name in enumerate(clip_cols): - if stats[f"n{i}"] < 4: - continue - q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] - iqr = q75 - q25 - lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr - actions.append(ActionRecord("outliers", name, "clip_iqr", {"lower": float(lower), "upper": float(upper)})) + add(ActionRecord("strings", name, "strip_strings", {}, rationale=_uncounted("a per-cell string pass"))) + # Mirror the pandas stage order: IQR bounds see the imputed column. + base = col + impute_strat = _impute_strategy(name, cfg_eff) + if impute_strat == "drop": + add(ActionRecord("missing", name, "drop_nulls", {}), col.null_count()) + elif impute_strat == "mode": + add(ActionRecord("missing", name, "impute_mode", {}), col.null_count()) + base = base.fill_null(_pl_mode(base)) + elif impute_strat == "ffill": + base = base.forward_fill() + add(ActionRecord("missing", name, "impute_ffill", {}), col.null_count() - base.null_count()) + elif dtype.is_numeric() and impute_strat in ("median", "auto"): + add(ActionRecord("missing", name, "impute_median", {}), col.null_count()) + base = base.fill_null(base.median()) + elif dtype.is_numeric() and impute_strat == "mean": + add(ActionRecord("missing", name, "impute_mean", {}), col.null_count()) + base = base.fill_null(base.mean()) + + outlier_strat = _outlier_strategy(name, cfg_eff) + if dtype.is_numeric() and outlier_strat in _OUTLIER_ACTIONS: + bounded[name] = (base, _OUTLIER_ACTIONS[outlier_strat]) if cfg_eff.drop_duplicates: - actions.append(ActionRecord("duplicates", "*", "drop_duplicates", {})) + add(ActionRecord("duplicates", "*", "drop_duplicates", {}, rationale=_uncounted("a full-row hash"))) + + k = cfg_eff.iqr_factor + stat_exprs = [e.alias(f"c{i}") for i, e in counts.items()] + for i, (base, _) in enumerate(bounded.values()): + stat_exprs += [base.count().alias(f"n{i}"), + base.quantile(0.25, interpolation="linear").alias(f"q25_{i}"), + base.quantile(0.75, interpolation="linear").alias(f"q75_{i}")] + if not stat_exprs: + return CleanPlan(actions=tuple(actions)) + # One 1-row aggregate query; apply_plan itself never collects. + stats = df.lazy().select(stat_exprs).collect().row(0, named=True) + actions = [ + dataclasses.replace(a, rows_affected=int(stats[f"c{i}"] or 0)) if i in counts else a + for i, a in enumerate(actions) + ] + out_names = {a.parameters["to"] for a in actions if a.action == "normalize_name"} | set(schema) + bounded_actions = [] + for i, (name, (_, action)) in enumerate(bounded.items()): + if stats[f"n{i}"] < 4: + continue + if action == "flag_iqr": + flag_column(name, out_names) + q25, q75 = stats[f"q25_{i}"], stats[f"q75_{i}"] + iqr = q75 - q25 + lower, upper = q25 - k * iqr, q75 + k * iqr + bounded_actions.append(ActionRecord("outliers", name, action, {"lower": float(lower), "upper": float(upper)}, + rationale=_uncounted("a second pass against the IQR fences"))) + # Keep the established order: outlier actions come before drop_duplicates. + if cfg_eff.drop_duplicates: + actions[-1:-1] = bounded_actions + else: + actions += bounded_actions return CleanPlan(actions=tuple(actions)) def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: CleanConfig) -> pl.DataFrame | pl.LazyFrame: @@ -184,30 +264,58 @@ def apply_plan(self, df: pl.DataFrame | pl.LazyFrame, plan: CleanPlan, cfg: Clea if renames: cur = cur.rename(renames) - # 2. Column expressions, chained per column so actions compose in plan order + # 2. Column expressions, chained per column so actions compose in plan order. + # Row filters run after the fused with_columns, on the transformed columns. exprs: dict[str, pl.Expr] = {} + filters: list[pl.Expr] = [] + flags: dict[str, pl.Expr] = {} + dedup = False for a in plan.actions: + key = (a.stage, a.action) + if key == ("column_names", "normalize_name"): + continue + if key == ("duplicates", "drop_duplicates"): + dedup = True + continue col_name = renames.get(a.column, a.column) expr = exprs.get(col_name, pl.col(col_name)) - if a.stage == "strings" and a.action == "strip_strings": + val = a.parameters.get("value") + if key == ("strings", "strip_strings"): exprs[col_name] = expr.str.strip_chars() - elif a.stage == "missing": - if a.action == "impute_median": - val = a.parameters.get("value") - exprs[col_name] = expr.fill_null(val if val is not None else expr.median()) - elif a.action == "impute_mean": - val = a.parameters.get("value") - exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) - elif a.stage == "outliers" and a.action == "clip_iqr": - lower = a.parameters.get("lower") - upper = a.parameters.get("upper") - if lower is not None and upper is not None: - exprs[col_name] = expr.clip(lower, upper) - if exprs: - cur = cur.with_columns(e.alias(c) for c, e in exprs.items()) + elif key == ("dtypes", "parse_currency"): + exprs[col_name] = ( + expr.cast(pl.String).str.replace_all(r"[^\d.-]", "").cast(pl.Float64, strict=False) + ) + elif key == ("missing", "impute_median"): + exprs[col_name] = expr.fill_null(val if val is not None else expr.median()) + elif key == ("missing", "impute_mean"): + exprs[col_name] = expr.fill_null(val if val is not None else expr.mean()) + elif key == ("missing", "impute_mode"): + exprs[col_name] = expr.fill_null(val if val is not None else _pl_mode(expr)) + elif key == ("missing", "impute_ffill"): + exprs[col_name] = expr.forward_fill() + elif key == ("missing", "drop_nulls"): + filters.append(pl.col(col_name).is_not_null()) + elif key in _BOUNDED_ACTIONS: + bounds = _bounds(a) + if bounds is None: + _reject(a, cfg, "polars", "parameters need numeric 'lower' and 'upper'") + elif a.action == "clip_iqr": + exprs[col_name] = expr.clip(*bounds) + elif a.action == "flag_iqr": + names = cur.collect_schema().names() # schema only, no data + flags[flag_column(col_name, names)] = ((expr < bounds[0]) | (expr > bounds[1])).fill_null(False) + else: + filters.append(pl.col(col_name).is_between(*bounds) | pl.col(col_name).is_null()) + else: + _reject(a, cfg, "polars") + if exprs or flags: + cur = cur.with_columns(e.alias(c) for c, e in {**exprs, **flags}.items()) + if filters: + cur = cur.filter(*filters) # 3. Duplicates - if any(a.stage == "duplicates" and a.action == "drop_duplicates" for a in plan.actions): + if dedup: if isinstance(cur, pl.LazyFrame): cur = cur.unique(maintain_order=False) else: diff --git a/src/freshdata/_core/engine.py b/src/freshdata/_core/engine.py index e0ed95fc..510f8d8e 100644 --- a/src/freshdata/_core/engine.py +++ b/src/freshdata/_core/engine.py @@ -1,5 +1,6 @@ # src/freshdata/_core/engine.py from __future__ import annotations +import dataclasses from typing import Any from freshdata._core.backends import resolve_backend from freshdata._core.models import ( @@ -12,6 +13,15 @@ ProfileReport, ) +def _with_overrides(config: CleanConfig | None, overrides: dict[str, Any]) -> CleanConfig: + cfg = config or CleanConfig() + return dataclasses.replace(cfg, **overrides) if overrides else cfg + +def _column_count(frame: Any) -> int: + if hasattr(frame, "collect_schema"): # Polars: schema only, no data and no PerformanceWarning + return len(frame.collect_schema()) + return len(frame.columns) if hasattr(frame, "columns") else 0 + def profile(df: Any) -> ProfileReport: return resolve_backend(df).profile(df) @@ -30,8 +40,8 @@ def apply_plan( if return_report: orig_r = 0 if type(df).__name__ == "LazyFrame" else len(df) clean_r = 0 if type(cleaned).__name__ == "LazyFrame" else len(cleaned) - orig_c = len(df.columns) if hasattr(df, "columns") else 0 - clean_c = len(cleaned.columns) if hasattr(cleaned, "columns") else 0 + orig_c = _column_count(df) + clean_c = _column_count(cleaned) rep = CleanReport(orig_r, clean_r, orig_c, clean_c, plan.actions) return cleaned, rep return cleaned @@ -40,14 +50,16 @@ def clean( df: Any, config: CleanConfig | None = None, return_report: bool = False, + **overrides: Any, ) -> Any: - cfg = config or CleanConfig() + """Plan and apply in one call. Keyword overrides replace CleanConfig fields.""" + cfg = _with_overrides(config, overrides) plan = suggest_plan(df, cfg) return apply_plan(df, plan, cfg, return_report=return_report) class Cleaner: - def __init__(self, config: CleanConfig | None = None): - self.config = config or CleanConfig() + def __init__(self, config: CleanConfig | None = None, **overrides: Any): + self.config = _with_overrides(config, overrides) def profile(self, df: Any) -> ProfileReport: return profile(df) diff --git a/src/freshdata/_core/models.py b/src/freshdata/_core/models.py index 55c2d0a5..1c3dc7f1 100644 --- a/src/freshdata/_core/models.py +++ b/src/freshdata/_core/models.py @@ -1,8 +1,21 @@ # src/freshdata/_core/models.py from __future__ import annotations +import warnings from dataclasses import asdict, dataclass, field +from numbers import Real from typing import Any, Literal, Mapping +_ROLES = ("id", "target", "numeric", "categorical", "datetime", "text") +_IMPUTES = ("preserve", "drop", "mean", "median", "mode", "ffill") +_OUTLIERS = ("preserve", "clip", "drop", "flag") + +def _check_choice(name: str, value: object, allowed: tuple[str, ...]) -> None: + if value not in allowed: + raise ValueError(f"{name}={value!r} is not valid. Use one of: {', '.join(allowed)}.") + +def _warn_no_effect(message: str) -> None: + warnings.warn(message, UserWarning, stacklevel=4) + @dataclass(frozen=True) class ColumnRule: role: Literal["id", "target", "numeric", "categorical", "datetime", "text"] | None = None @@ -10,6 +23,13 @@ class ColumnRule: outliers: Literal["preserve", "clip", "drop", "flag"] | None = None protected: bool = False + def __post_init__(self) -> None: + for name, allowed in (("role", _ROLES), ("impute", _IMPUTES), ("outliers", _OUTLIERS)): + if getattr(self, name) is not None: + _check_choice(name, getattr(self, name), allowed) + if self.role not in (None, "id", "target"): + _warn_no_effect(f"ColumnRule(role={self.role!r}) has no effect. Only 'id' and 'target' (protect the column) do.") + @dataclass(frozen=True) class CleanConfig: strategy: Literal["conservative", "balanced", "aggressive"] = "balanced" @@ -20,11 +40,30 @@ class CleanConfig: strip_strings: bool = True fix_dtypes: bool = True drop_duplicates: bool = False - default_impute: Literal["auto", "preserve", "median", "mode"] = "auto" - default_outliers: Literal["auto", "preserve", "clip", "flag"] = "preserve" + default_impute: Literal["auto", "preserve", "drop", "mean", "median", "mode", "ffill"] = "auto" + default_outliers: Literal["auto", "preserve", "clip", "drop", "flag"] = "preserve" iqr_factor: float = 1.5 strict_backend: bool = True + def __post_init__(self) -> None: + _check_choice("strategy", self.strategy, ("conservative", "balanced", "aggressive")) + _check_choice("default_impute", self.default_impute, ("auto", *_IMPUTES)) + _check_choice("default_outliers", self.default_outliers, ("auto", *_OUTLIERS)) + id_columns: object = self.id_columns # runtime input may not match the annotation + if isinstance(id_columns, str): + raise ValueError(f"id_columns must be a tuple of column names, not the string {id_columns!r}.") + object.__setattr__(self, "id_columns", tuple(self.id_columns)) + for col, rule in self.rules.items(): + if not isinstance(rule, ColumnRule): + raise TypeError(f"rules[{col!r}] must be a ColumnRule, not {type(rule).__name__}.") + factor: object = self.iqr_factor + if not isinstance(factor, Real) or factor < 0: + raise ValueError(f"iqr_factor must be a number >= 0, not {factor!r}.") + if self.strategy != "balanced": + _warn_no_effect(f"CleanConfig(strategy={self.strategy!r}) has no effect in freshdata 3.x.") + if self.default_outliers == "auto": + _warn_no_effect("CleanConfig(default_outliers='auto') leaves outliers unchanged, like 'preserve'.") + @dataclass(frozen=True) class ActionRecord: stage: str diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index 7a638bbb..b661cabb 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -2,10 +2,11 @@ from __future__ import annotations import dataclasses import re -from typing import Tuple, List +from collections.abc import Container +from typing import Any, List, Tuple import numpy as np import pandas as pd -from freshdata._core.models import ActionRecord, CleanConfig, ColumnRule +from freshdata._core.models import ActionRecord, CleanConfig CURRENCY_REGEX = re.compile(r"^\s*[-+]?\s*[\$€£¥]?\s*[\d,]+(?:\.\d+)?\s*[\$€£¥]?\s*$") @@ -13,6 +14,45 @@ def _normalize_name(name: str) -> str: s = re.sub(r"[^\w\s]", "", name).strip().lower() return re.sub(r"[\s\-]+", "_", s) +def _is_protected(col: Any, cfg: CleanConfig) -> bool: + """Single source of truth for columns that cleaning stages must not touch. + + ``cfg`` must already be normalized (see ``normalize_config``). + """ + name = str(col) + if col in cfg.id_columns or col == cfg.target_column or name.endswith("_id") or name == "id": + return True + rule = cfg.rules.get(col) + return rule is not None and (rule.protected or rule.role in ("id", "target")) + +def _impute_strategy(col: Any, cfg: CleanConfig) -> str | None: + """Per-field fallback: a rule that leaves ``impute`` unset uses the default.""" + rule = cfg.rules.get(col) + default = None if cfg.default_impute == "preserve" else cfg.default_impute + return (rule.impute if rule else None) or default + +def _outlier_strategy(col: Any, cfg: CleanConfig) -> str: + """Per-field fallback: a rule that leaves ``outliers`` unset uses the default.""" + rule = cfg.rules.get(col) + return (rule.outliers if rule else None) or cfg.default_outliers + +def flag_column(col: object, existing: Container[Any]) -> str: + """Name of the boolean column that outliers="flag" adds. Never overwrites a column.""" + name = f"{col}_outlier" + if name in existing: + raise ValueError(f"outliers='flag' on {col!r} would overwrite the existing column {name!r}") + return name + +def _strip_str(x: object) -> object: + """The one strip_strings executor, shared by planning and pandas apply_plan. + + Strips surrounding whitespace and turns empty strings into None. Missing + values become None. Other non-string cells are returned unchanged. + """ + if isinstance(x, str): + return x.strip() or None + return None if pd.isna(x) else x + def normalize_config(cfg: CleanConfig) -> CleanConfig: if not cfg.normalize_names: return cfg @@ -30,7 +70,7 @@ def stage_column_names(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame norm = _normalize_name(str(col)) if norm != col: rename_map[col] = norm - actions.append(ActionRecord("column_names", str(col), "normalize_name", {"to": norm}, len(df))) + actions.append(ActionRecord("column_names", str(col), "normalize_name", {"to": norm}, 0)) out = df.rename(columns=rename_map) if rename_map else df return out, actions @@ -39,21 +79,17 @@ def stage_strings(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis return df, [] actions: List[ActionRecord] = [] out = df.copy() + cfg_eff = normalize_config(cfg) for col in out.columns: + if _is_protected(col, cfg_eff): + continue if pd.api.types.is_string_dtype(out[col]) or pd.api.types.is_object_dtype(out[col]): - s = out[col].astype(object) - def _clean_str(x): - if pd.isna(x) or x is None: - return None - if isinstance(x, str): - val = re.sub(r"\s+", " ", x.strip()) - if not val or val.lower() in ("nan", "none", "null"): - return None - return val - return x - - out[col] = s.apply(_clean_str) - actions.append(ActionRecord("strings", str(col), "strip_strings", {}, len(out))) + # Series.apply (not .str) keeps pandas' result-dtype inference unchanged. + before = out[col].astype(object) + after = before.apply(_strip_str) + changed = (before.astype(str) != after.astype(str)) & ~(before.isna() & after.isna()) + out[col] = after + actions.append(ActionRecord("strings", str(col), "strip_strings", {}, int(changed.sum()))) return out, actions def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: @@ -61,7 +97,10 @@ def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List return df, [] actions: List[ActionRecord] = [] out = df.copy() + cfg_eff = normalize_config(cfg) for col in out.columns: + if _is_protected(col, cfg_eff): + continue if pd.api.types.is_object_dtype(out[col]) or pd.api.types.is_string_dtype(out[col]): sample = out[col].dropna().head(20).astype(str) if sample.empty: @@ -69,9 +108,10 @@ def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List has_symbol = sample.str.contains(r"[\$€£¥]").any() matches_pattern = sample.apply(lambda v: bool(CURRENCY_REGEX.match(v))) if has_symbol and (matches_pattern.sum() / len(sample)) >= 0.7: + parsed = int(out[col].notna().sum()) cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) out[col] = pd.to_numeric(cleaned, errors="coerce") - actions.append(ActionRecord("dtypes", str(col), "parse_currency", {"target_dtype": "float64"}, len(out))) + actions.append(ActionRecord("dtypes", str(col), "parse_currency", {"target_dtype": "float64"}, parsed)) return out, actions def stage_duplicates(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: @@ -89,12 +129,9 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis out = df.copy() cfg_eff = normalize_config(cfg) for col in out.columns: - if col in cfg_eff.id_columns or col == cfg_eff.target_column or str(col).endswith("_id") or str(col) == "id": + if _is_protected(col, cfg_eff): continue - rule = cfg_eff.rules.get(col, ColumnRule()) - if rule.protected: - continue - impute_strategy = rule.impute or (cfg_eff.default_impute if cfg_eff.default_impute != "preserve" else None) + impute_strategy = _impute_strategy(col, cfg_eff) null_count = int(out[col].isna().sum()) if impute_strategy == "drop" and (null_count > 0 or len(out) == 0): orig_len = len(out) @@ -118,22 +155,26 @@ def stage_missing(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Lis mode_vals = out[col].mode() if not mode_vals.empty and not pd.isna(mode_vals.iloc[0]): val = mode_vals.iloc[0] + val = val.item() if isinstance(val, np.generic) else val # plain Python scalar, original type out[col] = out[col].fillna(val) - actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": str(val)}, null_count)) + actions.append(ActionRecord("missing", str(col), "impute_mode", {"value": val}, null_count)) + elif impute_strategy == "ffill": + filled = out[col].ffill() + filled_count = null_count - int(filled.isna().sum()) + if filled_count: + out[col] = filled + actions.append(ActionRecord("missing", str(col), "impute_ffill", {}, filled_count)) return out, actions def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: actions: List[ActionRecord] = [] out = df.copy() cfg_eff = normalize_config(cfg) - for col in out.columns: - if col in cfg_eff.id_columns or col == cfg_eff.target_column or str(col).endswith("_id") or str(col) == "id": - continue - rule = cfg_eff.rules.get(col, ColumnRule()) - if rule.protected: + for col in list(out.columns): + if _is_protected(col, cfg_eff): continue - strategy = rule.outliers or cfg_eff.default_outliers - if strategy not in ("clip", "drop") or not pd.api.types.is_numeric_dtype(out[col]): + strategy = _outlier_strategy(col, cfg_eff) + if strategy not in ("clip", "drop", "flag") or not pd.api.types.is_numeric_dtype(out[col]): continue s = out[col].dropna() if len(s) < 4: @@ -142,8 +183,13 @@ def stage_outliers(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, Li iqr = q75 - q25 lower, upper = q25 - cfg_eff.iqr_factor * iqr, q75 + cfg_eff.iqr_factor * iqr if strategy == "clip": + clipped = int(((out[col] < lower) | (out[col] > upper)).sum()) out[col] = out[col].clip(lower=lower, upper=upper) - actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, len(out))) + actions.append(ActionRecord("outliers", str(col), "clip_iqr", {"lower": float(lower), "upper": float(upper)}, clipped)) + elif strategy == "flag": + flag_col = flag_column(col, out.columns) + out[flag_col] = (out[col] < lower) | (out[col] > upper) + actions.append(ActionRecord("outliers", str(col), "flag_iqr", {"lower": float(lower), "upper": float(upper)}, int(out[flag_col].sum()))) else: mask = ((out[col] >= lower) & (out[col] <= upper) | out[col].isna()).to_numpy() dropped = int((~mask).sum()) diff --git a/tests/test_backend_honesty.py b/tests/test_backend_honesty.py index a052cab8..c800be53 100644 --- a/tests/test_backend_honesty.py +++ b/tests/test_backend_honesty.py @@ -113,9 +113,126 @@ def test_lazy_plan_diff_detects_rule_changes(): import freshdata as fd lf = pl.LazyFrame({"score": [1.0, 2.0, None, 4.0, 100.0]}) - conservative = fd.suggest_plan(lf, CleanConfig(strategy="conservative", default_outliers="preserve")) + conservative = fd.suggest_plan(lf, CleanConfig(default_outliers="preserve")) aggressive = fd.suggest_plan(lf, CleanConfig( - strategy="aggressive", rules={"score": fd.ColumnRule(impute="median", outliers="clip")})) + rules={"score": fd.ColumnRule(impute="median", outliers="clip")})) diff = conservative.diff(aggressive) assert [(a.column, a.action) for a in diff["added"]] == [("score", "clip_iqr")] assert diff["removed"] == [] and diff["modified"] == [] + + +@pytest.mark.parametrize("rule, default", [ + ({"outliers": "clip"}, {"default_impute": "median"}), + ({"impute": "mean"}, {"default_outliers": "clip"}), +]) +def test_rules_fall_back_to_defaults_per_field_on_both_backends(rule, default): + import freshdata as fd + + data = {"v": [1.0, None, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(rules={"v": fd.ColumnRule(**rule)}, **default) + pd_actions = {a.action for a in fd.suggest_plan(pd.DataFrame(data), cfg).actions} + pl_actions = {a.action for a in fd.suggest_plan(pl.DataFrame(data), cfg).actions} + assert pd_actions == pl_actions and len(pl_actions) == 2 + + +def _rows(frame): + if isinstance(frame, pl.LazyFrame): + frame = frame.collect() + if isinstance(frame, pl.DataFrame): + frame = frame.to_pandas() + return sorted(map(tuple, frame.astype(object).where(frame.notna(), None).values.tolist()), key=repr) + + +def test_every_pandas_action_runs_identically_on_polars(): + import freshdata as fd + + data = {"price": ["$1", "$2", "$3", None, "$5", "$6"], "tag": ["a", "a", None, "b", "b", "a"], + "v": [1.0, None, 2.0, 3.0, 2.0, 2.5], "w": [1.0, 2.0, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(rules={"tag": fd.ColumnRule(impute="mode"), "v": fd.ColumnRule(impute="drop"), + "w": fd.ColumnRule(outliers="drop")}, strip_strings=False) + plan = fd.suggest_plan(pd.DataFrame(data), cfg) + assert {a.action for a in plan.actions} >= {"parse_currency", "impute_mode", "drop_nulls", "drop_outliers"} + expected = _rows(fd.apply_plan(pd.DataFrame(data), plan, cfg)) + assert _rows(fd.apply_plan(pl.DataFrame(data), plan, cfg)) == expected + lazy = fd.apply_plan(pl.LazyFrame(data), plan, cfg) + assert "PYTHON_UDF" not in lazy.explain() + assert _rows(lazy) == expected + + +def test_polars_planner_honors_mode_drop_and_outlier_drop_rules(): + import freshdata as fd + + data = {"tag": ["a", "a", None, "b"], "v": [1.0, None, 2.0, 3.0], "w": [1.0, 2.0, 2.0, 3.0]} + cfg = CleanConfig(rules={"tag": fd.ColumnRule(impute="mode"), "v": fd.ColumnRule(impute="drop"), + "w": fd.ColumnRule(outliers="drop")}) + actions = {(a.column, a.action) for a in fd.suggest_plan(pl.LazyFrame(data), cfg).actions} + assert {("tag", "impute_mode"), ("v", "drop_nulls"), ("w", "drop_outliers")} <= actions + + +def test_polars_plan_without_values_imputes_on_pandas(): + import freshdata as fd + + data = {"v": [1.0, None, 3.0], "t": ["x", None, "x"]} + cfg = CleanConfig(rules={"t": fd.ColumnRule(impute="mode")}) + plan = fd.suggest_plan(pl.DataFrame(data), cfg) + out = fd.apply_plan(pd.DataFrame(data), plan, cfg) + assert out["v"].tolist() == [1.0, 2.0, 3.0] and out["t"].tolist() == ["x", "x", "x"] + + +@pytest.mark.parametrize("frame", [pd.DataFrame({"v": [1.0]}), pl.DataFrame({"v": [1.0]})]) +def test_strict_backend_rejects_actions_it_cannot_run(frame): + import freshdata as fd + + for bad in (ActionRecord("missing", "v", "impute_magic"), ActionRecord("outliers", "v", "clip_iqr", {})): + with pytest.raises(ValueError, match="cannot run"): + fd.apply_plan(frame, CleanPlan((bad,))) + with pytest.warns(UserWarning, match="Skipped"): + out = fd.apply_plan(frame, CleanPlan((bad,)), CleanConfig(strict_backend=False)) + assert _rows(out) == _rows(frame) + + +def test_ffill_and_flag_run_on_every_backend(): + import freshdata as fd + + data = {"f": [1.0, None, None, 4.0, 5.0, 6.0], "w": [1.0, 2.0, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(rules={"f": fd.ColumnRule(impute="ffill"), "w": fd.ColumnRule(outliers="flag")}) + pd_out = fd.apply_plan(pd.DataFrame(data), fd.suggest_plan(pd.DataFrame(data), cfg), cfg) + assert pd_out["f"].tolist() == [1.0, 1.0, 1.0, 4.0, 5.0, 6.0] + assert pd_out["w_outlier"].tolist() == [False] * 5 + [True] + assert pd_out["w"].tolist() == data["w"] # flag never rewrites the values + for frame in (pl.DataFrame(data), pl.LazyFrame(data)): + out = fd.apply_plan(frame, fd.suggest_plan(frame, cfg), cfg) + assert _rows(out) == _rows(pd_out) + + +@pytest.mark.parametrize("make", [pd.DataFrame, pl.DataFrame]) +def test_flag_never_overwrites_an_existing_column(make): + import freshdata as fd + + frame = make({"w": [1.0, 2.0, 2.0, 3.0, 100.0], "w_outlier": ["keep"] * 5}) + with pytest.raises(ValueError, match="overwrite"): + fd.clean(frame, CleanConfig(default_outliers="flag", strip_strings=False)) + + +def test_lazyframe_profile_reports_real_counts(): + data = {"a": [1, 2, None, 2], "b": ["x", None, None, "y"]} + lazy = resolve_backend(pl.LazyFrame(data)).profile(pl.LazyFrame(data)) + eager = resolve_backend(pd.DataFrame(data)).profile(pd.DataFrame(data)) + assert lazy.row_count == 4 + for col in data: + p, q = eager.columns[col], lazy.columns[col] + assert (q.null_count, q.unique_count, q.null_ratio) == (p.null_count, p.unique_count, p.null_ratio) + + +def test_rows_affected_counts_changed_cells_on_both_backends(): + import freshdata as fd + + data = {"Name X": [" a", "b", "c ", None, "d", "e"], "v": [1.0, None, 2.0, 3.0, 2.0, 100.0]} + cfg = CleanConfig(default_outliers="clip") + exact = {("normalize_name", 0), ("strip_strings", 2), ("impute_median", 1), ("clip_iqr", 1)} + assert {(a.action, a.rows_affected) for a in fd.suggest_plan(pd.DataFrame(data), cfg).actions} == exact + # Polars counts only what its planning query gets for free, and says so for the rest. + for frame in (pl.DataFrame(data), pl.LazyFrame(data)): + plan = fd.suggest_plan(frame, cfg).actions + assert {(a.action, a.rows_affected) for a in plan if not a.rationale} == {("normalize_name", 0), ("impute_median", 1)} + assert {a.action for a in plan if "not computed" in a.rationale} == {"strip_strings", "clip_iqr"} diff --git a/tests/test_core_api.py b/tests/test_core_api.py index b903b55b..a3eb5298 100644 --- a/tests/test_core_api.py +++ b/tests/test_core_api.py @@ -73,3 +73,17 @@ def test_stages_disabled_flags(): cleaned = fd.clean(df, config=cfg) assert " Messy Name " in cleaned.columns assert cleaned[" Messy Name "].iloc[0] == " val " + + +def test_clean_and_cleaner_accept_config_overrides_like_migrate_emits(): + import pandas as pd + + import freshdata as fd + + df = pd.DataFrame({"v": [1.0, None, 5.0]}) + assert fd.clean(df, default_impute="mean")["v"].tolist() == [1.0, 3.0, 5.0] + cfg = fd.CleanConfig(default_impute="preserve") + assert fd.clean(df, cfg, default_impute="median")["v"].tolist() == [1.0, 3.0, 5.0] + assert fd.Cleaner(id_columns=["v"]).config.id_columns == ("v",) + with pytest.raises(TypeError): + fd.clean(df, not_an_option=True) diff --git a/tests/test_models.py b/tests/test_models.py index cb52db1a..fe9c454b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -62,3 +62,43 @@ def test_clean_plan_json_roundtrip(): assert len(loaded["actions"]) == 1 assert loaded["actions"][0]["stage"] == "missing" + + +@pytest.mark.parametrize("make", [ + lambda: ColumnRule(impute="medain"), + lambda: ColumnRule(outliers="cap"), + lambda: ColumnRule(role="key"), + lambda: CleanConfig(default_impute="knn"), + lambda: CleanConfig(strategy="fast"), + lambda: CleanConfig(id_columns="customer_id"), + lambda: CleanConfig(iqr_factor=-1), +]) +def test_invalid_options_are_rejected(make): + with pytest.raises(ValueError): + make() + + +def test_rules_must_be_column_rules_and_id_columns_become_a_tuple(): + with pytest.raises(TypeError): + CleanConfig(rules={"a": {"impute": "mean"}}) + assert CleanConfig(id_columns=["a", "b"]).id_columns == ("a", "b") + + +def test_options_without_effect_warn(): + with pytest.warns(UserWarning, match="no effect"): + CleanConfig(strategy="aggressive") + with pytest.warns(UserWarning, match="no effect"): + ColumnRule(role="numeric") + with pytest.warns(UserWarning, match="like 'preserve'"): + CleanConfig(default_outliers="auto") + + +def test_role_id_and_target_protect_the_column(): + import pandas as pd + + import freshdata as fd + + df = pd.DataFrame({"k": [" a ", None, " a "], "v": [1.0, None, 3.0]}) + cfg = CleanConfig(rules={"k": ColumnRule(role="id"), "v": ColumnRule(role="target")}, default_impute="mode") + out = fd.clean(df, cfg) + assert out["k"].tolist()[0] == " a " and out["v"].isna().sum() == 1 diff --git a/tests/test_property_invariants.py b/tests/test_property_invariants.py index fe53c24d..6633efb7 100644 --- a/tests/test_property_invariants.py +++ b/tests/test_property_invariants.py @@ -35,7 +35,6 @@ def dirty_dataframe_strategy(draw: st.DrawFn) -> pd.DataFrame: @given(df=dirty_dataframe_strategy()) def test_five_hard_invariants(df: pd.DataFrame) -> None: cfg = CleanConfig( - strategy="conservative", fix_dtypes=False, drop_duplicates=True, default_impute="preserve", diff --git a/tests/test_stages.py b/tests/test_stages.py index 05fb3bd2..8927da93 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -123,3 +123,55 @@ def test_stage_outliers_strategies(strategy, values, expected, actions): out, acts = stage_outliers(df, CleanConfig(rules={"x": ColumnRule(outliers=strategy)})) assert out["x"].tolist() == pytest.approx(expected, nan_ok=True) assert [a.action for a in acts] == actions + + +def test_strip_str_strips_and_keeps_non_strings(): + import numpy as np + + from freshdata._core.stages import _strip_str + + cells = (" a b ", " ", "null", None, np.nan, pd.NA, 3, 2.5, b"b") + assert [_strip_str(x) for x in cells] == ["a b", None, "null", None, None, None, 3, 2.5, b"b"] + + +def test_plan_statistics_match_applied_strings(): + import freshdata as fd + + df = pd.DataFrame({"s": [" x ", "x", "null", " ", None], "mixed": pd.Series([1, " a ", 2.5, None, "b"], dtype=object)}) + cfg = CleanConfig(rules={"s": ColumnRule(impute="mode")}) + plan = fd.suggest_plan(df, cfg) + out = fd.apply_plan(df, plan, cfg) + mode = next(a for a in plan.actions if a.action == "impute_mode") + assert mode.rows_affected == 2 # " " and None, the cells apply_plan leaves missing + assert out["s"].tolist() == ["x", "x", "null", "x", "x"] + assert out["mixed"].tolist()[:3] == [1, "a", 2.5] + + +def test_is_protected_covers_ids_target_suffix_and_rules(): + from freshdata._core.stages import _is_protected + + cfg = CleanConfig(id_columns=("key",), target_column="y", rules={"p": ColumnRule(protected=True)}) + assert [_is_protected(c, cfg) for c in ("key", "y", "user_id", "id", "p", "x", 7)] == [ + True, True, True, True, True, False, False, + ] + + +def test_mode_imputation_keeps_value_type(): + import freshdata as fd + + df = pd.DataFrame({"v": [1.0, 1.0, None, 2.0]}) + cfg = CleanConfig(rules={"v": ColumnRule(impute="mode")}) + plan = fd.suggest_plan(df, cfg) + assert plan.actions[-1].parameters == {"value": 1.0} + out = fd.apply_plan(df, plan, cfg) + assert out["v"].dtype == "float64" and out["v"].tolist() == [1.0, 1.0, 1.0, 2.0] + + +def test_protected_id_and_target_columns_are_not_stripped_or_parsed(): + import freshdata as fd + + df = pd.DataFrame({"p": [" $1,000 ", "$2", "$3"], "user_id": [" 7 ", "8", "9"], "free": [" $1 ", "$2", "$3"]}) + out = fd.clean(df, CleanConfig(rules={"p": ColumnRule(protected=True)})) + assert out["p"].tolist() == [" $1,000 ", "$2", "$3"] + assert out["user_id"].tolist() == [" 7 ", "8", "9"] + assert out["free"].tolist() == [1, 2, 3]