From a6ff073c2abe22bd166d07a293304cb0dc4c947c Mon Sep 17 00:00:00 2001 From: Muhammad Ali Siddiqui Date: Thu, 19 Nov 2026 15:16:09 +0500 Subject: [PATCH] fix: prevent data loss during currency parsing --- src/freshdata/_core/stages.py | 68 +++++++++++++++++++++++++++++------ tests/test_stages.py | 20 +++++++++++ 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/freshdata/_core/stages.py b/src/freshdata/_core/stages.py index b661cabb..fe74e27d 100644 --- a/src/freshdata/_core/stages.py +++ b/src/freshdata/_core/stages.py @@ -101,17 +101,63 @@ def stage_dtypes(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List for col in out.columns: if _is_protected(col, cfg_eff): continue - if pd.api.types.is_object_dtype(out[col]) or pd.api.types.is_string_dtype(out[col]): - sample = out[col].dropna().head(20).astype(str) - if sample.empty: - continue - has_symbol = sample.str.contains(r"[\$€£¥]").any() - matches_pattern = sample.apply(lambda v: bool(CURRENCY_REGEX.match(v))) - if has_symbol and (matches_pattern.sum() / len(sample)) >= 0.7: - parsed = int(out[col].notna().sum()) - cleaned = out[col].astype(str).str.replace(r"[^\d.-]", "", regex=True) - out[col] = pd.to_numeric(cleaned, errors="coerce") - actions.append(ActionRecord("dtypes", str(col), "parse_currency", {"target_dtype": "float64"}, parsed)) + + if not ( + pd.api.types.is_object_dtype(out[col]) + or pd.api.types.is_string_dtype(out[col]) + ): + continue + + sample = out[col].dropna().head(20).astype(str) + + if sample.empty: + continue + + has_symbol = sample.str.contains(r"[\$€£¥]").any() + matches_pattern = sample.apply( + lambda v: bool(CURRENCY_REGEX.fullmatch(v)) + ) + + if not has_symbol or (matches_pattern.sum() / len(sample)) < 0.7: + continue + + # Validate original values before removing currency symbols. + valid_mask = out[col].map( + lambda value: ( + True + if pd.isna(value) + else ( + True + if isinstance(value, str) and not value.strip() + else bool(CURRENCY_REGEX.fullmatch(str(value).strip())) + ) + ) + ) + + invalid_mask = out[col].notna() & ~valid_mask + + # Skip conversion if non-currency values are present. + if invalid_mask.any(): + continue + + cleaned = out[col].astype(str).str.replace( + r"[^\d.-]", "", regex=True + ) + parsed = pd.to_numeric(cleaned, errors="coerce") + + parsed_count = int(out[col].notna().sum()) + out[col] = parsed + + actions.append( + ActionRecord( + "dtypes", + str(col), + "parse_currency", + {"target_dtype": "float64"}, + parsed_count, + ) + ) + return out, actions def stage_duplicates(df: pd.DataFrame, cfg: CleanConfig) -> Tuple[pd.DataFrame, List[ActionRecord]]: diff --git a/tests/test_stages.py b/tests/test_stages.py index 8927da93..19c0f3a9 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -175,3 +175,23 @@ def test_protected_id_and_target_columns_are_not_stripped_or_parsed(): assert out["p"].tolist() == [" $1,000 ", "$2", "$3"] assert out["user_id"].tolist() == [" 7 ", "8", "9"] assert out["free"].tolist() == [1, 2, 3] + + +def test_currency_parser_preserves_identifiers(): + df = pd.DataFrame({ + "price": ["$10.00", "$20.00", "$30.00", "REF-9921"] + }) + + result, actions = stage_dtypes(df, CleanConfig()) + + assert result["price"].tolist() == [ + "$10.00", + "$20.00", + "$30.00", + "REF-9921", + ] + + assert not any( + action.action == "parse_currency" + for action in actions + )