diff --git a/docs/core_concepts/configuration.md b/docs/core_concepts/configuration.md index c5fa21d..f724512 100644 --- a/docs/core_concepts/configuration.md +++ b/docs/core_concepts/configuration.md @@ -51,6 +51,15 @@ example `{"feature_0": "rbf"}`. Check `numerical_features_` / `categorical_featu overrides, rather than guessing the column order. ``` +At `transform`, array columns are matched by position, so the training and serving +containers may differ, as in scikit-learn. A `Preprocessor` fitted on a `DataFrame` accepts +the same data as an array of the same width (for example `df.to_numpy()` in NumPy-based +serving or an explainer such as `shap.KernelExplainer`). It warns that `X` does not have +valid feature names and re-infers the numeric columns of an object array. One fitted on an +array accepts a `DataFrame` of the same width the same way, with a warning that `X` has +feature names. Input matched by position with a different number of columns raises the usual +`X has 4 features, but Preprocessor is expecting 3 features as input.` error. + ## Per-feature overrides Columns rarely want identical treatment. The `feature_preprocessing` dict assigns a strategy diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..3e384f3 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -613,7 +613,12 @@ def transform(self, X, embeddings=None, return_array=None): Parameters ---------- X : pandas.DataFrame, numpy.ndarray, or dict - Input features to transform. + Input features to transform. A DataFrame or dict is matched to the fitted + columns by name. A NumPy array is matched by position: after a fit on named + columns it takes those names (numeric columns of an object array are + re-inferred), with scikit-learn's usual warning that ``X`` does not have + valid feature names. Likewise, a DataFrame passed after a fit on an array + is matched by position, with a warning that ``X`` has feature names. embeddings : np.ndarray or list of np.ndarray, optional External embeddings to attach to dictionary output. Required when embeddings were supplied during ``fit`` and unsupported when the @@ -632,11 +637,17 @@ def transform(self, X, embeddings=None, return_array=None): (or ``return_array=False``); a SciPy CSR matrix (or CSR blocks) when ``output_format`` resolves to ``"sparse"``; or a pandas / polars DataFrame when configured via :meth:`set_output`. + + Raises + ------ + PretabDataError + If input matched by position has a different number of columns than + ``n_features_in_``: it could never be routed correctly. """ check_is_fitted(self) - X = to_dataframe(X, copy=True) + X = self._align_input(X) if self.missing_policy == "error": self._reject_missing(X) @@ -822,6 +833,47 @@ def output_dims_(self) -> dict: dims[columns[0]] = width return dims + def _align_input(self, X): + """Return ``X`` as a DataFrame whose columns line up with the fitted columns. + + A NumPy array is matched by position. Once the preprocessor was fitted on + named columns, an array of the fitted width takes those column labels + (numeric columns of an object array are re-inferred), and a DataFrame + passed to a preprocessor fitted on an array is matched by position too -- + both with scikit-learn's usual warning about the missing or unexpected + feature names. Any other DataFrame is matched by column label. + + Raises + ------ + PretabDataError + If array-like input matched by position has a different number of + columns than seen during ``fit``: it could never be routed correctly. + """ + # The ColumnTransformer records the labels it was fitted on whenever they are + # all strings; an array fit shows up as the synthetic positional labels. + fitted_labels = getattr(self.column_transformer_, "feature_names_in_", None) + positional_labels = [f"feature_{i}" for i in range(self.n_features_in_)] + fitted_on_array = fitted_labels is not None and list(fitted_labels) == positional_labels + + is_array = isinstance(X, np.ndarray) + X = to_dataframe(X, copy=True) + if not (is_array or (fitted_on_array and list(X.columns) != positional_labels)): + return X + if X.shape[1] != self.n_features_in_: + raise PretabDataError( + f"X has {X.shape[1]} features, but {type(self).__name__} is expecting " + f"{self.n_features_in_} features as input." + ) + if fitted_labels is None or list(X.columns) == list(fitted_labels): + return X + if fitted_on_array: + message = f"X has feature names, but {type(self).__name__} was fitted without feature names" + else: + message = f"X does not have valid feature names, but {type(self).__name__} was fitted with feature names" + warnings.warn(message, UserWarning, stacklevel=3) + X = X.set_axis(list(fitted_labels), axis=1) + return X.infer_objects() if is_array else X + def _output_itemsize(self) -> int: """Bytes per element of the dense transformed array. diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index df9d224..bbc1adf 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -377,3 +377,52 @@ def test_unknown_feature_preprocessing_key_raises(sample_data): X, y = sample_data with pytest.raises(InvalidParamError, match="feature_preprocessing"): Preprocessor(feature_preprocessing={"nnum1": "minmax"}).fit(X, y) + + +# --- positional input after a named fit (and vice versa) (issue #78) --------------- + + +@pytest.fixture +def mixed_named_frame(): + rng = np.random.default_rng(0) + frame = pd.DataFrame( + { + "age": rng.normal(40, 10, 100), + "income": rng.normal(50, 10, 100), + "city": rng.choice(["a", "b", "c"], 100), + } + ) + return frame, rng.normal(size=100) + + +def test_frame_fit_accepts_an_array_of_the_fitted_width(mixed_named_frame): + """Regression guard for issue #78: a DataFrame-fitted preprocessor rejected the same + data as an array ('columns are missing'), breaking NumPy-based serving and explainers.""" + from sklearn.linear_model import Ridge + from sklearn.pipeline import make_pipeline + + X, y = mixed_named_frame + pipe = make_pipeline(Preprocessor(random_state=0), Ridge()).fit(X, y) + with pytest.warns(UserWarning, match="does not have valid feature names"): + from_array = pipe.predict(X.to_numpy()) + np.testing.assert_allclose(from_array, pipe.predict(X)) + + +def test_array_fit_accepts_a_frame_of_the_fitted_width(mixed_named_frame): + X, y = mixed_named_frame + numeric = X[["age", "income"]] + pre = Preprocessor(numerical_method="minmax").fit(numeric.to_numpy(), y) + with pytest.warns(UserWarning, match="fitted without feature names"): + from_frame = pre.transform(numeric.rename(columns={"age": "u", "income": "v"})) + np.testing.assert_array_equal(from_frame, pre.transform(numeric.to_numpy())) + + +def test_positional_input_of_the_wrong_width_is_rejected(mixed_named_frame): + X, y = mixed_named_frame + pre = Preprocessor(random_state=0).fit(X, y) + with pytest.raises(PretabDataError, match="X has 2 features, but Preprocessor is expecting 3"): + pre.transform(X.to_numpy()[:, :2]) + numeric = X[["age", "income"]] + array_fit = Preprocessor(numerical_method="minmax").fit(numeric.to_numpy(), y) + with pytest.raises(PretabDataError, match="X has 1 features"): + array_fit.transform(X[["age"]])