Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,10 @@ Preprocessor(preset="adaptive") # same numerical method, int codes, adaptive w
### Automatic feature-type detection

By default PreTab inspects each column and classifies it as numerical or categorical.
String and object columns are treated as categorical, low-cardinality integer columns are
categorical, and integer columns with enough distinct values stay numerical. Tune the
behavior with `cat_cutoff` and `treat_all_integers_as_numerical`.
String, object and boolean columns are treated as categorical, low-cardinality integer
columns are categorical, and integer columns with enough distinct values stay numerical.
A boolean column is encoded like any other binary categorical column, with the categories
`False` and `True`. Tune the behavior with `cat_cutoff` and `treat_all_integers_as_numerical`.

```python
preprocessor = Preprocessor(
Expand Down
29 changes: 25 additions & 4 deletions pretab/compose/feature_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from ..exceptions import PretabDataError, invalid_param_error

__all__ = ["detect_column_types", "to_dataframe"]
__all__ = ["bool_columns_as_object", "detect_column_types", "to_dataframe"]


def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame:
Expand Down Expand Up @@ -41,14 +41,35 @@ def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame:
return X


def bool_columns_as_object(X: pd.DataFrame) -> pd.DataFrame:
"""Return ``X`` with its boolean columns cast to ``object``.

Boolean columns are categorical (see :func:`detect_column_types`), but the
categorical pipeline starts with a :class:`~sklearn.impute.SimpleImputer`,
which rejects the ``bool`` dtype. As ``object`` columns of ``True`` /
``False`` they are encoded like any other binary categorical column; a
missing value of pandas' nullable ``boolean`` dtype becomes ``NaN``, the
missing marker the imputer recognizes. ``X`` itself is returned when it has
no boolean column, so the caller's frame is never modified.
"""
bool_columns = [label for label, dtype in X.dtypes.items() if dtype.kind == "b"]
if not bool_columns:
return X
cast = X.copy(deep=False)
for label in bool_columns:
column = X[label].astype(object)
cast[label] = column.where(column.notna(), np.nan)
return cast


def detect_column_types(X, *, cat_cutoff, treat_all_integers_as_numerical, estimator_name="Preprocessor"):
"""Classify each column of ``X`` as numerical or categorical.

An integer column is treated as categorical when its cardinality falls below
``cat_cutoff`` -- interpreted as a unique-ratio cutoff when a float, or an
absolute unique-count cutoff when an int. Non-numeric dtypes are always
categorical; ``treat_all_integers_as_numerical`` bypasses the heuristic for
integer columns.
absolute unique-count cutoff when an int. Non-numeric dtypes, including
``bool`` and pandas' nullable ``boolean``, are always categorical;
``treat_all_integers_as_numerical`` bypasses the heuristic for integer columns.

Returns
-------
Expand Down
21 changes: 18 additions & 3 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

from .compose.config import PreprocessorConfig
from .compose.factory import build_column_transformer
from .compose.feature_detection import detect_column_types, to_dataframe
from .compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe
from .compose.inspection import (
build_feature_info,
build_feature_lineage,
Expand Down Expand Up @@ -341,6 +341,10 @@ class Preprocessor(TransformerMixin, BaseEstimator):
``"dummy"`` -> ``"one-hot"``, ``"ordinal"`` / ``"label"`` -> ``"int"``, ``"poly"`` ->
``"polynomial"``, and ``"passthrough"`` -> ``"none"``.

Boolean columns (``bool`` and pandas' nullable ``boolean``) are categorical and are encoded
like any other binary categorical column, with the categories ``False`` and ``True``; a
missing value of the nullable ``boolean`` dtype is treated as missing (``NaN``).

``transform`` returns a single stacked array by default (``output_structure="matrix"``),
or a dict of per-feature blocks keyed ``num_<col>`` / ``cat_<col>`` when
``output_structure="blocks"``. Passing ``return_array`` explicitly to ``transform`` /
Expand Down Expand Up @@ -580,7 +584,7 @@ def fit(self, X, y=None, embeddings=None):
categorical_features,
sparse_threshold=sparse_threshold,
)
self.column_transformer_.fit(X, y)
self.column_transformer_.fit(self._column_transformer_input(X), y)
self.n_features_in_ = X.shape[1]

self._enforce_output_budget(X.shape[0])
Expand Down Expand Up @@ -647,7 +651,7 @@ def transform(self, X, embeddings=None, return_array=None):
output_kind = container if container in ("pandas", "polars") else ("array" if resolved_return_array else "dict")
validate_embedding_request(embeddings, expected=self.embeddings_, output_kind=output_kind)

transformed_X = self.column_transformer_.transform(X)
transformed_X = self.column_transformer_.transform(self._column_transformer_input(X))
if not sp.issparse(transformed_X):
transformed_X = np.asarray(transformed_X)
if self.dtype is not None:
Expand Down Expand Up @@ -822,6 +826,17 @@ def output_dims_(self) -> dict:
dims[columns[0]] = width
return dims

@staticmethod
def _column_transformer_input(X):
"""Adapt a frame to what the internal ColumnTransformer expects.

Boolean columns become ``object`` columns, because the categorical
imputer rejects ``bool`` (see
:func:`~pretab.compose.feature_detection.bool_columns_as_object`). The
caller's frame is never modified.
"""
return bool_columns_as_object(X)

def _output_itemsize(self) -> int:
"""Bytes per element of the dense transformed array.

Expand Down
18 changes: 17 additions & 1 deletion tests/compose/test_feature_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pandas as pd
import pytest

from pretab.compose.feature_detection import detect_column_types, to_dataframe
from pretab.compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe
from pretab.exceptions import InvalidParamError, PretabDataError


Expand Down Expand Up @@ -105,3 +105,19 @@ def test_invalid_cat_cutoff_type_raises():
df = pd.DataFrame({"x": [1, 2, 3]})
with pytest.raises(InvalidParamError):
detect_column_types(df, cat_cutoff="bad", treat_all_integers_as_numerical=False)


def test_bool_columns_as_object_casts_only_boolean_columns():
frame = pd.DataFrame({"x": [0.5, 1.5], "flag": [True, False], "nullable": pd.array([True, None], dtype="boolean")})
cast = bool_columns_as_object(frame)
assert cast["x"].dtype == np.float64
assert cast["flag"].dtype == object and cast["flag"].tolist() == [True, False]
assert cast["nullable"].dtype == object
assert cast["nullable"][0] is True and np.isnan(cast["nullable"][1])
# The caller's frame keeps its dtypes.
assert frame["flag"].dtype == bool and frame["nullable"].dtype == "boolean"


def test_bool_columns_as_object_returns_frame_without_bool_columns_unchanged():
frame = pd.DataFrame({"x": [0.5, 1.5]})
assert bool_columns_as_object(frame) is frame
67 changes: 67 additions & 0 deletions tests/integration/test_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,3 +377,70 @@ def test_unknown_feature_preprocessing_key_raises(sample_data):
X, y = sample_data
with pytest.raises(InvalidParamError, match="feature_preprocessing"):
Preprocessor(feature_preprocessing={"nnum1": "minmax"}).fit(X, y)


# --- boolean columns (issue #54) ------------------------------------------------


@pytest.fixture
def bool_frame():
return pd.DataFrame({"x": [0.1, 0.4, 0.2, 0.9, 0.5, 0.7], "flag": [True, False, True, False, True, True]})


@pytest.mark.parametrize(
("options", "expected"),
[
({}, ["cat_flag"]),
({"categorical_method": "one-hot"}, ["cat_flag_False", "cat_flag_True"]),
pytest.param(
{"categorical_method": "onehot_from_ordinal"},
["cat_flag_bin_0", "cat_flag_bin_1"],
marks=pytest.mark.filterwarnings("ignore:OneHotFromOrdinalTransformer is deprecated:DeprecationWarning"),
),
({"categorical_method": "none"}, ["cat_flag"]),
({"preset": "standard"}, ["cat_flag"]),
({"preset": "expanded"}, ["cat_flag_False", "cat_flag_True"]),
({"preset": "adaptive"}, ["cat_flag"]),
({"missing_policy": "impute"}, ["cat_flag"]),
({"missing_policy": "impute_with_indicator"}, ["cat_flag"]),
({"missing_policy": "separate_state"}, ["cat_flag__representation__flag", "cat_flag__missing__flag__missing"]),
({"treat_all_integers_as_numerical": True}, ["cat_flag"]),
],
)
def test_bool_column_is_encoded_as_a_binary_categorical(bool_frame, options, expected):
"""Regression guard for issue #54: SimpleImputer rejected the bool dtype."""
pre = Preprocessor(**options).fit(bool_frame, np.arange(6.0))
names = pre.get_feature_names_out().tolist()
assert names[-len(expected) :] == expected
assert pre.categorical_features_ == ["flag"]
assert np.asarray(pre.transform(bool_frame)).shape == (6, len(names))


def test_bool_column_integer_codes_follow_the_values(bool_frame):
pre = Preprocessor(categorical_method="int").fit(bool_frame, np.arange(6.0))
codes = np.asarray(pre.transform(bool_frame))[:, -1]
np.testing.assert_array_equal(codes, np.where(bool_frame["flag"], 2.0, 1.0))


def test_nullable_boolean_column_with_missing_values_is_imputed(bool_frame):
frame = bool_frame.astype({"flag": "boolean"})
frame.loc[2, "flag"] = pd.NA
pre = Preprocessor(categorical_method="one-hot").fit(frame, np.arange(6.0))
assert pre.get_feature_names_out().tolist()[-2:] == ["cat_flag_False", "cat_flag_True"]
one_hot = np.asarray(pre.transform(frame))[:, -2:]
np.testing.assert_array_equal(one_hot[2], [0.0, 1.0]) # imputed with the most frequent value, True


def test_nullable_boolean_missing_value_gets_the_missing_code_without_imputation(bool_frame):
frame = bool_frame.astype({"flag": "boolean"})
frame.loc[2, "flag"] = pd.NA
pre = Preprocessor(categorical_method="int", missing_policy="propagate").fit(frame, np.arange(6.0))
codes = np.asarray(pre.transform(frame))[:, -1]
np.testing.assert_array_equal(codes, [2.0, 1.0, 0.0, 1.0, 2.0, 2.0]) # False -> 1, True -> 2, missing -> 0


def test_bool_numpy_array_is_supported():
X = np.array([[True], [False], [True], [True]])
pre = Preprocessor().fit(X)
assert pre.categorical_features_ == ["feature_0"]
assert np.asarray(pre.transform(X)).shape == (4, 1)
Loading