diff --git a/configs/common/covariance.json b/configs/common/covariance.json new file mode 100644 index 00000000..37b9d2f7 --- /dev/null +++ b/configs/common/covariance.json @@ -0,0 +1,15 @@ +{ + "INCLUDE": ["../common/sklearn.json"], + "PARAMETERS_SETS": { + "covariance parameters": { + "algorithm": { + "estimator": "EmpiricalCovariance", + "estimator_params": { + "store_precision": [true, false], + "assume_centered": false + }, + "estimator_methods": { "training": "fit" } + } + } + } +} diff --git a/configs/common/sklearn.json b/configs/common/sklearn.json index bae07393..a3e00aed 100644 --- a/configs/common/sklearn.json +++ b/configs/common/sklearn.json @@ -18,6 +18,24 @@ "data": { "format": "dpnp", "order": "C" } } ], + "sklearn-ex[cpu] preview implementations": { + "algorithm": [ + { "library": "sklearn", "device": "cpu" }, + { "library": "sklearnex.preview", "device": "cpu" } + ] + }, + "sklearn-ex[cpu,gpu] preview implementations": [ + { "algorithm": { "library": "sklearn", "device": "cpu" } }, + { "algorithm": { "library": "sklearnex.preview", "device": "cpu" } }, + { + "algorithm": { + "library": "sklearnex.preview", + "device": "gpu", + "sklearn_context": { "array_api_dispatch": true } + }, + "data": { "format": "dpnp", "order": "C" } + } + ], "sklearnex spmd implementation": { "algorithm": { "library": "sklearnex.spmd", diff --git a/configs/experiments/covariance.json b/configs/experiments/covariance.json new file mode 100644 index 00000000..55e673dd --- /dev/null +++ b/configs/experiments/covariance.json @@ -0,0 +1,29 @@ +{ + "INCLUDE": ["../common/sklearn.json", "../common/covariance.json"], + "PARAMETERS_SETS": { + "covariance datasets": [ + { + "data": { + "source": "make_blobs", + "generation_kwargs": [ + { "n_samples": 10000000, "n_features": 50, "centers": 1 }, + { "n_samples": 2000000, "n_features": 500, "centers": 1 }, + { "n_samples": 200000, "n_features": 2000, "centers": 1 } + ], + "split_kwargs": { "ignore": true } + } + }, + { + "data": { + "dataset": ["susy", "higgs"], + "split_kwargs": { "ignore": true } + } + } + ] + }, + "TEMPLATES": { + "sklearn covariance": { + "SETS": ["sklearn-ex[cpu,gpu] preview implementations", "covariance parameters", "covariance datasets"] + } + } +} diff --git a/configs/weekly/covariance.json b/configs/weekly/covariance.json new file mode 100644 index 00000000..d5b5b3c0 --- /dev/null +++ b/configs/weekly/covariance.json @@ -0,0 +1,33 @@ +{ + "INCLUDE": ["../common/sklearn.json", "../common/covariance.json"], + "PARAMETERS_SETS": { + "high-load covariance datasets": [ + { + "data": { + "source": "make_blobs", + "generation_kwargs": [ + { "n_samples": 50000000, "n_features": 50, "centers": 1 }, + { "n_samples": 10000000, "n_features": 500, "centers": 1 }, + { "n_samples": 1000000, "n_features": 5000, "centers": 1 } + ], + "split_kwargs": { "ignore": true } + } + }, + { + "data": { + "dataset": ["airline_depdelay", "epsilon"], + "split_kwargs": { "ignore": true } + } + } + ] + }, + "TEMPLATES": { + "sklearn covariance": { + "SETS": [ + "sklearn-ex[cpu,gpu] preview implementations", + "covariance parameters", + "high-load covariance datasets" + ] + } + } +} diff --git a/sklbench/benchmarks/sklearn_estimator.py b/sklbench/benchmarks/sklearn_estimator.py index 8c511ce3..456f5fc4 100644 --- a/sklbench/benchmarks/sklearn_estimator.py +++ b/sklbench/benchmarks/sklearn_estimator.py @@ -190,6 +190,15 @@ def get_subset_metrics_of_estimator( ) } ) + elif task == "utility": + # SPMD estimators are skipped: their score triggers an additional collective fit + if ( + "Covariance" in str(estimator_instance) + and stage == "training" + and hasattr(estimator_instance, "score") + and "spmd" not in type(estimator_instance).__module__ + ): + metrics.update({"average log-likelihood": float(estimator_instance.score(x))}) elif task == "clustering": if hasattr(estimator_instance, "inertia_"): # compute inertia manually using distances to cluster centers diff --git a/sklbench/report/implementation.py b/sklbench/report/implementation.py index 5d67b103..8189c7f4 100644 --- a/sklbench/report/implementation.py +++ b/sklbench/report/implementation.py @@ -74,7 +74,7 @@ "indifferent": [ # SVM "support vectors", - # PCA + # PCA, EmpiricalCovariance "average log-likelihood", "1st component variance ratio", # DBSCAN