diff --git a/docs/connectors.md b/docs/connectors.md index 84559bda1..4a14933a5 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -169,6 +169,30 @@ input order or adding samples can change it. No global random state is modified. package pins scikit-learn 1.9.1 because its allocation affects persisted dataset contents. Review generated assignments and version affected datasets when upgrading the splitter implementation. +If the split must not change with a library version, or must reproduce a split that other code +made, a script computes it once and writes the validation and test samples to a file beside the +connector. `PrecomputedSplitter` reads that file and checks that the split was made from the +pinned source file. Every sample that the file does not name goes to train. The file names a +sample by its position in the samples, or by the key that `key` returns: + +```python +from timenet_connectors.splitting import PrecomputedSplitter + +def get_series_id(sample): + return sample.series_id + +splitter = PrecomputedSplitter.from_file( + SPLITS_PATH, + source_sha256=SOURCE_SHA256, + key=get_series_id, +) +splits = splitter.split(samples) +``` + +The file is JSON, compressed with xz when its name ends in `.xz`. It holds `source_sha256`, +`validation` and `test`. The `opentslm/sleep-cot` and `opentslm/m4-caption` connectors keep the +script that wrote their file, `generate_splits.py`. + When constructing TimeNet tasks, copy the partition name onto each task: ```python diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py index feeca0e1f..83adade25 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py @@ -6,17 +6,15 @@ stratified by frequency, and wrote it to splits.json.xz. The connector reads that file. """ -from collections import Counter from collections.abc import Iterator, Sequence from dataclasses import dataclass from enum import StrEnum -import lzma from pathlib import Path import re -from typing import Annotated, Self +from typing import Annotated import numpy as np -from pydantic import AllowInfNan, BaseModel, ConfigDict, Field, Json, StrictFloat, ValidationError, model_validator +from pydantic import AllowInfNan, Field, Json, StrictFloat from timenet.connectors import BaseConnector from timenet.dataset import Record, Signal, Source, TimeFDataset @@ -25,6 +23,7 @@ from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec from timenet_connectors.download import ensure_archive from timenet_connectors.sources.readers import CsvDataReader, DataRow +from timenet_connectors.splitting import DatasetSplits, PrecomputedSplitter M4_CAPTION_URL = "https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj/download/M4TimeSeriesCaptionDatasetV02.zip" @@ -105,27 +104,16 @@ class _CaptionRow(DataRow): caption: str -@dataclass(frozen=True) -class M4CaptionedSeries: - """One M4 series of the release, with its frequency and caption, before the split is known.""" - - m4_id: str - frequency: M4Frequency - values: np.ndarray - caption: str - - @dataclass(frozen=True) class M4CaptionSeries: - """One M4 series of the release, with its frequency, caption and split.""" + """One M4 series of the release, with its frequency and caption.""" m4_id: str frequency: M4Frequency values: np.ndarray caption: str - split: Split - def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]: + def create_record_and_caption_task(self, *, split: Split) -> tuple[Record, AnswerTask]: """Create the Record of the series and the task that asks for its caption. The Record holds the values, the M4 id and the frequency. The task names the Record as its @@ -141,73 +129,10 @@ def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]: annotations=(frequency,), metadata={"m4_id": self.m4_id}, ) - task = AnswerTask(inputs=(record,), prompt=CAPTION_PROMPT, targets=(self.caption,), split=self.split) + task = AnswerTask(inputs=(record,), prompt=CAPTION_PROMPT, targets=(self.caption,), split=split) return record, task -class M4CaptionSplits(BaseModel): - """The validation and test series of the split, by M4 id. - - The release archive stores no split. generate_splits.py made an 80/10/10 split, stratified by - frequency, and wrote it to splits.json.xz. Every series that the file does not name is in the train - split. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - source_sha256: str - """SHA-256 digest of the release archive that the split was made from.""" - validation: tuple[str, ...] - test: tuple[str, ...] - - @model_validator(mode="after") - def _each_series_in_one_split(self) -> Self: - repeated = sorted(m4_id for m4_id, count in Counter(self.m4_ids).items() if count > 1) - if repeated: - raise ValueError(f"series {repeated} are named more than once") - return self - - @property - def m4_ids(self) -> tuple[str, ...]: - """The validation and test series together.""" - return (*self.validation, *self.test) - - @classmethod - def read_from_file(cls, path: Path) -> Self: - """Read the split from an xz-compressed JSON file. - - Returns: - The split. - - Raises: - TimeNetConnectorError: If the file does not hold a valid split. - """ - try: - return cls.model_validate_json(lzma.decompress(path.read_bytes())) - except (lzma.LZMAError, ValidationError) as exc: - raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc - - def assign_split_to_each_series(self, m4_ids: Sequence[str]) -> list[Split]: - """Assign a split to each series of the pinned release. - - Returns: - The split of each series, in the order of ``m4_ids``. - - Raises: - TimeNetConnectorError: If the split was made from another release, or names a series that - the release does not have. - """ - if self.source_sha256 != M4_CAPTION_SHA256: - raise TimeNetConnectorError( - f"the split was made from the release {self.source_sha256}, not from the pinned {M4_CAPTION_SHA256}" - ) - unknown = sorted(set(self.m4_ids) - set(m4_ids)) - if unknown: - raise TimeNetConnectorError(f"the split names series {unknown} that the release does not have") - split_of = dict.fromkeys(self.validation, Split.VALIDATION) | dict.fromkeys(self.test, Split.TEST) - return [split_of.get(m4_id, Split.TRAIN) for m4_id in m4_ids] - - @dataclass(frozen=True) class M4CaptionReader: """Read the series, the captions and the split of the extracted release.""" @@ -216,43 +141,33 @@ class M4CaptionReader: splits_path: Path def read_series(self) -> list[M4CaptionSeries]: - """Read every series of the release with its caption and split, in the order of OpenTSLM's loader. + """Read every series of the release with its caption, in the order of OpenTSLM's loader. The loader reads the files of each frequency, and skips ``m4_series.csv`` and ``m4_captions.csv``. - A malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and - its row. The split comes from the split file, which the method checks against every M4 id. + The method leaves out each series whose caption is an error message of the caption API. A + malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and + its row. Returns: - Each series with its caption and split. + Each series with its caption. """ - captioned = self.read_captioned_series() - split_file = M4CaptionSplits.read_from_file(self.splits_path) - splits = split_file.assign_split_to_each_series([series.m4_id for series in captioned]) - return [ - M4CaptionSeries( - m4_id=series.m4_id, - frequency=series.frequency, - values=series.values, - caption=series.caption, - split=split, - ) - for series, split in zip(captioned, splits, strict=True) - ] - - def read_captioned_series(self) -> list[M4CaptionedSeries]: - """Read every series of the release with its caption, in the order of :meth:`read_series`. + series: list[M4CaptionSeries] = [] + for frequency in M4Frequency: + series.extend(self._read_frequency(frequency)) + return series - The method leaves out each series whose caption is an error message of the caption API. + def split_series(self, series: Sequence[M4CaptionSeries]) -> DatasetSplits[M4CaptionSeries]: + """Put each series in the split that the split file names for its M4 id. Returns: - Each series with its caption, without the split. + The series of each split, in the order of ``series``. """ - captioned: list[M4CaptionedSeries] = [] - for frequency in M4Frequency: - captioned.extend(self._read_frequency(frequency)) - return captioned + splitter = PrecomputedSplitter[M4CaptionSeries].from_file( + self.splits_path, source_sha256=M4_CAPTION_SHA256, key=lambda one_series: one_series.m4_id + ) + return splitter.split(series) - def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries]: + def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionSeries]: """Read the series of one frequency, paired with the caption in the same row of its caption file. Yields: @@ -278,7 +193,7 @@ def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries] ) if caption_row.caption.startswith(CAPTION_ERROR_PREFIX): continue - yield M4CaptionedSeries( + yield M4CaptionSeries( m4_id=m4_id, frequency=frequency, values=np.array(series_row.series, dtype=np.float32), @@ -312,12 +227,13 @@ def convert(self, raw_refs: list[M4CaptionReader]) -> TimeFDataset: Returns: The populated dataset. """ - (release,) = raw_refs + (reader,) = raw_refs dataset = TimeFDataset(metadata=self.metadata()) - for series in release.read_series(): - record, task = series.create_record_and_caption_task() - dataset.add_record(record=record) - dataset.add_task(task=task) + for split, series_in_split in reader.split_series(reader.read_series()).items(): + for series in series_in_split: + record, task = series.create_record_and_caption_task(split=split) + dataset.add_record(record=record) + dataset.add_task(task=task) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py index 7f57381b5..b1271e8fd 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py @@ -1,4 +1,4 @@ -"""Write splits.json.xz, the frozen train, validation and test split of m4-caption. +"""Write splits.json.xz, the precomputed train, validation and test split of m4-caption. The release archive stores no split. This script makes an 80/10/10 split with StratifiedSplitter, stratified by M4 frequency, so that validation and test each hold a tenth of every frequency. @@ -30,8 +30,8 @@ M4_CAPTION_SHA256, RELEASE_FOLDER, SPLITS_PATH, - M4CaptionedSeries, M4CaptionReader, + M4CaptionSeries, M4Frequency, ) from timenet_connectors.splitting import StratifiedSplitter @@ -40,7 +40,7 @@ SEED = 42 -def frequency(series: M4CaptionedSeries) -> M4Frequency: +def frequency(series: M4CaptionSeries) -> M4Frequency: """Return the frequency of a series, which the split stratifies by. Returns: @@ -49,7 +49,7 @@ def frequency(series: M4CaptionedSeries) -> M4Frequency: return series.frequency -def values_digest(series: M4CaptionedSeries) -> str: +def values_digest(series: M4CaptionSeries) -> str: """Return the SHA-256 digest of the values of a series. Series with the same values form one group. Returns: @@ -80,9 +80,9 @@ def main() -> None: with tempfile.TemporaryDirectory() as directory, zipfile.ZipFile(archive_path) as archive: archive.extractall(directory) reader = M4CaptionReader(directory=Path(directory) / RELEASE_FOLDER, splits_path=SPLITS_PATH) - captioned = reader.read_captioned_series() + captioned = reader.read_series() - splitter = StratifiedSplitter[M4CaptionedSeries]( + splitter = StratifiedSplitter[M4CaptionSeries]( train_folds=8, validation_folds=1, test_folds=1, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py index 3e6c9cf1e..c3979249d 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py @@ -20,7 +20,6 @@ SPLITS_PATH, M4CaptionConnector, M4CaptionReader, - M4CaptionSplits, ) @@ -101,7 +100,10 @@ def _convert(directory: Path, **changes) -> TimeFDataset: def test_each_series_becomes_a_record_and_a_caption_task(tmp_path): tasks = _convert(tmp_path).tasks_of(AnswerTask) - expected = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids] + release_order = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids] + # The tasks come split by split, each split in release order. + split_order = [Split.TRAIN, Split.VALIDATION, Split.TEST] + expected = sorted(release_order, key=lambda series: split_order.index(_expected_split(series[1]))) assert len(tasks) == len(expected) for task, (frequency, m4_id) in zip(tasks, expected, strict=True): (record,) = task.inputs @@ -131,21 +133,13 @@ def test_a_series_whose_caption_is_an_error_message_is_left_out(tmp_path): "Q1": "Error generating caption: Error code: 404", } tasks = _convert(tmp_path, captions=captions).tasks_of(AnswerTask) - assert [task.inputs[0].metadata["m4_id"] for task in tasks] == ["D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"] + assert {task.inputs[0].metadata["m4_id"] for task in tasks} == {"D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"} -@pytest.mark.parametrize( - ("changes", "message"), - [ - ({"source_sha256": "0" * 64}, "the split was made from the release 0+, not from the pinned"), - ({"test": ["M3", "W1"]}, r"series \['W1'\] are named more than once"), - ({"test": ["M3", "M9"]}, r"names series \['M9'\] that the release does not have"), - ], - ids=["another release", "a series in two splits", "a series that the release does not have"], -) -def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, changes, message): - with pytest.raises(TimeNetConnectorError, match=message): - _convert(tmp_path, splits=_SPLITS | changes) +def test_a_split_file_of_another_release_raises(tmp_path): + # The connector checks the split file against its pinned digest, not against the digest in the file. + with pytest.raises(TimeNetConnectorError, match=r"the split was made from the source 0+, not from the pinned"): + _convert(tmp_path, splits=_SPLITS | {"source_sha256": "0" * 64}) _DATASET_SERIES_COUNTS = {"D": 3974, "H": 414, "M": 47943, "Q": 23604, "W": 359, "Y": 23000} @@ -153,10 +147,10 @@ def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, changes, me dataset leaves out 253 daily, 57 monthly and 396 quarterly series, whose caption is an error message.""" -def test_the_frozen_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): - splits = M4CaptionSplits.read_from_file(SPLITS_PATH) - assert splits.source_sha256 == M4_CAPTION_SHA256 - for m4_ids in (splits.validation, splits.test): +def test_the_precomputed_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): + splits = json.loads(lzma.decompress(SPLITS_PATH.read_bytes())) + assert splits["source_sha256"] == M4_CAPTION_SHA256 + for m4_ids in (splits["validation"], splits["test"]): counts = Counter(m4_id[0] for m4_id in m4_ids) for letter, dataset_count in _DATASET_SERIES_COUNTS.items(): # A tenth of the series of each frequency, rounded down or up. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index 75dfd8b49..104e77366 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -6,7 +6,7 @@ the stage. """ -from collections import Counter +from collections.abc import Sequence from dataclasses import dataclass import json from pathlib import Path @@ -14,7 +14,6 @@ from typing import Self import numpy as np -from pydantic import BaseModel, ConfigDict, NonNegativeInt, ValidationError, model_validator from timenet.connectors import BaseConnector from timenet.dataset import Record, Signal, Source, TimeFDataset @@ -23,6 +22,7 @@ from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg from timenet_connectors.download import Artifact, download_files from timenet_connectors.sources.readers import CsvDataReader, DataRow +from timenet_connectors.splitting import DatasetSplits, PrecomputedSplitter SLEEP_COT_URL = "https://polybox.ethz.ch/index.php/s/ZryWSdCFJZ9JR3R/download" @@ -160,73 +160,9 @@ def create_classification_task(self, record: Record, *, split: Split) -> Classif ) -class FrozenSplits(BaseModel): - """Validation and test row positions, starting at zero in file order. - - The connector reads the split from splits.json. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - source_sha256: str - """SHA-256 digest of the release CSV used to create the split.""" - validation: tuple[NonNegativeInt, ...] - test: tuple[NonNegativeInt, ...] - - @model_validator(mode="after") - def _each_row_in_one_split(self) -> Self: - repeated = sorted(position for position, count in Counter(self.positions).items() if count > 1) - if repeated: - raise ValueError(f"rows {repeated} are named more than once") - return self - - @property - def positions(self) -> tuple[int, ...]: - """The validation and test positions together.""" - return (*self.validation, *self.test) - - @classmethod - def read_from_file(cls, path: Path) -> Self: - """Read the split from a splits.json file. - - Returns: - The split. - - Raises: - TimeNetConnectorError: If the file does not hold a valid split. - """ - try: - return cls.model_validate_json(path.read_text(encoding="utf-8")) - except ValidationError as exc: - raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc - - def assign_split_to_each_row(self, n_rows: int) -> list[Split]: - """Assign a split to each row of the pinned release, in file order. - - Returns: - The split for each row; rows absent from the file belong to the train split. - - Raises: - TimeNetConnectorError: If the CSV digest differs or a row position is past the last row. - """ - if self.source_sha256 != SLEEP_COT_SHA256: - raise TimeNetConnectorError( - f"the split was made from the CSV {self.source_sha256}, not from the pinned {SLEEP_COT_SHA256}" - ) - last_named_row = max(self.positions) - if last_named_row >= n_rows: - raise TimeNetConnectorError(f"the split names row {last_named_row}, but the CSV has only {n_rows} rows") - splits = [Split.TRAIN] * n_rows - for position in self.validation: - splits[position] = Split.VALIDATION - for position in self.test: - splits[position] = Split.TEST - return splits - - @dataclass(frozen=True) class SleepCotReader: - """Reader for the release CSV and the frozen split of its rows.""" + """Reader for the release CSV and the precomputed split of its rows.""" csv_path: Path splits_path: Path @@ -251,13 +187,14 @@ def read_rows(self) -> list[SleepCotRow]: raise TimeNetConnectorError(f"{self.csv_path.name} data row {number}: {exc}") from exc return rows - def read_splits(self, n_rows: int) -> list[Split]: - """Read the split of each row from the splits file. + def split_rows(self, rows: Sequence[SleepCotRow]) -> DatasetSplits[SleepCotRow]: + """Put each row in the split that the splits file names for its position. Returns: - The split for each of the n_rows rows, in file order. + The rows of each split, in file order. """ - return FrozenSplits.read_from_file(self.splits_path).assign_split_to_each_row(n_rows) + splitter = PrecomputedSplitter[SleepCotRow].from_file(self.splits_path, source_sha256=SLEEP_COT_SHA256) + return splitter.split(rows) class SleepCotConnector(BaseConnector[SleepCotReader]): @@ -270,7 +207,7 @@ async def download_async(self, cache_dir: Path) -> list[SleepCotReader]: # ruff cache_dir: Directory for the downloaded file. Returns: - A reader for the release CSV and the frozen split of the connector. + A reader for the release CSV and the precomputed split of the connector. """ artifact = Artifact(SLEEP_COT_URL, cache_dir / "sleep_cot.csv", sha256=SLEEP_COT_SHA256) (csv_path,) = await download_files([artifact]) @@ -286,12 +223,11 @@ def convert(self, raw_refs: list[SleepCotReader]) -> TimeFDataset: The populated dataset. """ (reader,) = raw_refs - rows = reader.read_rows() - splits = reader.read_splits(len(rows)) dataset = TimeFDataset(metadata=self.metadata()) - for row, split in zip(rows, splits, strict=True): - record = dataset.add_record(record=row.create_record()) - dataset.add_task(task=row.create_classification_task(record, split=split)) + for split, rows in reader.split_rows(reader.read_rows()).items(): + for row in rows: + record = dataset.add_record(record=row.create_record()) + dataset.add_task(task=row.create_classification_task(record, split=split)) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py index e3d0e2c14..db4a5f750 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py @@ -7,7 +7,7 @@ # "scikit-learn==1.7.2", # ] # /// -"""Write splits.json, the frozen train, validation and test split of sleep-cot. +"""Write splits.json, the precomputed train, validation and test split of sleep-cot. sleep-cot does not store the train/val/test splits explicitly. Rather they give a random seed and allow scikit to do the split. This is hard to reproduce, because it means the official split is diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py index 4e83edfd8..e39887573 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py @@ -59,8 +59,6 @@ def _write_release( directory: Path, rows: list[dict[str, str]] = _ROWS, *, - validation: list[int] = _VALIDATION_POSITIONS, - test: list[int] = _TEST_POSITIONS, source_sha256: str = SLEEP_COT_SHA256, ) -> SleepCotReader: csv_path = directory / "sleep_cot.csv" @@ -69,15 +67,17 @@ def _write_release( writer.writeheader() writer.writerows(rows) splits_path = directory / "splits.json" - splits = {"source_sha256": source_sha256, "validation": validation, "test": test} + splits = {"source_sha256": source_sha256, "validation": _VALIDATION_POSITIONS, "test": _TEST_POSITIONS} splits_path.write_text(json.dumps(splits), encoding="utf-8") return SleepCotReader(csv_path=csv_path, splits_path=splits_path) def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): tasks = SleepCotConnector().convert([_write_release(tmp_path)]).tasks_of(ClassificationTask) - assert [task.split for task in tasks] == [Split.TRAIN, Split.VALIDATION, Split.TRAIN, Split.TEST] - for task, row in zip(tasks, _ROWS, strict=True): + # The tasks come split by split, each split in file order. + assert [task.split for task in tasks] == [Split.TRAIN, Split.TRAIN, Split.VALIDATION, Split.TEST] + rows_in_task_order = [_ROWS[0], _ROWS[2], _ROWS[1], _ROWS[3]] + for task, row in zip(tasks, rows_in_task_order, strict=True): (record,) = task.inputs (signal,) = record.signals assert signal.name == "EEG Fpz-Cz" @@ -95,18 +95,10 @@ def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): assert task.annotations == () -@pytest.mark.parametrize( - ("splits", "message"), - [ - ({"source_sha256": "0" * 64}, "the split was made from the CSV 0+, not from the pinned"), - ({"test": [1]}, r"rows \[1\] are named more than once"), - ({"test": [len(_ROWS)]}, f"names row {len(_ROWS)}, but the CSV has only {len(_ROWS)} rows"), - ], - ids=["another release CSV", "a row in two splits", "a row past the end"], -) -def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, splits, message): - with pytest.raises(TimeNetConnectorError, match=message): - SleepCotConnector().convert([_write_release(tmp_path, **splits)]) +def test_a_split_file_of_another_release_raises(tmp_path): + # The connector checks the split file against its pinned digest, not against the digest in the file. + with pytest.raises(TimeNetConnectorError, match=r"the split was made from the source 0+, not from the pinned"): + SleepCotConnector().convert([_write_release(tmp_path, source_sha256="0" * 64)]) @pytest.mark.parametrize( diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py index 66de829f9..f293f9985 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py @@ -1,7 +1,8 @@ """Generate dataset partitions during connector conversion.""" from timenet_connectors.splitting.base import DatasetSplits, Splitter +from timenet_connectors.splitting.precomputed import PrecomputedSplitter, SampleKey from timenet_connectors.splitting.stratified import StratifiedSplitter -__all__ = ["DatasetSplits", "Splitter", "StratifiedSplitter"] +__all__ = ["DatasetSplits", "PrecomputedSplitter", "SampleKey", "Splitter", "StratifiedSplitter"] diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py new file mode 100644 index 000000000..5b59582da --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py @@ -0,0 +1,111 @@ +"""A splitter that reads a split that a script computed once and wrote to a file beside its connector.""" + +from collections import Counter +from collections.abc import Callable, Iterable, Sequence +import lzma +from pathlib import Path +from typing import Self + +from pydantic import BaseModel, ConfigDict, NonNegativeInt, ValidationError + +from timenet.errors import TimeNetConnectorError +from timenet.types.splits import Split +from timenet_connectors.splitting.base import DatasetSplits, Splitter, T + + +SampleKey = NonNegativeInt | str +"""How a split file names a sample: by its position in the samples, or by a key such as a source id.""" + + +class _SplitFile(BaseModel): + """The content of a split file.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_sha256: str + """SHA-256 digest of the source file that the split was made from.""" + validation: tuple[SampleKey, ...] + test: tuple[SampleKey, ...] + + +class PrecomputedSplitter(Splitter[T]): + """Put each sample in the split that a split file names for it, and every other sample in train. + + Use it when the source has no split of its own and the split depends on the version of a library, + such as a seeded shuffle, so that making it again at build time could change it. A script makes + the split once and writes the file. The file is JSON, compressed with xz when its name ends in + ``.xz``. It names the validation and the test samples, either by their position in the samples or + by the key that ``key`` returns for each sample. + """ + + def __init__( + self, + *, + validation: Iterable[SampleKey], + test: Iterable[SampleKey], + key: Callable[[T], SampleKey] | None = None, + ) -> None: + """Create a splitter from the validation and test keys. + + Args: + validation: The keys of the validation samples. + test: The keys of the test samples. + key: Return the key of a sample. Without it, the key of a sample is its position. + + Raises: + TimeNetConnectorError: If a key is in both splits or twice in one split. + """ + validation, test = tuple(validation), tuple(test) + repeated = sorted((k for k, count in Counter((*validation, *test)).items() if count > 1), key=str) + if repeated: + raise TimeNetConnectorError(f"{repeated} are named more than once") + self._split_of = dict.fromkeys(validation, Split.VALIDATION) | dict.fromkeys(test, Split.TEST) + self._key = key + + @classmethod + def from_file(cls, path: Path, *, source_sha256: str, key: Callable[[T], SampleKey] | None = None) -> Self: + """Read the split from a file, and check that it was made from the pinned source. + + Args: + path: The split file. + source_sha256: The pinned digest of the source file. + key: Return the key of a sample. Without it, the key of a sample is its position. + + Returns: + The splitter. + + Raises: + TimeNetConnectorError: If the file does not hold a valid split, or the split was made from + another source. + """ + try: + content = path.read_bytes() + if path.suffix == ".xz": + content = lzma.decompress(content) + split_file = _SplitFile.model_validate_json(content) + splitter = cls(validation=split_file.validation, test=split_file.test, key=key) + except (lzma.LZMAError, ValidationError, TimeNetConnectorError) as exc: + raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc + if split_file.source_sha256 != source_sha256: + raise TimeNetConnectorError( + f"the split was made from the source {split_file.source_sha256}, not from the pinned {source_sha256}" + ) + return splitter + + def split(self, samples: Sequence[T]) -> DatasetSplits[T]: + """Put each sample in its split, keeping the order of the samples within each split. + + Returns: + The train, validation and test samples. + + Raises: + TimeNetConnectorError: If the split names a key that no sample has. + """ + keys: Sequence[SampleKey] = range(len(samples)) if self._key is None else [self._key(s) for s in samples] + unknown = sorted(set(self._split_of) - set(keys), key=str) + if unknown: + raise TimeNetConnectorError(f"the split names {unknown} that the source does not have") + partitions: dict[Split, list[T]] = {Split.TRAIN: [], Split.VALIDATION: [], Split.TEST: []} + for sample, sample_key in zip(samples, keys, strict=True): + partitions[self._split_of.get(sample_key, Split.TRAIN)].append(sample) + return DatasetSplits(partitions) diff --git a/packages/timenet-connectors/tests/test_precomputed_splits.py b/packages/timenet-connectors/tests/test_precomputed_splits.py new file mode 100644 index 000000000..be9a1a3a7 --- /dev/null +++ b/packages/timenet-connectors/tests/test_precomputed_splits.py @@ -0,0 +1,76 @@ +import json +import lzma +from pathlib import Path + +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.types.splits import Split +from timenet_connectors.splitting import PrecomputedSplitter + + +_SOURCE_SHA256 = "a" * 64 + + +def _write_split_file(path: Path, **splits: object) -> Path: + content = json.dumps({"source_sha256": _SOURCE_SHA256} | splits).encode() + path.write_bytes(lzma.compress(content) if path.suffix == ".xz" else content) + return path + + +def test_samples_by_position_get_their_split_and_the_rest_are_train(tmp_path): + path = _write_split_file(tmp_path / "splits.json", validation=[1], test=[3]) + splitter = PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256) + assert dict(splitter.split(["a", "b", "c", "d", "e"])) == { + Split.TRAIN: ("a", "c", "e"), + Split.VALIDATION: ("b",), + Split.TEST: ("d",), + } + + +def test_samples_by_key_get_their_split_from_an_xz_file(tmp_path): + path = _write_split_file(tmp_path / "splits.json.xz", validation=["W1"], test=["M3"]) + splitter = PrecomputedSplitter[tuple[str, int]].from_file( + path, source_sha256=_SOURCE_SHA256, key=lambda sample: sample[0] + ) + assert dict(splitter.split([("M3", 0), ("D1", 1), ("W1", 2)])) == { + Split.TRAIN: (("D1", 1),), + Split.VALIDATION: (("W1", 2),), + Split.TEST: (("M3", 0),), + } + + +def test_a_split_made_from_another_source_raises(tmp_path): + path = _write_split_file(tmp_path / "splits.json", validation=[], test=[]) + with pytest.raises(TimeNetConnectorError, match=f"made from the source {_SOURCE_SHA256}, not from the pinned 0+"): + PrecomputedSplitter[str].from_file(path, source_sha256="0" * 64) + + +def test_a_split_that_names_a_key_the_source_does_not_have_raises(tmp_path): + path = _write_split_file(tmp_path / "splits.json", validation=[1], test=[5]) + splitter = PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256) + with pytest.raises(TimeNetConnectorError, match=r"the split names \[5\] that the source does not have"): + splitter.split(["a", "b", "c", "d", "e"]) + + +@pytest.mark.parametrize( + ("content", "message"), + [ + (json.dumps({"source_sha256": _SOURCE_SHA256, "validation": [1], "test": [1]}), r"\[1\] are named more"), + (json.dumps({"source_sha256": _SOURCE_SHA256, "validation": [-1], "test": []}), "greater than or equal to 0"), + ("not JSON", "Invalid JSON"), + ], + ids=["a key in two splits", "a negative position", "a file that is not JSON"], +) +def test_a_malformed_split_file_raises_with_its_name(tmp_path, content, message): + path = tmp_path / "splits.json" + path.write_text(content) + with pytest.raises(TimeNetConnectorError, match=rf"(?s)splits\.json does not hold a valid split: .*{message}"): + PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256) + + +def test_an_xz_file_that_is_not_compressed_raises(tmp_path): + path = tmp_path / "splits.json.xz" + path.write_text("{}") + with pytest.raises(TimeNetConnectorError, match=r"splits\.json\.xz does not hold a valid split"): + PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256)