From 9f8f47f12be3d787854d67a31dc193bb99d80007 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 10:50:25 -0800 Subject: [PATCH 1/5] refactor(connectors): share one frozen-split type sleep-cot and m4-caption each had their own copy of a frozen split: the validation and test samples that a script wrote to a file, with the digest of the source file that the split was made from. FrozenSplits in timenet_connectors.splitting now owns this. It names samples by row position or by source id, reads plain JSON or JSON compressed with xz, and takes the pinned source digest as an argument. Both connectors use it. The real split files of both connectors give the same splits as before. --- .../datasets/opentslm/m4_caption/connector.py | 76 ++-------------- .../m4_caption/tests/test_connector.py | 10 +-- .../datasets/opentslm/sleep_cot/connector.py | 71 +-------------- .../sleep_cot/tests/test_connector.py | 6 +- .../timenet_connectors/splitting/__init__.py | 3 +- .../timenet_connectors/splitting/frozen.py | 88 +++++++++++++++++++ .../tests/test_frozen_splits.py | 78 ++++++++++++++++ 7 files changed, 186 insertions(+), 146 deletions(-) create mode 100644 packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py create mode 100644 packages/timenet-connectors/tests/test_frozen_splits.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py index feeca0e1f..ec9ddd90e 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py @@ -6,17 +6,15 @@ stratified by frequency, and wrote it to splits.json.xz. The connector reads that file. """ -from collections import Counter -from collections.abc import Iterator, Sequence +from collections.abc import Iterator from dataclasses import dataclass from enum import StrEnum -import lzma from pathlib import Path import re -from typing import Annotated, Self +from typing import Annotated import numpy as np -from pydantic import AllowInfNan, BaseModel, ConfigDict, Field, Json, StrictFloat, ValidationError, model_validator +from pydantic import AllowInfNan, Field, Json, StrictFloat from timenet.connectors import BaseConnector from timenet.dataset import Record, Signal, Source, TimeFDataset @@ -25,6 +23,7 @@ from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec from timenet_connectors.download import ensure_archive from timenet_connectors.sources.readers import CsvDataReader, DataRow +from timenet_connectors.splitting import FrozenSplits M4_CAPTION_URL = "https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj/download/M4TimeSeriesCaptionDatasetV02.zip" @@ -145,69 +144,6 @@ def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]: return record, task -class M4CaptionSplits(BaseModel): - """The validation and test series of the split, by M4 id. - - The release archive stores no split. generate_splits.py made an 80/10/10 split, stratified by - frequency, and wrote it to splits.json.xz. Every series that the file does not name is in the train - split. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - source_sha256: str - """SHA-256 digest of the release archive that the split was made from.""" - validation: tuple[str, ...] - test: tuple[str, ...] - - @model_validator(mode="after") - def _each_series_in_one_split(self) -> Self: - repeated = sorted(m4_id for m4_id, count in Counter(self.m4_ids).items() if count > 1) - if repeated: - raise ValueError(f"series {repeated} are named more than once") - return self - - @property - def m4_ids(self) -> tuple[str, ...]: - """The validation and test series together.""" - return (*self.validation, *self.test) - - @classmethod - def read_from_file(cls, path: Path) -> Self: - """Read the split from an xz-compressed JSON file. - - Returns: - The split. - - Raises: - TimeNetConnectorError: If the file does not hold a valid split. - """ - try: - return cls.model_validate_json(lzma.decompress(path.read_bytes())) - except (lzma.LZMAError, ValidationError) as exc: - raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc - - def assign_split_to_each_series(self, m4_ids: Sequence[str]) -> list[Split]: - """Assign a split to each series of the pinned release. - - Returns: - The split of each series, in the order of ``m4_ids``. - - Raises: - TimeNetConnectorError: If the split was made from another release, or names a series that - the release does not have. - """ - if self.source_sha256 != M4_CAPTION_SHA256: - raise TimeNetConnectorError( - f"the split was made from the release {self.source_sha256}, not from the pinned {M4_CAPTION_SHA256}" - ) - unknown = sorted(set(self.m4_ids) - set(m4_ids)) - if unknown: - raise TimeNetConnectorError(f"the split names series {unknown} that the release does not have") - split_of = dict.fromkeys(self.validation, Split.VALIDATION) | dict.fromkeys(self.test, Split.TEST) - return [split_of.get(m4_id, Split.TRAIN) for m4_id in m4_ids] - - @dataclass(frozen=True) class M4CaptionReader: """Read the series, the captions and the split of the extracted release.""" @@ -226,8 +162,8 @@ def read_series(self) -> list[M4CaptionSeries]: Each series with its caption and split. """ captioned = self.read_captioned_series() - split_file = M4CaptionSplits.read_from_file(self.splits_path) - splits = split_file.assign_split_to_each_series([series.m4_id for series in captioned]) + frozen = FrozenSplits[str].read_from_file(self.splits_path) + splits = frozen.assign_splits([series.m4_id for series in captioned], source_sha256=M4_CAPTION_SHA256) return [ M4CaptionSeries( m4_id=series.m4_id, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py index 3e6c9cf1e..4e80035e3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py @@ -20,8 +20,8 @@ SPLITS_PATH, M4CaptionConnector, M4CaptionReader, - M4CaptionSplits, ) +from timenet_connectors.splitting import FrozenSplits _SERIES_IDS = { @@ -137,9 +137,9 @@ def test_a_series_whose_caption_is_an_error_message_is_left_out(tmp_path): @pytest.mark.parametrize( ("changes", "message"), [ - ({"source_sha256": "0" * 64}, "the split was made from the release 0+, not from the pinned"), - ({"test": ["M3", "W1"]}, r"series \['W1'\] are named more than once"), - ({"test": ["M3", "M9"]}, r"names series \['M9'\] that the release does not have"), + ({"source_sha256": "0" * 64}, "the split was made from the source 0+, not from the pinned"), + ({"test": ["M3", "W1"]}, r"\['W1'\] are named more than once"), + ({"test": ["M3", "M9"]}, r"the split names \['M9'\] that the source does not have"), ], ids=["another release", "a series in two splits", "a series that the release does not have"], ) @@ -154,7 +154,7 @@ def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, changes, me def test_the_frozen_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): - splits = M4CaptionSplits.read_from_file(SPLITS_PATH) + splits = FrozenSplits[str].read_from_file(SPLITS_PATH) assert splits.source_sha256 == M4_CAPTION_SHA256 for m4_ids in (splits.validation, splits.test): counts = Counter(m4_id[0] for m4_id in m4_ids) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index 75dfd8b49..204c136a8 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -6,7 +6,6 @@ the stage. """ -from collections import Counter from dataclasses import dataclass import json from pathlib import Path @@ -14,7 +13,7 @@ from typing import Self import numpy as np -from pydantic import BaseModel, ConfigDict, NonNegativeInt, ValidationError, model_validator +from pydantic import NonNegativeInt from timenet.connectors import BaseConnector from timenet.dataset import Record, Signal, Source, TimeFDataset @@ -23,6 +22,7 @@ from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg from timenet_connectors.download import Artifact, download_files from timenet_connectors.sources.readers import CsvDataReader, DataRow +from timenet_connectors.splitting import FrozenSplits SLEEP_COT_URL = "https://polybox.ethz.ch/index.php/s/ZryWSdCFJZ9JR3R/download" @@ -160,70 +160,6 @@ def create_classification_task(self, record: Record, *, split: Split) -> Classif ) -class FrozenSplits(BaseModel): - """Validation and test row positions, starting at zero in file order. - - The connector reads the split from splits.json. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - source_sha256: str - """SHA-256 digest of the release CSV used to create the split.""" - validation: tuple[NonNegativeInt, ...] - test: tuple[NonNegativeInt, ...] - - @model_validator(mode="after") - def _each_row_in_one_split(self) -> Self: - repeated = sorted(position for position, count in Counter(self.positions).items() if count > 1) - if repeated: - raise ValueError(f"rows {repeated} are named more than once") - return self - - @property - def positions(self) -> tuple[int, ...]: - """The validation and test positions together.""" - return (*self.validation, *self.test) - - @classmethod - def read_from_file(cls, path: Path) -> Self: - """Read the split from a splits.json file. - - Returns: - The split. - - Raises: - TimeNetConnectorError: If the file does not hold a valid split. - """ - try: - return cls.model_validate_json(path.read_text(encoding="utf-8")) - except ValidationError as exc: - raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc - - def assign_split_to_each_row(self, n_rows: int) -> list[Split]: - """Assign a split to each row of the pinned release, in file order. - - Returns: - The split for each row; rows absent from the file belong to the train split. - - Raises: - TimeNetConnectorError: If the CSV digest differs or a row position is past the last row. - """ - if self.source_sha256 != SLEEP_COT_SHA256: - raise TimeNetConnectorError( - f"the split was made from the CSV {self.source_sha256}, not from the pinned {SLEEP_COT_SHA256}" - ) - last_named_row = max(self.positions) - if last_named_row >= n_rows: - raise TimeNetConnectorError(f"the split names row {last_named_row}, but the CSV has only {n_rows} rows") - splits = [Split.TRAIN] * n_rows - for position in self.validation: - splits[position] = Split.VALIDATION - for position in self.test: - splits[position] = Split.TEST - return splits - - @dataclass(frozen=True) class SleepCotReader: """Reader for the release CSV and the frozen split of its rows.""" @@ -257,7 +193,8 @@ def read_splits(self, n_rows: int) -> list[Split]: Returns: The split for each of the n_rows rows, in file order. """ - return FrozenSplits.read_from_file(self.splits_path).assign_split_to_each_row(n_rows) + splits = FrozenSplits[NonNegativeInt].read_from_file(self.splits_path) + return splits.assign_splits(range(n_rows), source_sha256=SLEEP_COT_SHA256) class SleepCotConnector(BaseConnector[SleepCotReader]): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py index 4e83edfd8..c1a5793c9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py @@ -98,9 +98,9 @@ def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): @pytest.mark.parametrize( ("splits", "message"), [ - ({"source_sha256": "0" * 64}, "the split was made from the CSV 0+, not from the pinned"), - ({"test": [1]}, r"rows \[1\] are named more than once"), - ({"test": [len(_ROWS)]}, f"names row {len(_ROWS)}, but the CSV has only {len(_ROWS)} rows"), + ({"source_sha256": "0" * 64}, "the split was made from the source 0+, not from the pinned"), + ({"test": [1]}, r"\[1\] are named more than once"), + ({"test": [len(_ROWS)]}, rf"the split names \[{len(_ROWS)}\] that the source does not have"), ], ids=["another release CSV", "a row in two splits", "a row past the end"], ) diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py index 66de829f9..364a9d443 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py @@ -1,7 +1,8 @@ """Generate dataset partitions during connector conversion.""" from timenet_connectors.splitting.base import DatasetSplits, Splitter +from timenet_connectors.splitting.frozen import FrozenSplits from timenet_connectors.splitting.stratified import StratifiedSplitter -__all__ = ["DatasetSplits", "Splitter", "StratifiedSplitter"] +__all__ = ["DatasetSplits", "FrozenSplits", "Splitter", "StratifiedSplitter"] diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py b/packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py new file mode 100644 index 000000000..397a7eadb --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py @@ -0,0 +1,88 @@ +"""A split that a script computed once and froze into a file beside its connector.""" + +from collections import Counter +from collections.abc import Sequence +import lzma +from pathlib import Path +from typing import Generic, Self, TypeVar + +from pydantic import BaseModel, ConfigDict, ValidationError, model_validator + +from timenet.errors import TimeNetConnectorError +from timenet.types.splits import Split + + +K = TypeVar("K", int, str) +"""How a split file names a sample: a row position or a source id.""" + + +class FrozenSplits(BaseModel, Generic[K]): + """The validation and test samples of a split that a script froze into a file. + + Use it when the source has no split of its own and the published split depends on a library that + the connector should not pin, such as a seeded shuffle. The script runs the published code once + and writes this file. The file names each sample by a key: a row position or a source id. Every + sample that the file does not name is in the train split. The file is JSON, compressed with xz + when its name ends in ``.xz``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_sha256: str + """SHA-256 digest of the source file that the split was made from.""" + validation: tuple[K, ...] + test: tuple[K, ...] + + @model_validator(mode="after") + def _each_key_in_one_split(self) -> Self: + repeated = sorted(key for key, count in Counter(self.named_keys).items() if count > 1) + if repeated: + raise ValueError(f"{repeated} are named more than once") + return self + + @property + def named_keys(self) -> tuple[K, ...]: + """The validation and test keys together.""" + return (*self.validation, *self.test) + + @classmethod + def read_from_file(cls, path: Path) -> Self: + """Read the split from a JSON file, or from an xz-compressed one if the name ends in ``.xz``. + + Returns: + The split. + + Raises: + TimeNetConnectorError: If the file does not hold a valid split. + """ + try: + content = path.read_bytes() + if path.suffix == ".xz": + content = lzma.decompress(content) + return cls.model_validate_json(content) + except (lzma.LZMAError, ValidationError) as exc: + raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc + + def assign_splits(self, keys: Sequence[K], *, source_sha256: str) -> list[Split]: + """Assign a split to each sample of the pinned source. + + Args: + keys: The key of each sample of the source, in the order of the samples. + source_sha256: The pinned digest of the source file. + + Returns: + The split of each sample, in the order of ``keys``. + + Raises: + TimeNetConnectorError: If the split was made from another source, or names a key that the + source does not have. + """ + if self.source_sha256 != source_sha256: + raise TimeNetConnectorError( + f"the split was made from the source {self.source_sha256}, not from the pinned {source_sha256}" + ) + unknown = sorted(set(self.named_keys) - set(keys)) + if unknown: + raise TimeNetConnectorError(f"the split names {unknown} that the source does not have") + split_of = dict.fromkeys(self.validation, Split.VALIDATION) | dict.fromkeys(self.test, Split.TEST) + return [split_of.get(key, Split.TRAIN) for key in keys] diff --git a/packages/timenet-connectors/tests/test_frozen_splits.py b/packages/timenet-connectors/tests/test_frozen_splits.py new file mode 100644 index 000000000..64531f209 --- /dev/null +++ b/packages/timenet-connectors/tests/test_frozen_splits.py @@ -0,0 +1,78 @@ +import json +import lzma +from pathlib import Path + +from pydantic import NonNegativeInt +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.types.splits import Split +from timenet_connectors.splitting import FrozenSplits + + +_SOURCE_SHA256 = "a" * 64 + + +def _write_split_file(path: Path, **splits: object) -> Path: + content = json.dumps({"source_sha256": _SOURCE_SHA256} | splits).encode() + path.write_bytes(lzma.compress(content) if path.suffix == ".xz" else content) + return path + + +def test_rows_by_position_get_their_split_and_the_rest_are_train(tmp_path): + path = _write_split_file(tmp_path / "splits.json", validation=[1], test=[3]) + splits = FrozenSplits[NonNegativeInt].read_from_file(path) + assert splits.assign_splits(range(5), source_sha256=_SOURCE_SHA256) == [ + Split.TRAIN, + Split.VALIDATION, + Split.TRAIN, + Split.TEST, + Split.TRAIN, + ] + + +def test_series_by_id_get_their_split_from_an_xz_file(tmp_path): + path = _write_split_file(tmp_path / "splits.json.xz", validation=["W1"], test=["M3"]) + splits = FrozenSplits[str].read_from_file(path) + assert splits.assign_splits(["M3", "D1", "W1"], source_sha256=_SOURCE_SHA256) == [ + Split.TEST, + Split.TRAIN, + Split.VALIDATION, + ] + + +def test_a_split_made_from_another_source_raises(tmp_path): + splits = FrozenSplits[str].read_from_file(_write_split_file(tmp_path / "splits.json", validation=[], test=[])) + with pytest.raises(TimeNetConnectorError, match=f"made from the source {_SOURCE_SHA256}, not from the pinned 0+"): + splits.assign_splits(["M1"], source_sha256="0" * 64) + + +def test_a_split_that_names_a_key_the_source_does_not_have_raises(tmp_path): + splits = FrozenSplits[NonNegativeInt].read_from_file( + _write_split_file(tmp_path / "splits.json", validation=[1], test=[5]) + ) + with pytest.raises(TimeNetConnectorError, match=r"the split names \[5\] that the source does not have"): + splits.assign_splits(range(5), source_sha256=_SOURCE_SHA256) + + +@pytest.mark.parametrize( + ("content", "message"), + [ + (json.dumps({"source_sha256": _SOURCE_SHA256, "validation": [1], "test": [1]}), r"\[1\] are named more"), + (json.dumps({"source_sha256": _SOURCE_SHA256, "validation": [-1], "test": []}), "greater than or equal to 0"), + ("not JSON", "Invalid JSON"), + ], + ids=["a key in two splits", "a negative position", "a file that is not JSON"], +) +def test_a_malformed_split_file_raises_with_its_name(tmp_path, content, message): + path = tmp_path / "splits.json" + path.write_text(content) + with pytest.raises(TimeNetConnectorError, match=rf"(?s)splits\.json does not hold a valid split: .*{message}"): + FrozenSplits[NonNegativeInt].read_from_file(path) + + +def test_an_xz_file_that_is_not_compressed_raises(tmp_path): + path = tmp_path / "splits.json.xz" + path.write_text("{}") + with pytest.raises(TimeNetConnectorError, match=r"splits\.json\.xz does not hold a valid split"): + FrozenSplits[str].read_from_file(path) From 5cffd10f077878280ad328debbe678a50261b162 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 15:54:04 -0800 Subject: [PATCH 2/5] test(connectors): keep only the pinned-digest split test in each connector --- .../m4_caption/tests/test_connector.py | 16 ++++----------- .../sleep_cot/tests/test_connector.py | 20 +++++-------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py index 4e80035e3..205209c5c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py @@ -134,18 +134,10 @@ def test_a_series_whose_caption_is_an_error_message_is_left_out(tmp_path): assert [task.inputs[0].metadata["m4_id"] for task in tasks] == ["D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"] -@pytest.mark.parametrize( - ("changes", "message"), - [ - ({"source_sha256": "0" * 64}, "the split was made from the source 0+, not from the pinned"), - ({"test": ["M3", "W1"]}, r"\['W1'\] are named more than once"), - ({"test": ["M3", "M9"]}, r"the split names \['M9'\] that the source does not have"), - ], - ids=["another release", "a series in two splits", "a series that the release does not have"], -) -def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, changes, message): - with pytest.raises(TimeNetConnectorError, match=message): - _convert(tmp_path, splits=_SPLITS | changes) +def test_a_split_file_of_another_release_raises(tmp_path): + # The connector checks the split file against its pinned digest, not against the digest in the file. + with pytest.raises(TimeNetConnectorError, match=r"the split was made from the source 0+, not from the pinned"): + _convert(tmp_path, splits=_SPLITS | {"source_sha256": "0" * 64}) _DATASET_SERIES_COUNTS = {"D": 3974, "H": 414, "M": 47943, "Q": 23604, "W": 359, "Y": 23000} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py index c1a5793c9..573373098 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py @@ -59,8 +59,6 @@ def _write_release( directory: Path, rows: list[dict[str, str]] = _ROWS, *, - validation: list[int] = _VALIDATION_POSITIONS, - test: list[int] = _TEST_POSITIONS, source_sha256: str = SLEEP_COT_SHA256, ) -> SleepCotReader: csv_path = directory / "sleep_cot.csv" @@ -69,7 +67,7 @@ def _write_release( writer.writeheader() writer.writerows(rows) splits_path = directory / "splits.json" - splits = {"source_sha256": source_sha256, "validation": validation, "test": test} + splits = {"source_sha256": source_sha256, "validation": _VALIDATION_POSITIONS, "test": _TEST_POSITIONS} splits_path.write_text(json.dumps(splits), encoding="utf-8") return SleepCotReader(csv_path=csv_path, splits_path=splits_path) @@ -95,18 +93,10 @@ def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): assert task.annotations == () -@pytest.mark.parametrize( - ("splits", "message"), - [ - ({"source_sha256": "0" * 64}, "the split was made from the source 0+, not from the pinned"), - ({"test": [1]}, r"\[1\] are named more than once"), - ({"test": [len(_ROWS)]}, rf"the split names \[{len(_ROWS)}\] that the source does not have"), - ], - ids=["another release CSV", "a row in two splits", "a row past the end"], -) -def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, splits, message): - with pytest.raises(TimeNetConnectorError, match=message): - SleepCotConnector().convert([_write_release(tmp_path, **splits)]) +def test_a_split_file_of_another_release_raises(tmp_path): + # The connector checks the split file against its pinned digest, not against the digest in the file. + with pytest.raises(TimeNetConnectorError, match=r"the split was made from the source 0+, not from the pinned"): + SleepCotConnector().convert([_write_release(tmp_path, source_sha256="0" * 64)]) @pytest.mark.parametrize( From 5f537da3ae5946bd1cbdf80b04c75b51376a350b Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 16:08:41 -0800 Subject: [PATCH 3/5] refactor(connectors): rename FrozenSplits to PrecomputedSplits --- .../datasets/opentslm/m4_caption/connector.py | 6 +++--- .../opentslm/m4_caption/generate_splits.py | 2 +- .../opentslm/m4_caption/tests/test_connector.py | 6 +++--- .../datasets/opentslm/sleep_cot/connector.py | 8 ++++---- .../datasets/opentslm/sleep_cot/generate_splits.py | 2 +- .../src/timenet_connectors/splitting/__init__.py | 4 ++-- .../splitting/{frozen.py => precomputed.py} | 6 +++--- ...frozen_splits.py => test_precomputed_splits.py} | 14 +++++++------- 8 files changed, 24 insertions(+), 24 deletions(-) rename packages/timenet-connectors/src/timenet_connectors/splitting/{frozen.py => precomputed.py} (95%) rename packages/timenet-connectors/tests/{test_frozen_splits.py => test_precomputed_splits.py} (84%) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py index ec9ddd90e..81de1e205 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py @@ -23,7 +23,7 @@ from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec from timenet_connectors.download import ensure_archive from timenet_connectors.sources.readers import CsvDataReader, DataRow -from timenet_connectors.splitting import FrozenSplits +from timenet_connectors.splitting import PrecomputedSplits M4_CAPTION_URL = "https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj/download/M4TimeSeriesCaptionDatasetV02.zip" @@ -162,8 +162,8 @@ def read_series(self) -> list[M4CaptionSeries]: Each series with its caption and split. """ captioned = self.read_captioned_series() - frozen = FrozenSplits[str].read_from_file(self.splits_path) - splits = frozen.assign_splits([series.m4_id for series in captioned], source_sha256=M4_CAPTION_SHA256) + precomputed = PrecomputedSplits[str].read_from_file(self.splits_path) + splits = precomputed.assign_splits([series.m4_id for series in captioned], source_sha256=M4_CAPTION_SHA256) return [ M4CaptionSeries( m4_id=series.m4_id, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py index 7f57381b5..9b8e59ac2 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py @@ -1,4 +1,4 @@ -"""Write splits.json.xz, the frozen train, validation and test split of m4-caption. +"""Write splits.json.xz, the precomputed train, validation and test split of m4-caption. The release archive stores no split. This script makes an 80/10/10 split with StratifiedSplitter, stratified by M4 frequency, so that validation and test each hold a tenth of every frequency. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py index 205209c5c..6df0c24f9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py @@ -21,7 +21,7 @@ M4CaptionConnector, M4CaptionReader, ) -from timenet_connectors.splitting import FrozenSplits +from timenet_connectors.splitting import PrecomputedSplits _SERIES_IDS = { @@ -145,8 +145,8 @@ def test_a_split_file_of_another_release_raises(tmp_path): dataset leaves out 253 daily, 57 monthly and 396 quarterly series, whose caption is an error message.""" -def test_the_frozen_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): - splits = FrozenSplits[str].read_from_file(SPLITS_PATH) +def test_the_precomputed_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): + splits = PrecomputedSplits[str].read_from_file(SPLITS_PATH) assert splits.source_sha256 == M4_CAPTION_SHA256 for m4_ids in (splits.validation, splits.test): counts = Counter(m4_id[0] for m4_id in m4_ids) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index 204c136a8..6c86c30a5 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -22,7 +22,7 @@ from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg from timenet_connectors.download import Artifact, download_files from timenet_connectors.sources.readers import CsvDataReader, DataRow -from timenet_connectors.splitting import FrozenSplits +from timenet_connectors.splitting import PrecomputedSplits SLEEP_COT_URL = "https://polybox.ethz.ch/index.php/s/ZryWSdCFJZ9JR3R/download" @@ -162,7 +162,7 @@ def create_classification_task(self, record: Record, *, split: Split) -> Classif @dataclass(frozen=True) class SleepCotReader: - """Reader for the release CSV and the frozen split of its rows.""" + """Reader for the release CSV and the precomputed split of its rows.""" csv_path: Path splits_path: Path @@ -193,7 +193,7 @@ def read_splits(self, n_rows: int) -> list[Split]: Returns: The split for each of the n_rows rows, in file order. """ - splits = FrozenSplits[NonNegativeInt].read_from_file(self.splits_path) + splits = PrecomputedSplits[NonNegativeInt].read_from_file(self.splits_path) return splits.assign_splits(range(n_rows), source_sha256=SLEEP_COT_SHA256) @@ -207,7 +207,7 @@ async def download_async(self, cache_dir: Path) -> list[SleepCotReader]: # ruff cache_dir: Directory for the downloaded file. Returns: - A reader for the release CSV and the frozen split of the connector. + A reader for the release CSV and the precomputed split of the connector. """ artifact = Artifact(SLEEP_COT_URL, cache_dir / "sleep_cot.csv", sha256=SLEEP_COT_SHA256) (csv_path,) = await download_files([artifact]) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py index e3d0e2c14..db4a5f750 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/generate_splits.py @@ -7,7 +7,7 @@ # "scikit-learn==1.7.2", # ] # /// -"""Write splits.json, the frozen train, validation and test split of sleep-cot. +"""Write splits.json, the precomputed train, validation and test split of sleep-cot. sleep-cot does not store the train/val/test splits explicitly. Rather they give a random seed and allow scikit to do the split. This is hard to reproduce, because it means the official split is diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py index 364a9d443..4870aaef8 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py @@ -1,8 +1,8 @@ """Generate dataset partitions during connector conversion.""" from timenet_connectors.splitting.base import DatasetSplits, Splitter -from timenet_connectors.splitting.frozen import FrozenSplits +from timenet_connectors.splitting.precomputed import PrecomputedSplits from timenet_connectors.splitting.stratified import StratifiedSplitter -__all__ = ["DatasetSplits", "FrozenSplits", "Splitter", "StratifiedSplitter"] +__all__ = ["DatasetSplits", "PrecomputedSplits", "Splitter", "StratifiedSplitter"] diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py similarity index 95% rename from packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py rename to packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py index 397a7eadb..16b640824 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/frozen.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py @@ -1,4 +1,4 @@ -"""A split that a script computed once and froze into a file beside its connector.""" +"""A split that a script computed once and wrote to a file beside its connector.""" from collections import Counter from collections.abc import Sequence @@ -16,8 +16,8 @@ """How a split file names a sample: a row position or a source id.""" -class FrozenSplits(BaseModel, Generic[K]): - """The validation and test samples of a split that a script froze into a file. +class PrecomputedSplits(BaseModel, Generic[K]): + """The validation and test samples of a split that a script computed once and wrote to a file. Use it when the source has no split of its own and the published split depends on a library that the connector should not pin, such as a seeded shuffle. The script runs the published code once diff --git a/packages/timenet-connectors/tests/test_frozen_splits.py b/packages/timenet-connectors/tests/test_precomputed_splits.py similarity index 84% rename from packages/timenet-connectors/tests/test_frozen_splits.py rename to packages/timenet-connectors/tests/test_precomputed_splits.py index 64531f209..dd6162806 100644 --- a/packages/timenet-connectors/tests/test_frozen_splits.py +++ b/packages/timenet-connectors/tests/test_precomputed_splits.py @@ -7,7 +7,7 @@ from timenet.errors import TimeNetConnectorError from timenet.types.splits import Split -from timenet_connectors.splitting import FrozenSplits +from timenet_connectors.splitting import PrecomputedSplits _SOURCE_SHA256 = "a" * 64 @@ -21,7 +21,7 @@ def _write_split_file(path: Path, **splits: object) -> Path: def test_rows_by_position_get_their_split_and_the_rest_are_train(tmp_path): path = _write_split_file(tmp_path / "splits.json", validation=[1], test=[3]) - splits = FrozenSplits[NonNegativeInt].read_from_file(path) + splits = PrecomputedSplits[NonNegativeInt].read_from_file(path) assert splits.assign_splits(range(5), source_sha256=_SOURCE_SHA256) == [ Split.TRAIN, Split.VALIDATION, @@ -33,7 +33,7 @@ def test_rows_by_position_get_their_split_and_the_rest_are_train(tmp_path): def test_series_by_id_get_their_split_from_an_xz_file(tmp_path): path = _write_split_file(tmp_path / "splits.json.xz", validation=["W1"], test=["M3"]) - splits = FrozenSplits[str].read_from_file(path) + splits = PrecomputedSplits[str].read_from_file(path) assert splits.assign_splits(["M3", "D1", "W1"], source_sha256=_SOURCE_SHA256) == [ Split.TEST, Split.TRAIN, @@ -42,13 +42,13 @@ def test_series_by_id_get_their_split_from_an_xz_file(tmp_path): def test_a_split_made_from_another_source_raises(tmp_path): - splits = FrozenSplits[str].read_from_file(_write_split_file(tmp_path / "splits.json", validation=[], test=[])) + splits = PrecomputedSplits[str].read_from_file(_write_split_file(tmp_path / "splits.json", validation=[], test=[])) with pytest.raises(TimeNetConnectorError, match=f"made from the source {_SOURCE_SHA256}, not from the pinned 0+"): splits.assign_splits(["M1"], source_sha256="0" * 64) def test_a_split_that_names_a_key_the_source_does_not_have_raises(tmp_path): - splits = FrozenSplits[NonNegativeInt].read_from_file( + splits = PrecomputedSplits[NonNegativeInt].read_from_file( _write_split_file(tmp_path / "splits.json", validation=[1], test=[5]) ) with pytest.raises(TimeNetConnectorError, match=r"the split names \[5\] that the source does not have"): @@ -68,11 +68,11 @@ def test_a_malformed_split_file_raises_with_its_name(tmp_path, content, message) path = tmp_path / "splits.json" path.write_text(content) with pytest.raises(TimeNetConnectorError, match=rf"(?s)splits\.json does not hold a valid split: .*{message}"): - FrozenSplits[NonNegativeInt].read_from_file(path) + PrecomputedSplits[NonNegativeInt].read_from_file(path) def test_an_xz_file_that_is_not_compressed_raises(tmp_path): path = tmp_path / "splits.json.xz" path.write_text("{}") with pytest.raises(TimeNetConnectorError, match=r"splits\.json\.xz does not hold a valid split"): - FrozenSplits[str].read_from_file(path) + PrecomputedSplits[str].read_from_file(path) From 42dbfd1e74573ea6f1599620941df98d55683486 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 20 Nov 2026 10:34:03 -0800 Subject: [PATCH 4/5] refactor(connectors): name the key type of PrecomputedSplits SampleKeyT K becomes SampleKeyT, like RowT of the readers. The docstring no longer says that the split comes from a library the connector should not pin. It now says that the split depends on a library version, so that making it again at build time could change it. --- .../splitting/precomputed.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py index 16b640824..5d0c76151 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py @@ -12,26 +12,26 @@ from timenet.types.splits import Split -K = TypeVar("K", int, str) +SampleKeyT = TypeVar("SampleKeyT", int, str) """How a split file names a sample: a row position or a source id.""" -class PrecomputedSplits(BaseModel, Generic[K]): +class PrecomputedSplits(BaseModel, Generic[SampleKeyT]): """The validation and test samples of a split that a script computed once and wrote to a file. - Use it when the source has no split of its own and the published split depends on a library that - the connector should not pin, such as a seeded shuffle. The script runs the published code once - and writes this file. The file names each sample by a key: a row position or a source id. Every - sample that the file does not name is in the train split. The file is JSON, compressed with xz - when its name ends in ``.xz``. + Use it when the source has no split of its own and the split depends on the version of a library, + such as a seeded shuffle, so that making it again at build time could change it. The script makes + the split once and writes this file. The file names each sample by a key: a row position or a + source id. Every sample that the file does not name is in the train split. The file is JSON, + compressed with xz when its name ends in ``.xz``. """ model_config = ConfigDict(extra="forbid", frozen=True) source_sha256: str """SHA-256 digest of the source file that the split was made from.""" - validation: tuple[K, ...] - test: tuple[K, ...] + validation: tuple[SampleKeyT, ...] + test: tuple[SampleKeyT, ...] @model_validator(mode="after") def _each_key_in_one_split(self) -> Self: @@ -41,7 +41,7 @@ def _each_key_in_one_split(self) -> Self: return self @property - def named_keys(self) -> tuple[K, ...]: + def named_keys(self) -> tuple[SampleKeyT, ...]: """The validation and test keys together.""" return (*self.validation, *self.test) @@ -63,7 +63,7 @@ def read_from_file(cls, path: Path) -> Self: except (lzma.LZMAError, ValidationError) as exc: raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc - def assign_splits(self, keys: Sequence[K], *, source_sha256: str) -> list[Split]: + def assign_splits(self, keys: Sequence[SampleKeyT], *, source_sha256: str) -> list[Split]: """Assign a split to each sample of the pinned source. Args: From 939fd8c7ae6370d92270aa4689ffdb3f11c62fa3 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 20 Nov 2026 11:48:28 -0800 Subject: [PATCH 5/5] refactor(connectors): make the precomputed split a Splitter PrecomputedSplits becomes PrecomputedSplitter, a Splitter like StratifiedSplitter. PrecomputedSplitter.from_file reads the split file and checks it against the pinned source digest. split() returns DatasetSplits. The file names a sample by its position, or by the key that the key function returns. The file format does not change. sleep_cot and m4_caption now loop over the DatasetSplits, like tsqa. So their records come split by split, each split in source order. M4 keeps one series class, M4CaptionSeries, and the reader gets split_series. The connector docs describe the new splitter. --- docs/connectors.md | 24 ++++ .../datasets/opentslm/m4_caption/connector.py | 78 +++++------ .../opentslm/m4_caption/generate_splits.py | 10 +- .../m4_caption/tests/test_connector.py | 14 +- .../datasets/opentslm/sleep_cot/connector.py | 23 ++-- .../sleep_cot/tests/test_connector.py | 6 +- .../timenet_connectors/splitting/__init__.py | 4 +- .../splitting/precomputed.py | 121 +++++++++++------- .../tests/test_precomputed_splits.py | 50 ++++---- 9 files changed, 179 insertions(+), 151 deletions(-) diff --git a/docs/connectors.md b/docs/connectors.md index 84559bda1..4a14933a5 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -169,6 +169,30 @@ input order or adding samples can change it. No global random state is modified. package pins scikit-learn 1.9.1 because its allocation affects persisted dataset contents. Review generated assignments and version affected datasets when upgrading the splitter implementation. +If the split must not change with a library version, or must reproduce a split that other code +made, a script computes it once and writes the validation and test samples to a file beside the +connector. `PrecomputedSplitter` reads that file and checks that the split was made from the +pinned source file. Every sample that the file does not name goes to train. The file names a +sample by its position in the samples, or by the key that `key` returns: + +```python +from timenet_connectors.splitting import PrecomputedSplitter + +def get_series_id(sample): + return sample.series_id + +splitter = PrecomputedSplitter.from_file( + SPLITS_PATH, + source_sha256=SOURCE_SHA256, + key=get_series_id, +) +splits = splitter.split(samples) +``` + +The file is JSON, compressed with xz when its name ends in `.xz`. It holds `source_sha256`, +`validation` and `test`. The `opentslm/sleep-cot` and `opentslm/m4-caption` connectors keep the +script that wrote their file, `generate_splits.py`. + When constructing TimeNet tasks, copy the partition name onto each task: ```python diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py index 81de1e205..83adade25 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py @@ -6,7 +6,7 @@ stratified by frequency, and wrote it to splits.json.xz. The connector reads that file. """ -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from dataclasses import dataclass from enum import StrEnum from pathlib import Path @@ -23,7 +23,7 @@ from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec from timenet_connectors.download import ensure_archive from timenet_connectors.sources.readers import CsvDataReader, DataRow -from timenet_connectors.splitting import PrecomputedSplits +from timenet_connectors.splitting import DatasetSplits, PrecomputedSplitter M4_CAPTION_URL = "https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj/download/M4TimeSeriesCaptionDatasetV02.zip" @@ -104,27 +104,16 @@ class _CaptionRow(DataRow): caption: str -@dataclass(frozen=True) -class M4CaptionedSeries: - """One M4 series of the release, with its frequency and caption, before the split is known.""" - - m4_id: str - frequency: M4Frequency - values: np.ndarray - caption: str - - @dataclass(frozen=True) class M4CaptionSeries: - """One M4 series of the release, with its frequency, caption and split.""" + """One M4 series of the release, with its frequency and caption.""" m4_id: str frequency: M4Frequency values: np.ndarray caption: str - split: Split - def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]: + def create_record_and_caption_task(self, *, split: Split) -> tuple[Record, AnswerTask]: """Create the Record of the series and the task that asks for its caption. The Record holds the values, the M4 id and the frequency. The task names the Record as its @@ -140,7 +129,7 @@ def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]: annotations=(frequency,), metadata={"m4_id": self.m4_id}, ) - task = AnswerTask(inputs=(record,), prompt=CAPTION_PROMPT, targets=(self.caption,), split=self.split) + task = AnswerTask(inputs=(record,), prompt=CAPTION_PROMPT, targets=(self.caption,), split=split) return record, task @@ -152,43 +141,33 @@ class M4CaptionReader: splits_path: Path def read_series(self) -> list[M4CaptionSeries]: - """Read every series of the release with its caption and split, in the order of OpenTSLM's loader. + """Read every series of the release with its caption, in the order of OpenTSLM's loader. The loader reads the files of each frequency, and skips ``m4_series.csv`` and ``m4_captions.csv``. - A malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and - its row. The split comes from the split file, which the method checks against every M4 id. + The method leaves out each series whose caption is an error message of the caption API. A + malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and + its row. Returns: - Each series with its caption and split. + Each series with its caption. """ - captioned = self.read_captioned_series() - precomputed = PrecomputedSplits[str].read_from_file(self.splits_path) - splits = precomputed.assign_splits([series.m4_id for series in captioned], source_sha256=M4_CAPTION_SHA256) - return [ - M4CaptionSeries( - m4_id=series.m4_id, - frequency=series.frequency, - values=series.values, - caption=series.caption, - split=split, - ) - for series, split in zip(captioned, splits, strict=True) - ] - - def read_captioned_series(self) -> list[M4CaptionedSeries]: - """Read every series of the release with its caption, in the order of :meth:`read_series`. + series: list[M4CaptionSeries] = [] + for frequency in M4Frequency: + series.extend(self._read_frequency(frequency)) + return series - The method leaves out each series whose caption is an error message of the caption API. + def split_series(self, series: Sequence[M4CaptionSeries]) -> DatasetSplits[M4CaptionSeries]: + """Put each series in the split that the split file names for its M4 id. Returns: - Each series with its caption, without the split. + The series of each split, in the order of ``series``. """ - captioned: list[M4CaptionedSeries] = [] - for frequency in M4Frequency: - captioned.extend(self._read_frequency(frequency)) - return captioned + splitter = PrecomputedSplitter[M4CaptionSeries].from_file( + self.splits_path, source_sha256=M4_CAPTION_SHA256, key=lambda one_series: one_series.m4_id + ) + return splitter.split(series) - def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries]: + def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionSeries]: """Read the series of one frequency, paired with the caption in the same row of its caption file. Yields: @@ -214,7 +193,7 @@ def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries] ) if caption_row.caption.startswith(CAPTION_ERROR_PREFIX): continue - yield M4CaptionedSeries( + yield M4CaptionSeries( m4_id=m4_id, frequency=frequency, values=np.array(series_row.series, dtype=np.float32), @@ -248,12 +227,13 @@ def convert(self, raw_refs: list[M4CaptionReader]) -> TimeFDataset: Returns: The populated dataset. """ - (release,) = raw_refs + (reader,) = raw_refs dataset = TimeFDataset(metadata=self.metadata()) - for series in release.read_series(): - record, task = series.create_record_and_caption_task() - dataset.add_record(record=record) - dataset.add_task(task=task) + for split, series_in_split in reader.split_series(reader.read_series()).items(): + for series in series_in_split: + record, task = series.create_record_and_caption_task(split=split) + dataset.add_record(record=record) + dataset.add_task(task=task) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py index 9b8e59ac2..b1271e8fd 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py @@ -30,8 +30,8 @@ M4_CAPTION_SHA256, RELEASE_FOLDER, SPLITS_PATH, - M4CaptionedSeries, M4CaptionReader, + M4CaptionSeries, M4Frequency, ) from timenet_connectors.splitting import StratifiedSplitter @@ -40,7 +40,7 @@ SEED = 42 -def frequency(series: M4CaptionedSeries) -> M4Frequency: +def frequency(series: M4CaptionSeries) -> M4Frequency: """Return the frequency of a series, which the split stratifies by. Returns: @@ -49,7 +49,7 @@ def frequency(series: M4CaptionedSeries) -> M4Frequency: return series.frequency -def values_digest(series: M4CaptionedSeries) -> str: +def values_digest(series: M4CaptionSeries) -> str: """Return the SHA-256 digest of the values of a series. Series with the same values form one group. Returns: @@ -80,9 +80,9 @@ def main() -> None: with tempfile.TemporaryDirectory() as directory, zipfile.ZipFile(archive_path) as archive: archive.extractall(directory) reader = M4CaptionReader(directory=Path(directory) / RELEASE_FOLDER, splits_path=SPLITS_PATH) - captioned = reader.read_captioned_series() + captioned = reader.read_series() - splitter = StratifiedSplitter[M4CaptionedSeries]( + splitter = StratifiedSplitter[M4CaptionSeries]( train_folds=8, validation_folds=1, test_folds=1, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py index 6df0c24f9..c3979249d 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py @@ -21,7 +21,6 @@ M4CaptionConnector, M4CaptionReader, ) -from timenet_connectors.splitting import PrecomputedSplits _SERIES_IDS = { @@ -101,7 +100,10 @@ def _convert(directory: Path, **changes) -> TimeFDataset: def test_each_series_becomes_a_record_and_a_caption_task(tmp_path): tasks = _convert(tmp_path).tasks_of(AnswerTask) - expected = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids] + release_order = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids] + # The tasks come split by split, each split in release order. + split_order = [Split.TRAIN, Split.VALIDATION, Split.TEST] + expected = sorted(release_order, key=lambda series: split_order.index(_expected_split(series[1]))) assert len(tasks) == len(expected) for task, (frequency, m4_id) in zip(tasks, expected, strict=True): (record,) = task.inputs @@ -131,7 +133,7 @@ def test_a_series_whose_caption_is_an_error_message_is_left_out(tmp_path): "Q1": "Error generating caption: Error code: 404", } tasks = _convert(tmp_path, captions=captions).tasks_of(AnswerTask) - assert [task.inputs[0].metadata["m4_id"] for task in tasks] == ["D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"] + assert {task.inputs[0].metadata["m4_id"] for task in tasks} == {"D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"} def test_a_split_file_of_another_release_raises(tmp_path): @@ -146,9 +148,9 @@ def test_a_split_file_of_another_release_raises(tmp_path): def test_the_precomputed_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): - splits = PrecomputedSplits[str].read_from_file(SPLITS_PATH) - assert splits.source_sha256 == M4_CAPTION_SHA256 - for m4_ids in (splits.validation, splits.test): + splits = json.loads(lzma.decompress(SPLITS_PATH.read_bytes())) + assert splits["source_sha256"] == M4_CAPTION_SHA256 + for m4_ids in (splits["validation"], splits["test"]): counts = Counter(m4_id[0] for m4_id in m4_ids) for letter, dataset_count in _DATASET_SERIES_COUNTS.items(): # A tenth of the series of each frequency, rounded down or up. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index 6c86c30a5..104e77366 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -6,6 +6,7 @@ the stage. """ +from collections.abc import Sequence from dataclasses import dataclass import json from pathlib import Path @@ -13,7 +14,6 @@ from typing import Self import numpy as np -from pydantic import NonNegativeInt from timenet.connectors import BaseConnector from timenet.dataset import Record, Signal, Source, TimeFDataset @@ -22,7 +22,7 @@ from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg from timenet_connectors.download import Artifact, download_files from timenet_connectors.sources.readers import CsvDataReader, DataRow -from timenet_connectors.splitting import PrecomputedSplits +from timenet_connectors.splitting import DatasetSplits, PrecomputedSplitter SLEEP_COT_URL = "https://polybox.ethz.ch/index.php/s/ZryWSdCFJZ9JR3R/download" @@ -187,14 +187,14 @@ def read_rows(self) -> list[SleepCotRow]: raise TimeNetConnectorError(f"{self.csv_path.name} data row {number}: {exc}") from exc return rows - def read_splits(self, n_rows: int) -> list[Split]: - """Read the split of each row from the splits file. + def split_rows(self, rows: Sequence[SleepCotRow]) -> DatasetSplits[SleepCotRow]: + """Put each row in the split that the splits file names for its position. Returns: - The split for each of the n_rows rows, in file order. + The rows of each split, in file order. """ - splits = PrecomputedSplits[NonNegativeInt].read_from_file(self.splits_path) - return splits.assign_splits(range(n_rows), source_sha256=SLEEP_COT_SHA256) + splitter = PrecomputedSplitter[SleepCotRow].from_file(self.splits_path, source_sha256=SLEEP_COT_SHA256) + return splitter.split(rows) class SleepCotConnector(BaseConnector[SleepCotReader]): @@ -223,12 +223,11 @@ def convert(self, raw_refs: list[SleepCotReader]) -> TimeFDataset: The populated dataset. """ (reader,) = raw_refs - rows = reader.read_rows() - splits = reader.read_splits(len(rows)) dataset = TimeFDataset(metadata=self.metadata()) - for row, split in zip(rows, splits, strict=True): - record = dataset.add_record(record=row.create_record()) - dataset.add_task(task=row.create_classification_task(record, split=split)) + for split, rows in reader.split_rows(reader.read_rows()).items(): + for row in rows: + record = dataset.add_record(record=row.create_record()) + dataset.add_task(task=row.create_classification_task(record, split=split)) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py index 573373098..e39887573 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py @@ -74,8 +74,10 @@ def _write_release( def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): tasks = SleepCotConnector().convert([_write_release(tmp_path)]).tasks_of(ClassificationTask) - assert [task.split for task in tasks] == [Split.TRAIN, Split.VALIDATION, Split.TRAIN, Split.TEST] - for task, row in zip(tasks, _ROWS, strict=True): + # The tasks come split by split, each split in file order. + assert [task.split for task in tasks] == [Split.TRAIN, Split.TRAIN, Split.VALIDATION, Split.TEST] + rows_in_task_order = [_ROWS[0], _ROWS[2], _ROWS[1], _ROWS[3]] + for task, row in zip(tasks, rows_in_task_order, strict=True): (record,) = task.inputs (signal,) = record.signals assert signal.name == "EEG Fpz-Cz" diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py index 4870aaef8..f293f9985 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/__init__.py @@ -1,8 +1,8 @@ """Generate dataset partitions during connector conversion.""" from timenet_connectors.splitting.base import DatasetSplits, Splitter -from timenet_connectors.splitting.precomputed import PrecomputedSplits +from timenet_connectors.splitting.precomputed import PrecomputedSplitter, SampleKey from timenet_connectors.splitting.stratified import StratifiedSplitter -__all__ = ["DatasetSplits", "PrecomputedSplits", "Splitter", "StratifiedSplitter"] +__all__ = ["DatasetSplits", "PrecomputedSplitter", "SampleKey", "Splitter", "StratifiedSplitter"] diff --git a/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py index 5d0c76151..5b59582da 100644 --- a/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py +++ b/packages/timenet-connectors/src/timenet_connectors/splitting/precomputed.py @@ -1,88 +1,111 @@ -"""A split that a script computed once and wrote to a file beside its connector.""" +"""A splitter that reads a split that a script computed once and wrote to a file beside its connector.""" from collections import Counter -from collections.abc import Sequence +from collections.abc import Callable, Iterable, Sequence import lzma from pathlib import Path -from typing import Generic, Self, TypeVar +from typing import Self -from pydantic import BaseModel, ConfigDict, ValidationError, model_validator +from pydantic import BaseModel, ConfigDict, NonNegativeInt, ValidationError from timenet.errors import TimeNetConnectorError from timenet.types.splits import Split +from timenet_connectors.splitting.base import DatasetSplits, Splitter, T -SampleKeyT = TypeVar("SampleKeyT", int, str) -"""How a split file names a sample: a row position or a source id.""" +SampleKey = NonNegativeInt | str +"""How a split file names a sample: by its position in the samples, or by a key such as a source id.""" -class PrecomputedSplits(BaseModel, Generic[SampleKeyT]): - """The validation and test samples of a split that a script computed once and wrote to a file. - - Use it when the source has no split of its own and the split depends on the version of a library, - such as a seeded shuffle, so that making it again at build time could change it. The script makes - the split once and writes this file. The file names each sample by a key: a row position or a - source id. Every sample that the file does not name is in the train split. The file is JSON, - compressed with xz when its name ends in ``.xz``. - """ +class _SplitFile(BaseModel): + """The content of a split file.""" model_config = ConfigDict(extra="forbid", frozen=True) source_sha256: str """SHA-256 digest of the source file that the split was made from.""" - validation: tuple[SampleKeyT, ...] - test: tuple[SampleKeyT, ...] + validation: tuple[SampleKey, ...] + test: tuple[SampleKey, ...] - @model_validator(mode="after") - def _each_key_in_one_split(self) -> Self: - repeated = sorted(key for key, count in Counter(self.named_keys).items() if count > 1) - if repeated: - raise ValueError(f"{repeated} are named more than once") - return self - @property - def named_keys(self) -> tuple[SampleKeyT, ...]: - """The validation and test keys together.""" - return (*self.validation, *self.test) +class PrecomputedSplitter(Splitter[T]): + """Put each sample in the split that a split file names for it, and every other sample in train. + + Use it when the source has no split of its own and the split depends on the version of a library, + such as a seeded shuffle, so that making it again at build time could change it. A script makes + the split once and writes the file. The file is JSON, compressed with xz when its name ends in + ``.xz``. It names the validation and the test samples, either by their position in the samples or + by the key that ``key`` returns for each sample. + """ + + def __init__( + self, + *, + validation: Iterable[SampleKey], + test: Iterable[SampleKey], + key: Callable[[T], SampleKey] | None = None, + ) -> None: + """Create a splitter from the validation and test keys. + + Args: + validation: The keys of the validation samples. + test: The keys of the test samples. + key: Return the key of a sample. Without it, the key of a sample is its position. + + Raises: + TimeNetConnectorError: If a key is in both splits or twice in one split. + """ + validation, test = tuple(validation), tuple(test) + repeated = sorted((k for k, count in Counter((*validation, *test)).items() if count > 1), key=str) + if repeated: + raise TimeNetConnectorError(f"{repeated} are named more than once") + self._split_of = dict.fromkeys(validation, Split.VALIDATION) | dict.fromkeys(test, Split.TEST) + self._key = key @classmethod - def read_from_file(cls, path: Path) -> Self: - """Read the split from a JSON file, or from an xz-compressed one if the name ends in ``.xz``. + def from_file(cls, path: Path, *, source_sha256: str, key: Callable[[T], SampleKey] | None = None) -> Self: + """Read the split from a file, and check that it was made from the pinned source. + + Args: + path: The split file. + source_sha256: The pinned digest of the source file. + key: Return the key of a sample. Without it, the key of a sample is its position. Returns: - The split. + The splitter. Raises: - TimeNetConnectorError: If the file does not hold a valid split. + TimeNetConnectorError: If the file does not hold a valid split, or the split was made from + another source. """ try: content = path.read_bytes() if path.suffix == ".xz": content = lzma.decompress(content) - return cls.model_validate_json(content) - except (lzma.LZMAError, ValidationError) as exc: + split_file = _SplitFile.model_validate_json(content) + splitter = cls(validation=split_file.validation, test=split_file.test, key=key) + except (lzma.LZMAError, ValidationError, TimeNetConnectorError) as exc: raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc + if split_file.source_sha256 != source_sha256: + raise TimeNetConnectorError( + f"the split was made from the source {split_file.source_sha256}, not from the pinned {source_sha256}" + ) + return splitter - def assign_splits(self, keys: Sequence[SampleKeyT], *, source_sha256: str) -> list[Split]: - """Assign a split to each sample of the pinned source. - - Args: - keys: The key of each sample of the source, in the order of the samples. - source_sha256: The pinned digest of the source file. + def split(self, samples: Sequence[T]) -> DatasetSplits[T]: + """Put each sample in its split, keeping the order of the samples within each split. Returns: - The split of each sample, in the order of ``keys``. + The train, validation and test samples. Raises: - TimeNetConnectorError: If the split was made from another source, or names a key that the - source does not have. + TimeNetConnectorError: If the split names a key that no sample has. """ - if self.source_sha256 != source_sha256: - raise TimeNetConnectorError( - f"the split was made from the source {self.source_sha256}, not from the pinned {source_sha256}" - ) - unknown = sorted(set(self.named_keys) - set(keys)) + keys: Sequence[SampleKey] = range(len(samples)) if self._key is None else [self._key(s) for s in samples] + unknown = sorted(set(self._split_of) - set(keys), key=str) if unknown: raise TimeNetConnectorError(f"the split names {unknown} that the source does not have") - split_of = dict.fromkeys(self.validation, Split.VALIDATION) | dict.fromkeys(self.test, Split.TEST) - return [split_of.get(key, Split.TRAIN) for key in keys] + partitions: dict[Split, list[T]] = {Split.TRAIN: [], Split.VALIDATION: [], Split.TEST: []} + for sample, sample_key in zip(samples, keys, strict=True): + partitions[self._split_of.get(sample_key, Split.TRAIN)].append(sample) + return DatasetSplits(partitions) diff --git a/packages/timenet-connectors/tests/test_precomputed_splits.py b/packages/timenet-connectors/tests/test_precomputed_splits.py index dd6162806..be9a1a3a7 100644 --- a/packages/timenet-connectors/tests/test_precomputed_splits.py +++ b/packages/timenet-connectors/tests/test_precomputed_splits.py @@ -2,12 +2,11 @@ import lzma from pathlib import Path -from pydantic import NonNegativeInt import pytest from timenet.errors import TimeNetConnectorError from timenet.types.splits import Split -from timenet_connectors.splitting import PrecomputedSplits +from timenet_connectors.splitting import PrecomputedSplitter _SOURCE_SHA256 = "a" * 64 @@ -19,40 +18,39 @@ def _write_split_file(path: Path, **splits: object) -> Path: return path -def test_rows_by_position_get_their_split_and_the_rest_are_train(tmp_path): +def test_samples_by_position_get_their_split_and_the_rest_are_train(tmp_path): path = _write_split_file(tmp_path / "splits.json", validation=[1], test=[3]) - splits = PrecomputedSplits[NonNegativeInt].read_from_file(path) - assert splits.assign_splits(range(5), source_sha256=_SOURCE_SHA256) == [ - Split.TRAIN, - Split.VALIDATION, - Split.TRAIN, - Split.TEST, - Split.TRAIN, - ] + splitter = PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256) + assert dict(splitter.split(["a", "b", "c", "d", "e"])) == { + Split.TRAIN: ("a", "c", "e"), + Split.VALIDATION: ("b",), + Split.TEST: ("d",), + } -def test_series_by_id_get_their_split_from_an_xz_file(tmp_path): +def test_samples_by_key_get_their_split_from_an_xz_file(tmp_path): path = _write_split_file(tmp_path / "splits.json.xz", validation=["W1"], test=["M3"]) - splits = PrecomputedSplits[str].read_from_file(path) - assert splits.assign_splits(["M3", "D1", "W1"], source_sha256=_SOURCE_SHA256) == [ - Split.TEST, - Split.TRAIN, - Split.VALIDATION, - ] + splitter = PrecomputedSplitter[tuple[str, int]].from_file( + path, source_sha256=_SOURCE_SHA256, key=lambda sample: sample[0] + ) + assert dict(splitter.split([("M3", 0), ("D1", 1), ("W1", 2)])) == { + Split.TRAIN: (("D1", 1),), + Split.VALIDATION: (("W1", 2),), + Split.TEST: (("M3", 0),), + } def test_a_split_made_from_another_source_raises(tmp_path): - splits = PrecomputedSplits[str].read_from_file(_write_split_file(tmp_path / "splits.json", validation=[], test=[])) + path = _write_split_file(tmp_path / "splits.json", validation=[], test=[]) with pytest.raises(TimeNetConnectorError, match=f"made from the source {_SOURCE_SHA256}, not from the pinned 0+"): - splits.assign_splits(["M1"], source_sha256="0" * 64) + PrecomputedSplitter[str].from_file(path, source_sha256="0" * 64) def test_a_split_that_names_a_key_the_source_does_not_have_raises(tmp_path): - splits = PrecomputedSplits[NonNegativeInt].read_from_file( - _write_split_file(tmp_path / "splits.json", validation=[1], test=[5]) - ) + path = _write_split_file(tmp_path / "splits.json", validation=[1], test=[5]) + splitter = PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256) with pytest.raises(TimeNetConnectorError, match=r"the split names \[5\] that the source does not have"): - splits.assign_splits(range(5), source_sha256=_SOURCE_SHA256) + splitter.split(["a", "b", "c", "d", "e"]) @pytest.mark.parametrize( @@ -68,11 +66,11 @@ def test_a_malformed_split_file_raises_with_its_name(tmp_path, content, message) path = tmp_path / "splits.json" path.write_text(content) with pytest.raises(TimeNetConnectorError, match=rf"(?s)splits\.json does not hold a valid split: .*{message}"): - PrecomputedSplits[NonNegativeInt].read_from_file(path) + PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256) def test_an_xz_file_that_is_not_compressed_raises(tmp_path): path = tmp_path / "splits.json.xz" path.write_text("{}") with pytest.raises(TimeNetConnectorError, match=r"splits\.json\.xz does not hold a valid split"): - PrecomputedSplits[str].read_from_file(path) + PrecomputedSplitter[str].from_file(path, source_sha256=_SOURCE_SHA256)