Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions docs/connectors.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,30 @@ input order or adding samples can change it. No global random state is modified.
package pins scikit-learn 1.9.1 because its allocation affects persisted dataset contents. Review
generated assignments and version affected datasets when upgrading the splitter implementation.

If the split must not change with a library version, or must reproduce a split that other code
made, a script computes it once and writes the validation and test samples to a file beside the
connector. `PrecomputedSplitter` reads that file and checks that the split was made from the
pinned source file. Every sample that the file does not name goes to train. The file names a
sample by its position in the samples, or by the key that `key` returns:

```python
from timenet_connectors.splitting import PrecomputedSplitter

def get_series_id(sample):
return sample.series_id

splitter = PrecomputedSplitter.from_file(
SPLITS_PATH,
source_sha256=SOURCE_SHA256,
key=get_series_id,
)
splits = splitter.split(samples)
```

The file is JSON, compressed with xz when its name ends in `.xz`. It holds `source_sha256`,
`validation` and `test`. The `opentslm/sleep-cot` and `opentslm/m4-caption` connectors keep the
script that wrote their file, `generate_splits.py`.

When constructing TimeNet tasks, copy the partition name onto each task:

```python
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,15 @@
stratified by frequency, and wrote it to splits.json.xz. The connector reads that file.
"""

from collections import Counter
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from enum import StrEnum
import lzma
from pathlib import Path
import re
from typing import Annotated, Self
from typing import Annotated

import numpy as np
from pydantic import AllowInfNan, BaseModel, ConfigDict, Field, Json, StrictFloat, ValidationError, model_validator
from pydantic import AllowInfNan, Field, Json, StrictFloat

from timenet.connectors import BaseConnector
from timenet.dataset import Record, Signal, Source, TimeFDataset
Expand All @@ -25,6 +23,7 @@
from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec
from timenet_connectors.download import ensure_archive
from timenet_connectors.sources.readers import CsvDataReader, DataRow
from timenet_connectors.splitting import DatasetSplits, PrecomputedSplitter


M4_CAPTION_URL = "https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj/download/M4TimeSeriesCaptionDatasetV02.zip"
Expand Down Expand Up @@ -105,27 +104,16 @@ class _CaptionRow(DataRow):
caption: str


@dataclass(frozen=True)
class M4CaptionedSeries:
"""One M4 series of the release, with its frequency and caption, before the split is known."""

m4_id: str
frequency: M4Frequency
values: np.ndarray
caption: str


@dataclass(frozen=True)
class M4CaptionSeries:
"""One M4 series of the release, with its frequency, caption and split."""
"""One M4 series of the release, with its frequency and caption."""

m4_id: str
frequency: M4Frequency
values: np.ndarray
caption: str
split: Split

def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]:
def create_record_and_caption_task(self, *, split: Split) -> tuple[Record, AnswerTask]:
"""Create the Record of the series and the task that asks for its caption.

The Record holds the values, the M4 id and the frequency. The task names the Record as its
Expand All @@ -141,73 +129,10 @@ def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]:
annotations=(frequency,),
metadata={"m4_id": self.m4_id},
)
task = AnswerTask(inputs=(record,), prompt=CAPTION_PROMPT, targets=(self.caption,), split=self.split)
task = AnswerTask(inputs=(record,), prompt=CAPTION_PROMPT, targets=(self.caption,), split=split)
return record, task


class M4CaptionSplits(BaseModel):
"""The validation and test series of the split, by M4 id.

The release archive stores no split. generate_splits.py made an 80/10/10 split, stratified by
frequency, and wrote it to splits.json.xz. Every series that the file does not name is in the train
split.
"""

model_config = ConfigDict(extra="forbid", frozen=True)

source_sha256: str
"""SHA-256 digest of the release archive that the split was made from."""
validation: tuple[str, ...]
test: tuple[str, ...]

@model_validator(mode="after")
def _each_series_in_one_split(self) -> Self:
repeated = sorted(m4_id for m4_id, count in Counter(self.m4_ids).items() if count > 1)
if repeated:
raise ValueError(f"series {repeated} are named more than once")
return self

@property
def m4_ids(self) -> tuple[str, ...]:
"""The validation and test series together."""
return (*self.validation, *self.test)

@classmethod
def read_from_file(cls, path: Path) -> Self:
"""Read the split from an xz-compressed JSON file.

Returns:
The split.

Raises:
TimeNetConnectorError: If the file does not hold a valid split.
"""
try:
return cls.model_validate_json(lzma.decompress(path.read_bytes()))
except (lzma.LZMAError, ValidationError) as exc:
raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc

def assign_split_to_each_series(self, m4_ids: Sequence[str]) -> list[Split]:
"""Assign a split to each series of the pinned release.

Returns:
The split of each series, in the order of ``m4_ids``.

Raises:
TimeNetConnectorError: If the split was made from another release, or names a series that
the release does not have.
"""
if self.source_sha256 != M4_CAPTION_SHA256:
raise TimeNetConnectorError(
f"the split was made from the release {self.source_sha256}, not from the pinned {M4_CAPTION_SHA256}"
)
unknown = sorted(set(self.m4_ids) - set(m4_ids))
if unknown:
raise TimeNetConnectorError(f"the split names series {unknown} that the release does not have")
split_of = dict.fromkeys(self.validation, Split.VALIDATION) | dict.fromkeys(self.test, Split.TEST)
return [split_of.get(m4_id, Split.TRAIN) for m4_id in m4_ids]


@dataclass(frozen=True)
class M4CaptionReader:
"""Read the series, the captions and the split of the extracted release."""
Expand All @@ -216,43 +141,33 @@ class M4CaptionReader:
splits_path: Path

def read_series(self) -> list[M4CaptionSeries]:
"""Read every series of the release with its caption and split, in the order of OpenTSLM's loader.
"""Read every series of the release with its caption, in the order of OpenTSLM's loader.

The loader reads the files of each frequency, and skips ``m4_series.csv`` and ``m4_captions.csv``.
A malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and
its row. The split comes from the split file, which the method checks against every M4 id.
The method leaves out each series whose caption is an error message of the caption API. A
malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and
its row.

Returns:
Each series with its caption and split.
Each series with its caption.
"""
captioned = self.read_captioned_series()
split_file = M4CaptionSplits.read_from_file(self.splits_path)
splits = split_file.assign_split_to_each_series([series.m4_id for series in captioned])
return [
M4CaptionSeries(
m4_id=series.m4_id,
frequency=series.frequency,
values=series.values,
caption=series.caption,
split=split,
)
for series, split in zip(captioned, splits, strict=True)
]

def read_captioned_series(self) -> list[M4CaptionedSeries]:
"""Read every series of the release with its caption, in the order of :meth:`read_series`.
series: list[M4CaptionSeries] = []
for frequency in M4Frequency:
series.extend(self._read_frequency(frequency))
return series

The method leaves out each series whose caption is an error message of the caption API.
def split_series(self, series: Sequence[M4CaptionSeries]) -> DatasetSplits[M4CaptionSeries]:
"""Put each series in the split that the split file names for its M4 id.

Returns:
Each series with its caption, without the split.
The series of each split, in the order of ``series``.
"""
captioned: list[M4CaptionedSeries] = []
for frequency in M4Frequency:
captioned.extend(self._read_frequency(frequency))
return captioned
splitter = PrecomputedSplitter[M4CaptionSeries].from_file(
self.splits_path, source_sha256=M4_CAPTION_SHA256, key=lambda one_series: one_series.m4_id
)
return splitter.split(series)

def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries]:
def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionSeries]:
"""Read the series of one frequency, paired with the caption in the same row of its caption file.

Yields:
Expand All @@ -278,7 +193,7 @@ def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries]
)
if caption_row.caption.startswith(CAPTION_ERROR_PREFIX):
continue
yield M4CaptionedSeries(
yield M4CaptionSeries(
m4_id=m4_id,
frequency=frequency,
values=np.array(series_row.series, dtype=np.float32),
Expand Down Expand Up @@ -312,12 +227,13 @@ def convert(self, raw_refs: list[M4CaptionReader]) -> TimeFDataset:
Returns:
The populated dataset.
"""
(release,) = raw_refs
(reader,) = raw_refs
dataset = TimeFDataset(metadata=self.metadata())
for series in release.read_series():
record, task = series.create_record_and_caption_task()
dataset.add_record(record=record)
dataset.add_task(task=task)
for split, series_in_split in reader.split_series(reader.read_series()).items():
for series in series_in_split:
record, task = series.create_record_and_caption_task(split=split)
dataset.add_record(record=record)
dataset.add_task(task=task)
return dataset


Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Write splits.json.xz, the frozen train, validation and test split of m4-caption.
"""Write splits.json.xz, the precomputed train, validation and test split of m4-caption.

The release archive stores no split. This script makes an 80/10/10 split with StratifiedSplitter,
stratified by M4 frequency, so that validation and test each hold a tenth of every frequency.
Expand Down Expand Up @@ -30,8 +30,8 @@
M4_CAPTION_SHA256,
RELEASE_FOLDER,
SPLITS_PATH,
M4CaptionedSeries,
M4CaptionReader,
M4CaptionSeries,
M4Frequency,
)
from timenet_connectors.splitting import StratifiedSplitter
Expand All @@ -40,7 +40,7 @@
SEED = 42


def frequency(series: M4CaptionedSeries) -> M4Frequency:
def frequency(series: M4CaptionSeries) -> M4Frequency:
"""Return the frequency of a series, which the split stratifies by.

Returns:
Expand All @@ -49,7 +49,7 @@ def frequency(series: M4CaptionedSeries) -> M4Frequency:
return series.frequency


def values_digest(series: M4CaptionedSeries) -> str:
def values_digest(series: M4CaptionSeries) -> str:
"""Return the SHA-256 digest of the values of a series. Series with the same values form one group.

Returns:
Expand Down Expand Up @@ -80,9 +80,9 @@ def main() -> None:
with tempfile.TemporaryDirectory() as directory, zipfile.ZipFile(archive_path) as archive:
archive.extractall(directory)
reader = M4CaptionReader(directory=Path(directory) / RELEASE_FOLDER, splits_path=SPLITS_PATH)
captioned = reader.read_captioned_series()
captioned = reader.read_series()

splitter = StratifiedSplitter[M4CaptionedSeries](
splitter = StratifiedSplitter[M4CaptionSeries](
train_folds=8,
validation_folds=1,
test_folds=1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
SPLITS_PATH,
M4CaptionConnector,
M4CaptionReader,
M4CaptionSplits,
)


Expand Down Expand Up @@ -101,7 +100,10 @@ def _convert(directory: Path, **changes) -> TimeFDataset:

def test_each_series_becomes_a_record_and_a_caption_task(tmp_path):
tasks = _convert(tmp_path).tasks_of(AnswerTask)
expected = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids]
release_order = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids]
# The tasks come split by split, each split in release order.
split_order = [Split.TRAIN, Split.VALIDATION, Split.TEST]
expected = sorted(release_order, key=lambda series: split_order.index(_expected_split(series[1])))
assert len(tasks) == len(expected)
for task, (frequency, m4_id) in zip(tasks, expected, strict=True):
(record,) = task.inputs
Expand Down Expand Up @@ -131,32 +133,24 @@ def test_a_series_whose_caption_is_an_error_message_is_left_out(tmp_path):
"Q1": "Error generating caption: Error code: 404",
}
tasks = _convert(tmp_path, captions=captions).tasks_of(AnswerTask)
assert [task.inputs[0].metadata["m4_id"] for task in tasks] == ["D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"]
assert {task.inputs[0].metadata["m4_id"] for task in tasks} == {"D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"}


@pytest.mark.parametrize(
("changes", "message"),
[
({"source_sha256": "0" * 64}, "the split was made from the release 0+, not from the pinned"),
({"test": ["M3", "W1"]}, r"series \['W1'\] are named more than once"),
({"test": ["M3", "M9"]}, r"names series \['M9'\] that the release does not have"),
],
ids=["another release", "a series in two splits", "a series that the release does not have"],
)
def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, changes, message):
with pytest.raises(TimeNetConnectorError, match=message):
_convert(tmp_path, splits=_SPLITS | changes)
def test_a_split_file_of_another_release_raises(tmp_path):
# The connector checks the split file against its pinned digest, not against the digest in the file.
with pytest.raises(TimeNetConnectorError, match=r"the split was made from the source 0+, not from the pinned"):
_convert(tmp_path, splits=_SPLITS | {"source_sha256": "0" * 64})


_DATASET_SERIES_COUNTS = {"D": 3974, "H": 414, "M": 47943, "Q": 23604, "W": 359, "Y": 23000}
"""The number of series of each frequency in the dataset, by the letter that starts their M4 ids. The
dataset leaves out 253 daily, 57 monthly and 396 quarterly series, whose caption is an error message."""


def test_the_frozen_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test():
splits = M4CaptionSplits.read_from_file(SPLITS_PATH)
assert splits.source_sha256 == M4_CAPTION_SHA256
for m4_ids in (splits.validation, splits.test):
def test_the_precomputed_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test():
splits = json.loads(lzma.decompress(SPLITS_PATH.read_bytes()))
assert splits["source_sha256"] == M4_CAPTION_SHA256
for m4_ids in (splits["validation"], splits["test"]):
counts = Counter(m4_id[0] for m4_id in m4_ids)
for letter, dataset_count in _DATASET_SERIES_COUNTS.items():
# A tenth of the series of each frequency, rounded down or up.
Expand Down
Loading