diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/README.md new file mode 100644 index 000000000..438e31c47 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/README.md @@ -0,0 +1,63 @@ +# M4-Caption + +M4-Caption pairs each of the 100,000 series of the M4 forecasting competition with one caption that +GPT-4o wrote. OpenTSLM publishes it as a zip archive of CSV files, one series file and one caption +file for each M4 frequency. + +For 706 series, the release has an error message of the caption API instead of a caption. There are +three messages, each after the prefix "Error generating caption: ": "Connection error." (630 +series), "Error code: 404" (75) and "Request timed out." (1). The connector leaves out these 253 +daily, 57 monthly and 396 quarterly series, so the dataset has 99,294 series. + +The connector converts each series into one Record and one `AnswerTask`. The Record holds the +series as one float32 Signal on an ordinal axis, the M4 id in its metadata, and the M4 frequency as +an annotation. The task target is the caption. The task prompt is the prompt of the OpenTSLM loader +(the post-prompt of `M4QADataset`), the same for every series. + +## Split + +The release archive stores no split. `generate_splits.py` makes an 80/10/10 split with +`StratifiedSplitter`, stratified by M4 frequency. Validation and test each hold a tenth of every +frequency, so the small hourly and weekly frequencies keep their share in each split. The result +depends on the version of scikit-learn, so the script ran once. It wrote the 9,929 validation and +9,929 test series, by M4 id, to `splits.json.xz`. The connector reads that file and puts every +other series in the train split. + +M4 has 90 pairs of series whose histories are equal value for value: 19 monthly, 35 quarterly and +36 yearly pairs. The two series of a pair have different M4 ids and different M4 test horizons. All +pairs but one also have different captions. A model that trains on one series of a pair sees the +exact input of the other. So the script groups the series by their values with the `group_by` +option of `StratifiedSplitter`, and each pair stays in one split. + +This split is not the split of the OpenTSLM paper. The OpenTSLM loader makes a random split of all +frequencies together, so its share of the small frequencies drifts. For example, its test split +holds 6.5% of the hourly series. + +The file is JSON compressed with xz, about 19 KB. Python reads it with the `lzma` module of the +standard library. To read it yourself, run `xz -dc splits.json.xz`. The file also states the SHA-256 +of the release archive. If it does not match the pinned archive, the build stops with an error. + +## Relation to the M4 data + +Each series of the release is the train part of the M4 series with the same id, from the +`Dataset/Train` files of the M4-methods repository. The release does not hold the test horizon or +the series information of `M4-info.csv`. Each value is the float32 rounding of the M4 value. The +largest relative difference is 6e-8. + +## Future work: an M4 parent dataset + +TimeNet can model this relation with a parent dataset. A connector for M4, for example +`mcompetitions/m4`, can convert the `Dataset/` folder of M4-methods: the train and test files and +`M4-info.csv`. M4-Caption can then declare that release under `parents` and import its Records. +The caption task then names the train part of each series with a `StepInterval` scope. + +With a parent, the 24 million values exist once, and a reader gets the M4 category, horizon and +test part. The values become the float64 M4 values, not the float32 values that OpenTSLM plotted +for GPT-4o. + +## Future work: change axis types + +We currently use an ordinal axis for consistency, as some of the frequencies are only regular in +calendar terms, e.g. "monthly". If +[Proposal: calendar axes, value cells and civil-date origins](https://github.com/OpenTSLM/TimeNet/issues/555) +lands we can reconsider this choice. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/__init__.py new file mode 100644 index 000000000..210792a47 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/__init__.py @@ -0,0 +1,6 @@ +"""M4-Caption: the M4 competition series, each with a caption that GPT-4o wrote.""" + +from timenet_connectors.datasets.opentslm.m4_caption.connector import ( + CONNECTOR as CONNECTOR, + M4CaptionConnector as M4CaptionConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py new file mode 100644 index 000000000..57c1c6656 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/connector.py @@ -0,0 +1,323 @@ +"""The M4-Caption connector: M4 competition series, each with a caption that GPT-4o wrote. + +The OpenTSLM release holds the train part of all 100,000 M4 series, one CSV of series and one CSV of +captions for each of the six M4 frequencies. Each series becomes one Record and one AnswerTask. The +target is the caption. The connector leaves out the 706 series whose caption is an error message. The release archive stores no split. generate_splits.py made an 80/10/10 split, +stratified by frequency, and wrote it to splits.json.xz. The connector reads that file. +""" + +from collections import Counter +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from enum import StrEnum +import lzma +from pathlib import Path +import re +from typing import Annotated, Self + +import numpy as np +from pydantic import AllowInfNan, BaseModel, ConfigDict, Field, Json, StrictFloat, ValidationError, model_validator + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import OrdinalAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec +from timenet_connectors.download import ensure_archive +from timenet_connectors.sources.readers import CsvDataReader, DataRow + + +M4_CAPTION_URL = "https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj/download/M4TimeSeriesCaptionDatasetV02.zip" +M4_CAPTION_SHA256 = "4acb748d94d7287fe58dbb5c3cf6314ea259010b7720f42506739269dec54edf" +RELEASE_FOLDER = "M4TimeSeriesCaptionDataset" +"""The folder in the archive that holds the CSV files.""" +SPLITS_PATH = Path(__file__).with_name("splits.json.xz") +"""The stratified split of the release, as generate_splits.py wrote it.""" + +CAPTION_PROMPT = "Please generate a detailed caption for this time-series, describing it as accurately as possible." +"""The prompt of OpenTSLM's loader, the same for every series: the post-prompt of ``M4QADataset``.""" + +CAPTION_ERROR_PREFIX = "Error generating caption: " +"""The start of an error message of the caption API, which the release has instead of a caption for +706 series. The dataset leaves out these series.""" + +# The release states no unit, and the M4 series come from many domains. +M4_SERIES = TimeSeriesSpec(spec_type="m4_series", name="M4 series", unit_value=None, dtype="float32") + +_SERIES_ID = re.compile(r"series-(?P(?P[A-Z])[1-9][0-9]*)") +"""A release id: the M4 id, such as M1, after a ``series-`` prefix that OpenTSLM added.""" + +_SeriesValue = Annotated[StrictFloat, AllowInfNan(False)] +"""A finite JSON number. Strict, so that a Boolean or a string in the list is an error, not a number.""" + + +class M4Frequency(StrEnum): + """An M4 frequency. The members are in the order that OpenTSLM's loader reads them.""" + + DAILY = "Daily" + HOURLY = "Hourly" + MONTHLY = "Monthly" + QUARTERLY = "Quarterly" + WEEKLY = "Weekly" + YEARLY = "Yearly" + + @property + def id_letter(self) -> str: + """The letter that starts the M4 id of each series of this frequency, such as M in M1.""" + return _M4_ID_LETTERS[self] + + def parse_m4_id(self, series_id: str) -> str: + """Get the M4 id from a release id, and check that it belongs to this frequency. + + Returns: + The M4 id, such as ``M1``. + + Raises: + TimeNetConnectorError: If the id is not an M4 series id of this frequency. + """ + match = _SERIES_ID.fullmatch(series_id) + if match is None or match["letter"] != self.id_letter: + raise TimeNetConnectorError(f"{series_id!r} is not a {self} M4 series id") + return match["m4_id"] + + +_M4_ID_LETTERS = { + M4Frequency.DAILY: "D", + M4Frequency.HOURLY: "H", + M4Frequency.MONTHLY: "M", + M4Frequency.QUARTERLY: "Q", + M4Frequency.WEEKLY: "W", + M4Frequency.YEARLY: "Y", +} + + +class _SeriesRow(DataRow): + """A row of a series file: the release id and the values as a JSON list.""" + + id: str + series: Json[Annotated[list[_SeriesValue], Field(min_length=1)]] + + +class _CaptionRow(DataRow): + """A row of a caption file: the release id and the caption.""" + + id: str + caption: str + + +@dataclass(frozen=True) +class M4CaptionedSeries: + """One M4 series of the release, with its frequency and caption, before the split is known.""" + + m4_id: str + frequency: M4Frequency + values: np.ndarray + caption: str + + +@dataclass(frozen=True) +class M4CaptionSeries: + """One M4 series of the release, with its frequency, caption and split.""" + + m4_id: str + frequency: M4Frequency + values: np.ndarray + caption: str + split: Split + + def create_record_and_caption_task(self) -> tuple[Record, AnswerTask]: + """Create the Record of the series and the task that asks for its caption. + + The Record holds the values, the M4 id and the frequency. The task names the Record as its + input. Add the Record to the dataset before the task. + + Returns: + The Record and the task, not yet added to a dataset. + """ + signal = Signal.from_values(self.values, spec=M4_SERIES, name="value", time_axis=OrdinalAxis()) + record = Record( + sources=[Source(name="M4", signals=[signal])], + annotations=[Annotation("frequency", value=self.frequency.value)], + metadata={"m4_id": self.m4_id}, + ) + task = AnswerTask(inputs=[record], prompt=CAPTION_PROMPT, targets=self.caption, split=self.split) + return record, task + + +class M4CaptionSplits(BaseModel): + """The validation and test series of the split, by M4 id. + + The release archive stores no split. generate_splits.py made an 80/10/10 split, stratified by + frequency, and wrote it to splits.json.xz. Every series that the file does not name is in the train + split. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_sha256: str + """SHA-256 digest of the release archive that the split was made from.""" + validation: tuple[str, ...] + test: tuple[str, ...] + + @model_validator(mode="after") + def _each_series_in_one_split(self) -> Self: + repeated = sorted(m4_id for m4_id, count in Counter(self.m4_ids).items() if count > 1) + if repeated: + raise ValueError(f"series {repeated} are named more than once") + return self + + @property + def m4_ids(self) -> tuple[str, ...]: + """The validation and test series together.""" + return (*self.validation, *self.test) + + @classmethod + def read_from_file(cls, path: Path) -> Self: + """Read the split from an xz-compressed JSON file. + + Returns: + The split. + + Raises: + TimeNetConnectorError: If the file does not hold a valid split. + """ + try: + return cls.model_validate_json(lzma.decompress(path.read_bytes())) + except (lzma.LZMAError, ValidationError) as exc: + raise TimeNetConnectorError(f"{path.name} does not hold a valid split: {exc}") from exc + + def assign_split_to_each_series(self, m4_ids: Sequence[str]) -> list[Split]: + """Assign a split to each series of the pinned release. + + Returns: + The split of each series, in the order of ``m4_ids``. + + Raises: + TimeNetConnectorError: If the split was made from another release, or names a series that + the release does not have. + """ + if self.source_sha256 != M4_CAPTION_SHA256: + raise TimeNetConnectorError( + f"the split was made from the release {self.source_sha256}, not from the pinned {M4_CAPTION_SHA256}" + ) + unknown = sorted(set(self.m4_ids) - set(m4_ids)) + if unknown: + raise TimeNetConnectorError(f"the split names series {unknown} that the release does not have") + split_of = dict.fromkeys(self.validation, Split.VALIDATION) | dict.fromkeys(self.test, Split.TEST) + return [split_of.get(m4_id, Split.TRAIN) for m4_id in m4_ids] + + +@dataclass(frozen=True) +class M4CaptionReader: + """Read the series, the captions and the split of the extracted release.""" + + directory: Path + splits_path: Path + + def read_series(self) -> list[M4CaptionSeries]: + """Read every series of the release with its caption and split, in the order of OpenTSLM's loader. + + The loader reads the files of each frequency, and skips ``m4_series.csv`` and ``m4_captions.csv``. + A malformed file raises a :class:`~timenet.errors.TimeNetConnectorError` that names the file and + its row. The split comes from the split file, which the method checks against every M4 id. + + Returns: + Each series with its caption and split. + """ + captioned = self.read_captioned_series() + split_file = M4CaptionSplits.read_from_file(self.splits_path) + splits = split_file.assign_split_to_each_series([series.m4_id for series in captioned]) + return [ + M4CaptionSeries( + m4_id=series.m4_id, + frequency=series.frequency, + values=series.values, + caption=series.caption, + split=split, + ) + for series, split in zip(captioned, splits, strict=True) + ] + + def read_captioned_series(self) -> list[M4CaptionedSeries]: + """Read every series of the release with its caption, in the order of :meth:`read_series`. + + The method leaves out each series whose caption is an error message of the caption API. + + Returns: + Each series with its caption, without the split. + """ + captioned: list[M4CaptionedSeries] = [] + for frequency in M4Frequency: + captioned.extend(self._read_frequency(frequency)) + return captioned + + def _read_frequency(self, frequency: M4Frequency) -> Iterator[M4CaptionedSeries]: + """Read the series of one frequency, paired with the caption in the same row of its caption file. + + Yields: + Each series with its caption, except a series whose caption is an error message. + + Raises: + TimeNetConnectorError: If the files do not pair one to one, or a series row is malformed. + """ + series_path = self.directory / f"m4_series_{frequency}.csv" + captions_path = self.directory / f"m4_captions_{frequency}.csv" + series_rows = list(CsvDataReader(series_path, _SeriesRow, expected_columns=("id", "series")).iter_rows()) + caption_rows = list(CsvDataReader(captions_path, _CaptionRow, expected_columns=("id", "caption")).iter_rows()) + if len(series_rows) != len(caption_rows): + raise TimeNetConnectorError( + f"{series_path.name} has {len(series_rows)} series rows but {len(caption_rows)} caption rows" + ) + for number, (series_row, caption_row) in enumerate(zip(series_rows, caption_rows, strict=True), start=1): + try: + m4_id = frequency.parse_m4_id(series_row.id) + if series_row.id != caption_row.id: + raise TimeNetConnectorError( + f"series id {series_row.id!r} does not match caption id {caption_row.id!r}" + ) + if caption_row.caption.startswith(CAPTION_ERROR_PREFIX): + continue + yield M4CaptionedSeries( + m4_id=m4_id, + frequency=frequency, + values=np.array(series_row.series, dtype=np.float32), + caption=caption_row.caption, + ) + except TimeNetConnectorError as exc: + raise TimeNetConnectorError(f"{series_path.name} data row {number}: {exc}") from exc + + +class M4CaptionConnector(BaseConnector[list[M4CaptionReader]]): + """Connector for the M4-Caption release of OpenTSLM.""" + + async def download_async(self, cache_dir: Path) -> list[M4CaptionReader]: # ruff: ignore[no-self-use] + """Download and extract the release archive once. + + Args: + cache_dir: The directory that holds the extracted files. + + Returns: + A reader of the extracted release and of the split file of the connector. + """ + root = await ensure_archive(M4_CAPTION_URL, cache_dir, filename="m4_caption.zip", sha256=M4_CAPTION_SHA256) + return [M4CaptionReader(directory=root / RELEASE_FOLDER, splits_path=SPLITS_PATH)] + + def convert(self, releases: list[M4CaptionReader]) -> TimeFDataset: + """Create one Record and one AnswerTask for each series. + + Args: + releases: The reader from :meth:`download_async`. + + Returns: + The populated dataset. + """ + (release,) = releases + dataset = TimeFDataset(metadata=self.metadata()) + for series in release.read_series(): + record, task = series.create_record_and_caption_task() + dataset.add_record(record) + dataset.add_task(task) + return dataset + + +CONNECTOR = M4CaptionConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/dataset.yaml new file mode 100644 index 000000000..1f60e0bfb --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/dataset.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: opentslm/m4-caption +dataset_version: 1.0.0 +name: M4-Caption +description: >- + 99,294 univariate series from the M4 forecasting competition (79,436 train, 9,929 validation, + 9,929 test). Each series has one caption in English. GPT-4o wrote each caption from a plot of the + series, with the prompt "Generate a detailed caption for the following time-series data:". + OpenTSLM publishes the series and the captions as one zip archive of CSV files on Polybox. The + series are the train part of M4, the history that the competitors received, without the forecast + horizon. There are 4,227 daily, 414 hourly, 48,000 monthly, 24,000 quarterly, 359 weekly and + 23,000 yearly series, with 13 to 9,919 values each. The OpenTSLM paper gives 64 to 512 values, but + the Polybox release has the full length. The Polybox release has no timestamps and states no unit. + The M4 series come from the micro, industry, macro, finance, demographic and other domains. Each + record states the M4 id, such as M1, and the M4 frequency. For 706 series, the Polybox release has + an error message of the caption API instead of a caption, such as "Error generating caption: + Connection error.". The dataset leaves out these 253 daily, 57 monthly and 396 quarterly series. Each task asks for the caption with the + prompt of OpenTSLM's loader: "Please generate a detailed caption for this time-series, describing + it as accurately as possible.". The Polybox release has no split column, so the dataset gives + each series an 80/10/10 split, stratified by M4 frequency. M4 has 90 pairs of series whose + histories are equal value for value, and each pair stays in one split. This split is not the + random split of OpenTSLM's loader. + + The Polybox release also holds m4_series.csv, a copy of the weekly series, and m4_captions.csv, + with 40 other captions of weekly series. OpenTSLM does not use these two files, and the dataset + does not include them. + + The M4 organizers at the University of Nicosia state no license for the M4 data. OpenTSLM + releases the captions under MIT. +license: NONE +source_url: https://polybox.ethz.ch/index.php/s/MT3y9WdEebT8wfj +citation: >- + Langer, P., et al. (2025). OpenTSLM: Time-Series Language Models for Reasoning over Multivariate + Medical Text- and Time-Series Data. https://doi.org/10.48550/arXiv.2510.02410. M4 series: + Makridakis, S., Spiliotis, E., & Assimakopoulos, V. (2020). The M4 Competition: 100,000 time series + and 61 forecasting methods. International Journal of Forecasting, 36(1), 54-74. Makridakis, S., + Spiliotis, E., & Assimakopoulos, V. (2018). The M4 Competition: Results, findings, conclusion and + way forward. International Journal of Forecasting, 34(4), 802-808. +domains: + - economics + - finance + - general +tags: + - m4 + - forecasting + - captioning + - generated-supervision + - demographics diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py new file mode 100644 index 000000000..7f57381b5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/generate_splits.py @@ -0,0 +1,103 @@ +"""Write splits.json.xz, the frozen train, validation and test split of m4-caption. + +The release archive stores no split. This script makes an 80/10/10 split with StratifiedSplitter, +stratified by M4 frequency, so that validation and test each hold a tenth of every frequency. + +M4 has 90 pairs of series whose train parts are equal value for value. Series with the same values +must stay in one split, so the splitter groups the series by their values. The script reads the +series through the connector, so it leaves out the series whose caption is an error message. + +The result depends on the version of scikit-learn. So the script ran once and wrote the validation +and test series to splits.json.xz, and the connector reads that file. The script is kept here for +reference. + +Run it once from the root of the repository. The argument is the path to the release archive, +M4TimeSeriesCaptionDatasetV02.zip:: + + uv run python -m timenet_connectors.datasets.opentslm.m4_caption.generate_splits +""" + +import hashlib +import json +import lzma +from pathlib import Path +import sys +import tempfile +import zipfile + +from timenet.types import Split +from timenet_connectors.datasets.opentslm.m4_caption.connector import ( + M4_CAPTION_SHA256, + RELEASE_FOLDER, + SPLITS_PATH, + M4CaptionedSeries, + M4CaptionReader, + M4Frequency, +) +from timenet_connectors.splitting import StratifiedSplitter + + +SEED = 42 + + +def frequency(series: M4CaptionedSeries) -> M4Frequency: + """Return the frequency of a series, which the split stratifies by. + + Returns: + The frequency. + """ + return series.frequency + + +def values_digest(series: M4CaptionedSeries) -> str: + """Return the SHA-256 digest of the values of a series. Series with the same values form one group. + + Returns: + The digest. + """ + return hashlib.sha256(series.values.tobytes()).hexdigest() + + +def m4_id_order(m4_id: str) -> tuple[str, int]: + """Sort M4 ids by frequency letter, then by number: M2 before M10. + + Returns: + The sort key. + """ + return m4_id[0], int(m4_id[1:]) + + +def main() -> None: + """Split the release archive that the first argument names, and write splits.json.xz. + + Raises: + SystemExit: If the archive is not the pinned release. + """ + archive_path = Path(sys.argv[1]) + source_sha256 = hashlib.sha256(archive_path.read_bytes()).hexdigest() + if source_sha256 != M4_CAPTION_SHA256: + raise SystemExit(f"{archive_path} has the SHA-256 {source_sha256}, not the pinned {M4_CAPTION_SHA256}") + with tempfile.TemporaryDirectory() as directory, zipfile.ZipFile(archive_path) as archive: + archive.extractall(directory) + reader = M4CaptionReader(directory=Path(directory) / RELEASE_FOLDER, splits_path=SPLITS_PATH) + captioned = reader.read_captioned_series() + + splitter = StratifiedSplitter[M4CaptionedSeries]( + train_folds=8, + validation_folds=1, + test_folds=1, + stratify_by=frequency, + group_by=values_digest, + seed=SEED, + ) + splits = splitter.split(captioned) + content = { + "source_sha256": source_sha256, + "validation": sorted((series.m4_id for series in splits[Split.VALIDATION]), key=m4_id_order), + "test": sorted((series.m4_id for series in splits[Split.TEST]), key=m4_id_order), + } + SPLITS_PATH.write_bytes(lzma.compress((json.dumps(content, indent=2) + "\n").encode())) + + +if __name__ == "__main__": + main() diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/splits.json.xz b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/splits.json.xz new file mode 100644 index 000000000..b49688eb9 Binary files /dev/null and b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/splits.json.xz differ diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py new file mode 100644 index 000000000..3e6c9cf1e --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/m4_caption/tests/test_connector.py @@ -0,0 +1,198 @@ +from collections import Counter +import csv +import json +import lzma +from pathlib import Path + +import numpy as np +import pytest + +from timenet.dataset import TimeFDataset +from timenet.dataset.axis import OrdinalAxis +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing import assert_datasets_equal +from timenet.types import AnswerTask, Split +from timenet.types.text import read_value +from timenet_connectors.datasets.opentslm.m4_caption.connector import ( + M4_CAPTION_SHA256, + SPLITS_PATH, + M4CaptionConnector, + M4CaptionReader, + M4CaptionSplits, +) + + +_SERIES_IDS = { + "Daily": ["D1", "D2"], + "Hourly": ["H1"], + "Monthly": ["M1", "M2", "M3"], + "Quarterly": ["Q1", "Q2"], + "Weekly": ["W1"], + "Yearly": ["Y1"], +} +"""Ten series in the order that OpenTSLM's loader reads the frequencies.""" + +_SPLITS = {"source_sha256": M4_CAPTION_SHA256, "validation": ["W1"], "test": ["M3"]} +"""A split file for the ten series. Every series that it does not name is in the train split.""" +_EXPECTED_SPLIT = {"M3": Split.TEST, "W1": Split.VALIDATION} + +_FLOAT32_VALUES = [1089.199951171875, 1078.9100341796875, 8000.0, 195.92799377441406] +"""Values as the release writes them: M4 values rounded to float32, printed with every digit.""" + + +_LONGEST_SERIES = 9919 +"""The number of values of the longest series in the release, a daily one. Its CSV cell has more +characters than the default cell limit of the csv module.""" + + +def _values(m4_id: str) -> list[float]: + # A distinct length and offset for each series, so a mix-up of two series changes the values. The sum + # is in float32, so each value stays a float32 value, like in the release. + shift = np.float32(sum(map(ord, m4_id))) + values = np.array(_FLOAT32_VALUES, dtype=np.float32) + shift + if m4_id == "D1": + return np.resize(values, _LONGEST_SERIES).tolist() + return values.tolist()[: 2 + len(m4_id)] + + +def _caption(m4_id: str) -> str: + return f"Caption of {m4_id}.\n\nA second paragraph." + + +def _write_csv(path: Path, columns: tuple[str, str], rows: list[tuple[str, str]]) -> None: + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.writer(stream) + writer.writerow(columns) + writer.writerows(rows) + + +def _expected_split(m4_id: str) -> Split: + return _EXPECTED_SPLIT.get(m4_id, Split.TRAIN) + + +def _write_release( + directory: Path, + *, + series_rows: dict[str, list[tuple[str, str]]] | None = None, + captions: dict[str, str] | None = None, + splits: dict[str, str | list[str]] = _SPLITS, +) -> M4CaptionReader: + # Like the archive, the files sit in a folder of their own, beside the files that the loader skips. + root = directory / "M4TimeSeriesCaptionDataset" + root.mkdir() + for frequency, m4_ids in _SERIES_IDS.items(): + default_rows = [(f"series-{m4_id}", json.dumps(_values(m4_id))) for m4_id in m4_ids] + caption_rows = [(f"series-{m4_id}", (captions or {}).get(m4_id, _caption(m4_id))) for m4_id in m4_ids] + _write_csv( + root / f"m4_series_{frequency}.csv", ("id", "series"), (series_rows or {}).get(frequency, default_rows) + ) + _write_csv(root / f"m4_captions_{frequency}.csv", ("id", "caption"), caption_rows) + _write_csv(root / "m4_captions.csv", ("id", "caption"), [("series-W1", "A caption that OpenTSLM never used.")]) + splits_path = directory / "splits.json.xz" + splits_path.write_bytes(lzma.compress(json.dumps(splits).encode())) + return M4CaptionReader(directory=root, splits_path=splits_path) + + +def _convert(directory: Path, **changes) -> TimeFDataset: + return M4CaptionConnector().convert([_write_release(directory, **changes)]) + + +def test_each_series_becomes_a_record_and_a_caption_task(tmp_path): + tasks = _convert(tmp_path).tasks_of(AnswerTask) + expected = [(frequency, m4_id) for frequency, m4_ids in _SERIES_IDS.items() for m4_id in m4_ids] + assert len(tasks) == len(expected) + for task, (frequency, m4_id) in zip(tasks, expected, strict=True): + (record,) = task.inputs + (signal,) = record.signals + # The release values are float32 values, so float32 keeps every digit. + assert signal.to_numpy().dtype == np.float32 + np.testing.assert_array_equal(signal.to_numpy(), np.array(_values(m4_id), dtype=np.float32)) + # The release has no timestamps, and a month or a year has no fixed length. + assert signal.time_axis == OrdinalAxis() + assert signal.spec.unit_value is None + assert record.metadata == {"m4_id": m4_id} + (annotation,) = record.annotations + assert (annotation.key, read_value(annotation.value)) == ("frequency", frequency) + assert task.targets is not None + assert [read_value(target) for target in task.targets] == [_caption(m4_id)] + # The prompt of OpenTSLM's loader, the same for every series. + assert read_value(task.prompt) == ( + "Please generate a detailed caption for this time-series, describing it as accurately as possible." + ) + assert task.split == _expected_split(m4_id) + + +def test_a_series_whose_caption_is_an_error_message_is_left_out(tmp_path): + # Two of the three error messages of the caption API that the release has instead of a caption. + captions = { + "D2": "Error generating caption: Connection error.", + "Q1": "Error generating caption: Error code: 404", + } + tasks = _convert(tmp_path, captions=captions).tasks_of(AnswerTask) + assert [task.inputs[0].metadata["m4_id"] for task in tasks] == ["D1", "H1", "M1", "M2", "M3", "Q2", "W1", "Y1"] + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ({"source_sha256": "0" * 64}, "the split was made from the release 0+, not from the pinned"), + ({"test": ["M3", "W1"]}, r"series \['W1'\] are named more than once"), + ({"test": ["M3", "M9"]}, r"names series \['M9'\] that the release does not have"), + ], + ids=["another release", "a series in two splits", "a series that the release does not have"], +) +def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, changes, message): + with pytest.raises(TimeNetConnectorError, match=message): + _convert(tmp_path, splits=_SPLITS | changes) + + +_DATASET_SERIES_COUNTS = {"D": 3974, "H": 414, "M": 47943, "Q": 23604, "W": 359, "Y": 23000} +"""The number of series of each frequency in the dataset, by the letter that starts their M4 ids. The +dataset leaves out 253 daily, 57 monthly and 396 quarterly series, whose caption is an error message.""" + + +def test_the_frozen_split_is_of_the_pinned_release_and_has_a_tenth_of_each_frequency_in_validation_and_test(): + splits = M4CaptionSplits.read_from_file(SPLITS_PATH) + assert splits.source_sha256 == M4_CAPTION_SHA256 + for m4_ids in (splits.validation, splits.test): + counts = Counter(m4_id[0] for m4_id in m4_ids) + for letter, dataset_count in _DATASET_SERIES_COUNTS.items(): + # A tenth of the series of each frequency, rounded down or up. + assert dataset_count // 10 <= counts[letter] <= -(-dataset_count // 10) + + +@pytest.mark.parametrize( + ("series_rows", "message"), + [ + ({"Daily": [("series-D1", "[1.0, null]"), ("series-D2", "[1.0]")]}, "line 2: .*Input should be a valid number"), + ({"Daily": [("series-D1", "[1.0, true]"), ("series-D2", "[1.0]")]}, "line 2: .*Input should be a valid number"), + ({"Daily": [("series-D1", "[]"), ("series-D2", "[1.0]")]}, "line 2: .*List should have at least 1 item"), + ({"Daily": [("series-D2", "[1.0]"), ("series-D1", "[1.0]")]}, "series id 'series-D2' does not match"), + ({"Daily": [("series-M9", "[1.0]"), ("series-D2", "[1.0]")]}, "is not a Daily M4 series id"), + ({"Daily": [("series-D1", "[1.0]")]}, "has 1 series rows but 2 caption rows"), + ], + ids=[ + "a null value", + "a Boolean value", + "an empty series", + "series and captions out of step", + "an id of another frequency", + "a series row too few", + ], +) +def test_a_malformed_series_file_raises_with_its_location(tmp_path, series_rows, message): + # The CSV reader rejects a cell that does not hold a list of numbers itself, and names the line of the file. + with pytest.raises(TimeNetConnectorError, match=rf"(?s)m4_series_Daily\.csv.*{message}"): + _convert(tmp_path, series_rows=series_rows) + + +def test_the_dataset_round_trips_through_storage(tmp_path): + source = tmp_path / "source" + source.mkdir() + dataset = _convert(source) + version_dir = dataset.write(path=tmp_path / "registry") + with TimeFReader(open_local_version(version_dir)) as reader: + reader.verify() + assert_datasets_equal(dataset, reader.read())