diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/__init__.py new file mode 100644 index 000000000..bdeae32ca --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/__init__.py @@ -0,0 +1 @@ +"""Connectors for datasets published by the ``AutonLab`` org.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/__init__.py new file mode 100644 index 000000000..ac7115411 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/__init__.py @@ -0,0 +1,6 @@ +"""The TimeSeriesExam connector package: its connector implementation and dataset card.""" + +from timenet_connectors.datasets.autonlab.timeseriesexam1.connector import ( + CONNECTOR as CONNECTOR, + TimeSeriesExam1Connector as TimeSeriesExam1Connector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py new file mode 100644 index 000000000..7ce5534e4 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py @@ -0,0 +1,104 @@ +"""The TimeSeriesExam connector: multiple-choice questions about time series, from the HuggingFace Hub. + +The source repository is ``AutonLab/TimeSeriesExam1``. The connector pins one commit and reads only the published +test split. A row has a question, its options, the answer, and one series (``ts``) or two (``ts1`` and ``ts2``). +Each series becomes a Record. + +Each row becomes one :class:`~timenet.types.ClassificationTask`. Its labels are the option texts, and its target is +the published answer text. The same options are in a different order in different rows, so the order has no meaning +and the label set is not ordered. + +The task metadata holds the other published columns. It is provenance and never model input. +The prompt is the question alone. The options are the labels of the task, and ``format_hint`` names option +letters that the task does not use. +""" + +from typing import Self + +from pydantic import model_validator + +from timenet.dataset import OrdinalAxis, Record, Signal, Source, TimeFDataset +from timenet.types import ClassificationTask, LabelSet, Split, TimeSeriesSpec, ureg +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.sources.readers import DataRow, ParquetDataReader + + +_SPEC = TimeSeriesSpec( + spec_type="tse1_series", name="TimeSeriesExam series", unit_value=ureg.dimensionless, dtype="float64" +) +_AXIS = OrdinalAxis() +_SERIES_COLUMNS = ("ts", "ts1", "ts2") +_NOT_METADATA = {"question", "options", "answer", *_SERIES_COLUMNS} +"""The published columns that a task does not keep in its metadata. It keeps the others, as published.""" + + +class _SourceRow(DataRow): + """A published row. A series column is ``None`` where the row has no such series.""" + + id: int + question: str + options: list[str] + answer: str + question_type: str + tid: int + difficulty: str + format_hint: str + relevant_concepts: list[str] + question_hint: str + category: str + subcategory: str + ts: list[float] | None + ts1: list[float] | None + ts2: list[float] | None + + @property + def series(self) -> dict[str, list[float]]: + """The series that the row has, by column name.""" + return {column: values for column in _SERIES_COLUMNS if (values := getattr(self, column)) is not None} + + @model_validator(mode="after") + def _has_ts_or_ts1_and_ts2(self) -> Self: + if list(self.series) not in (["ts"], ["ts1", "ts2"]): + raise ValueError(f"row id {self.id}: expected ts, or ts1 and ts2, found {list(self.series)}") + return self + + +def _record(row_id: int, role: str, values: list[float]) -> Record: + signal = Signal.from_values(values, spec=_SPEC, name="series", time_axis=_AXIS) + return Record(sources=[Source("series", signals=[signal])], metadata={"row_id": row_id, "role": role}) + + +class TimeSeriesExam1Connector( + BaseHuggingFaceConnector, + hf_repo="AutonLab/TimeSeriesExam1", + source_revision="628f686757bb484d9b608b6855fb41f6a302ecda", + source_files={"test": "data/test-00000-of-00001.parquet"}, +): + """Build TimeSeriesExam from the pinned test parquet file of its Hugging Face release.""" + + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Build one Record per series and one classification task per question. + + Args: + raw_refs: The downloaded test file. + + Returns: + The populated :class:`~timenet.dataset.TimeFDataset`. + """ + dataset = TimeFDataset(metadata=self.metadata()) + for row in ParquetDataReader(raw_refs[0].path, _SourceRow).iter_rows(): + inputs = [dataset.add_record(_record(row.id, role, values)) for role, values in row.series.items()] + dataset.add_task( + ClassificationTask( + inputs=inputs, + prompt=row.question, + label_set=LabelSet(row.options), + targets=row.answer, + split=Split.TEST, + metadata=row.model_dump(exclude=_NOT_METADATA), + ) + ) + return dataset + + +CONNECTOR = TimeSeriesExam1Connector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/dataset.yaml new file mode 100644 index 000000000..af3cd804f --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/dataset.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: autonlab/timeseriesexam1 +dataset_version: 1.0.0 +name: TimeSeriesExam +description: >- + TimeSeriesExam is a multiple-choice exam that tests how well a model + understands basic time series concepts. It has 746 questions in five + categories: pattern recognition, noise understanding, similarity analysis, + anomaly detection, and causality analysis. A question is about one series, + or about two series that it compares. The 996 series are synthetic. The + authors build them from baseline components such as linear and non-linear + trends and cyclic patterns. Most series have 1,024 values. Each question has + two to four answer options and an easy, medium, or hard difficulty. The + authors publish only a test split. +license: MIT +license_url: https://github.com/moment-timeseries-foundation-model/TimeSeriesExam/blob/main/LICENSE +source_url: https://huggingface.co/datasets/AutonLab/TimeSeriesExam1 +citation: >- + Cai, Choudhry, Goswami, Dubrawski. TimeSeriesExam: A Time Series Understanding Exam. + NeurIPS 2024 Workshop on Time Series in the Age of Large Models. + https://arxiv.org/abs/2410.14752 +domains: + - general +tags: + - multiple-choice + - question-answering + - synthetic + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/requirements.txt new file mode 100644 index 000000000..fe57efebd --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/requirements.txt @@ -0,0 +1,2 @@ +# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class. +huggingface_hub>=1.1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/fixtures/test.parquet b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/fixtures/test.parquet new file mode 100644 index 000000000..3249cb703 Binary files /dev/null and b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/fixtures/test.parquet differ diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py new file mode 100644 index 000000000..285334c29 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py @@ -0,0 +1,126 @@ +from pathlib import Path +import re + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from timenet.dataset import TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.types import ClassificationTask, Split, read_value +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.datasets.autonlab.timeseriesexam1 import TimeSeriesExam1Connector + + +# Eight real rows of the published test split, in this order. The fixture keeps the published schema. +_FIXTURE = Path(__file__).parent / "fixtures" / "test.parquet" +_SERIES_COLUMNS = ("ts", "ts1", "ts2") + + +def _source_rows() -> list[dict]: + return pq.read_table(_FIXTURE).to_pylist() + + +def _convert(tmp_path: Path, rows: list[dict] | None = None) -> TimeFDataset: + path = _FIXTURE + if rows is not None: + path = tmp_path / "rows.parquet" + pq.write_table(pa.Table.from_pylist(rows, schema=pq.read_schema(_FIXTURE)), path) + return TimeSeriesExam1Connector().convert([HubFileRef(path, "test")]) + + +def _task_for(dataset: TimeFDataset, row_id: int) -> ClassificationTask: + return next(task for task in dataset.tasks_of(ClassificationTask) if task.metadata["id"] == row_id) + + +def test_metadata(): + metadata = TimeSeriesExam1Connector().metadata() + assert metadata.dataset_id == "autonlab/timeseriesexam1" + assert str(metadata.license) == "MIT" + assert str(metadata.dataset_version) == "1.0.0" + assert metadata.license_url + assert metadata.citation + + +def test_records_and_tasks_match_rows(tmp_path): + dataset = _convert(tmp_path) + rows = _source_rows() + two_series = [row for row in rows if row["ts1"] is not None] + assert len(dataset.tasks) == len(dataset.tasks_of(ClassificationTask)) == len(rows) + assert len(dataset.records) == len(rows) + len(two_series) + roles = [(record.metadata["row_id"], record.metadata["role"]) for record in dataset.records] + expected = [(row["id"], role) for row in rows for role in (("ts1", "ts2") if row["ts1"] is not None else ("ts",))] + assert roles == expected + + +def test_series_values_equal_source(tmp_path): + dataset = _convert(tmp_path) + by_key: dict[object, list[float]] = { + (row["id"], column): row[column] for row in _source_rows() for column in _SERIES_COLUMNS + } + for record in dataset.records: + (signal,) = record.signals + expected = by_key[record.metadata["row_id"], record.metadata["role"]] + assert signal.n_values == len(expected) + assert signal.to_numpy().tolist() == expected + + +def test_inputs_follow_ts1_then_ts2(tmp_path): + dataset = _convert(tmp_path) + assert [record.metadata["role"] for record in _task_for(dataset, 1).inputs] == ["ts1", "ts2"] + assert [record.metadata["role"] for record in _task_for(dataset, 18).inputs] == ["ts"] + + +def test_records_are_not_shared_between_rows(tmp_path): + dataset = _convert(tmp_path) + ids = [id(record) for task in dataset.tasks for record in task.inputs] + assert len(set(ids)) == len(ids) + + +def test_task_fields(tmp_path): + dataset = _convert(tmp_path) + rows = {row["id"]: row for row in _source_rows()} + for task in dataset.tasks_of(ClassificationTask): + row = rows[task.metadata["id"]] + assert task.split == Split.TEST + assert task.targets == (row["answer"],) + assert task.label_set.names == tuple(row["options"]) + assert not task.label_set.ordered + assert task.label_set.multi_label is False + assert read_value(task.prompt) == row["question"] + + +def test_metadata_is_as_published(tmp_path): + dataset = _convert(tmp_path) + keys = ["category", "subcategory", "question_type", "difficulty", "relevant_concepts", "question_hint"] + keys += ["format_hint", "tid"] + for row in _source_rows(): + metadata = _task_for(dataset, row["id"]).metadata + assert metadata == {"id": row["id"], **{key: row[key] for key in keys}} + + +def test_options_and_answers_survive_a_write(tmp_path): + with TimeFReader(open_local_version(_convert(tmp_path).write(path=tmp_path / "out"))) as reader: + stored = reader.read() + rows = {row["id"]: row for row in _source_rows()} + for task in stored.tasks_of(ClassificationTask): + row = rows[task.metadata["id"]] + assert set(task.label_set.names) == set(row["options"]) + assert [read_value(target) for target in task.targets or ()] == [row["answer"]] + + +@pytest.mark.parametrize( + ("row_id", "changes", "found"), + [ + (18, {"ts1": [1.0, 2.0], "ts2": [1.0, 2.0]}, ["ts", "ts1", "ts2"]), + (18, {"ts": None}, []), + (1, {"ts2": None}, ["ts1"]), + ], +) +def test_rejects_row_without_ts_or_ts1_and_ts2(tmp_path, row_id, changes, found): + rows = [{**row, **changes} if row["id"] == row_id else row for row in _source_rows()] + message = f"row id {row_id}: expected ts, or ts1 and ts2, found {found}" + with pytest.raises(TimeNetConnectorError, match=re.escape(message)): + _convert(tmp_path, rows)