Skip to content
Merged
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Connectors for datasets published by the ``AutonLab`` org."""
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
"""The TimeSeriesExam connector package: its connector implementation and dataset card."""

from timenet_connectors.datasets.autonlab.timeseriesexam1.connector import (
CONNECTOR as CONNECTOR,
TimeSeriesExam1Connector as TimeSeriesExam1Connector,
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
"""The TimeSeriesExam connector: multiple-choice questions about time series, from the HuggingFace Hub.

The source repository is ``AutonLab/TimeSeriesExam1``. The connector pins one commit and reads only the published
test split. A row has a question, its options, the answer, and one series (``ts``) or two (``ts1`` and ``ts2``).
Each series becomes a Record.

Each row becomes one :class:`~timenet.types.ClassificationTask`. Its labels are the option texts, and its target is
the published answer text. The same options are in a different order in different rows, so the order has no meaning
and the label set is not ordered.

The task metadata holds the other published columns. It is provenance and never model input.
The prompt is the question alone. The options are the labels of the task, and ``format_hint`` names option
letters that the task does not use.
"""

from typing import Self

from pydantic import model_validator

from timenet.dataset import OrdinalAxis, Record, Signal, Source, TimeFDataset
from timenet.types import ClassificationTask, LabelSet, Split, TimeSeriesSpec, ureg
from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef
from timenet_connectors.sources.readers import DataRow, ParquetDataReader


_SPEC = TimeSeriesSpec(
spec_type="tse1_series", name="TimeSeriesExam series", unit_value=ureg.dimensionless, dtype="float64"
)
_AXIS = OrdinalAxis()
_SERIES_COLUMNS = ("ts", "ts1", "ts2")
_NOT_METADATA = {"question", "options", "answer", *_SERIES_COLUMNS}
"""The published columns that a task does not keep in its metadata. It keeps the others, as published."""


class _SourceRow(DataRow):
"""A published row. A series column is ``None`` where the row has no such series."""

id: int
question: str
options: list[str]
answer: str
question_type: str
tid: int
difficulty: str
format_hint: str
relevant_concepts: list[str]
question_hint: str
category: str
subcategory: str
ts: list[float] | None
ts1: list[float] | None
ts2: list[float] | None

@property
def series(self) -> dict[str, list[float]]:
"""The series that the row has, by column name."""
return {column: values for column in _SERIES_COLUMNS if (values := getattr(self, column)) is not None}

@model_validator(mode="after")
def _has_ts_or_ts1_and_ts2(self) -> Self:
if list(self.series) not in (["ts"], ["ts1", "ts2"]):
raise ValueError(f"row id {self.id}: expected ts, or ts1 and ts2, found {list(self.series)}")
return self


def _record(row_id: int, role: str, values: list[float]) -> Record:
signal = Signal.from_values(values, spec=_SPEC, name="series", time_axis=_AXIS)
return Record(sources=[Source("series", signals=[signal])], metadata={"row_id": row_id, "role": role})


class TimeSeriesExam1Connector(
BaseHuggingFaceConnector,
hf_repo="AutonLab/TimeSeriesExam1",
source_revision="628f686757bb484d9b608b6855fb41f6a302ecda",
source_files={"test": "data/test-00000-of-00001.parquet"},
):
"""Build TimeSeriesExam from the pinned test parquet file of its Hugging Face release."""

def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset:
"""Build one Record per series and one classification task per question.

Args:
raw_refs: The downloaded test file.

Returns:
The populated :class:`~timenet.dataset.TimeFDataset`.
"""
dataset = TimeFDataset(metadata=self.metadata())
for row in ParquetDataReader(raw_refs[0].path, _SourceRow).iter_rows():
inputs = [dataset.add_record(_record(row.id, role, values)) for role, values in row.series.items()]
dataset.add_task(
ClassificationTask(
inputs=inputs,
prompt=row.question,
label_set=LabelSet(row.options),
targets=row.answer,
split=Split.TEST,
metadata=row.model_dump(exclude=_NOT_METADATA),
)
)
return dataset


CONNECTOR = TimeSeriesExam1Connector
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json
yaml_schema_version: 1
dataset_id: autonlab/timeseriesexam1
dataset_version: 1.0.0
name: TimeSeriesExam
description: >-
TimeSeriesExam is a multiple-choice exam that tests how well a model
understands basic time series concepts. It has 746 questions in five
categories: pattern recognition, noise understanding, similarity analysis,
anomaly detection, and causality analysis. A question is about one series,
or about two series that it compares. The 996 series are synthetic. The
authors build them from baseline components such as linear and non-linear
trends and cyclic patterns. Most series have 1,024 values. Each question has
two to four answer options and an easy, medium, or hard difficulty. The
authors publish only a test split.
license: MIT
license_url: https://github.com/moment-timeseries-foundation-model/TimeSeriesExam/blob/main/LICENSE
source_url: https://huggingface.co/datasets/AutonLab/TimeSeriesExam1
citation: >-
Cai, Choudhry, Goswami, Dubrawski. TimeSeriesExam: A Time Series Understanding Exam.
NeurIPS 2024 Workshop on Time Series in the Age of Large Models.
https://arxiv.org/abs/2410.14752
domains:
- general
tags:
- multiple-choice
- question-answering
- synthetic
- huggingface
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class.
huggingface_hub>=1.1
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
from pathlib import Path
import re

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from timenet.dataset import TimeFDataset
from timenet.errors import TimeNetConnectorError
from timenet.reader import TimeFReader
from timenet.registry import open_local_version
from timenet.types import ClassificationTask, Split, read_value
from timenet_connectors.bases.huggingface import HubFileRef
from timenet_connectors.datasets.autonlab.timeseriesexam1 import TimeSeriesExam1Connector


# Eight real rows of the published test split, in this order. The fixture keeps the published schema.
_FIXTURE = Path(__file__).parent / "fixtures" / "test.parquet"
_SERIES_COLUMNS = ("ts", "ts1", "ts2")


def _source_rows() -> list[dict]:
return pq.read_table(_FIXTURE).to_pylist()


def _convert(tmp_path: Path, rows: list[dict] | None = None) -> TimeFDataset:
path = _FIXTURE
if rows is not None:
path = tmp_path / "rows.parquet"
pq.write_table(pa.Table.from_pylist(rows, schema=pq.read_schema(_FIXTURE)), path)
return TimeSeriesExam1Connector().convert([HubFileRef(path, "test")])


def _task_for(dataset: TimeFDataset, row_id: int) -> ClassificationTask:
return next(task for task in dataset.tasks_of(ClassificationTask) if task.metadata["id"] == row_id)


def test_metadata():
metadata = TimeSeriesExam1Connector().metadata()
assert metadata.dataset_id == "autonlab/timeseriesexam1"
assert str(metadata.license) == "MIT"
assert str(metadata.dataset_version) == "1.0.0"
assert metadata.license_url
assert metadata.citation


def test_records_and_tasks_match_rows(tmp_path):
dataset = _convert(tmp_path)
rows = _source_rows()
two_series = [row for row in rows if row["ts1"] is not None]
assert len(dataset.tasks) == len(dataset.tasks_of(ClassificationTask)) == len(rows)
assert len(dataset.records) == len(rows) + len(two_series)
roles = [(record.metadata["row_id"], record.metadata["role"]) for record in dataset.records]
expected = [(row["id"], role) for row in rows for role in (("ts1", "ts2") if row["ts1"] is not None else ("ts",))]
assert roles == expected


def test_series_values_equal_source(tmp_path):
dataset = _convert(tmp_path)
by_key: dict[object, list[float]] = {
(row["id"], column): row[column] for row in _source_rows() for column in _SERIES_COLUMNS
}
for record in dataset.records:
(signal,) = record.signals
expected = by_key[record.metadata["row_id"], record.metadata["role"]]
assert signal.n_values == len(expected)
assert signal.to_numpy().tolist() == expected


def test_inputs_follow_ts1_then_ts2(tmp_path):
dataset = _convert(tmp_path)
assert [record.metadata["role"] for record in _task_for(dataset, 1).inputs] == ["ts1", "ts2"]
assert [record.metadata["role"] for record in _task_for(dataset, 18).inputs] == ["ts"]


def test_records_are_not_shared_between_rows(tmp_path):
dataset = _convert(tmp_path)
ids = [id(record) for task in dataset.tasks for record in task.inputs]
assert len(set(ids)) == len(ids)


def test_task_fields(tmp_path):
dataset = _convert(tmp_path)
rows = {row["id"]: row for row in _source_rows()}
for task in dataset.tasks_of(ClassificationTask):
row = rows[task.metadata["id"]]
assert task.split == Split.TEST
assert task.targets == (row["answer"],)
assert task.label_set.names == tuple(row["options"])
assert not task.label_set.ordered
assert task.label_set.multi_label is False
assert read_value(task.prompt) == row["question"]


def test_metadata_is_as_published(tmp_path):
dataset = _convert(tmp_path)
keys = ["category", "subcategory", "question_type", "difficulty", "relevant_concepts", "question_hint"]
keys += ["format_hint", "tid"]
for row in _source_rows():
metadata = _task_for(dataset, row["id"]).metadata
assert metadata == {"id": row["id"], **{key: row[key] for key in keys}}


def test_options_and_answers_survive_a_write(tmp_path):
with TimeFReader(open_local_version(_convert(tmp_path).write(path=tmp_path / "out"))) as reader:
stored = reader.read()
rows = {row["id"]: row for row in _source_rows()}
for task in stored.tasks_of(ClassificationTask):
row = rows[task.metadata["id"]]
assert set(task.label_set.names) == set(row["options"])
assert [read_value(target) for target in task.targets or ()] == [row["answer"]]


@pytest.mark.parametrize(
("row_id", "changes", "found"),
[
(18, {"ts1": [1.0, 2.0], "ts2": [1.0, 2.0]}, ["ts", "ts1", "ts2"]),
(18, {"ts": None}, []),
(1, {"ts2": None}, ["ts1"]),
],
)
def test_rejects_row_without_ts_or_ts1_and_ts2(tmp_path, row_id, changes, found):
rows = [{**row, **changes} if row["id"] == row_id else row for row in _source_rows()]
message = f"row id {row_id}: expected ts, or ts1 and ts2, found {found}"
with pytest.raises(TimeNetConnectorError, match=re.escape(message)):
_convert(tmp_path, rows)
Loading