From e18f1bb826cc9179b9a624b61f948795d6efc394 Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Thu, 29 Oct 2026 13:18:05 +0100 Subject: [PATCH 1/9] feat(connectors): add DatasetLoader and DatasetTask bases Connectors that read several files or define several kinds of task have each grown their own loader: TRACE has TraceDatasetLoader, and RATs40K repeats the same shape. The [TRACE review](https://review.aioniclabs.dev/OpenTSLM/TimeNet/pull/577#diff-comment-PRRC_kwDOTR9sUs8AAAABBJn8LA) asked for a generic class those loaders inherit, without duplicating what BaseConnector or TimeFDataset already do. This adds two small bases for that, in timenet_connectors.bases.loader. - DatasetTask is one task as the source defines it, such as TRACE's retrieval task. It declares its TIMENET_TYPE and its split, and builds the TimeNet task on demand with to_timenet_task(). - DatasetLoader[TTask] parses the downloaded files once. Each dataset fills in parse_from(), records() and tasks(), plus dataset_annotations() if it needs any. The base provides tasks_of(kind, split=), timenet_tasks() and populate(dataset). Where the boundaries are: - The connector keeps the outside world: its card, its download, its pinned source, and the engine's download/convert interface. Its convert() becomes parse_from(...) followed by populate(...). The download bases (Hugging Face, PhysioNet, EDF, Excel) are unchanged. - The loader keeps the downloaded files: parsing, Records and source tasks. It holds state, so it is its own object rather than more methods on a stateless connector. - TimeNet core is untouched. TimeFDataset still holds, selects and stores the TimeNet tasks. The loader's tasks_of() selects source task kinds, which a TimeNet type can't tell apart when two kinds become the same type. The base adds no runtime checks of its own. The type checker rejects a TASK_KINDS entry that isn't a DatasetTask, and populate() declares the kinds' TimeNet types, so the writer rejects a task of any other type. populate() streams the tasks, declaring each TimeNet type once. Shortcuts such as get_train_tasks(), pinned counts, key lookups and a default convert() are left out. Each joins the base once two connectors need it. TRACE and RATs40K move onto these bases in follow-up PRs. docs/connectors.md gains a "Loaders and source tasks" section. Two tests cover a toy loader with three task kinds, two of which become the same TimeNet type. Validation: make check, make lint-fix, make check-ci and make test (1606 passed, 1 skipped). make test-connectors passes for every connector this commit tracks. --- docs/connectors.md | 48 ++++++ .../src/timenet_connectors/bases/loader.py | 130 ++++++++++++++++ .../tests/test_dataset_loader.py | 142 ++++++++++++++++++ 3 files changed, 320 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/bases/loader.py create mode 100644 packages/timenet-connectors/tests/test_dataset_loader.py diff --git a/docs/connectors.md b/docs/connectors.md index 4ff6a6246..d54744b03 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -58,6 +58,54 @@ dataclass, or an S3 key. The connector is generic through PEP 695: --- +## Loaders and source tasks + +A connector that parses several files, or defines several kinds of task, can put that work in a +loader. `timenet_connectors.bases.loader` holds two generic bases. The connector keeps the outside +world: its card, its download, and its pinned source. The loader keeps the downloaded files. + +```python +from timenet_connectors.bases.loader import DatasetLoader, DatasetTask + + +class MyRetrievalTask(DatasetTask): + TIMENET_TYPE = TSCorrespondenceTask + # split (a property) and to_timenet_task() build the TimeNet task. + + +class MyDatasetLoader(DatasetLoader[MyRetrievalTask]): + """The dataset's tasks: one retrieval task per published row.""" + + TASK_KINDS = (MyRetrievalTask,) + # parse_from(refs), records() and tasks() are the dataset's own. + + +class MyConnector(BaseConnector[MyRawRef]): + def convert(self, raw_refs: list[MyRawRef]) -> TimeFDataset: + loader = MyDatasetLoader.parse_from(raw_refs) + dataset = TimeFDataset(metadata=self.metadata()) + return loader.populate(dataset) +``` + +| Member | Provided by | Contract | +| --- | --- | --- | +| `parse_from(refs)` | the dataset | Parse and check the downloaded files once. | +| `records()` | the dataset | Every Record, in a stable order. | +| `tasks()` | the dataset | Every source task. Each call starts again from the first. | +| `dataset_annotations()` | the dataset, optional | Annotations of the whole dataset, such as a label vocabulary. | +| `tasks_of(kind, split=)` | the base | The source tasks of one kind, optionally of one split. | +| `timenet_tasks()` | the base | Every task as TimeNet stores it, in the order of `tasks()`. | +| `populate(dataset)` | the base | Add the annotations and Records, and stream the tasks. | + +A source task is the dataset's own: TRACE has an event-type, a retrieval and a forecasting task. +`tasks_of` selects by that kind, which a TimeNet task type can't tell apart when two kinds become +the same type. To select stored tasks by TimeNet type or split, use `TimeFDataset.tasks_of` and +`TimeFDataset.get_split` instead. A loader declares its kinds in `TASK_KINDS` and lists them in its +docstring. `populate` declares their TimeNet types, so the writer rejects a task of a kind missing +from `TASK_KINDS`. + +--- + ## Reusing descriptors Each Signal has exactly one owning Source. Do not attach one Signal to several Sources or Records. diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/loader.py b/packages/timenet-connectors/src/timenet_connectors/bases/loader.py new file mode 100644 index 000000000..b9300bb1a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/bases/loader.py @@ -0,0 +1,130 @@ +"""Generic bases for a dataset's loader and its typed source tasks. + +A connector deals with the outside world: its card, its download, its pinned source, and the +engine's interface. A :class:`DatasetLoader` deals with the downloaded files: it parses them once, +then hands out the dataset's Records and its tasks as the source defines them. Each source task is a +:class:`DatasetTask`, which knows the TimeNet task it becomes. The connector's ``convert`` parses with +the loader and lets it fill the dataset:: + + def convert(self, raw_refs): + loader = MyDatasetLoader.parse_from(raw_refs) + return loader.populate(TimeFDataset(metadata=self.metadata())) + +The loader returns source tasks, not TimeNet tasks, so it repeats nothing ``TimeFDataset`` does: +``tasks_of`` selects by source task kind, which a TimeNet task type can't tell apart. +""" + +from abc import ABC, abstractmethod +from collections.abc import Iterable, Iterator, Sequence +from typing import Any, ClassVar, Generic, Self, TypeVar + +from timenet.dataset import Record, TimeFDataset +from timenet.types import Annotation, Split, Task + + +class DatasetTask(ABC): + """One task as its source defines it, which knows the TimeNet task it becomes. + + A subclass is one kind of task of one dataset, such as TRACE's retrieval task. It holds what the + task is built from, and builds the TimeNet task on demand, so a loader can keep its tasks small + and stream them. + """ + + TIMENET_TYPE: ClassVar[type[Task]] + """The TimeNet task type ``to_timenet_task`` returns.""" + + @property + @abstractmethod + def split(self) -> Split | None: + """Return the task's split, as the source assigns it, or ``None`` when the source gives it none.""" + + @abstractmethod + def to_timenet_task(self) -> Task: + """Return the task as TimeNet stores it, an instance of :attr:`TIMENET_TYPE`.""" + + +TTask = TypeVar("TTask", bound=DatasetTask) +K = TypeVar("K", bound=DatasetTask) + + +class DatasetLoader(ABC, Generic[TTask]): + """A dataset's downloaded files, parsed: its Records, its typed source tasks, and its annotations. + + A subclass implements :meth:`parse_from`, :meth:`records` and :meth:`tasks`, declares the kinds + of task it yields in :attr:`TASK_KINDS`, and lists them in its docstring. It may override + :meth:`dataset_annotations`. The base provides :meth:`tasks_of`, :meth:`timenet_tasks` and + :meth:`populate`. + """ + + TASK_KINDS: ClassVar[tuple[type[DatasetTask], ...]] + """Every kind of task :meth:`tasks` yields, in the order a reader should meet them. + + :meth:`populate` declares their TimeNet types. The writer rejects a streamed task of any other + type, so a kind missing here fails when the dataset is written. + """ + + @classmethod + @abstractmethod + def parse_from(cls, refs: Iterable[Any]) -> Self: + """Parse and check the downloaded files once. + + Args: + refs: The references the connector's ``download`` returned. + + Returns: + The loader, holding what its Records and tasks are built from. + """ + + @abstractmethod + def records(self) -> Iterator[Record]: + """Return every Record, in a stable order.""" + + @abstractmethod + def tasks(self) -> Iterator[TTask]: + """Return every source task, in a stable order. Each call starts again from the first.""" + + def dataset_annotations(self) -> Sequence[Annotation]: # noqa: PLR6301 - the loader interface + """Return the annotations the dataset holds as a whole, such as a label vocabulary. + + Returns: + No annotations, unless a subclass overrides this. + """ + return () + + def tasks_of(self, kind: type[K], *, split: Split | None = None) -> Iterator[K]: + """Return the tasks of one kind, optionally of one split, in the order of :meth:`tasks`. + + Args: + kind: A source task kind from :attr:`TASK_KINDS`, or a base class of several. + split: Only the tasks of this split, if given. + + Yields: + The matching tasks. + """ + for task in self.tasks(): + if isinstance(task, kind) and (split is None or task.split is split): + yield task + + def timenet_tasks(self) -> Iterator[Task]: + """Return every task as TimeNet stores it, in the order of :meth:`tasks`.""" + return (task.to_timenet_task() for task in self.tasks()) + + def populate(self, dataset: TimeFDataset) -> TimeFDataset: + """Fill a dataset with this loader's annotations and Records, and stream its tasks. + + Tasks are not held: the dataset calls :meth:`timenet_tasks` each time it reads them. + + Args: + dataset: A dataset, usually empty, carrying the connector's metadata. + + Returns: + The same dataset. + """ + for annotation in self.dataset_annotations(): + dataset.annotate(annotation) + for record in self.records(): + dataset.add_record(record=record) + # One TimeNet type per kind, without repeats, in TASK_KINDS order. + task_types = tuple(dict.fromkeys(kind.TIMENET_TYPE for kind in self.TASK_KINDS)) + dataset.set_task_stream(task_types, self.timenet_tasks) + return dataset diff --git a/packages/timenet-connectors/tests/test_dataset_loader.py b/packages/timenet-connectors/tests/test_dataset_loader.py new file mode 100644 index 000000000..45af4c9c4 --- /dev/null +++ b/packages/timenet-connectors/tests/test_dataset_loader.py @@ -0,0 +1,142 @@ +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass +from typing import Any, ClassVar, Self + +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import OrdinalAxis +from timenet.types import ( + Annotation, + AnswerTask, + ClassificationTask, + DatasetMetadata, + License, + Split, + Task, + TimeSeriesSpec, + Version, +) +from timenet_connectors.bases.loader import DatasetLoader, DatasetTask + + +_SPEC = TimeSeriesSpec(spec_type="toy", name="Toy", unit_value=None, dtype="float64") +_METADATA = DatasetMetadata( + dataset_id="timenet/toy", + dataset_version=Version(1, 0, 0), + name="Toy", + description="demo", + license=License.CC_BY_4_0, +) + + +@dataclass(frozen=True) +class _ToyTask(DatasetTask): + """A toy source task on one Record.""" + + record: Record + toy_split: Split | None + + @property + def split(self) -> Split | None: + return self.toy_split + + +@dataclass(frozen=True) +class _Label(_ToyTask): + TIMENET_TYPE: ClassVar[type[Task]] = ClassificationTask + + def to_timenet_task(self) -> ClassificationTask: + return ClassificationTask(inputs=(self.record,), targets=("spike",), split=self.split) + + +@dataclass(frozen=True) +class _Severity(_ToyTask): + # A second kind that becomes the same TimeNet type as _Label. + TIMENET_TYPE: ClassVar[type[Task]] = ClassificationTask + + def to_timenet_task(self) -> ClassificationTask: + return ClassificationTask(inputs=(self.record,), targets=("high",), split=self.split) + + +@dataclass(frozen=True) +class _Caption(_ToyTask): + TIMENET_TYPE: ClassVar[type[Task]] = AnswerTask + + def to_timenet_task(self) -> AnswerTask: + return AnswerTask(inputs=(self.record,), prompt="Describe it.", targets=("A spike.",), split=self.split) + + +class _ToyLoader(DatasetLoader[_ToyTask]): + """Toy tasks: a label and a severity per Record, and a caption on the train Record.""" + + TASK_KINDS = (_Label, _Severity, _Caption) + + def __init__(self, records: tuple[Record, ...]) -> None: + self._records = records + + @classmethod + def parse_from(cls, refs: Iterable[Any]) -> Self: + records = tuple(Record(sources=(Source(name=name, signals=(_signal(values),)),)) for name, values in refs) + return cls(records) + + def records(self) -> Iterator[Record]: + return iter(self._records) + + def tasks(self) -> Iterator[_ToyTask]: + train, test = self._records + yield _Label(train, Split.TRAIN) + yield _Severity(train, Split.TRAIN) + yield _Caption(train, Split.TRAIN) + yield _Label(test, Split.TEST) + yield _Severity(test, Split.TEST) + + def dataset_annotations(self) -> Sequence[Annotation]: + return (Annotation(key="label_vocabulary", value=["spike", "flat"]),) + + +def _signal(values: list[float]) -> Signal: + return Signal.from_values(values, spec=_SPEC, name="series", time_axis=OrdinalAxis()) + + +def _loader() -> _ToyLoader: + return _ToyLoader.parse_from([("train", [1.0, 2.0]), ("test", [3.0])]) + + +def test_tasks_of_selects_a_source_kind_and_a_split(): + loader = _loader() + assert [task.split for task in loader.tasks_of(_Label, split=Split.TEST)] == [Split.TEST] + # The two classification kinds stay apart, which a TimeNet task type can't tell. + assert [type(task) for task in loader.tasks_of(_Severity, split=Split.TRAIN)] == [_Severity] + # A base class selects every kind derived from it, in the order of tasks(). + assert list(loader.tasks_of(_ToyTask, split=Split.TRAIN)) == list(loader.tasks())[:3] + + +class _UnsplitLoader(_ToyLoader): + """A source that splits only its label tasks.""" + + def tasks(self) -> Iterator[_ToyTask]: + train, test = self._records + yield _Label(train, Split.TRAIN) + yield _Caption(test, None) + + +def test_a_task_without_a_split_is_selected_only_without_a_split(): + loader = _UnsplitLoader.parse_from([("train", [1.0, 2.0]), ("test", [3.0])]) + assert [task.split for task in loader.tasks_of(_ToyTask)] == [Split.TRAIN, None] + assert [task.split for task in loader.tasks_of(_ToyTask, split=Split.TRAIN)] == [Split.TRAIN] + dataset = loader.populate(TimeFDataset(metadata=_METADATA)) + assert [task.split for task in dataset.iter_streamed_tasks_validated()] == [Split.TRAIN, None] + + +def test_populate_fills_the_dataset_and_streams_the_tasks(): + loader = _loader() + dataset = TimeFDataset(metadata=_METADATA) + assert loader.populate(dataset) is dataset + assert [(a.key, a.value) for a in dataset.annotations] == [("label_vocabulary", ["spike", "flat"])] + assert [record.sources[0].name for record in dataset.records] == ["train", "test"] + # The stream starts again on each read, in the order of tasks(). + first, second = list(dataset.iter_tasks()), list(dataset.iter_tasks()) + expected = [(task.TIMENET_TYPE, task.split) for task in loader.tasks()] + assert [(type(task), task.split) for task in first] == expected + assert [(type(task), task.split) for task in second] == expected + # Each TimeNet type is declared once, in TASK_KINDS order. + assert dataset.derive_schema().tasks == (ClassificationTask, AnswerTask) From c503389e97e98b6ff308d9f1d0de4688c2639dd5 Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 13:25:02 +0100 Subject: [PATCH 2/9] refactor(connectors): let DatasetLoader yield TimeNet tasks directly The review found the DatasetTask class an extra step. Each task of a dataset needed its own class that held the Record and the parsed row, gave a split, and then built the TimeNet task. Simple connectors got more code, not less. This removes DatasetTask. tasks() now yields TimeNet tasks, and TASK_TYPES lists their TimeNet types, as TASK_KINDS listed the kinds before. timenet_tasks() goes, because tasks() already gives TimeNet tasks. tasks_of() goes too. When the dataset authors define two tasks of the same TimeNet type, the loader gives each one its own method, named as the authors name it, and tasks() yields from each method. Calling that method selects the tasks of the authors. After populate(), TimeFDataset.tasks_of and get_split select the tasks, so the base needs no selection method and no TTask type variable. A stored name for the task of the authors is left out until a connector needs it. The module docstring and the docs section now use simple English. The docs section is now "Converting with DatasetLoader" and matches the style of the page. The tests use a toy loader with two tasks of the same TimeNet type and a task without a split. Validation: make lint-fix, make check, make check-ci and make test (2142 passed, 1 skipped). --- docs/connectors.md | 60 +++++----- .../src/timenet_connectors/bases/loader.py | 110 +++++------------ .../tests/test_dataset_loader.py | 113 +++++------------- 3 files changed, 96 insertions(+), 187 deletions(-) diff --git a/docs/connectors.md b/docs/connectors.md index d54744b03..b071886ee 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -58,26 +58,33 @@ dataclass, or an S3 key. The connector is generic through PEP 695: --- -## Loaders and source tasks +## Converting with `DatasetLoader` -A connector that parses several files, or defines several kinds of task, can put that work in a -loader. `timenet_connectors.bases.loader` holds two generic bases. The connector keeps the outside -world: its card, its download, and its pinned source. The loader keeps the downloaded files. +Use `timenet_connectors.bases.loader.DatasetLoader` to keep the parsing out of the connector. The +loader reads the downloaded files once, builds the Records and the tasks, and adds them to the +dataset. The connector keeps `download` and calls the loader in `convert`. ```python -from timenet_connectors.bases.loader import DatasetLoader, DatasetTask +from timenet_connectors.bases.loader import DatasetLoader -class MyRetrievalTask(DatasetTask): - TIMENET_TYPE = TSCorrespondenceTask - # split (a property) and to_timenet_task() build the TimeNet task. +class MyDatasetLoader(DatasetLoader): + TASK_TYPES = (ClassificationTask,) + # Each dataset also implements parse_from(refs) and records(). + def binary_label_tasks(self) -> Iterator[ClassificationTask]: + """Is the beat normal or abnormal?""" + for row in self._rows: + yield ClassificationTask(inputs=(self._records[row.key],), targets=(row.binary,)) -class MyDatasetLoader(DatasetLoader[MyRetrievalTask]): - """The dataset's tasks: one retrieval task per published row.""" + def rhythm_label_tasks(self) -> Iterator[ClassificationTask]: + """Which of the 5 rhythm classes is the beat?""" + for row in self._rows: + yield ClassificationTask(inputs=(self._records[row.key],), targets=(row.rhythm,)) - TASK_KINDS = (MyRetrievalTask,) - # parse_from(refs), records() and tasks() are the dataset's own. + def tasks(self) -> Iterator[Task]: + yield from self.binary_label_tasks() + yield from self.rhythm_label_tasks() class MyConnector(BaseConnector[MyRawRef]): @@ -87,22 +94,21 @@ class MyConnector(BaseConnector[MyRawRef]): return loader.populate(dataset) ``` -| Member | Provided by | Contract | +| Method | Provided by | Contract | | --- | --- | --- | -| `parse_from(refs)` | the dataset | Parse and check the downloaded files once. | -| `records()` | the dataset | Every Record, in a stable order. | -| `tasks()` | the dataset | Every source task. Each call starts again from the first. | -| `dataset_annotations()` | the dataset, optional | Annotations of the whole dataset, such as a label vocabulary. | -| `tasks_of(kind, split=)` | the base | The source tasks of one kind, optionally of one split. | -| `timenet_tasks()` | the base | Every task as TimeNet stores it, in the order of `tasks()`. | -| `populate(dataset)` | the base | Add the annotations and Records, and stream the tasks. | - -A source task is the dataset's own: TRACE has an event-type, a retrieval and a forecasting task. -`tasks_of` selects by that kind, which a TimeNet task type can't tell apart when two kinds become -the same type. To select stored tasks by TimeNet type or split, use `TimeFDataset.tasks_of` and -`TimeFDataset.get_split` instead. A loader declares its kinds in `TASK_KINDS` and lists them in its -docstring. `populate` declares their TimeNet types, so the writer rejects a task of a kind missing -from `TASK_KINDS`. +| `parse_from(refs)` | the dataset | Read the downloaded files once and make sure that they are correct. | +| `records()` | the dataset | Return each Record, always in the same order. | +| `tasks()` | the dataset | Return each TimeNet task. Each call starts again at the first task. | +| `dataset_annotations()` | the dataset, optional | Return the annotations of the full dataset, such as the list of its labels. | +| `populate(dataset)` | the base | Add the annotations, the Records and the tasks to the dataset. | + +A loader lists each TimeNet task type that it makes in `TASK_TYPES`. `populate` gives these types to +the dataset. When the dataset is written, the writer stops on a task whose type is not in this list. + +The dataset authors often define several tasks. Two of them can have the same TimeNet type, such as +the binary and the rhythm label in the example. Give each task of the authors its own method, with +the name that the authors use. Then `loader.binary_label_tasks()` returns only the binary-label +tasks. After `populate`, use `TimeFDataset.tasks_of` and `TimeFDataset.get_split` to select tasks. --- diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/loader.py b/packages/timenet-connectors/src/timenet_connectors/bases/loader.py index b9300bb1a..49b93a970 100644 --- a/packages/timenet-connectors/src/timenet_connectors/bases/loader.py +++ b/packages/timenet-connectors/src/timenet_connectors/bases/loader.py @@ -1,121 +1,79 @@ -"""Generic bases for a dataset's loader and its typed source tasks. +"""A shared base class that reads a dataset's files and turns them into TimeNet tasks. -A connector deals with the outside world: its card, its download, its pinned source, and the -engine's interface. A :class:`DatasetLoader` deals with the downloaded files: it parses them once, -then hands out the dataset's Records and its tasks as the source defines them. Each source task is a -:class:`DatasetTask`, which knows the TimeNet task it becomes. The connector's ``convert`` parses with -the loader and lets it fill the dataset:: +A connector gets the files of a dataset from its source, and turns them into Records and tasks. +Most connectors do the second job with the same steps: read the files once, build the Records and +the tasks, and add them to a TimeFDataset. :class:`DatasetLoader` does these steps, so each +connector only writes the parts that are special to its dataset. + +The connector still downloads the files. Its ``convert`` gives them to the loader:: def convert(self, raw_refs): loader = MyDatasetLoader.parse_from(raw_refs) return loader.populate(TimeFDataset(metadata=self.metadata())) - -The loader returns source tasks, not TimeNet tasks, so it repeats nothing ``TimeFDataset`` does: -``tasks_of`` selects by source task kind, which a TimeNet task type can't tell apart. """ from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Sequence -from typing import Any, ClassVar, Generic, Self, TypeVar +from typing import Any, ClassVar, Self from timenet.dataset import Record, TimeFDataset -from timenet.types import Annotation, Split, Task - - -class DatasetTask(ABC): - """One task as its source defines it, which knows the TimeNet task it becomes. - - A subclass is one kind of task of one dataset, such as TRACE's retrieval task. It holds what the - task is built from, and builds the TimeNet task on demand, so a loader can keep its tasks small - and stream them. - """ - - TIMENET_TYPE: ClassVar[type[Task]] - """The TimeNet task type ``to_timenet_task`` returns.""" - - @property - @abstractmethod - def split(self) -> Split | None: - """Return the task's split, as the source assigns it, or ``None`` when the source gives it none.""" +from timenet.types import Annotation, Task - @abstractmethod - def to_timenet_task(self) -> Task: - """Return the task as TimeNet stores it, an instance of :attr:`TIMENET_TYPE`.""" - - -TTask = TypeVar("TTask", bound=DatasetTask) -K = TypeVar("K", bound=DatasetTask) +class DatasetLoader(ABC): + """Reads the downloaded files of one dataset and gives its Records, tasks and annotations. -class DatasetLoader(ABC, Generic[TTask]): - """A dataset's downloaded files, parsed: its Records, its typed source tasks, and its annotations. + A subclass must implement :meth:`parse_from`, :meth:`records` and :meth:`tasks`. It must list + each TimeNet task type that it makes in :attr:`TASK_TYPES`. It can override + :meth:`dataset_annotations`. This base gives :meth:`populate`. - A subclass implements :meth:`parse_from`, :meth:`records` and :meth:`tasks`, declares the kinds - of task it yields in :attr:`TASK_KINDS`, and lists them in its docstring. It may override - :meth:`dataset_annotations`. The base provides :meth:`tasks_of`, :meth:`timenet_tasks` and - :meth:`populate`. + When the dataset authors define several tasks, give each one its own method with the name that + the authors use, such as ``binary_label_tasks``. Then :meth:`tasks` yields from each method. After + :meth:`populate`, ``TimeFDataset.tasks_of`` and ``TimeFDataset.get_split`` select the tasks. """ - TASK_KINDS: ClassVar[tuple[type[DatasetTask], ...]] - """Every kind of task :meth:`tasks` yields, in the order a reader should meet them. + TASK_TYPES: ClassVar[tuple[type[Task], ...]] + """Each TimeNet task type that :meth:`tasks` makes, in the order that a reader sees them first. - :meth:`populate` declares their TimeNet types. The writer rejects a streamed task of any other - type, so a kind missing here fails when the dataset is written. + :meth:`populate` gives these types to the dataset. When the dataset is written, the writer stops + on a task whose type is not in this list. """ @classmethod @abstractmethod def parse_from(cls, refs: Iterable[Any]) -> Self: - """Parse and check the downloaded files once. + """Read the downloaded files one time, and make sure that they are correct. Args: - refs: The references the connector's ``download`` returned. + refs: The references that the ``download`` of the connector returned. Returns: - The loader, holding what its Records and tasks are built from. + The loader. It keeps the data that its Records and tasks need. """ @abstractmethod def records(self) -> Iterator[Record]: - """Return every Record, in a stable order.""" + """Return each Record, always in the same order.""" @abstractmethod - def tasks(self) -> Iterator[TTask]: - """Return every source task, in a stable order. Each call starts again from the first.""" + def tasks(self) -> Iterator[Task]: + """Return each TimeNet task, always in the same order. Each call starts again at the first task.""" def dataset_annotations(self) -> Sequence[Annotation]: # noqa: PLR6301 - the loader interface - """Return the annotations the dataset holds as a whole, such as a label vocabulary. + """Return the annotations of the full dataset, such as the list of its labels. Returns: - No annotations, unless a subclass overrides this. + No annotations. A subclass can override this. """ return () - def tasks_of(self, kind: type[K], *, split: Split | None = None) -> Iterator[K]: - """Return the tasks of one kind, optionally of one split, in the order of :meth:`tasks`. - - Args: - kind: A source task kind from :attr:`TASK_KINDS`, or a base class of several. - split: Only the tasks of this split, if given. - - Yields: - The matching tasks. - """ - for task in self.tasks(): - if isinstance(task, kind) and (split is None or task.split is split): - yield task - - def timenet_tasks(self) -> Iterator[Task]: - """Return every task as TimeNet stores it, in the order of :meth:`tasks`.""" - return (task.to_timenet_task() for task in self.tasks()) - def populate(self, dataset: TimeFDataset) -> TimeFDataset: - """Fill a dataset with this loader's annotations and Records, and stream its tasks. + """Add the annotations, the Records and the tasks of this loader to a dataset. - Tasks are not held: the dataset calls :meth:`timenet_tasks` each time it reads them. + The dataset does not keep the tasks. It calls :meth:`tasks` each time that it reads them. Args: - dataset: A dataset, usually empty, carrying the connector's metadata. + dataset: A dataset, usually empty, with the metadata of the connector. Returns: The same dataset. @@ -124,7 +82,5 @@ def populate(self, dataset: TimeFDataset) -> TimeFDataset: dataset.annotate(annotation) for record in self.records(): dataset.add_record(record=record) - # One TimeNet type per kind, without repeats, in TASK_KINDS order. - task_types = tuple(dict.fromkeys(kind.TIMENET_TYPE for kind in self.TASK_KINDS)) - dataset.set_task_stream(task_types, self.timenet_tasks) + dataset.set_task_stream(self.TASK_TYPES, self.tasks) return dataset diff --git a/packages/timenet-connectors/tests/test_dataset_loader.py b/packages/timenet-connectors/tests/test_dataset_loader.py index 45af4c9c4..80f2af63c 100644 --- a/packages/timenet-connectors/tests/test_dataset_loader.py +++ b/packages/timenet-connectors/tests/test_dataset_loader.py @@ -1,6 +1,5 @@ from collections.abc import Iterable, Iterator, Sequence -from dataclasses import dataclass -from typing import Any, ClassVar, Self +from typing import Any, Self from timenet.dataset import Record, Signal, Source, TimeFDataset from timenet.dataset.axis import OrdinalAxis @@ -15,7 +14,7 @@ TimeSeriesSpec, Version, ) -from timenet_connectors.bases.loader import DatasetLoader, DatasetTask +from timenet_connectors.bases.loader import DatasetLoader _SPEC = TimeSeriesSpec(spec_type="toy", name="Toy", unit_value=None, dtype="float64") @@ -28,103 +27,52 @@ ) -@dataclass(frozen=True) -class _ToyTask(DatasetTask): - """A toy source task on one Record.""" +def _signal() -> Signal: + return Signal.from_values([1.0, 2.0], spec=_SPEC, name="series", time_axis=OrdinalAxis()) - record: Record - toy_split: Split | None - @property - def split(self) -> Split | None: - return self.toy_split +class _ToyLoader(DatasetLoader): + """Two classification tasks of the authors for each Record, and a caption without a split.""" - -@dataclass(frozen=True) -class _Label(_ToyTask): - TIMENET_TYPE: ClassVar[type[Task]] = ClassificationTask - - def to_timenet_task(self) -> ClassificationTask: - return ClassificationTask(inputs=(self.record,), targets=("spike",), split=self.split) - - -@dataclass(frozen=True) -class _Severity(_ToyTask): - # A second kind that becomes the same TimeNet type as _Label. - TIMENET_TYPE: ClassVar[type[Task]] = ClassificationTask - - def to_timenet_task(self) -> ClassificationTask: - return ClassificationTask(inputs=(self.record,), targets=("high",), split=self.split) - - -@dataclass(frozen=True) -class _Caption(_ToyTask): - TIMENET_TYPE: ClassVar[type[Task]] = AnswerTask - - def to_timenet_task(self) -> AnswerTask: - return AnswerTask(inputs=(self.record,), prompt="Describe it.", targets=("A spike.",), split=self.split) - - -class _ToyLoader(DatasetLoader[_ToyTask]): - """Toy tasks: a label and a severity per Record, and a caption on the train Record.""" - - TASK_KINDS = (_Label, _Severity, _Caption) + TASK_TYPES = (ClassificationTask, AnswerTask) def __init__(self, records: tuple[Record, ...]) -> None: self._records = records @classmethod def parse_from(cls, refs: Iterable[Any]) -> Self: - records = tuple(Record(sources=(Source(name=name, signals=(_signal(values),)),)) for name, values in refs) - return cls(records) + return cls(tuple(Record(sources=(Source(name=name, signals=(_signal(),)),)) for name in refs)) def records(self) -> Iterator[Record]: return iter(self._records) - def tasks(self) -> Iterator[_ToyTask]: + def label_tasks(self) -> Iterator[ClassificationTask]: + """Is the series a spike or flat?""" train, test = self._records - yield _Label(train, Split.TRAIN) - yield _Severity(train, Split.TRAIN) - yield _Caption(train, Split.TRAIN) - yield _Label(test, Split.TEST) - yield _Severity(test, Split.TEST) + yield ClassificationTask(inputs=(train,), targets=("spike",), split=Split.TRAIN) + yield ClassificationTask(inputs=(test,), targets=("flat",), split=Split.TEST) - def dataset_annotations(self) -> Sequence[Annotation]: - return (Annotation(key="label_vocabulary", value=["spike", "flat"]),) + def severity_tasks(self) -> Iterator[ClassificationTask]: + """Is the spike high or low? The same TimeNet type as the label tasks.""" + train, _ = self._records + yield ClassificationTask(inputs=(train,), targets=("high",), split=Split.TRAIN) + def tasks(self) -> Iterator[Task]: + yield from self.label_tasks() + yield from self.severity_tasks() + train, _ = self._records + yield AnswerTask(inputs=(train,), prompt="Describe it.", targets=("A spike.",)) -def _signal(values: list[float]) -> Signal: - return Signal.from_values(values, spec=_SPEC, name="series", time_axis=OrdinalAxis()) + def dataset_annotations(self) -> Sequence[Annotation]: + return (Annotation(key="label_vocabulary", value=["spike", "flat"]),) def _loader() -> _ToyLoader: - return _ToyLoader.parse_from([("train", [1.0, 2.0]), ("test", [3.0])]) - - -def test_tasks_of_selects_a_source_kind_and_a_split(): - loader = _loader() - assert [task.split for task in loader.tasks_of(_Label, split=Split.TEST)] == [Split.TEST] - # The two classification kinds stay apart, which a TimeNet task type can't tell. - assert [type(task) for task in loader.tasks_of(_Severity, split=Split.TRAIN)] == [_Severity] - # A base class selects every kind derived from it, in the order of tasks(). - assert list(loader.tasks_of(_ToyTask, split=Split.TRAIN)) == list(loader.tasks())[:3] - - -class _UnsplitLoader(_ToyLoader): - """A source that splits only its label tasks.""" - - def tasks(self) -> Iterator[_ToyTask]: - train, test = self._records - yield _Label(train, Split.TRAIN) - yield _Caption(test, None) + return _ToyLoader.parse_from(["train", "test"]) -def test_a_task_without_a_split_is_selected_only_without_a_split(): - loader = _UnsplitLoader.parse_from([("train", [1.0, 2.0]), ("test", [3.0])]) - assert [task.split for task in loader.tasks_of(_ToyTask)] == [Split.TRAIN, None] - assert [task.split for task in loader.tasks_of(_ToyTask, split=Split.TRAIN)] == [Split.TRAIN] - dataset = loader.populate(TimeFDataset(metadata=_METADATA)) - assert [task.split for task in dataset.iter_streamed_tasks_validated()] == [Split.TRAIN, None] +def _targets(tasks: Iterable[Task]) -> list[tuple[Any, ...] | None]: + return [task.targets for task in tasks] def test_populate_fills_the_dataset_and_streams_the_tasks(): @@ -134,9 +82,8 @@ def test_populate_fills_the_dataset_and_streams_the_tasks(): assert [(a.key, a.value) for a in dataset.annotations] == [("label_vocabulary", ["spike", "flat"])] assert [record.sources[0].name for record in dataset.records] == ["train", "test"] # The stream starts again on each read, in the order of tasks(). - first, second = list(dataset.iter_tasks()), list(dataset.iter_tasks()) - expected = [(task.TIMENET_TYPE, task.split) for task in loader.tasks()] - assert [(type(task), task.split) for task in first] == expected - assert [(type(task), task.split) for task in second] == expected - # Each TimeNet type is declared once, in TASK_KINDS order. + first, second = list(dataset.iter_streamed_tasks_validated()), list(dataset.iter_streamed_tasks_validated()) + assert _targets(first) == _targets(second) == _targets(loader.tasks()) + # The caption keeps having no split. + assert [task.split for task in first] == [Split.TRAIN, Split.TEST, Split.TRAIN, None] assert dataset.derive_schema().tasks == (ClassificationTask, AnswerTask) From 2db87d961d9a584c97de615798ff92ee03af03ad Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 13:54:47 +0100 Subject: [PATCH 3/9] feat(connectors): share typed record metadata across connectors TRACE and RATs40K each carried the same RecordMetadata prototype: a small shared core (subject_id) plus typed per-dataset extensions, each stored under its own name. This moves it to timenet_connectors.metadata so both connectors use one copy. It is still a prototype of the shared metadata proposed on TimeNet #433, kept in the connectors package until the core design is agreed. Nothing in it is specific to one dataset, so it can move to timenet.types unchanged. The API nav in zensical.toml is regenerated for the new module. That also adds four entries main was already missing: timenet.json, timenet_connectors.sources, timenet_connectors.splitting and timenet_connectors.time_axes. Validation: make check, make lint-fix, make check-ci and make test (1607 passed, 1 skipped). --- .../src/timenet_connectors/metadata.py | 118 ++++++++++++++++++ .../timenet-connectors/tests/test_metadata.py | 27 ++++ zensical.toml | 2 +- 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 packages/timenet-connectors/src/timenet_connectors/metadata.py create mode 100644 packages/timenet-connectors/tests/test_metadata.py diff --git a/packages/timenet-connectors/src/timenet_connectors/metadata.py b/packages/timenet-connectors/src/timenet_connectors/metadata.py new file mode 100644 index 000000000..18d7a1577 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/metadata.py @@ -0,0 +1,118 @@ +"""A Record's metadata as a small shared core plus typed extensions, one per dataset or source. + +This is a prototype of the shared metadata proposed on TimeNet #433, kept in the connectors package until +the core design is agreed. Nothing here is specific to one dataset, so it can move to ``timenet.types`` +unchanged. + +A Record's metadata is a ``RecordMetadata``: the fields every dataset shares, then any number of +extensions, each a ``MetadataExtension``. Each extension is stored under its own ``NAME``, so no extension can +overwrite a core field or another extension, and ``from_json`` restores each one as its own type:: + + {"subject_id": "USW00093242", "trace": {"coordinates": ..., "sampling": ..., "windows": [...]}} + +Metadata is provenance and never model input. +""" + +from collections.abc import Iterable +from typing import Annotated, Any, ClassVar, Self, TypeVar + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, StringConstraints + +from timenet.errors import TimeFValidationError +from timenet.json import JsonMapping + + +class MetadataExtension(BaseModel): + """A dataset's or source's typed addition to ``RecordMetadata``, stored under its ``NAME``.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + NAME: ClassVar[str] + """The key the extension is stored under. It must differ from every core field and other extension.""" + + +_Extension = TypeVar("_Extension", bound=MetadataExtension) + + +class RecordMetadata(BaseModel): + """The metadata every Record shares, and the typed extensions its datasets add.""" + + model_config = ConfigDict(extra="forbid", frozen=True, title="Record metadata") + + subject_id: Annotated[StrictStr, StringConstraints(pattern=r"\S")] | None = None + """What was measured: a person, a device, or a site such as a weather station.""" + extensions: tuple[MetadataExtension, ...] = Field(default=(), exclude=True) + """The extensions, in the order they were added. Each is serialized under its ``NAME``.""" + + def extend(self, extension: MetadataExtension) -> Self: + """Return this metadata with one more extension. + + Returns: + New metadata; this one is unchanged. + + Raises: + TimeFValidationError: If the extension's name is a core field's, or another extension's. + """ + if extension.NAME in _CORE_FIELDS or extension.NAME == "extensions": + raise TimeFValidationError(f"extension name {extension.NAME!r} is a core metadata field") + if any(existing.NAME == extension.NAME for existing in self.extensions): + raise TimeFValidationError(f"metadata already has a {extension.NAME!r} extension") + return self.model_copy(update={"extensions": (*self.extensions, extension)}) + + def extension(self, kind: type[_Extension]) -> _Extension: + """Return the extension of one type. + + Returns: + The extension. + + Raises: + TimeFValidationError: If the metadata has no extension of that type. + """ + for extension in self.extensions: + if isinstance(extension, kind): + return extension + raise TimeFValidationError(f"metadata has no {kind.NAME!r} extension") + + def to_json(self) -> JsonMapping: + """Return the metadata as TimeNet stores it: the core fields, then each extension under its name. + + Fields that are ``None`` are left out, and enums are written as their values. + + Returns: + The JSON object. + """ + stored: dict[str, Any] = self.model_dump(mode="json", exclude_none=True) + for extension in self.extensions: + stored[extension.NAME] = extension.model_dump(mode="json", exclude_none=True) + return stored + + @classmethod + def from_json(cls, stored: JsonMapping, kinds: Iterable[type[MetadataExtension]]) -> Self: + """Read stored metadata back, restoring each extension as its own type. + + Args: + stored: The JSON object ``to_json`` wrote. + kinds: The extension types the metadata may hold. + + Returns: + The metadata, with its extensions in the order they were stored. + + Raises: + TimeFValidationError: If a stored key is neither a core field nor one of ``kinds``. + """ + by_name = {kind.NAME: kind for kind in kinds} + core = {key: value for key, value in stored.items() if key in _CORE_FIELDS} + unknown = sorted(key for key in stored if key not in _CORE_FIELDS and key not in by_name) + if unknown: + raise TimeFValidationError( + f"metadata has keys that are neither core fields nor known extensions: {unknown}" + ) + extensions = tuple(by_name[key].model_validate(value) for key, value in stored.items() if key in by_name) + metadata = cls.model_validate(core) + for extension in extensions: + metadata = metadata.extend(extension) + return metadata + + +# The fields stored at the top level; the extensions are stored under their names instead. +_CORE_FIELDS = frozenset(name for name in RecordMetadata.model_fields if name != "extensions") diff --git a/packages/timenet-connectors/tests/test_metadata.py b/packages/timenet-connectors/tests/test_metadata.py new file mode 100644 index 000000000..84bace2e4 --- /dev/null +++ b/packages/timenet-connectors/tests/test_metadata.py @@ -0,0 +1,27 @@ +import pytest + +from timenet.errors import TimeFValidationError +from timenet_connectors.metadata import MetadataExtension, RecordMetadata + + +class _OtherMetadata(MetadataExtension): + NAME = "other" + note: str + + +class _SubjectMetadata(MetadataExtension): + NAME = "subject_id" + + +def test_metadata_extensions_keep_their_own_names(): + metadata = RecordMetadata(subject_id="S1").extend(_OtherMetadata(note="x")) + assert metadata.to_json() == {"subject_id": "S1", "other": {"note": "x"}} + assert RecordMetadata.from_json(metadata.to_json(), [_OtherMetadata]) == metadata + # No extension may overwrite a core field or another extension. + with pytest.raises(TimeFValidationError, match="already has a 'other' extension"): + metadata.extend(_OtherMetadata(note="y")) + with pytest.raises(TimeFValidationError, match="'subject_id' is a core metadata field"): + metadata.extend(_SubjectMetadata()) + # A stored key that is neither a core field nor a known extension is not dropped silently. + with pytest.raises(TimeFValidationError, match=r"neither core fields nor known extensions: \['other'\]"): + RecordMetadata.from_json(metadata.to_json(), []) diff --git a/zensical.toml b/zensical.toml index 472e5e3f4..b6f4ebb22 100644 --- a/zensical.toml +++ b/zensical.toml @@ -22,7 +22,7 @@ nav = [ { "Datasets" = [{ "Use datasets" = [{ "The client (SDK)" = "client.md" }, { "Registries" = "registry.md" }, { "Load into your stack" = "usage.md" }] }, { "Build datasets" = [{ "Connectors" = "connectors.md" }, { "Build & publish" = "build.md" }] }, { "Catalog" = [{ "Datasets" = "catalog/datasets.md" }, { "Benchmarks" = "catalog/benchmarks.md" }, { "Licensing" = "catalog/licensing.md" }] }] }, { "CLI" = [{ "Overview" = "cli/index.md" }, { "timenet (consumer)" = "cli/timenet.md" }, { "timenet-build (producer)" = "cli/build.md" }] }, # >>> api-nav: generated by scripts/gen_api_docs.py — do not edit >>> - { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }] }, + { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.metadata" = "api/metadata.md" }, { "timenet_connectors.sources" = "api/sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, # <<< api-nav <<< ] From f7b28cef50687798c1f92917966bfe34e944a61e Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 13:54:47 +0100 Subject: [PATCH 4/9] refactor(connectors): nest metadata extensions and keep them out of the fields --- .../src/timenet_connectors/metadata.py | 130 +++++++++++------- .../timenet-connectors/tests/test_metadata.py | 53 ++++++- 2 files changed, 128 insertions(+), 55 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/metadata.py b/packages/timenet-connectors/src/timenet_connectors/metadata.py index 18d7a1577..e7619321d 100644 --- a/packages/timenet-connectors/src/timenet_connectors/metadata.py +++ b/packages/timenet-connectors/src/timenet_connectors/metadata.py @@ -6,58 +6,71 @@ A Record's metadata is a ``RecordMetadata``: the fields every dataset shares, then any number of extensions, each a ``MetadataExtension``. Each extension is stored under its own ``NAME``, so no extension can -overwrite a core field or another extension, and ``from_json`` restores each one as its own type:: +overwrite a field or another extension, and ``from_json`` restores each one as its own type. An extension can +hold extensions of its own, stored the same way one level down:: {"subject_id": "USW00093242", "trace": {"coordinates": ..., "sampling": ..., "windows": [...]}} +An extension that may change shape over time declares a ``version`` field as a ``Literal``, so a stored +version it does not read is rejected rather than misread. + +Design choices: + +- The stored JSON is plain keys and values, with no type tags or wrappers, so a reader without these classes, + such as a consumer who has not installed the connector, can still read it. +- Values keep their types: each extension is a pydantic model with typed fields, not a bag of strings. +- Metadata is frozen, and ``extend`` returns a new copy, so metadata shared between objects never changes. +- ``from_json`` takes the extension types explicitly rather than looking them up in a global registry, so a + name only has to be unique among the types passed, and reading does not depend on which modules are imported. + Metadata is provenance and never model input. """ -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from typing import Annotated, Any, ClassVar, Self, TypeVar -from pydantic import BaseModel, ConfigDict, Field, StrictStr, StringConstraints +from pydantic import BaseModel, ConfigDict, PrivateAttr, StrictStr, StringConstraints from timenet.errors import TimeFValidationError from timenet.json import JsonMapping -class MetadataExtension(BaseModel): - """A dataset's or source's typed addition to ``RecordMetadata``, stored under its ``NAME``.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - NAME: ClassVar[str] - """The key the extension is stored under. It must differ from every core field and other extension.""" +_Extension = TypeVar("_Extension", bound="MetadataExtension") -_Extension = TypeVar("_Extension", bound=MetadataExtension) +class Metadata(BaseModel): + """Typed fields, then extensions, each stored under its ``NAME``.""" + model_config = ConfigDict(extra="forbid", frozen=True) -class RecordMetadata(BaseModel): - """The metadata every Record shares, and the typed extensions its datasets add.""" + _extensions: dict[str, "MetadataExtension"] = PrivateAttr(default_factory=dict) + """The extensions by name, in the order they were added. A private attribute rather than a field, so its + own name never collides with an extension's.""" - model_config = ConfigDict(extra="forbid", frozen=True, title="Record metadata") + def __repr_args__(self) -> Iterator[tuple[str | None, Any]]: # noqa: PLW3201 - pydantic's repr hook + # pydantic leaves private attributes out of repr; show each extension under its name, as in the JSON. + yield from super().__repr_args__() + yield from self._extensions.items() - subject_id: Annotated[StrictStr, StringConstraints(pattern=r"\S")] | None = None - """What was measured: a person, a device, or a site such as a weather station.""" - extensions: tuple[MetadataExtension, ...] = Field(default=(), exclude=True) - """The extensions, in the order they were added. Each is serialized under its ``NAME``.""" - - def extend(self, extension: MetadataExtension) -> Self: - """Return this metadata with one more extension. + def extend(self, *extensions: "MetadataExtension") -> Self: + """Return this metadata with more extensions. Returns: New metadata; this one is unchanged. Raises: - TimeFValidationError: If the extension's name is a core field's, or another extension's. + TimeFValidationError: If an extension's name is one of this metadata's fields, or another extension's. """ - if extension.NAME in _CORE_FIELDS or extension.NAME == "extensions": - raise TimeFValidationError(f"extension name {extension.NAME!r} is a core metadata field") - if any(existing.NAME == extension.NAME for existing in self.extensions): - raise TimeFValidationError(f"metadata already has a {extension.NAME!r} extension") - return self.model_copy(update={"extensions": (*self.extensions, extension)}) + extended = dict(self._extensions) + for extension in extensions: + if extension.NAME in type(self).model_fields: + raise TimeFValidationError(f"extension name {extension.NAME!r} is a field of {type(self).__name__}") + if extension.NAME in extended: + raise TimeFValidationError(f"metadata already has a {extension.NAME!r} extension") + extended[extension.NAME] = extension + copy = self.model_copy() + copy._extensions = extended + return copy def extension(self, kind: type[_Extension]) -> _Extension: """Return the extension of one type. @@ -68,13 +81,13 @@ def extension(self, kind: type[_Extension]) -> _Extension: Raises: TimeFValidationError: If the metadata has no extension of that type. """ - for extension in self.extensions: - if isinstance(extension, kind): - return extension - raise TimeFValidationError(f"metadata has no {kind.NAME!r} extension") + extension = self._extensions.get(kind.NAME) + if not isinstance(extension, kind): + raise TimeFValidationError(f"metadata has no {kind.NAME!r} extension") + return extension def to_json(self) -> JsonMapping: - """Return the metadata as TimeNet stores it: the core fields, then each extension under its name. + """Return the metadata as TimeNet stores it: the fields, then each extension under its name. Fields that are ``None`` are left out, and enums are written as their values. @@ -82,37 +95,54 @@ def to_json(self) -> JsonMapping: The JSON object. """ stored: dict[str, Any] = self.model_dump(mode="json", exclude_none=True) - for extension in self.extensions: - stored[extension.NAME] = extension.model_dump(mode="json", exclude_none=True) + for name, extension in self._extensions.items(): + stored[name] = extension.to_json() return stored @classmethod - def from_json(cls, stored: JsonMapping, kinds: Iterable[type[MetadataExtension]]) -> Self: - """Read stored metadata back, restoring each extension as its own type. + def from_json(cls, stored: JsonMapping, kinds: Iterable[type["MetadataExtension"]]) -> Self: + """Read stored metadata back, restoring each extension, at any depth, as its own type. Args: stored: The JSON object ``to_json`` wrote. - kinds: The extension types the metadata may hold. + kinds: The extension types the metadata may hold, at any depth. Returns: The metadata, with its extensions in the order they were stored. Raises: - TimeFValidationError: If a stored key is neither a core field nor one of ``kinds``. + TimeFValidationError: If two of ``kinds`` share a name, a stored key is neither a field nor one of + ``kinds``, or an extension is not stored as a JSON object. """ + kinds = tuple(kinds) by_name = {kind.NAME: kind for kind in kinds} - core = {key: value for key, value in stored.items() if key in _CORE_FIELDS} - unknown = sorted(key for key in stored if key not in _CORE_FIELDS and key not in by_name) + if len(by_name) != len(kinds): + raise TimeFValidationError("two extension types share a name") + fields = {key: value for key, value in stored.items() if key in cls.model_fields} + unknown = sorted(key for key in stored if key not in fields and key not in by_name) if unknown: - raise TimeFValidationError( - f"metadata has keys that are neither core fields nor known extensions: {unknown}" - ) - extensions = tuple(by_name[key].model_validate(value) for key, value in stored.items() if key in by_name) - metadata = cls.model_validate(core) - for extension in extensions: - metadata = metadata.extend(extension) - return metadata + raise TimeFValidationError(f"metadata has keys that are neither fields nor known extensions: {unknown}") + extensions: list[MetadataExtension] = [] + for key, value in stored.items(): + if key in fields: + continue + if not isinstance(value, dict): + raise TimeFValidationError(f"metadata extension {key!r} is not a JSON object") + extensions.append(by_name[key].from_json(value, kinds)) + return cls.model_validate(fields).extend(*extensions) + + +class MetadataExtension(Metadata): + """A dataset's or source's typed addition to ``RecordMetadata``, stored under its ``NAME``.""" + + NAME: ClassVar[str] + """The key the extension is stored under. It must differ from its holder's fields and other extensions.""" -# The fields stored at the top level; the extensions are stored under their names instead. -_CORE_FIELDS = frozenset(name for name in RecordMetadata.model_fields if name != "extensions") +class RecordMetadata(Metadata): + """The metadata every Record shares, and the typed extensions its datasets add.""" + + model_config = ConfigDict(title="Record metadata") + + subject_id: Annotated[StrictStr, StringConstraints(pattern=r"\S")] | None = None + """What was measured: a person, a device, or a site such as a weather station.""" diff --git a/packages/timenet-connectors/tests/test_metadata.py b/packages/timenet-connectors/tests/test_metadata.py index 84bace2e4..b705fc4f8 100644 --- a/packages/timenet-connectors/tests/test_metadata.py +++ b/packages/timenet-connectors/tests/test_metadata.py @@ -1,9 +1,15 @@ +from enum import StrEnum + import pytest from timenet.errors import TimeFValidationError from timenet_connectors.metadata import MetadataExtension, RecordMetadata +class _Sampling(StrEnum): + RANDOM = "random" + + class _OtherMetadata(MetadataExtension): NAME = "other" note: str @@ -13,15 +19,52 @@ class _SubjectMetadata(MetadataExtension): NAME = "subject_id" +class _DeviceMetadata(MetadataExtension): + NAME = "device" + name: str + + +class _TypedMetadata(MetadataExtension): + NAME = "typed" + grams: float + sampling: _Sampling + + def test_metadata_extensions_keep_their_own_names(): - metadata = RecordMetadata(subject_id="S1").extend(_OtherMetadata(note="x")) + core = RecordMetadata(subject_id="S1") + metadata = core.extend(_OtherMetadata(note="x")) + assert core.to_json() == {"subject_id": "S1"} assert metadata.to_json() == {"subject_id": "S1", "other": {"note": "x"}} assert RecordMetadata.from_json(metadata.to_json(), [_OtherMetadata]) == metadata - # No extension may overwrite a core field or another extension. + # No extension may overwrite a field or another extension. with pytest.raises(TimeFValidationError, match="already has a 'other' extension"): metadata.extend(_OtherMetadata(note="y")) - with pytest.raises(TimeFValidationError, match="'subject_id' is a core metadata field"): + with pytest.raises(TimeFValidationError, match="'subject_id' is a field of RecordMetadata"): metadata.extend(_SubjectMetadata()) - # A stored key that is neither a core field nor a known extension is not dropped silently. - with pytest.raises(TimeFValidationError, match=r"neither core fields nor known extensions: \['other'\]"): + # A stored key that is neither a field nor a known extension is not dropped silently. + with pytest.raises(TimeFValidationError, match=r"neither fields nor known extensions: \['other'\]"): RecordMetadata.from_json(metadata.to_json(), []) + with pytest.raises(TimeFValidationError, match="two extension types share a name"): + RecordMetadata.from_json({}, [_OtherMetadata, _OtherMetadata]) + with pytest.raises(TimeFValidationError, match="'other' is not a JSON object"): + RecordMetadata.from_json({"other": "x"}, [_OtherMetadata]) + + +def test_metadata_extensions_nest_and_keep_their_types(): + typed = _TypedMetadata(grams=45, sampling=_Sampling.RANDOM).extend(_DeviceMetadata(name="watch")) + metadata = RecordMetadata(subject_id="S1").extend(typed) + stored = metadata.to_json() + assert stored == { + "subject_id": "S1", + "typed": {"grams": 45.0, "sampling": "random", "device": {"name": "watch"}}, + } + assert repr(metadata) == ( + "RecordMetadata(subject_id='S1', typed=_TypedMetadata(grams=45.0, sampling=<_Sampling.RANDOM: 'random'>, " + "device=_DeviceMetadata(name='watch')))" + ) + restored = RecordMetadata.from_json(stored, [_TypedMetadata, _DeviceMetadata]) + assert restored == metadata + assert restored.extension(_TypedMetadata).sampling is _Sampling.RANDOM + assert restored.extension(_TypedMetadata).extension(_DeviceMetadata).name == "watch" + with pytest.raises(TimeFValidationError, match="has no 'device' extension"): + restored.extension(_DeviceMetadata) From aac5e5f8ba1673ff04cf98115f2ed1f101d82e94 Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 13:54:47 +0100 Subject: [PATCH 5/9] refactor(connectors): make record metadata one flat class per dataset A dataset now declares its Record metadata as one subclass of RecordMetadata that adds its own typed fields, and a Record holds one such object. to_json writes it as one flat JSON object, and XRecordMetadata.from_json reads it back as that class. - Removes MetadataExtension, extend, extension() and the kinds argument. - A subclass that redefines a shared field such as subject_id is rejected when it is defined. - from_json raises TimeFValidationError on an unknown key or another version; core_of reads only the shared fields. Validation: make check, make check-ci, make test (2146 passed, 1 skipped) and make test-connectors (every connector on this branch passes). --- .../src/timenet_connectors/metadata.py | 155 ++++++------------ .../timenet-connectors/tests/test_metadata.py | 78 ++++----- 2 files changed, 79 insertions(+), 154 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/metadata.py b/packages/timenet-connectors/src/timenet_connectors/metadata.py index e7619321d..91ca901e9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/metadata.py +++ b/packages/timenet-connectors/src/timenet_connectors/metadata.py @@ -1,93 +1,57 @@ -"""A Record's metadata as a small shared core plus typed extensions, one per dataset or source. +"""A Record's metadata: the fields every dataset shares, plus one dataset's own typed fields. -This is a prototype of the shared metadata proposed on TimeNet #433, kept in the connectors package until -the core design is agreed. Nothing here is specific to one dataset, so it can move to ``timenet.types`` -unchanged. +A connector declares its Record metadata as one subclass of ``RecordMetadata`` that adds its own fields, and a +Record holds one such object. ``to_json`` writes it as one flat JSON object:: -A Record's metadata is a ``RecordMetadata``: the fields every dataset shares, then any number of -extensions, each a ``MetadataExtension``. Each extension is stored under its own ``NAME``, so no extension can -overwrite a field or another extension, and ``from_json`` restores each one as its own type. An extension can -hold extensions of its own, stored the same way one level down:: - - {"subject_id": "USW00093242", "trace": {"coordinates": ..., "sampling": ..., "windows": [...]}} - -An extension that may change shape over time declares a ``version`` field as a ``Literal``, so a stored -version it does not read is rejected rather than misread. + {"subject_id": "USW00093242", "coordinates": ..., "sampling": "random", "windows": [...]} Design choices: -- The stored JSON is plain keys and values, with no type tags or wrappers, so a reader without these classes, - such as a consumer who has not installed the connector, can still read it. -- Values keep their types: each extension is a pydantic model with typed fields, not a bag of strings. -- Metadata is frozen, and ``extend`` returns a new copy, so metadata shared between objects never changes. -- ``from_json`` takes the extension types explicitly rather than looking them up in a global registry, so a - name only has to be unique among the types passed, and reading does not depend on which modules are imported. +- One class per dataset, one level of keys: the stored JSON is plain keys and values, with no type tags or + wrappers, so a reader without the connector, such as a consumer who has not installed it, can still read it. + A field may still hold a structured value, such as a pydantic model or a tuple of them. +- Values keep their types: the fields are typed pydantic fields, not a bag of strings. +- Metadata is frozen, so metadata shared between objects never changes. +- A subclass adds fields but never redefines a shared one, so a shared field means the same in every dataset. +- Reading needs the dataset's class: ``CGMRecordMetadata.from_json(...)``. Code without it reads the shared + fields with ``RecordMetadata.core_of(...)``. + +A dataset whose metadata may change shape over time declares a ``version`` field as a ``Literal``, so a stored +version it does not read is rejected rather than misread. Metadata is provenance and never model input. """ -from collections.abc import Iterable, Iterator -from typing import Annotated, Any, ClassVar, Self, TypeVar +from typing import Annotated, Any, Self -from pydantic import BaseModel, ConfigDict, PrivateAttr, StrictStr, StringConstraints +from pydantic import BaseModel, ConfigDict, StrictStr, StringConstraints, ValidationError from timenet.errors import TimeFValidationError from timenet.json import JsonMapping -_Extension = TypeVar("_Extension", bound="MetadataExtension") - - -class Metadata(BaseModel): - """Typed fields, then extensions, each stored under its ``NAME``.""" +class RecordMetadata(BaseModel): + """The metadata every Record shares. A dataset subclasses it to add its own fields.""" - model_config = ConfigDict(extra="forbid", frozen=True) + model_config = ConfigDict(extra="forbid", frozen=True, title="Record metadata") - _extensions: dict[str, "MetadataExtension"] = PrivateAttr(default_factory=dict) - """The extensions by name, in the order they were added. A private attribute rather than a field, so its - own name never collides with an extension's.""" - - def __repr_args__(self) -> Iterator[tuple[str | None, Any]]: # noqa: PLW3201 - pydantic's repr hook - # pydantic leaves private attributes out of repr; show each extension under its name, as in the JSON. - yield from super().__repr_args__() - yield from self._extensions.items() - - def extend(self, *extensions: "MetadataExtension") -> Self: - """Return this metadata with more extensions. - - Returns: - New metadata; this one is unchanged. - - Raises: - TimeFValidationError: If an extension's name is one of this metadata's fields, or another extension's. - """ - extended = dict(self._extensions) - for extension in extensions: - if extension.NAME in type(self).model_fields: - raise TimeFValidationError(f"extension name {extension.NAME!r} is a field of {type(self).__name__}") - if extension.NAME in extended: - raise TimeFValidationError(f"metadata already has a {extension.NAME!r} extension") - extended[extension.NAME] = extension - copy = self.model_copy() - copy._extensions = extended - return copy - - def extension(self, kind: type[_Extension]) -> _Extension: - """Return the extension of one type. + subject_id: Annotated[StrictStr, StringConstraints(pattern=r"\S")] | None = None + """What was measured: a person, a device, or a site such as a weather station.""" - Returns: - The extension. + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: # noqa: PLW3201 - pydantic's subclass hook + """Reject a subclass that redefines a shared field. Raises: - TimeFValidationError: If the metadata has no extension of that type. + TimeFValidationError: If the subclass changes a shared field's type. """ - extension = self._extensions.get(kind.NAME) - if not isinstance(extension, kind): - raise TimeFValidationError(f"metadata has no {kind.NAME!r} extension") - return extension + super().__pydantic_init_subclass__(**kwargs) + for name, field in RecordMetadata.model_fields.items(): + if cls.model_fields[name].annotation != field.annotation: + raise TimeFValidationError(f"{cls.__name__} redefines the shared metadata field {name!r}") def to_json(self) -> JsonMapping: - """Return the metadata as TimeNet stores it: the fields, then each extension under its name. + """Return the metadata as TimeNet stores it. Fields that are ``None`` are left out, and enums are written as their values. @@ -95,54 +59,37 @@ def to_json(self) -> JsonMapping: The JSON object. """ stored: dict[str, Any] = self.model_dump(mode="json", exclude_none=True) - for name, extension in self._extensions.items(): - stored[name] = extension.to_json() return stored @classmethod - def from_json(cls, stored: JsonMapping, kinds: Iterable[type["MetadataExtension"]]) -> Self: - """Read stored metadata back, restoring each extension, at any depth, as its own type. + def from_json(cls, stored: JsonMapping) -> Self: + """Read stored metadata back as this class. Args: stored: The JSON object ``to_json`` wrote. - kinds: The extension types the metadata may hold, at any depth. Returns: - The metadata, with its extensions in the order they were stored. + The metadata. Raises: - TimeFValidationError: If two of ``kinds`` share a name, a stored key is neither a field nor one of - ``kinds``, or an extension is not stored as a JSON object. + TimeFValidationError: If the stored object does not match this class, for example an unknown key or + another version. """ - kinds = tuple(kinds) - by_name = {kind.NAME: kind for kind in kinds} - if len(by_name) != len(kinds): - raise TimeFValidationError("two extension types share a name") - fields = {key: value for key, value in stored.items() if key in cls.model_fields} - unknown = sorted(key for key in stored if key not in fields and key not in by_name) - if unknown: - raise TimeFValidationError(f"metadata has keys that are neither fields nor known extensions: {unknown}") - extensions: list[MetadataExtension] = [] - for key, value in stored.items(): - if key in fields: - continue - if not isinstance(value, dict): - raise TimeFValidationError(f"metadata extension {key!r} is not a JSON object") - extensions.append(by_name[key].from_json(value, kinds)) - return cls.model_validate(fields).extend(*extensions) - - -class MetadataExtension(Metadata): - """A dataset's or source's typed addition to ``RecordMetadata``, stored under its ``NAME``.""" - - NAME: ClassVar[str] - """The key the extension is stored under. It must differ from its holder's fields and other extensions.""" + try: + return cls.model_validate(stored) + except ValidationError as error: + raise TimeFValidationError(f"invalid {cls.__name__}: {error}") from error + @classmethod + def core_of(cls, stored: JsonMapping) -> "RecordMetadata": + """Read only the shared fields, for code that doesn't have the dataset's class. -class RecordMetadata(Metadata): - """The metadata every Record shares, and the typed extensions its datasets add.""" - - model_config = ConfigDict(title="Record metadata") + Args: + stored: The JSON object a dataset's ``to_json`` wrote. - subject_id: Annotated[StrictStr, StringConstraints(pattern=r"\S")] | None = None - """What was measured: a person, a device, or a site such as a weather station.""" + Returns: + The shared fields; the dataset's own fields are left out. + """ + return RecordMetadata.from_json( + {key: value for key, value in stored.items() if key in RecordMetadata.model_fields} + ) diff --git a/packages/timenet-connectors/tests/test_metadata.py b/packages/timenet-connectors/tests/test_metadata.py index b705fc4f8..9d5e6c8f0 100644 --- a/packages/timenet-connectors/tests/test_metadata.py +++ b/packages/timenet-connectors/tests/test_metadata.py @@ -1,70 +1,48 @@ from enum import StrEnum +from typing import Literal +from pydantic import BaseModel, ConfigDict import pytest from timenet.errors import TimeFValidationError -from timenet_connectors.metadata import MetadataExtension, RecordMetadata +from timenet_connectors.metadata import RecordMetadata class _Sampling(StrEnum): RANDOM = "random" -class _OtherMetadata(MetadataExtension): - NAME = "other" - note: str - - -class _SubjectMetadata(MetadataExtension): - NAME = "subject_id" - - -class _DeviceMetadata(MetadataExtension): - NAME = "device" +class _Device(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) name: str -class _TypedMetadata(MetadataExtension): - NAME = "typed" +class _DatasetMetadata(RecordMetadata): + version: Literal[2] = 2 grams: float sampling: _Sampling + watch: _Device + strap: _Device | None = None -def test_metadata_extensions_keep_their_own_names(): - core = RecordMetadata(subject_id="S1") - metadata = core.extend(_OtherMetadata(note="x")) - assert core.to_json() == {"subject_id": "S1"} - assert metadata.to_json() == {"subject_id": "S1", "other": {"note": "x"}} - assert RecordMetadata.from_json(metadata.to_json(), [_OtherMetadata]) == metadata - # No extension may overwrite a field or another extension. - with pytest.raises(TimeFValidationError, match="already has a 'other' extension"): - metadata.extend(_OtherMetadata(note="y")) - with pytest.raises(TimeFValidationError, match="'subject_id' is a field of RecordMetadata"): - metadata.extend(_SubjectMetadata()) - # A stored key that is neither a field nor a known extension is not dropped silently. - with pytest.raises(TimeFValidationError, match=r"neither fields nor known extensions: \['other'\]"): - RecordMetadata.from_json(metadata.to_json(), []) - with pytest.raises(TimeFValidationError, match="two extension types share a name"): - RecordMetadata.from_json({}, [_OtherMetadata, _OtherMetadata]) - with pytest.raises(TimeFValidationError, match="'other' is not a JSON object"): - RecordMetadata.from_json({"other": "x"}, [_OtherMetadata]) - - -def test_metadata_extensions_nest_and_keep_their_types(): - typed = _TypedMetadata(grams=45, sampling=_Sampling.RANDOM).extend(_DeviceMetadata(name="watch")) - metadata = RecordMetadata(subject_id="S1").extend(typed) +def test_dataset_metadata_is_one_flat_typed_object(): + metadata = _DatasetMetadata(subject_id="S1", grams=45, sampling=_Sampling.RANDOM, watch=_Device(name="w")) stored = metadata.to_json() - assert stored == { - "subject_id": "S1", - "typed": {"grams": 45.0, "sampling": "random", "device": {"name": "watch"}}, - } - assert repr(metadata) == ( - "RecordMetadata(subject_id='S1', typed=_TypedMetadata(grams=45.0, sampling=<_Sampling.RANDOM: 'random'>, " - "device=_DeviceMetadata(name='watch')))" - ) - restored = RecordMetadata.from_json(stored, [_TypedMetadata, _DeviceMetadata]) + assert stored == {"subject_id": "S1", "version": 2, "grams": 45.0, "sampling": "random", "watch": {"name": "w"}} + restored = _DatasetMetadata.from_json(stored) assert restored == metadata - assert restored.extension(_TypedMetadata).sampling is _Sampling.RANDOM - assert restored.extension(_TypedMetadata).extension(_DeviceMetadata).name == "watch" - with pytest.raises(TimeFValidationError, match="has no 'device' extension"): - restored.extension(_DeviceMetadata) + assert restored.sampling is _Sampling.RANDOM + # Code without the dataset's class still reads the shared fields. + assert RecordMetadata.core_of(stored) == RecordMetadata(subject_id="S1") + # A key the class doesn't declare, or a version it doesn't read, is rejected rather than misread. + with pytest.raises(TimeFValidationError, match="invalid _DatasetMetadata"): + _DatasetMetadata.from_json({**stored, "note": "x"}) + with pytest.raises(TimeFValidationError, match="version"): + _DatasetMetadata.from_json({**stored, "version": 1}) + + +def test_a_dataset_cannot_redefine_a_shared_field(): + with pytest.raises(TimeFValidationError, match="redefines the shared metadata field 'subject_id'"): + + class _Clash(RecordMetadata): + subject_id: int = 0 # ty: ignore[invalid-attribute-override] From 61615b9eb1253d31d758fffe4a44af0430f03d26 Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 13:54:47 +0100 Subject: [PATCH 6/9] refactor(connectors): drop the unused RecordMetadata.core_of Every connector reads its metadata with its own class's from_json, so nothing called core_of. --- .../src/timenet_connectors/metadata.py | 17 +---------------- .../timenet-connectors/tests/test_metadata.py | 2 -- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/metadata.py b/packages/timenet-connectors/src/timenet_connectors/metadata.py index 91ca901e9..df1d8a426 100644 --- a/packages/timenet-connectors/src/timenet_connectors/metadata.py +++ b/packages/timenet-connectors/src/timenet_connectors/metadata.py @@ -13,8 +13,7 @@ - Values keep their types: the fields are typed pydantic fields, not a bag of strings. - Metadata is frozen, so metadata shared between objects never changes. - A subclass adds fields but never redefines a shared one, so a shared field means the same in every dataset. -- Reading needs the dataset's class: ``CGMRecordMetadata.from_json(...)``. Code without it reads the shared - fields with ``RecordMetadata.core_of(...)``. +- Reading needs the dataset's class: ``CGMRecordMetadata.from_json(...)``. A dataset whose metadata may change shape over time declares a ``version`` field as a ``Literal``, so a stored version it does not read is rejected rather than misread. @@ -79,17 +78,3 @@ def from_json(cls, stored: JsonMapping) -> Self: return cls.model_validate(stored) except ValidationError as error: raise TimeFValidationError(f"invalid {cls.__name__}: {error}") from error - - @classmethod - def core_of(cls, stored: JsonMapping) -> "RecordMetadata": - """Read only the shared fields, for code that doesn't have the dataset's class. - - Args: - stored: The JSON object a dataset's ``to_json`` wrote. - - Returns: - The shared fields; the dataset's own fields are left out. - """ - return RecordMetadata.from_json( - {key: value for key, value in stored.items() if key in RecordMetadata.model_fields} - ) diff --git a/packages/timenet-connectors/tests/test_metadata.py b/packages/timenet-connectors/tests/test_metadata.py index 9d5e6c8f0..87f92b5e7 100644 --- a/packages/timenet-connectors/tests/test_metadata.py +++ b/packages/timenet-connectors/tests/test_metadata.py @@ -32,8 +32,6 @@ def test_dataset_metadata_is_one_flat_typed_object(): restored = _DatasetMetadata.from_json(stored) assert restored == metadata assert restored.sampling is _Sampling.RANDOM - # Code without the dataset's class still reads the shared fields. - assert RecordMetadata.core_of(stored) == RecordMetadata(subject_id="S1") # A key the class doesn't declare, or a version it doesn't read, is rejected rather than misread. with pytest.raises(TimeFValidationError, match="invalid _DatasetMetadata"): _DatasetMetadata.from_json({**stored, "note": "x"}) From 3d2354fe40e39bc2ec1d87b4ad788216e8333cdd Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 14:41:40 +0100 Subject: [PATCH 7/9] fix(connectors): keep None metadata fields when storing a Record RecordMetadata.to_json dumped with exclude_none=True, which drops every None field, including fields of nested models. Metadata with a legitimate None then fails to read back (a required nullable field is reported missing) or reads back wrong (a None that overrides a non-None default comes back as the default, without an error). to_json now writes every field, None as null, so from_json returns the values that were stored. A test covers a required nullable field, a None over a non-None default, and a None inside a nested model. Capture24, the only connector upstack built on RecordMetadata, always sets its fields, so its stored metadata only gains an explicit null where a field is unset. --- .../src/timenet_connectors/metadata.py | 5 ++-- .../timenet-connectors/tests/test_metadata.py | 28 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/metadata.py b/packages/timenet-connectors/src/timenet_connectors/metadata.py index df1d8a426..e947a7f4b 100644 --- a/packages/timenet-connectors/src/timenet_connectors/metadata.py +++ b/packages/timenet-connectors/src/timenet_connectors/metadata.py @@ -52,12 +52,13 @@ def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: # noqa: PLW3201 - p def to_json(self) -> JsonMapping: """Return the metadata as TimeNet stores it. - Fields that are ``None`` are left out, and enums are written as their values. + Every field is written, ``None`` as ``null``, so ``from_json`` reads back the same values; enums are + written as their values. Returns: The JSON object. """ - stored: dict[str, Any] = self.model_dump(mode="json", exclude_none=True) + stored: dict[str, Any] = self.model_dump(mode="json") return stored @classmethod diff --git a/packages/timenet-connectors/tests/test_metadata.py b/packages/timenet-connectors/tests/test_metadata.py index 87f92b5e7..e87f788cf 100644 --- a/packages/timenet-connectors/tests/test_metadata.py +++ b/packages/timenet-connectors/tests/test_metadata.py @@ -28,7 +28,14 @@ class _DatasetMetadata(RecordMetadata): def test_dataset_metadata_is_one_flat_typed_object(): metadata = _DatasetMetadata(subject_id="S1", grams=45, sampling=_Sampling.RANDOM, watch=_Device(name="w")) stored = metadata.to_json() - assert stored == {"subject_id": "S1", "version": 2, "grams": 45.0, "sampling": "random", "watch": {"name": "w"}} + assert stored == { + "subject_id": "S1", + "version": 2, + "grams": 45.0, + "sampling": "random", + "watch": {"name": "w"}, + "strap": None, + } restored = _DatasetMetadata.from_json(stored) assert restored == metadata assert restored.sampling is _Sampling.RANDOM @@ -39,6 +46,25 @@ def test_dataset_metadata_is_one_flat_typed_object(): _DatasetMetadata.from_json({**stored, "version": 1}) +class _Sensor(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + serial: str | None + + +class _NullableMetadata(RecordMetadata): + battery: float | None + unit: str | None = "mg/dL" + sensor: _Sensor + + +def test_a_none_value_is_stored_and_read_back(): + # A required nullable field, a None that overrides a non-None default, and a None inside a nested model. + metadata = _NullableMetadata(battery=None, unit=None, sensor=_Sensor(serial=None)) + stored = metadata.to_json() + assert stored == {"subject_id": None, "battery": None, "unit": None, "sensor": {"serial": None}} + assert _NullableMetadata.from_json(stored) == metadata + + def test_a_dataset_cannot_redefine_a_shared_field(): with pytest.raises(TimeFValidationError, match="redefines the shared metadata field 'subject_id'"): From 76bd6288bd39efbd905b53bbfceb84575c476f89 Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 15:25:29 +0100 Subject: [PATCH 8/9] feat(connectors): add EngineMT-QA aero-engine question answering dataset This adds pandalin98/enginemt-qa: question answering over simulated N-CMAPSS aero-engine cycles in four tasks (Wang et al., ITFormer, ICML 2025). The data comes from the Hugging Face release of the authors, pinned at ea2051a, the revision that their own channel audit checked. The connector uses the flat record metadata of #593, and the same module layout as RATs40K, CAPTURE-24 and TRACE: release, keys, questions, qa, signals, records, tasks and loader, and also model_input. The design follows the code of the authors at ITFormer-ICML25 b80142f (dataset/dataset.py, utils/metrics.py, inference.py) and the paper. The dataset holds: - Records: one for each HDF5 row of seq_data, with 33 lazy float64 signals, channel_00 to channel_32, on a 1 Hz axis, 600 values each. The release names no channel, and the audit of the authors says not to guess, so the names are indices. The values are z-scored, as published. The 1 Hz rate is the "10-minute (600 points)" of the paper. Metadata holds the row_id that the QAs name. By default, the connector builds the 7,755 rows that a QA names. EngineMTQAConnector.ALL_ROWS = True builds all 118,921 rows, which the encoder pretraining of the authors reads. - Tasks: one for each published QA pair, one for each stage, under the file split of the authors: Understanding (AnswerTask, 1 cycle), Perception (ClassificationTask, 1 cycle), Reasoning (ClassificationTask, 10 cycles) and Decision-Making (AnswerTask, 10 cycles). There are 142,273 tasks: 100,296 train and 41,977 test. - Closed questions: each of the nine closed questions has one fixed text and one option list. The model of the authors predicts letters, and closed_question_metrics scores them as a set. So the targets are the published letters. The fault cause can hold several, such as ("a", "b", "e"). One vocabulary annotation for each option list maps each letter to its option. - Inputs with 10 cycles: the 10 Records, in published order. The model of the authors reads the first 60 steps of each, joined. One scope cannot cut 10 inputs, so these tasks state steps_per_cycle: 60, and model_input.authors_series rebuilds the (600, 33) array of the authors. - Prompt and metadata: the published question, with its marker. The task metadata holds source_line and qa_offset, as the authors name them. The data keeps these issues visible: - 1,769 Perception answers are empty: fault cause 1,054 train and 422 test, health state 215 train and 78 test. They are missing labels, not "no fault". So these pairs get no task, and their count is pinned for each question and file. - 2,689 of the 3,384 test lists of 10 ids are also train lists, with the same Reasoning answers, and 3,223 rows are in both files. The connector keeps the split of the authors, and the README says so. - The paper says 32 channels and 11k or 110k pairs. The release has 33 channels and 144,042 pairs. The README lists these differences. check_layout makes sure that the HDF5 file holds exactly seq_data and data_ID, with the published shape and dtype, and the ids 1 to 118,921. Each QA line is validated strictly: exact keys, name repeating id, 10 different ids in each list, alternating turns, stage against format and row count, one marker, closed questions by their exact text, and answers as different letters of their options. The build checks the pinned counts of each file (lines, pairs for each stage, empty answers for each question) and the 7,755 rows that a QA names. A signal reads its row from the HDF5 file once, for all 33 channels. This adds h5py to the requirements of the connector and to the dev group, as wfdb and huggingface-hub are, so that ty resolves it. On the real release, at the pinned digests, a full build through run_pipeline stores 7,755 Records and 142,273 tasks (69,902 answer, 72,371 classification) in 903 MB, in about one minute. Read back through TimeNet(...).load, 50 random Records equal their HDF5 rows, and the authors_series of 200 random tasks equal the slicing of the authors. With ALL_ROWS = True, a full build stores 118,921 Records, ids 1 to 118,921, and the same 142,273 tasks in 13 GB, in 12 minutes. 50 random Records read back equal to their HDF5 rows. The pinned counts match the README and the audit of the authors: 101,565 train and 42,477 test pairs, for each stage. This commit builds its tasks on the DatasetTask classes of the first version of DatasetLoader. The next commit moves them to the current DatasetLoader, and gives the validation for both commits. --- .../datasets/pandalin98/__init__.py | 4 + .../datasets/pandalin98/enginemt_qa/README.md | 244 ++++++++++++++ .../pandalin98/enginemt_qa/__init__.py | 6 + .../pandalin98/enginemt_qa/connector.py | 54 +++ .../pandalin98/enginemt_qa/dataset.yaml | 33 ++ .../datasets/pandalin98/enginemt_qa/keys.py | 33 ++ .../datasets/pandalin98/enginemt_qa/loader.py | 142 ++++++++ .../pandalin98/enginemt_qa/model_input.py | 51 +++ .../datasets/pandalin98/enginemt_qa/qa.py | 183 +++++++++++ .../pandalin98/enginemt_qa/questions.py | 225 +++++++++++++ .../pandalin98/enginemt_qa/records.py | 93 ++++++ .../pandalin98/enginemt_qa/release.py | 123 +++++++ .../pandalin98/enginemt_qa/requirements.txt | 4 + .../pandalin98/enginemt_qa/signals.py | 77 +++++ .../datasets/pandalin98/enginemt_qa/tasks.py | 129 ++++++++ .../enginemt_qa/tests/test_connector.py | 309 ++++++++++++++++++ pyproject.toml | 1 + uv.lock | 58 +++- 18 files changed, 1764 insertions(+), 5 deletions(-) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/keys.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/model_input.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/qa.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/questions.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/release.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/signals.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py new file mode 100644 index 000000000..2c96ddf0d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py @@ -0,0 +1,4 @@ +"""Connectors for datasets released by Yilin Wang (``pandalin98``) and co-authors. + +Keep this regular package so pkgutil.walk_packages can discover its connector children. +""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md new file mode 100644 index 000000000..b19e13629 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md @@ -0,0 +1,244 @@ +# EngineMT-QA + +EngineMT-QA asks questions about simulated aero-engine flights. Each question is about one 600-step cycle +or a group of 10 cycles, of 33 signals from NASA's N-CMAPSS simulator. The authors define four tasks, one +per stage: Understanding, Perception, Reasoning and Decision-Making. They release it with the ITFormer +paper (Wang et al., ICML 2025, [arXiv 2506.20093](https://arxiv.org/abs/2506.20093)) and their code, +[ITFormer-ICML25](https://github.com/Pandalin98/ITFormer-ICML25). + +The connector builds one Record per cycle and one task per published question-answer pair, as the +authors' code reads them. Each statement below names its source: the paper, the dataset card on the +Hub, the released files, or the authors' code at `b80142f` (2026-07-19). + +## The source data + +The release is the Hub dataset +[pandalin98/EngineMT-QA](https://huggingface.co/datasets/pandalin98/EngineMT-QA), pinned at +`ea2051a15512212976c22c1a075ec549c7d48eef`. The authors' audit of channel counts +(`docs/ISSUE_23_CHANNEL_AUDIT.md`) checked the same revision and the same SHA-256 digests. It holds +three files: + +| File | Size | Holds | +| --- | --- | --- | +| `time_series_data.h5` | 18.8 GB | `seq_data`: float64 shaped (118,921, 600, 33); `data_ID`: the ids 1 to 118,921 | +| `train_qa.jsonl` | 40.5 MB | 13,291 lines, 101,565 QA pairs | +| `test_qa.jsonl` | 17.0 MB | 5,539 lines, 42,477 QA pairs | + +**Signals (released files).** Row `i` of `seq_data` has id `i + 1`, so a QA's id `n` is row `n - 1`, +which is how the authors' `TsQaDataset` reads it. The HDF5 file has no attributes and names no channel. +The authors' audit says not to name the 33 channels without a mapping from them, so the connector +doesn't. The values are z-scored: over a sample of 400 rows, every channel has a mean near 0 and a +standard deviation near 1. No row holds NaN or infinity. + +**QA (released files).** Each line has an `id`, a `name`, and a `conversations` list of `human` +questions, each followed by its `gpt` answer. Here is one line, shortened to one of its 15 pairs: + +```json +{ + "id": "7323", + "name": "7323.csv", + "conversations": [ + {"from": "human", "stage": "2", "attribute": "close", + "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, + {"from": "gpt", "value": "a"} + ] +} +``` + +A line has one of two shapes. Every line of the pinned files has one or the other: + +| Line | `id` | Pairs | train lines | test lines | +| --- | --- | --- | --- | --- | +| Single-cycle | one row id | 8 Understanding, then 7 Perception | 5,141 | 2,155 | +| Multi-cycle | a list of 10 row ids | 2 Reasoning, then 1 Decision-Making | 8,150 | 3,384 | + +Every question holds one `` marker where the series goes. + +**The four tasks (paper Section 5.1, Appendix C, and the card).** + +| Stage | Task | Input | Format | Authors' metric | +| --- | --- | --- | --- | --- | +| 1 | Understanding | 1 cycle | open | BLEU, ROUGE | +| 2 | Perception | 1 cycle | multiple choice | accuracy, F1 | +| 3 | Reasoning | 10 cycles | multiple choice | accuracy, F1 | +| 4 | Decision-Making | 10 cycles | open | BLEU, ROUGE | + +Understanding asks what the change in a named signal means, such as altitude or the total +temperature at the HPT outlet. Its wording varies from line to line. Decision-Making asks for +maintenance actions. The closed questions have fixed wording. Nine questions exist, and each is +published with exactly one text and one option list (released files): + +| Stage | Question | Options | +| --- | --- | --- | +| 2 | Health status of the fan, LPC, HPC, HPT, LPT (5 questions) | a: Health, b: Fault | +| 2 | Fault cause: "caused by efficiency or flow problems?" | a to f: HPT, LPT, Fan efficiency modifier; HPT, LPT, Fan flow modifier | +| 2 | Potential health state | a: Operating Normally, b: Abnormal Degradation, c: Imminent Failure | +| 3 | Qualitative condition of the engine | a to e: Good, Moderate, Poor, Bad, Extremely Poor Condition | +| 3 | Probability range of engine failure | a to e: 1%-10%, 10%-30%, 30%-50%, 50%-70%, 70%-100% | + +An answer is one letter, except for the fault cause. Its answers hold 1, 2, 3 or all 6 letters, joined +by `", "`, such as `a, b, e`. + +## The TimeNet dataset + +### Records + +Each HDF5 row becomes one Record: + +- One Source, `N-CMAPSS engine`, with 33 signals, `channel_00` to `channel_32`, in `seq_data` column + order. The values are the published float64 values, unchanged. They are z-scored and have no unit. +- Each signal has 600 values on a regular 1 Hz axis that starts at 0. The paper describes the inputs as + "10-minute (600 points) time-series segments" (Section 5.2). The HDF5 file states no rate. +- Typed metadata, `EngineMTQARecordMetadata`, which holds `row_id`: the row's `data_ID`, the id the QA + files use. The release names no engine unit or flight, so the shared `subject_id` is unset. + +By default the dataset holds the 7,755 rows that a QA pair names. With `EngineMTQAConnector.ALL_ROWS = +True` it holds all 118,921 rows, which the authors' encoder pretraining reads (`train_pretrain.py` reads +every row of `seq_data`). The extra Records have no task. + +A row named by several lines, in train and in test, is one Record that their tasks share. + +### Tasks + +Each published QA pair with a label becomes one task. There are four kinds, one per stage, and each +task takes its split from its file: + +| Kind | TimeNet type | Inputs | Target | train | test | +| --- | --- | --- | --- | --- | --- | +| `UnderstandingTask` | `AnswerTask` | 1 Record | the published answer | 41,128 | 17,240 | +| `PerceptionTask` | `ClassificationTask` | 1 Record | the published letters | 34,718 | 14,585 | +| `ReasoningTask` | `ClassificationTask` | 10 Records | the published letter | 16,300 | 6,768 | +| `DecisionTask` | `AnswerTask` | 10 Records | the published answer | 8,150 | 3,384 | + +That is 142,273 tasks: 100,296 train and 41,977 test. The 1,769 Perception pairs with an empty +answer have no task (see [Data limitations](#data-limitations)). + +**Prompt.** The prompt is the published question, unchanged, with its `` marker and its options. +The authors send it as the user message, after the system message `You are a helpful assistant.` +Their `add_adaptive_prompt`, which would append a stage hint, is commented out in `__getitem__`, so the +prompt carries no hint. + +**Closed questions are classifications over letters.** Each closed question always offers the same +options in the same order, so a letter has one meaning per question. The authors' model answers with +the letters, and `closed_question_metrics` scores the set of letters. So the targets are the published +letters, one item per letter, in published order, such as `("a", "b", "e")`. `target_schema` names the +question's vocabulary annotation. Task metadata `closed_question` names the question, such as +`hpc_health` or `fault_cause`. + +**Multi-cycle inputs.** A Reasoning or Decision-Making task has its 10 Records as inputs, in published +order. The order is the release's, which isn't sorted by id in 256 of the 11,534 lists. The authors' +model reads only the first 60 steps of each row and concatenates them into one (600, 33) array +(`seq_data[int(i) - 1][: 600 // 10]` in `TsQaDataset.__getitem__`). One task scope can't cut 10 inputs, +so the task holds the whole Records, and its metadata states `steps_per_cycle: 60`. +`model_input.authors_series(task)` rebuilds the authors' array from the task's Records. + +**Metadata.** Every task names its pair as the authors' evaluation does: `source_file`, `source_line` +(0-based) and `qa_offset` (the 0-based pair within the line). + +### Dataset annotations + +| Key | Value | +| --- | --- | +| `source_repository`, `source_revision` | the pinned Hub source | +| `component_health_vocabulary` | `["a", "b"]`; metadata `options` maps a to Health, b to Fault | +| `fault_cause_vocabulary` | `["a", ..., "f"]` with the six modifiers | +| `health_state_vocabulary` | `["a", "b", "c"]` with the three states | +| `engine_condition_vocabulary` | `["a", ..., "e"]` with the five conditions | +| `failure_probability_vocabulary` | `["a", ..., "e"]` with the five ranges | + +The five health-status questions share `component_health_vocabulary`. + +## How the connector follows the authors + +| The authors (code at `b80142f`) | The connector | +| --- | --- | +| `_build_index` flattens each line into one sample per pair | one task per labelled pair | +| `seq_data[int(id) - 1]` | the Record of row id `id` | +| first 60 steps of each of 10 rows, concatenated | 10 input Records, `steps_per_cycle: 60`, `authors_series` | +| SFT target: the published answer | the target: the answer, or its letters | +| `closed_question_metrics`: the set of letters a to f | a `ClassificationTask` over letters | +| train on `train_qa.jsonl`, evaluate on `test_qa.jsonl` | `Split.TRAIN`, `Split.TEST` | +| `sample_index`, `source_line`, `qa_offset` | `source_line`, `qa_offset` in task metadata | + +TimeNet ships no parser or metric for the answers. + +## Comparing with the paper + +- **Size.** The paper says "over 110k" QA pairs (Section 1) and "11k QA pairs" (Section 5.1 and Figure + 3). The card says 87k train and 25k test, 113k in all. The released files hold 101,565 and 42,477, + 144,042 in all, which the authors' README and audit state. The connector follows the files. +- **Channels.** The paper says 32 channels (Sections 1 and 5.2, Appendix D). The release has 33. The + authors' audit says it can't tell which column differs, and that dropping one would be a guess. The + connector keeps all 33. +- **Reasoning.** The paper says Reasoning predicts "the probability of future failures and the + remaining useful life" (Section 5.1). The release has no remaining useful life question: Reasoning asks + for the condition and the failure probability. `utils/metrics.py` has an unused `compute_rul`. +- **Time.** The paper calls an input both a "10-minute (600 points)" segment (Section 5.2) and a cycle + of "600 time steps" (Appendix D). N-CMAPSS flights last hours. Within a row, channel 0 spans a median + of 0.28 standard deviations, against -1.89 to 2.28 across 2,000 rows. So a row is a window of a + flight, not a whole flight resampled. The 1 Hz axis rests on the paper. + +## Data limitations + +- **Empty closed answers.** 1,769 Perception answers are the empty string: the fault cause in 1,054 + train and 422 test pairs, and the health state in 215 train and 78 test pairs. They are missing labels. + An empty fault cause appears whether or not all five components are healthy, so it doesn't mean "no + fault". The health state takes exactly one option. The authors trained on these pairs, and their + metric before `b80142f` scored an empty answer as an empty set. Their current metric raises on an + empty reference, so it fails on the test file's Perception references, 500 of which are empty. The connector + leaves them out and pins their count per question and file. +- **Train and test share rows.** 3,223 of the 7,755 rows appear in both files. They include 1,350 rows + that a single-cycle line names in each file, and all 1,900 rows of the multi-cycle lists, which both + files draw from. 2,689 of the 3,384 test lists of 10 ids (79%) are also train lists, and each has the + same two Reasoning answers in both files. The split is the authors' and is kept. A test score is not a + score on unseen signals, and for Reasoning it is mostly a score on train questions. +- **Repeated id.** Two train lines, 2,003 and 10,533 (1-based), are both about row id 40034. They ask + different questions, and both are kept. +- **Simulated, normalized, unnamed.** The signals come from a simulator, are z-scored with statistics + the release doesn't give, and have no channel names. The questions name signals such as altitude, + but no Record says which channel is which. +- **Generated text.** The paper says an LLM rewrote the questions and answers (ChatGPT, DeepSeek), and + experts checked them (Appendix A). The Understanding and Decision-Making answers vary in wording for + the same content. + +## Data flow and validation + +```text +time_series_data.h5 --check_layout--> RowLoader --> one lazy Record per row +train_qa.jsonl, test_qa.jsonl --read_pairs--> QAPair --> one task per pair +``` + +1. `signals.check_layout` checks the HDF5 file. It must hold exactly `seq_data` and `data_ID`, of the + published shape and dtype, with the ids 1 to 118,921 in order. +2. `qa.read_pairs` validates every line strictly. The keys are exact, and `name` must repeat `id`. A + list of ids holds 10 distinct ids. The conversation alternates `human` and `gpt` turns. A stage + decides the format (`open` or `close`) and the number of rows. Each question holds one ``. A + closed question must be one of the nine, with its exact text. A closed answer must be distinct + letters of its options, and one letter where one is expected. An open answer must not be empty. +3. The loader counts each file's lines, pairs per stage, and empty answers by question, and compares + them with `release.PINNED_RELEASE`. It also compares the number of rows that QAs name. Any + difference raises `TimeFValidationError`. +4. Each signal reads its row when the writer asks for it, one row per Record. A row that the file no + longer holds raises `TimeFFormatError`. + +## Open questions + +- **License.** The Hub metadata says MIT, but the card's License section is an unfilled placeholder. + The repository's `LICENSE` is MIT and covers the code. The paper names no data license. The card + here says MIT. Confirm with the authors. +- **Channel names.** Which N-CMAPSS variable each of the 33 channels holds, and which one the paper's + 32 leaves out. +- **Sampling rate.** Whether a row is a 10-minute window at 1 Hz, as the paper says, and where in its + flight the window starts. +- **Cycles.** Whether the 10 rows of a multi-cycle question are consecutive cycles of one engine unit. + The release names neither the unit nor the cycle. + +## Local build + +```bash +uv run timenet-build build pandalin98/enginemt-qa --out .timenet-registry +``` + +The build downloads 18.8 GB. The default dataset holds 7,755 Records and 142,273 tasks, and takes about +900 MB as TimeF. Writing it takes about a minute after the download. With `ALL_ROWS = True` it holds +118,921 Records and the same tasks, and takes about 13 GB and 12 minutes. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py new file mode 100644 index 000000000..0f217e8c8 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py @@ -0,0 +1,6 @@ +"""EngineMT-QA: aero-engine question answering over simulated N-CMAPSS cycles, in four tasks.""" + +from timenet_connectors.datasets.pandalin98.enginemt_qa.connector import ( + CONNECTOR as CONNECTOR, + EngineMTQAConnector as EngineMTQAConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py new file mode 100644 index 000000000..5b838285a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py @@ -0,0 +1,54 @@ +"""Convert the pinned EngineMT-QA files into lazy Records and one task per QA pair. + +Each HDF5 row a QA names becomes a Record whose 33 signals read the row on demand, and each labelled QA +pair becomes the task of its stage. README.md explains how this follows the authors. + +Metadata is provenance and never model input. +""" + +from collections.abc import Mapping +from typing import ClassVar + +from timenet.dataset import TimeFDataset +from timenet.types import Annotation +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.datasets.pandalin98.enginemt_qa import release +from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import AnnotationKey +from timenet_connectors.datasets.pandalin98.enginemt_qa.loader import EngineMTQADatasetLoader +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import Part, PinnedRelease + + +class EngineMTQAConnector(BaseHuggingFaceConnector): + """Build EngineMT-QA from the pinned files of its Hugging Face release.""" + + HF_REPO = release.HF_REPO + SOURCE_REVISION = release.SOURCE_REVISION + SOURCE_FILES: ClassVar[Mapping[str, str]] = {part.value: part.source_file for part in Part} + SOURCE_RELEASE: ClassVar[PinnedRelease] = release.PINNED_RELEASE + """What the pinned files hold, which every build checks.""" + ALL_ROWS: ClassVar[bool] = False + """Build a Record for every one of the 118,921 HDF5 rows, as the authors' encoder pretraining reads + them, rather than for the 7,755 rows a QA names. The extra Records have no task.""" + + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Convert the files into one lazy Record per row, and a stream of one task per labelled QA pair. + + The connector adds the pinned source's provenance. ``EngineMTQADatasetLoader`` checks the HDF5 + layout, reads and checks both QA files, and builds the Records; its ``populate`` adds them and + streams the tasks, not holding them. A malformed file, or counts that differ from + :attr:`SOURCE_RELEASE`, raise ``TimeFValidationError``. + + Returns: + A dataset with the provenance and vocabulary annotations, the Records, and a stream of every task. + """ + dataset = TimeFDataset(metadata=self.metadata()) + for key, value in ( + (AnnotationKey.SOURCE_REPOSITORY, self.HF_REPO), + (AnnotationKey.SOURCE_REVISION, self.SOURCE_REVISION), + ): + dataset.annotate(Annotation(key=key, value=value)) + loader = EngineMTQADatasetLoader.parse_from(raw_refs, all_rows=self.ALL_ROWS, release=self.SOURCE_RELEASE) + return loader.populate(dataset) + + +CONNECTOR = EngineMTQAConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml new file mode 100644 index 000000000..570ecafc5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml @@ -0,0 +1,33 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# Dataset Card for the pandalin98/enginemt-qa connector: the single source for this dataset's metadata. +# +# Training convention: the tasks are the authors' four stages, one task per published QA pair, split as +# the authors train and test (train_qa.jsonl, test_qa.jsonl). Closed questions are classifications over +# their option letters, as the authors' model answers and their metric scores. 1,769 closed pairs with an +# empty answer have no task. Signals are the release's z-scored values at 1 Hz, 33 unnamed channels per +# 600-step cycle. +yaml_schema_version: 1 +dataset_id: pandalin98/enginemt-qa +dataset_version: 1.0.0 +name: EngineMT-QA +description: >- + Aero-engine time-series question answering over simulated N-CMAPSS flight cycles: 142,273 labelled + QA pairs (100,296 train, 41,977 test) in four tasks. Understanding and Perception ask about one + 600-step cycle of 33 z-scored channels; Reasoning and Decision-Making ask about 10 cycles. + Perception and Reasoning are multiple choice; Understanding and Decision-Making are open-ended + answers written with LLMs and checked by experts. +license: MIT +source_url: https://huggingface.co/datasets/pandalin98/EngineMT-QA +citation: >- + Wang et al. ITFormer: Bridging Time Series and Natural Language for Multi-Modal QA with + Large-Scale Multitask Dataset. ICML 2025. https://arxiv.org/abs/2506.20093 +domains: + - industrial +tags: + - aero-engine + - n-cmapss + - simulated + - question-answering + - classification + - generated-supervision + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/keys.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/keys.py new file mode 100644 index 000000000..8e3f09a66 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/keys.py @@ -0,0 +1,33 @@ +"""The names EngineMT-QA writes into a dataset, so writing and reading them share one spelling.""" + +from enum import StrEnum, unique + +from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import Vocabulary + + +@unique +class AnnotationKey(StrEnum): + """The key of an annotation the connector builds.""" + + SOURCE_REPOSITORY = "source_repository" + SOURCE_REVISION = "source_revision" + + +def vocabulary_key(vocabulary: Vocabulary) -> str: + """Give the key of the dataset annotation that holds a closed question's option letters. + + Returns: + The key, which each task of the question also states as its ``target_schema``. + """ + return f"{vocabulary}_vocabulary" + + +def channel_name(index: int) -> str: + """Name a ``seq_data`` channel by its index, since the release names none: ``channel_00`` to ``channel_32``. + + The names are zero-padded, so their sorted order is the source's. + + Returns: + The signal name. + """ + return f"channel_{index:02d}" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py new file mode 100644 index 000000000..30d7b4a4c --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py @@ -0,0 +1,142 @@ +"""Load EngineMT-QA's three files into one lazy Record per row and one task per QA pair.""" + +from collections import Counter +from collections.abc import Iterable, Iterator, Mapping +from pathlib import Path +from typing import Self + +from timenet.dataset import Record +from timenet.errors import TimeFValidationError +from timenet.types import Annotation +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.bases.loader import DatasetLoader +from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import vocabulary_key +from timenet_connectors.datasets.pandalin98.enginemt_qa.qa import QAPair, read_pairs +from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import Vocabulary +from timenet_connectors.datasets.pandalin98.enginemt_qa.records import RowLoader, build_record +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import ( + PINNED_RELEASE, + QA_PARTS, + Part, + PinnedInventory, + PinnedRelease, +) +from timenet_connectors.datasets.pandalin98.enginemt_qa.signals import check_layout +from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import KIND_OF_STAGE, TASK_KINDS, EngineMTQATask + + +class EngineMTQADatasetLoader(DatasetLoader[EngineMTQATask]): + """EngineMT-QA's files, read and checked: one lazy Record per row, and one task per QA pair. + + EngineMT-QA defines four tasks, the paper's stages, each under the authors' split: + + - Understanding (:class:`~.tasks.UnderstandingTask`): describe one cycle's change in a named signal. + - Perception (:class:`~.tasks.PerceptionTask`): from one cycle, choose a component's health, the + fault's causes, or the engine's health state. + - Reasoning (:class:`~.tasks.ReasoningTask`): from 10 cycles, choose the engine's condition or its + failure probability range. + - Decision-making (:class:`~.tasks.DecisionTask`): from 10 cycles, recommend maintenance. + + Select one kind, or one kind of one split, with ``tasks_of``, such as + ``tasks_of(PerceptionTask, split=Split.TEST)``. + + The loader keeps the QA pairs, not the values: the signals read their row from the HDF5 file, so the + files must stay available until writing ends. + """ + + TASK_KINDS = TASK_KINDS + + def __init__(self, pairs: tuple[QAPair, ...], records: Mapping[int, Record]) -> None: + self._pairs = pairs + self._records = records + + @classmethod + def parse_from( + cls, refs: Iterable[HubFileRef], *, all_rows: bool = False, release: PinnedRelease = PINNED_RELEASE + ) -> Self: + """Read and check the three files once, and build one lazy Record per row. + + A closed QA pair with an empty answer has no label, so it gets no task; ``release`` pins how many + there are. Every other malformed pair, and any count that differs from ``release``, raises. + + Args: + refs: The downloaded files, one per part. + all_rows: Build a Record for every HDF5 row, as the authors' encoder pretraining reads them, + rather than only for the rows a QA names. + release: What the files must hold. + + Returns: + The loaded release. + + Raises: + TimeFValidationError: If a part is missing or unknown, or the files differ from ``release``. + """ + files = list(refs) + paths = {Part(ref.part): ref.path for ref in files if ref.part in set(Part)} + if set(paths) != set(Part) or len(paths) != len(files): + raise TimeFValidationError(f"EngineMT-QA loads exactly the parts {[part.value for part in Part]}") + signals = paths[Part.SIGNALS] + check_layout(signals, n_rows=release.rows) + pairs: list[QAPair] = [] + for part in QA_PARTS: + pairs.extend(_read_labelled_pairs(paths[part], part, release.qa[part])) + referenced = sorted({row for pair in pairs for row in pair.rows}) + release.check_referenced_rows(len(referenced)) + if referenced[-1] > release.rows: + raise TimeFValidationError(f"a QA names row id {referenced[-1]}, past the file's {release.rows} rows") + # One loader for the Records of this load, so a row's channels share one read. + rows = RowLoader(signals) + row_ids = range(1, release.rows + 1) if all_rows else referenced + return cls(tuple(pairs), {row_id: build_record(row_id, rows) for row_id in row_ids}) + + def dataset_annotations(self) -> tuple[Annotation, ...]: # noqa: PLR6301 - the loader interface + """Return each closed question's option letters, with the option each letter names. + + A closed task names its vocabulary's key as its ``target_schema``. The connector adds the provenance. + + Returns: + One vocabulary per option list. + """ + return tuple( + Annotation( + key=vocabulary_key(vocabulary), + value=list(vocabulary.letters), + description="The option letters a closed question offers. Its answer is one or more of them.", + metadata={"options": dict(zip(vocabulary.letters, vocabulary.options, strict=True))}, + ) + for vocabulary in Vocabulary + ) + + def records(self) -> Iterator[Record]: + """Return one Record per row, in row id order, whose values stay in the HDF5 file until written.""" + return iter(self._records.values()) + + def tasks(self) -> Iterator[EngineMTQATask]: + """Return one task per labelled QA pair: the train file's pairs, then the test file's, in file order. + + Yields: + The tasks. + """ + for pair in self._pairs: + yield KIND_OF_STAGE[pair.stage](pair, tuple(self._records[row] for row in pair.rows)) + + +def _read_labelled_pairs(path: Path, part: Part, inventory: PinnedInventory) -> list[QAPair]: + """Read one QA file's pairs, leave out the closed ones with an empty answer, and check the inventory. + + Returns: + The labelled pairs, in file order. + """ + lines: set[int] = set() + per_stage: Counter[int] = Counter() + empty: Counter[str] = Counter() + labelled = [] + for pair in read_pairs(path, part): + lines.add(pair.ref.line) + per_stage[pair.stage.value] += 1 + if pair.closed is not None and not pair.answer: + empty[pair.closed.value] += 1 + else: + labelled.append(pair) + inventory.check(path, PinnedInventory(len(lines), dict(per_stage), dict(empty))) + return labelled diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/model_input.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/model_input.py new file mode 100644 index 000000000..6a3a50422 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/model_input.py @@ -0,0 +1,51 @@ +"""Rebuild the series the authors' model reads for a task, from the task's Records. + +The authors' ``TsQaDataset`` gives the model one array shaped (600, 33) per QA pair. A single-cycle +pair reads its row whole. A multi-cycle pair reads the first :data:`~.tasks.STEPS_PER_CYCLE` steps of +each of its 10 rows and concatenates them in published order. A task holds the whole Records instead, +so :func:`authors_series` cuts and joins them the same way. +""" + +import numpy as np + +from timenet.dataset import Record +from timenet.errors import TimeFValidationError +from timenet.types import Task +from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import channel_name +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import MULTI_CYCLE_ROWS, N_CHANNELS +from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import STEPS_PER_CYCLE + + +_CHANNELS = tuple(channel_name(index) for index in range(N_CHANNELS)) + + +def record_values(record: Record) -> np.ndarray: + """Return a Record's values as the HDF5 row holds them. + + Returns: + The values shaped (steps, channels), the channels in source order. + + Raises: + TimeFValidationError: If the Record doesn't hold the 33 EngineMT-QA channels. + """ + signals = {signal.name: signal for signal in record.signals} + if tuple(sorted(signals)) != _CHANNELS: + raise TimeFValidationError("the Record is not an EngineMT-QA Record") + return np.stack([signals[name].to_numpy() for name in _CHANNELS], axis=1) + + +def authors_series(task: Task) -> np.ndarray: + """Rebuild the array the authors' ``TsQaDataset`` feeds the model for this task. + + Returns: + The values shaped (600, 33): one row whole, or the first 60 steps of each of 10 rows, joined. + + Raises: + TimeFValidationError: If the task names neither one nor 10 Records. + """ + records = task.inputs + if len(records) == 1: + return record_values(records[0]) + if len(records) == MULTI_CYCLE_ROWS: + return np.concatenate([record_values(record)[:STEPS_PER_CYCLE] for record in records]) + raise TimeFValidationError(f"an EngineMT-QA task names 1 or {MULTI_CYCLE_ROWS} Records, not {len(records)}") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/qa.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/qa.py new file mode 100644 index 000000000..5a1edc41d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/qa.py @@ -0,0 +1,183 @@ +"""Read the published QA files: one JSON line per row or 10-row group, flattened into QA pairs. + +A line is ``{"id": ..., "name": ..., "conversations": [question, answer, ...]}``. ``id`` names one HDF5 +row (a string such as ``"7323"``) or ten (a list), and ``name`` repeats each as ``".csv"``. The +conversation alternates a ``human`` question, carrying its ``stage`` and ``attribute``, and a ``gpt`` +answer. The authors' ``TsQaDataset._build_index`` flattens every line into one sample per pair, in file +order, and :func:`read_pairs` does the same. +""" + +from collections.abc import Iterator +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, StringConstraints, ValidationError, model_validator + +from timenet.errors import TimeFValidationError +from timenet.types import Split +from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import ClosedQuestion, Stage +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import MULTI_CYCLE_ROWS, N_ROWS, Part, split_of + + +# A row id is a decimal string, as published; the HDF5 row is the id minus one. +RowId = Annotated[StrictStr, StringConstraints(pattern=r"^[1-9][0-9]*$")] +Text = Annotated[StrictStr, StringConstraints(pattern=r"\S")] + + +class _Question(BaseModel): + model_config = ConfigDict(strict=True, frozen=True, extra="forbid", populate_by_name=True) + + speaker: Literal["human"] = Field(alias="from") + stage: Literal["1", "2", "3", "4"] + attribute: Literal["open", "close"] + value: Text + + +class _Answer(BaseModel): + model_config = ConfigDict(strict=True, frozen=True, extra="forbid", populate_by_name=True) + + speaker: Literal["gpt"] = Field(alias="from") + value: StrictStr + """Empty for some closed answers, as published.""" + + +class QALine(BaseModel): + """One published line, validated: its rows, and its questions with their answers.""" + + model_config = ConfigDict(strict=True, frozen=True, extra="forbid") + + id: RowId | list[RowId] + name: StrictStr | list[StrictStr] + conversations: list[_Question | _Answer] = Field(min_length=2) + + @model_validator(mode="after") + def _check(self) -> "QALine": + ids = self.id if isinstance(self.id, list) else [self.id] + names = self.name if isinstance(self.name, list) else [self.name] + if isinstance(self.id, list) != isinstance(self.name, list) or names != [f"{i}.csv" for i in ids]: + raise TimeFValidationError(f"name {self.name!r} doesn't repeat id {self.id!r}") + if isinstance(self.id, list) and len(ids) != MULTI_CYCLE_ROWS: + raise TimeFValidationError(f"a multi-cycle id names {len(ids)} rows, not {MULTI_CYCLE_ROWS}") + if len(set(ids)) != len(ids) or not all(int(i) <= N_ROWS for i in ids): + raise TimeFValidationError(f"id {self.id!r} repeats a row or names one past {N_ROWS}") + self.pairs() + return self + + @property + def rows(self) -> tuple[int, ...]: + """Return the row ids the line names, in published order.""" + return tuple(int(i) for i in (self.id if isinstance(self.id, list) else [self.id])) + + def pairs(self) -> list[tuple[_Question, _Answer]]: + """Return the conversation's questions, each with its answer, in published order. + + Raises: + TimeFValidationError: If the conversation doesn't alternate a question and an answer. + """ + turns = self.conversations + if len(turns) % 2: + raise TimeFValidationError("conversations must alternate a human question and a gpt answer") + pairs = [] + for question, answer in zip(turns[::2], turns[1::2], strict=True): + if not isinstance(question, _Question) or not isinstance(answer, _Answer): + raise TimeFValidationError("conversations must alternate a human question and a gpt answer") + pairs.append((question, answer)) + return pairs + + +@dataclass(frozen=True) +class QARef: + """Where one QA pair was published: its file, its 0-based line, and its 0-based pair in the line. + + ``line`` and ``qa_offset`` are the authors' ``source_line`` and ``qa_offset``. + """ + + part: Part + line: int + qa_offset: int + + @property + def split(self) -> Split: + """Return the pair's split: its file's.""" + return split_of(self.part) + + @property + def context(self) -> str: + """Return where the pair is, for error messages.""" + return f"{self.part.source_file} line {self.line + 1}, pair {self.qa_offset}" + + +@dataclass(frozen=True) +class QAPair: + """One published question and its answer, with the rows it is about.""" + + ref: QARef + rows: tuple[int, ...] + """The row ids, one for a single-cycle question and ten for a multi-cycle one, in published order.""" + stage: Stage + question: str + """The question exactly as published, with its ```` marker.""" + answer: str + """The answer exactly as published: free text, or letters for a closed question.""" + closed: ClosedQuestion | None + """The closed question, or None for an open one.""" + + @property + def letters(self) -> tuple[str, ...]: + """Return a closed answer's letters, in published order; empty for a missing answer. + + Raises: + TimeFValidationError: If the pair is an open question. + """ + if self.closed is None: + raise TimeFValidationError(f"{self.ref.context}: an open question has no letters") + return self.closed.parse_answer(self.answer) if self.answer else () + + +def _pair(ref: QARef, rows: tuple[int, ...], question: _Question, answer: _Answer) -> QAPair: + """Check a question against its stage's format and number of rows, and pair it with its answer. + + Returns: + The pair. + + Raises: + TimeFValidationError: If the stage, attribute and rows disagree, a closed question is unknown, or an + answer is malformed. + """ + stage = Stage(int(question.stage)) + if (question.attribute == "close") != stage.is_closed: + raise TimeFValidationError(f"stage {stage} questions are not {question.attribute!r}") + if (len(rows) > 1) != stage.is_multi_cycle: + raise TimeFValidationError(f"a stage {stage} question names {len(rows)} rows") + if question.value.count("") != 1: + raise TimeFValidationError("a question must hold one marker") + closed = ClosedQuestion.of(stage, question.value) if stage.is_closed else None + if closed is None and not answer.value.strip(): + raise TimeFValidationError("an open question has an empty answer") + if closed is not None and answer.value: + # Parse once here, so a malformed answer fails the read, not the task stream. + closed.parse_answer(answer.value) + return QAPair(ref, rows, stage, question.value, answer.value, closed) + + +def read_pairs(path: Path, part: Part) -> Iterator[QAPair]: + """Validate every line of one QA file and flatten it into pairs, in file order. + + Yields: + Each published QA pair, empty answers included. + + Raises: + TimeFValidationError: If a line or pair is malformed, with file, line and pair. + """ + with path.open(encoding="utf-8") as lines: + for number, text in enumerate(lines): + ref = QARef(part, number, 0) + try: + line = QALine.model_validate(json.loads(text)) + for offset, (question, answer) in enumerate(line.pairs()): + ref = QARef(part, number, offset) + yield _pair(ref, line.rows, question, answer) + except (ValidationError, ValueError) as exc: + raise TimeFValidationError(f"{ref.context}: {exc}") from exc diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/questions.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/questions.py new file mode 100644 index 000000000..4b67659d5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/questions.py @@ -0,0 +1,225 @@ +"""EngineMT-QA's four stages and its nine closed questions, each with its option letters. + +The paper's four tasks are the release's ``stage`` values. Stages 1 and 4 are open questions, scored with +BLEU and ROUGE; stages 2 and 3 are closed, scored with accuracy and F1 (paper Section 5.1). + +Each closed question is published with one text and one option list in every row, so a letter always +means the same option. The authors' model answers with the letters, such as ``a, b, e``, and their +``closed_question_metrics`` scores the set of letters. So the letters are the classes, and each closed +question's :class:`Vocabulary` names the option each letter stands for. +""" + +from enum import IntEnum, StrEnum, unique +import re +from typing import NamedTuple + +from timenet.errors import TimeFValidationError + + +@unique +class Stage(IntEnum): + """The release's ``stage``: one of the paper's four tasks.""" + + UNDERSTANDING = 1 + PERCEPTION = 2 + REASONING = 3 + DECISION_MAKING = 4 + + @property + def is_closed(self) -> bool: + """Return whether the stage asks closed questions, which the release marks ``close``.""" + return self in {Stage.PERCEPTION, Stage.REASONING} + + @property + def is_multi_cycle(self) -> bool: + """Return whether the stage's questions name 10 cycles rather than one.""" + return self in {Stage.REASONING, Stage.DECISION_MAKING} + + +class _Options(NamedTuple): + options: tuple[str, ...] + """The option names, in letter order from ``a``.""" + multiple: bool + """Whether an answer may hold several letters.""" + + +@unique +class Vocabulary(StrEnum): + """The option list of one or more closed questions: its letters are the classes.""" + + COMPONENT_HEALTH = "component_health" + FAULT_CAUSE = "fault_cause" + HEALTH_STATE = "health_state" + ENGINE_CONDITION = "engine_condition" + FAILURE_PROBABILITY = "failure_probability" + + @property + def options(self) -> tuple[str, ...]: + """Return the option names, in letter order from ``a``, as the question text spells them.""" + return _VOCABULARIES[self].options + + @property + def letters(self) -> tuple[str, ...]: + """Return the option letters, ``a`` first.""" + return tuple("abcdef"[: len(self.options)]) + + @property + def multiple(self) -> bool: + """Return whether an answer may hold several letters. Only the fault cause does.""" + return _VOCABULARIES[self].multiple + + +_VOCABULARIES = { + Vocabulary.COMPONENT_HEALTH: _Options(("Health", "Fault"), multiple=False), + Vocabulary.FAULT_CAUSE: _Options( + ( + "HPT efficiency modifier", + "LPT efficiency modifier", + "Fan efficiency modifier", + "HPT flow modifier", + "LPT flow modifier", + "Fan flow modifier", + ), + multiple=True, + ), + Vocabulary.HEALTH_STATE: _Options( + ("Operating Normally", "Abnormal Degradation", "Imminent Failure"), multiple=False + ), + Vocabulary.ENGINE_CONDITION: _Options( + ("Good Condition", "Moderate Condition", "Poor Condition", "Bad Condition", "Extremely Poor Condition"), + multiple=False, + ), + Vocabulary.FAILURE_PROBABILITY: _Options(("1%-10%", "10%-30%", "30%-50%", "50%-70%", "70%-100%"), multiple=False), +} + + +class _Closed(NamedTuple): + stage: Stage + vocabulary: Vocabulary + text: str + + +def _health_text(component: str) -> str: + return f"What is the health status of {component} of the given engine signal in one cycle?\n a: Health\n b: Fault\n" + + +_REASONING_PREFIX = ( + "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different " + "health states and performing temporal inference to predict the health decline trend, " +) + + +@unique +class ClosedQuestion(StrEnum): + """A closed question, which the release always publishes with the same text.""" + + FAN_HEALTH = "fan_health" + LPC_HEALTH = "lpc_health" + HPC_HEALTH = "hpc_health" + HPT_HEALTH = "hpt_health" + LPT_HEALTH = "lpt_health" + FAULT_CAUSE = "fault_cause" + HEALTH_STATE = "health_state" + ENGINE_CONDITION = "engine_condition" + FAILURE_PROBABILITY = "failure_probability" + + @property + def stage(self) -> Stage: + """Return the question's stage.""" + return _CLOSED[self].stage + + @property + def vocabulary(self) -> Vocabulary: + """Return the question's option list.""" + return _CLOSED[self].vocabulary + + @property + def text(self) -> str: + """Return the question exactly as published, ```` and options included.""" + return _CLOSED[self].text + + @classmethod + def of(cls, stage: Stage, text: str) -> "ClosedQuestion": + """Identify a published closed question by its exact text. + + Returns: + The question. + + Raises: + TimeFValidationError: If no closed question of the stage has this text. + """ + question = _BY_TEXT.get(text) + if question is None or question.stage is not stage: + raise TimeFValidationError(f"unknown stage {stage} closed question {text[:80]!r}") + return question + + def parse_answer(self, answer: str) -> tuple[str, ...]: + """Parse a published answer, such as ``a`` or ``a, b, e``, into its letters. + + Returns: + The letters, in published order. + + Raises: + TimeFValidationError: If the answer isn't distinct letters of the question's options, joined by + ``", "``, or holds several letters where one is expected. + """ + letters = tuple(answer.split(", ")) + vocabulary = self.vocabulary + if not set(letters) <= set(vocabulary.letters) or len(set(letters)) != len(letters): + raise TimeFValidationError(f"{self}: answer {answer!r} is not distinct letters of {vocabulary.letters}") + if len(letters) > 1 and not vocabulary.multiple: + raise TimeFValidationError(f"{self}: answer {answer!r} holds several letters, where one is expected") + return letters + + +_CLOSED = { + ClosedQuestion.FAN_HEALTH: _Closed(Stage.PERCEPTION, Vocabulary.COMPONENT_HEALTH, _health_text("fan")), + ClosedQuestion.LPC_HEALTH: _Closed(Stage.PERCEPTION, Vocabulary.COMPONENT_HEALTH, _health_text("LPC")), + ClosedQuestion.HPC_HEALTH: _Closed(Stage.PERCEPTION, Vocabulary.COMPONENT_HEALTH, _health_text("HPC")), + ClosedQuestion.HPT_HEALTH: _Closed(Stage.PERCEPTION, Vocabulary.COMPONENT_HEALTH, _health_text("HPT")), + ClosedQuestion.LPT_HEALTH: _Closed(Stage.PERCEPTION, Vocabulary.COMPONENT_HEALTH, _health_text("LPT")), + ClosedQuestion.FAULT_CAUSE: _Closed( + Stage.PERCEPTION, + Vocabulary.FAULT_CAUSE, + "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or " + "flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency " + "modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n", + ), + ClosedQuestion.HEALTH_STATE: _Closed( + Stage.PERCEPTION, + Vocabulary.HEALTH_STATE, + "What is the potential health state of the given engine signal in one cycle?\n a: Operating " + "Normally\n b: Abnormal Degradation\n c: Imminent Failure\n", + ), + # The published text has two spaces after "a:". + ClosedQuestion.ENGINE_CONDITION: _Closed( + Stage.REASONING, + Vocabulary.ENGINE_CONDITION, + _REASONING_PREFIX + "what is the qualitative condition of the engine? a: Good Condition\n b: Moderate " + "Condition\n c: Poor Condition\n d: Bad Condition\n e: Extremely Poor Condition\n", + ), + ClosedQuestion.FAILURE_PROBABILITY: _Closed( + Stage.REASONING, + Vocabulary.FAILURE_PROBABILITY, + _REASONING_PREFIX + "what is the probability range of engine failure? a: 1%-10%\n b: 10%-30%\n " + "c: 30%-50%\n d: 50%-70%\n e: 70%-100%\n", + ), +} +_BY_TEXT = {closed.text: question for question, closed in _CLOSED.items()} + +_OPTION = re.compile(r"([a-f]):\s+([^\n]+)\n") + + +def _check_texts() -> None: + """Check at import that each question's text offers exactly its vocabulary's options. + + Raises: + TimeFValidationError: If a text and its vocabulary disagree. + """ + for question, closed in _CLOSED.items(): + offered = tuple(_OPTION.findall(closed.text.split("", 1)[1].split("?")[-1])) + if offered != tuple(zip(closed.vocabulary.letters, closed.vocabulary.options, strict=True)): + raise TimeFValidationError(f"{question}: its text offers {offered}, not its vocabulary") + + +_check_texts() diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py new file mode 100644 index 000000000..3827a920a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py @@ -0,0 +1,93 @@ +"""One Record per HDF5 row: its 33 channels as lazy signals, and its row id in typed metadata.""" + +from functools import lru_cache, partial +from pathlib import Path +from typing import Annotated + +import pyarrow as pa +from pydantic import Field, StrictInt + +from timenet.dataset import Record, Signal, Source +from timenet.dataset.axis import RegularAxis +from timenet.types import TimeSeriesSpec +from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import channel_name +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import N_CHANNELS, N_STEPS, SAMPLE_RATE_HZ +from timenet_connectors.datasets.pandalin98.enginemt_qa.signals import read_row +from timenet_connectors.metadata import RecordMetadata + + +SOURCE_NAME = "N-CMAPSS engine" + +# The authors z-score the values and publish neither the units nor the statistics. +_SPEC = TimeSeriesSpec( + spec_type="enginemt_qa_channel", name="EngineMT-QA channel, z-scored", unit_value=None, dtype="float64" +) + + +class EngineMTQARecordMetadata(RecordMetadata): + """EngineMT-QA's Record metadata: the row the Record holds. + + The release names no engine unit or flight cycle, so the shared ``subject_id`` stays unset. + """ + + row_id: Annotated[StrictInt, Field(ge=1)] + """The row's ``data_ID``, which QAs name as their ``id``; the HDF5 row is ``row_id - 1``.""" + + +def record_metadata(record: Record) -> EngineMTQARecordMetadata: + """Read an EngineMT-QA Record's typed metadata back. + + Returns: + The row the Record holds. + + Raises: + TimeFValidationError: If the Record has no EngineMT-QA metadata. + """ # noqa: DOC502 - raised by from_json + return EngineMTQARecordMetadata.from_json(record.metadata) + + +class RowLoader: + """Load one channel of a row lazily, reading each row once for all its channels. + + A loader serves the Records of one load. A Record's signals load one after another, so one cached + row suffices. + """ + + def __init__(self, path: Path) -> None: + self._path = path + self._read = lru_cache(maxsize=1)(partial(read_row, path)) + + def __call__(self, row_id: int, channel: int) -> pa.Array: + """Return one channel of a row. + + Returns: + The channel's 600 float64 values. + """ + return pa.array(self._read(row_id)[:, channel], type=pa.float64()) + + +def build_record(row_id: int, rows: RowLoader) -> Record: + """Build a row's Record: one Source of 33 lazy signals at 1 Hz, and the row id as metadata. + + Args: + row_id: The row's id. + rows: The loader the signals read their values through. + + Returns: + The Record, whose values stay in the HDF5 file until read. + """ + time_axis = RegularAxis.from_rate_hz(SAMPLE_RATE_HZ) + signals = tuple( + Signal.from_loader( + name=channel_name(channel), + spec=_SPEC, + time_axis=time_axis, + n_values=N_STEPS, + loader=partial(rows, row_id, channel), + ) + for channel in range(N_CHANNELS) + ) + return Record( + sources=(Source(name=SOURCE_NAME, signals=signals),), + metadata=EngineMTQARecordMetadata(row_id=row_id).to_json(), + ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/release.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/release.py new file mode 100644 index 000000000..c5a289b1a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/release.py @@ -0,0 +1,123 @@ +"""The EngineMT-QA release as published: its Hub pin, its three files, their shapes, and their pinned inventory.""" + +from collections.abc import Mapping +from enum import StrEnum, unique +from pathlib import Path +from typing import NamedTuple + +from timenet.errors import TimeFValidationError +from timenet.types import Split + + +HF_REPO = "pandalin98/EngineMT-QA" +SOURCE_REVISION = "ea2051a15512212976c22c1a075ec549c7d48eef" +"""The revision the authors' audit (``docs/ISSUE_23_CHANNEL_AUDIT.md`` at ITFormer-ICML25 ``b80142f``) checked.""" + + +@unique +class Part(StrEnum): + """The release's three files: the signals, and the train and test QA.""" + + SIGNALS = "signals" + TRAIN = "train" + TEST = "test" + + @property + def source_file(self) -> str: + """Return the file's name in the Hub repository, such as ``train_qa.jsonl``.""" + return "time_series_data.h5" if self is Part.SIGNALS else f"{self.value}_qa.jsonl" + + +QA_PARTS = (Part.TRAIN, Part.TEST) +"""The QA files, in the order the loader reads them.""" + + +def split_of(part: Part) -> Split: + """Return the TimeNet split of a QA file: the authors train on ``train_qa`` and test on ``test_qa``. + + Raises: + TimeFValidationError: If the part is not a QA file. + """ + if part not in QA_PARTS: + raise TimeFValidationError(f"{part} is not a QA file") + return Split(part.value) + + +SEQ_DATA = "seq_data" +"""The HDF5 dataset of every cycle's values, shaped ``(rows, steps, channels)``.""" +DATA_ID = "data_ID" +"""The HDF5 dataset of each row's id, which a QA's ``id`` names. It is ``1, 2, ...``: row ``i`` has id ``i + 1``.""" + +N_ROWS = 118_921 +N_STEPS = 600 +N_CHANNELS = 33 +"""The released channel count. The paper says 32; the authors' audit confirms 33 and names none of them.""" + +SAMPLE_RATE_HZ = 1 +"""The paper's "10-minute (600 points) time-series segments" (Section 5.2). The HDF5 states no rate.""" + +MULTI_CYCLE_ROWS = 10 +"""How many rows a multi-cycle QA names.""" + + +class PinnedInventory(NamedTuple): + """One QA file's pinned inventory at :data:`SOURCE_REVISION`.""" + + lines: int + qa_per_stage: Mapping[int, int] + """The published QA pairs per stage, empty answers included.""" + empty_answers: Mapping[str, int] + """The closed QA pairs with an empty answer, by closed question, which the connector skips.""" + + def check(self, path: Path, found: "PinnedInventory") -> None: + """Compare a file's counted inventory with this one. + + Raises: + TimeFValidationError: If the file isn't the pinned revision's. + """ + if (found.lines, dict(found.qa_per_stage), dict(found.empty_answers)) != ( + self.lines, + dict(self.qa_per_stage), + dict(self.empty_answers), + ): + raise TimeFValidationError(f"{path}: EngineMT-QA holds {found}, the pinned file holds {self}") + + +class PinnedRelease(NamedTuple): + """What the pinned release holds, which every build checks.""" + + rows: int + """The HDF5 rows.""" + referenced_rows: int + """The distinct rows the QA files name, which the default build holds.""" + qa: Mapping[Part, PinnedInventory] + """Each QA file's inventory.""" + + def check_referenced_rows(self, found: int) -> None: + """Compare the distinct rows the QA files name with the pinned count. + + Raises: + TimeFValidationError: If the QA files aren't the pinned revision's. + """ + if found != self.referenced_rows: + raise TimeFValidationError(f"the QA files name {found} rows, the pinned files {self.referenced_rows}") + + +# The QA counts are the authors' README's, for test, and their audit's, for train. The empty answers are +# missing labels: an empty fault cause doesn't follow from the five components' health. +PINNED_RELEASE = PinnedRelease( + rows=N_ROWS, + referenced_rows=7_755, + qa={ + Part.TRAIN: PinnedInventory( + lines=13_291, + qa_per_stage={1: 41_128, 2: 35_987, 3: 16_300, 4: 8_150}, + empty_answers={"fault_cause": 1_054, "health_state": 215}, + ), + Part.TEST: PinnedInventory( + lines=5_539, + qa_per_stage={1: 17_240, 2: 15_085, 3: 6_768, 4: 3_384}, + empty_answers={"fault_cause": 422, "health_state": 78}, + ), + }, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt new file mode 100644 index 000000000..ef2960923 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt @@ -0,0 +1,4 @@ +# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class. +huggingface_hub>=1.1 +# h5py reads the signals file, time_series_data.h5. +h5py>=3.10 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/signals.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/signals.py new file mode 100644 index 000000000..4491520ec --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/signals.py @@ -0,0 +1,77 @@ +"""Read the published ``time_series_data.h5``: check its layout once, then read one row at a time. + +The file holds two HDF5 datasets and no attributes: ``seq_data``, every row's values shaped +``(118921, 600, 33)`` in float64, and ``data_ID``, every row's id. The ids are ``1, 2, ...``, so a QA's +id ``i`` is row ``i - 1``, as the authors' ``TsQaDataset`` reads it (``seq_data[int(id) - 1]``). The values +are z-scored, and the file names no channel, so neither does the connector. +""" + +from pathlib import Path +from types import ModuleType + +import numpy as np + +from timenet.errors import TimeFFormatError, TimeFValidationError +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import DATA_ID, N_CHANNELS, N_ROWS, N_STEPS, SEQ_DATA + + +def _h5py() -> ModuleType: + """Import ``h5py`` lazily. Raise an error that states the fix if it is missing. + + Returns: + The imported ``h5py`` module. + + Raises: + ImportError: If ``h5py``, declared in this connector's requirements, is not installed. + """ + try: + import h5py # noqa: PLC0415 + except ImportError as exc: + raise ImportError( + "reading EngineMT-QA needs h5py, declared in this connector's requirements.txt. Run the build " + "without --no-isolation, or install it yourself" + ) from exc + return h5py + + +def check_layout(path: Path, *, n_rows: int = N_ROWS) -> None: + """Check the file holds exactly ``seq_data`` and ``data_ID``, of the published shapes, with ids ``1..n``. + + Args: + path: The downloaded ``time_series_data.h5``. + n_rows: The rows the file must hold. + + Raises: + TimeFValidationError: If the layout or the ids differ from the published file's. + """ + with _h5py().File(path, "r") as file: + if set(file) != {SEQ_DATA, DATA_ID}: + raise TimeFValidationError(f"{path}: expected {SEQ_DATA} and {DATA_ID}, got {sorted(file)}") + values, ids = file[SEQ_DATA], file[DATA_ID] + if values.shape != (n_rows, N_STEPS, N_CHANNELS) or values.dtype != np.float64: + raise TimeFValidationError(f"{path}: {SEQ_DATA} is {values.dtype} {values.shape}") + if ids.shape != (n_rows,) or not np.array_equal(ids[:], np.arange(1, n_rows + 1)): + raise TimeFValidationError(f"{path}: {DATA_ID} is not the row ids 1 to {n_rows}") + + +def read_row(path: Path, row_id: int) -> np.ndarray: + """Read one row's values. + + Args: + path: The ``time_series_data.h5`` that :func:`check_layout` checked. + row_id: The row's id, which QAs name; the row is ``row_id - 1``. + + Returns: + The row's values, shaped ``(600, 33)``: steps, then channels. + + Raises: + TimeFFormatError: If the file no longer holds the row, as when it changed since conversion. + """ + try: + with _h5py().File(path, "r") as file: + values = np.asarray(file[SEQ_DATA][row_id - 1], dtype=np.float64) + except (OSError, KeyError, IndexError, ValueError) as exc: + raise TimeFFormatError(f"{path}: cannot read row id {row_id}: {exc}") from exc + if values.shape != (N_STEPS, N_CHANNELS): + raise TimeFFormatError(f"{path}: row id {row_id} is shaped {values.shape}; the file changed") + return values diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py new file mode 100644 index 000000000..9ce29409b --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py @@ -0,0 +1,129 @@ +"""EngineMT-QA's four tasks, one per stage, each QA pair built as the TimeNet task it becomes. + +Every task keeps the published question as its prompt, ```` marker included, and its file as its +split. Its metadata names the pair as the authors' evaluation does: the file, the 0-based line +(``source_line``) and the 0-based pair in the line (``qa_offset``). +""" + +from dataclasses import dataclass +from typing import Any, ClassVar + +from timenet.dataset import Record +from timenet.errors import TimeFValidationError +from timenet.types import AnswerTask, ClassificationTask, Split, Task +from timenet_connectors.bases.loader import DatasetTask +from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import vocabulary_key +from timenet_connectors.datasets.pandalin98.enginemt_qa.qa import QAPair +from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import Stage +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import MULTI_CYCLE_ROWS, N_STEPS + + +STEPS_PER_CYCLE = N_STEPS // MULTI_CYCLE_ROWS +"""The steps of each cycle the authors' model reads in a multi-cycle task: the first 60 of 600. + +``TsQaDataset.__getitem__`` concatenates ``seq_data[id - 1][: 600 // 10]`` over the 10 ids, so its input +keeps the single-cycle shape (600, 33). The task's inputs are the 10 whole Records; ``model_input`` +rebuilds the authors' input from them. +""" + + +@dataclass(frozen=True) +class EngineMTQATask(DatasetTask): + """One published QA pair, on the Records of the rows it names, in published order.""" + + STAGE: ClassVar[Stage] + + pair: QAPair + records: tuple[Record, ...] + + @property + def split(self) -> Split: + """Return the task's split: its QA file's.""" + return self.pair.ref.split + + def _metadata(self) -> dict[str, Any]: + ref = self.pair.ref + metadata: dict[str, Any] = { + "source_file": ref.part.source_file, + "source_line": ref.line, + "qa_offset": ref.qa_offset, + } + if self.STAGE.is_multi_cycle: + metadata["steps_per_cycle"] = STEPS_PER_CYCLE + return metadata + + +@dataclass(frozen=True) +class _OpenTask(EngineMTQATask): + """An open question, which the authors score with BLEU and ROUGE against the published answer.""" + + TIMENET_TYPE: ClassVar[type[Task]] = AnswerTask + + def to_timenet_task(self) -> AnswerTask: + """Return the answer task, with the published answer as its target.""" + return AnswerTask( + inputs=self.records, + prompt=self.pair.question, + targets=(self.pair.answer,), + split=self.split, + metadata=self._metadata(), + ) + + +@dataclass(frozen=True) +class _ClosedTask(EngineMTQATask): + """A closed question, whose classes are its option letters, which the authors score as a set.""" + + TIMENET_TYPE: ClassVar[type[Task]] = ClassificationTask + + def to_timenet_task(self) -> ClassificationTask: + """Return the classification task, with the published letters as targets. + + Raises: + TimeFValidationError: If the pair is open, or its answer is empty. + """ + closed, letters = self.pair.closed, self.pair.letters + if closed is None or not letters: + raise TimeFValidationError(f"{self.pair.ref.context}: a closed task needs a published answer") + return ClassificationTask( + inputs=self.records, + prompt=self.pair.question, + targets=letters, + target_schema=vocabulary_key(closed.vocabulary), + split=self.split, + metadata=self._metadata() | {"closed_question": closed.value}, + ) + + +@dataclass(frozen=True) +class UnderstandingTask(_OpenTask): + """Stage 1: describe one cycle's change in a named signal and what it implies.""" + + STAGE: ClassVar[Stage] = Stage.UNDERSTANDING + + +@dataclass(frozen=True) +class PerceptionTask(_ClosedTask): + """Stage 2: from one cycle, name a component's health, the fault's causes, or the engine's health state.""" + + STAGE: ClassVar[Stage] = Stage.PERCEPTION + + +@dataclass(frozen=True) +class ReasoningTask(_ClosedTask): + """Stage 3: from 10 cycles, name the engine's condition or its failure probability range.""" + + STAGE: ClassVar[Stage] = Stage.REASONING + + +@dataclass(frozen=True) +class DecisionTask(_OpenTask): + """Stage 4: from 10 cycles, recommend maintenance actions.""" + + STAGE: ClassVar[Stage] = Stage.DECISION_MAKING + + +TASK_KINDS: tuple[type[EngineMTQATask], ...] = (UnderstandingTask, PerceptionTask, ReasoningTask, DecisionTask) +"""The task kinds, in stage order.""" + +KIND_OF_STAGE = {kind.STAGE: kind for kind in TASK_KINDS} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py new file mode 100644 index 000000000..6575c545a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py @@ -0,0 +1,309 @@ +import json +from pathlib import Path +from typing import Any, ClassVar + +import h5py +import numpy as np +import pytest + +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeFFormatError, TimeFValidationError +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry +from timenet.types import AnswerTask, ClassificationTask, Split +from timenet.types.text import read_value +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.datasets.pandalin98.enginemt_qa import EngineMTQAConnector +from timenet_connectors.datasets.pandalin98.enginemt_qa.loader import EngineMTQADatasetLoader +from timenet_connectors.datasets.pandalin98.enginemt_qa.model_input import authors_series +from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import ClosedQuestion, Vocabulary +from timenet_connectors.datasets.pandalin98.enginemt_qa.records import record_metadata +from timenet_connectors.datasets.pandalin98.enginemt_qa.release import ( + N_CHANNELS, + N_STEPS, + Part, + PinnedInventory, + PinnedRelease, +) +from timenet_connectors.datasets.pandalin98.enginemt_qa.signals import check_layout +from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import ( + DecisionTask, + PerceptionTask, + ReasoningTask, + UnderstandingTask, +) + + +_ROWS = 13 +# Unsorted, as 256 published lists are. Row 13 is named by no QA. +_MULTI = ["5", "4", "6", "7", "8", "9", "10", "11", "12", "1"] +_PERCEPTION = [question for question in ClosedQuestion if question.stage == 2] +# Each file's two lines hold one pair of each stage layout, with one empty closed answer each. +_INVENTORY = PinnedInventory(lines=2, qa_per_stage={1: 8, 2: 7, 3: 2, 4: 1}, empty_answers={}) +_RELEASE = PinnedRelease( + rows=_ROWS, + referenced_rows=12, + qa={ + Part.TRAIN: _INVENTORY._replace(empty_answers={"health_state": 1}), + Part.TEST: _INVENTORY._replace(empty_answers={"fault_cause": 1}), + }, +) + + +class _FixtureConnector(EngineMTQAConnector): + """The connector, pinned to the fixture's counts instead of the release's.""" + + CARD = Path(__file__).parents[1] / "dataset.yaml" + SOURCE_RELEASE: ClassVar[PinnedRelease] = _RELEASE + + +def _values(row_id: int) -> np.ndarray: + """Every value names its row, step and channel, so a misplaced value can't pass.""" + steps = np.arange(N_STEPS, dtype=np.float64)[:, None] + channels = np.arange(N_CHANNELS, dtype=np.float64)[None, :] + return row_id * 1000 + steps + channels / 100 + + +def _write_signals(path: Path, rows: int = _ROWS) -> None: + with h5py.File(path, "w") as file: + file["data_ID"] = np.arange(1, rows + 1, dtype=np.int64) + file["seq_data"] = np.stack([_values(row_id) for row_id in range(1, rows + 1)]) + + +def _turns(question: str, answer: str, stage: int, attribute: str) -> list[dict[str, str]]: + return [ + {"from": "human", "stage": str(stage), "attribute": attribute, "value": question}, + {"from": "gpt", "value": answer}, + ] + + +def _single(row_id: str, answers: dict[ClosedQuestion, str]) -> dict[str, Any]: + turns = [] + for k in range(8): + turns += _turns(f"What does the change in signal {k} represent in one cycle?\n ", f"steady {k}", 1, "open") + for question in _PERCEPTION: + turns += _turns(question.text, answers.get(question, "a"), 2, "close") + return {"id": row_id, "name": f"{row_id}.csv", "conversations": turns} + + +def _multi(condition: str) -> dict[str, Any]: + turns = _turns(ClosedQuestion.ENGINE_CONDITION.text, condition, 3, "close") + turns += _turns(ClosedQuestion.FAILURE_PROBABILITY.text, "d", 3, "close") + turns += _turns( + "Hey, what's the plan for this engine signal across 10 cycles?\n ", "Replace the HPT.", 4, "open" + ) + return {"id": _MULTI, "name": [f"{i}.csv" for i in _MULTI], "conversations": turns} + + +def _lines(part: Part) -> list[dict[str, Any]]: + if part is Part.TRAIN: + single = _single("3", {ClosedQuestion.FAULT_CAUSE: "a, b, e", ClosedQuestion.HEALTH_STATE: ""}) + return [single, _multi("c")] + # Row 3 is in both files, as published rows are. + return [_single("2", {ClosedQuestion.FAULT_CAUSE: "", ClosedQuestion.HPT_HEALTH: "b"}), _multi("e")] + + +def _write_qa(path: Path, lines: list[dict[str, Any]]) -> None: + path.write_text("".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8") + + +def _release(tmp_path: Path) -> list[HubFileRef]: + paths = {part: tmp_path / part.source_file for part in Part} + _write_signals(paths[Part.SIGNALS]) + for part in (Part.TRAIN, Part.TEST): + _write_qa(paths[part], _lines(part)) + return [HubFileRef(path, part.value) for part, path in paths.items()] + + +def _output(dataset) -> dict: + """What the dataset holds, as plain values.""" + records = { + record_metadata(record).row_id: sorted((s.name, s.n_values) for s in record.signals) + for record in dataset.records + } + tasks = [ + ( + type(task).__name__, + task.split.value, + tuple(record_metadata(record).row_id for record in task.inputs), + read_value(task.prompt), + tuple(read_value(target) for target in task.targets), + getattr(task, "target_schema", None), + dict(task.metadata), + ) + for task in dataset.iter_tasks() + ] + annotations = {a.key: (read_value(a.value), dict(a.metadata)) for a in dataset.annotations} + return {"records": records, "tasks": tasks, "annotations": annotations} + + +def test_converts_the_fixture(tmp_path): + output = _output(_FixtureConnector().convert(_release(tmp_path))) + channels = sorted((f"channel_{index:02d}", N_STEPS) for index in range(N_CHANNELS)) + # Only the rows a QA names, and each once though row 3 is in both files. + assert output["records"] == dict.fromkeys(range(1, 13), channels) + # Each file: 8 Understanding, 6 Perception (one empty answer left out), 2 Reasoning, 1 Decision. + kinds = [(kind, split) for kind, split, *_ in output["tasks"]] + per_file = ["AnswerTask"] * 8 + ["ClassificationTask"] * 8 + ["AnswerTask"] + assert kinds == [(kind, "train") for kind in per_file] + [(kind, "test") for kind in per_file] + assert output["annotations"]["source_repository"] == ("pandalin98/EngineMT-QA", {}) + assert output["annotations"]["source_revision"] == ("ea2051a15512212976c22c1a075ec549c7d48eef", {}) + + +def test_closed_questions_target_their_letters(tmp_path): + tasks = _output(_FixtureConnector().convert(_release(tmp_path)))["tasks"] + fault_cause = next(task for task in tasks if task[-1].get("closed_question") == "fault_cause") + assert fault_cause == ( + "ClassificationTask", + "train", + (3,), + ClosedQuestion.FAULT_CAUSE.text, + ("a", "b", "e"), + "fault_cause_vocabulary", + {"source_file": "train_qa.jsonl", "source_line": 0, "qa_offset": 13, "closed_question": "fault_cause"}, + ) + hpt = next(task for task in tasks if task[1] == "test" and task[-1].get("closed_question") == "hpt_health") + assert hpt[3:6] == (ClosedQuestion.HPT_HEALTH.text, ("b",), "component_health_vocabulary") + # The train health state and the test fault cause are empty, so neither has a task. + asked = [(split, metadata.get("closed_question")) for _, split, _, _, _, _, metadata in tasks] + assert ("train", "health_state") not in asked + assert ("test", "fault_cause") not in asked + + +def test_open_questions_keep_the_published_text(tmp_path): + tasks = _output(_FixtureConnector().convert(_release(tmp_path)))["tasks"] + assert tasks[1] == ( + "AnswerTask", + "train", + (3,), + "What does the change in signal 1 represent in one cycle?\n ", + ("steady 1",), + None, + {"source_file": "train_qa.jsonl", "source_line": 0, "qa_offset": 1}, + ) + decision = tasks[-1] + assert decision[0] == "AnswerTask" + assert decision[3:5] == ( + "Hey, what's the plan for this engine signal across 10 cycles?\n ", + ("Replace the HPT.",), + ) + assert decision[-1] == {"source_file": "test_qa.jsonl", "source_line": 1, "qa_offset": 2, "steps_per_cycle": 60} + + +def test_multi_cycle_tasks_take_ten_records_in_published_order(tmp_path): + dataset = _FixtureConnector().convert(_release(tmp_path)) + task = next(task for task in dataset.iter_tasks() if len(task.inputs) > 1) + assert [record_metadata(record).row_id for record in task.inputs] == [int(i) for i in _MULTI] + # The authors' TsQaDataset: the first 60 steps of each row, joined in published order. + expected = np.concatenate([_values(int(i))[:60] for i in _MULTI]) + np.testing.assert_array_equal(authors_series(task), expected) + + +def test_signals_read_their_row(tmp_path): + dataset = _FixtureConnector().convert(_release(tmp_path)) + record = next(record for record in dataset.records if record_metadata(record).row_id == 7) + for signal in record.signals: + channel = int(signal.name.removeprefix("channel_")) + assert isinstance(signal.time_axis, RegularAxis) + assert signal.time_axis.period_us == 1_000_000 + np.testing.assert_array_equal(signal.to_numpy(), _values(7)[:, channel]) + single = next(task for task in dataset.iter_tasks() if len(task.inputs) == 1) + np.testing.assert_array_equal(authors_series(single), _values(3)) + + +def test_storage_round_trip_keeps_the_whole_conversion(tmp_path): + original = _FixtureConnector().convert(_release(tmp_path)) + registry = LocalRegistry(tmp_path / "registry") + registry.store(original) + with TimeFReader(registry.open_version("pandalin98/enginemt-qa", "1.0.0")) as reader: + restored = reader.read() + assert _output(restored) == _output(original) + record = next(record for record in restored.records if record_metadata(record).row_id == 12) + np.testing.assert_array_equal( + next(s for s in record.signals if s.name == "channel_32").to_numpy(), _values(12)[:, 32] + ) + + +def test_vocabularies_name_each_letter(tmp_path): + annotations = _output(_FixtureConnector().convert(_release(tmp_path)))["annotations"] + assert annotations["component_health_vocabulary"] == (["a", "b"], {"options": {"a": "Health", "b": "Fault"}}) + assert annotations["failure_probability_vocabulary"][0] == ["a", "b", "c", "d", "e"] + assert {key for key in annotations if key.endswith("_vocabulary")} == {f"{v}_vocabulary" for v in Vocabulary} + + +def test_all_rows_builds_a_record_per_row(tmp_path): + loader = EngineMTQADatasetLoader.parse_from(_release(tmp_path), all_rows=True, release=_RELEASE) + assert [record_metadata(record).row_id for record in loader.records()] == list(range(1, _ROWS + 1)) + + +def test_tasks_of_selects_a_kind_and_split(tmp_path): + loader = EngineMTQADatasetLoader.parse_from(_release(tmp_path), release=_RELEASE) + assert len(list(loader.tasks_of(UnderstandingTask, split=Split.TEST))) == 8 + assert len(list(loader.tasks_of(PerceptionTask, split=Split.TRAIN))) == 6 + assert len(list(loader.tasks_of(ReasoningTask))) == 4 + assert [task.split for task in loader.tasks_of(DecisionTask)] == [Split.TRAIN, Split.TEST] + assert {kind.TIMENET_TYPE for kind in loader.TASK_KINDS} == {AnswerTask, ClassificationTask} + + +def test_a_changed_inventory_fails(tmp_path): + release = _RELEASE._replace(qa={**_RELEASE.qa, Part.TEST: _INVENTORY}) + with pytest.raises(TimeFValidationError, match="the pinned file holds"): + EngineMTQADatasetLoader.parse_from(_release(tmp_path), release=release) + with pytest.raises(TimeFValidationError, match="name 12 rows"): + EngineMTQADatasetLoader.parse_from(_release(tmp_path), release=_RELEASE._replace(referenced_rows=13)) + + +def _with_pair(line: dict[str, Any], offset: int, **changes: str) -> dict[str, Any]: + conversations = [dict(turn) for turn in line["conversations"]] + question, answer = conversations[2 * offset], conversations[2 * offset + 1] + answer["value"] = changes.pop("answer", answer["value"]) + question.update(changes) + return {**line, "conversations": conversations} + + +@pytest.mark.parametrize( + ("line", "message"), + [ + (_with_pair(_single("3", {}), 13, answer="a, g"), "not distinct letters"), + (_with_pair(_single("3", {}), 13, answer="a,b"), "not distinct letters"), + (_with_pair(_single("3", {}), 13, answer="a, a"), "not distinct letters"), + (_with_pair(_single("3", {}), 14, answer="a, b"), "several letters"), + (_with_pair(_single("3", {}), 8, value="What is the health status of the fan?\n "), "unknown stage 2"), + (_with_pair(_single("3", {}), 0, answer=""), "open question has an empty answer"), + (_with_pair(_single("3", {}), 0, attribute="close"), "stage 1 questions are not 'close'"), + (_with_pair(_single("3", {}), 0, value="Describe the signal."), "one marker"), + (_with_pair(_multi("c"), 2, stage="1"), "stage 1 question names 10 rows"), + ({**_single("3", {}), "name": "4.csv"}, "doesn't repeat id"), + ({**_multi("c"), "id": _MULTI[:9], "name": [f"{i}.csv" for i in _MULTI[:9]]}, "names 9 rows"), + ({**_single("3", {}), "id": "03", "name": "03.csv"}, "String should match pattern"), + ({**_single("3", {}), "extra": 1}, "Extra inputs are not permitted"), + ({**_single("3", {}), "conversations": _single("3", {})["conversations"][:-1]}, "alternate"), + ], +) +def test_a_malformed_line_fails_with_its_place(tmp_path, line, message): + refs = _release(tmp_path) + _write_qa(tmp_path / Part.TRAIN.source_file, [_lines(Part.TRAIN)[0], line]) + with pytest.raises(TimeFValidationError, match=message) as raised: + EngineMTQADatasetLoader.parse_from(refs, release=_RELEASE) + assert "train_qa.jsonl line 2" in str(raised.value) + + +def test_check_layout_rejects_another_file(tmp_path): + path = tmp_path / "time_series_data.h5" + _write_signals(path) + check_layout(path, n_rows=_ROWS) + with pytest.raises(TimeFValidationError, match="seq_data is"): + check_layout(path, n_rows=_ROWS + 1) + with h5py.File(path, "a") as file: + file["data_ID"][0] = 2 + with pytest.raises(TimeFValidationError, match="not the row ids"): + check_layout(path, n_rows=_ROWS) + + +def test_a_file_changed_after_conversion_fails_when_read(tmp_path): + refs = _release(tmp_path) + dataset = _FixtureConnector().convert(refs) + _write_signals(tmp_path / Part.SIGNALS.source_file, rows=4) + record = next(record for record in dataset.records if record_metadata(record).row_id == 12) + with pytest.raises(TimeFFormatError, match="cannot read row id 12"): + record.signals[0].to_numpy() diff --git a/pyproject.toml b/pyproject.toml index 0b7f019a1..f81a5e4ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dev = [ "ruff>=0.15.12", "ty>=0.0.92", "edfio>=0.4.14", + "h5py>=3.10", "wfdb>=4.1", "xlrd>=2.0", ] diff --git a/uv.lock b/uv.lock index 48aec0303..0b12ae56b 100644 --- a/uv.lock +++ b/uv.lock @@ -25,6 +25,7 @@ members = [ [manifest.dependency-groups] dev = [ { name = "edfio", specifier = ">=0.4.14" }, + { name = "h5py", specifier = ">=3.10" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "hypothesis", specifier = ">=6.140" }, { name = "jsonschema", specifier = ">=4.18" }, @@ -1106,6 +1107,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h5py" +version = "3.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/95/a825894f3e45cbac7554c4e97314ce886b233a20033787eda755ca8fecc7/h5py-3.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:719439d14b83f74eeb080e9650a6c7aa6d0d9ea0ca7f804347b05fac6fbf18af", size = 3721663, upload-time = "2026-03-06T13:47:49.599Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3b/38ff88b347c3e346cda1d3fc1b65a7aa75d40632228d8b8a5d7b58508c24/h5py-3.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3f0a0e136f2e95dd0b67146abb6668af4f1a69c81ef8651a2d316e8e01de447", size = 3087630, upload-time = "2026-03-06T13:47:51.249Z" }, + { url = "https://files.pythonhosted.org/packages/98/a8/2594cef906aee761601eff842c7dc598bea2b394a3e1c00966832b8eeb7c/h5py-3.16.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a6fbc5367d4046801f9b7db9191b31895f22f1c6df1f9987d667854cac493538", size = 4823472, upload-time = "2026-03-06T13:47:53.085Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/c1f604538ff6db22a0690be2dc44ab59178e115f63c917794e529356ab23/h5py-3.16.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fb1720028d99040792bb2fb31facb8da44a6f29df7697e0b84f0d79aff2e9bd3", size = 5027150, upload-time = "2026-03-06T13:47:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fd/301739083c2fc4fd89950f9bcfce75d6e14b40b0ca3d40e48a8993d1722c/h5py-3.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:314b6054fe0b1051c2b0cb2df5cbdab15622fb05e80f202e3b6a5eee0d6fe365", size = 4814544, upload-time = "2026-03-06T13:47:56.893Z" }, + { url = "https://files.pythonhosted.org/packages/4c/42/2193ed41ccee78baba8fcc0cff2c925b8b9ee3793305b23e1f22c20bf4c7/h5py-3.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ffbab2fedd6581f6aa31cf1639ca2cb86e02779de525667892ebf4cc9fd26434", size = 5034013, upload-time = "2026-03-06T13:47:59.01Z" }, + { url = "https://files.pythonhosted.org/packages/f7/20/e6c0ff62ca2ad1a396a34f4380bafccaaf8791ff8fccf3d995a1fc12d417/h5py-3.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:17d1f1630f92ad74494a9a7392ab25982ce2b469fc62da6074c0ce48366a2999", size = 3191673, upload-time = "2026-03-06T13:48:00.626Z" }, + { url = "https://files.pythonhosted.org/packages/f2/48/239cbe352ac4f2b8243a8e620fa1a2034635f633731493a7ff1ed71e8658/h5py-3.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:85b9c49dd58dc44cf70af944784e2c2038b6f799665d0dcbbc812a26e0faa859", size = 2673834, upload-time = "2026-03-06T13:48:02.579Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c0/5d4119dba94093bbafede500d3defd2f5eab7897732998c04b54021e530b/h5py-3.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c5313566f4643121a78503a473f0fb1e6dcc541d5115c44f05e037609c565c4d", size = 3685604, upload-time = "2026-03-06T13:48:04.198Z" }, + { url = "https://files.pythonhosted.org/packages/b0/42/c84efcc1d4caebafb1ecd8be4643f39c85c47a80fe254d92b8b43b1eadaf/h5py-3.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42b012933a83e1a558c673176676a10ce2fd3759976a0fedee1e672d1e04fc9d", size = 3061940, upload-time = "2026-03-06T13:48:05.783Z" }, + { url = "https://files.pythonhosted.org/packages/89/84/06281c82d4d1686fde1ac6b0f307c50918f1c0151062445ab3b6fa5a921d/h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527", size = 5198852, upload-time = "2026-03-06T13:48:07.482Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e9/1a19e42cd43cc1365e127db6aae85e1c671da1d9a5d746f4d34a50edb577/h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e", size = 5405250, upload-time = "2026-03-06T13:48:09.628Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8e/9790c1655eabeb85b92b1ecab7d7e62a2069e53baefd58c98f0909c7a948/h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794", size = 5190108, upload-time = "2026-03-06T13:48:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/51/d7/ab693274f1bd7e8c5f9fdd6c7003a88d59bedeaf8752716a55f532924fbb/h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074", size = 5419216, upload-time = "2026-03-06T13:48:13.322Z" }, + { url = "https://files.pythonhosted.org/packages/03/c1/0976b235cf29ead553e22f2fb6385a8252b533715e00d0ae52ed7b900582/h5py-3.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:96b422019a1c8975c2d5dadcf61d4ba6f01c31f92bbde6e4649607885fe502d6", size = 3182868, upload-time = "2026-03-06T13:48:15.759Z" }, + { url = "https://files.pythonhosted.org/packages/14/d9/866b7e570b39070f92d47b0ff1800f0f8239b6f9e45f02363d7112336c1f/h5py-3.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:39c2838fb1e8d97bcf1755e60ad1f3dd76a7b2a475928dc321672752678b96db", size = 2653286, upload-time = "2026-03-06T13:48:17.279Z" }, + { url = "https://files.pythonhosted.org/packages/0f/9e/6142ebfda0cb6e9349c091eae73c2e01a770b7659255248d637bec54a88b/h5py-3.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:370a845f432c2c9619db8eed334d1e610c6015796122b0e57aa46312c22617d9", size = 3671808, upload-time = "2026-03-06T13:48:19.737Z" }, + { url = "https://files.pythonhosted.org/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42108e93326c50c2810025aade9eac9d6827524cdccc7d4b75a546e5ab308edb", size = 3045837, upload-time = "2026-03-06T13:48:21.854Z" }, + { url = "https://files.pythonhosted.org/packages/da/1e/6172269e18cc5a484e2913ced33339aad588e02ba407fafd00d369e22ef3/h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524", size = 5193860, upload-time = "2026-03-06T13:48:24.071Z" }, + { url = "https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402", size = 5400417, upload-time = "2026-03-06T13:48:25.728Z" }, + { url = "https://files.pythonhosted.org/packages/bc/81/5b62d760039eed64348c98129d17061fdfc7839fc9c04eaaad6dee1004e4/h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7", size = 5185214, upload-time = "2026-03-06T13:48:27.436Z" }, + { url = "https://files.pythonhosted.org/packages/28/c4/532123bcd9080e250696779c927f2cb906c8bf3447df98f5ceb8dcded539/h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff", size = 5414598, upload-time = "2026-03-06T13:48:29.49Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/a27997f84341fc0dfcdd1fe4179b6ba6c32a7aa880fdb8c514d4dad6fba3/h5py-3.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:18f2bbcd545e6991412253b98727374c356d67caa920e68dc79eab36bf5fedad", size = 3175509, upload-time = "2026-03-06T13:48:31.131Z" }, + { url = "https://files.pythonhosted.org/packages/a5/23/bb8647521d4fd770c30a76cfc6cb6a2f5495868904054e92f2394c5a78ff/h5py-3.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:656f00e4d903199a1d58df06b711cf3ca632b874b4207b7dbec86185b5c8c7d4", size = 2647362, upload-time = "2026-03-06T13:48:33.411Z" }, + { url = "https://files.pythonhosted.org/packages/48/3c/7fcd9b4c9eed82e91fb15568992561019ae7a829d1f696b2c844355d95dd/h5py-3.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9c9d307c0ef862d1cd5714f72ecfafe0a5d7529c44845afa8de9f46e5ba8bd65", size = 3678608, upload-time = "2026-03-06T13:48:35.183Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b7/9366ed44ced9b7ef357ab48c94205280276db9d7f064aa3012a97227e966/h5py-3.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c1eff849cdd53cbc73c214c30ebdb6f1bb8b64790b4b4fc36acdb5e43570210", size = 3054773, upload-time = "2026-03-06T13:48:37.139Z" }, + { url = "https://files.pythonhosted.org/packages/58/a5/4964bc0e91e86340c2bbda83420225b2f770dcf1eb8a39464871ad769436/h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965", size = 5198886, upload-time = "2026-03-06T13:48:38.879Z" }, + { url = "https://files.pythonhosted.org/packages/f1/16/d905e7f53e661ce2c24686c38048d8e2b750ffc4350009d41c4e6c6c9826/h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd", size = 5404883, upload-time = "2026-03-06T13:48:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/58f34cb74af46d39f4cd18ea20909a8514960c5a3e5b92fd06a28161e0a8/h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c", size = 5192039, upload-time = "2026-03-06T13:48:43.117Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ca/934a39c24ce2e2db017268c08da0537c20fa0be7e1549be3e977313fc8f5/h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc", size = 5421526, upload-time = "2026-03-06T13:48:44.838Z" }, + { url = "https://files.pythonhosted.org/packages/3e/14/615a450205e1b56d16c6783f5ccd116cde05550faad70ae077c955654a75/h5py-3.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:fa48993a0b799737ba7fd21e2350fa0a60701e58180fae9f2de834bc39a147ab", size = 3183263, upload-time = "2026-03-06T13:48:47.117Z" }, + { url = "https://files.pythonhosted.org/packages/7b/48/a6faef5ed632cae0c65ac6b214a6614a0b510c3183532c521bdb0055e117/h5py-3.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:1897a771a7f40d05c262fc8f37376ec37873218544b70216872876c627640f63", size = 2663450, upload-time = "2026-03-06T13:48:48.707Z" }, + { url = "https://files.pythonhosted.org/packages/5d/32/0c8bb8aedb62c772cf7c1d427c7d1951477e8c2835f872bc0a13d1f85f86/h5py-3.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15922e485844f77c0b9d275396d435db3baa58292a9c2176a386e072e0cf2491", size = 3760693, upload-time = "2026-03-06T13:48:50.453Z" }, + { url = "https://files.pythonhosted.org/packages/1d/1f/fcc5977d32d6387c5c9a694afee716a5e20658ac08b3ff24fdec79fb05f2/h5py-3.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:df02dd29bd247f98674634dfe41f89fd7c16ba3d7de8695ec958f58404a4e618", size = 3181305, upload-time = "2026-03-06T13:48:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a1/af87f64b9f986889884243643621ebbd4ac72472ba8ec8cec891ac8e2ca1/h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242", size = 5074061, upload-time = "2026-03-06T13:48:54.089Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d0/146f5eaff3dc246a9c7f6e5e4f42bd45cc613bce16693bcd4d1f7c958bf5/h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16", size = 5279216, upload-time = "2026-03-06T13:48:56.75Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/12a13424f1e604fc7df9497b73c0356fb78c2fb206abd7465ce47226e8fd/h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7", size = 5070068, upload-time = "2026-03-06T13:48:59.169Z" }, + { url = "https://files.pythonhosted.org/packages/41/8c/bbe98f813722b4873818a8db3e15aa3e625b59278566905ac439725e8070/h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725", size = 5300253, upload-time = "2026-03-06T13:49:02.033Z" }, + { url = "https://files.pythonhosted.org/packages/32/9e/87e6705b4d6890e7cecdf876e2a7d3e40654a2ae37482d79a6f1b87f7b92/h5py-3.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4c6ab014ab704b4feaa719ae783b86522ed0bf1f82184704ed3c9e4e3228796e", size = 3381671, upload-time = "2026-03-06T13:49:04.351Z" }, + { url = "https://files.pythonhosted.org/packages/96/91/9fad90cfc5f9b2489c7c26ad897157bce82f0e9534a986a221b99760b23b/h5py-3.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:faca8fb4e4319c09d83337adc80b2ca7d5c5a343c2d6f1b6388f32cfecca13c1", size = 2740706, upload-time = "2026-03-06T13:49:06.347Z" }, +] + [[package]] name = "hf-xet" version = "1.6.0" @@ -4206,13 +4259,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/21/02db3e1fb3904016bfac310037c95b9f1eaaf0ffe7b4a84f14263a7d95df/zensical-0.0.51-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:134d776afa526098e05e34713e2f577c075e57a232e01b97842bb0206716afce", size = 12791154, upload-time = "2026-07-17T18:07:20.748Z" }, { url = "https://files.pythonhosted.org/packages/a2/35/b0d96f58253514cb3d08f5779020ab01ee5472334fb984b92e3fc9e9c9ac/zensical-0.0.51-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e97ab39668ae3b452c550634e921a0336443743aae5e1fe031c7bb57d049e535", size = 12692190, upload-time = "2026-07-17T18:07:24.553Z" }, { url = "https://files.pythonhosted.org/packages/2e/90/7a60e126a10c37c6b789938ff17e73fe76bba707fa029cb40ac659aeaa82/zensical-0.0.51-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c9579809f88608e7aa2cff516fff9d267d74a843cf6088a5f4227de2f092bb5", size = 13139337, upload-time = "2026-07-17T18:07:27.885Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c3/9101c97b90d4713ef2816db03366a45ae4762efebffd296737a2dd2df325/zensical-0.0.51-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296dc7a14aa28b81a58eb57df2d5c9c9a4b0de7e90c11d99c943354287952925", size = 13069851, upload-time = "2026-07-17T18:07:31.814Z" }, - { url = "https://files.pythonhosted.org/packages/d2/79/0474df9e15a2c18f6281a786e10177c1b6e16feac1c568e7f36ad39b339c/zensical-0.0.51-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f779d2d87b4bf228cf2e279bc0ae6bcf3b36a9335ff283a317d01f7c15ae46b2", size = 13451083, upload-time = "2026-07-17T18:07:35.543Z" }, { url = "https://files.pythonhosted.org/packages/fe/6f/91bbf78f704d5fd4c0c9be27d6bce3b6e4c2c339e4dcd6e7cf19ecda643c/zensical-0.0.51-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f813a1514a90890ca86248a8d54b81b2164bcbff11a6bcf11b01e1c01a1454", size = 13110446, upload-time = "2026-07-17T18:07:38.783Z" }, { url = "https://files.pythonhosted.org/packages/d9/89/aa9a95f81771614c37bdc52b8ab21fcdef4c8de7c9cedf34e9bf62674281/zensical-0.0.51-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:186ef37e0eee0e969e2cfae47b1b97775e3164e2cba95c71faa4dd6ef47ed009", size = 13315871, upload-time = "2026-07-17T18:07:42.43Z" }, - { url = "https://files.pythonhosted.org/packages/08/11/1bf6e9ded29d376f8c12644cc4de04676b010fee8caa17f682606b1f16d5/zensical-0.0.51-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f5d91ce246ed930224603083cef02ae8947132fc7c52901d72015ea03526fa58", size = 13344382, upload-time = "2026-07-17T18:07:46.066Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ec/663f16ff82d08b212e7c3236a88bd332f73331f94ddac1c91aaf882bbd1e/zensical-0.0.51-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:b1108eae82c6e8ffc33026f60b485c1512647a5333be4f547166b7c8877b98af", size = 13499628, upload-time = "2026-07-17T18:07:49.196Z" }, { url = "https://files.pythonhosted.org/packages/60/b4/7f1b6c3cf06d9f6ff5216523168a5d6ccc693444d5ceeb911eca97b30d98/zensical-0.0.51-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6fa0ecaf14f56841bfc595fa141396350c72aafbec73a016ebe3c824ed21ac72", size = 13451420, upload-time = "2026-07-17T18:07:52.563Z" }, - { url = "https://files.pythonhosted.org/packages/0a/44/be4bc09ec8f69e7be1b07b875887961c4e0e478b03a10d2cc624ef28fbe6/zensical-0.0.51-cp310-abi3-win32.whl", hash = "sha256:fb7ff4946b72168759c6af0a29cf5de4c38aebe633a83292e8cd4145b5213cc2", size = 12375639, upload-time = "2026-07-17T18:07:56.081Z" }, { url = "https://files.pythonhosted.org/packages/1f/85/aa827c244ed4f404e99a91c3ecf5e5adb62eca806a9e9c8e3333bbad8660/zensical-0.0.51-cp310-abi3-win_amd64.whl", hash = "sha256:12529d3d3991b63820952111dc1d1edc29b2c9b3a3abb16c243bcb649631ebf2", size = 12628965, upload-time = "2026-07-17T18:07:59.741Z" }, ] From 93cb7e3a0b990d13abddd9f93944301c7162db0c Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Fri, 30 Oct 2026 15:25:29 +0100 Subject: [PATCH 9/9] refactor(connectors): give each EngineMT-QA stage its own method DatasetLoader no longer has DatasetTask. A loader now yields TimeNet tasks, and gives each task of the authors its own method. This moves EngineMT-QA to that design. Two stages make AnswerTasks and two make ClassificationTasks, so the methods are what keeps the stages apart. - tasks.py: the classes UnderstandingTask, PerceptionTask, ReasoningTask and DecisionTask become the functions open_task and closed_task. stage_task picks one of them with Stage.is_closed. The content of the tasks does not change. - loader.py: EngineMTQADatasetLoader gets the methods understanding_tasks, perception_tasks, reasoning_tasks and decision_tasks. tasks() keeps the file order, in which the stages alternate, so the stream and the task ids do not change. TASK_TYPES lists AnswerTask and ClassificationTask. - tests: the test that selected tasks by kind and split now uses the four methods. It also makes sure that tasks() holds exactly the tasks of the four methods. - README.md: the task table names the four loader methods. Validation: make lint-fix, make check, make check-ci and make test (2144 passed, 1 skipped). make test-connectors passes for each connector in this commit, with the 26 EngineMT-QA tests. It also reports failures for folders that hold only __pycache__ from other branches. The full build of the real release was not run again. --- .../datasets/pandalin98/enginemt_qa/README.md | 14 +- .../datasets/pandalin98/enginemt_qa/loader.py | 74 +++++-- .../datasets/pandalin98/enginemt_qa/tasks.py | 181 ++++++++---------- .../enginemt_qa/tests/test_connector.py | 30 +-- 4 files changed, 158 insertions(+), 141 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md index b19e13629..becb83775 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md @@ -100,15 +100,15 @@ A row named by several lines, in train and in test, is one Record that their tas ### Tasks -Each published QA pair with a label becomes one task. There are four kinds, one per stage, and each -task takes its split from its file: +Each published QA pair with a label becomes one task. There are four tasks, one per stage, and each +task takes its split from its file. Each stage has its own loader method: -| Kind | TimeNet type | Inputs | Target | train | test | +| Loader method | TimeNet type | Inputs | Target | train | test | | --- | --- | --- | --- | --- | --- | -| `UnderstandingTask` | `AnswerTask` | 1 Record | the published answer | 41,128 | 17,240 | -| `PerceptionTask` | `ClassificationTask` | 1 Record | the published letters | 34,718 | 14,585 | -| `ReasoningTask` | `ClassificationTask` | 10 Records | the published letter | 16,300 | 6,768 | -| `DecisionTask` | `AnswerTask` | 10 Records | the published answer | 8,150 | 3,384 | +| `understanding_tasks()` | `AnswerTask` | 1 Record | the published answer | 41,128 | 17,240 | +| `perception_tasks()` | `ClassificationTask` | 1 Record | the published letters | 34,718 | 14,585 | +| `reasoning_tasks()` | `ClassificationTask` | 10 Records | the published letter | 16,300 | 6,768 | +| `decision_tasks()` | `AnswerTask` | 10 Records | the published answer | 8,150 | 3,384 | That is 142,273 tasks: 100,296 train and 41,977 test. The 1,769 Perception pairs with an empty answer have no task (see [Data limitations](#data-limitations)). diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py index 30d7b4a4c..b0c86fe42 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/loader.py @@ -7,12 +7,12 @@ from timenet.dataset import Record from timenet.errors import TimeFValidationError -from timenet.types import Annotation +from timenet.types import Annotation, AnswerTask, ClassificationTask, Task from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.bases.loader import DatasetLoader from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import vocabulary_key from timenet_connectors.datasets.pandalin98.enginemt_qa.qa import QAPair, read_pairs -from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import Vocabulary +from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import Stage, Vocabulary from timenet_connectors.datasets.pandalin98.enginemt_qa.records import RowLoader, build_record from timenet_connectors.datasets.pandalin98.enginemt_qa.release import ( PINNED_RELEASE, @@ -22,29 +22,29 @@ PinnedRelease, ) from timenet_connectors.datasets.pandalin98.enginemt_qa.signals import check_layout -from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import KIND_OF_STAGE, TASK_KINDS, EngineMTQATask +from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import closed_task, open_task, stage_task -class EngineMTQADatasetLoader(DatasetLoader[EngineMTQATask]): +class EngineMTQADatasetLoader(DatasetLoader): """EngineMT-QA's files, read and checked: one lazy Record per row, and one task per QA pair. - EngineMT-QA defines four tasks, the paper's stages, each under the authors' split: + EngineMT-QA defines four tasks, the paper's stages, each under the authors' split. Two stages are + ``AnswerTask``s and two are ``ClassificationTask``s, so each stage has its own method: - - Understanding (:class:`~.tasks.UnderstandingTask`): describe one cycle's change in a named signal. - - Perception (:class:`~.tasks.PerceptionTask`): from one cycle, choose a component's health, the - fault's causes, or the engine's health state. - - Reasoning (:class:`~.tasks.ReasoningTask`): from 10 cycles, choose the engine's condition or its - failure probability range. - - Decision-making (:class:`~.tasks.DecisionTask`): from 10 cycles, recommend maintenance. + - Understanding (:meth:`understanding_tasks`): describe one cycle's change in a named signal. + - Perception (:meth:`perception_tasks`): from one cycle, choose a component's health, the fault's + causes, or the engine's health state. + - Reasoning (:meth:`reasoning_tasks`): from 10 cycles, choose the engine's condition or its failure + probability range. + - Decision-making (:meth:`decision_tasks`): from 10 cycles, recommend maintenance. - Select one kind, or one kind of one split, with ``tasks_of``, such as - ``tasks_of(PerceptionTask, split=Split.TEST)``. + :meth:`tasks` keeps the file order, in which the stages alternate. The loader keeps the QA pairs, not the values: the signals read their row from the HDF5 file, so the files must stay available until writing ends. """ - TASK_KINDS = TASK_KINDS + TASK_TYPES = (AnswerTask, ClassificationTask) def __init__(self, pairs: tuple[QAPair, ...], records: Mapping[int, Record]) -> None: self._pairs = pairs @@ -111,14 +111,56 @@ def records(self) -> Iterator[Record]: """Return one Record per row, in row id order, whose values stay in the HDF5 file until written.""" return iter(self._records.values()) - def tasks(self) -> Iterator[EngineMTQATask]: + def tasks(self) -> Iterator[Task]: """Return one task per labelled QA pair: the train file's pairs, then the test file's, in file order. Yields: The tasks. """ for pair in self._pairs: - yield KIND_OF_STAGE[pair.stage](pair, tuple(self._records[row] for row in pair.rows)) + yield stage_task(pair, self._inputs(pair)) + + def understanding_tasks(self) -> Iterator[AnswerTask]: + """Return the understanding tasks (stage 1), in file order. + + Yields: + One answer task for each understanding pair. + """ + for pair in self._pairs_of(Stage.UNDERSTANDING): + yield open_task(pair, self._inputs(pair)) + + def perception_tasks(self) -> Iterator[ClassificationTask]: + """Return the perception tasks (stage 2), in file order. + + Yields: + One classification task for each labelled perception pair. + """ + for pair in self._pairs_of(Stage.PERCEPTION): + yield closed_task(pair, self._inputs(pair)) + + def reasoning_tasks(self) -> Iterator[ClassificationTask]: + """Return the reasoning tasks (stage 3), in file order. + + Yields: + One classification task for each reasoning pair. + """ + for pair in self._pairs_of(Stage.REASONING): + yield closed_task(pair, self._inputs(pair)) + + def decision_tasks(self) -> Iterator[AnswerTask]: + """Return the decision-making tasks (stage 4), in file order. + + Yields: + One answer task for each decision-making pair. + """ + for pair in self._pairs_of(Stage.DECISION_MAKING): + yield open_task(pair, self._inputs(pair)) + + def _pairs_of(self, stage: Stage) -> Iterator[QAPair]: + return (pair for pair in self._pairs if pair.stage is stage) + + def _inputs(self, pair: QAPair) -> tuple[Record, ...]: + return tuple(self._records[row] for row in pair.rows) def _read_labelled_pairs(path: Path, part: Part, inventory: PinnedInventory) -> list[QAPair]: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py index 9ce29409b..a013a5594 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py @@ -1,20 +1,17 @@ -"""EngineMT-QA's four tasks, one per stage, each QA pair built as the TimeNet task it becomes. +"""EngineMT-QA's tasks, one per stage, each QA pair built as a TimeNet task. Every task keeps the published question as its prompt, ```` marker included, and its file as its split. Its metadata names the pair as the authors' evaluation does: the file, the 0-based line (``source_line``) and the 0-based pair in the line (``qa_offset``). """ -from dataclasses import dataclass -from typing import Any, ClassVar +from typing import Any from timenet.dataset import Record from timenet.errors import TimeFValidationError -from timenet.types import AnswerTask, ClassificationTask, Split, Task -from timenet_connectors.bases.loader import DatasetTask +from timenet.types import AnswerTask, ClassificationTask from timenet_connectors.datasets.pandalin98.enginemt_qa.keys import vocabulary_key from timenet_connectors.datasets.pandalin98.enginemt_qa.qa import QAPair -from timenet_connectors.datasets.pandalin98.enginemt_qa.questions import Stage from timenet_connectors.datasets.pandalin98.enginemt_qa.release import MULTI_CYCLE_ROWS, N_STEPS @@ -27,103 +24,75 @@ """ -@dataclass(frozen=True) -class EngineMTQATask(DatasetTask): - """One published QA pair, on the Records of the rows it names, in published order.""" - - STAGE: ClassVar[Stage] - - pair: QAPair - records: tuple[Record, ...] - - @property - def split(self) -> Split: - """Return the task's split: its QA file's.""" - return self.pair.ref.split - - def _metadata(self) -> dict[str, Any]: - ref = self.pair.ref - metadata: dict[str, Any] = { - "source_file": ref.part.source_file, - "source_line": ref.line, - "qa_offset": ref.qa_offset, - } - if self.STAGE.is_multi_cycle: - metadata["steps_per_cycle"] = STEPS_PER_CYCLE - return metadata - - -@dataclass(frozen=True) -class _OpenTask(EngineMTQATask): - """An open question, which the authors score with BLEU and ROUGE against the published answer.""" - - TIMENET_TYPE: ClassVar[type[Task]] = AnswerTask - - def to_timenet_task(self) -> AnswerTask: - """Return the answer task, with the published answer as its target.""" - return AnswerTask( - inputs=self.records, - prompt=self.pair.question, - targets=(self.pair.answer,), - split=self.split, - metadata=self._metadata(), - ) - - -@dataclass(frozen=True) -class _ClosedTask(EngineMTQATask): - """A closed question, whose classes are its option letters, which the authors score as a set.""" - - TIMENET_TYPE: ClassVar[type[Task]] = ClassificationTask - - def to_timenet_task(self) -> ClassificationTask: - """Return the classification task, with the published letters as targets. - - Raises: - TimeFValidationError: If the pair is open, or its answer is empty. - """ - closed, letters = self.pair.closed, self.pair.letters - if closed is None or not letters: - raise TimeFValidationError(f"{self.pair.ref.context}: a closed task needs a published answer") - return ClassificationTask( - inputs=self.records, - prompt=self.pair.question, - targets=letters, - target_schema=vocabulary_key(closed.vocabulary), - split=self.split, - metadata=self._metadata() | {"closed_question": closed.value}, - ) - - -@dataclass(frozen=True) -class UnderstandingTask(_OpenTask): - """Stage 1: describe one cycle's change in a named signal and what it implies.""" - - STAGE: ClassVar[Stage] = Stage.UNDERSTANDING - - -@dataclass(frozen=True) -class PerceptionTask(_ClosedTask): - """Stage 2: from one cycle, name a component's health, the fault's causes, or the engine's health state.""" - - STAGE: ClassVar[Stage] = Stage.PERCEPTION - - -@dataclass(frozen=True) -class ReasoningTask(_ClosedTask): - """Stage 3: from 10 cycles, name the engine's condition or its failure probability range.""" - - STAGE: ClassVar[Stage] = Stage.REASONING - - -@dataclass(frozen=True) -class DecisionTask(_OpenTask): - """Stage 4: from 10 cycles, recommend maintenance actions.""" - - STAGE: ClassVar[Stage] = Stage.DECISION_MAKING - - -TASK_KINDS: tuple[type[EngineMTQATask], ...] = (UnderstandingTask, PerceptionTask, ReasoningTask, DecisionTask) -"""The task kinds, in stage order.""" - -KIND_OF_STAGE = {kind.STAGE: kind for kind in TASK_KINDS} +def _metadata(pair: QAPair) -> dict[str, Any]: + ref = pair.ref + metadata: dict[str, Any] = { + "source_file": ref.part.source_file, + "source_line": ref.line, + "qa_offset": ref.qa_offset, + } + if pair.stage.is_multi_cycle: + metadata["steps_per_cycle"] = STEPS_PER_CYCLE + return metadata + + +def open_task(pair: QAPair, records: tuple[Record, ...]) -> AnswerTask: + """Return an open question's task, which the authors score with BLEU and ROUGE against the published answer. + + Understanding (stage 1) and decision-making (stage 4) ask open questions. + + Args: + pair: The QA pair, whose file gives the split. + records: The Records of the rows the pair names, in published order. + + Returns: + The answer task, with the published answer as its target. + """ + return AnswerTask( + inputs=records, + prompt=pair.question, + targets=(pair.answer,), + split=pair.ref.split, + metadata=_metadata(pair), + ) + + +def closed_task(pair: QAPair, records: tuple[Record, ...]) -> ClassificationTask: + """Return a closed question's task, whose classes are its option letters, which the authors score as a set. + + Perception (stage 2) and reasoning (stage 3) ask closed questions. + + Args: + pair: The QA pair, whose file gives the split. + records: The Records of the rows the pair names, in published order. + + Returns: + The classification task, with the published letters as targets. + + Raises: + TimeFValidationError: If the pair is open, or its answer is empty. + """ + closed, letters = pair.closed, pair.letters + if closed is None or not letters: + raise TimeFValidationError(f"{pair.ref.context}: a closed task needs a published answer") + return ClassificationTask( + inputs=records, + prompt=pair.question, + targets=letters, + target_schema=vocabulary_key(closed.vocabulary), + split=pair.ref.split, + metadata=_metadata(pair) | {"closed_question": closed.value}, + ) + + +def stage_task(pair: QAPair, records: tuple[Record, ...]) -> AnswerTask | ClassificationTask: + """Return the task of a pair of any stage: closed for perception and reasoning, open otherwise. + + Args: + pair: The QA pair. + records: The Records of the rows the pair names, in published order. + + Returns: + The pair's task. + """ + return closed_task(pair, records) if pair.stage.is_closed else open_task(pair, records) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py index 6575c545a..dc0a65be1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py @@ -26,12 +26,6 @@ PinnedRelease, ) from timenet_connectors.datasets.pandalin98.enginemt_qa.signals import check_layout -from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import ( - DecisionTask, - PerceptionTask, - ReasoningTask, - UnderstandingTask, -) _ROWS = 13 @@ -236,13 +230,25 @@ def test_all_rows_builds_a_record_per_row(tmp_path): assert [record_metadata(record).row_id for record in loader.records()] == list(range(1, _ROWS + 1)) -def test_tasks_of_selects_a_kind_and_split(tmp_path): +def _pair_key(task) -> tuple: + return task.metadata["source_file"], task.metadata["source_line"], task.metadata["qa_offset"] + + +def test_each_stage_has_its_own_method(tmp_path): loader = EngineMTQADatasetLoader.parse_from(_release(tmp_path), release=_RELEASE) - assert len(list(loader.tasks_of(UnderstandingTask, split=Split.TEST))) == 8 - assert len(list(loader.tasks_of(PerceptionTask, split=Split.TRAIN))) == 6 - assert len(list(loader.tasks_of(ReasoningTask))) == 4 - assert [task.split for task in loader.tasks_of(DecisionTask)] == [Split.TRAIN, Split.TEST] - assert {kind.TIMENET_TYPE for kind in loader.TASK_KINDS} == {AnswerTask, ClassificationTask} + assert len([task for task in loader.understanding_tasks() if task.split is Split.TEST]) == 8 + assert len([task for task in loader.perception_tasks() if task.split is Split.TRAIN]) == 6 + assert len(list(loader.reasoning_tasks())) == 4 + assert [task.split for task in loader.decision_tasks()] == [Split.TRAIN, Split.TEST] + # tasks() keeps the file order, and holds exactly the tasks of the four methods. + by_stage = [ + *loader.understanding_tasks(), + *loader.perception_tasks(), + *loader.reasoning_tasks(), + *loader.decision_tasks(), + ] + assert sorted(map(_pair_key, loader.tasks())) == sorted(map(_pair_key, by_stage)) + assert (AnswerTask, ClassificationTask) == loader.TASK_TYPES def test_a_changed_inventory_fails(tmp_path):