From d240f60fcba64e95e8aa8e1c997c44e3a668d3fe Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:43:39 -0800 Subject: [PATCH 1/2] refactor(connectors)!: give Hugging Face connectors a mapping from part to path BaseHuggingFaceConnector implements convert once. It checks the refs and calls convert_files with a mapping from each part name to its local path. A connector reads files["questions"] and no longer calls ref_paths, which is removed. --- .agents/skills/add-dataset-connector/SKILL.md | 4 +- .../references/connector-anatomy.md | 18 ++++-- docs/connectors.md | 4 ++ .../timenet_connectors/bases/huggingface.py | 64 +++++++++---------- .../datasets/bash_lab/sensorcaps/connector.py | 47 ++++++++------ .../datasets/chengsenwang/tsqa/connector.py | 11 ++-- .../datasets/datadog/arfbench/connector.py | 14 ++-- .../datadog/arfbench_charts/connector.py | 12 ++-- .../tsaqa/tsaqa_benchmark/connector.py | 15 ++--- .../tests/test_huggingface_hub.py | 22 ++----- 10 files changed, 106 insertions(+), 105 deletions(-) diff --git a/.agents/skills/add-dataset-connector/SKILL.md b/.agents/skills/add-dataset-connector/SKILL.md index c38373cad..8f3ada4c0 100644 --- a/.agents/skills/add-dataset-connector/SKILL.md +++ b/.agents/skills/add-dataset-connector/SKILL.md @@ -42,8 +42,8 @@ latter). For PhysioNet, inspect the archive layout and a WFDB header (`fs`, `sig ### 3. Pick the base connector - Hub dataset: subclass `BaseHuggingFaceConnector`, pass the class keywords `hf_repo`, - `source_revision` (a commit), and `source_files`, inherit `download`, and implement `convert` - over the `HubFileRef` list. The base fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and `SOURCE_REVISION`. + `source_revision` (a commit), and `source_files`, inherit `download` and `convert`, and implement + `convert_files` over the part-to-path mapping. The base fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and `SOURCE_REVISION`. - PhysioNet/WFDB: subclass `BasePhysioNetConnector`, implement `download` (use `ensure_archive` / `download_files` from `timenet_connectors.download`) and `convert` (use `_read_header` / `_lead_loader`). - Neither: subclass `BaseConnector[TRaw]` and implement `download` + `convert` yourself. diff --git a/.agents/skills/add-dataset-connector/references/connector-anatomy.md b/.agents/skills/add-dataset-connector/references/connector-anatomy.md index ca351cdc6..65fe0618a 100644 --- a/.agents/skills/add-dataset-connector/references/connector-anatomy.md +++ b/.agents/skills/add-dataset-connector/references/connector-anatomy.md @@ -89,8 +89,10 @@ card sets one of the fields to a different value, the build fails. Do not copy ` `download` is inherited and returns one `HubFileRef(path, part)` per file. Declare `huggingface_hub>=1.1` in the connector's `requirements.txt`, and set `HF_TOKEN` for gated or -private datasets. The base checks the refs before your `convert` runs: they must be nonempty and -name each declared part at most once. Do not call `check_refs` yourself. +private datasets. Do not override `convert`. The base's `convert` checks the refs: they must be nonempty +and name each declared part at most once. Then it calls your +`convert_files(self, files: Mapping[str, Path])` with the local path of each part, so read a file as +`files["train"]`. To read the Hub's parquet conversion instead of the original files, pin the commit that `refs/convert/parquet` points to and read the files with `ParquetDataReader` and a connector-owned @@ -184,11 +186,13 @@ the bounds of every `Span` the task carries. ```python import json +from collections.abc import Mapping +from pathlib import Path from pydantic import ConfigDict, Field from timenet.dataset import OrdinalAxis, Record, Signal, Source, TimeFDataset from timenet.types import Annotation, AnswerTask, TimeSeriesSpec, ureg -from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.sources.readers import DataRow, ParquetDataReader @@ -217,18 +221,18 @@ class TSQAConnector( ): """Connector for the TSQA time-series QA dataset (Hub repo ``ChengsenWang/TSQA``).""" - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: """Build one record per row: the parsed series plus its question-and-answer task. Args: - raw_refs: The parquet files from :meth:`download`. + files: The parquet file of each part. Returns: The populated :class:`~timenet.dataset.TimeFDataset`. """ dataset = TimeFDataset(metadata=self.metadata()) - for ref in raw_refs: - for row in ParquetDataReader(ref.path, TSQARow).iter_rows(): + for path in files.values(): + for row in ParquetDataReader(path, TSQARow).iter_rows(): series = json.loads(row.series) signals = series if series and isinstance(series[0], list) else [series] time_series = [ diff --git a/docs/connectors.md b/docs/connectors.md index 228560848..87a0438fa 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -295,6 +295,10 @@ falls back to throttled text lines. The `--quiet` flag silences all output. PhysioNet connectors extend `BasePhysioNetConnector` for WFDB record I/O. They use `ensure_archive` to pull their database archive. +Hugging Face Hub connectors extend `BaseHuggingFaceConnector`. The base downloads the files in +`SOURCE_FILES` and implements `convert`. The connector implements `convert_files(files)` instead, where +`files` maps each part name to its local path. + ## Example connectors - `timenet/hello-world` is a synthetic, offline reference connector. It needs no network and produces diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py index 4f3f63df7..03a8684b9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py +++ b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py @@ -1,17 +1,17 @@ """A reusable base for connectors that read files from a pinned HuggingFace Hub dataset revision. Subclasses pass ``hf_repo``, ``source_revision``, and ``source_files`` as class keywords, ship a -``dataset.yaml`` card beside the connector, and implement ``convert()`` over the :class:`HubFileRef` list ``download()`` -returns. The base checks that list before the subclass's ``convert()`` runs. It also fills -``source_url`` and ``source_revision`` in the dataset metadata from ``HF_REPO`` and ``SOURCE_REVISION``. +``dataset.yaml`` card beside the connector, and implement ``convert_files()``. ``download()`` returns a +:class:`HubFileRef` list. The base's ``convert()`` checks that list, then passes +``convert_files()`` the path of each part. The base also fills ``source_url`` and ``source_revision`` +in the dataset metadata from ``HF_REPO`` and ``SOURCE_REVISION``. :mod:`timenet_connectors.sources.huggingface_hub` describes how the files download. For the Hub's parquet conversion, pin the commit ``refs/convert/parquet`` points to. """ -from abc import ABC +from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from dataclasses import dataclass -import functools from pathlib import Path import re from types import MappingProxyType @@ -19,7 +19,7 @@ from timenet.connectors import BaseConnector from timenet.dataset import TimeFDataset -from timenet.errors import TimeFValidationError, TimeNetConnectorError, TimeNetDownloadError +from timenet.errors import TimeFValidationError, TimeNetDownloadError from timenet.types import DatasetMetadata from timenet_connectors.sources.huggingface_hub import hub_snapshot @@ -33,18 +33,17 @@ class HubFileRef: path: Path part: str - """The connector's own name for this file, such as ``"hhar_v2"``: the key in ``SOURCE_FILES``, which - ``convert()`` branches on and builds ids from.""" + """The connector's own name for this file, such as ``"hhar_v2"``: the key in ``SOURCE_FILES`` and in the + mapping that ``convert_files()`` receives.""" class BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]], ABC): """Base class for connectors that read files from a pinned Hub dataset revision. Subclasses pass ``hf_repo``, ``source_revision``, and ``source_files`` as class keywords, and - implement ``convert()`` over the returned :class:`HubFileRef` list. The base stores the keywords as - :attr:`HF_REPO`, :attr:`SOURCE_REVISION`, and :attr:`SOURCE_FILES`. A subclass that passes no - keywords keeps the values of its parent. The base wraps each subclass's ``convert()`` to run - :meth:`check_refs` first. + implement :meth:`convert_files`. The base stores the keywords as :attr:`HF_REPO`, + :attr:`SOURCE_REVISION`, and :attr:`SOURCE_FILES`. A subclass that passes no keywords keeps the + values of its parent. :meth:`convert` runs :meth:`check_refs` first. """ HF_REPO: ClassVar[str] @@ -78,16 +77,6 @@ def __init_subclass__( ) if not getattr(cls, "SOURCE_FILES", None): raise TimeFValidationError(f"{cls.__name__} source_files must declare at least one file") - # Every subclass's convert() checks its refs first, so no connector has to call check_refs itself - convert = cls.__dict__.get("convert") - if convert is not None: - - @functools.wraps(convert) - def checked(self: BaseHuggingFaceConnector, raw_refs: list[HubFileRef]) -> TimeFDataset: - self.check_refs(raw_refs) - return convert(self, raw_refs) - - cls.convert = checked def metadata(self) -> DatasetMetadata: """Return the card, with ``source_url`` and ``source_revision`` from :attr:`HF_REPO` and :attr:`SOURCE_REVISION`. @@ -117,7 +106,7 @@ def check_refs(self, raw_refs: Sequence[HubFileRef]) -> None: """Check that ``raw_refs`` is nonempty and names each part once, only parts :attr:`SOURCE_FILES` declares. Args: - raw_refs: The references ``convert()`` received. + raw_refs: The references :meth:`convert` received. Raises: TimeFValidationError: If ``raw_refs`` is empty, repeats a part, or names an undeclared part. @@ -129,24 +118,31 @@ def check_refs(self, raw_refs: Sequence[HubFileRef]) -> None: if unknown: raise TimeFValidationError(f"{type(self).__name__} has no source part {', '.join(unknown)}") - def ref_paths(self, raw_refs: Sequence[HubFileRef], *parts: str) -> dict[str, Path]: - """Map each part in ``raw_refs`` to its local path. + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Check ``raw_refs``, then convert the files they name. Args: - raw_refs: The references ``convert()`` received. - *parts: The parts the caller needs. + raw_refs: The references from :meth:`download_async`. Returns: - The path of each part: a file, or the directory of a pattern part. + The dataset from :meth:`convert_files`. Raises: - TimeNetConnectorError: If one of ``parts`` is not in ``raw_refs``. + TimeFValidationError: If ``raw_refs`` is empty, repeats a part, or names an undeclared part. + """ # ruff: ignore[docstring-extraneous-exception] - raised by check_refs + self.check_refs(raw_refs) + return self.convert_files({ref.part: ref.path for ref in raw_refs}) + + @abstractmethod + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + """Convert the downloaded files into a dataset. + + Args: + files: The local path of each part: a file, or the directory of a pattern part. + + Returns: + The populated dataset. """ - paths = {ref.part: ref.path for ref in raw_refs} - missing = [part for part in parts if part not in paths] - if missing: - raise TimeNetConnectorError(f"{self.HF_REPO} refs lack {', '.join(missing)}") - return paths async def download_async(self, cache_dir: Path) -> list[HubFileRef]: """Download :attr:`SOURCE_FILES` at :attr:`SOURCE_REVISION` concurrently. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py index 6bdd13197..ac812fed4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py @@ -7,6 +7,7 @@ from collections.abc import Iterator, Mapping from dataclasses import dataclass from functools import partial +from pathlib import Path from typing import Annotated, ClassVar import pyarrow as pa @@ -15,7 +16,7 @@ from timenet.dataset import Record, Signal, Source, TimeFDataset from timenet.errors import TimeFValidationError from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec -from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.sources.readers import DataRow, JsonlDataReader @@ -78,16 +79,17 @@ class _SourceRow(DataRow): @dataclass(frozen=True) class _RowRef: - file: HubFileRef + part: str + path: Path index: int @property def key(self) -> tuple[str, int]: - return self.file.part, self.index + return self.part, self.index @property def context(self) -> str: - return f"{self.file.path} ({self.file.part}, row {self.index})" + return f"{self.path} ({self.part}, row {self.index})" @dataclass(frozen=True) @@ -218,25 +220,25 @@ def _parse_row(row: _SourceRow, context: str) -> _ParsedRow: raise TimeFValidationError(f"{context}: {exc}") from exc -def _iter_rows(file: HubFileRef) -> Iterator[tuple[_RowRef, _ParsedRow]]: +def _iter_rows(part: str, path: Path) -> Iterator[tuple[_RowRef, _ParsedRow]]: """Read a JSONL file one row at a time. Yields: Each row reference and its decoded contents. """ - for index, row in enumerate(JsonlDataReader(file.path, _SourceRow).iter_rows()): - ref = _RowRef(file, index) + for index, row in enumerate(JsonlDataReader(path, _SourceRow).iter_rows()): + ref = _RowRef(part, path, index) yield ref, _parse_row(row, ref.context) -def _iter_tasks(files: tuple[HubFileRef, ...], records: Mapping[tuple[str, int], Record]) -> Iterator[AnswerTask]: +def _iter_tasks(files: Mapping[str, Path], records: Mapping[tuple[str, int], Record]) -> Iterator[AnswerTask]: """Re-read original system prompts and caption targets without retaining the corpus. Yields: A caption task with its instruction prompt and registered input Record. """ - for file in files: - for ref, row in _iter_rows(file): + for part, path in files.items(): + for ref, row in _iter_rows(part, path): record = records[ref.key] context = [ annotation @@ -272,14 +274,17 @@ class SensorCapsConnector( SOURCE_ROWS: ClassVar[Mapping[str, int]] = _SOURCE_ROWS - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: """Index local rows into six-channel Records and a re-iterable caption stream. + Args: + files: The JSONL file of each partition. + Returns: A dataset with materialized signals and streamed caption tasks. Raises: - TimeFValidationError: If references or source rows violate the SensorCaps mapping. + TimeFValidationError: If source rows violate the SensorCaps mapping. """ dataset = TimeFDataset(metadata=self.metadata()) for key, value in ( @@ -290,18 +295,18 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: ): dataset.annotate(Annotation(key, value=value)) records: dict[tuple[str, int], Record] = {} - for file in raw_refs: + for part, path in files.items(): count = 0 - for ref, row in _iter_rows(file): + for ref, row in _iter_rows(part, path): record = self._make_record(ref, row) dataset.add_record(record) records[ref.key] = record count += 1 # Every pinned count is nonzero, so this also rejects an empty partition. - expected = self.SOURCE_ROWS[file.part] + expected = self.SOURCE_ROWS[part] if count != expected: - raise TimeFValidationError(f"{file.path} ({file.part}): expected {expected} rows, found {count}") - dataset.set_task_stream([AnswerTask], partial(_iter_tasks, tuple(raw_refs), records)) + raise TimeFValidationError(f"{path} ({part}): expected {expected} rows, found {count}") + dataset.set_task_stream([AnswerTask], partial(_iter_tasks, files, records)) return dataset def _make_record(self, ref: _RowRef, row: _ParsedRow) -> Record: @@ -310,7 +315,7 @@ def _make_record(self, ref: _RowRef, row: _ParsedRow) -> Record: Returns: A Record with materialized sensor values and annotations from the original row. """ - channels = _PAMAP2_CHANNELS if ref.file.part == "pamap2_v2" else _CHANNELS + channels = _PAMAP2_CHANNELS if ref.part == "pamap2_v2" else _CHANNELS # Retain published values, including upstream zero-filled PAMAP2 gaps. signals = [ Signal.from_irregular( @@ -322,11 +327,11 @@ def _make_record(self, ref: _RowRef, row: _ParsedRow) -> Record: for spec, name, sensor, dimension in channels ] record = Record(sources=[Source("IMU", signals=signals)]) - source = ref.file.part.rsplit("_", 1)[0] + source = ref.part.rsplit("_", 1)[0] metadata: dict[str, str | int] = { "source_dataset": source, - "source_partition": ref.file.part, - "source_file": self.SOURCE_FILES[ref.file.part], + "source_partition": ref.part, + "source_file": self.SOURCE_FILES[ref.part], "source_row": ref.index, "activity_summary": row.summary, } diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py index e260dbffa..6b5e9295e 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py @@ -6,11 +6,14 @@ lists. Each row becomes one record that carries the series and an :class:`~timenet.types.AnswerTask`. """ +from collections.abc import Mapping +from pathlib import Path + from pydantic import ConfigDict, Field from timenet.dataset import OrdinalAxis, Record, Signal, Source, TimeFDataset from timenet.types import Annotation, AnswerTask, Split, TimeSeriesSpec, ureg -from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.datasets.chengsenwang.tsqa.sample import TSQASample, get_category, get_signal_id from timenet_connectors.sources.readers import DataRow, ParquetDataReader from timenet_connectors.splitting import Splitter, StratifiedSplitter @@ -59,16 +62,16 @@ def __init__(self) -> None: seed=42, ) - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: """Build one record per row: the parsed series plus its question-and-answer task. Args: - raw_refs: The parquet files from :meth:`download`. + files: The parquet file of each part. Returns: The populated :class:`~timenet.dataset.TimeFDataset`. """ - rows = (row for ref in raw_refs for row in ParquetDataReader(ref.path, _SourceRow).iter_rows()) + rows = (row for path in files.values() for row in ParquetDataReader(path, _SourceRow).iter_rows()) samples = [TSQASample.from_row(row.model_dump(by_alias=True)) for row in rows] splits = self.splitter.split(samples) dataset = TimeFDataset(metadata=self.metadata()) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py index 2d23dccf5..c9af56464 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py @@ -17,7 +17,8 @@ if TYPE_CHECKING: - from timenet_connectors.bases.huggingface import HubFileRef + from collections.abc import Mapping + from pathlib import Path REPO = "Datadog/ARFBench" REVISION = "1cc8ae54e9633f596b755f7c4bce54ccb0cb9f5a" @@ -36,22 +37,21 @@ class ARFBenchConnector( The values are plain series, so the dataset uses the default Parquet backend. """ - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: """Create a Record for each metric and a series task for each question. Args: - raw_refs: The question table and the series directory from :meth:`download_async`. + files: The question table and the series directory. Returns: The populated dataset. """ - paths = self.ref_paths(raw_refs, "questions", "series") dataset = TimeFDataset(metadata=self.metadata()) metric_records = { - files.metric: dataset.add_record(files.create_record()) - for files in MetricSeriesFiles.from_series_dir(paths["series"]) + metric_files.metric: dataset.add_record(metric_files.create_record()) + for metric_files in MetricSeriesFiles.from_series_dir(files["series"]) } - for question in read_questions(paths["questions"]): + for question in read_questions(files["questions"]): dataset.add_task(question.create_series_task(metric_records)) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py index 769b5c362..fa7ecedb9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py @@ -17,10 +17,9 @@ if TYPE_CHECKING: + from collections.abc import Mapping from pathlib import Path - from timenet_connectors.bases.huggingface import HubFileRef - CHART_DIR = "arfbench-images" # {metric}.png, and {metric}-{metric}.png for the pairs that the questions cite @@ -46,21 +45,20 @@ class ARFBenchChartsConnector( values_backend = ValuesBackend.ZARR # the chart tensors need the Zarr backend - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: """Create a Record for each chart and a chart task for each question. Args: - raw_refs: The question table and the chart directory from :meth:`download_async`. + files: The question table and the chart directory. Returns: The populated dataset. """ - paths = self.ref_paths(raw_refs, "questions", "charts") dataset = TimeFDataset(metadata=self.metadata()) chart_records = { - path.stem: dataset.add_record(_create_chart_record(path)) for path in sorted(paths["charts"].glob("*.png")) + path.stem: dataset.add_record(_create_chart_record(path)) for path in sorted(files["charts"].glob("*.png")) } - for question in read_questions(paths["questions"]): + for question in read_questions(files["questions"]): dataset.add_task(question.create_chart_task(chart_records)) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index 97c09db23..0d7c9cbf5 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -30,7 +30,7 @@ from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetConnectorError from timenet.json import JsonValue from timenet.types import Annotation, AnswerTask, ClassificationTask, Label, LabelSet, Split, Task, TimeSeriesSpec -from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.sources.readers import DataRow, ParquetDataReader @@ -746,7 +746,7 @@ class TSAQAConnector( ): """Build TSAQA from the pinned train, val and test parquet files of its Hugging Face release.""" - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: """Convert each split file into lazy Records and a stream of one task per row. Every file is read once up front to check each row and keep its small fields. The source issues @@ -754,7 +754,7 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: in the files until written, so the files must stay available until then. Args: - raw_refs: The downloaded split files. + files: The downloaded file of each split. Returns: A dataset with the provenance annotations, the Records, and a stream of every task. @@ -762,16 +762,15 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: Raises: TimeFValidationError: If a split file is missing, or a row is malformed. """ - paths = {ref.part: ref.path for ref in raw_refs} - if paths.keys() != _SPLITS.keys(): + if files.keys() != _SPLITS.keys(): raise TimeFValidationError( - f"TSAQA needs all its split files, missing {sorted(split.value for split in _SplitFile if split not in paths)}" + f"TSAQA needs all its split files, missing {sorted(split.value for split in _SplitFile if split not in files)}" ) rows: list[_Row] = [] for split in _SplitFile: - rows.extend(_read_rows(paths[split], split)) + rows.extend(_read_rows(files[split], split)) issues = _find_issues(rows) - loader = _SeriesLoader(paths) + loader = _SeriesLoader(files) records = [_records(row, loader) for row in rows] dataset = TimeFDataset(metadata=self.metadata()) diff --git a/packages/timenet-connectors/tests/test_huggingface_hub.py b/packages/timenet-connectors/tests/test_huggingface_hub.py index 514b180e4..34382f782 100644 --- a/packages/timenet-connectors/tests/test_huggingface_hub.py +++ b/packages/timenet-connectors/tests/test_huggingface_hub.py @@ -9,7 +9,7 @@ import pytest -from timenet.errors import TimeFValidationError, TimeNetConnectorError, TimeNetDownloadError +from timenet.errors import TimeFValidationError, TimeNetDownloadError from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef from timenet_connectors.download.progress import DownloadProgress, progress_sink from timenet_connectors.sources.huggingface_hub import hub_snapshot @@ -24,8 +24,8 @@ class _Pinned( source_revision=_SHA, source_files={"train": "data/train.jsonl", "test": "data/test.jsonl"}, ): - def convert(self, raw_refs): - return raw_refs + def convert_files(self, files): + return files class _HubTqdm: @@ -173,19 +173,11 @@ def test_check_refs_rejects_empty_repeated_or_undeclared_parts(parts, message): _Pinned().check_refs(refs) -def test_convert_checks_refs_before_the_subclass_runs(): +def test_convert_checks_refs_then_passes_the_path_of_each_part(): with pytest.raises(TimeFValidationError, match=r"has no source part dev$"): _Pinned().convert([HubFileRef(Path("dev.jsonl"), "dev")]) refs = [HubFileRef(Path("test.jsonl"), "test")] - assert _Pinned().convert(refs) is refs - assert _Pinned.convert.__name__ == "convert" - - -def test_ref_paths_maps_parts_and_names_the_repo_when_one_is_missing(): - refs = [HubFileRef(Path("train.jsonl"), "train")] - assert _Pinned().ref_paths(refs, "train") == {"train": Path("train.jsonl")} - with pytest.raises(TimeNetConnectorError, match=r"^org/repo refs lack test$"): - _Pinned().ref_paths(refs, "train", "test") + assert _Pinned().convert(refs) == {"test": Path("test.jsonl")} def test_snapshot_downloads_the_matching_files_and_returns_the_snapshot_root(monkeypatch, tmp_path): @@ -206,8 +198,8 @@ class _Patterned( source_revision=_SHA, source_files={"questions": "qa.csv", "series": "data/*.parquet"}, ): - def convert(self, raw_refs): - return raw_refs + def convert_files(self, files): + return files def test_a_pattern_part_resolves_to_the_directory_of_its_files(monkeypatch, tmp_path): From 79b4915cfc1943e01954cbd36901d68413d377a4 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:58:05 -0800 Subject: [PATCH 2/2] refactor(connectors)!: make the Hugging Face download result the mapping itself download_async returns the mapping from each part name to its local path, and a connector implements the normal convert(files). HubFileRef, check_refs and convert_files are removed. TimeSeriesExam uses the same form. --- .agents/skills/add-dataset-connector/SKILL.md | 4 +- .../references/connector-anatomy.md | 12 ++- docs/connectors.md | 11 +-- .../timenet_connectors/bases/huggingface.py | 83 ++++--------------- .../autonlab/timeseriesexam1/connector.py | 10 ++- .../timeseriesexam1/tests/test_connector.py | 3 +- .../datasets/bash_lab/sensorcaps/connector.py | 2 +- .../sensorcaps/tests/test_connector.py | 25 +++--- .../datasets/chengsenwang/tsqa/connector.py | 2 +- .../chengsenwang/tsqa/tests/test_connector.py | 3 +- .../datasets/datadog/arfbench/connector.py | 2 +- .../datadog/arfbench/tests/release.py | 10 +-- .../datadog/arfbench/tests/test_connector.py | 17 ++-- .../datadog/arfbench_charts/connector.py | 2 +- .../arfbench_charts/tests/test_connector.py | 5 +- .../tsaqa/tsaqa_benchmark/connector.py | 10 +-- .../tsaqa_benchmark/tests/test_connector.py | 21 +---- .../tests/test_huggingface_hub.py | 44 +++------- 18 files changed, 77 insertions(+), 189 deletions(-) diff --git a/.agents/skills/add-dataset-connector/SKILL.md b/.agents/skills/add-dataset-connector/SKILL.md index 8f3ada4c0..0347e5ea5 100644 --- a/.agents/skills/add-dataset-connector/SKILL.md +++ b/.agents/skills/add-dataset-connector/SKILL.md @@ -42,8 +42,8 @@ latter). For PhysioNet, inspect the archive layout and a WFDB header (`fs`, `sig ### 3. Pick the base connector - Hub dataset: subclass `BaseHuggingFaceConnector`, pass the class keywords `hf_repo`, - `source_revision` (a commit), and `source_files`, inherit `download` and `convert`, and implement - `convert_files` over the part-to-path mapping. The base fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and `SOURCE_REVISION`. + `source_revision` (a commit), and `source_files`, inherit `download`, and implement `convert` + over the part-to-path mapping. The base fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and `SOURCE_REVISION`. - PhysioNet/WFDB: subclass `BasePhysioNetConnector`, implement `download` (use `ensure_archive` / `download_files` from `timenet_connectors.download`) and `convert` (use `_read_header` / `_lead_loader`). - Neither: subclass `BaseConnector[TRaw]` and implement `download` + `convert` yourself. diff --git a/.agents/skills/add-dataset-connector/references/connector-anatomy.md b/.agents/skills/add-dataset-connector/references/connector-anatomy.md index 65fe0618a..754d40bec 100644 --- a/.agents/skills/add-dataset-connector/references/connector-anatomy.md +++ b/.agents/skills/add-dataset-connector/references/connector-anatomy.md @@ -12,7 +12,7 @@ this branch. Anchor files: ## The `BaseConnector` contract `BaseConnector(ABC, Generic[TRaw])` in `timenet.connectors`. `TRaw` is the whole value that `download` hands to -`convert`: one handle dataclass, a list of file refs (`list[HubFileRef]` for Hub connectors), or `None`. +`convert`: one handle dataclass, a list of file refs, a part-to-path mapping (Hub connectors), or `None`. - `download(self, cache_dir: Path) -> TRaw` (abstract): fetch/discover raw source files, return a lightweight handle. I/O only, no parsing, idempotent for a given `cache_dir`. @@ -71,7 +71,7 @@ tags: ## Base connectors to reuse ### `BaseHuggingFaceConnector` (`bases/huggingface.py`) -`BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]])` downloads files from a Hub dataset at a pinned +`BaseHuggingFaceConnector(BaseConnector[Mapping[str, Path]])` downloads files from a Hub dataset at a pinned commit. Pass these class keywords: - `hf_repo`: the Hub repo id. @@ -87,11 +87,9 @@ and `get_card()` for a release that is not built, read the card file without the card sets one of the fields to a different value, the build fails. Do not copy `HF_REPO` or `SOURCE_REVISION` into dataset annotations. -`download` is inherited and returns one `HubFileRef(path, part)` per file. Declare +`download` is inherited. It returns the local path of each part, in `SOURCE_FILES` order. Declare `huggingface_hub>=1.1` in the connector's `requirements.txt`, and set `HF_TOKEN` for gated or -private datasets. Do not override `convert`. The base's `convert` checks the refs: they must be nonempty -and name each declared part at most once. Then it calls your -`convert_files(self, files: Mapping[str, Path])` with the local path of each part, so read a file as +private datasets. Implement `convert(self, files: Mapping[str, Path])` and read a file as `files["train"]`. To read the Hub's parquet conversion instead of the original files, pin the commit that @@ -221,7 +219,7 @@ class TSQAConnector( ): """Connector for the TSQA time-series QA dataset (Hub repo ``ChengsenWang/TSQA``).""" - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Build one record per row: the parsed series plus its question-and-answer task. Args: diff --git a/docs/connectors.md b/docs/connectors.md index 87a0438fa..e0c2922b1 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -296,8 +296,8 @@ PhysioNet connectors extend `BasePhysioNetConnector` for WFDB record I/O. They u pull their database archive. Hugging Face Hub connectors extend `BaseHuggingFaceConnector`. The base downloads the files in -`SOURCE_FILES` and implements `convert`. The connector implements `convert_files(files)` instead, where -`files` maps each part name to its local path. +`SOURCE_FILES`. Its `download` returns a mapping from each part name to its local path. The connector +implements `convert(self, files)` and reads a file as `files["train"]`. ## Example connectors @@ -411,7 +411,8 @@ See the [API reference for `timenet.connectors`](api/timenet.connectors.md) for ## Reading source rows For CSV, JSON arrays, JSON Lines, and Parquet, declare the source fields in a connector-owned -`DataRow` and read each downloaded file during `convert`. This example reads a list of file references: +`DataRow` and read each downloaded file during `convert`. This example is from a Hub connector. Its +`files` maps each part name to its local path: ```python from pydantic import Field @@ -421,8 +422,8 @@ class SourceRow(DataRow): question: str = Field(alias="Question") count: int = Field(alias="Count") -for ref in raw_refs: - for row in ParquetDataReader(ref.path, SourceRow).iter_rows(): +for path in files.values(): + for row in ParquetDataReader(path, SourceRow).iter_rows(): # Convert row.question and row.count into TimeNet objects. ... ``` diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py index 03a8684b9..71982a629 100644 --- a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py +++ b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py @@ -1,24 +1,21 @@ """A reusable base for connectors that read files from a pinned HuggingFace Hub dataset revision. Subclasses pass ``hf_repo``, ``source_revision``, and ``source_files`` as class keywords, ship a -``dataset.yaml`` card beside the connector, and implement ``convert_files()``. ``download()`` returns a -:class:`HubFileRef` list. The base's ``convert()`` checks that list, then passes -``convert_files()`` the path of each part. The base also fills ``source_url`` and ``source_revision`` -in the dataset metadata from ``HF_REPO`` and ``SOURCE_REVISION``. +``dataset.yaml`` card beside the connector, and implement ``convert()``. ``download()`` returns a mapping +from each part in ``SOURCE_FILES`` to its local path. ``convert()`` receives that mapping. The base also +fills ``source_url`` and ``source_revision`` in the dataset metadata from ``HF_REPO`` and ``SOURCE_REVISION``. :mod:`timenet_connectors.sources.huggingface_hub` describes how the files download. For the Hub's parquet conversion, pin the commit ``refs/convert/parquet`` points to. """ -from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from abc import ABC +from collections.abc import Mapping from pathlib import Path import re from types import MappingProxyType from typing import Any, ClassVar from timenet.connectors import BaseConnector -from timenet.dataset import TimeFDataset from timenet.errors import TimeFValidationError, TimeNetDownloadError from timenet.types import DatasetMetadata from timenet_connectors.sources.huggingface_hub import hub_snapshot @@ -27,23 +24,13 @@ _COMMIT_SHA = re.compile(r"[0-9a-f]{40}") -@dataclass(frozen=True) -class HubFileRef: - """A downloaded file (or a pattern's directory) and the part it belongs to.""" - - path: Path - part: str - """The connector's own name for this file, such as ``"hhar_v2"``: the key in ``SOURCE_FILES`` and in the - mapping that ``convert_files()`` receives.""" - - -class BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]], ABC): +class BaseHuggingFaceConnector(BaseConnector[Mapping[str, Path]], ABC): """Base class for connectors that read files from a pinned Hub dataset revision. Subclasses pass ``hf_repo``, ``source_revision``, and ``source_files`` as class keywords, and - implement :meth:`convert_files`. The base stores the keywords as :attr:`HF_REPO`, - :attr:`SOURCE_REVISION`, and :attr:`SOURCE_FILES`. A subclass that passes no keywords keeps the - values of its parent. :meth:`convert` runs :meth:`check_refs` first. + implement :meth:`convert`, which receives the local path of each part. The base stores the keywords + as :attr:`HF_REPO`, :attr:`SOURCE_REVISION`, and :attr:`SOURCE_FILES`. A subclass that passes no + keywords keeps the values of its parent. """ HF_REPO: ClassVar[str] @@ -102,66 +89,24 @@ def metadata(self) -> DatasetMetadata: ) return card.model_copy(update=source) - def check_refs(self, raw_refs: Sequence[HubFileRef]) -> None: - """Check that ``raw_refs`` is nonempty and names each part once, only parts :attr:`SOURCE_FILES` declares. - - Args: - raw_refs: The references :meth:`convert` received. - - Raises: - TimeFValidationError: If ``raw_refs`` is empty, repeats a part, or names an undeclared part. - """ - parts = [ref.part for ref in raw_refs] - if not parts or len(set(parts)) != len(parts): - raise TimeFValidationError(f"{type(self).__name__} requires nonempty, distinct part references") - unknown = sorted(set(parts) - self.SOURCE_FILES.keys()) - if unknown: - raise TimeFValidationError(f"{type(self).__name__} has no source part {', '.join(unknown)}") - - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: - """Check ``raw_refs``, then convert the files they name. - - Args: - raw_refs: The references from :meth:`download_async`. - - Returns: - The dataset from :meth:`convert_files`. - - Raises: - TimeFValidationError: If ``raw_refs`` is empty, repeats a part, or names an undeclared part. - """ # ruff: ignore[docstring-extraneous-exception] - raised by check_refs - self.check_refs(raw_refs) - return self.convert_files({ref.part: ref.path for ref in raw_refs}) - - @abstractmethod - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: - """Convert the downloaded files into a dataset. - - Args: - files: The local path of each part: a file, or the directory of a pattern part. - - Returns: - The populated dataset. - """ - - async def download_async(self, cache_dir: Path) -> list[HubFileRef]: + async def download_async(self, cache_dir: Path) -> dict[str, Path]: """Download :attr:`SOURCE_FILES` at :attr:`SOURCE_REVISION` concurrently. Args: cache_dir: Directory where the connector caches Hub files. Returns: - One reference per part, in :attr:`SOURCE_FILES` order. + The local path of each part, in :attr:`SOURCE_FILES` order: a file, or the directory of a pattern part. Raises: ImportError: If ``huggingface_hub``, declared in the connector's requirements, is missing. TimeNetDownloadError: If a declared file is absent from the revision or cannot be downloaded. """ # ruff: ignore[docstring-extraneous-exception] - also raised by hub_snapshot root = await hub_snapshot(self.HF_REPO, self.SOURCE_REVISION, cache_dir, tuple(self.SOURCE_FILES.values())) - refs = [] + files = {} for part, name in self.SOURCE_FILES.items(): path = root / name if "*" not in name else root / Path(name).parent if not path.exists(): raise TimeNetDownloadError(f"{self.HF_REPO}@{self.SOURCE_REVISION} has no {name}") - refs.append(HubFileRef(path, part)) - return refs + files[part] = path + return files diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py index 7ce5534e4..0781d7174 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/connector.py @@ -13,13 +13,15 @@ letters that the task does not use. """ +from collections.abc import Mapping +from pathlib import Path from typing import Self from pydantic import model_validator from timenet.dataset import OrdinalAxis, Record, Signal, Source, TimeFDataset from timenet.types import ClassificationTask, LabelSet, Split, TimeSeriesSpec, ureg -from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.sources.readers import DataRow, ParquetDataReader @@ -76,17 +78,17 @@ class TimeSeriesExam1Connector( ): """Build TimeSeriesExam from the pinned test parquet file of its Hugging Face release.""" - def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Build one Record per series and one classification task per question. Args: - raw_refs: The downloaded test file. + files: The downloaded test file. Returns: The populated :class:`~timenet.dataset.TimeFDataset`. """ dataset = TimeFDataset(metadata=self.metadata()) - for row in ParquetDataReader(raw_refs[0].path, _SourceRow).iter_rows(): + for row in ParquetDataReader(files["test"], _SourceRow).iter_rows(): inputs = [dataset.add_record(_record(row.id, role, values)) for role, values in row.series.items()] dataset.add_task( ClassificationTask( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py index 285334c29..d4f76e20d 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/autonlab/timeseriesexam1/tests/test_connector.py @@ -10,7 +10,6 @@ from timenet.reader import TimeFReader from timenet.registry import open_local_version from timenet.types import ClassificationTask, Split, read_value -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.autonlab.timeseriesexam1 import TimeSeriesExam1Connector @@ -28,7 +27,7 @@ def _convert(tmp_path: Path, rows: list[dict] | None = None) -> TimeFDataset: if rows is not None: path = tmp_path / "rows.parquet" pq.write_table(pa.Table.from_pylist(rows, schema=pq.read_schema(_FIXTURE)), path) - return TimeSeriesExam1Connector().convert([HubFileRef(path, "test")]) + return TimeSeriesExam1Connector().convert({"test": path}) def _task_for(dataset: TimeFDataset, row_id: int) -> ClassificationTask: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py index ac812fed4..690cf98ce 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py @@ -274,7 +274,7 @@ class SensorCapsConnector( SOURCE_ROWS: ClassVar[Mapping[str, int]] = _SOURCE_ROWS - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Index local rows into six-channel Records and a re-iterable caption stream. Args: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py index 2ffbdbcaf..f901e4ca6 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py @@ -14,7 +14,6 @@ from timenet.reader import TimeFReader from timenet.registry import open_local_version from timenet.types import AnswerTask, InputModality, Split, TimeSeriesSpec, read_value -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.bash_lab.sensorcaps import SensorCapsConnector from timenet_connectors.datasets.bash_lab.sensorcaps.connector import DECLARED_SAMPLING_RATE_HZ @@ -98,13 +97,13 @@ def _fixture_inventory(monkeypatch): def _refs(tmp_path): - refs = [] + refs = {} for partition in _PARTITIONS: path = tmp_path / f"{partition}.jsonl" path.write_text( "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in _rows(partition)), encoding="utf-8" ) - refs.append(HubFileRef(path, partition)) + refs[partition] = path return refs @@ -115,7 +114,7 @@ def _annotations(owner): def _write_row(tmp_path, row): path = tmp_path / "row.jsonl" path.write_text(json.dumps(row) + "\n", encoding="utf-8") - return HubFileRef(path, "hhar_v2") + return {"hhar_v2": path} def _assert_records_and_tasks(records, tasks): @@ -191,9 +190,9 @@ def test_conversion_preserves_source_semantics_and_reiterable_tasks(tmp_path): def test_signal_values_remain_available_after_source_removal(tmp_path): - refs = _refs(tmp_path)[:1] + refs = {"hhar_v2": _refs(tmp_path)["hhar_v2"]} dataset = SensorCapsConnector().convert(refs) - refs[0].path.unlink() + refs["hhar_v2"].unlink() first, second = dataset.records np.testing.assert_array_equal(first.signals[0].to_numpy(), _expected_values(0, 0, 0)) np.testing.assert_array_equal(second.signals[3].to_numpy(), _expected_values(0, 1, 3)) @@ -203,7 +202,7 @@ def test_jsonl_reader_rejects_duplicate_keys(tmp_path): path = tmp_path / "duplicate.jsonl" path.write_text('{"messages":[],"messages":[]}\n', encoding="utf-8") with pytest.raises(TimeNetConnectorError, match="duplicate key") as error: - SensorCapsConnector().convert([HubFileRef(path, "hhar_v2")]) + SensorCapsConnector().convert({"hhar_v2": path}) assert str(path) in str(error.value) assert "line 1" in str(error.value) @@ -260,8 +259,8 @@ def test_invalid_sensor_data_has_source_context(tmp_path, sensor, pattern): row["messages"][1]["content"] = "\n".join(lines) ref = _write_row(tmp_path, row) with pytest.raises(TimeFValidationError, match=pattern) as error: - SensorCapsConnector().convert([ref]) - assert str(ref.path) in str(error.value) + SensorCapsConnector().convert(ref) + assert str(ref["hhar_v2"]) in str(error.value) assert "hhar_v2, row 0" in str(error.value) @@ -274,7 +273,7 @@ def test_integer_and_float_sensor_readings_are_accepted(monkeypatch, tmp_path): if heading in {"Accelerometer", "Gyroscope"}: lines[index] = f"{heading}: {json.dumps([[1, 2.5, -3]] * 60)}" row["messages"][1]["content"] = "\n".join(lines) - dataset = SensorCapsConnector().convert([_write_row(tmp_path, row)]) + dataset = SensorCapsConnector().convert(_write_row(tmp_path, row)) for signal, expected in zip(dataset.records[0].signals, [1, 2.5, -3] * 2, strict=True): np.testing.assert_array_equal(signal.to_numpy(), np.full(60, expected, dtype=np.float32)) @@ -282,7 +281,7 @@ def test_integer_and_float_sensor_readings_are_accepted(monkeypatch, tmp_path): def test_pinned_inventory_mismatch_fails(monkeypatch, tmp_path): monkeypatch.setitem(SensorCapsConnector.SOURCE_ROWS, "hhar_v2", 3) with pytest.raises(TimeFValidationError, match=r"hhar_v2\): expected 3 rows, found 2"): - SensorCapsConnector().convert(_refs(tmp_path)[:1]) + SensorCapsConnector().convert({"hhar_v2": _refs(tmp_path)["hhar_v2"]}) @pytest.mark.parametrize("n_readings", [1, 59, 61]) @@ -296,5 +295,5 @@ def test_unexpected_sample_count_rejects_timing_reconstruction(tmp_path, n_readi row["messages"][1]["content"] = "\n".join(lines) ref = _write_row(tmp_path, row) with pytest.raises(TimeFValidationError, match="expected 60 readings per sensor") as error: - SensorCapsConnector().convert([ref]) - assert str(ref.path) in str(error.value) + SensorCapsConnector().convert(ref) + assert str(ref["hhar_v2"]) in str(error.value) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py index 6b5e9295e..12acffa4c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py @@ -62,7 +62,7 @@ def __init__(self) -> None: seed=42, ) - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Build one record per row: the parsed series plus its question-and-answer task. Args: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/tests/test_connector.py index 146e0d4c3..71dce1efa 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/tests/test_connector.py @@ -10,7 +10,6 @@ from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version from timenet.types import AnswerTask, Split, read_value -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.chengsenwang.tsqa import TSQAConnector @@ -64,7 +63,7 @@ def _refs(tmp_path, rows): # Write the rows to a parquet file shaped like the Hub's converted copy; convert() reads it back. path = tmp_path / f"rows-{len(list(tmp_path.iterdir()))}.parquet" pq.write_table(pa.Table.from_pylist(rows), path) - return [HubFileRef(path, "train")] + return {"train": path} def _convert(tmp_path) -> TimeFDataset: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py index c9af56464..536a6af80 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/connector.py @@ -37,7 +37,7 @@ class ARFBenchConnector( The values are plain series, so the dataset uses the default Parquet backend. """ - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Create a Record for each metric and a series task for each question. Args: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/release.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/release.py index bf489c80c..e7528f273 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/release.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/release.py @@ -12,7 +12,6 @@ import pyarrow.parquet as pq from timenet.types import SupportsAnnotate, read_value -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.datadog.arfbench.connector import QA_CSV, SERIES_DIR from timenet_connectors.datasets.datadog.arfbench.questions import QUESTION_COLUMNS @@ -116,13 +115,10 @@ def facts(annotated: SupportsAnnotate) -> dict[str, object]: return {annotation.key: read_value(annotation.value) for annotation in annotated.annotations} -def write_release(root: Path) -> list[HubFileRef]: +def write_release(root: Path) -> dict[str, Path]: """Write the question table and the series files. Returns: - The refs that the series connector downloads. + The files that the series connector downloads. """ - return [ - HubFileRef(write_qa_csv(root, ROWS), "questions"), - HubFileRef(write_series_dir(root), "series"), - ] + return {"questions": write_qa_csv(root, ROWS), "series": write_series_dir(root)} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py index 6b598e234..408ace7b9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py @@ -14,7 +14,6 @@ from timenet.registry import open_local_version from timenet.types import ClassificationTask, InputModality, MissingReason, Split, read_value, read_values from timenet.writer import TimeFWriter -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.datadog.arfbench.connector import ARFBenchConnector from timenet_connectors.datasets.datadog.arfbench.questions import QUESTION_COLUMNS from timenet_connectors.datasets.datadog.arfbench.tests.release import ( @@ -30,15 +29,11 @@ @pytest.fixture -def release(tmp_path: Path) -> list[HubFileRef]: +def release(tmp_path: Path) -> dict[str, Path]: """A release with two metrics and one question about each metric count.""" return write_release(tmp_path) -def _path(refs: list[HubFileRef], part: str) -> Path: - return next(ref.path for ref in refs if ref.part == part) - - def test_a_metric_record_keeps_every_rollup_per_tag_group(release): dataset = ARFBenchConnector().convert(release) first, second = dataset.records @@ -110,7 +105,7 @@ def test_each_question_is_a_classification_task_over_its_options(release): def test_questions_with_equal_options_in_equal_order_share_a_label_set(release): up_down = {**PRESENCE_ROW, "Unnamed: 0": "2", "options_str": '["Up", "Down"]', "correct_answer": "Up"} no_yes = {**PRESENCE_ROW, "Unnamed: 0": "3", "options_str": '["No", "Yes"]', "correct_answer": "No"} - write_qa_csv(_path(release, "questions").parent, [*ROWS, up_down, no_yes]) + write_qa_csv(release["questions"].parent, [*ROWS, up_down, no_yes]) presence, correlation, up_down_task, no_yes_task = ARFBenchConnector().convert(release).get_all() assert isinstance(presence, ClassificationTask) and isinstance(correlation, ClassificationTask) @@ -166,25 +161,25 @@ def test_the_dataset_round_trips_through_parquet_with_its_missing_timesteps(rele ) def test_a_malformed_question_raises_with_its_location(release, change, message): rows = [*ROWS, {**CORRELATION_ROW, "Unnamed: 0": "2"} | change] - write_qa_csv(_path(release, "questions").parent, rows) + write_qa_csv(release["questions"].parent, rows) with pytest.raises(TimeNetConnectorError, match=rf"(arfbench-qa\.csv )?{message}"): ARFBenchConnector().convert(release) def test_an_unexpected_header_raises(release): - write_qa_csv(_path(release, "questions").parent, [PRESENCE_ROW], columns=QUESTION_COLUMNS[1:]) + write_qa_csv(release["questions"].parent, [PRESENCE_ROW], columns=QUESTION_COLUMNS[1:]) with pytest.raises(TimeNetConnectorError, match=r"arfbench-qa\.csv: expected columns"): ARFBenchConnector().convert(release) def test_a_row_off_the_rollup_grid_raises(release): - write_series_file(_path(release, "series") / "35997_1_10.parquet", [(30, "", 0.5), (35, "", 0.6)]) + write_series_file(release["series"] / "35997_1_10.parquet", [(30, "", 0.5), (35, "", 0.6)]) with pytest.raises(TimeNetConnectorError, match=r"35997_1_10\.parquet group '': rows are off the 10 s grid"): ARFBenchConnector().convert(release) def test_a_series_file_without_an_interval_raises(release): - series_dir = _path(release, "series") + series_dir = release["series"] (series_dir / "35997_1_10.parquet").rename(series_dir / "35997_1_10s.parquet") with pytest.raises(TimeNetConnectorError, match=r"35997_1_10s\.parquet is not named"): ARFBenchConnector().convert(release) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py index fa7ecedb9..fbca28f1a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/connector.py @@ -45,7 +45,7 @@ class ARFBenchChartsConnector( values_backend = ValuesBackend.ZARR # the chart tensors need the Zarr backend - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Create a Record for each chart and a chart task for each question. Args: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/tests/test_connector.py index a1a0cf502..99c058161 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench_charts/tests/test_connector.py @@ -14,7 +14,6 @@ from timenet.registry import open_local_version from timenet.types import ClassificationTask, InputModality, Split, read_values from timenet.writer import TimeFWriter -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.datadog.arfbench.tests.release import CORRELATION_ROW, ROWS, write_qa_csv from timenet_connectors.datasets.datadog.arfbench_charts.connector import CHART_DIR, ARFBenchChartsConnector @@ -29,9 +28,9 @@ def _write_chart_dir(root: Path) -> Path: @pytest.fixture -def release(tmp_path: Path) -> list[HubFileRef]: +def release(tmp_path: Path) -> dict[str, Path]: """The question table and the three charts of the test release.""" - return [HubFileRef(write_qa_csv(tmp_path, ROWS), "questions"), HubFileRef(_write_chart_dir(tmp_path), "charts")] + return {"questions": write_qa_csv(tmp_path, ROWS), "charts": _write_chart_dir(tmp_path)} def _ids(records) -> list[str]: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index 0d7c9cbf5..a4263f400 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -746,7 +746,7 @@ class TSAQAConnector( ): """Build TSAQA from the pinned train, val and test parquet files of its Hugging Face release.""" - def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: """Convert each split file into lazy Records and a stream of one task per row. Every file is read once up front to check each row and keep its small fields. The source issues @@ -760,12 +760,8 @@ def convert_files(self, files: Mapping[str, Path]) -> TimeFDataset: A dataset with the provenance annotations, the Records, and a stream of every task. Raises: - TimeFValidationError: If a split file is missing, or a row is malformed. - """ - if files.keys() != _SPLITS.keys(): - raise TimeFValidationError( - f"TSAQA needs all its split files, missing {sorted(split.value for split in _SplitFile if split not in files)}" - ) + TimeFValidationError: If a row is malformed. + """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_rows rows: list[_Row] = [] for split in _SplitFile: rows.extend(_read_rows(files[split], split)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py index 112151423..fc9c24ec4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py @@ -15,7 +15,6 @@ from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version from timenet.types import AnswerTask, ClassificationTask, Split -from timenet_connectors.bases.huggingface import HubFileRef from timenet_connectors.datasets.tsaqa.tsaqa_benchmark import connector from timenet_connectors.datasets.tsaqa.tsaqa_benchmark.connector import ( SourceIssue, @@ -50,8 +49,8 @@ class _FixtureConnector(TSAQAConnector): CARD = Path(__file__).parents[1] / "dataset.yaml" -def _refs(directory: Path = _FIXTURES) -> list[HubFileRef]: - return [HubFileRef(directory / f"{split}.parquet", split) for split in _SPLITS] +def _refs(directory: Path = _FIXTURES) -> dict[str, Path]: + return {split: directory / f"{split}.parquet" for split in _SPLITS} def _published() -> dict[tuple[str, int], dict[str, Any]]: @@ -285,16 +284,6 @@ def test_declares_the_pinned_release(): } -def test_a_file_that_names_no_split_fails(): - with pytest.raises(TimeFValidationError, match="has no source part dev"): - _FixtureConnector().convert([HubFileRef(_FIXTURES / "val.parquet", "dev")]) - - -def test_no_split_file_fails(): - with pytest.raises(TimeFValidationError, match="nonempty"): - _FixtureConnector().convert([]) - - def _rewrite(path: Path, position: int, **columns: str) -> None: """Rewrite one row of a fixture file in place, keeping its schema.""" table = pq.read_table(path) @@ -398,12 +387,6 @@ def test_a_printed_series_that_loses_its_line_fails_naming_the_labels(tmp_path): _FixtureConnector().convert(_refs(tmp_path)) -def test_a_conversion_needs_every_split_file(): - refs = [ref for ref in _refs() if ref.part == "test"] - with pytest.raises(TimeFValidationError, match=r"missing \['train', 'val'\]"): - _FixtureConnector().convert(refs) - - def test_a_file_that_is_not_parquet_fails_as_a_format_error(tmp_path): shutil.copytree(_FIXTURES, tmp_path, dirs_exist_ok=True) (tmp_path / "test.parquet").write_text("not parquet") diff --git a/packages/timenet-connectors/tests/test_huggingface_hub.py b/packages/timenet-connectors/tests/test_huggingface_hub.py index 34382f782..65ec6a9c9 100644 --- a/packages/timenet-connectors/tests/test_huggingface_hub.py +++ b/packages/timenet-connectors/tests/test_huggingface_hub.py @@ -10,7 +10,7 @@ import pytest from timenet.errors import TimeFValidationError, TimeNetDownloadError -from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.download.progress import DownloadProgress, progress_sink from timenet_connectors.sources.huggingface_hub import hub_snapshot @@ -24,7 +24,7 @@ class _Pinned( source_revision=_SHA, source_files={"train": "data/train.jsonl", "test": "data/test.jsonl"}, ): - def convert_files(self, files): + def convert(self, files): return files @@ -74,15 +74,15 @@ def _snapshot_bars(tqdm_class): file_bytes.update(6) -def test_download_returns_one_ref_per_source_file_in_declared_order(monkeypatch, tmp_path): +def test_download_returns_the_path_of_each_part_in_declared_order(monkeypatch, tmp_path): # No sink is installed, so the progress class must stay silent without one. calls = _fake_hub(monkeypatch, tmp_path, bars=_snapshot_bars) - refs = asyncio.run(_Pinned().download_async(tmp_path / "cache")) + files = asyncio.run(_Pinned().download_async(tmp_path / "cache")) - assert refs == [ - HubFileRef(tmp_path / "snapshot" / "data/train.jsonl", "train"), - HubFileRef(tmp_path / "snapshot" / "data/test.jsonl", "test"), + assert list(files.items()) == [ + ("train", tmp_path / "snapshot" / "data/train.jsonl"), + ("test", tmp_path / "snapshot" / "data/test.jsonl"), ] ((repo, kwargs),) = calls assert issubclass(kwargs.pop("tqdm_class"), _HubTqdm) @@ -159,27 +159,6 @@ def test_subclass_stores_the_keywords_and_inherits_them(): ) -@pytest.mark.parametrize( - ("parts", "message"), - [ - ((), r"requires nonempty, distinct part references"), - (("train", "train"), r"requires nonempty, distinct part references"), - (("train", "valid", "dev"), r"has no source part dev, valid$"), - ], -) -def test_check_refs_rejects_empty_repeated_or_undeclared_parts(parts, message): - refs = [HubFileRef(Path(f"{part}.jsonl"), part) for part in parts] - with pytest.raises(TimeFValidationError, match=message): - _Pinned().check_refs(refs) - - -def test_convert_checks_refs_then_passes_the_path_of_each_part(): - with pytest.raises(TimeFValidationError, match=r"has no source part dev$"): - _Pinned().convert([HubFileRef(Path("dev.jsonl"), "dev")]) - refs = [HubFileRef(Path("test.jsonl"), "test")] - assert _Pinned().convert(refs) == {"test": Path("test.jsonl")} - - def test_snapshot_downloads_the_matching_files_and_returns_the_snapshot_root(monkeypatch, tmp_path): calls = _fake_hub(monkeypatch, tmp_path, files=("meta.csv", "data/train.parquet", "README.md")) @@ -198,19 +177,16 @@ class _Patterned( source_revision=_SHA, source_files={"questions": "qa.csv", "series": "data/*.parquet"}, ): - def convert_files(self, files): + def convert(self, files): return files def test_a_pattern_part_resolves_to_the_directory_of_its_files(monkeypatch, tmp_path): calls = _fake_hub(monkeypatch, tmp_path, files=("qa.csv", "data/a.parquet", "data/b.parquet", "README.md")) - refs = asyncio.run(_Patterned().download_async(tmp_path / "cache")) + files = asyncio.run(_Patterned().download_async(tmp_path / "cache")) - assert refs == [ - HubFileRef(tmp_path / "snapshot" / "qa.csv", "questions"), - HubFileRef(tmp_path / "snapshot" / "data", "series"), - ] + assert files == {"questions": tmp_path / "snapshot" / "qa.csv", "series": tmp_path / "snapshot" / "data"} assert calls[0][1]["allow_patterns"] == ["qa.csv", "data/*.parquet"]