diff --git a/.agents/skills/add-dataset-connector/SKILL.md b/.agents/skills/add-dataset-connector/SKILL.md index 0b32a171e..f68aed4af 100644 --- a/.agents/skills/add-dataset-connector/SKILL.md +++ b/.agents/skills/add-dataset-connector/SKILL.md @@ -42,7 +42,8 @@ latter). For PhysioNet, inspect the archive layout and a WFDB header (`fs`, `sig ### 3. Pick the base connector - Hub dataset: subclass `BaseHuggingFaceConnector`, set `HF_REPO`, `SOURCE_REVISION` (a commit), and - `SOURCE_FILES`, inherit `download`, and implement `convert` over the `HubFileRef` list. + `SOURCE_FILES`, inherit `download`, and implement `convert` over the `HubFileRef` list. The base + fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and `SOURCE_REVISION`. - PhysioNet/WFDB: subclass `BasePhysioNetConnector`, implement `download` (use `ensure_archive` / `download_files` from `timenet_connectors.download`) and `convert` (use `_read_header` / `_lead_loader`). - Neither: subclass `BaseConnector[TRaw]` and implement `download` + `convert` yourself. diff --git a/.agents/skills/add-dataset-connector/references/connector-anatomy.md b/.agents/skills/add-dataset-connector/references/connector-anatomy.md index 45463b585..14574a14f 100644 --- a/.agents/skills/add-dataset-connector/references/connector-anatomy.md +++ b/.agents/skills/add-dataset-connector/references/connector-anatomy.md @@ -77,6 +77,12 @@ commit. Set: - `SOURCE_REVISION`: a 40-character commit, not a branch, so every rebuild reads the same files. - `SOURCE_FILES`: part name -> file path in the repo. +The base fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and +`SOURCE_REVISION`. The card can omit both fields. Keep `source_url` in the card. The docs catalog, +and `get_card()` for a release that is not built, read the card file without the connector. If the +card sets one of the fields to a different value, the build fails. Do not copy `HF_REPO` or +`SOURCE_REVISION` into dataset annotations. + `download` is inherited and returns one `HubFileRef(path, part)` per file. Declare `huggingface_hub>=1.1` in the connector's `requirements.txt`, and set `HF_TOKEN` for gated or private datasets. The base checks the refs before your `convert` runs: they must be nonempty and diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py index e1a83ec49..091ad8796 100644 --- a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py +++ b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py @@ -2,7 +2,8 @@ Subclasses set ``HF_REPO``, ``SOURCE_REVISION``, and ``SOURCE_FILES``, ship a ``dataset.yaml`` card beside the connector, and implement ``convert()`` over the :class:`HubFileRef` list ``download()`` -returns. The base checks that list before the subclass's ``convert()`` runs. +returns. The base checks that list before the subclass's ``convert()`` runs. It also fills +``source_url`` and ``source_revision`` in the dataset metadata from ``HF_REPO`` and ``SOURCE_REVISION``. :mod:`timenet_connectors.sources.huggingface_hub` describes how the files download. For the Hub's parquet conversion, pin the commit ``refs/convert/parquet`` points to. """ @@ -18,6 +19,7 @@ from timenet.connectors import BaseConnector from timenet.dataset import TimeFDataset from timenet.errors import TimeFValidationError, TimeNetConnectorError, TimeNetDownloadError +from timenet.types import DatasetMetadata from timenet_connectors.sources.huggingface_hub import hub_snapshot @@ -70,6 +72,30 @@ def checked(self: BaseHuggingFaceConnector, raw_refs: list[HubFileRef]) -> TimeF cls.convert = checked + def metadata(self) -> DatasetMetadata: + """Return the card, with ``source_url`` and ``source_revision`` from :attr:`HF_REPO` and :attr:`SOURCE_REVISION`. + + The card can omit both fields. If it sets one, the value must match the connector. + + Returns: + The dataset's :class:`~timenet.types.DatasetMetadata`. + + Raises: + TimeFValidationError: If the card sets ``source_url`` or ``source_revision`` to another value. + """ + card = super().metadata() + source = { + "source_url": f"https://huggingface.co/datasets/{self.HF_REPO}", + "source_revision": self.SOURCE_REVISION, + } + for field, value in source.items(): + declared = getattr(card, field) + if declared is not None and declared != value: + raise TimeFValidationError( + f"{type(self).__name__} card sets {field} to {declared!r}, but the connector reads {value!r}" + ) + return card.model_copy(update=source) + def check_refs(self, raw_refs: Sequence[HubFileRef]) -> None: """Check that ``raw_refs`` is nonempty and names each part once, only parts :attr:`SOURCE_FILES` declares. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py index 3bee9d6fd..32d4c2d55 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py @@ -280,8 +280,6 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: """ dataset = TimeFDataset(metadata=self.metadata()) for key, value in ( - ("source_repository", self.HF_REPO), - ("source_revision", self.SOURCE_REVISION), ("declared_sampling_rate_hz", DECLARED_SAMPLING_RATE_HZ), ("source_sampling_rate_hz", _SOURCE_SAMPLING_RATE_HZ), ("source_window_samples", _SOURCE_WINDOW_SAMPLES), diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py index a70877ca8..4a780f81e 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py @@ -42,10 +42,7 @@ ("gyroscope", "z"), ("magnetometer", "x"), ) -# Literal pins, so moving to another repo or commit is a deliberate test edit. _DATASET_ANNOTATIONS = { - "source_repository": "BASH-Lab/SensorCaps", - "source_revision": "4242e07feaacb0ee3aff86182a62914fa7abacef", "declared_sampling_rate_hz": DECLARED_SAMPLING_RATE_HZ, "source_sampling_rate_hz": 20, "source_window_samples": 120, @@ -236,6 +233,8 @@ async def download_async(self, cache_dir): records = list(reader.iter_records()) _assert_records_and_tasks(records, list(reader.iter_tasks(inputs=records))) assert _annotations(restored) == _DATASET_ANNOTATIONS + assert restored.metadata.source_url == "https://huggingface.co/datasets/BASH-Lab/SensorCaps" + assert restored.metadata.source_revision == "4242e07feaacb0ee3aff86182a62914fa7abacef" reader.verify() diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index ad105afe7..589757b4c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -775,8 +775,6 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: records = [_records(row, loader) for row in rows] dataset = TimeFDataset(metadata=self.metadata()) - for key, value in (("source_repository", self.HF_REPO), ("source_revision", self.SOURCE_REVISION)): - dataset.annotate(Annotation(key=key, value=value)) # The model reads the published task and question_type, so each distinct value is a dataset annotation. labels = { label: dataset.annotate(Annotation(key=label[0], value=label[1])) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py index f97e78213..a5d29c807 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py @@ -129,10 +129,9 @@ def test_storage_round_trip_keeps_the_whole_conversion(tmp_path): published = _published() with TimeFReader(open_local_version(registry.root / "tsaqa/tsaqa-benchmark/1.0.0")) as reader: restored = reader.read() - assert {a.key: a.value for a in restored.annotations if a.key not in {"task", "question_type"}} == { - "source_repository": "TSAQA/TSAQA-Benchmark", - "source_revision": "883dcb0f96e0d1c8fd3806f35548ff1f2058a9cd", - } + assert all(a.key in {"task", "question_type"} for a in restored.annotations) + assert restored.metadata.source_url == "https://huggingface.co/datasets/TSAQA/TSAQA-Benchmark" + assert restored.metadata.source_revision == "883dcb0f96e0d1c8fd3806f35548ff1f2058a9cd" labels = [(a.key, a.value) for a in restored.annotations if a.key in {"task", "question_type"}] expected = {("task", row["task"]) for row in published.values()} expected |= {("question_type", row["question_type"]) for row in published.values()} diff --git a/packages/timenet-connectors/tests/test_huggingface_hub.py b/packages/timenet-connectors/tests/test_huggingface_hub.py index 080345cc1..69cd9dd78 100644 --- a/packages/timenet-connectors/tests/test_huggingface_hub.py +++ b/packages/timenet-connectors/tests/test_huggingface_hub.py @@ -210,3 +210,35 @@ def test_a_declared_file_or_pattern_without_a_match_fails(monkeypatch, tmp_path) _fake_hub(monkeypatch, tmp_path, files=("qa.csv", "README.md")) with pytest.raises(TimeNetDownloadError, match=rf"org/repo@{_SHA} has no data/\*\.parquet$"): asyncio.run(_Patterned().download_async(tmp_path / "cache")) + + +_TSQA_CARD = Path(__file__).parents[1] / "src/timenet_connectors/datasets/chengsenwang/tsqa/dataset.yaml" + + +def _card(tmp_path: Path, **fields: str) -> Path: + lines = [line for line in _TSQA_CARD.read_text().splitlines() if not line.startswith("source_")] + path = tmp_path / "dataset.yaml" + path.write_text("\n".join([*lines, *(f'{key}: "{value}"' for key, value in fields.items())]) + "\n") + return path + + +def test_metadata_takes_the_source_from_the_connector(tmp_path): + connector = type("_Carded", (_Pinned,), {"CARD": _card(tmp_path)})() + metadata = connector.metadata() + assert metadata.source_url == "https://huggingface.co/datasets/org/repo" + assert metadata.source_revision == _SHA + + +def test_metadata_accepts_a_card_that_repeats_the_source(tmp_path): + card = _card(tmp_path, source_url="https://huggingface.co/datasets/org/repo", source_revision=_SHA) + assert type("_Carded", (_Pinned,), {"CARD": card})().metadata().source_revision == _SHA + + +@pytest.mark.parametrize( + ("field", "value"), + [("source_url", "https://huggingface.co/datasets/org/other"), ("source_revision", "1" * 40)], +) +def test_metadata_rejects_a_card_source_that_differs(tmp_path, field, value): + connector = type("_Carded", (_Pinned,), {"CARD": _card(tmp_path, **{field: value})})() + with pytest.raises(TimeFValidationError, match=f"card sets {field}"): + connector.metadata()