Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .agents/skills/add-dataset-connector/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ latter). For PhysioNet, inspect the archive layout and a WFDB header (`fs`, `sig

### 3. Pick the base connector
- Hub dataset: subclass `BaseHuggingFaceConnector`, set `HF_REPO`, `SOURCE_REVISION` (a commit), and
`SOURCE_FILES`, inherit `download`, and implement `convert` over the `HubFileRef` list.
`SOURCE_FILES`, inherit `download`, and implement `convert` over the `HubFileRef` list. The base
fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and `SOURCE_REVISION`.
- PhysioNet/WFDB: subclass `BasePhysioNetConnector`, implement `download` (use `ensure_archive` /
`download_files` from `timenet_connectors.download`) and `convert` (use `_read_header` / `_lead_loader`).
- Neither: subclass `BaseConnector[TRaw]` and implement `download` + `convert` yourself.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ commit. Set:
- `SOURCE_REVISION`: a 40-character commit, not a branch, so every rebuild reads the same files.
- `SOURCE_FILES`: part name -> file path in the repo.

The base fills `source_url` and `source_revision` in the dataset metadata from `HF_REPO` and
`SOURCE_REVISION`. The card can omit both fields. Keep `source_url` in the card. The docs catalog,
and `get_card()` for a release that is not built, read the card file without the connector. If the
card sets one of the fields to a different value, the build fails. Do not copy `HF_REPO` or
`SOURCE_REVISION` into dataset annotations.

`download` is inherited and returns one `HubFileRef(path, part)` per file. Declare
`huggingface_hub>=1.1` in the connector's `requirements.txt`, and set `HF_TOKEN` for gated or
private datasets. The base checks the refs before your `convert` runs: they must be nonempty and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

Subclasses set ``HF_REPO``, ``SOURCE_REVISION``, and ``SOURCE_FILES``, ship a ``dataset.yaml`` card
beside the connector, and implement ``convert()`` over the :class:`HubFileRef` list ``download()``
returns. The base checks that list before the subclass's ``convert()`` runs.
returns. The base checks that list before the subclass's ``convert()`` runs. It also fills
``source_url`` and ``source_revision`` in the dataset metadata from ``HF_REPO`` and ``SOURCE_REVISION``.
:mod:`timenet_connectors.sources.huggingface_hub` describes how the files download. For the Hub's
parquet conversion, pin the commit ``refs/convert/parquet`` points to.
"""
Expand All @@ -18,6 +19,7 @@
from timenet.connectors import BaseConnector
from timenet.dataset import TimeFDataset
from timenet.errors import TimeFValidationError, TimeNetConnectorError, TimeNetDownloadError
from timenet.types import DatasetMetadata
from timenet_connectors.sources.huggingface_hub import hub_snapshot


Expand Down Expand Up @@ -70,6 +72,30 @@ def checked(self: BaseHuggingFaceConnector, raw_refs: list[HubFileRef]) -> TimeF

cls.convert = checked

def metadata(self) -> DatasetMetadata:
"""Return the card, with ``source_url`` and ``source_revision`` from :attr:`HF_REPO` and :attr:`SOURCE_REVISION`.

The card can omit both fields. If it sets one, the value must match the connector.

Returns:
The dataset's :class:`~timenet.types.DatasetMetadata`.

Raises:
TimeFValidationError: If the card sets ``source_url`` or ``source_revision`` to another value.
"""
card = super().metadata()
source = {
"source_url": f"https://huggingface.co/datasets/{self.HF_REPO}",
"source_revision": self.SOURCE_REVISION,
}
for field, value in source.items():
declared = getattr(card, field)
if declared is not None and declared != value:
raise TimeFValidationError(
f"{type(self).__name__} card sets {field} to {declared!r}, but the connector reads {value!r}"
)
return card.model_copy(update=source)

def check_refs(self, raw_refs: Sequence[HubFileRef]) -> None:
"""Check that ``raw_refs`` is nonempty and names each part once, only parts :attr:`SOURCE_FILES` declares.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,6 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset:
"""
dataset = TimeFDataset(metadata=self.metadata())
for key, value in (
("source_repository", self.HF_REPO),
("source_revision", self.SOURCE_REVISION),
("declared_sampling_rate_hz", DECLARED_SAMPLING_RATE_HZ),
("source_sampling_rate_hz", _SOURCE_SAMPLING_RATE_HZ),
("source_window_samples", _SOURCE_WINDOW_SAMPLES),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,7 @@
("gyroscope", "z"),
("magnetometer", "x"),
)
# Literal pins, so moving to another repo or commit is a deliberate test edit.
_DATASET_ANNOTATIONS = {
"source_repository": "BASH-Lab/SensorCaps",
"source_revision": "4242e07feaacb0ee3aff86182a62914fa7abacef",
"declared_sampling_rate_hz": DECLARED_SAMPLING_RATE_HZ,
"source_sampling_rate_hz": 20,
"source_window_samples": 120,
Expand Down Expand Up @@ -236,6 +233,8 @@ async def download_async(self, cache_dir):
records = list(reader.iter_records())
_assert_records_and_tasks(records, list(reader.iter_tasks(inputs=records)))
assert _annotations(restored) == _DATASET_ANNOTATIONS
assert restored.metadata.source_url == "https://huggingface.co/datasets/BASH-Lab/SensorCaps"
assert restored.metadata.source_revision == "4242e07feaacb0ee3aff86182a62914fa7abacef"
reader.verify()


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -775,8 +775,6 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset:
records = [_records(row, loader) for row in rows]

dataset = TimeFDataset(metadata=self.metadata())
for key, value in (("source_repository", self.HF_REPO), ("source_revision", self.SOURCE_REVISION)):
dataset.annotate(Annotation(key=key, value=value))
# The model reads the published task and question_type, so each distinct value is a dataset annotation.
labels = {
label: dataset.annotate(Annotation(key=label[0], value=label[1]))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,9 @@ def test_storage_round_trip_keeps_the_whole_conversion(tmp_path):
published = _published()
with TimeFReader(open_local_version(registry.root / "tsaqa/tsaqa-benchmark/1.0.0")) as reader:
restored = reader.read()
assert {a.key: a.value for a in restored.annotations if a.key not in {"task", "question_type"}} == {
"source_repository": "TSAQA/TSAQA-Benchmark",
"source_revision": "883dcb0f96e0d1c8fd3806f35548ff1f2058a9cd",
}
assert all(a.key in {"task", "question_type"} for a in restored.annotations)
assert restored.metadata.source_url == "https://huggingface.co/datasets/TSAQA/TSAQA-Benchmark"
assert restored.metadata.source_revision == "883dcb0f96e0d1c8fd3806f35548ff1f2058a9cd"
labels = [(a.key, a.value) for a in restored.annotations if a.key in {"task", "question_type"}]
expected = {("task", row["task"]) for row in published.values()}
expected |= {("question_type", row["question_type"]) for row in published.values()}
Expand Down
32 changes: 32 additions & 0 deletions packages/timenet-connectors/tests/test_huggingface_hub.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,3 +210,35 @@ def test_a_declared_file_or_pattern_without_a_match_fails(monkeypatch, tmp_path)
_fake_hub(monkeypatch, tmp_path, files=("qa.csv", "README.md"))
with pytest.raises(TimeNetDownloadError, match=rf"org/repo@{_SHA} has no data/\*\.parquet$"):
asyncio.run(_Patterned().download_async(tmp_path / "cache"))


_TSQA_CARD = Path(__file__).parents[1] / "src/timenet_connectors/datasets/chengsenwang/tsqa/dataset.yaml"


def _card(tmp_path: Path, **fields: str) -> Path:
lines = [line for line in _TSQA_CARD.read_text().splitlines() if not line.startswith("source_")]
path = tmp_path / "dataset.yaml"
path.write_text("\n".join([*lines, *(f'{key}: "{value}"' for key, value in fields.items())]) + "\n")
return path


def test_metadata_takes_the_source_from_the_connector(tmp_path):
connector = type("_Carded", (_Pinned,), {"CARD": _card(tmp_path)})()
metadata = connector.metadata()
assert metadata.source_url == "https://huggingface.co/datasets/org/repo"
assert metadata.source_revision == _SHA


def test_metadata_accepts_a_card_that_repeats_the_source(tmp_path):
card = _card(tmp_path, source_url="https://huggingface.co/datasets/org/repo", source_revision=_SHA)
assert type("_Carded", (_Pinned,), {"CARD": card})().metadata().source_revision == _SHA


@pytest.mark.parametrize(
("field", "value"),
[("source_url", "https://huggingface.co/datasets/org/other"), ("source_revision", "1" * 40)],
)
def test_metadata_rejects_a_card_source_that_differs(tmp_path, field, value):
connector = type("_Carded", (_Pinned,), {"CARD": _card(tmp_path, **{field: value})})()
with pytest.raises(TimeFValidationError, match=f"card sets {field}"):
connector.metadata()