diff --git a/.agents/skills/add-dataset-connector/references/connector-anatomy.md b/.agents/skills/add-dataset-connector/references/connector-anatomy.md
index e6581907a..ca351cdc6 100644
--- a/.agents/skills/add-dataset-connector/references/connector-anatomy.md
+++ b/.agents/skills/add-dataset-connector/references/connector-anatomy.md
@@ -11,13 +11,14 @@ this branch. Anchor files:
## The `BaseConnector` contract
-`BaseConnector(ABC, Generic[TRaw])` in `timenet.connectors`. `TRaw` is whatever `download` hands to
-`convert` (a dict per Hub row, a dataclass ref per PhysioNet record, etc.).
-
-- `download(self, cache_dir: Path) -> list[TRaw]` (abstract): fetch/discover raw source files, return
- lightweight refs. I/O only, no parsing, idempotent for a given `cache_dir`.
-- `convert(self, raw_refs: list[TRaw]) -> TimeFDataset` (abstract): parse refs into a `TimeFDataset`.
- CPU only, no network.
+`BaseConnector(ABC, Generic[TRaw])` in `timenet.connectors`. `TRaw` is the whole value that `download` hands to
+`convert`: one handle dataclass, a list of file refs (`list[HubFileRef]` for Hub connectors), or `None`.
+
+- `download(self, cache_dir: Path) -> TRaw` (abstract): fetch/discover raw source files, return
+ a lightweight handle. I/O only, no parsing, idempotent for a given `cache_dir`.
+- `convert(self, raw: TRaw, /) -> TimeFDataset` (abstract): parse the download result into a
+ `TimeFDataset`. CPU only, no network. The parameter is positional-only, so name it as you like
+ (`source`, `recordings`).
- `metadata(self) -> DatasetMetadata` (**concrete**, do not override): loads and validates the card via
`DatasetMetadata.from_yaml`. By convention the card is `dataset.yaml` beside the connector module;
set the `CARD` class var to point elsewhere. (Some docs call `metadata` abstract; it isn't.)
@@ -70,7 +71,7 @@ tags:
## Base connectors to reuse
### `BaseHuggingFaceConnector` (`bases/huggingface.py`)
-`BaseHuggingFaceConnector(BaseConnector[HubFileRef])` downloads files from a Hub dataset at a pinned
+`BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]])` downloads files from a Hub dataset at a pinned
commit. Pass these class keywords:
- `hf_repo`: the Hub repo id.
diff --git a/docs/connectors.md b/docs/connectors.md
index 86a3e09b8..228560848 100644
--- a/docs/connectors.md
+++ b/docs/connectors.md
@@ -29,9 +29,9 @@ from pathlib import Path
from timenet.connectors import BaseConnector
from timenet.dataset import TimeFDataset
-class MyConnector(BaseConnector[MyRawRef]):
- def download(self, cache_dir: Path) -> list[MyRawRef]: ...
- def convert(self, raw_refs: list[MyRawRef]) -> TimeFDataset: ...
+class MyConnector(BaseConnector[MySource]):
+ def download(self, cache_dir: Path) -> MySource: ...
+ def convert(self, source: MySource) -> TimeFDataset: ...
```
The two stages stay separate. This lets the engine drive
@@ -39,9 +39,9 @@ The two stages stay separate. This lets the engine drive
| Method | Nature | Contract |
| --- | --- | --- |
-| `download(cache_dir)` | I/O only | Fetch or find raw files and return lightweight references. The method is idempotent and does not parse data. |
-| `convert(raw_refs)` | local work | Parse the references into a `TimeFDataset`. Do not use the network. |
-| `compose(raw_refs, context)` | local work | Build a child dataset from the references and the opened parent releases. The default calls `convert(raw_refs)`. |
+| `download(cache_dir)` | I/O only | Fetch or find raw files and return a lightweight handle to them. The method is idempotent and does not parse data. |
+| `convert(raw)` | local work | Parse the download result into a `TimeFDataset`. Do not use the network. |
+| `compose(raw, context)` | local work | Build a child dataset from the download result and the opened parent releases. The default calls `convert(raw)`. |
A connector must implement `download` or `download_async`. It must also implement `convert` or
`compose`. If either pair has no implementation, Python raises `TypeError` during connector
@@ -63,9 +63,11 @@ writer after conversion.
card. `metadata().dataset_id` must match the connector's built id.
A connector takes no constructor arguments. Configuration comes from environment variables that
-`__init__` reads. `TRaw` is the reference type that the connector defines, for example a path, a small
-dataclass, or an S3 key. The connector is generic over `TRaw`:
-`class MyConnector(BaseConnector[MyRawRef])`.
+`__init__` reads. `TRaw` is the type of the value that `download` returns. The connector defines it.
+It can be any value, for example one small dataclass, a list of file references, or `None`. The
+connector is generic over `TRaw`: `class MyConnector(BaseConnector[MySource])`. The engine passes the
+value to `compose` unchanged. The base takes it as a positional argument, so a connector can name the
+parameter, for example `source` or `recordings`.
If an assumption about your source data, downloads, or parents does not hold, raise
`TimeNetConnectorError` from `timenet.errors`.
@@ -269,7 +271,7 @@ helpers from `download_async`:
```python
from timenet_connectors.download import Artifact, ensure_archive, download_files
-class MyConnector(BaseConnector[MyRawRef]):
+class MyConnector(BaseConnector[MySource]):
async def download_async(self, cache_dir):
await download_files(
[
@@ -280,7 +282,7 @@ class MyConnector(BaseConnector[MyRawRef]):
headers={"Authorization": "Bearer …"},
)
await ensure_archive("https://host/records.zip", cache_dir)
- return [...] # lightweight references into cache_dir
+ return MySource(...) # a lightweight handle to the files in cache_dir
```
An ambient sink reports progress, through `timenet_connectors.download.progress`. The download helpers
@@ -318,9 +320,9 @@ A successful build removes the raw downloads of the dataset, at
`timenet.build()`. This helps while you write a connector for a large source, because each rebuild
downloads the source again.
-Keeping `download` and `convert` apart makes a connector testable offline. `convert` takes raw
-references and does not touch the network. A test can therefore pass small local inputs or synthetic
-references directly to `convert()` and skip `download`.
+Keeping `download` and `convert` apart makes a connector testable offline. `convert` takes the
+download result and does not touch the network. A test can therefore pass a small local input or a
+synthetic handle directly to `convert()` and skip `download`.
After the build, you can load and inspect a dataset with the SDK. See `examples/load_tsqa.py`. This
example loads a dataset and calls `describe()` to print its identity, its counts, its columns per
@@ -337,8 +339,8 @@ parents:
ptbxl: physionet/ptb-xl@1.0.0
```
-The connector of a child overrides `compose(raw_refs, context)`. The default `compose` calls
-`convert(raw_refs)`. If the card declares parents and the connector does not override `compose`,
+The connector of a child overrides `compose(raw, context)`. The default `compose` calls
+`convert(raw)`. If the card declares parents and the connector does not override `compose`,
the build raises before it opens a parent or downloads anything.
Before the connector runs, the build prepares each parent release in the local registry, as
@@ -352,7 +354,7 @@ from timenet.dataset import TimeFDataset
from timenet.types.reference_keys import ParentAlias
-def compose(self, raw_refs, context: BuildContext) -> TimeFDataset:
+def compose(self, source, context: BuildContext) -> TimeFDataset:
dataset = TimeFDataset(metadata=self.metadata())
ptbxl = context.parents[ParentAlias("ptbxl")]
# Find parent Records by a key of the source in their metadata.
@@ -405,7 +407,7 @@ See the [API reference for `timenet.connectors`](api/timenet.connectors.md) for
## Reading source rows
For CSV, JSON arrays, JSON Lines, and Parquet, declare the source fields in a connector-owned
-`DataRow` and read each downloaded file during `convert`:
+`DataRow` and read each downloaded file during `convert`. This example reads a list of file references:
```python
from pydantic import Field
diff --git a/docs/data-model/index.md b/docs/data-model/index.md
index d059f9a69..b5bf1d213 100644
--- a/docs/data-model/index.md
+++ b/docs/data-model/index.md
@@ -31,7 +31,7 @@ database, and Parquet or Zarr Signal values. Reading it never runs connector cod
```mermaid
flowchart LR
- D["download()
I/O · fetch raw refs"]
+ D["download()
I/O · fetch raw files"]
C["convert()
CPU only"]
S["derive_schema()
types from data"]
W["store_dataset()
TimeFWriter"]
diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py
index 629cc96ee..4f3f63df7 100644
--- a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py
+++ b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py
@@ -37,7 +37,7 @@ class HubFileRef:
``convert()`` branches on and builds ids from."""
-class BaseHuggingFaceConnector(BaseConnector[HubFileRef], ABC):
+class BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]], ABC):
"""Base class for connectors that read files from a pinned Hub dataset revision.
Subclasses pass ``hf_repo``, ``source_revision``, and ``source_files`` as class keywords, and
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py
index 902330aa3..8541cc3e4 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py
@@ -184,7 +184,7 @@ def read_rows(self) -> Iterator[HarCotRow]:
yield parsed
-class HarCotConnector(BaseConnector[HarCotSplitFile]):
+class HarCotConnector(BaseConnector[list[HarCotSplitFile]]):
"""Connector for the HAR-CoT release of OpenTSLM."""
async def download_async(self, cache_dir: Path) -> list[HarCotSplitFile]: # ruff: ignore[no-self-use]
@@ -199,17 +199,17 @@ async def download_async(self, cache_dir: Path) -> list[HarCotSplitFile]: # ruf
root = await ensure_archive(HAR_COT_URL, cache_dir, filename="har_cot.zip", sha256=HAR_COT_SHA256)
return [HarCotSplitFile(split=split, path=root / name) for split, name in SPLIT_FILES.items()]
- def convert(self, raw_refs: list[HarCotSplitFile]) -> TimeFDataset:
+ def convert(self, split_files: list[HarCotSplitFile]) -> TimeFDataset:
"""Create one Record and one ClassificationTask for each window.
Args:
- raw_refs: The split files from :meth:`download_async`.
+ split_files: The split files from :meth:`download_async`.
Returns:
The populated dataset.
"""
dataset = TimeFDataset(metadata=self.metadata())
- for split_file in raw_refs:
+ for split_file in split_files:
for row in split_file.read_rows():
record = dataset.add_record(row.create_record())
dataset.add_task(row.create_classification_task(record, split=split_file.split))
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py
index a93726eeb..4e2ce3e6b 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py
@@ -262,7 +262,7 @@ def read_splits(self, n_rows: int) -> list[Split]:
class SleepCotConnector(BaseConnector[SleepCotReader]):
"""Connector for the SleepEDF-CoT release of OpenTSLM."""
- async def download_async(self, cache_dir: Path) -> list[SleepCotReader]: # ruff: ignore[no-self-use]
+ async def download_async(self, cache_dir: Path) -> SleepCotReader: # ruff: ignore[no-self-use]
"""Download the release CSV.
Args:
@@ -273,18 +273,17 @@ async def download_async(self, cache_dir: Path) -> list[SleepCotReader]: # ruff
"""
artifact = Artifact(SLEEP_COT_URL, cache_dir / "sleep_cot.csv", sha256=SLEEP_COT_SHA256)
(csv_path,) = await download_files([artifact])
- return [SleepCotReader(csv_path=csv_path, splits_path=SPLITS_PATH)]
+ return SleepCotReader(csv_path=csv_path, splits_path=SPLITS_PATH)
- def convert(self, raw_refs: list[SleepCotReader]) -> TimeFDataset:
+ def convert(self, reader: SleepCotReader) -> TimeFDataset:
"""Create one Record and one ClassificationTask for each window.
Args:
- raw_refs: The reader from :meth:`download_async`.
+ reader: The reader from :meth:`download_async`.
Returns:
The populated dataset.
"""
- (reader,) = raw_refs
rows = reader.read_rows()
splits = reader.read_splits(len(rows))
dataset = TimeFDataset(metadata=self.metadata())
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py
index 639b0e83e..cffa8ff3f 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py
@@ -74,7 +74,7 @@ def _write_release(
def test_each_window_becomes_a_record_and_a_classification_task(tmp_path):
- tasks = SleepCotConnector().convert([_write_release(tmp_path)]).tasks_of(ClassificationTask)
+ tasks = SleepCotConnector().convert(_write_release(tmp_path)).tasks_of(ClassificationTask)
assert [task.split for task in tasks] == [Split.TRAIN, Split.VALIDATION, Split.TRAIN, Split.TEST]
for task, row in zip(tasks, _ROWS, strict=True):
(record,) = task.inputs
@@ -105,7 +105,7 @@ def test_each_window_becomes_a_record_and_a_classification_task(tmp_path):
)
def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, splits, message):
with pytest.raises(TimeNetConnectorError, match=message):
- SleepCotConnector().convert([_write_release(tmp_path, **splits)])
+ SleepCotConnector().convert(_write_release(tmp_path, **splits))
@pytest.mark.parametrize(
@@ -121,13 +121,13 @@ def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, splits, mes
def test_a_malformed_row_raises_with_its_location(tmp_path, change, message):
rows = [*_ROWS[:2], _ROWS[2] | change]
with pytest.raises(TimeNetConnectorError, match=rf"sleep_cot\.csv data row 3: {message}"):
- SleepCotConnector().convert([_write_release(tmp_path, rows)])
+ SleepCotConnector().convert(_write_release(tmp_path, rows))
def test_the_dataset_round_trips_through_storage(tmp_path):
source = tmp_path / "source"
source.mkdir()
- dataset = SleepCotConnector().convert([_write_release(source)])
+ dataset = SleepCotConnector().convert(_write_release(source))
version_dir = dataset.write(path=tmp_path / "registry")
with TimeFReader(open_local_version(version_dir)) as reader:
reader.verify()
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py
index f25e8d80f..58548e9a4 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py
@@ -81,10 +81,9 @@
@dataclass(frozen=True)
class EcgQaCotSource:
- """A lightweight handle to the fetched sources, so ``compose`` streams rather than holding refs.
+ """A lightweight handle to the fetched sources.
- ``download`` returns one of these instead of a row per question, so the ~230k questions are read
- lazily during ``compose`` and the task stream, never materialized as a list.
+ ``compose`` and the task stream read the ~230k questions lazily. Nothing holds them as a list.
"""
answers_path: Path
@@ -286,7 +285,7 @@ class EcgQaCotConnector(BaseConnector[EcgQaCotSource]):
(Split.TEST, "ecg_qa_cot_test.csv"),
)
- async def download_async(self, cache_dir: Path) -> list[EcgQaCotSource]:
+ async def download_async(self, cache_dir: Path) -> EcgQaCotSource:
"""Fetch the template answers and CoT CSVs and return one lightweight handle.
The template-answers CSV and the CoT archive download concurrently. This method reads no
@@ -296,7 +295,7 @@ async def download_async(self, cache_dir: Path) -> list[EcgQaCotSource]:
cache_dir: The directory that holds downloaded archives.
Returns:
- A single-element list holding the :class:`EcgQaCotSource` handle.
+ The :class:`EcgQaCotSource` handle.
"""
answers_path = cache_dir / "answers_for_each_template.csv"
# The two HTTP artifacts download concurrently.
@@ -309,9 +308,9 @@ async def download_async(self, cache_dir: Path) -> list[EcgQaCotSource]:
cot_csvs = tuple(
(split, find_dir_containing(cot_root, csv_name) / csv_name) for split, csv_name in self._COT_CSVS
)
- return [EcgQaCotSource(answers_path=answers_path, cot_csvs=cot_csvs)]
+ return EcgQaCotSource(answers_path=answers_path, cot_csvs=cot_csvs)
- def compose(self, raw_refs: list[EcgQaCotSource], context: BuildContext) -> TimeFDataset:
+ def compose(self, source: EcgQaCotSource, context: BuildContext) -> TimeFDataset:
"""Build one record per recording and stream one :class:`AnswerTask` per CoT row.
A first pass over the CoT CSVs collects each recording's split and the distinct question
@@ -321,7 +320,7 @@ def compose(self, raw_refs: list[EcgQaCotSource], context: BuildContext) -> Time
themselves stream from :meth:`_iter_tasks`, so the ~230k questions never all live in memory.
Args:
- raw_refs: The single-element list from :meth:`download`.
+ source: The handle from :meth:`download`.
context: The opened PTB-XL parent, under the parent alias ``ptbxl``.
Returns:
@@ -332,7 +331,6 @@ def compose(self, raw_refs: list[EcgQaCotSource], context: BuildContext) -> Time
name, or if an imported Record has no patient, age, or sex, or has a sex other than
male or female.
"""
- source = raw_refs[0]
answers = _load_template_answers(source.answers_path)
dataset = TimeFDataset(metadata=self.metadata())
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py
index 66f0843b5..d464893a9 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py
@@ -213,7 +213,7 @@ def _store_ptbxl(
for row in sorted(rows, key=lambda row: int(row["ecg_id"]), reverse=True):
filename = f"records500/00000/{int(row['ecg_id']):05d}_hr"
writer.writerow({column: row.get(column, "") for column in _PTBXL_COLUMNS} | {"filename_hr": filename})
- parent = PtbXlConnector().convert([PtbXlSource(tmp_path, database)])
+ parent = PtbXlConnector().convert(PtbXlSource(tmp_path, database))
if change is not None:
ecg_id, key, value = change
(record,) = (record for record in parent.records if record.metadata["ecg_id"] == ecg_id)
@@ -252,7 +252,7 @@ def _composed(
_store_ptbxl(tmp_path, records_dir, _PTBXL_ROWS if ptbxl_rows is None else ptbxl_rows, root)
connector = EcgQaCotConnector()
with open_parents(connector.metadata(), LocalRegistry(root)) as context:
- yield connector.compose([_source(tmp_path, split)], context)
+ yield connector.compose(_source(tmp_path, split), context)
def test_is_a_connector():
@@ -383,7 +383,7 @@ def test_an_incomplete_ptbxl_record_raises(tmp_path, records_dir, key, message):
pytest.raises(TimeNetConnectorError, match=f"ecg_id 2: {message}"),
open_parents(connector.metadata(), LocalRegistry(root)) as context,
):
- connector.compose([_source(tmp_path)], context)
+ connector.compose(_source(tmp_path), context)
def test_a_ptbxl_sex_other_than_male_or_female_raises(tmp_path, records_dir):
@@ -396,7 +396,7 @@ def test_a_ptbxl_sex_other_than_male_or_female_raises(tmp_path, records_dir):
pytest.raises(TimeNetConnectorError, match="ecg_id 2: sex 'unknown' is not male or female"),
open_parents(connector.metadata(), LocalRegistry(root)) as context,
):
- connector.compose([_source(tmp_path)], context)
+ connector.compose(_source(tmp_path), context)
def test_series_values_match_fixture_record(tmp_path, records_dir):
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py
index c00e25173..69d3a7922 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py
@@ -120,14 +120,14 @@ def _parse_patient_id(cells: dict[str, str]) -> str:
class PtbXlConnector(BasePhysioNetConnector[PtbXlSource]):
"""Preserve PTB-XL waveforms and clinical metadata without downstream tasks."""
- async def download_async(self, cache_dir: Path) -> list[PtbXlSource]: # ruff: ignore[no-self-use]
+ async def download_async(self, cache_dir: Path) -> PtbXlSource: # ruff: ignore[no-self-use]
"""Return the extracted pinned PTB-XL release."""
root = find_dir_containing(
await ensure_archive(PTBXL_ZIP_URL, cache_dir, sha256=PTBXL_ZIP_SHA256), "ptbxl_database.csv"
)
- return [PtbXlSource(root, root / "ptbxl_database.csv")]
+ return PtbXlSource(root, root / "ptbxl_database.csv")
- def convert(self, raw_refs: list[PtbXlSource]) -> TimeFDataset:
+ def convert(self, source: PtbXlSource) -> TimeFDataset:
"""Return records without tasks, with lazy WFDB value loaders.
Each record carries its PTB-XL ``ecg_id`` in its metadata. A child dataset selects the
@@ -141,7 +141,6 @@ def convert(self, raw_refs: list[PtbXlSource]) -> TimeFDataset:
``ecg_id``. For a repeated ``ecg_id``, the message also names the earlier row.
"""
dataset = TimeFDataset(metadata=self.metadata())
- source = raw_refs[0]
# The CSV row where each ecg_id first appears.
first_rows: dict[int, int] = {}
for line, source_row in enumerate(CsvDataReader(source.database_csv, _SourceRow).iter_rows(), 2):
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py
index cf0f6137a..7949a91d4 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py
@@ -61,12 +61,12 @@ def _write_source(tmp_path: Path, **cells: str) -> PtbXlSource:
def _annotations(tmp_path: Path, **cells: str) -> dict[str, Annotation]:
"""Convert the fixture row with ``cells`` in place of its cells, and return its annotations by key."""
- dataset = PtbXlConnector().convert([_write_source(tmp_path, **cells)])
+ dataset = PtbXlConnector().convert(_write_source(tmp_path, **cells))
return {ann.key: ann for ann in dataset.records[0].annotations}
def test_taskless_clinical_records_round_trip(tmp_path):
- dataset = PtbXlConnector().convert([_write_source(tmp_path)])
+ dataset = PtbXlConnector().convert(_write_source(tmp_path))
assert dataset.tasks == ()
assert len(dataset.records[0].signals) == 12
# Children find a recording by its upstream ecg_id, which the record's metadata carries.
@@ -127,7 +127,7 @@ def test_a_recording_that_is_not_twelve_leads_at_500_hz_raises_a_connector_error
"record", fs=250, units=["mV"] * 12, sig_name=_LEADS, p_signal=values, fmt=["16"] * 12, write_dir=str(tmp_path)
)
with pytest.raises(TimeNetConnectorError, match=r"row 2 \(ecg_id 1\) .*: PTB-XL requires twelve leads at 500 Hz"):
- PtbXlConnector().convert([source])
+ PtbXlConnector().convert(source)
# The optional columns that the fixture row fills. The cell of each adds the annotation of the same key.
@@ -190,11 +190,11 @@ def test_a_repeated_ecg_id_raises_and_names_both_rows(tmp_path):
with pytest.raises(
TimeNetConnectorError, match=r"invalid PTB-XL row 4 \(ecg_id 1\) in .*: ecg_id 1 repeats row 2$"
):
- PtbXlConnector().convert([source])
+ PtbXlConnector().convert(source)
def test_recording_date_and_findings_are_annotated_when_their_cell_has_a_value(tmp_path):
- dataset = PtbXlConnector().convert([_write_source(tmp_path)])
+ dataset = PtbXlConnector().convert(_write_source(tmp_path))
by_key = {ann.key: ann.value for ann in dataset.records[0].annotations}
# Each value loses its surrounding whitespace. Cells with only whitespace add no annotation.
expected = {
@@ -222,6 +222,6 @@ def test_convert_needs_no_wfdb_for_format16(tmp_path, monkeypatch):
# wfdb installed. wfdb is only a fallback for other signal formats.
source = _write_source(tmp_path)
monkeypatch.setitem(sys.modules, "wfdb", None) # `import wfdb` -> ImportError if anything reaches for it
- dataset = PtbXlConnector().convert([source])
+ dataset = PtbXlConnector().convert(source)
assert dataset.records
assert len(dataset.records[0].signals[0].to_numpy()) # runs the direct loader, no wfdb
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py
index 7b78f6ff6..3822c322a 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py
@@ -65,10 +65,9 @@
@dataclass(frozen=True)
class SleepEdfxSource:
- """A small handle to the fetched release, which ``convert`` walks in place of a list of refs.
+ """A small handle to the fetched release.
- ``download`` returns one handle and not an entry for each recording. During ``convert``,
- :func:`_iter_recordings` finds the recordings. Nothing holds them as a list.
+ During ``convert``, :func:`_iter_recordings` finds the recordings. Nothing holds them as a list.
It names the subject table of every study by path, and carries no row of any of them.
``download`` resolves paths and opens nothing. ``convert`` reads the table of each study it
@@ -222,7 +221,7 @@ class SleepEdfxConnector(BasePhysioNetConnector[SleepEdfxSource]):
(_TELEMETRY_STUDY, tables.TELEMETRY_SHEET),
)
- async def download_async(self, cache_dir: Path) -> list[SleepEdfxSource]:
+ async def download_async(self, cache_dir: Path) -> SleepEdfxSource:
"""Get the Sleep-EDF archive and give one small handle to it.
One archive holds the full release, and this is one fetch.
@@ -237,7 +236,7 @@ async def download_async(self, cache_dir: Path) -> list[SleepEdfxSource]:
cache_dir: The directory for the archive and its extracted contents.
Returns:
- A list of one :class:`SleepEdfxSource` handle.
+ One :class:`SleepEdfxSource` handle.
Raises:
TimeNetDownloadError: If the archive holds no directory for a study, or no subject table.
@@ -258,11 +257,11 @@ async def download_async(self, cache_dir: Path) -> list[SleepEdfxSource]:
subject_tables.append((study, table_path))
studies.append((study, study_dir))
- return [SleepEdfxSource(studies=tuple(studies), subject_tables=tuple(subject_tables))]
+ return SleepEdfxSource(studies=tuple(studies), subject_tables=tuple(subject_tables))
# PLR0914: the loop holds one name for each fact a record is made of, on purpose. A reader
# of it sees the join and each annotation, and not one call that hides both behind a name.
- def convert(self, raw_refs: list[SleepEdfxSource]) -> TimeFDataset:
+ def convert(self, source: SleepEdfxSource) -> TimeFDataset:
"""Turn the fetched release into a dataset.
This reads the subject table of each study one time, before the loop. It then joins each
@@ -274,12 +273,11 @@ def convert(self, raw_refs: list[SleepEdfxSource]) -> TimeFDataset:
one time.
Args:
- raw_refs: The list of one handle from :meth:`download`.
+ source: The handle from :meth:`download`.
Returns:
The dataset, with one record for each recording it walked.
"""
- source = raw_refs[0]
dataset = TimeFDataset(metadata=self.metadata())
metadata_annotation = metadata.MetadataAnnotation()
subject_tables = metadata.SubjectTables(
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py
index b0aa23c1f..84f13fd09 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py
@@ -129,7 +129,7 @@ def _convert(
studies=((study, release / study),),
subject_tables=((study, release / table),),
)
- return SleepEdfxConnector().convert([source])
+ return SleepEdfxConnector().convert(source)
def _streamed(dataset: TimeFDataset) -> list:
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py
index 4981fa0bb..d73f39b83 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py
@@ -103,7 +103,7 @@ def _wave(
return lambda: pa.array(_wave_values(fn, n, phase))
-class HelloWorldConnector(BaseConnector[HelloWorldRecording]):
+class HelloWorldConnector(BaseConnector[list[HelloWorldRecording]]):
"""A deterministic, offline demo connector for the ``timenet/hello-world`` dataset."""
# Override: the data is synthetic, so this method needs no cache.
@@ -122,13 +122,13 @@ def download(self, cache_dir: Path) -> list[HelloWorldRecording]:
"""
return [HelloWorldRecording(index=0, n_values=16), HelloWorldRecording(index=1, n_values=512)]
- def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset:
+ def convert(self, recordings: list[HelloWorldRecording]) -> TimeFDataset:
"""Build the complete dataset from the recording descriptions.
The dataset includes every TimeF feature.
Args:
- raw_refs: The recordings from :meth:`download`.
+ recordings: The recordings from :meth:`download`.
Returns:
The populated :class:`~timenet.dataset.TimeFDataset`.
@@ -137,7 +137,7 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset:
# Record 0's sine signal.
shared = Signal.from_values(
- _wave_values(np.sin, raw_refs[0].n_values, phase=0.0),
+ _wave_values(np.sin, recordings[0].n_values, phase=0.0),
spec=_SINE,
name="a",
time_axis=_AXIS,
@@ -148,7 +148,7 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset:
# Record 0: the full recording, with two modalities, all annotation shapes, and a task chain.
cosine = Signal.from_values(
- _wave_values(np.cos, raw_refs[0].n_values, phase=0.0),
+ _wave_values(np.cos, recordings[0].n_values, phase=0.0),
spec=_COSINE,
name="b",
time_axis=_AXIS,
@@ -210,8 +210,8 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset:
spec=_SINE,
name="a",
time_axis=_AXIS,
- loader=_wave(np.sin, raw_refs[1].n_values, phase=1.0),
- n_values=raw_refs[1].n_values,
+ loader=_wave(np.sin, recordings[1].n_values, phase=1.0),
+ n_values=recordings[1].n_values,
)
record1 = Record(
sources=[
@@ -219,7 +219,7 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset:
"Synthetic generator",
signals=[
Signal.from_values(
- _wave_values(np.sin, raw_refs[0].n_values, phase=0.0),
+ _wave_values(np.sin, recordings[0].n_values, phase=0.0),
spec=_SINE,
name="a",
time_axis=_AXIS,
@@ -236,11 +236,11 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset:
# half of rec-0. As a result, the window is a true offset window and not a byte-identical
# prefix of the shared sine signal. The phase offset continues the same wave. The values
# therefore match rec-0 over the window.
- window_start = raw_refs[0].n_values // 2
+ window_start = recordings[0].n_values // 2
window = Signal.from_values(
_wave_values(
np.sin,
- raw_refs[0].n_values - window_start,
+ recordings[0].n_values - window_start,
phase=2.0 * np.pi * window_start / _SAMPLING_RATE_HZ,
),
spec=_SINE,
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py
index 358f8f4f1..cd7510f37 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py
@@ -40,27 +40,22 @@
class TestMeanConnector(BaseConnector[None]):
"""A deterministic, offline connector for the ``timenet/test-mean`` classification demo.
- Synthetic: :meth:`download` returns no references and :meth:`convert` generates every record directly.
+ Synthetic: :meth:`download` returns nothing and :meth:`convert` generates every record directly.
"""
- # ruff: ignore[unused-method-argument, no-self-use] (synthetic: nothing to download)
- def download(self, cache_dir: Path) -> list[None]:
- """Return no references; the data is synthetic and built entirely in :meth:`convert`.
+ def download(self, cache_dir: Path) -> None:
+ """Return nothing; the data is synthetic and built entirely in :meth:`convert`.
Args:
cache_dir: Unused; nothing is fetched.
-
- Returns:
- An empty list.
"""
- return []
# ruff: ignore[unused-method-argument] (synthetic: nothing to convert)
- def convert(self, raw_refs: list[None]) -> TimeFDataset:
+ def convert(self, raw: None) -> TimeFDataset:
"""Generate the balanced set of labeled, single-signal records.
Args:
- raw_refs: Unused; the data is synthetic (``download`` returns nothing).
+ raw: Unused; the data is synthetic (``download`` returns nothing).
Returns:
The populated dataset: one single-signal series and one ``ClassificationTask`` per record.
diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py
index 47ae40737..de8d3f142 100644
--- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py
+++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py
@@ -1,5 +1,4 @@
from collections import Counter
-from pathlib import Path
from typing import cast
from timenet.connectors import BaseConnector
@@ -10,7 +9,7 @@
def _convert() -> TimeFDataset:
- return TestMeanConnector().convert([])
+ return TestMeanConnector().convert(None)
def _labels_by_record(dataset: TimeFDataset) -> dict[int | None, str | None]:
@@ -30,11 +29,6 @@ def test_metadata():
assert TestMeanConnector().metadata().dataset_id == "timenet/test-mean"
-def test_download_defaults_to_no_op():
- # Synthetic connector: it generates everything in convert and inherits the no-op download.
- assert TestMeanConnector().download(Path("cache")) == []
-
-
def test_convert_is_deterministic():
assert_datasets_equal(_convert(), _convert())
diff --git a/packages/timenet-connectors/tests/test_build_cli.py b/packages/timenet-connectors/tests/test_build_cli.py
index 2e087451b..4290ae944 100644
--- a/packages/timenet-connectors/tests/test_build_cli.py
+++ b/packages/timenet-connectors/tests/test_build_cli.py
@@ -222,7 +222,7 @@ def _demo_card(dataset_id: str, **update) -> DatasetMetadata:
)
-class _Parent(BaseConnector[str]):
+class _Parent(BaseConnector[list[str]]):
"""Builds the demo dataset as test/parent."""
def metadata(self):
diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py
index f1939b1c3..789971de1 100644
--- a/packages/timenet-connectors/tests/test_build_plan.py
+++ b/packages/timenet-connectors/tests/test_build_plan.py
@@ -270,7 +270,7 @@ def test_the_plan_reads_each_card_from_the_backend_that_claims_the_dataset(tmp_p
_parent_steps(CHILD.dataset_id, LocalRegistry(tmp_path), parents_from=[InstalledConnectors()])
-class _Grand(BaseConnector[str]):
+class _Grand(BaseConnector[list[str]]):
"""Rebuilds GRAND, the demo dataset."""
def metadata(self):
diff --git a/packages/timenet/src/timenet/connectors/base.py b/packages/timenet/src/timenet/connectors/base.py
index 4d265fcbe..b04ff4ebb 100644
--- a/packages/timenet/src/timenet/connectors/base.py
+++ b/packages/timenet/src/timenet/connectors/base.py
@@ -86,8 +86,10 @@ def metadata(self) -> DatasetMetadata:
"""
return DatasetMetadata.from_yaml(self._card_path())
- def download(self, cache_dir: Path) -> list[TRaw]:
- """Fetch or discover raw source files and return lightweight references to them.
+ def download(self, cache_dir: Path) -> TRaw:
+ """Fetch or discover raw source files and return a lightweight handle to them.
+
+ The handle can be any value: one object, a list of file references, or ``None``.
I/O only: no parsing, no array work. Must be idempotent for a given ``cache_dir``. Override
this for a synchronous connector. For an I/O-bound one, override :meth:`download_async`
@@ -98,11 +100,11 @@ def download(self, cache_dir: Path) -> list[TRaw]:
cache_dir: Directory to write downloaded files into (created by the engine).
Returns:
- Raw references passed directly to :meth:`convert`.
+ The download result. The engine passes it to :meth:`compose` unchanged.
"""
return asyncio.run(self.download_async(cache_dir))
- async def download_async(self, cache_dir: Path) -> list[TRaw]:
+ async def download_async(self, cache_dir: Path) -> TRaw:
"""Async variant of :meth:`download` for connectors whose downloads are I/O-bound.
Override this to fetch artifacts concurrently, for example with the connector HTTP download
@@ -112,7 +114,7 @@ async def download_async(self, cache_dir: Path) -> list[TRaw]:
cache_dir: Directory to write downloaded files into (created by the engine).
Returns:
- Raw references passed directly to :meth:`convert`.
+ The download result. The engine passes it to :meth:`compose` unchanged.
Raises:
NotImplementedError: If a subclass overrides neither :meth:`download` nor
@@ -122,8 +124,8 @@ async def download_async(self, cache_dir: Path) -> list[TRaw]:
"a connector must implement download() (synchronous) or download_async() (asynchronous)"
)
- def compose(self, raw_refs: list[TRaw], context: "BuildContext") -> TimeFDataset:
- """Build the dataset from raw references and the opened parent releases.
+ def compose(self, raw: TRaw, /, context: "BuildContext") -> TimeFDataset:
+ """Build the dataset from the download result and the opened parent releases.
A connector whose card declares ``parents`` must override this hook. The connector reads
each parent through ``context.parents[alias]`` and copies objects with
@@ -132,23 +134,23 @@ def compose(self, raw_refs: list[TRaw], context: "BuildContext") -> TimeFDataset
:meth:`convert`.
Args:
- raw_refs: The references returned by :meth:`download`.
+ raw: The value that :meth:`download` returned.
context: The opened parents, keyed by the parent aliases on the card.
Returns:
The populated dataset.
"""
del context
- return self.convert(raw_refs)
+ return self.convert(raw)
- def convert(self, raw_refs: list[TRaw]) -> TimeFDataset:
- """Parse raw references and populate a :class:`~timenet.dataset.TimeFDataset`.
+ def convert(self, raw: TRaw, /) -> TimeFDataset:
+ """Parse the download result and populate a :class:`~timenet.dataset.TimeFDataset`.
CPU-bound: no network I/O. Attach time-series values as lazy loaders instead of materializing
them.
Args:
- raw_refs: The references returned by :meth:`download`.
+ raw: The value that :meth:`download` returned.
Returns:
The populated dataset.
diff --git a/packages/timenet/src/timenet/engine/engine.py b/packages/timenet/src/timenet/engine/engine.py
index cf9405a68..13d3b54c8 100644
--- a/packages/timenet/src/timenet/engine/engine.py
+++ b/packages/timenet/src/timenet/engine/engine.py
@@ -38,7 +38,7 @@ def run_pipeline( # ruff: ignore[too-many-arguments]
This function is idempotent. If the target version is already committed, it skips the expensive
``download``, ``convert``, and ``store`` stages and returns the existing directory. Pass ``force``
to rebuild it. Otherwise the stages run in order: create the cache directory, open the declared
- parents from ``root``, ``download`` raw references into the cache, ``compose`` them into a
+ parents from ``root``, ``download`` the raw source into the cache, ``compose`` the result into a
dataset, check that the dataset has the card that the engine read, record each declared parent
in it, ``derive_schema``, ``store``, then delete the cache directory. Parents must already be in
``root``; the engine never fetches or builds them. A card that declares parents needs a
diff --git a/packages/timenet/tests/connectors/test_base.py b/packages/timenet/tests/connectors/test_base.py
index 6f80dfeba..ad86ed3b6 100644
--- a/packages/timenet/tests/connectors/test_base.py
+++ b/packages/timenet/tests/connectors/test_base.py
@@ -8,7 +8,7 @@
def test_connector_without_convert_or_compose_fails_at_construction():
- class NoConversion(BaseConnector[str]):
+ class NoConversion(BaseConnector[list[str]]):
def download(self, cache_dir):
return []
@@ -17,7 +17,7 @@ def download(self, cache_dir):
def test_connector_whose_init_skips_the_base_fails_without_convert_or_compose():
- class NoConversion(BaseConnector[str]):
+ class NoConversion(BaseConnector[list[str]]):
def __init__(self) -> None:
# Does not call super().__init__(), so the check runs in __new__.
self.ready = True
@@ -33,7 +33,7 @@ def test_concrete_connector_implements_contract(tmp_path):
card = tmp_path / "dataset.yaml"
card.write_text("dataset_id: demo/thing\ndataset_version: 1.0.0\nname: Demo\ndescription: d\nlicense: MIT\n")
- class DemoConnector(BaseConnector[str]):
+ class DemoConnector(BaseConnector[list[str]]):
CARD = card
def download(self, cache_dir: Path) -> list[str]:
@@ -57,7 +57,7 @@ def _write_card(tmp_path: Path) -> Path:
def test_async_connector_download_bridges_to_download_async(tmp_path):
card = _write_card(tmp_path)
- class AsyncConnector(BaseConnector[str]):
+ class AsyncConnector(BaseConnector[list[str]]):
CARD = card
async def download_async(self, cache_dir: Path) -> list[str]:
@@ -73,7 +73,7 @@ def convert(self, raw_refs: list[str]) -> TimeFDataset:
def test_connector_implementing_neither_download_raises(tmp_path):
card = _write_card(tmp_path)
- class NoDownload(BaseConnector[str]):
+ class NoDownload(BaseConnector[list[str]]):
CARD = card
def convert(self, raw_refs: list[str]) -> TimeFDataset:
@@ -86,7 +86,7 @@ def convert(self, raw_refs: list[str]) -> TimeFDataset:
def test_connector_whose_init_skips_the_base_fails_without_download(tmp_path):
card = _write_card(tmp_path)
- class NoDownload(BaseConnector[str]):
+ class NoDownload(BaseConnector[list[str]]):
CARD = card
def __init__(self) -> None:
diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py
index 0fc57cba4..bf3ffeda5 100644
--- a/packages/timenet/tests/test_client.py
+++ b/packages/timenet/tests/test_client.py
@@ -92,7 +92,7 @@ def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path
assert not (cache / reference.dataset_id).exists()
-class _Resampled(BaseConnector[str]):
+class _Resampled(BaseConnector[list[str]]):
"""Writes the first demo Signal at half its rate, from its declared parent, into a Record of its own."""
def metadata(self):
diff --git a/packages/timenet/tests/test_engine.py b/packages/timenet/tests/test_engine.py
index d7fa7fd63..689b2d8e0 100644
--- a/packages/timenet/tests/test_engine.py
+++ b/packages/timenet/tests/test_engine.py
@@ -54,7 +54,7 @@ def _write_demo_card() -> Path:
_DEMO_CARD = _write_demo_card()
-class _DemoConnector(BaseConnector[str]):
+class _DemoConnector(BaseConnector[list[str]]):
CARD = _DEMO_CARD
def download(self, cache_dir: Path) -> list[str]:
@@ -235,7 +235,7 @@ def _store_credentialed_parent(root: Path) -> None:
LocalRegistry(root).put_dataset(parent)
-class _ChildConnector(BaseConnector[str]):
+class _ChildConnector(BaseConnector[list[str]]):
"""Declares the demo dataset under the parent alias ``parent`` and imports one record with ``IMPORT``."""
IMPORT: ClassVar[dict[str, Any] | None] = {}
@@ -248,7 +248,7 @@ def metadata(self):
def download(self, cache_dir):
return []
- def compose(self, raw_refs, context):
+ def compose(self, raw_refs, /, context):
dataset = TimeFDataset(metadata=self.metadata())
if self.IMPORT is not None:
dataset.import_from(