diff --git a/.agents/skills/add-dataset-connector/references/connector-anatomy.md b/.agents/skills/add-dataset-connector/references/connector-anatomy.md index e6581907a..ca351cdc6 100644 --- a/.agents/skills/add-dataset-connector/references/connector-anatomy.md +++ b/.agents/skills/add-dataset-connector/references/connector-anatomy.md @@ -11,13 +11,14 @@ this branch. Anchor files: ## The `BaseConnector` contract -`BaseConnector(ABC, Generic[TRaw])` in `timenet.connectors`. `TRaw` is whatever `download` hands to -`convert` (a dict per Hub row, a dataclass ref per PhysioNet record, etc.). - -- `download(self, cache_dir: Path) -> list[TRaw]` (abstract): fetch/discover raw source files, return - lightweight refs. I/O only, no parsing, idempotent for a given `cache_dir`. -- `convert(self, raw_refs: list[TRaw]) -> TimeFDataset` (abstract): parse refs into a `TimeFDataset`. - CPU only, no network. +`BaseConnector(ABC, Generic[TRaw])` in `timenet.connectors`. `TRaw` is the whole value that `download` hands to +`convert`: one handle dataclass, a list of file refs (`list[HubFileRef]` for Hub connectors), or `None`. + +- `download(self, cache_dir: Path) -> TRaw` (abstract): fetch/discover raw source files, return + a lightweight handle. I/O only, no parsing, idempotent for a given `cache_dir`. +- `convert(self, raw: TRaw, /) -> TimeFDataset` (abstract): parse the download result into a + `TimeFDataset`. CPU only, no network. The parameter is positional-only, so name it as you like + (`source`, `recordings`). - `metadata(self) -> DatasetMetadata` (**concrete**, do not override): loads and validates the card via `DatasetMetadata.from_yaml`. By convention the card is `dataset.yaml` beside the connector module; set the `CARD` class var to point elsewhere. (Some docs call `metadata` abstract; it isn't.) @@ -70,7 +71,7 @@ tags: ## Base connectors to reuse ### `BaseHuggingFaceConnector` (`bases/huggingface.py`) -`BaseHuggingFaceConnector(BaseConnector[HubFileRef])` downloads files from a Hub dataset at a pinned +`BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]])` downloads files from a Hub dataset at a pinned commit. Pass these class keywords: - `hf_repo`: the Hub repo id. diff --git a/docs/connectors.md b/docs/connectors.md index 86a3e09b8..228560848 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -29,9 +29,9 @@ from pathlib import Path from timenet.connectors import BaseConnector from timenet.dataset import TimeFDataset -class MyConnector(BaseConnector[MyRawRef]): - def download(self, cache_dir: Path) -> list[MyRawRef]: ... - def convert(self, raw_refs: list[MyRawRef]) -> TimeFDataset: ... +class MyConnector(BaseConnector[MySource]): + def download(self, cache_dir: Path) -> MySource: ... + def convert(self, source: MySource) -> TimeFDataset: ... ``` The two stages stay separate. This lets the engine drive @@ -39,9 +39,9 @@ The two stages stay separate. This lets the engine drive | Method | Nature | Contract | | --- | --- | --- | -| `download(cache_dir)` | I/O only | Fetch or find raw files and return lightweight references. The method is idempotent and does not parse data. | -| `convert(raw_refs)` | local work | Parse the references into a `TimeFDataset`. Do not use the network. | -| `compose(raw_refs, context)` | local work | Build a child dataset from the references and the opened parent releases. The default calls `convert(raw_refs)`. | +| `download(cache_dir)` | I/O only | Fetch or find raw files and return a lightweight handle to them. The method is idempotent and does not parse data. | +| `convert(raw)` | local work | Parse the download result into a `TimeFDataset`. Do not use the network. | +| `compose(raw, context)` | local work | Build a child dataset from the download result and the opened parent releases. The default calls `convert(raw)`. | A connector must implement `download` or `download_async`. It must also implement `convert` or `compose`. If either pair has no implementation, Python raises `TypeError` during connector @@ -63,9 +63,11 @@ writer after conversion. card. `metadata().dataset_id` must match the connector's built id. A connector takes no constructor arguments. Configuration comes from environment variables that -`__init__` reads. `TRaw` is the reference type that the connector defines, for example a path, a small -dataclass, or an S3 key. The connector is generic over `TRaw`: -`class MyConnector(BaseConnector[MyRawRef])`. +`__init__` reads. `TRaw` is the type of the value that `download` returns. The connector defines it. +It can be any value, for example one small dataclass, a list of file references, or `None`. The +connector is generic over `TRaw`: `class MyConnector(BaseConnector[MySource])`. The engine passes the +value to `compose` unchanged. The base takes it as a positional argument, so a connector can name the +parameter, for example `source` or `recordings`. If an assumption about your source data, downloads, or parents does not hold, raise `TimeNetConnectorError` from `timenet.errors`. @@ -269,7 +271,7 @@ helpers from `download_async`: ```python from timenet_connectors.download import Artifact, ensure_archive, download_files -class MyConnector(BaseConnector[MyRawRef]): +class MyConnector(BaseConnector[MySource]): async def download_async(self, cache_dir): await download_files( [ @@ -280,7 +282,7 @@ class MyConnector(BaseConnector[MyRawRef]): headers={"Authorization": "Bearer …"}, ) await ensure_archive("https://host/records.zip", cache_dir) - return [...] # lightweight references into cache_dir + return MySource(...) # a lightweight handle to the files in cache_dir ``` An ambient sink reports progress, through `timenet_connectors.download.progress`. The download helpers @@ -318,9 +320,9 @@ A successful build removes the raw downloads of the dataset, at `timenet.build()`. This helps while you write a connector for a large source, because each rebuild downloads the source again. -Keeping `download` and `convert` apart makes a connector testable offline. `convert` takes raw -references and does not touch the network. A test can therefore pass small local inputs or synthetic -references directly to `convert()` and skip `download`. +Keeping `download` and `convert` apart makes a connector testable offline. `convert` takes the +download result and does not touch the network. A test can therefore pass a small local input or a +synthetic handle directly to `convert()` and skip `download`. After the build, you can load and inspect a dataset with the SDK. See `examples/load_tsqa.py`. This example loads a dataset and calls `describe()` to print its identity, its counts, its columns per @@ -337,8 +339,8 @@ parents: ptbxl: physionet/ptb-xl@1.0.0 ``` -The connector of a child overrides `compose(raw_refs, context)`. The default `compose` calls -`convert(raw_refs)`. If the card declares parents and the connector does not override `compose`, +The connector of a child overrides `compose(raw, context)`. The default `compose` calls +`convert(raw)`. If the card declares parents and the connector does not override `compose`, the build raises before it opens a parent or downloads anything. Before the connector runs, the build prepares each parent release in the local registry, as @@ -352,7 +354,7 @@ from timenet.dataset import TimeFDataset from timenet.types.reference_keys import ParentAlias -def compose(self, raw_refs, context: BuildContext) -> TimeFDataset: +def compose(self, source, context: BuildContext) -> TimeFDataset: dataset = TimeFDataset(metadata=self.metadata()) ptbxl = context.parents[ParentAlias("ptbxl")] # Find parent Records by a key of the source in their metadata. @@ -405,7 +407,7 @@ See the [API reference for `timenet.connectors`](api/timenet.connectors.md) for ## Reading source rows For CSV, JSON arrays, JSON Lines, and Parquet, declare the source fields in a connector-owned -`DataRow` and read each downloaded file during `convert`: +`DataRow` and read each downloaded file during `convert`. This example reads a list of file references: ```python from pydantic import Field diff --git a/docs/data-model/index.md b/docs/data-model/index.md index d059f9a69..b5bf1d213 100644 --- a/docs/data-model/index.md +++ b/docs/data-model/index.md @@ -31,7 +31,7 @@ database, and Parquet or Zarr Signal values. Reading it never runs connector cod ```mermaid flowchart LR - D["download()
I/O · fetch raw refs"] + D["download()
I/O · fetch raw files"] C["convert()
CPU only"] S["derive_schema()
types from data"] W["store_dataset()
TimeFWriter"] diff --git a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py index 629cc96ee..4f3f63df7 100644 --- a/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py +++ b/packages/timenet-connectors/src/timenet_connectors/bases/huggingface.py @@ -37,7 +37,7 @@ class HubFileRef: ``convert()`` branches on and builds ids from.""" -class BaseHuggingFaceConnector(BaseConnector[HubFileRef], ABC): +class BaseHuggingFaceConnector(BaseConnector[list[HubFileRef]], ABC): """Base class for connectors that read files from a pinned Hub dataset revision. Subclasses pass ``hf_repo``, ``source_revision``, and ``source_files`` as class keywords, and diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py index 902330aa3..8541cc3e4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py @@ -184,7 +184,7 @@ def read_rows(self) -> Iterator[HarCotRow]: yield parsed -class HarCotConnector(BaseConnector[HarCotSplitFile]): +class HarCotConnector(BaseConnector[list[HarCotSplitFile]]): """Connector for the HAR-CoT release of OpenTSLM.""" async def download_async(self, cache_dir: Path) -> list[HarCotSplitFile]: # ruff: ignore[no-self-use] @@ -199,17 +199,17 @@ async def download_async(self, cache_dir: Path) -> list[HarCotSplitFile]: # ruf root = await ensure_archive(HAR_COT_URL, cache_dir, filename="har_cot.zip", sha256=HAR_COT_SHA256) return [HarCotSplitFile(split=split, path=root / name) for split, name in SPLIT_FILES.items()] - def convert(self, raw_refs: list[HarCotSplitFile]) -> TimeFDataset: + def convert(self, split_files: list[HarCotSplitFile]) -> TimeFDataset: """Create one Record and one ClassificationTask for each window. Args: - raw_refs: The split files from :meth:`download_async`. + split_files: The split files from :meth:`download_async`. Returns: The populated dataset. """ dataset = TimeFDataset(metadata=self.metadata()) - for split_file in raw_refs: + for split_file in split_files: for row in split_file.read_rows(): record = dataset.add_record(row.create_record()) dataset.add_task(row.create_classification_task(record, split=split_file.split)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index a93726eeb..4e2ce3e6b 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -262,7 +262,7 @@ def read_splits(self, n_rows: int) -> list[Split]: class SleepCotConnector(BaseConnector[SleepCotReader]): """Connector for the SleepEDF-CoT release of OpenTSLM.""" - async def download_async(self, cache_dir: Path) -> list[SleepCotReader]: # ruff: ignore[no-self-use] + async def download_async(self, cache_dir: Path) -> SleepCotReader: # ruff: ignore[no-self-use] """Download the release CSV. Args: @@ -273,18 +273,17 @@ async def download_async(self, cache_dir: Path) -> list[SleepCotReader]: # ruff """ artifact = Artifact(SLEEP_COT_URL, cache_dir / "sleep_cot.csv", sha256=SLEEP_COT_SHA256) (csv_path,) = await download_files([artifact]) - return [SleepCotReader(csv_path=csv_path, splits_path=SPLITS_PATH)] + return SleepCotReader(csv_path=csv_path, splits_path=SPLITS_PATH) - def convert(self, raw_refs: list[SleepCotReader]) -> TimeFDataset: + def convert(self, reader: SleepCotReader) -> TimeFDataset: """Create one Record and one ClassificationTask for each window. Args: - raw_refs: The reader from :meth:`download_async`. + reader: The reader from :meth:`download_async`. Returns: The populated dataset. """ - (reader,) = raw_refs rows = reader.read_rows() splits = reader.read_splits(len(rows)) dataset = TimeFDataset(metadata=self.metadata()) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py index 639b0e83e..cffa8ff3f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/tests/test_connector.py @@ -74,7 +74,7 @@ def _write_release( def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): - tasks = SleepCotConnector().convert([_write_release(tmp_path)]).tasks_of(ClassificationTask) + tasks = SleepCotConnector().convert(_write_release(tmp_path)).tasks_of(ClassificationTask) assert [task.split for task in tasks] == [Split.TRAIN, Split.VALIDATION, Split.TRAIN, Split.TEST] for task, row in zip(tasks, _ROWS, strict=True): (record,) = task.inputs @@ -105,7 +105,7 @@ def test_each_window_becomes_a_record_and_a_classification_task(tmp_path): ) def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, splits, message): with pytest.raises(TimeNetConnectorError, match=message): - SleepCotConnector().convert([_write_release(tmp_path, **splits)]) + SleepCotConnector().convert(_write_release(tmp_path, **splits)) @pytest.mark.parametrize( @@ -121,13 +121,13 @@ def test_a_split_file_that_does_not_fit_the_release_raises(tmp_path, splits, mes def test_a_malformed_row_raises_with_its_location(tmp_path, change, message): rows = [*_ROWS[:2], _ROWS[2] | change] with pytest.raises(TimeNetConnectorError, match=rf"sleep_cot\.csv data row 3: {message}"): - SleepCotConnector().convert([_write_release(tmp_path, rows)]) + SleepCotConnector().convert(_write_release(tmp_path, rows)) def test_the_dataset_round_trips_through_storage(tmp_path): source = tmp_path / "source" source.mkdir() - dataset = SleepCotConnector().convert([_write_release(source)]) + dataset = SleepCotConnector().convert(_write_release(source)) version_dir = dataset.write(path=tmp_path / "registry") with TimeFReader(open_local_version(version_dir)) as reader: reader.verify() diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py index f25e8d80f..58548e9a4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/connector.py @@ -81,10 +81,9 @@ @dataclass(frozen=True) class EcgQaCotSource: - """A lightweight handle to the fetched sources, so ``compose`` streams rather than holding refs. + """A lightweight handle to the fetched sources. - ``download`` returns one of these instead of a row per question, so the ~230k questions are read - lazily during ``compose`` and the task stream, never materialized as a list. + ``compose`` and the task stream read the ~230k questions lazily. Nothing holds them as a list. """ answers_path: Path @@ -286,7 +285,7 @@ class EcgQaCotConnector(BaseConnector[EcgQaCotSource]): (Split.TEST, "ecg_qa_cot_test.csv"), ) - async def download_async(self, cache_dir: Path) -> list[EcgQaCotSource]: + async def download_async(self, cache_dir: Path) -> EcgQaCotSource: """Fetch the template answers and CoT CSVs and return one lightweight handle. The template-answers CSV and the CoT archive download concurrently. This method reads no @@ -296,7 +295,7 @@ async def download_async(self, cache_dir: Path) -> list[EcgQaCotSource]: cache_dir: The directory that holds downloaded archives. Returns: - A single-element list holding the :class:`EcgQaCotSource` handle. + The :class:`EcgQaCotSource` handle. """ answers_path = cache_dir / "answers_for_each_template.csv" # The two HTTP artifacts download concurrently. @@ -309,9 +308,9 @@ async def download_async(self, cache_dir: Path) -> list[EcgQaCotSource]: cot_csvs = tuple( (split, find_dir_containing(cot_root, csv_name) / csv_name) for split, csv_name in self._COT_CSVS ) - return [EcgQaCotSource(answers_path=answers_path, cot_csvs=cot_csvs)] + return EcgQaCotSource(answers_path=answers_path, cot_csvs=cot_csvs) - def compose(self, raw_refs: list[EcgQaCotSource], context: BuildContext) -> TimeFDataset: + def compose(self, source: EcgQaCotSource, context: BuildContext) -> TimeFDataset: """Build one record per recording and stream one :class:`AnswerTask` per CoT row. A first pass over the CoT CSVs collects each recording's split and the distinct question @@ -321,7 +320,7 @@ def compose(self, raw_refs: list[EcgQaCotSource], context: BuildContext) -> Time themselves stream from :meth:`_iter_tasks`, so the ~230k questions never all live in memory. Args: - raw_refs: The single-element list from :meth:`download`. + source: The handle from :meth:`download`. context: The opened PTB-XL parent, under the parent alias ``ptbxl``. Returns: @@ -332,7 +331,6 @@ def compose(self, raw_refs: list[EcgQaCotSource], context: BuildContext) -> Time name, or if an imported Record has no patient, age, or sex, or has a sex other than male or female. """ - source = raw_refs[0] answers = _load_template_answers(source.answers_path) dataset = TimeFDataset(metadata=self.metadata()) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py index 66f0843b5..d464893a9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ecg_qa_cot/tests/test_connector.py @@ -213,7 +213,7 @@ def _store_ptbxl( for row in sorted(rows, key=lambda row: int(row["ecg_id"]), reverse=True): filename = f"records500/00000/{int(row['ecg_id']):05d}_hr" writer.writerow({column: row.get(column, "") for column in _PTBXL_COLUMNS} | {"filename_hr": filename}) - parent = PtbXlConnector().convert([PtbXlSource(tmp_path, database)]) + parent = PtbXlConnector().convert(PtbXlSource(tmp_path, database)) if change is not None: ecg_id, key, value = change (record,) = (record for record in parent.records if record.metadata["ecg_id"] == ecg_id) @@ -252,7 +252,7 @@ def _composed( _store_ptbxl(tmp_path, records_dir, _PTBXL_ROWS if ptbxl_rows is None else ptbxl_rows, root) connector = EcgQaCotConnector() with open_parents(connector.metadata(), LocalRegistry(root)) as context: - yield connector.compose([_source(tmp_path, split)], context) + yield connector.compose(_source(tmp_path, split), context) def test_is_a_connector(): @@ -383,7 +383,7 @@ def test_an_incomplete_ptbxl_record_raises(tmp_path, records_dir, key, message): pytest.raises(TimeNetConnectorError, match=f"ecg_id 2: {message}"), open_parents(connector.metadata(), LocalRegistry(root)) as context, ): - connector.compose([_source(tmp_path)], context) + connector.compose(_source(tmp_path), context) def test_a_ptbxl_sex_other_than_male_or_female_raises(tmp_path, records_dir): @@ -396,7 +396,7 @@ def test_a_ptbxl_sex_other_than_male_or_female_raises(tmp_path, records_dir): pytest.raises(TimeNetConnectorError, match="ecg_id 2: sex 'unknown' is not male or female"), open_parents(connector.metadata(), LocalRegistry(root)) as context, ): - connector.compose([_source(tmp_path)], context) + connector.compose(_source(tmp_path), context) def test_series_values_match_fixture_record(tmp_path, records_dir): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py index c00e25173..69d3a7922 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py @@ -120,14 +120,14 @@ def _parse_patient_id(cells: dict[str, str]) -> str: class PtbXlConnector(BasePhysioNetConnector[PtbXlSource]): """Preserve PTB-XL waveforms and clinical metadata without downstream tasks.""" - async def download_async(self, cache_dir: Path) -> list[PtbXlSource]: # ruff: ignore[no-self-use] + async def download_async(self, cache_dir: Path) -> PtbXlSource: # ruff: ignore[no-self-use] """Return the extracted pinned PTB-XL release.""" root = find_dir_containing( await ensure_archive(PTBXL_ZIP_URL, cache_dir, sha256=PTBXL_ZIP_SHA256), "ptbxl_database.csv" ) - return [PtbXlSource(root, root / "ptbxl_database.csv")] + return PtbXlSource(root, root / "ptbxl_database.csv") - def convert(self, raw_refs: list[PtbXlSource]) -> TimeFDataset: + def convert(self, source: PtbXlSource) -> TimeFDataset: """Return records without tasks, with lazy WFDB value loaders. Each record carries its PTB-XL ``ecg_id`` in its metadata. A child dataset selects the @@ -141,7 +141,6 @@ def convert(self, raw_refs: list[PtbXlSource]) -> TimeFDataset: ``ecg_id``. For a repeated ``ecg_id``, the message also names the earlier row. """ dataset = TimeFDataset(metadata=self.metadata()) - source = raw_refs[0] # The CSV row where each ecg_id first appears. first_rows: dict[int, int] = {} for line, source_row in enumerate(CsvDataReader(source.database_csv, _SourceRow).iter_rows(), 2): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py index cf0f6137a..7949a91d4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/tests/test_connector.py @@ -61,12 +61,12 @@ def _write_source(tmp_path: Path, **cells: str) -> PtbXlSource: def _annotations(tmp_path: Path, **cells: str) -> dict[str, Annotation]: """Convert the fixture row with ``cells`` in place of its cells, and return its annotations by key.""" - dataset = PtbXlConnector().convert([_write_source(tmp_path, **cells)]) + dataset = PtbXlConnector().convert(_write_source(tmp_path, **cells)) return {ann.key: ann for ann in dataset.records[0].annotations} def test_taskless_clinical_records_round_trip(tmp_path): - dataset = PtbXlConnector().convert([_write_source(tmp_path)]) + dataset = PtbXlConnector().convert(_write_source(tmp_path)) assert dataset.tasks == () assert len(dataset.records[0].signals) == 12 # Children find a recording by its upstream ecg_id, which the record's metadata carries. @@ -127,7 +127,7 @@ def test_a_recording_that_is_not_twelve_leads_at_500_hz_raises_a_connector_error "record", fs=250, units=["mV"] * 12, sig_name=_LEADS, p_signal=values, fmt=["16"] * 12, write_dir=str(tmp_path) ) with pytest.raises(TimeNetConnectorError, match=r"row 2 \(ecg_id 1\) .*: PTB-XL requires twelve leads at 500 Hz"): - PtbXlConnector().convert([source]) + PtbXlConnector().convert(source) # The optional columns that the fixture row fills. The cell of each adds the annotation of the same key. @@ -190,11 +190,11 @@ def test_a_repeated_ecg_id_raises_and_names_both_rows(tmp_path): with pytest.raises( TimeNetConnectorError, match=r"invalid PTB-XL row 4 \(ecg_id 1\) in .*: ecg_id 1 repeats row 2$" ): - PtbXlConnector().convert([source]) + PtbXlConnector().convert(source) def test_recording_date_and_findings_are_annotated_when_their_cell_has_a_value(tmp_path): - dataset = PtbXlConnector().convert([_write_source(tmp_path)]) + dataset = PtbXlConnector().convert(_write_source(tmp_path)) by_key = {ann.key: ann.value for ann in dataset.records[0].annotations} # Each value loses its surrounding whitespace. Cells with only whitespace add no annotation. expected = { @@ -222,6 +222,6 @@ def test_convert_needs_no_wfdb_for_format16(tmp_path, monkeypatch): # wfdb installed. wfdb is only a fallback for other signal formats. source = _write_source(tmp_path) monkeypatch.setitem(sys.modules, "wfdb", None) # `import wfdb` -> ImportError if anything reaches for it - dataset = PtbXlConnector().convert([source]) + dataset = PtbXlConnector().convert(source) assert dataset.records assert len(dataset.records[0].signals[0].to_numpy()) # runs the direct loader, no wfdb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py index 7b78f6ff6..3822c322a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py @@ -65,10 +65,9 @@ @dataclass(frozen=True) class SleepEdfxSource: - """A small handle to the fetched release, which ``convert`` walks in place of a list of refs. + """A small handle to the fetched release. - ``download`` returns one handle and not an entry for each recording. During ``convert``, - :func:`_iter_recordings` finds the recordings. Nothing holds them as a list. + During ``convert``, :func:`_iter_recordings` finds the recordings. Nothing holds them as a list. It names the subject table of every study by path, and carries no row of any of them. ``download`` resolves paths and opens nothing. ``convert`` reads the table of each study it @@ -222,7 +221,7 @@ class SleepEdfxConnector(BasePhysioNetConnector[SleepEdfxSource]): (_TELEMETRY_STUDY, tables.TELEMETRY_SHEET), ) - async def download_async(self, cache_dir: Path) -> list[SleepEdfxSource]: + async def download_async(self, cache_dir: Path) -> SleepEdfxSource: """Get the Sleep-EDF archive and give one small handle to it. One archive holds the full release, and this is one fetch. @@ -237,7 +236,7 @@ async def download_async(self, cache_dir: Path) -> list[SleepEdfxSource]: cache_dir: The directory for the archive and its extracted contents. Returns: - A list of one :class:`SleepEdfxSource` handle. + One :class:`SleepEdfxSource` handle. Raises: TimeNetDownloadError: If the archive holds no directory for a study, or no subject table. @@ -258,11 +257,11 @@ async def download_async(self, cache_dir: Path) -> list[SleepEdfxSource]: subject_tables.append((study, table_path)) studies.append((study, study_dir)) - return [SleepEdfxSource(studies=tuple(studies), subject_tables=tuple(subject_tables))] + return SleepEdfxSource(studies=tuple(studies), subject_tables=tuple(subject_tables)) # PLR0914: the loop holds one name for each fact a record is made of, on purpose. A reader # of it sees the join and each annotation, and not one call that hides both behind a name. - def convert(self, raw_refs: list[SleepEdfxSource]) -> TimeFDataset: + def convert(self, source: SleepEdfxSource) -> TimeFDataset: """Turn the fetched release into a dataset. This reads the subject table of each study one time, before the loop. It then joins each @@ -274,12 +273,11 @@ def convert(self, raw_refs: list[SleepEdfxSource]) -> TimeFDataset: one time. Args: - raw_refs: The list of one handle from :meth:`download`. + source: The handle from :meth:`download`. Returns: The dataset, with one record for each recording it walked. """ - source = raw_refs[0] dataset = TimeFDataset(metadata=self.metadata()) metadata_annotation = metadata.MetadataAnnotation() subject_tables = metadata.SubjectTables( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py index b0aa23c1f..84f13fd09 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_connector.py @@ -129,7 +129,7 @@ def _convert( studies=((study, release / study),), subject_tables=((study, release / table),), ) - return SleepEdfxConnector().convert([source]) + return SleepEdfxConnector().convert(source) def _streamed(dataset: TimeFDataset) -> list: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py index 4981fa0bb..d73f39b83 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py @@ -103,7 +103,7 @@ def _wave( return lambda: pa.array(_wave_values(fn, n, phase)) -class HelloWorldConnector(BaseConnector[HelloWorldRecording]): +class HelloWorldConnector(BaseConnector[list[HelloWorldRecording]]): """A deterministic, offline demo connector for the ``timenet/hello-world`` dataset.""" # Override: the data is synthetic, so this method needs no cache. @@ -122,13 +122,13 @@ def download(self, cache_dir: Path) -> list[HelloWorldRecording]: """ return [HelloWorldRecording(index=0, n_values=16), HelloWorldRecording(index=1, n_values=512)] - def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset: + def convert(self, recordings: list[HelloWorldRecording]) -> TimeFDataset: """Build the complete dataset from the recording descriptions. The dataset includes every TimeF feature. Args: - raw_refs: The recordings from :meth:`download`. + recordings: The recordings from :meth:`download`. Returns: The populated :class:`~timenet.dataset.TimeFDataset`. @@ -137,7 +137,7 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset: # Record 0's sine signal. shared = Signal.from_values( - _wave_values(np.sin, raw_refs[0].n_values, phase=0.0), + _wave_values(np.sin, recordings[0].n_values, phase=0.0), spec=_SINE, name="a", time_axis=_AXIS, @@ -148,7 +148,7 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset: # Record 0: the full recording, with two modalities, all annotation shapes, and a task chain. cosine = Signal.from_values( - _wave_values(np.cos, raw_refs[0].n_values, phase=0.0), + _wave_values(np.cos, recordings[0].n_values, phase=0.0), spec=_COSINE, name="b", time_axis=_AXIS, @@ -210,8 +210,8 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset: spec=_SINE, name="a", time_axis=_AXIS, - loader=_wave(np.sin, raw_refs[1].n_values, phase=1.0), - n_values=raw_refs[1].n_values, + loader=_wave(np.sin, recordings[1].n_values, phase=1.0), + n_values=recordings[1].n_values, ) record1 = Record( sources=[ @@ -219,7 +219,7 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset: "Synthetic generator", signals=[ Signal.from_values( - _wave_values(np.sin, raw_refs[0].n_values, phase=0.0), + _wave_values(np.sin, recordings[0].n_values, phase=0.0), spec=_SINE, name="a", time_axis=_AXIS, @@ -236,11 +236,11 @@ def convert(self, raw_refs: list[HelloWorldRecording]) -> TimeFDataset: # half of rec-0. As a result, the window is a true offset window and not a byte-identical # prefix of the shared sine signal. The phase offset continues the same wave. The values # therefore match rec-0 over the window. - window_start = raw_refs[0].n_values // 2 + window_start = recordings[0].n_values // 2 window = Signal.from_values( _wave_values( np.sin, - raw_refs[0].n_values - window_start, + recordings[0].n_values - window_start, phase=2.0 * np.pi * window_start / _SAMPLING_RATE_HZ, ), spec=_SINE, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py index 358f8f4f1..cd7510f37 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py @@ -40,27 +40,22 @@ class TestMeanConnector(BaseConnector[None]): """A deterministic, offline connector for the ``timenet/test-mean`` classification demo. - Synthetic: :meth:`download` returns no references and :meth:`convert` generates every record directly. + Synthetic: :meth:`download` returns nothing and :meth:`convert` generates every record directly. """ - # ruff: ignore[unused-method-argument, no-self-use] (synthetic: nothing to download) - def download(self, cache_dir: Path) -> list[None]: - """Return no references; the data is synthetic and built entirely in :meth:`convert`. + def download(self, cache_dir: Path) -> None: + """Return nothing; the data is synthetic and built entirely in :meth:`convert`. Args: cache_dir: Unused; nothing is fetched. - - Returns: - An empty list. """ - return [] # ruff: ignore[unused-method-argument] (synthetic: nothing to convert) - def convert(self, raw_refs: list[None]) -> TimeFDataset: + def convert(self, raw: None) -> TimeFDataset: """Generate the balanced set of labeled, single-signal records. Args: - raw_refs: Unused; the data is synthetic (``download`` returns nothing). + raw: Unused; the data is synthetic (``download`` returns nothing). Returns: The populated dataset: one single-signal series and one ``ClassificationTask`` per record. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py index 47ae40737..de8d3f142 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py @@ -1,5 +1,4 @@ from collections import Counter -from pathlib import Path from typing import cast from timenet.connectors import BaseConnector @@ -10,7 +9,7 @@ def _convert() -> TimeFDataset: - return TestMeanConnector().convert([]) + return TestMeanConnector().convert(None) def _labels_by_record(dataset: TimeFDataset) -> dict[int | None, str | None]: @@ -30,11 +29,6 @@ def test_metadata(): assert TestMeanConnector().metadata().dataset_id == "timenet/test-mean" -def test_download_defaults_to_no_op(): - # Synthetic connector: it generates everything in convert and inherits the no-op download. - assert TestMeanConnector().download(Path("cache")) == [] - - def test_convert_is_deterministic(): assert_datasets_equal(_convert(), _convert()) diff --git a/packages/timenet-connectors/tests/test_build_cli.py b/packages/timenet-connectors/tests/test_build_cli.py index 2e087451b..4290ae944 100644 --- a/packages/timenet-connectors/tests/test_build_cli.py +++ b/packages/timenet-connectors/tests/test_build_cli.py @@ -222,7 +222,7 @@ def _demo_card(dataset_id: str, **update) -> DatasetMetadata: ) -class _Parent(BaseConnector[str]): +class _Parent(BaseConnector[list[str]]): """Builds the demo dataset as test/parent.""" def metadata(self): diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index f1939b1c3..789971de1 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -270,7 +270,7 @@ def test_the_plan_reads_each_card_from_the_backend_that_claims_the_dataset(tmp_p _parent_steps(CHILD.dataset_id, LocalRegistry(tmp_path), parents_from=[InstalledConnectors()]) -class _Grand(BaseConnector[str]): +class _Grand(BaseConnector[list[str]]): """Rebuilds GRAND, the demo dataset.""" def metadata(self): diff --git a/packages/timenet/src/timenet/connectors/base.py b/packages/timenet/src/timenet/connectors/base.py index 4d265fcbe..b04ff4ebb 100644 --- a/packages/timenet/src/timenet/connectors/base.py +++ b/packages/timenet/src/timenet/connectors/base.py @@ -86,8 +86,10 @@ def metadata(self) -> DatasetMetadata: """ return DatasetMetadata.from_yaml(self._card_path()) - def download(self, cache_dir: Path) -> list[TRaw]: - """Fetch or discover raw source files and return lightweight references to them. + def download(self, cache_dir: Path) -> TRaw: + """Fetch or discover raw source files and return a lightweight handle to them. + + The handle can be any value: one object, a list of file references, or ``None``. I/O only: no parsing, no array work. Must be idempotent for a given ``cache_dir``. Override this for a synchronous connector. For an I/O-bound one, override :meth:`download_async` @@ -98,11 +100,11 @@ def download(self, cache_dir: Path) -> list[TRaw]: cache_dir: Directory to write downloaded files into (created by the engine). Returns: - Raw references passed directly to :meth:`convert`. + The download result. The engine passes it to :meth:`compose` unchanged. """ return asyncio.run(self.download_async(cache_dir)) - async def download_async(self, cache_dir: Path) -> list[TRaw]: + async def download_async(self, cache_dir: Path) -> TRaw: """Async variant of :meth:`download` for connectors whose downloads are I/O-bound. Override this to fetch artifacts concurrently, for example with the connector HTTP download @@ -112,7 +114,7 @@ async def download_async(self, cache_dir: Path) -> list[TRaw]: cache_dir: Directory to write downloaded files into (created by the engine). Returns: - Raw references passed directly to :meth:`convert`. + The download result. The engine passes it to :meth:`compose` unchanged. Raises: NotImplementedError: If a subclass overrides neither :meth:`download` nor @@ -122,8 +124,8 @@ async def download_async(self, cache_dir: Path) -> list[TRaw]: "a connector must implement download() (synchronous) or download_async() (asynchronous)" ) - def compose(self, raw_refs: list[TRaw], context: "BuildContext") -> TimeFDataset: - """Build the dataset from raw references and the opened parent releases. + def compose(self, raw: TRaw, /, context: "BuildContext") -> TimeFDataset: + """Build the dataset from the download result and the opened parent releases. A connector whose card declares ``parents`` must override this hook. The connector reads each parent through ``context.parents[alias]`` and copies objects with @@ -132,23 +134,23 @@ def compose(self, raw_refs: list[TRaw], context: "BuildContext") -> TimeFDataset :meth:`convert`. Args: - raw_refs: The references returned by :meth:`download`. + raw: The value that :meth:`download` returned. context: The opened parents, keyed by the parent aliases on the card. Returns: The populated dataset. """ del context - return self.convert(raw_refs) + return self.convert(raw) - def convert(self, raw_refs: list[TRaw]) -> TimeFDataset: - """Parse raw references and populate a :class:`~timenet.dataset.TimeFDataset`. + def convert(self, raw: TRaw, /) -> TimeFDataset: + """Parse the download result and populate a :class:`~timenet.dataset.TimeFDataset`. CPU-bound: no network I/O. Attach time-series values as lazy loaders instead of materializing them. Args: - raw_refs: The references returned by :meth:`download`. + raw: The value that :meth:`download` returned. Returns: The populated dataset. diff --git a/packages/timenet/src/timenet/engine/engine.py b/packages/timenet/src/timenet/engine/engine.py index cf9405a68..13d3b54c8 100644 --- a/packages/timenet/src/timenet/engine/engine.py +++ b/packages/timenet/src/timenet/engine/engine.py @@ -38,7 +38,7 @@ def run_pipeline( # ruff: ignore[too-many-arguments] This function is idempotent. If the target version is already committed, it skips the expensive ``download``, ``convert``, and ``store`` stages and returns the existing directory. Pass ``force`` to rebuild it. Otherwise the stages run in order: create the cache directory, open the declared - parents from ``root``, ``download`` raw references into the cache, ``compose`` them into a + parents from ``root``, ``download`` the raw source into the cache, ``compose`` the result into a dataset, check that the dataset has the card that the engine read, record each declared parent in it, ``derive_schema``, ``store``, then delete the cache directory. Parents must already be in ``root``; the engine never fetches or builds them. A card that declares parents needs a diff --git a/packages/timenet/tests/connectors/test_base.py b/packages/timenet/tests/connectors/test_base.py index 6f80dfeba..ad86ed3b6 100644 --- a/packages/timenet/tests/connectors/test_base.py +++ b/packages/timenet/tests/connectors/test_base.py @@ -8,7 +8,7 @@ def test_connector_without_convert_or_compose_fails_at_construction(): - class NoConversion(BaseConnector[str]): + class NoConversion(BaseConnector[list[str]]): def download(self, cache_dir): return [] @@ -17,7 +17,7 @@ def download(self, cache_dir): def test_connector_whose_init_skips_the_base_fails_without_convert_or_compose(): - class NoConversion(BaseConnector[str]): + class NoConversion(BaseConnector[list[str]]): def __init__(self) -> None: # Does not call super().__init__(), so the check runs in __new__. self.ready = True @@ -33,7 +33,7 @@ def test_concrete_connector_implements_contract(tmp_path): card = tmp_path / "dataset.yaml" card.write_text("dataset_id: demo/thing\ndataset_version: 1.0.0\nname: Demo\ndescription: d\nlicense: MIT\n") - class DemoConnector(BaseConnector[str]): + class DemoConnector(BaseConnector[list[str]]): CARD = card def download(self, cache_dir: Path) -> list[str]: @@ -57,7 +57,7 @@ def _write_card(tmp_path: Path) -> Path: def test_async_connector_download_bridges_to_download_async(tmp_path): card = _write_card(tmp_path) - class AsyncConnector(BaseConnector[str]): + class AsyncConnector(BaseConnector[list[str]]): CARD = card async def download_async(self, cache_dir: Path) -> list[str]: @@ -73,7 +73,7 @@ def convert(self, raw_refs: list[str]) -> TimeFDataset: def test_connector_implementing_neither_download_raises(tmp_path): card = _write_card(tmp_path) - class NoDownload(BaseConnector[str]): + class NoDownload(BaseConnector[list[str]]): CARD = card def convert(self, raw_refs: list[str]) -> TimeFDataset: @@ -86,7 +86,7 @@ def convert(self, raw_refs: list[str]) -> TimeFDataset: def test_connector_whose_init_skips_the_base_fails_without_download(tmp_path): card = _write_card(tmp_path) - class NoDownload(BaseConnector[str]): + class NoDownload(BaseConnector[list[str]]): CARD = card def __init__(self) -> None: diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 0fc57cba4..bf3ffeda5 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -92,7 +92,7 @@ def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path assert not (cache / reference.dataset_id).exists() -class _Resampled(BaseConnector[str]): +class _Resampled(BaseConnector[list[str]]): """Writes the first demo Signal at half its rate, from its declared parent, into a Record of its own.""" def metadata(self): diff --git a/packages/timenet/tests/test_engine.py b/packages/timenet/tests/test_engine.py index d7fa7fd63..689b2d8e0 100644 --- a/packages/timenet/tests/test_engine.py +++ b/packages/timenet/tests/test_engine.py @@ -54,7 +54,7 @@ def _write_demo_card() -> Path: _DEMO_CARD = _write_demo_card() -class _DemoConnector(BaseConnector[str]): +class _DemoConnector(BaseConnector[list[str]]): CARD = _DEMO_CARD def download(self, cache_dir: Path) -> list[str]: @@ -235,7 +235,7 @@ def _store_credentialed_parent(root: Path) -> None: LocalRegistry(root).put_dataset(parent) -class _ChildConnector(BaseConnector[str]): +class _ChildConnector(BaseConnector[list[str]]): """Declares the demo dataset under the parent alias ``parent`` and imports one record with ``IMPORT``.""" IMPORT: ClassVar[dict[str, Any] | None] = {} @@ -248,7 +248,7 @@ def metadata(self): def download(self, cache_dir): return [] - def compose(self, raw_refs, context): + def compose(self, raw_refs, /, context): dataset = TimeFDataset(metadata=self.metadata()) if self.IMPORT is not None: dataset.import_from(