From b0a7af17856f6715ca54803e23c00c634ec30f67 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:44:14 -0800 Subject: [PATCH 01/15] feat(format)!: annotate label sets like other entities and store them by id Remove LabelSet.name. A LabelSet is now an annotated object like a Record, Source, Signal, Task or the dataset: it has annotations and annotate(), and annotation(key) works on it. A ClassificationTask yields its label set in annotation_owners(), so the schema lists label set keys like any other keys. LabelSet(["a", "b"]) takes plain strings as labels without a description. Two label sets with the same labels (in any order, with the same descriptions) and the same multi_label are one set, and == says so. The dataset no longer keeps a label set registry. The writer stores each such set once, keeps the label order of the first set, keeps every distinct annotation occurrence of the equal sets, and does not change the caller's sets. label_sets is keyed by an integer label_set_id, tasks.label_set_id refers to it, and label set annotations are annotation_occurrences rows of a new LabelSet owner kind. The reader shares one set object for each id in a read. A copied task gets its own copy of its label set. The connectors that named their label sets attach the name as a name annotation: har_cot, sleep_cot, sleep_edfx, test_mean, hello_world and TSAQA. ARFBench stops deriving a name from its options. The control schema version stays 2. BREAKING CHANGE: LabelSet takes no name, so move it into annotations. Releases with classification tasks that were written before this change must be built again. --- benchmarks/end_to_end/corpus.py | 8 +- docs/data-model/annotations.md | 21 +- docs/data-model/records.md | 5 +- docs/data-model/tasks.md | 23 +- docs/timef-dataset.md | 6 +- docs/timef-format.md | 19 +- docs/timef-reader.md | 4 +- .../datasets/datadog/arfbench/questions.py | 24 +- .../datadog/arfbench/tests/test_connector.py | 4 +- .../datasets/opentslm/har_cot/connector.py | 4 +- .../datasets/opentslm/sleep_cot/connector.py | 2 +- .../datasets/physionet/sleep_edfx/README.md | 4 +- .../datasets/physionet/sleep_edfx/tasks.py | 18 +- .../physionet/sleep_edfx/tests/test_tasks.py | 3 +- .../datasets/timenet/hello_world/connector.py | 4 +- .../datasets/timenet/test_mean/connector.py | 4 +- .../timenet/test_mean/tests/test_connector.py | 3 +- .../datasets/tsaqa/tsaqa_benchmark/README.md | 4 +- .../tsaqa/tsaqa_benchmark/connector.py | 2 +- .../tsaqa_benchmark/tests/test_connector.py | 3 +- .../src/timenet/dataset/composition.py | 20 +- .../timenet/src/timenet/dataset/dataset.py | 32 +-- packages/timenet/src/timenet/dataset/edit.py | 5 +- .../src/timenet/format/control_audit.py | 7 +- .../src/timenet/format/control_reader.py | 93 +++++--- .../src/timenet/format/control_schema.py | 27 +-- .../src/timenet/format/control_writer.py | 99 +++++++- .../timenet/src/timenet/format/task_codec.py | 11 +- .../timenet/src/timenet/reader/load_plan.py | 11 +- .../timenet/src/timenet/testing/builders.py | 5 +- .../timenet/src/timenet/types/__init__.py | 3 +- .../timenet/src/timenet/types/annotations.py | 22 +- .../timenet/src/timenet/types/label_sets.py | 222 +++++++++++------ .../timenet/src/timenet/types/object_kind.py | 1 + packages/timenet/src/timenet/types/tasks.py | 48 ++-- .../timenet/tests/dataset/test_composition.py | 4 +- .../timenet/tests/dataset/test_dataset.py | 70 ++++-- .../tests/format/test_control_writer.py | 2 +- packages/timenet/tests/format/test_duckdb.py | 1 + .../tests/format/test_label_set_storage.py | 223 +++++++++++++----- .../timenet/tests/format/test_task_codec.py | 1 - packages/timenet/tests/registry/conftest.py | 1 - .../tests/registry/test_conformance.py | 1 - .../timenet/tests/registry/test_namespaced.py | 2 +- .../timenet/tests/types/test_label_sets.py | 104 +++++++- .../timenet/tests/writer/test_id_encoding.py | 2 +- .../timenet/tests/writer/test_sharding.py | 1 - packages/timenet/tests/writer/test_writer.py | 10 +- 48 files changed, 789 insertions(+), 404 deletions(-) diff --git a/benchmarks/end_to_end/corpus.py b/benchmarks/end_to_end/corpus.py index 2dbe348a6..478119b36 100644 --- a/benchmarks/end_to_end/corpus.py +++ b/benchmarks/end_to_end/corpus.py @@ -31,10 +31,10 @@ from timenet.types.wire import ValueDtype -CONDITION = LabelSet(name="condition", labels=(Label(name="healthy"), Label(name="outer-race-fault"))) -RHYTHM = LabelSet(name="rhythm", labels=(Label(name="normal-sinus"), Label(name="atrial-fibrillation"))) -SLEEP_STAGE = LabelSet(name="sleep-stage", labels=tuple(Label(name=stage) for stage in ("W", "N1", "N2", "N3", "REM"))) -MEAN_SIGN = LabelSet(name="mean-sign", labels=(Label(name="above-zero"), Label(name="below-zero"))) +CONDITION = LabelSet(labels=(Label(name="healthy"), Label(name="outer-race-fault"))) +RHYTHM = LabelSet(labels=(Label(name="normal-sinus"), Label(name="atrial-fibrillation"))) +SLEEP_STAGE = LabelSet(labels=tuple(Label(name=stage) for stage in ("W", "N1", "N2", "N3", "REM"))) +MEAN_SIGN = LabelSet(labels=(Label(name="above-zero"), Label(name="below-zero"))) @dataclass(frozen=True) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 27a7b5c35..be62c9d26 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -9,7 +9,7 @@ tags: # Annotations An `Annotation` stores reusable content. An annotation occurrence attaches that content to a Dataset, -Task, Record, Source, or Signal. +Task, Record, Source, Signal, or label set. ```python from timenet.types import Annotation @@ -116,6 +116,25 @@ a separate occurrence with shared content. Other Signals remain unchanged. Time spans use whole microseconds on the Record timeline. Step spans are reserved for Tasks on ordinal Signals. +## Label set annotations + +A `LabelSet` takes annotations like the other owners. Pass them to the constructor, or call +`annotate()` later. Use them for a name, a description, or other context. No key is required. + +```python +from timenet.types import Annotation, Label, LabelSet + +SLEEP_STAGE = LabelSet( + labels=(Label(name="W"), Label(name="N1"), Label(name="R")), + annotations=(Annotation(key="name", value="sleep_stage"),), +) +SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) +SLEEP_STAGE.annotation("name") +``` + +A label set annotation has no span. Annotations are not part of the identity of a set. TimeF stores +equal label sets once. If equal sets carry different annotations, the stored set keeps all of them. + ## Valid values An annotation value can be a string, integer, float, Boolean, or list of strings. For example, a diff --git a/docs/data-model/records.md b/docs/data-model/records.md index 76717a056..412b77435 100644 --- a/docs/data-model/records.md +++ b/docs/data-model/records.md @@ -121,10 +121,7 @@ Records. ```python from timenet.types import ClassificationTask, Label, LabelSet -CONDITION = LabelSet( - name="condition", - labels=(Label(name="healthy"), Label(name="faulty")), -) +CONDITION = LabelSet(labels=(Label(name="healthy"), Label(name="faulty"))) task = ClassificationTask( inputs=(record,), diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index fa7f17039..aab54d651 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -14,7 +14,7 @@ expected output. ```python from timenet.types import ClassificationTask, Label, LabelSet -CONDITION = LabelSet(name="condition", labels=(Label(name="healthy"), Label(name="faulty"))) +CONDITION = LabelSet(labels=(Label(name="healthy"), Label(name="faulty"))) task = ClassificationTask( inputs=(record,), @@ -87,10 +87,7 @@ own: one Record may serve Tasks of several splits. ```python from timenet.types import ClassificationTask, Label, LabelSet, Split -ACTIVITY = LabelSet( - name="activity", - labels=(Label(name="walking"), Label(name="sitting")), -) +ACTIVITY = LabelSet(labels=(Label(name="walking"), Label(name="sitting"))) dataset.add_task( ClassificationTask( @@ -225,11 +222,9 @@ dataset.add_task(age_question(record, 64)) from timenet.types import Label, LabelSet CONDITION = LabelSet( - name="condition", labels=(Label(name="healthy"), Label(name="faulty", description="A bearing fault.")), ) FINDINGS = LabelSet( - name="findings", labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), multi_label=True, ) @@ -238,8 +233,9 @@ ClassificationTask(inputs=(record,), targets="healthy", label_set=CONDITION) ClassificationTask(inputs=(record,), targets=("PVC", "AF"), label_set=FINDINGS) ``` -A `LabelSet` defines the allowed labels for a classification target. The `multi_label` flag controls -how many labels a target can contain. +A `LabelSet` defines the allowed labels for a classification target. Like other owners, it can carry +[annotations](annotations.md#label-set-annotations). The `multi_label` flag controls how many labels +a target can contain. If `multi_label=False`, the default, a target must contain exactly one label. If `multi_label=True`, a target contains all labels that apply. Omitted labels do not apply. An empty target means that no @@ -252,9 +248,12 @@ must contain exactly one label. Labels come from the targets stored directly on the task or from the text of its target annotations. -Tasks can share the same label set object. Each name identifies one label set within a dataset. -If two different sets have the same name, `add_task()`, the task stream, and the writer raise an -error. TimeF stores each set once. The reader gives tasks that use the same set a shared object. +A label set has no name. TimeF stores equal sets once. For this, two sets are equal if they have the +same labels, label descriptions, and `multi_label`. The order of labels and the annotations do not +matter, and `==` follows the same rule. The stored set gets one integer id, `label_set_id`, and keeps +the label order of the first equal set that the writer meets. If you need the order of each task, +put it in the task metadata. If equal sets carry different annotation occurrences, the stored set +keeps each distinct occurrence. In one read, the reader gives tasks that use the same set a shared object. ### Answer and rationale diff --git a/docs/timef-dataset.md b/docs/timef-dataset.md index 407b08a4d..056379fa0 100644 --- a/docs/timef-dataset.md +++ b/docs/timef-dataset.md @@ -213,7 +213,7 @@ inserts that stream once without retaining every Task in memory. ## Annotations -The same `Annotation` type can annotate a Dataset, Task, Record, Source, or Signal. Annotation +The same `Annotation` type can annotate a Dataset, Task, Record, Source, Signal, or label set. Annotation content is immutable and reusable. Every call to `annotate()` creates a separate occurrence, so a long payload can be stored once while its occurrences carry independent placement, confidence, and provenance. `Annotation` is final: reuse content through a constant, or a builder function for @@ -238,8 +238,8 @@ lead_i.annotate( `derive_schema()` collects the time-series specifications, annotation descriptors, and concrete Task types found in the dataset. Annotation descriptors come from every owner: the Dataset, Records, -Sources, Signals, and Tasks. The Tasks of a task stream are not read here. The writer adds their -keys to the schema while it writes them. It does not load Signal values. +Sources, Signals, Tasks, and label sets. The Tasks of a task stream are not read here. The writer +adds their keys to the schema while it writes them. It does not load Signal values. Write a version with the declarative helper: diff --git a/docs/timef-format.md b/docs/timef-format.md index c3b939d79..948ae7f58 100644 --- a/docs/timef-format.md +++ b/docs/timef-format.md @@ -171,9 +171,9 @@ The `annotation_occurrences` table records where the content applies. It points the content through `content_id`. It also carries the span as `span_type`, `start_us`, and `end_us`, plus provenance, confidence, and occurrence metadata. -An occurrence can annotate a Dataset, Task, Record, Source, or Signal. It names its owner with -`object_type` and `object_id`. For a Record, Signal, or Task, `object_id` is the entity ID of the -owner. For a Source, it is the storage ID of the Source. `object_id` is `NULL` exactly when the +An occurrence can annotate a Dataset, Task, Record, Source, Signal, or label set. It names its owner with +`object_type` and `object_id`. For a Record, Signal, Task, or label set, `object_id` is the ID of the +owner. For a Source, it is the storage ID of that row. `object_id` is `NULL` exactly when the owner is the Dataset. IDs are unique only within one kind, so the type is part of the reference. A read of every Record or every Task rejects an occurrence with an unknown owner type or a missing @@ -188,14 +188,17 @@ its own `occurrence_id` and temporal placement. Python subclass. Four typed columns store the scope: `scope_type`, `scope_start`, `scope_end`, and `scope_signal_ids`. -Configuration for each task type uses the columns `label_set`, `unit`, `target_name`, and `mode`. +Configuration for each task type uses the columns `label_set_id`, `unit`, `target_name`, and `mode`. A column that a task type does not use is `NULL`. SQL can filter these columns without parsing JSON. -Every classification task has a label set, and `tasks.label_set` stores its name. Each row in -`label_sets` stores one set with the fields `name`, `labels`, and `multi_label`. The name is unique -within a release. +Every classification task has a label set, and `tasks.label_set_id` stores its ID. Each row in +`label_sets` stores one set with the fields `label_set_id`, `labels`, and `multi_label`. Sets with the +same labels, label descriptions, and `multi_label` are one set, so a release stores them once. The +writer numbers the sets in the order that tasks first use them. The annotations of a set are rows of +`annotation_occurrences` with `object_type` `LabelSet` and the `label_set_id` as `object_id`. -The `labels` field holds one `{name, description}` entry per label, in label order. If a label has +The `labels` field holds one `{name, description}` entry per label, in the label order of the first +equal set that the writer meets. If a label has no description, its `description` is `NULL`. Each inline target is one row in `task_targets`. Its `target_kind` names the typed value column diff --git a/docs/timef-reader.md b/docs/timef-reader.md index a22afcc54..594669d3e 100644 --- a/docs/timef-reader.md +++ b/docs/timef-reader.md @@ -112,8 +112,8 @@ Tasks have four useful read paths: The Record, Signal, and Task columns hold the ids that the dataset assigned, for example `task_id`, `input_record_ids`, `from_task_ids`, and `span_signal_ids`. `occurrence_id` and `content_id` are storage row numbers of this version. The next write numbers them again. In - `read_annotation_table()`, `object_id` is a Record, Signal, or Task id, a storage row number for - a Source, or `NULL` for the dataset. + `read_annotation_table()`, `object_id` is a Record, Signal, or Task id, a label set id, a storage + row number for a Source, or `NULL` for the dataset. One call returns one consistent set of objects. A Record that two tasks use is one object, and the Signals on one irregular axis share one handle to its time offsets. `read_tasks(inputs=records)` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py index 6d80560cc..62c8b2524 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py @@ -5,12 +5,11 @@ import csv from dataclasses import dataclass from enum import Enum -import hashlib import json from typing import TYPE_CHECKING, Self, TypeVar from timenet.errors import TimeNetConnectorError -from timenet.types import Annotation, ClassificationTask, InputModality, Label, LabelSet, Split +from timenet.types import Annotation, ClassificationTask, InputModality, LabelSet, Split if TYPE_CHECKING: @@ -108,15 +107,6 @@ def _parse_answer_options(cell: str) -> tuple[str, ...]: return tuple(options) -def _label_set_name(options: tuple[str, ...]) -> str: - """Name a label set by its options, so that equal option lists share one set. - - Returns: - ``options_`` and the first 8 hex digits of the SHA-256 of the options as a JSON list. - """ - return "options_" + hashlib.sha256(json.dumps(options).encode()).hexdigest()[:8] - - @dataclass(frozen=True) class ARFBenchQuestion: """One row of the question table: a multiple-choice question about one or two metrics.""" @@ -216,10 +206,9 @@ def create_chart_task(self, chart_records: Mapping[str, Record]) -> Classificati def _create_task(self, inputs: list[Record], modality: InputModality) -> ClassificationTask: """Create a test task that asks for the correct answer from the options of the question. - The options are the label set of the task. The set is named after a hash of the options, so - questions with the same options share one set. The row index in the metadata pairs the - series task with the chart task of ``datadog/arfbench-charts``. The interpolation flags - describe what an engineer saw. This method does not interpolate. + The options are the label set of the task. Questions with the same options share one set. The row + index in the metadata pairs the series task with the chart task of ``datadog/arfbench-charts``. + The interpolation flags describe what an engineer saw. This method does not interpolate. Returns: The task, not yet added to a dataset. @@ -231,10 +220,7 @@ def _create_task(self, inputs: list[Record], modality: InputModality) -> Classif input_modalities=frozenset({InputModality.TEXT, modality}), split=Split.TEST, metadata={"question_index": self.index}, - label_set=LabelSet( - name=_label_set_name(self.answer_options), - labels=tuple(Label(name=option) for option in self.answer_options), - ), + label_set=LabelSet(self.answer_options), ) task.annotate(Annotation("task_category", value=self.task_category.value)) task.annotate(Annotation("difficulty", value=self.difficulty.value)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py index f75e74c8d..21c19c564 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py @@ -115,8 +115,8 @@ def test_questions_with_equal_options_share_a_label_set(release): assert isinstance(presence, ClassificationTask) and isinstance(correlation, ClassificationTask) assert isinstance(up_down_task, ClassificationTask) assert presence.label_set == correlation.label_set - assert presence.label_set.name.startswith("options_") - assert up_down_task.label_set.name != presence.label_set.name + assert presence.label_set.annotations == () + assert up_down_task.label_set != presence.label_set def test_the_dataset_round_trips_through_parquet_with_its_missing_timesteps(release, tmp_path): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py index 9ab4d7136..36d100dde 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py @@ -16,7 +16,7 @@ from timenet.connectors import BaseConnector from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset from timenet.errors import TimeNetConnectorError -from timenet.types import ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg +from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg from timenet_connectors.download import ensure_archive from timenet_connectors.sources.readers import CsvDataReader, DataRow @@ -35,7 +35,6 @@ # The label order is the order of the OpenTSLM training prompt. ACTIVITY = LabelSet( - name="activity", labels=( Label(name="biking"), Label(name="lying"), @@ -46,6 +45,7 @@ Label(name="walking_down", description="Walking down stairs."), Label(name="walking_up", description="Walking up stairs."), ), + annotations=(Annotation(key="name", value="activity"),), ) # The release states no unit. A resting window has a magnitude of about 9.8, which is gravity in m/s². ACCELERATION = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index b2ecf5f96..01f5fd0ea 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -35,7 +35,6 @@ # The release writes the stage codes of the Rechtschaffen and Kales manual. The prompts and the # rationales write the names in the descriptions. SLEEP_STAGE = LabelSet( - name="sleep_stage", labels=( Label(name="W", description="Wake"), Label(name="N1", description="Non-REM stage 1"), @@ -44,6 +43,7 @@ Label(name="N4", description="Non-REM stage 4"), Label(name="REM", description="REM sleep"), ), + annotations=(Annotation(key="name", value="sleep_stage"),), ) # OpenTSLM converted the microvolts of Sleep-EDF to volts and kept every second sample of 100 Hz. EEG = TimeSeriesSpec(spec_type="eeg", name="EEG", unit_value=ureg.volt, dtype="float64") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md index c2e49a27b..a7a7bf5c1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md @@ -122,8 +122,8 @@ where the recording came from. ### How the connector writes them Each classification task has a set of allowed target labels. The three sets contain eight sleep -stages, two sexes, and two study conditions. Each `LabelSet` uses its annotation key as its name: -`sleep_stage`, `sex`, or `condition`. +stages, two sexes, and two study conditions. Each `LabelSet` carries a `name` annotation with its +annotation key: `sleep_stage`, `sex`, or `condition`. Tasks that ask the same question share one label set. Each target must contain exactly one label from that set. TimeNet enforces this requirement when it writes the dataset. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py index ded52053a..ccff401ad 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py @@ -6,8 +6,9 @@ No task names its own id. Nothing resolves one by id: each task holds its record in ``inputs``, and no task here states a lineage. Every task takes the id the dataset gives it. -Each classification task draws its target from one closed set, a ``LabelSet`` named after the -annotation key whose values it holds. The tasks that ask one question share one set. +Each classification task draws its target from one closed set. The ``name`` annotation of each +``LabelSet`` holds the annotation key whose values the set lists. The tasks that ask one question +share one set. No task carries a prompt. The release states no question in words. """ @@ -38,10 +39,9 @@ # The labels the release writes for a scored epoch. Five name a stage of sleep. `Sleep stage W` # is wake, `Movement time` marks a disturbed epoch, and `Sleep stage ?` marks one the -# technician left unscored. Each classification task draws its target from one closed set, named -# after the key whose values it holds, and each target is exactly one label. +# technician left unscored. Each classification task draws its target from one closed set, whose +# name annotation is the key whose values it holds, and each target is exactly one label. SLEEP_STAGES = LabelSet( - name=str(AnnotationKey.SLEEP_STAGE), labels=( Label(name="Sleep stage W", description="Wake."), Label(name="Sleep stage 1"), @@ -52,6 +52,7 @@ Label(name="Movement time", description="An epoch that body movement obscures, so it has no stage."), Label(name="Sleep stage ?", description="An epoch with no sleep-stage score from data collection."), ), + annotations=(Annotation(key="name", value=str(AnnotationKey.SLEEP_STAGE)),), ) STAGE_LABELS = SLEEP_STAGES.names @@ -61,14 +62,17 @@ # The sexes, after each sheet is decoded. The two sheets code the column with opposite # meanings, so a raw code never reaches a task. SEXES = LabelSet( - name=str(AnnotationKey.SEX), labels=(Label(name="F", description="Female."), Label(name="M", description="Male.")), + annotations=(Annotation(key="name", value=str(AnnotationKey.SEX)),), ) SEX_LABELS = SEXES.names # The two nights of the telemetry study. CONDITION_LABELS = tuple(Condition) -CONDITIONS = LabelSet(name=str(AnnotationKey.CONDITION), labels=tuple(Label(name=str(one)) for one in Condition)) +CONDITIONS = LabelSet( + labels=[str(one) for one in Condition], + annotations=(Annotation(key="name", value=str(AnnotationKey.CONDITION)),), +) def build_epoch_tasks(recording_id: str, span_annotations: Sequence[Annotation]) -> Iterator[ClassificationTask]: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py index cdac61de4..948885b3d 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py @@ -11,6 +11,7 @@ TimeInterval, TimePoint, ) +from timenet.types.text import read_value from timenet_connectors.datasets.physionet.sleep_edfx import tasks from timenet_connectors.datasets.physionet.sleep_edfx.keys import AnnotationKey @@ -153,7 +154,7 @@ def _pick(built, kind, label_set=None): def _label_sets(built): - return {one.label_set.name for one in built if isinstance(one, ClassificationTask)} + return {read_value(one.label_set.annotation("name").value) for one in built if isinstance(one, ClassificationTask)} def test_age_is_a_scalar_with_a_unit(): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py index 781e71d64..e40de0f96 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py @@ -49,12 +49,12 @@ # Each classification task draws its target from a label set that every task with the same question # shares. _RHYTHM = LabelSet( - name="rhythm", labels=(Label(name="normal", description="A regular wave."), Label(name="abnormal")), + annotations=(Annotation(key="name", value="rhythm"),), ) _EVENT = LabelSet( - name="event", labels=(Label(name="onset", description="The moment the wave starts."), Label(name="offset")), + annotations=(Annotation(key="name", value="event"),), ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py index 3e84a3fef..da086bcf8 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py @@ -14,7 +14,7 @@ from timenet.connectors import BaseConnector from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset -from timenet.types import ClassificationTask, Label, LabelSet, TimeSeriesSpec, ureg +from timenet.types import Annotation, ClassificationTask, Label, LabelSet, TimeSeriesSpec, ureg _N_RECORDS = 1000 @@ -23,11 +23,11 @@ _NOISE_STD = 0.5 _SEED = 20260715 # fixed base seed keeps every build's values identical _MEAN_SIGN = LabelSet( - name="mean_sign", labels=( Label(name="above_zero", description="The mean of the signal is above zero."), Label(name="below_zero", description="The mean of the signal is below zero."), ), + annotations=(Annotation(key="name", value="mean_sign"),), ) _SIGNAL = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py index 4c0cef688..2acf80378 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py @@ -46,7 +46,8 @@ def test_has_balanced_binary_labels(): def test_every_task_shares_the_mean_sign_label_set(): label_sets = [task.label_set for task in _convert().tasks_of(ClassificationTask)] - assert {label_set.name for label_set in label_sets} == {"mean_sign"} + names = [label_set.annotation("name") for label_set in label_sets] + assert {None if name is None else read_value(name.value) for name in names} == {"mean_sign"} assert len({id(label_set) for label_set in label_sets}) == 1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md index edfb7bed6..90fe18238 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md @@ -156,8 +156,8 @@ metadata. | `puzzling` | `AnswerTask` | Four letters, such as `C,B,D,A` | 20,932 | 2,979 | 5,978 | - **A true-or-false question** is a classification, since `T` and `F` mean the same in every - question. Every true-or-false task shares the `true_or_false` label set: `T` (True.) and `F` - (False.). + question. Every true-or-false task shares one label set, with a `name` annotation of `true_or_false`: `T` + (True.) and `F` (False.). - **A multiple-choice question or a puzzle** is an answer task. Its letters name choices that differ from one question to the next. A classification question's choices are its UCR dataset's class names, such as `normal process`. The `answer_text` metadata holds the right choice's text when the diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index dfa5c70e4..0edd4cefc 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -74,8 +74,8 @@ class _Role(StrEnum): _TRUE_OR_FALSE_LABEL_SET = LabelSet( - name="true_or_false", labels=(Label(name="T", description="True."), Label(name="F", description="False.")), + annotations=(Annotation(key="name", value="true_or_false"),), ) """The label set that every true-or-false task shares.""" _CONTEXT_KEYS = ("meta_info", "domain", "dataset") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py index a5d29c807..a56687def 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py @@ -109,7 +109,8 @@ def _check_task(task, row: dict[str, Any]) -> None: assert task.metadata["question_type"] == row["question_type"] assert task.metadata["source_issues"] == _EXPECTED_ISSUES.get(_row_of(task), []) if isinstance(task, ClassificationTask): - assert (task.label_set.name, task.label_set.names) == ("true_or_false", ("T", "F")) + assert task.label_set.names == ("T", "F") + assert task.label_set.annotation("name").value == "true_or_false" def test_every_task_holds_its_published_row(): diff --git a/packages/timenet/src/timenet/dataset/composition.py b/packages/timenet/src/timenet/dataset/composition.py index 430ff4099..b58c55756 100644 --- a/packages/timenet/src/timenet/dataset/composition.py +++ b/packages/timenet/src/timenet/dataset/composition.py @@ -3,7 +3,7 @@ from collections import deque from collections.abc import Container, Hashable, Iterable, Iterator, Mapping from copy import deepcopy -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypeAlias from timenet.dataset.record import Record @@ -18,6 +18,7 @@ TimePoint, TSCorrespondenceTask, ) +from timenet.types.annotations import _copy_annotation from timenet.types.clock import TimeOrigin from timenet.types.ids import EntityId, check_id from timenet.types.load_plan import LoadPlan @@ -163,23 +164,6 @@ def _fits(annotation: Annotation, signals: tuple[Signal, ...], owner: str, time_ return True -def _copy_annotation(original: Annotation) -> Annotation: - """Copy an annotation occurrence as a value, with the handle of a stored value. - - The copy has deep copies of the value, the metadata, and the occurrence metadata. So a change to - one of them in the copy leaves the original as it is. Imports and edits copy annotations this way. - - Returns: - The copy. - """ - return replace( - original, - value=deepcopy(original.value), - metadata=deepcopy(original.metadata), - occurrence_metadata=deepcopy(original.occurrence_metadata), - ) - - def _hierarchy(record: Record) -> list[tuple[Record | Source, dict[str, tuple]]]: """Return the fields that an import changes when it adds to the copy of a Record. diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index 9a4532450..c019ed46c 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -23,7 +23,6 @@ DatasetSchema, EntityId, InputModality, - LabelSetRegistry, ObjectKind, StepSpan, SupportsAnnotate, @@ -92,8 +91,6 @@ def __init__(self, *, metadata: DatasetMetadata) -> None: """The Record that holds each registered Signal, by Signal ID.""" self._tasks: list[Task] = [] self._tasks_by_id: dict[EntityId, Task] = {} - self._label_sets = LabelSetRegistry() - """The label sets that the registered tasks use, one per name.""" self._ids = IdCounters() self._streamed_task_base = 0 self._annotations: list[Annotation] = [] @@ -475,8 +472,8 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: the cross-task checks (``from_tasks`` derivations) that need every task at once do not run for a stream. - If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks are - added to it. After the last task, the schema lists them. + If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks and + their label sets are added to it. After the last task, the schema lists them. Yields: Each validated task, in the source's order. @@ -487,12 +484,11 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: """ # ruff: ignore[docstring-extraneous-exception] (raised by _validate_streamed_task, not directly here) index = _TaskValidationIndex(self) declared = set(self._streamed_task_types) - label_sets = LabelSetRegistry() schema = self._schema descriptors = {} if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations} for position, task in enumerate(self.iter_tasks(), start=1): task._id = self._ids.claim(ObjectKind.TASK, self._streamed_task_base + position) - self._validate_streamed_task(task, index, declared, label_sets) + self._validate_streamed_task(task, index, declared) if schema is not None: self._add_descriptors(descriptors, task.walk_annotations()) yield task @@ -504,7 +500,6 @@ def _validate_streamed_task( task: Task, index: _TaskValidationIndex, declared: set[type[Task]], - label_sets: LabelSetRegistry, ) -> None: """Make sure that each streamed task follows the dataset rules. @@ -512,8 +507,6 @@ def _validate_streamed_task( task: The streamed task. index: The shared index of dataset objects and annotations. declared: The task types :meth:`set_task_stream` declared. - label_sets: The label sets used so far in the stream. - This method adds the label set of this task. Raises: TimeFValidationError: If the task's type is not declared, or the task fails a check of @@ -524,9 +517,9 @@ def _validate_streamed_task( f"streamed task {task.id!r} has type {type(task).__name__}, not one of the declared " f"{sorted(t.__name__ for t in declared)}" ) - self._check_task(task, index, label_sets) + self._check_task(task, index) - def _check_task(self, task: Task, index: _TaskValidationIndex, label_sets: LabelSetRegistry) -> None: + def _check_task(self, task: Task, index: _TaskValidationIndex) -> None: """Make sure that a task follows the dataset rules. This method handles tasks added alone, in a batch, or in a stream. @@ -534,7 +527,6 @@ def _check_task(self, task: Task, index: _TaskValidationIndex, label_sets: Label Args: task: The task to check. index: The shared index of dataset objects and annotations. - label_sets: The label sets used so far. This method adds the label set of this task. Raises: TimeFValidationError: If the task's record attachment, references, answer, labels, label @@ -544,7 +536,6 @@ def _check_task(self, task: Task, index: _TaskValidationIndex, label_sets: Label task.check_against_scope() self._check_task_answer(task) task.check_targets() - task.register_label_set(label_sets) self._check_signal_refs(task, index) self._validate_task_spans(task, index) self._check_annotation_refs(task, self._referenced_records(task), index) @@ -582,7 +573,7 @@ def _register_batch(self, batch: tuple[Task, ...]) -> tuple[Task, ...]: for offset, task in enumerate(batch): task._id = first + offset try: - label_sets = self._validate_task_batch(batch) + self._validate_task_batch(batch) except BaseException: for task, task_id in zip(batch, previous, strict=True): task._id = task_id @@ -591,14 +582,13 @@ def _register_batch(self, batch: tuple[Task, ...]) -> tuple[Task, ...]: self._ids.claim(ObjectKind.TASK, task.id) self._tasks.append(task) self._tasks_by_id[task.id] = task - self._label_sets = label_sets return batch def _is_registered(self, task: Task) -> bool: """Return whether this exact task object is in the dataset.""" return task._id is not None and self._tasks_by_id.get(task._id) is task - def _validate_task_batch(self, batch: tuple[Task, ...]) -> LabelSetRegistry: + def _validate_task_batch(self, batch: tuple[Task, ...]) -> None: """Make sure that a batch follows the dataset rules before attaching its tasks. This method examines each task and the relationships between tasks. @@ -608,9 +598,6 @@ def _validate_task_batch(self, batch: tuple[Task, ...]) -> LabelSetRegistry: Args: batch: The tasks to register together. - Returns: - A copy of the registry that includes the label sets from this batch. - Raises: TimeFValidationError: This error occurs under the conditions documented on :meth:`add_tasks`. @@ -627,10 +614,8 @@ def _validate_task_batch(self, batch: tuple[Task, ...]) -> LabelSetRegistry: ) self._check_no_derivation_cycle(batch) index = _TaskValidationIndex(self) - label_sets = self._label_sets.copy() for task in batch: - self._check_task(task, index, label_sets) - return label_sets + self._check_task(task, index) @classmethod def _validate_task_spans(cls, task: Task, index: _TaskValidationIndex) -> None: @@ -787,7 +772,6 @@ def from_parts( # ruff: ignore[too-many-arguments] - every part of a stored dat dataset._tasks = list(tasks) for task in dataset._tasks: dataset._tasks_by_id[dataset._ids.claim(ObjectKind.TASK, task.id)] = task - task.register_label_set(dataset._label_sets) dataset._annotations = list(annotations) dataset._schema = schema if import_state is not None: diff --git a/packages/timenet/src/timenet/dataset/edit.py b/packages/timenet/src/timenet/dataset/edit.py index c51fba7eb..7967907ef 100644 --- a/packages/timenet/src/timenet/dataset/edit.py +++ b/packages/timenet/src/timenet/dataset/edit.py @@ -13,7 +13,7 @@ from dataclasses import replace from pathlib import Path -from timenet.dataset.composition import _copy_annotation, _fits +from timenet.dataset.composition import _fits from timenet.dataset.dataset import TimeFDataset from timenet.dataset.record import Record from timenet.dataset.source import Source @@ -24,6 +24,7 @@ from timenet.registry._paths import release_dir, split_release_dir from timenet.registry.dependencies import open_local_version from timenet.types import Annotation, DatasetSchema, EntityId, Task, TimeInterval, TimeSpan, Version +from timenet.types.annotations import _copy_annotation from timenet.types.object_kind import ObjectKind from timenet.types.references import DatasetRef from timenet.writer import TimeFWriter @@ -242,7 +243,7 @@ def reachable(task: Task, annotation: Annotation) -> bool: def _copied(annotations: tuple[Annotation, ...], copies: dict[int, Annotation]) -> tuple[Annotation, ...]: """Return a copy of each annotation, and keep each copy in ``copies``. - The copies have their own value and metadata (see ``composition._copy_annotation``). An + The copies have their own value and metadata (see ``annotations._copy_annotation``). An annotation that ``copies`` already holds keeps that copy, so an annotation object that two owners share gets one copy. diff --git a/packages/timenet/src/timenet/format/control_audit.py b/packages/timenet/src/timenet/format/control_audit.py index bb41882f2..faa99bfc0 100644 --- a/packages/timenet/src/timenet/format/control_audit.py +++ b/packages/timenet/src/timenet/format/control_audit.py @@ -56,6 +56,7 @@ def _sql_in(values: Iterable[str]) -> str: ("signals", "signal_id"), ("annotation_contents", "content_id"), ("annotation_occurrences", "occurrence_id"), + ("label_sets", "label_set_id"), ("tasks", "task_id"), ) """The primary id column of each entity table.""" @@ -112,8 +113,8 @@ def _sql_in(values: Iterable[str]) -> str: OR list_contains([label.name FOR label IN labels], '') OR list_contains([label.description FOR label IN labels], '') UNION ALL - SELECT 'tasks.label_set' FROM tasks - WHERE (label_set IS NOT NULL) <> (task_type = '{TaskType.CLASSIFICATION}') + SELECT 'tasks.label_set_id' FROM tasks + WHERE (label_set_id IS NOT NULL) <> (task_type = '{TaskType.CLASSIFICATION}') UNION ALL {_ENTITY_ID_CHECKS} LIMIT 1 @@ -122,7 +123,7 @@ def _sql_in(values: Iterable[str]) -> str: _LABELED_TARGETS = f""" WITH labeled AS ( SELECT t.task_id, [label.name FOR label IN s.labels] AS labels, s.multi_label - FROM tasks t JOIN label_sets s ON s.name = t.label_set + FROM tasks t JOIN label_sets s ON s.label_set_id = t.label_set_id ), targets AS ( SELECT x.task_id, x.target_kind = '{TaskTargetKind.TEXT}' AS is_text, x.text_value AS label, diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index d8bf92455..59527b43d 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -6,7 +6,7 @@ from collections import ChainMap, Counter, defaultdict from collections.abc import Callable, Collection, Container, Iterable, Iterator, Mapping, MutableMapping from fractions import Fraction -from functools import cached_property, partial +from functools import partial from graphlib import CycleError, TopologicalSorter import json import os @@ -50,6 +50,7 @@ Annotation, ArrayHandle, InputModality, + Label, LabelSet, MissingValues, Task, @@ -73,7 +74,8 @@ ValueLoaderFactory = Callable[[int, TimeSeriesSpec, int], ArrayHandle] _OwnerId: TypeAlias = int -"""The ID of an annotation's owner: the public ID of a Record, Signal, or Task, or the catalog row ID of a Source.""" +"""The ID of an annotation's owner: the key of its row in the table that +:data:`~timenet.format.control_schema.ANNOTATED_OWNER_TABLES` names for its kind.""" _AnnotationOwner: TypeAlias = tuple[ObjectKind, _OwnerId | None] """An annotation's owner: its kind and its ID. The ID is ``None`` for the Dataset itself.""" _OccurrenceId: TypeAlias = int @@ -251,7 +253,7 @@ class _TaskRow(NamedTuple): scope_end: int | None scope_signal_ids: list[int] | None has_inline_targets: bool - label_set: str | None + label_set_id: int | None unit: str | None target_name: str | None mode: str | None @@ -790,8 +792,6 @@ def close(self) -> None: self._disconnect() self._clock_cache.clear() self._task_facts = None - # Drop the cached label sets, so that a reopened reader reads them again. - self.__dict__.pop("_label_sets_by_name", None) self._known_record_ids.clear() self._chunk_strings.clear() @@ -1629,7 +1629,7 @@ def read_task_table( ) SELECT t.task_id, t.task_type, t.prompt, t.rationale, t.scope_type, t.scope_start, t.scope_end, t.scope_signal_ids, - t.label_set, t.unit, t.target_name, t.mode, t.has_inline_targets, + t.label_set_id, t.unit, t.target_name, t.mode, t.has_inline_targets, t.input_modalities, t.split, coalesce(inputs.input_record_ids, []) AS input_record_ids, coalesce(parents.from_task_ids, []) AS from_task_ids, @@ -1853,44 +1853,38 @@ def _task_rows(self, task_ids: list[int] | None) -> list[_TaskRow]: _TaskRow._make, ) - @cached_property - def _label_sets_by_name(self) -> dict[str, LabelSet]: - """The stored label sets by name. + def _read_label_sets(self) -> dict[int, LabelSet]: + """Read every stored label set with its annotations. - The reader reads each set once. Tasks that use the same set share one object. + Returns: + The label sets by id. Raises: - TimeFFormatError: If a stored label set is not valid. + TimeFFormatError: If a stored label set or one of its annotations is not valid. """ columns = ", ".join(TABLES["label_sets"].column_names) # ruff: ignore[hardcoded-sql-expression] - fixed identifiers rows = self._query_table(f"SELECT {columns} FROM label_sets").to_pylist() - label_sets: dict[str, LabelSet] = {} + annotations = self._read_annotations({ObjectKind.LABEL_SET: None}) + stored_ids = {row["label_set_id"] for row in rows} + for _, label_set_id in annotations: + if label_set_id not in stored_ids: + raise TimeFFormatError( + f"annotation occurrence refers to missing {ObjectKind.LABEL_SET} {label_set_id!r}" + ) + label_sets: dict[int, LabelSet] = {} for row in rows: + label_set_id = row["label_set_id"] try: - label_set = LabelSet.model_validate(row) - except ValidationError as exc: - raise TimeFFormatError(f"the stored label set {row['name']!r} is not valid: {exc}") from exc - label_sets[label_set.name] = label_set + label_sets[label_set_id] = LabelSet( + labels=tuple(Label.model_validate(label) for label in row["labels"]), + multi_label=row["multi_label"], + annotations=annotations.get((ObjectKind.LABEL_SET, label_set_id), ()), + ) + except (ValidationError, TimeFValidationError) as exc: + raise TimeFFormatError(f"the stored label set {label_set_id!r} is not valid: {exc}") from exc return label_sets - def _label_set(self, task_id: int, name: str | None) -> LabelSet: - """Return the stored label set that a classification task names. - - Returns: - The shared label set. - - Raises: - TimeFFormatError: If the task names no label set, no label set has this name, or a stored - label set is not valid. - """ - if name is None: - raise TimeFFormatError(f"classification task {task_id!r} has no label set") - label_set = self._label_sets_by_name.get(name) - if label_set is None: - raise TimeFFormatError(f"tasks refer to a missing label set {name!r}") - return label_set - def _target_rows(self, task_ids: list[int] | None) -> list[_TargetRow]: """Read the target rows for one batch of tasks, or every task, in task and position order. @@ -2268,7 +2262,7 @@ class _TaskBuilder: Records, Signals, and annotations that tasks refer to outside that set once, and keeps every task that another task derives from until its dependants are built. A one-pass read is one scope. A stream starts a new scope for each batch, so its batches share only the caller's - objects. + objects and the label sets, which the builder reads once. """ def __init__( # ruff: ignore[too-many-arguments] - one argument per task selection axis @@ -2306,6 +2300,8 @@ def __init__( # ruff: ignore[too-many-arguments] - one argument per task select self.dataset_occurrences: list[Annotation] = [] """The occurrences of the dataset itself that the scope read for its tasks' references.""" self.retained: _TasksById = {} + self._label_sets: dict[int, LabelSet] | None = None + """The stored label sets by id, which the read builds once and all its tasks share.""" # The lookups of the caller's objects that a stream's batches share, or None for one pass. self._given: _Lookups | None = None self._dataset_annotations_indexed = dataset_annotations is not None @@ -2326,8 +2322,8 @@ def iter_batches(self) -> Iterator[list[Task]]: """Yield the selected tasks in task ID order, one bounded batch at a time, each in its own scope. The tasks of a batch share the objects the batch builds, and no batch shares them with - another. The builder forgets a batch's objects before it yields them, so it holds nothing - of a batch it has handed out. + another, apart from the label sets. The builder forgets a batch's other objects before it + yields them. Yields: Each batch's built tasks. @@ -2454,6 +2450,27 @@ def _build_batch(self, task_ids: list[EntityId] | None) -> list[Task]: self.retained[task_id] = task return [self._task(task_id, built) for task_id in task_ids] + def _label_set(self, task_id: int, label_set_id: int | None) -> LabelSet: + """Return the stored label set that a classification task names. + + The read builds every label set once. Tasks that use the same set share one object. + + Returns: + The shared label set. + + Raises: + TimeFFormatError: If the task names no label set, no label set has this id, or a stored + label set is not valid. + """ + if label_set_id is None: + raise TimeFFormatError(f"classification task {task_id!r} has no label set") + if self._label_sets is None: + self._label_sets = self.reader._read_label_sets() + label_set = self._label_sets.get(label_set_id) + if label_set is None: + raise TimeFFormatError(f"classification task {task_id!r} refers to missing label set {label_set_id!r}") + return label_set + def _task(self, task_id: EntityId, built: _TasksById) -> Task: """Return a task from this batch or the retained parents. @@ -2556,8 +2573,8 @@ def _build(self, task_ids: list[EntityId] | None) -> _TasksById: **decode_task_payload(task_type, row.payload_columns), } if task_type is TaskType.CLASSIFICATION: - kwargs["label_set"] = reader._label_set(row.task_id, row.label_set) - elif row.label_set is not None: + kwargs["label_set"] = self._label_set(row.task_id, row.label_set_id) + elif row.label_set_id is not None: raise TimeFFormatError( f"{task_type} task {row.task_id!r} has a label set, but only a classification task can have one" ) diff --git a/packages/timenet/src/timenet/format/control_schema.py b/packages/timenet/src/timenet/format/control_schema.py index d4b417fe2..da9cb00b5 100644 --- a/packages/timenet/src/timenet/format/control_schema.py +++ b/packages/timenet/src/timenet/format/control_schema.py @@ -3,11 +3,9 @@ The SQL table definitions and the writer's Arrow batch schemas come from :data:`CONTROL_TABLES`. The audit also uses it to examine keys and references. Define schema changes there. -This is version 2 of the control schema. Every table except ``label_sets`` uses ids as keys. -Records, Signals, and Tasks use the public ids that the caller sees. All other objects use the -writer's storage ids. - -Each label set has a unique name in a dataset. The ``label_sets`` table uses this name as its key. A foreign key stores the key of the row that it refers to. +This is version 2 of the control schema. Every table uses ids as keys. +Records, Signals, and Tasks use the public ids that the caller sees. All other objects, label sets +included, use the writer's storage ids. A foreign key stores the key of the row that it refers to. Fixed-shape fields are typed columns, so SQL can filter them. Only the free-form ``metadata`` dictionaries are JSON. The database has no persisted constraints. The writer keeps @@ -124,6 +122,7 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column ObjectKind.SOURCE: ("sources", "source_id"), ObjectKind.SIGNAL: ("signals", "signal_id"), ObjectKind.TASK: ("tasks", "task_id"), + ObjectKind.LABEL_SET: ("label_sets", "label_set_id"), } """The owner table and key column of each annotated object type. The Dataset has no owner row.""" @@ -281,24 +280,24 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column unique=(("occurrence_id",),), foreign_keys=(ForeignKey("content_id", "annotation_contents", "content_id"),), ) -"""The owner is ``(object_type, object_id)``. ``object_id`` is the public id of a Record, Signal, -or Task, or the storage id of a Source. Ids are unique only within one kind, so the type is part of -the reference. ``object_id`` is NULL exactly when the owner is the Dataset.""" +"""The owner is ``(object_type, object_id)``. ``object_id`` is the key of the owner row in the table +that :data:`ANNOTATED_OWNER_TABLES` names for its type. Ids are unique only within one kind, so the type +is part of the reference. ``object_id`` is NULL exactly when the owner is the Dataset.""" LABEL_SETS = Table( "label_sets", ( - _required("name"), + _key("label_set_id"), _required("labels", ColumnType.LABEL_LIST), _required("multi_label", ColumnType.BOOLEAN), ), - unique=(("name",),), + unique=(("label_set_id",),), ) """Each row stores one label set that a classification task uses. The ``labels`` column holds one ``{name, description}`` entry per label, in label order. -If a label has no description, ``description`` is NULL. Tasks refer to a set by its name in -``tasks.label_set``.""" +If a label has no description, ``description`` is NULL. Tasks refer to a set by its id in +``tasks.label_set_id``. The writer gives each distinct set one id, in the order that tasks first use it.""" TASKS = Table( "tasks", @@ -313,7 +312,7 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column Column("scope_end", ColumnType.BIGINT), Column("scope_signal_ids", ColumnType.BIGINT_LIST), _required("has_inline_targets", ColumnType.BOOLEAN), - Column("label_set", ColumnType.VARCHAR), + Column("label_set_id", ColumnType.BIGINT), Column("unit", ColumnType.VARCHAR), Column("target_name", ColumnType.VARCHAR), Column("mode", ColumnType.VARCHAR), @@ -323,7 +322,7 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column unique=(("task_id",),), foreign_keys=( ForeignKey("scope_signal_ids", "signals", "signal_id", many=True), - ForeignKey("label_set", "label_sets", "name"), + ForeignKey("label_set_id", "label_sets", "label_set_id"), ), ) diff --git a/packages/timenet/src/timenet/format/control_writer.py b/packages/timenet/src/timenet/format/control_writer.py index 51d05a10a..c2f6158ab 100644 --- a/packages/timenet/src/timenet/format/control_writer.py +++ b/packages/timenet/src/timenet/format/control_writer.py @@ -8,7 +8,7 @@ import json from pathlib import Path from typing import TYPE_CHECKING, Any, NamedTuple, assert_never, cast -from weakref import WeakValueDictionary +from weakref import WeakValueDictionary, ref import duckdb import numpy as np @@ -39,7 +39,8 @@ from timenet.types import ( Annotation, ArrayHandle, - LabelSetRegistry, + ClassificationTask, + LabelSet, ObjectKind, Task, TSCorrespondenceTask, @@ -207,6 +208,89 @@ def flush(self) -> None: self.signals.flush() +class _LabelSetRows: + """The ``label_sets`` rows of one write, one row for each distinct label set. + + Equal sets (see :meth:`LabelSet._content_key`) get one id, in the order that tasks first use them. + The row keeps the labels of the first set. The stored annotations are all the occurrences of the + first set, then each occurrence of a later equal set that differs from every stored one in a field. + A set object that many tasks share is checked and added once. The caller's sets do not change. + """ + + def __init__(self, ids: Iterator[int]) -> None: + """Start with no label sets. + + Args: + ids: The storage ids for new label sets. + """ + self._new_ids = ids + self._sets: dict[Hashable, tuple[dict[str, object], list[Annotation], set[Hashable]]] = {} + self._known: dict[int, tuple[ref[LabelSet], int]] = {} + + def register(self, label_set: LabelSet) -> int: + """Add a label set and its annotations. + + Returns: + The stored id of the set. + + Raises: + TimeFValidationError: If the set is not valid. A set can change after a task gets it. + """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it + known = self._known.get(id(label_set)) + if known is not None and known[0]() is label_set: + return known[1] + label_set._validate() + key = label_set._content_key() + stored = self._sets.get(key) + if stored is None: + row: dict[str, object] = { + "label_set_id": next(self._new_ids), + "labels": [label.model_dump() for label in label_set.labels], + "multi_label": label_set.multi_label, + } + occurrences = list(label_set.annotations) + self._sets[key] = (row, occurrences, {_occurrence_key(annotation) for annotation in occurrences}) + else: + row, occurrences, held = stored + for annotation in label_set.annotations: + occurrence = _occurrence_key(annotation) + if occurrence not in held: + held.add(occurrence) + occurrences.append(annotation) + label_set_id = cast("int", row["label_set_id"]) + object_id = id(label_set) + self._known[object_id] = (ref(label_set, lambda _: self._known.pop(object_id, None)), label_set_id) + return label_set_id + + def write( + self, connection: duckdb.DuckDBPyConnection, annotations: list[tuple[ObjectKind, int | None, Annotation]] + ) -> None: + """Insert the rows and queue the annotations of each set. + + Args: + connection: The open write transaction. + annotations: The annotation triples that the writer collects. This method adds to them. + """ + insert_rows(connection, TABLES["label_sets"], [row for row, _, _ in self._sets.values()]) + for row, occurrences, _ in self._sets.values(): + label_set_id = cast("int", row["label_set_id"]) + annotations.extend((ObjectKind.LABEL_SET, label_set_id, annotation) for annotation in occurrences) + + +def _occurrence_key(annotation: Annotation) -> tuple[Hashable, ...]: + """Return a key that is equal for two occurrences only if every field of them is equal. + + Returns: + The hashable key. + """ + return ( + annotation._content_key, + annotation.source, + annotation.confidence, + json.dumps(annotation.occurrence_metadata, sort_keys=True), + ) + + class DuckDBControlWriter: """Serialize the relational part of one immutable TimeF version.""" @@ -493,14 +577,13 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume TimeFValidationError: If a task refers to a Record or Signal outside the dataset. Also if the dataset registered a different Record or Signal object under that id. Also if two tasks share an id, or a task derives from itself or a task outside the dataset. - Also if two tasks use different label sets with the same name. """ # Weak: if no other task names a streamed task, Python can free it after the writer writes it. written: WeakValueDictionary[int, Task] = WeakValueDictionary() dependencies: list[tuple[int, int, Task]] = [] annotation_refs: list[_AnnotationRef] = [] task_counts: dict[str, int] = {} - label_sets = LabelSetRegistry() + label_sets = _LabelSetRows(self._storage_ids["label_set"]) index = _TaskValidationIndex(dataset) for batch in _task_batches(tasks_source): task_rows: list[dict[str, object]] = [] @@ -511,7 +594,6 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume TimeFDataset._check_signal_refs(task, index) # A task can change after add_task, so check its labels and its label set again. task.check_targets() - task.register_label_set(label_sets) task_id = task.id written[task_id] = task task_type = str(task.task_type) @@ -526,6 +608,9 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume **self._scope_columns(task, signal_ids), "has_inline_targets": task.targets is not None, **encode_task_payload(task), + "label_set_id": ( + label_sets.register(task.label_set) if isinstance(task, ClassificationTask) else None + ), "rationale": self._pack(ObjectKind.TASK, task_id, PackedField.RATIONALE, task.rationale), "metadata": _json(task.metadata), } @@ -561,8 +646,8 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume insert_rows(connection, TABLES["task_record_refs"], record_ref_rows) # Store this batch's locators now, so a streamed task source never holds them all. self.external.write_locators(connection) - # Tasks refer to a label set by its name, so the sets can follow the tasks that use them. - insert_rows(connection, TABLES["label_sets"], [label_set.model_dump() for label_set in label_sets]) + # Tasks refer to a label set by its id, so the sets can follow the tasks that use them. + label_sets.write(connection, annotations) duplicate = connection.execute( "SELECT task_id FROM tasks GROUP BY task_id HAVING count(*) > 1 LIMIT 1" ).fetchone() diff --git a/packages/timenet/src/timenet/format/task_codec.py b/packages/timenet/src/timenet/format/task_codec.py index 400335be1..97f11aa30 100644 --- a/packages/timenet/src/timenet/format/task_codec.py +++ b/packages/timenet/src/timenet/format/task_codec.py @@ -20,7 +20,6 @@ from timenet.format.task_target_kind import SPAN_KINDS, TaskTargetKind from timenet.format.task_target_row import TargetValueRow from timenet.types import ( - ClassificationTask, EntityId, InputModality, LocalizationMode, @@ -57,11 +56,11 @@ class TaskRelationshipField(StrEnum): """The fields stored in ``task_annotation_refs``.""" -PAYLOAD_COLUMNS = ("label_set", "unit", "target_name", "mode") +PAYLOAD_COLUMNS = ("unit", "target_name", "mode") """The ``tasks`` columns that hold configuration for each task type. -Unused columns are NULL. A classification task stores its label set name in ``label_set``. -The ``label_sets`` table stores the set itself.""" +Unused columns are NULL. The writer sets the ``label_set_id`` column of a classification task itself. +The ``label_sets`` table stores the set.""" def decode_input_modalities(values: list[str] | None) -> frozenset[InputModality]: @@ -182,9 +181,7 @@ def encode_task_payload(task: Task) -> dict[str, str | None]: One value per :data:`PAYLOAD_COLUMNS`, ``None`` where the task type has no such field. """ columns: dict[str, str | None] = dict.fromkeys(PAYLOAD_COLUMNS) - if isinstance(task, ClassificationTask): - columns["label_set"] = task.label_set.name - elif isinstance(task, ScalarPredictionTask): + if isinstance(task, ScalarPredictionTask): columns["unit"] = None if task.unit is None else str(task.unit) columns["target_name"] = task.target_name elif isinstance(task, TemporalLocalizationTask): diff --git a/packages/timenet/src/timenet/reader/load_plan.py b/packages/timenet/src/timenet/reader/load_plan.py index 3b2e8db01..4ff3fe32d 100644 --- a/packages/timenet/src/timenet/reader/load_plan.py +++ b/packages/timenet/src/timenet/reader/load_plan.py @@ -17,7 +17,7 @@ from timenet.dataset import Record, Signal, Source from timenet.format.stored_array import ChunkLocation, StoredArray from timenet.format.stored_values import PackedLocation, StoredOffsets, StoredText, StoredTextList -from timenet.types import Annotation, Task +from timenet.types import Annotation, LabelSet, Task from timenet.types.load_plan import Loading, LoadPlan @@ -60,14 +60,21 @@ def _annotation_children(item: Annotation) -> tuple[object, ...]: return (item.value,) +@_load_plan_children.register +def _label_set_children(item: LabelSet) -> tuple[object, ...]: + """Return a label set's annotations.""" + return item.annotations + + @_load_plan_children.register def _task_children(item: Task) -> tuple[object, ...]: - """Return a task's text, annotations, input and candidate records, targets, and the annotations it references.""" + """Return a task's text, annotations, owned objects, records, targets, and the annotations it references.""" return ( item.prompt, item.rationale, *(item.targets or ()), *item.annotations, + *(owner for owner in item.annotation_owners() if owner is not item), *item.inputs, *getattr(item, "candidate_records", ()), *item.input_annotations, diff --git a/packages/timenet/src/timenet/testing/builders.py b/packages/timenet/src/timenet/testing/builders.py index 2a9c81cdb..d76164c07 100644 --- a/packages/timenet/src/timenet/testing/builders.py +++ b/packages/timenet/src/timenet/testing/builders.py @@ -16,7 +16,6 @@ DatasetMetadata, Domain, EntityId, - Label, LabelSet, License, LocalizationMode, @@ -31,9 +30,9 @@ ) -_RHYTHM_LABELS = LabelSet(name="rhythm", labels=(Label(name="normal"), Label(name="abnormal"))) +_RHYTHM_LABELS = LabelSet(["normal", "abnormal"]) """The label set of the fixture's rhythm classification task.""" -_EVENT_LABELS = LabelSet(name="event", labels=(Label(name="onset"), Label(name="offset"))) +_EVENT_LABELS = LabelSet(["onset", "offset"]) """The label set of the fixture's scoped event classification task.""" diff --git a/packages/timenet/src/timenet/types/__init__.py b/packages/timenet/src/timenet/types/__init__.py index 7fcface81..343bf76d6 100644 --- a/packages/timenet/src/timenet/types/__init__.py +++ b/packages/timenet/src/timenet/types/__init__.py @@ -13,7 +13,7 @@ from timenet.types.domains import Domain from timenet.types.handles import ArrayHandle, ValueHandle from timenet.types.ids import EntityId, check_id -from timenet.types.label_sets import Label, LabelSet, LabelSetRegistry +from timenet.types.label_sets import Label, LabelSet from timenet.types.licenses import License from timenet.types.metadata import DatasetFilter, DatasetMetadata, DatasetSchema, ListedRelease from timenet.types.modalities import InputModality @@ -68,7 +68,6 @@ "InputModality", "Label", "LabelSet", - "LabelSetRegistry", "LazyText", "LazyTextList", "License", diff --git a/packages/timenet/src/timenet/types/annotations.py b/packages/timenet/src/timenet/types/annotations.py index 417540a5a..fece3bd8c 100644 --- a/packages/timenet/src/timenet/types/annotations.py +++ b/packages/timenet/src/timenet/types/annotations.py @@ -13,6 +13,7 @@ class synthesis. :class:`AnnotationDescriptor` is the type-level projection stor """ from collections.abc import Hashable +from copy import deepcopy from dataclasses import KW_ONLY, dataclass, field, replace from enum import StrEnum, unique import json @@ -184,6 +185,23 @@ def _new_occurrence(self) -> "Annotation": return replace(self) +def _copy_annotation(original: Annotation) -> Annotation: + """Copy an annotation occurrence as a value, with the handle of a stored value. + + The copy has deep copies of the value, the metadata, and the occurrence metadata. So a change to + one of them in the copy leaves the original as it is. Imports and edits copy annotations this way. + + Returns: + The copy. + """ + return replace( + original, + value=deepcopy(original.value), + metadata=deepcopy(original.metadata), + occurrence_metadata=deepcopy(original.occurrence_metadata), + ) + + def annotation_type_of(annotation: Annotation) -> AnnotationType: """Return the :class:`AnnotationType` shape of an annotation instance. @@ -203,8 +221,8 @@ class SupportsAnnotate(Protocol): ``annotate`` binds a fresh occurrence of ``annotation`` to this object and returns it; the returned occurrence is what a Task names in ``input_annotations`` or ``target_annotations``. - ``annotations`` lists every occurrence attached so far. Records, Sources, Signals, Tasks, and - the dataset itself implement it. A :class:`~timenet.dataset.SignalSelection` does not: it fans + ``annotations`` lists every occurrence attached so far. Records, Sources, Signals, Tasks, label + sets, and the dataset itself implement it. A :class:`~timenet.dataset.SignalSelection` does not: it fans one annotation out over several Signals and returns every occurrence it created. The contract is checked statically; there is no runtime ``isinstance`` support because ``annotations`` is a property. diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index 0a33f469d..bbddea3e6 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -1,12 +1,12 @@ """The allowed labels for a classification target.""" -from collections.abc import Iterator, Sequence -from typing import Self - -from pydantic import StrictBool, model_validator +from collections.abc import Hashable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import TypeVar from timenet.errors import TimeFValidationError from timenet.types._model import TimeFModel +from timenet.types.annotations import Annotation, SupportsAnnotate from timenet.types.wire import NonEmptyString @@ -19,7 +19,43 @@ class Label(TimeFModel): """What the label means, such as ``The subject is awake.``.""" -class LabelSet(TimeFModel): +_Item = TypeVar("_Item") + + +def _as_tuple(value: object, item_type: type[_Item]) -> tuple[_Item, ...]: + """Return the items of a label set field as a tuple. + + Returns: + The items. :meth:`LabelSet._validate` checks their type. + + Raises: + TimeFValidationError: If the value is one item, a string, a mapping, or not iterable. + """ + if isinstance(value, (item_type, str, Mapping)) or not isinstance(value, Iterable): + raise TimeFValidationError( + f"a label set takes a list of {item_type.__name__} items, got {type(value).__name__} {value!r}" + ) + return tuple(value) + + +def _as_label(item: object) -> object: + """Turn a plain string into a label without a description. Leave every other item as it is. + + Returns: + The label, or the item. :meth:`LabelSet._validate` checks its type. + + Raises: + TimeFValidationError: If the string is empty. + """ + if not isinstance(item, str): + return item + if not item: + raise TimeFValidationError("a label set cannot have an empty label") + return Label(name=item) + + +@dataclass(init=False, eq=False) +class LabelSet(SupportsAnnotate): """The allowed labels for a classification target. A target must contain one label unless ``multi_label=True``. @@ -27,41 +63,128 @@ class LabelSet(TimeFModel): Tasks that ask the same kind of question share one label set:: SLEEP_STAGE = LabelSet( - name="sleep_stage", - labels=( - Label(name="awake", description="The subject is awake."), - Label(name="non_rem_sleep"), - Label(name="rem_sleep"), - ), + [Label(name="awake", description="The subject is awake."), "non_rem_sleep", "rem_sleep"], + annotations=(Annotation(key="name", value="sleep_stage"),), ) ClassificationTask(inputs=(record,), targets=("awake",), label_set=SLEEP_STAGE) - Each name identifies one label set in a dataset. + A plain string is a label without a description, so ``LabelSet(["yes", "no"])`` is enough when no label + needs one. + + A label set has no name. Two sets with the same labels (in any order, with the same descriptions) + and the same ``multi_label`` are the same set, so ``==`` is true for them. A dataset stores them once, + with the label order of the first of them that it writes. Annotations are not part of that identity. """ - name: NonEmptyString - """The name of the set within a dataset, such as ``sleep_stage``.""" labels: tuple[Label, ...] - """The allowed labels in a fixed order.""" - multi_label: StrictBool = False + """The allowed labels in order. A dataset keeps the order of the first equal set that it writes.""" + multi_label: bool = False """If ``False``, a target holds exactly one label from the set. If ``True``, a target holds all labels from the set that apply. An empty target means that no label applies.""" + annotations: tuple[Annotation, ...] = field(default=(), compare=False) + """Annotations that describe the set, such as ``Annotation(key="name", value="sleep_stage")``. + + An annotation cannot have a span. The set holds its own occurrence of each annotation. + When a dataset holds several equal sets, it stores each distinct annotation occurrence of them once.""" - @model_validator(mode="after") - def _valid_set(self) -> Self: + def __init__( + self, + labels: Iterable[Label | str], + *, + multi_label: bool = False, + annotations: Iterable[Annotation] = (), + ) -> None: + """Build a label set, make sure that it is valid, and attach the annotations. + + Args: + labels: The allowed labels in order. A plain string is a label without a description. + multi_label: Whether a target can hold more than one label. + annotations: Annotations that describe the set. + + Raises: + TimeFValidationError: If the set is not valid (see :meth:`_validate`). + """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it + self.labels = tuple(_as_label(label) for label in _as_tuple(labels, Label)) # ty: ignore[invalid-assignment] + self.multi_label = multi_label + self.annotations = _as_tuple(annotations, Annotation) + self._validate() + self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) + + def _validate(self) -> None: + """Make sure that the set is valid. The writer calls this again, because a set can change after it is built. + + Raises: + TimeFValidationError: If a label is not a Label, the set has no label or repeats a name, + ``multi_label`` is not a bool, or an annotation is not an Annotation or has a span. + """ + for label in self.labels: + if not isinstance(label, Label): + raise TimeFValidationError(f"a label set takes Label objects, got {type(label).__name__}") if not self.labels: - raise ValueError(f"label set {self.name!r} must list at least one label") + raise TimeFValidationError("a label set must list at least one label") if len(set(self.names)) != len(self.labels): - raise ValueError(f"label set {self.name!r} lists a label more than once") - return self + raise TimeFValidationError(f"{self} lists a label more than once") + if not isinstance(self.multi_label, bool): + raise TimeFValidationError(f"{self} multi_label must be a bool, got {type(self.multi_label).__name__}") + for annotation in self.annotations: + self._check_annotation(annotation) + + def _check_annotation(self, annotation: object) -> None: + """Make sure that an annotation can describe the set. + + Raises: + TimeFValidationError: If the annotation is not an Annotation or has a span. + """ + if not isinstance(annotation, Annotation): + raise TimeFValidationError(f"{self} takes Annotation objects, got {type(annotation).__name__}") + if annotation.span is not None: + raise TimeFValidationError( + f"annotation {annotation.key!r} has a span, but {self} takes annotations without a span" + ) + + def annotate(self, annotation: Annotation) -> Annotation: + """Attach one annotation occurrence and return it. + + Returns: + The attached occurrence. + + Raises: + TimeFValidationError: If the annotation has a span. A label set has no timeline. + """ # ruff: ignore[docstring-extraneous-exception] - _check_annotation raises it + self._check_annotation(annotation) + attached = annotation._new_occurrence() + self.annotations = (*self.annotations, attached) + return attached @property def names(self) -> tuple[str, ...]: """The name of each label, in label order.""" return tuple(label.name for label in self.labels) + def __str__(self) -> str: + """Return the labels of the set, for error messages.""" + return f"label set {list(self.names)}" + + def __eq__(self, other: object) -> bool: + """Return whether the sets have the same labels, in any order, and the same ``multi_label``.""" + if not isinstance(other, LabelSet): + return NotImplemented + return self._content_key() == other._content_key() + + __hash__ = None + + def _content_key(self) -> tuple[Hashable, ...]: + """Return the key that makes two sets equal. + + The key ignores label order and annotations. + + Returns: + The hashable key. + """ + return (self.multi_label, frozenset((label.name, label.description) for label in self.labels)) + def check_targets(self, targets: Sequence[str]) -> None: """Make sure that the target labels follow the rules of this set. @@ -82,60 +205,9 @@ def check_targets(self, targets: Sequence[str]) -> None: names = set(self.names) unknown = sorted({label for label in targets if label not in names}) if unknown: - raise TimeFValidationError(f"labels not in label set {self.name!r}: {unknown}") + raise TimeFValidationError(f"labels not in {self}: {unknown}") if not self.multi_label and len(targets) != 1: - raise TimeFValidationError(f"{len(targets)} labels, but label set {self.name!r} takes exactly one") + raise TimeFValidationError(f"{len(targets)} labels, but {self} takes exactly one") repeated = sorted({label for label in targets if targets.count(label) > 1}) if repeated: - raise TimeFValidationError(f"repeated labels of label set {self.name!r}: {repeated}") - - -class LabelSetRegistry: - """The label sets of a dataset, with one set per name. - - The sets stay in the order in which tasks first added them. - """ - - def __init__(self) -> None: - """Start with no label sets.""" - self._by_name: dict[str, LabelSet] = {} - - def register_or_reuse_matching_label_set(self, label_set: LabelSet) -> LabelSet: - """Store a label set by name. - - If the registry already holds an equal set with this name, return that set. - - Returns: - The set that the registry holds under the name. - - Raises: - TimeFValidationError: If a different set already has the same name. - """ - known = self._by_name.setdefault(label_set.name, label_set) - if known is not label_set and known != label_set: - raise TimeFValidationError( - f"label set {label_set.name!r} differs from another set with that name. " - "A name stands for one label set in a dataset" - ) - return known - - def copy(self) -> "LabelSetRegistry": - """Return a separate registry with the same label sets. - - The dataset uses the copy to check a batch before adding its tasks. - If a task fails, the dataset discards the copy and keeps its original registry. - - Returns: - The copy. - """ - copy = LabelSetRegistry() - copy._by_name = dict(self._by_name) - return copy - - def __iter__(self) -> Iterator[LabelSet]: - """Iterate over the sets in the order in which tasks first added them. - - Returns: - The iterator. - """ - return iter(self._by_name.values()) + raise TimeFValidationError(f"repeated labels of {self}: {repeated}") diff --git a/packages/timenet/src/timenet/types/object_kind.py b/packages/timenet/src/timenet/types/object_kind.py index 03a975777..19b3cb11c 100644 --- a/packages/timenet/src/timenet/types/object_kind.py +++ b/packages/timenet/src/timenet/types/object_kind.py @@ -12,6 +12,7 @@ class ObjectKind(StrEnum): SOURCE = "Source" SIGNAL = "Signal" TASK = "Task" + LABEL_SET = "LabelSet" ANNOTATION = "Annotation" ANNOTATION_CONTENT = "Content" AXIS = "Axis" diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 4d1754cdb..e22b5123f 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -1,6 +1,7 @@ """Declarative modelling tasks over records and signals.""" from collections.abc import Iterable, Iterator, Mapping +from copy import copy as shallow_copy from dataclasses import dataclass, field, replace from enum import StrEnum, unique from functools import lru_cache @@ -11,9 +12,9 @@ from timenet.errors import TimeFValidationError from timenet.json import JsonMapping, validate_json_mapping from timenet.types._fields import TupleField, ValueField, as_tuple -from timenet.types.annotations import Annotation, SupportsAnnotate, value_type_of +from timenet.types.annotations import Annotation, SupportsAnnotate, _copy_annotation, value_type_of from timenet.types.ids import EntityId, assigned_id -from timenet.types.label_sets import LabelSet, LabelSetRegistry +from timenet.types.label_sets import LabelSet from timenet.types.modalities import InputModality from timenet.types.spans import Span, StepSpan, TimeSpan from timenet.types.splits import Split, parse_split @@ -360,15 +361,6 @@ def check_against_scope(self) -> None: def check_targets(self) -> None: """Make sure that the final targets follow the rules of this task class.""" - def register_label_set(self, label_sets: LabelSetRegistry) -> None: - """Add the label set of the task to the dataset's label sets. - - Only a classification task has a label set. - - Args: - label_sets: The label sets of the dataset so far. - """ - @final @dataclass(kw_only=True) @@ -408,8 +400,7 @@ def check_targets(self) -> None: for value in values: if not isinstance(value, TextHandle): raise TimeFValidationError( - f"{name} has label set {self.label_set.name!r}, so each label must be text, " - f"not {type(value).__name__}" + f"{name} has {self.label_set}, so each label must be text, not {type(value).__name__}" ) labels.append(value.read()) try: @@ -417,19 +408,32 @@ def check_targets(self) -> None: except TimeFValidationError as exc: raise TimeFValidationError(f"{name}: {exc}") from exc - def register_label_set(self, label_sets: LabelSetRegistry) -> None: - """Add the label set of the task to the dataset's label sets. + def annotation_owners(self) -> Iterator[SupportsAnnotate]: + """Yield the task, then its label set. + + Yields: + The task and its label set. + """ + yield self + yield self.label_set + + def with_references(self, records: Mapping[EntityId, "Record"], signals: Mapping[EntityId, "Signal"]) -> Self: + """Return a copy of the task that references the given Records and Signals, by ID. + + See :meth:`Task.with_references`. The copy also gets its own copy of the label set and of its + annotations, so a change to the set of the copy leaves the set of the task as it is. Args: - label_sets: The label sets of the dataset so far. + records: The Records for the copy, by ID. + signals: The Signals for the copy, by ID. - Raises: - TimeFValidationError: If a different set already has the same name. + Returns: + The copy. """ - try: - label_sets.register_or_reuse_matching_label_set(self.label_set) - except TimeFValidationError as exc: - raise TimeFValidationError(f"ClassificationTask {self.id!r}: {exc}") from exc + copy = super().with_references(records, signals) + copy.label_set = shallow_copy(self.label_set) + copy.label_set.annotations = tuple(_copy_annotation(annotation) for annotation in self.label_set.annotations) + return copy @final diff --git a/packages/timenet/tests/dataset/test_composition.py b/packages/timenet/tests/dataset/test_composition.py index 9c541d8ca..80a9b17e2 100644 --- a/packages/timenet/tests/dataset/test_composition.py +++ b/packages/timenet/tests/dataset/test_composition.py @@ -58,12 +58,12 @@ from timenet.writer import TimeFWriter -ORDER = LabelSet(name="order", labels=(Label(name="late"),)) +ORDER = LabelSet(labels=(Label(name="late"),)) _ALIAS = ParentAlias("base") _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) -SCP = LabelSet(name="scp", labels=(Label(name="normal", description="No finding."), Label(name="abnormal"))) +SCP = LabelSet(labels=(Label(name="normal", description="No finding."), Label(name="abnormal"))) def store(root, dataset): diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 7c2f24426..99740ef3d 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -1,3 +1,4 @@ +from dataclasses import replace from typing import Any import numpy as np @@ -34,7 +35,6 @@ TEST_LABELS = LabelSet( - name="labels", labels=( Label(name="N2"), Label(name="a"), @@ -179,26 +179,45 @@ def test_derive_schema(make_series): assert ds.schema is schema -def test_derive_schema_collects_dataset_and_task_annotations(make_series): +def test_derive_schema_collects_label_set_annotations(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) - record.annotate(Annotation(key="age", value=64)) - ds.annotate(Annotation(key="cohort", value="adult")) - task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - task.annotate(Annotation(key="difficulty", value=2)) + labels = (Label(name="a"), Label(name="b")) + ds.add_task( + task=ClassificationTask( + inputs=(record,), + targets=("a",), + label_set=LabelSet(labels=labels, annotations=(Annotation(key="name", value="letters", unit="meter"),)), + ) + ) + ds.add_task( + task=ClassificationTask( + inputs=(record,), + targets=("b",), + label_set=LabelSet(labels=labels, annotations=(Annotation(key="note", value="extra"),)), + ) + ) schema = ds.derive_schema() - assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"] + assert [descriptor.key for descriptor in schema.annotations] == ["name", "note"] + assert schema.annotations[0] == AnnotationDescriptor( + key="name", annotation_type=AnnotationType.STATIC, value_type="str", unit="meter" + ) -def test_derive_schema_rejects_a_task_annotation_that_conflicts_with_a_record_annotation(make_series): +def test_derive_schema_rejects_a_label_set_annotation_that_conflicts_with_a_record_annotation(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) record.annotate(Annotation(key="name", value=1)) - task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - task.annotate(Annotation(key="name", value="letters")) - with pytest.raises(TimeFValidationError, match="conflicting descriptors"): + ds.add_task( + task=ClassificationTask( + inputs=(record,), + targets=("a",), + label_set=LabelSet(labels=(Label(name="a"),), annotations=(Annotation(key="name", value="letters"),)), + ) + ) + with pytest.raises(ValueError, match="conflicting descriptors"): ds.derive_schema() @@ -206,7 +225,8 @@ def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) record.annotate(Annotation(key="age", value=64)) - task = _classification_task(record, "a") + label_set = LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)) + task = ClassificationTask(inputs=(record,), targets=("a",), label_set=label_set) task.annotate(Annotation(key="difficulty", value=2)) ds.set_task_stream([ClassificationTask], lambda: iter((task,))) assert [descriptor.key for descriptor in ds.derive_schema().annotations] == ["age"] @@ -214,15 +234,15 @@ def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series): list(ds.iter_streamed_tasks_validated()) assert ds.schema is not None - assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty"] + assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty", "name"] def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) record.annotate(Annotation(key="name", value=1)) - task = _classification_task(record, "a") - task.annotate(Annotation(key="name", value="letters")) + label_set = LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)) + task = ClassificationTask(inputs=(record,), targets=("a",), label_set=label_set) ds.set_task_stream([ClassificationTask], lambda: iter((task,))) ds.derive_schema() @@ -230,6 +250,19 @@ def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema( list(ds.iter_streamed_tasks_validated()) +def test_derive_schema_collects_dataset_and_task_annotations(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="age", value=64)) + ds.annotate(Annotation(key="cohort", value="adult")) + task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + task.annotate(Annotation(key="difficulty", value=2)) + + schema = ds.derive_schema() + + assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"] + + def test_derive_schema_dedupes_specs(make_series): ds = _dataset() # Two signals of the same modality => one distinct spec. @@ -888,10 +921,13 @@ def test_walk_annotations_goes_from_the_dataset_to_records_to_tasks(make_series) record = ds.add_record(record=Record(sources=(source,))) task = ds.add_task(task=_classification_task(record, "a")) answer = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - for owner, key in ((answer, "answer"), (task, "task"), (signal, "signal"), (source, "source"), (record, "record")): + task.label_set = replace(task.label_set) + owners = ((answer, "answer"), (task.label_set, "label_set"), (task, "task"), (signal, "signal")) + for owner, key in (*owners, (source, "source"), (record, "record")): owner.annotate(Annotation(key=key, value=key)) ds.annotate(Annotation(key="dataset", value="dataset")) - assert [a.key for a in ds.walk_annotations()] == ["dataset", "record", "source", "signal", "task", "answer"] + keys = [a.key for a in ds.walk_annotations()] + assert keys == ["dataset", "record", "source", "signal", "task", "label_set", "answer"] def test_add_tasks_allows_deriving_from_another_task_in_the_same_batch(make_series): diff --git a/packages/timenet/tests/format/test_control_writer.py b/packages/timenet/tests/format/test_control_writer.py index 764024477..6c5902e6e 100644 --- a/packages/timenet/tests/format/test_control_writer.py +++ b/packages/timenet/tests/format/test_control_writer.py @@ -99,7 +99,7 @@ def test_control_writer_serializes_recursive_hierarchy_and_shared_axis(tmp_path, assert connection.execute("SELECT count(*) FROM signals").fetchone() == (2,) assert connection.execute("SELECT count(*) FROM axes").fetchone() == (1,) assert connection.execute( - "SELECT task_type, prompt, scope_type, label_set, unit, target_name, mode FROM tasks" + "SELECT task_type, prompt, scope_type, label_set_id, unit, target_name, mode FROM tasks" ).fetchone() == ("answer", "Alive?", None, None, None, None, None) assert connection.execute( "SELECT position, target_kind, text_value, integer_value, record_id FROM task_targets" diff --git a/packages/timenet/tests/format/test_duckdb.py b/packages/timenet/tests/format/test_duckdb.py index 18ef89646..6c05ea44a 100644 --- a/packages/timenet/tests/format/test_duckdb.py +++ b/packages/timenet/tests/format/test_duckdb.py @@ -59,6 +59,7 @@ def test_entity_id_columns_are_bigint_and_dataset_id_stays_varchar(tmp_path): "signal_id", "content_id", "occurrence_id", + "label_set_id", "task_id", } diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index c8ef67ffa..2cf6b068e 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -21,6 +21,7 @@ Label, LabelSet, License, + TimePoint, TimeSeriesSpec, Version, ureg, @@ -29,9 +30,10 @@ SPEC = TimeSeriesSpec(spec_type="voltage", name="Voltage", unit_value=ureg.millivolt) STAGE_LABELS = LabelSet( - name="stage", labels=(Label(name="W"), Label(name="N1"), Label(name="R", description="REM sleep.")) + labels=(Label(name="W"), Label(name="N1"), Label(name="R", description="REM sleep.")), + annotations=(Annotation(key="name", value="stage", description="Sleep stage"),), ) -FINDINGS = LabelSet(name="findings", labels=(Label(name="AF"), Label(name="PVC")), multi_label=True) +FINDINGS = LabelSet(labels=(Label(name="AF"), Label(name="PVC")), multi_label=True) def _record() -> Record: @@ -62,6 +64,10 @@ def _write(dataset: TimeFDataset, path): DuckDBControlWriter(path, external=PackedValueWriter(path.parent, StorageSources())).write_hierarchy(dataset) +def _label_sets(tasks) -> list[LabelSet]: + return [task.label_set for task in tasks if isinstance(task, ClassificationTask)] + + def _read_tasks(path): with DuckDBControlReader(path) as reader: return reader.read_tasks(reader.read_records()) @@ -83,7 +89,7 @@ def release(tmp_path_factory) -> Path: ClassificationTask(inputs=(record,), targets=("PVC", "AF"), label_set=FINDINGS), # 3 ClassificationTask(inputs=(record,), target_annotations=(stage,), label_set=STAGE_LABELS), # 4 # An equal set built separately is the same set. - ClassificationTask(inputs=(record,), targets=("R",), label_set=STAGE_LABELS.model_copy()), # 5 + ClassificationTask(inputs=(record,), targets=("R",), label_set=replace(STAGE_LABELS)), # 5 # A multi-label target with no label says that none applies. ClassificationTask(inputs=(record,), targets=(), label_set=FINDINGS), # 6 ] @@ -107,10 +113,9 @@ def release_copy(release, tmp_path) -> Path: def test_each_label_set_is_stored_once_and_read_as_one_object(release): with connect_control(release, read_only=True) as connection: audit_control_database(connection, require_chunks=False) - assert connection.execute("SELECT name, labels, multi_label FROM label_sets ORDER BY name").fetchall() == [ - ("findings", [{"name": "AF", "description": None}, {"name": "PVC", "description": None}], True), + assert connection.execute("SELECT label_set_id, labels, multi_label FROM label_sets ORDER BY 1").fetchall() == [ ( - "stage", + 1, [ {"name": "W", "description": None}, {"name": "N1", "description": None}, @@ -118,6 +123,7 @@ def test_each_label_set_is_stored_once_and_read_as_one_object(release): ], False, ), + (2, [{"name": "AF", "description": None}, {"name": "PVC", "description": None}], True), ] stage_w, _, findings, stage_n1, stage_r, none_apply = _read_tasks(release) @@ -130,84 +136,170 @@ def test_each_label_set_is_stored_once_and_read_as_one_object(release): assert none_apply.targets == () -def test_the_task_table_shows_the_label_set_name(release): +def test_a_filtered_read_still_loads_the_label_set_annotations(release): with DuckDBControlReader(release) as reader: - assert reader.read_task_table()["label_set"].to_pylist() == [ - "stage", - None, - "findings", - "stage", - "stage", - "findings", - ] + tasks = reader.read_tasks_by_id([4, 5]) + first, second = _label_sets(tasks.values()) + assert first is second + annotation = first.annotation("name") + assert annotation is not None + assert annotation.value == "stage" -OTHER_STAGE_LABELS = LabelSet(name="stage", labels=(Label(name="W"), Label(name="R"))) -SAME_NAME = r"ClassificationTask [0-9]+: label set 'stage' differs from another set" +def test_the_batches_of_a_task_stream_share_one_label_set_object(release): + with DuckDBControlReader(release) as reader: + batches = list(reader.iter_task_batches(batch_size=1)) + + first, fourth = _label_sets(batches[0] + batches[3]) + assert first is fourth -def test_add_task_rejects_a_different_set_with_a_used_name(): - dataset, record = _dataset() - dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS)) - with pytest.raises(TimeFValidationError, match=SAME_NAME): - dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS)) - assert len(dataset.tasks) == 1 +def _stored_sets(path): + with connect_control(path, read_only=True) as connection: + return connection.execute("SELECT label_set_id, labels, multi_label FROM label_sets ORDER BY 1").fetchall() -def test_a_rejected_batch_leaves_no_label_set_behind(): +def _write_sets(tmp_path, *label_sets): dataset, record = _dataset() + dataset.add_tasks( + tasks=[ + ClassificationTask(inputs=(record,), targets=(label_set.labels[0].name,), label_set=label_set) + for label_set in label_sets + ] + ) + path = tmp_path / "control.duckdb" + _write(dataset, path) + return path - with pytest.raises(TimeFValidationError, match=SAME_NAME): - dataset.add_tasks( - tasks=[ - ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), - ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS), - ] - ) - assert dataset.tasks == () - dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS)) +def test_sets_are_stored_once_per_content_and_keep_the_first_label_order(tmp_path): + reordered = LabelSet(labels=tuple(reversed(STAGE_LABELS.labels))) + described = LabelSet(labels=(Label(name="W", description="Awake."), Label(name="N1"), Label(name="R"))) + multi = LabelSet(labels=STAGE_LABELS.labels, multi_label=True) + path = _write_sets(tmp_path, STAGE_LABELS, reordered, described, multi) -def test_a_stream_rejects_a_different_set_with_a_used_name(): - dataset, record = _dataset() - dataset.set_task_stream( - (ClassificationTask,), - lambda: iter( - [ - ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), - ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS), - ] + assert [row[0] for row in _stored_sets(path)] == [1, 2, 3] + first, second, *_ = _read_tasks(path) + assert first.label_set is second.label_set + assert first.label_set.names == ("W", "N1", "R") + + +def test_equal_sets_with_different_annotations_are_stored_once_with_both(tmp_path): + first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) + second = LabelSet( + labels=STAGE_LABELS.labels, + annotations=(Annotation(key="name", value="stage"), Annotation(key="source", value="manual")), + ) + bare = LabelSet(labels=STAGE_LABELS.labels) + path = _write_sets(tmp_path, first, bare, second) + + assert [row[0] for row in _stored_sets(path)] == [1] + with connect_control(path, read_only=True) as connection: + audit_control_database(connection, require_chunks=False) + assert connection.execute( + "SELECT count(*) FROM annotation_occurrences WHERE object_type = 'LabelSet'" + ).fetchone() == (2,) + tasks = _read_tasks(path) + assert tasks[0].label_set is tasks[1].label_set is tasks[2].label_set + assert [(a.key, a.value) for a in tasks[0].label_set.annotations] == [("name", "stage"), ("source", "manual")] + + +def test_occurrences_that_differ_only_in_occurrence_fields_are_all_stored(tmp_path): + first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage", confidence=0.5),)) + first.annotate(Annotation(key="name", value="stage", occurrence_metadata={"k": 1})) + second = LabelSet( + labels=STAGE_LABELS.labels, + annotations=( + Annotation(key="name", value="stage", confidence=0.5), + Annotation(key="name", value="stage", source="rater", confidence=0.9), ), ) + path = _write_sets(tmp_path, first, second) - with pytest.raises(TimeFValidationError, match=SAME_NAME): - list(dataset.iter_streamed_tasks_validated()) + (label_set, _) = _label_sets(_read_tasks(path)) + assert [(a.source, a.confidence, a.occurrence_metadata) for a in label_set.annotations] == [ + (None, 0.5, {}), + (None, None, {"k": 1}), + ("rater", 0.9, {}), + ] + + +def test_writing_does_not_change_the_callers_label_sets(tmp_path): + first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) + second = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="source", value="manual"),)) + first_annotations, second_annotations = first.annotations, second.annotations + + _write_sets(tmp_path, first, second) + + assert first.annotations is first_annotations + assert second.annotations is second_annotations + + +def test_equal_annotations_of_two_sets_share_one_content_row(tmp_path): + path = _write_sets( + tmp_path, + LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)), + LabelSet(labels=FINDINGS.labels, annotations=(Annotation(key="name", value="stage"),)), + ) + + with connect_control(path, read_only=True) as connection: + assert connection.execute("SELECT count(*) FROM annotation_contents").fetchone() == (1,) + assert connection.execute( + "SELECT object_id FROM annotation_occurrences WHERE object_type = 'LabelSet' ORDER BY 1" + ).fetchall() == [(1,), (2,)] + + +def test_the_writer_rejects_a_label_set_changed_after_add_task(tmp_path): + dataset, record = _dataset() + second = ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS) + dataset.add_tasks(tasks=[ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), second]) + second.label_set = LabelSet(labels=(Label(name="X"),)) + + with pytest.raises(TimeFValidationError, match=r"ClassificationTask 2: labels not in label set \['X'\]"): + _write(dataset, tmp_path / "control.duckdb") @pytest.mark.parametrize( - ("changed", "message"), + ("field", "value", "message"), [ - (OTHER_STAGE_LABELS, SAME_NAME), - (LabelSet(name="other", labels=(Label(name="X"),)), "ClassificationTask 2: labels not in label set 'other'"), + ("labels", (Label(name="W"), Label(name="W")), "lists a label more than once"), + ("multi_label", 1, "multi_label must be a bool, got int"), + ("annotations", (Annotation(key="x", span=TimePoint.seconds(1.0)),), "has a span"), ], ) -def test_the_writer_rejects_a_label_set_changed_after_add_task(tmp_path, changed, message): +def test_the_writer_checks_a_label_set_changed_after_it_was_built(tmp_path, field, value, message): + label_set = LabelSet(labels=(Label(name="W"),)) dataset, record = _dataset() - second = ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS) - dataset.add_tasks(tasks=[ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), second]) - second.label_set = changed + dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=label_set)) + setattr(label_set, field, value) with pytest.raises(TimeFValidationError, match=message): _write(dataset, tmp_path / "control.duckdb") +def test_the_reader_rejects_an_annotation_of_a_missing_label_set(release_copy): + with connect_control(release_copy) as connection: + connection.execute("UPDATE annotation_occurrences SET object_id = 99 WHERE object_type = 'LabelSet'") + + with pytest.raises(TimeFFormatError, match="refers to missing LabelSet 99"): + _read_tasks(release_copy) + + +def test_the_reader_rejects_a_task_that_names_a_missing_label_set(release_copy): + with connect_control(release_copy) as connection: + connection.execute("UPDATE tasks SET label_set_id = 99 WHERE task_id = 1") + + with pytest.raises(TimeFFormatError, match="classification task 1 refers to missing label set 99"): + _read_tasks(release_copy) + + @pytest.mark.parametrize( ("task", "message"), [ ( ClassificationTask(targets=("X",), label_set=STAGE_LABELS), - "^ClassificationTask 1: labels not in label set 'stage': \\['X'\\]", + "^ClassificationTask 1: labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]", ), (ClassificationTask(targets=(1,), label_set=STAGE_LABELS), "each label must be text, not int"), ], @@ -226,7 +318,7 @@ def test_add_task_checks_target_annotations_against_the_label_set(): unknown = record.annotate(Annotation(key="stage", value="X")) dataset.add_task(task=ClassificationTask(inputs=(record,), target_annotations=(awake,), label_set=STAGE_LABELS)) - with pytest.raises(TimeFValidationError, match="labels not in label set 'stage': \\['X'\\]"): + with pytest.raises(TimeFValidationError, match="labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]"): dataset.add_task( task=ClassificationTask(inputs=(record,), target_annotations=(unknown,), label_set=STAGE_LABELS) ) @@ -239,22 +331,25 @@ def test_a_stream_rejects_targets_that_do_not_fit_the_label_set(): lambda: iter([ClassificationTask(inputs=(record,), targets=("X",), label_set=STAGE_LABELS)]), ) - with pytest.raises(TimeFValidationError, match="labels not in label set 'stage': \\['X'\\]"): + with pytest.raises(TimeFValidationError, match="labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]"): list(dataset.iter_streamed_tasks_validated()) @pytest.mark.parametrize( ("corruption", "field"), [ - ("UPDATE label_sets SET labels = [] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': 'W'}] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': ''}] WHERE name = 'stage'", "label_sets.labels"), + ("UPDATE label_sets SET labels = [] WHERE label_set_id = 1", "label_sets.labels"), + ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': 'W'}] WHERE label_set_id = 1", "label_sets.labels"), + ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': ''}] WHERE label_set_id = 1", "label_sets.labels"), # A NULL name drops out of list_distinct, so the check for distinct names finds it. - ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': NULL}] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE label_sets SET labels = [{'name': 'W', 'description': ''}] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE tasks SET label_set = 'stage' WHERE task_type = 'answer'", "tasks.label_set"), - ("UPDATE tasks SET label_set = NULL WHERE task_id = 1", "tasks.label_set"), - ("UPDATE tasks SET label_set = 'missing' WHERE task_id = 1", "label_sets.name"), + ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': NULL}] WHERE label_set_id = 1", "label_sets.labels"), + ( + "UPDATE label_sets SET labels = [{'name': 'W', 'description': ''}] WHERE label_set_id = 1", + "label_sets.labels", + ), + ("UPDATE tasks SET label_set_id = 1 WHERE task_type = 'answer'", "tasks.label_set_id"), + ("UPDATE tasks SET label_set_id = NULL WHERE task_id = 1", "tasks.label_set_id"), + ("UPDATE tasks SET label_set_id = 99 WHERE task_id = 1", "label_sets.label_set_id"), ("UPDATE task_targets SET text_value = 'BOGUS' WHERE task_id = 1", "task 1 has a label that is not"), ( "INSERT INTO task_targets (task_id, position, target_kind, text_value) VALUES (1, 1, 'text', 'R')", @@ -280,7 +375,7 @@ def test_the_audit_rejects_a_corrupt_label_set(release_copy, corruption, field): def test_the_reader_rejects_a_classification_task_without_a_label_set(release_copy): with connect_control(release_copy) as connection: - connection.execute("UPDATE tasks SET label_set = NULL WHERE task_id = 1") + connection.execute("UPDATE tasks SET label_set_id = NULL WHERE task_id = 1") with pytest.raises(TimeFFormatError, match="classification task 1 has no label set"): _read_tasks(release_copy) @@ -288,7 +383,7 @@ def test_the_reader_rejects_a_classification_task_without_a_label_set(release_co def test_the_reader_rejects_a_label_set_on_another_task_type(release_copy): with connect_control(release_copy) as connection: - connection.execute("UPDATE tasks SET label_set = 'stage' WHERE task_type = 'answer'") + connection.execute("UPDATE tasks SET label_set_id = 1 WHERE task_type = 'answer'") with pytest.raises(TimeFFormatError, match="answer task 2 has a label set"): _read_tasks(release_copy) diff --git a/packages/timenet/tests/format/test_task_codec.py b/packages/timenet/tests/format/test_task_codec.py index 562a95828..e0aa3432c 100644 --- a/packages/timenet/tests/format/test_task_codec.py +++ b/packages/timenet/tests/format/test_task_codec.py @@ -29,7 +29,6 @@ def test_task_payload_keeps_targets_out_of_configuration_columns(): task = AnswerTask(prompt="Alive?", targets=("Yes", 1)) assert encode_task_payload(task) == { - "label_set": None, "unit": None, "target_name": None, "mode": None, diff --git a/packages/timenet/tests/registry/conftest.py b/packages/timenet/tests/registry/conftest.py index 8f8c51258..0fbf47810 100644 --- a/packages/timenet/tests/registry/conftest.py +++ b/packages/timenet/tests/registry/conftest.py @@ -23,7 +23,6 @@ RHYTHM = LabelSet( - name="rhythm", labels=( Label(name="afib"), Label(name="normal"), diff --git a/packages/timenet/tests/registry/test_conformance.py b/packages/timenet/tests/registry/test_conformance.py index 1fd42dc36..6bb546928 100644 --- a/packages/timenet/tests/registry/test_conformance.py +++ b/packages/timenet/tests/registry/test_conformance.py @@ -35,7 +35,6 @@ RHYTHM = LabelSet( - name="rhythm", labels=( Label(name="afib"), Label(name="normal"), diff --git a/packages/timenet/tests/registry/test_namespaced.py b/packages/timenet/tests/registry/test_namespaced.py index 78db80630..946790cc6 100644 --- a/packages/timenet/tests/registry/test_namespaced.py +++ b/packages/timenet/tests/registry/test_namespaced.py @@ -16,7 +16,7 @@ from timenet.writer import TimeFWriter -LABELS = LabelSet(name="labels", labels=(Label(name="x"),)) +LABELS = LabelSet(labels=(Label(name="x"),)) def _namespaced_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index 7087e619b..5d9ef8a30 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -1,24 +1,36 @@ -from pydantic import ValidationError +import pickle + import pytest from timenet.errors import TimeFValidationError -from timenet.types import ClassificationTask, Label, LabelSet +from timenet.types import Annotation, AnswerTask, ClassificationTask, Label, LabelSet, TimePoint -STAGES = LabelSet(name="stage", labels=(Label(name="W", description="Awake."), Label(name="N1"), Label(name="R"))) -FINDINGS = LabelSet(name="findings", labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), multi_label=True) +STAGES = LabelSet(labels=(Label(name="W", description="Awake."), Label(name="N1"), Label(name="R"))) +FINDINGS = LabelSet(labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), multi_label=True) @pytest.mark.parametrize( - ("labels", "message"), + ("kwargs", "message"), [ - ((), "at least one label"), - ((Label(name="W"), Label(name="W", description="Awake.")), "more than once"), + ({"labels": ()}, "at least one label"), + ({"labels": None}, "takes a list of Label items, got NoneType"), + ({"labels": 3}, "takes a list of Label items, got int"), + ({"labels": "WR"}, "takes a list of Label items, got str 'WR'"), + ({"labels": {"W": "Wake."}}, "takes a list of Label items, got dict"), + ({"labels": Label(name="W")}, "takes a list of Label items, got Label"), + ({"labels": ("W", "")}, "cannot have an empty label"), + ({"labels": (Label(name="W"),), "annotations": None}, "takes a list of Annotation items, got NoneType"), + ({"labels": (Label(name="W"), Label(name="W", description="Awake."))}, "more than once"), + ({"labels": (3,)}, "takes Label objects, got int"), + ({"labels": ("W", "W")}, "more than once"), + ({"labels": (Label(name="W"),), "multi_label": 1}, "multi_label must be a bool, got int"), + ({"labels": (Label(name="W"),), "annotations": ("name",)}, "takes Annotation objects, got str"), ], ) -def test_a_label_set_needs_at_least_one_label_and_distinct_names(labels, message): - with pytest.raises(ValidationError, match=message): - LabelSet(name="stage", labels=labels) +def test_a_label_set_rejects_invalid_fields(kwargs, message): + with pytest.raises(TimeFValidationError, match=message): + LabelSet(**kwargs) @pytest.mark.parametrize(("label_set", "type_name"), [(None, "NoneType"), ("stage", "str")]) @@ -30,8 +42,8 @@ def test_a_classification_task_requires_a_label_set(label_set, type_name): @pytest.mark.parametrize( ("targets", "message"), [ - (("X",), "labels not in label set 'stage': \\['X'\\]"), - ((), "0 labels, but label set 'stage' takes exactly one"), + (("X",), "labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]"), + ((), "0 labels, but label set \\['W', 'N1', 'R'\\] takes exactly one"), (("W", "R"), "2 labels"), ], ) @@ -46,5 +58,71 @@ def test_a_multi_label_target_accepts_any_distinct_listed_labels(targets): def test_a_multi_label_target_rejects_a_repeated_label(): - with pytest.raises(TimeFValidationError, match="repeated labels of label set 'findings': \\['AF'\\]"): + with pytest.raises(TimeFValidationError, match=r"repeated labels of label set \['AF', 'PVC', 'LBBB'\]: \['AF'\]"): FINDINGS.check_targets(("AF", "PVC", "AF")) + + +def test_equality_ignores_label_order_and_annotations_but_not_descriptions_or_multi_label(): + first = LabelSet(labels=(Label(name="F"), Label(name="M")), annotations=(Annotation(key="name", value="sex"),)) + second = LabelSet(labels=(Label(name="M"), Label(name="F"))) + described = LabelSet(labels=(Label(name="F", description="Female."), Label(name="M"))) + multi = LabelSet(labels=(Label(name="F"), Label(name="M")), multi_label=True) + assert first._content_key() == second._content_key() + assert len({first._content_key(), described._content_key(), multi._content_key()}) == 3 + assert first == LabelSet(labels=(Label(name="F"), Label(name="M"))) + assert first == second + assert first != described + assert first != multi + + +def test_annotate_attaches_an_occurrence_and_rejects_a_span(): + annotation = Annotation(key="name", value="sex") + built = LabelSet(labels=(Label(name="F"),), annotations=(annotation,)) + assert built.annotations == (annotation,) + assert built.annotations[0] is not annotation + label_set = LabelSet(labels=(Label(name="F"),)) + attached = label_set.annotate(annotation) + assert label_set.annotations == (attached,) + assert attached is not annotation + with pytest.raises(TimeFValidationError, match="has a span"): + label_set.annotate(Annotation(key="x", span=TimePoint.seconds(1.0))) + with pytest.raises(TimeFValidationError, match="has a span"): + LabelSet(labels=(Label(name="F"),), annotations=(Annotation(key="x", span=TimePoint.seconds(1.0)),)) + + +def test_a_classification_task_yields_itself_then_its_label_set(): + task = ClassificationTask(targets=("W",), label_set=STAGES) + assert list(task.annotation_owners()) == [task, STAGES] + answer = AnswerTask(targets=("x",)) + assert list(answer.annotation_owners()) == [answer] + + +def test_a_copied_task_gets_its_own_label_set(): + label_set = LabelSet(labels=(Label(name="W"),), annotations=(Annotation(key="name", value="stage"),)) + task = ClassificationTask(targets=("W",), label_set=label_set) + copy = task.with_references({}, {}) + assert copy.label_set == label_set + assert copy.label_set is not label_set + assert copy.label_set.annotations == label_set.annotations + assert copy.label_set.annotations[0] is not label_set.annotations[0] + copy.label_set.annotate(Annotation(key="note", value="child")) + assert [a.key for a in label_set.annotations] == ["name"] + + +def test_a_task_with_an_annotated_set_survives_a_pickle_round_trip(): + label_set = LabelSet( + labels=(Label(name="W", description="Awake."),), + multi_label=True, + annotations=(Annotation(key="name", value="stage"),), + ) + task = ClassificationTask(targets=("W",), label_set=label_set) + restored = pickle.loads(pickle.dumps(task)).label_set + assert restored == label_set + assert [(a.key, a.value) for a in restored.annotations] == [("name", "stage")] + + +def test_a_set_takes_its_labels_as_plain_strings_or_label_objects(): + label_set = LabelSet(["W", Label(name="R", description="REM sleep.")], multi_label=True) + assert label_set.labels == (Label(name="W"), Label(name="R", description="REM sleep.")) + assert label_set.multi_label is True + assert LabelSet(labels=("W", "R")) == LabelSet([Label(name="W"), Label(name="R")]) diff --git a/packages/timenet/tests/writer/test_id_encoding.py b/packages/timenet/tests/writer/test_id_encoding.py index b97597dfe..148f0f3c4 100644 --- a/packages/timenet/tests/writer/test_id_encoding.py +++ b/packages/timenet/tests/writer/test_id_encoding.py @@ -27,7 +27,7 @@ from timenet.writer import TimeFWriter -LABELS = LabelSet(name="labels", labels=(Label(name="x"),)) +LABELS = LabelSet(labels=(Label(name="x"),)) def _spec(): diff --git a/packages/timenet/tests/writer/test_sharding.py b/packages/timenet/tests/writer/test_sharding.py index 4ec1840f6..edbe605d2 100644 --- a/packages/timenet/tests/writer/test_sharding.py +++ b/packages/timenet/tests/writer/test_sharding.py @@ -29,7 +29,6 @@ CLASSES = LabelSet( - name="classes", labels=( Label(name="c0"), Label(name="c1"), diff --git a/packages/timenet/tests/writer/test_writer.py b/packages/timenet/tests/writer/test_writer.py index 7759d9442..adc9dca27 100644 --- a/packages/timenet/tests/writer/test_writer.py +++ b/packages/timenet/tests/writer/test_writer.py @@ -31,7 +31,7 @@ from timenet.writer.value_encoding import ValueEncoding -LABELS = LabelSet(name="labels", labels=(Label(name="x"),)) +LABELS = LabelSet(labels=(Label(name="x"),)) def _written(tmp_path, dataset=None, **kwargs): @@ -108,7 +108,7 @@ def test_manifest_is_valid_and_matches_dataset(tmp_path): assert (version_dir / rel).exists() -def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_path): +def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_label_sets(tmp_path): dataset = make_dataset() record = dataset.records[0] streamed = TimeFDataset.from_parts( @@ -118,14 +118,14 @@ def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_pat schema=dataset.derive_schema(), last_ids=dataset.last_ids, ) - task = ClassificationTask(inputs=(record,), targets=("x",), label_set=LABELS) - task.annotate(Annotation(key="task_note", value="letters")) + label_set = LabelSet(["x"], annotations=(Annotation(key="label_set_note", value="letters"),)) + task = ClassificationTask(inputs=(record,), targets=("x",), label_set=label_set) streamed.set_task_stream([ClassificationTask], lambda: iter((task,))) version_dir = _written(tmp_path, streamed) manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text()) - assert "task_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations] + assert "label_set_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations] def test_manifest_counts_match_the_control_database(tmp_path): From 6c44af4f98e551102a4ffc57a1ea5d19076f3fe6 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:59:02 -0800 Subject: [PATCH 02/15] refactor(types): use annotatable_entities for label sets Follow the rename in the base change: a ClassificationTask yields its label set from annotatable_entities(), and the load plan reads it there. --- packages/timenet/src/timenet/reader/load_plan.py | 4 ++-- packages/timenet/src/timenet/types/tasks.py | 2 +- packages/timenet/tests/types/test_label_sets.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/timenet/src/timenet/reader/load_plan.py b/packages/timenet/src/timenet/reader/load_plan.py index 4ff3fe32d..a56cc57b2 100644 --- a/packages/timenet/src/timenet/reader/load_plan.py +++ b/packages/timenet/src/timenet/reader/load_plan.py @@ -68,13 +68,13 @@ def _label_set_children(item: LabelSet) -> tuple[object, ...]: @_load_plan_children.register def _task_children(item: Task) -> tuple[object, ...]: - """Return a task's text, annotations, owned objects, records, targets, and the annotations it references.""" + """Return a task's text, annotations, the entities it holds, records, targets, and the annotations it references.""" return ( item.prompt, item.rationale, *(item.targets or ()), *item.annotations, - *(owner for owner in item.annotation_owners() if owner is not item), + *(entity for entity in item.annotatable_entities() if entity is not item), *item.inputs, *getattr(item, "candidate_records", ()), *item.input_annotations, diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index e22b5123f..29f478c33 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -408,7 +408,7 @@ def check_targets(self) -> None: except TimeFValidationError as exc: raise TimeFValidationError(f"{name}: {exc}") from exc - def annotation_owners(self) -> Iterator[SupportsAnnotate]: + def annotatable_entities(self) -> Iterator[SupportsAnnotate]: """Yield the task, then its label set. Yields: diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index 5d9ef8a30..a670b4ecf 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -92,9 +92,9 @@ def test_annotate_attaches_an_occurrence_and_rejects_a_span(): def test_a_classification_task_yields_itself_then_its_label_set(): task = ClassificationTask(targets=("W",), label_set=STAGES) - assert list(task.annotation_owners()) == [task, STAGES] + assert list(task.annotatable_entities()) == [task, STAGES] answer = AnswerTask(targets=("x",)) - assert list(answer.annotation_owners()) == [answer] + assert list(answer.annotatable_entities()) == [answer] def test_a_copied_task_gets_its_own_label_set(): From d96b373d02eb0167e110b17bf930adda56ef8e26 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:59:45 -0800 Subject: [PATCH 03/15] test: read label set annotations without the removed lookup The tests and the annotations docs read a label set's name annotation from its annotations, since annotation(key) is gone. --- docs/data-model/annotations.md | 1 - .../datasets/physionet/sleep_edfx/tests/test_tasks.py | 3 ++- .../datasets/timenet/test_mean/tests/test_connector.py | 4 ++-- .../datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py | 2 +- packages/timenet/tests/format/test_label_set_storage.py | 4 +--- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index be62c9d26..21afac4da 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -129,7 +129,6 @@ SLEEP_STAGE = LabelSet( annotations=(Annotation(key="name", value="sleep_stage"),), ) SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) -SLEEP_STAGE.annotation("name") ``` A label set annotation has no span. Annotations are not part of the identity of a set. TimeF stores diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py index 948885b3d..7a4b0ecb4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py @@ -154,7 +154,8 @@ def _pick(built, kind, label_set=None): def _label_sets(built): - return {read_value(one.label_set.annotation("name").value) for one in built if isinstance(one, ClassificationTask)} + tasks = [one for one in built if isinstance(one, ClassificationTask)] + return {read_value(a.value) for one in tasks for a in one.label_set.annotations if a.key == "name"} def test_age_is_a_scalar_with_a_unit(): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py index 2acf80378..47ae40737 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py @@ -46,8 +46,8 @@ def test_has_balanced_binary_labels(): def test_every_task_shares_the_mean_sign_label_set(): label_sets = [task.label_set for task in _convert().tasks_of(ClassificationTask)] - names = [label_set.annotation("name") for label_set in label_sets] - assert {None if name is None else read_value(name.value) for name in names} == {"mean_sign"} + names = {read_value(a.value) for label_set in label_sets for a in label_set.annotations if a.key == "name"} + assert names == {"mean_sign"} assert len({id(label_set) for label_set in label_sets}) == 1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py index a56687def..112151423 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py @@ -110,7 +110,7 @@ def _check_task(task, row: dict[str, Any]) -> None: assert task.metadata["source_issues"] == _EXPECTED_ISSUES.get(_row_of(task), []) if isinstance(task, ClassificationTask): assert task.label_set.names == ("T", "F") - assert task.label_set.annotation("name").value == "true_or_false" + assert [(a.key, a.value) for a in task.label_set.annotations] == [("name", "true_or_false")] def test_every_task_holds_its_published_row(): diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 2cf6b068e..060160ed2 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -142,9 +142,7 @@ def test_a_filtered_read_still_loads_the_label_set_annotations(release): first, second = _label_sets(tasks.values()) assert first is second - annotation = first.annotation("name") - assert annotation is not None - assert annotation.value == "stage" + assert [(annotation.key, annotation.value) for annotation in first.annotations] == [("name", "stage")] def test_the_batches_of_a_task_stream_share_one_label_set_object(release): From 2259a80a49ee6c667fc9cf87c37fbd3d476fa469 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:02:51 -0800 Subject: [PATCH 04/15] refactor(types): add label set annotations in ClassificationTask.walk_annotations A ClassificationTask yields its own annotations, then those of its label set, from walk_annotations(). The load plan reads a task's annotations through walk_annotations(), so it needs no separate label set handler, and annotatable_entities() is gone. --- packages/timenet/src/timenet/reader/load_plan.py | 13 +++---------- packages/timenet/src/timenet/types/tasks.py | 10 +++++----- packages/timenet/tests/types/test_label_sets.py | 12 ++++++++---- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/timenet/src/timenet/reader/load_plan.py b/packages/timenet/src/timenet/reader/load_plan.py index a56cc57b2..d32ae7367 100644 --- a/packages/timenet/src/timenet/reader/load_plan.py +++ b/packages/timenet/src/timenet/reader/load_plan.py @@ -17,7 +17,7 @@ from timenet.dataset import Record, Signal, Source from timenet.format.stored_array import ChunkLocation, StoredArray from timenet.format.stored_values import PackedLocation, StoredOffsets, StoredText, StoredTextList -from timenet.types import Annotation, LabelSet, Task +from timenet.types import Annotation, Task from timenet.types.load_plan import Loading, LoadPlan @@ -60,21 +60,14 @@ def _annotation_children(item: Annotation) -> tuple[object, ...]: return (item.value,) -@_load_plan_children.register -def _label_set_children(item: LabelSet) -> tuple[object, ...]: - """Return a label set's annotations.""" - return item.annotations - - @_load_plan_children.register def _task_children(item: Task) -> tuple[object, ...]: - """Return a task's text, annotations, the entities it holds, records, targets, and the annotations it references.""" + """Return a task's text, annotations, input and candidate records, targets, and the annotations it references.""" return ( item.prompt, item.rationale, *(item.targets or ()), - *item.annotations, - *(entity for entity in item.annotatable_entities() if entity is not item), + *item.walk_annotations(), *item.inputs, *getattr(item, "candidate_records", ()), *item.input_annotations, diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 29f478c33..899c0ac1e 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -408,14 +408,14 @@ def check_targets(self) -> None: except TimeFValidationError as exc: raise TimeFValidationError(f"{name}: {exc}") from exc - def annotatable_entities(self) -> Iterator[SupportsAnnotate]: - """Yield the task, then its label set. + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of the task, then those of its label set. Yields: - The task and its label set. + Each annotation occurrence, in the order above. """ - yield self - yield self.label_set + yield from self.annotations + yield from self.label_set.annotations def with_references(self, records: Mapping[EntityId, "Record"], signals: Mapping[EntityId, "Signal"]) -> Self: """Return a copy of the task that references the given Records and Signals, by ID. diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index a670b4ecf..edd7314f4 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -90,11 +90,15 @@ def test_annotate_attaches_an_occurrence_and_rejects_a_span(): LabelSet(labels=(Label(name="F"),), annotations=(Annotation(key="x", span=TimePoint.seconds(1.0)),)) -def test_a_classification_task_yields_itself_then_its_label_set(): - task = ClassificationTask(targets=("W",), label_set=STAGES) - assert list(task.annotatable_entities()) == [task, STAGES] +def test_a_classification_task_walks_its_annotations_then_those_of_its_label_set(): + task = ClassificationTask( + targets=("W",), label_set=LabelSet(["W", "R"], annotations=(Annotation(key="set", value="s"),)) + ) + task.annotate(Annotation(key="task", value="t")) + assert [annotation.key for annotation in task.walk_annotations()] == ["task", "set"] answer = AnswerTask(targets=("x",)) - assert list(answer.annotatable_entities()) == [answer] + answer.annotate(Annotation(key="answer", value="a")) + assert [annotation.key for annotation in answer.walk_annotations()] == ["answer"] def test_a_copied_task_gets_its_own_label_set(): From 7997385935f52e6ee722b4ff25ba192870bfd8ef Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:15:55 -0800 Subject: [PATCH 05/15] feat(types): add ordered label sets LabelSet(labels, ordered=True) makes the label order part of the identity of the set: the same labels in another order are a different set, and the stored set keeps its own order. By default a set is unordered, as before. label_sets gets an ordered column. ARFBench builds its option sets with ordered=True, because a model sees the options in their order. --- docs/data-model/tasks.md | 21 ++++++++---- docs/timef-format.md | 13 ++++---- .../datasets/datadog/arfbench/questions.py | 5 +-- .../datadog/arfbench/tests/test_connector.py | 12 ++++--- .../src/timenet/format/control_reader.py | 1 + .../src/timenet/format/control_schema.py | 2 ++ .../src/timenet/format/control_writer.py | 1 + .../timenet/src/timenet/types/label_sets.py | 32 +++++++++++++------ .../tests/format/test_label_set_storage.py | 28 ++++++++++++++-- .../timenet/tests/types/test_label_sets.py | 14 ++++++++ 10 files changed, 100 insertions(+), 29 deletions(-) diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index aab54d651..f6ce0c54f 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -239,7 +239,11 @@ a target can contain. If `multi_label=False`, the default, a target must contain exactly one label. If `multi_label=True`, a target contains all labels that apply. Omitted labels do not apply. An empty target means that no -label applies. The order of labels does not matter. +label applies. + +By default the order of labels does not matter. Set `ordered=True` when it does. Then the label order is +part of the identity of the set. Use it for ordinal labels, such as `low`, `medium` and `high`, or for +answer options whose order matters. When the dataset adds or streams a task, it makes sure that the target follows the rules of its label set. Each label must be in the set. No label can repeat. A target with `multi_label=False` @@ -249,11 +253,16 @@ Labels come from the targets stored directly on the task or from the text of its annotations. A label set has no name. TimeF stores equal sets once. For this, two sets are equal if they have the -same labels, label descriptions, and `multi_label`. The order of labels and the annotations do not -matter, and `==` follows the same rule. The stored set gets one integer id, `label_set_id`, and keeps -the label order of the first equal set that the writer meets. If you need the order of each task, -put it in the task metadata. If equal sets carry different annotation occurrences, the stored set -keeps each distinct occurrence. In one read, the reader gives tasks that use the same set a shared object. +same labels, label descriptions, `multi_label`, and `ordered`. The annotations do not matter. The +order of labels matters only for an ordered set. `==` follows the same rule. + +Use `ordered=True` when the label order has a meaning, for example for ordinal labels or for answer +options that a model sees in a fixed order. Then each order is its own set, and the stored set keeps +it. An unordered set keeps the label order of the first equal set that the writer meets. + +The stored set gets one integer id, `label_set_id`. If equal sets carry different annotation +occurrences, the stored set keeps each distinct occurrence. In one read, the reader gives tasks that +use the same set a shared object. ### Answer and rationale diff --git a/docs/timef-format.md b/docs/timef-format.md index 948ae7f58..a8878ee87 100644 --- a/docs/timef-format.md +++ b/docs/timef-format.md @@ -192,14 +192,15 @@ Configuration for each task type uses the columns `label_set_id`, `unit`, `targe A column that a task type does not use is `NULL`. SQL can filter these columns without parsing JSON. Every classification task has a label set, and `tasks.label_set_id` stores its ID. Each row in -`label_sets` stores one set with the fields `label_set_id`, `labels`, and `multi_label`. Sets with the -same labels, label descriptions, and `multi_label` are one set, so a release stores them once. The -writer numbers the sets in the order that tasks first use them. The annotations of a set are rows of -`annotation_occurrences` with `object_type` `LabelSet` and the `label_set_id` as `object_id`. +`label_sets` stores one set with the fields `label_set_id`, `labels`, `multi_label`, and `ordered`. +Sets with the same labels, label descriptions, `multi_label`, and `ordered` are one set, so a release +stores them once. If `ordered` is true, the label order is part of the set, so the same labels in +another order are a different set. The writer numbers the sets in the order that tasks first use them. +The annotations of a set are rows of `annotation_occurrences` with `object_type` `LabelSet` and the +`label_set_id` as `object_id`. The `labels` field holds one `{name, description}` entry per label, in the label order of the first -equal set that the writer meets. If a label has -no description, its `description` is `NULL`. +equal set that the writer meets. If a label has no description, its `description` is `NULL`. Each inline target is one row in `task_targets`. Its `target_kind` names the typed value column that is set. A target stores entity IDs directly: `record_id` for a Record, `signal_id` for a diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py index 62c8b2524..ce42facad 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py @@ -206,7 +206,8 @@ def create_chart_task(self, chart_records: Mapping[str, Record]) -> Classificati def _create_task(self, inputs: list[Record], modality: InputModality) -> ClassificationTask: """Create a test task that asks for the correct answer from the options of the question. - The options are the label set of the task. Questions with the same options share one set. The row + The options are the label set of the task. The set is ordered, because the order of the options is + part of the question. Questions with the same options in the same order share one set. The row index in the metadata pairs the series task with the chart task of ``datadog/arfbench-charts``. The interpolation flags describe what an engineer saw. This method does not interpolate. @@ -220,7 +221,7 @@ def _create_task(self, inputs: list[Record], modality: InputModality) -> Classif input_modalities=frozenset({InputModality.TEXT, modality}), split=Split.TEST, metadata={"question_index": self.index}, - label_set=LabelSet(self.answer_options), + label_set=LabelSet(self.answer_options, ordered=True), ) task.annotate(Annotation("task_category", value=self.task_category.value)) task.annotate(Annotation("difficulty", value=self.difficulty.value)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py index 21c19c564..6b598e234 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py @@ -107,16 +107,20 @@ def test_each_question_is_a_classification_task_over_its_options(release): } -def test_questions_with_equal_options_share_a_label_set(release): +def test_questions_with_equal_options_in_equal_order_share_a_label_set(release): up_down = {**PRESENCE_ROW, "Unnamed: 0": "2", "options_str": '["Up", "Down"]', "correct_answer": "Up"} - write_qa_csv(_path(release, "questions").parent, [*ROWS, up_down]) - presence, correlation, up_down_task = ARFBenchConnector().convert(release).get_all() + no_yes = {**PRESENCE_ROW, "Unnamed: 0": "3", "options_str": '["No", "Yes"]', "correct_answer": "No"} + write_qa_csv(_path(release, "questions").parent, [*ROWS, up_down, no_yes]) + presence, correlation, up_down_task, no_yes_task = ARFBenchConnector().convert(release).get_all() assert isinstance(presence, ClassificationTask) and isinstance(correlation, ClassificationTask) - assert isinstance(up_down_task, ClassificationTask) + assert isinstance(up_down_task, ClassificationTask) and isinstance(no_yes_task, ClassificationTask) + assert presence.label_set.ordered assert presence.label_set == correlation.label_set assert presence.label_set.annotations == () assert up_down_task.label_set != presence.label_set + assert no_yes_task.label_set != presence.label_set + assert no_yes_task.label_set.names == ("No", "Yes") def test_the_dataset_round_trips_through_parquet_with_its_missing_timesteps(release, tmp_path): diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index 59527b43d..4df39ba92 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -1879,6 +1879,7 @@ def _read_label_sets(self) -> dict[int, LabelSet]: label_sets[label_set_id] = LabelSet( labels=tuple(Label.model_validate(label) for label in row["labels"]), multi_label=row["multi_label"], + ordered=row["ordered"], annotations=annotations.get((ObjectKind.LABEL_SET, label_set_id), ()), ) except (ValidationError, TimeFValidationError) as exc: diff --git a/packages/timenet/src/timenet/format/control_schema.py b/packages/timenet/src/timenet/format/control_schema.py index da9cb00b5..882920a99 100644 --- a/packages/timenet/src/timenet/format/control_schema.py +++ b/packages/timenet/src/timenet/format/control_schema.py @@ -290,12 +290,14 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column _key("label_set_id"), _required("labels", ColumnType.LABEL_LIST), _required("multi_label", ColumnType.BOOLEAN), + _required("ordered", ColumnType.BOOLEAN), ), unique=(("label_set_id",),), ) """Each row stores one label set that a classification task uses. The ``labels`` column holds one ``{name, description}`` entry per label, in label order. +``ordered`` says whether that order is part of the identity of the set. If a label has no description, ``description`` is NULL. Tasks refer to a set by its id in ``tasks.label_set_id``. The writer gives each distinct set one id, in the order that tasks first use it.""" diff --git a/packages/timenet/src/timenet/format/control_writer.py b/packages/timenet/src/timenet/format/control_writer.py index c2f6158ab..92f6d78d3 100644 --- a/packages/timenet/src/timenet/format/control_writer.py +++ b/packages/timenet/src/timenet/format/control_writer.py @@ -247,6 +247,7 @@ def register(self, label_set: LabelSet) -> int: "label_set_id": next(self._new_ids), "labels": [label.model_dump() for label in label_set.labels], "multi_label": label_set.multi_label, + "ordered": label_set.ordered, } occurrences = list(label_set.annotations) self._sets[key] = (row, occurrences, {_occurrence_key(annotation) for annotation in occurrences}) diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index bbddea3e6..b28e3f392 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -58,7 +58,8 @@ def _as_label(item: object) -> object: class LabelSet(SupportsAnnotate): """The allowed labels for a classification target. - A target must contain one label unless ``multi_label=True``. + A target must contain one label unless ``multi_label=True``. By default the label order does not matter. + Use ``ordered=True`` for ordinal labels or for options whose order matters. Tasks that ask the same kind of question share one label set:: @@ -71,18 +72,25 @@ class LabelSet(SupportsAnnotate): A plain string is a label without a description, so ``LabelSet(["yes", "no"])`` is enough when no label needs one. - A label set has no name. Two sets with the same labels (in any order, with the same descriptions) - and the same ``multi_label`` are the same set, so ``==`` is true for them. A dataset stores them once, - with the label order of the first of them that it writes. Annotations are not part of that identity. + A label set has no name. Two sets with the same labels (with the same descriptions), the same + ``multi_label`` and the same ``ordered`` are the same set, so ``==`` is true for them. If ``ordered`` is + ``False``, the labels can be in any order. If ``ordered`` is ``True``, the labels must be in the same order. + A dataset stores equal sets once, with the label order of the first of them that it writes. + Annotations are not part of that identity. """ labels: tuple[Label, ...] - """The allowed labels in order. A dataset keeps the order of the first equal set that it writes.""" + """The allowed labels in order. For an unordered set, a dataset keeps the order of the first equal set that + it writes.""" multi_label: bool = False """If ``False``, a target holds exactly one label from the set. If ``True``, a target holds all labels from the set that apply. An empty target means that no label applies.""" + ordered: bool = False + """If ``True``, the label order is part of the identity of the set. If ``False``, the order does not matter. + + An ordered set needs labels in a fixed order, so pass a list or a tuple, not a Python set.""" annotations: tuple[Annotation, ...] = field(default=(), compare=False) """Annotations that describe the set, such as ``Annotation(key="name", value="sleep_stage")``. @@ -94,6 +102,7 @@ def __init__( labels: Iterable[Label | str], *, multi_label: bool = False, + ordered: bool = False, annotations: Iterable[Annotation] = (), ) -> None: """Build a label set, make sure that it is valid, and attach the annotations. @@ -101,6 +110,7 @@ def __init__( Args: labels: The allowed labels in order. A plain string is a label without a description. multi_label: Whether a target can hold more than one label. + ordered: Whether the label order is part of the identity of the set. annotations: Annotations that describe the set. Raises: @@ -108,6 +118,7 @@ def __init__( """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it self.labels = tuple(_as_label(label) for label in _as_tuple(labels, Label)) # ty: ignore[invalid-assignment] self.multi_label = multi_label + self.ordered = ordered self.annotations = _as_tuple(annotations, Annotation) self._validate() self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) @@ -117,7 +128,7 @@ def _validate(self) -> None: Raises: TimeFValidationError: If a label is not a Label, the set has no label or repeats a name, - ``multi_label`` is not a bool, or an annotation is not an Annotation or has a span. + ``multi_label`` or ``ordered`` is not a bool, or an annotation is not an Annotation or has a span. """ for label in self.labels: if not isinstance(label, Label): @@ -128,6 +139,8 @@ def _validate(self) -> None: raise TimeFValidationError(f"{self} lists a label more than once") if not isinstance(self.multi_label, bool): raise TimeFValidationError(f"{self} multi_label must be a bool, got {type(self.multi_label).__name__}") + if not isinstance(self.ordered, bool): + raise TimeFValidationError(f"{self} ordered must be a bool, got {type(self.ordered).__name__}") for annotation in self.annotations: self._check_annotation(annotation) @@ -168,7 +181,7 @@ def __str__(self) -> str: return f"label set {list(self.names)}" def __eq__(self, other: object) -> bool: - """Return whether the sets have the same labels, in any order, and the same ``multi_label``.""" + """Return whether the sets have the same content key (see :meth:`_content_key`).""" if not isinstance(other, LabelSet): return NotImplemented return self._content_key() == other._content_key() @@ -178,12 +191,13 @@ def __eq__(self, other: object) -> bool: def _content_key(self) -> tuple[Hashable, ...]: """Return the key that makes two sets equal. - The key ignores label order and annotations. + The key ignores annotations. It ignores label order unless the set is ordered. Returns: The hashable key. """ - return (self.multi_label, frozenset((label.name, label.description) for label in self.labels)) + labels = ((label.name, label.description) for label in self.labels) + return (self.multi_label, self.ordered, tuple(labels) if self.ordered else frozenset(labels)) def check_targets(self, targets: Sequence[str]) -> None: """Make sure that the target labels follow the rules of this set. diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 060160ed2..7a23c0704 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -113,7 +113,9 @@ def release_copy(release, tmp_path) -> Path: def test_each_label_set_is_stored_once_and_read_as_one_object(release): with connect_control(release, read_only=True) as connection: audit_control_database(connection, require_chunks=False) - assert connection.execute("SELECT label_set_id, labels, multi_label FROM label_sets ORDER BY 1").fetchall() == [ + assert connection.execute( + "SELECT label_set_id, labels, multi_label, ordered FROM label_sets ORDER BY 1" + ).fetchall() == [ ( 1, [ @@ -122,8 +124,9 @@ def test_each_label_set_is_stored_once_and_read_as_one_object(release): {"name": "R", "description": "REM sleep."}, ], False, + False, ), - (2, [{"name": "AF", "description": None}, {"name": "PVC", "description": None}], True), + (2, [{"name": "AF", "description": None}, {"name": "PVC", "description": None}], True, False), ] stage_w, _, findings, stage_n1, stage_r, none_apply = _read_tasks(release) @@ -183,6 +186,27 @@ def test_sets_are_stored_once_per_content_and_keep_the_first_label_order(tmp_pat assert first.label_set.names == ("W", "N1", "R") +def test_ordered_sets_in_another_order_are_stored_as_two_rows_in_their_own_order(tmp_path): + low_high = LabelSet(["low", "high"], ordered=True) + high_low = LabelSet(["high", "low"], ordered=True) + unordered = LabelSet(["low", "high"]) + path = _write_sets(tmp_path, low_high, high_low, LabelSet(["low", "high"], ordered=True), unordered) + + with connect_control(path, read_only=True) as connection: + audit_control_database(connection, require_chunks=False) + assert connection.execute("SELECT label_set_id, ordered FROM label_sets ORDER BY 1").fetchall() == [ + (1, True), + (2, True), + (3, False), + ] + first, second, third, fourth = _label_sets(_read_tasks(path)) + assert first == low_high and first.ordered + assert first.names == ("low", "high") + assert second.names == ("high", "low") + assert first is third + assert fourth == unordered and not fourth.ordered + + def test_equal_sets_with_different_annotations_are_stored_once_with_both(tmp_path): first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) second = LabelSet( diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index edd7314f4..e47a20039 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -25,6 +25,8 @@ ({"labels": (3,)}, "takes Label objects, got int"), ({"labels": ("W", "W")}, "more than once"), ({"labels": (Label(name="W"),), "multi_label": 1}, "multi_label must be a bool, got int"), + ({"labels": (Label(name="W"),), "ordered": 1}, "ordered must be a bool, got int"), + ({"labels": (Label(name="W"),), "ordered": None}, "ordered must be a bool, got NoneType"), ({"labels": (Label(name="W"),), "annotations": ("name",)}, "takes Annotation objects, got str"), ], ) @@ -75,6 +77,18 @@ def test_equality_ignores_label_order_and_annotations_but_not_descriptions_or_mu assert first != multi +def test_an_ordered_set_uses_its_label_order_for_equality(): + low_high = LabelSet(["low", "high"], ordered=True) + high_low = LabelSet(["high", "low"], ordered=True) + assert low_high == LabelSet(["low", "high"], ordered=True) + assert low_high._content_key() == LabelSet(["low", "high"], ordered=True)._content_key() + assert low_high != high_low + assert low_high._content_key() != high_low._content_key() + assert low_high != LabelSet(["low", "high"]) + assert LabelSet(["low", "high"]) == LabelSet({"high", "low"}) + assert LabelSet(["low"]).ordered is False + + def test_annotate_attaches_an_occurrence_and_rejects_a_span(): annotation = Annotation(key="name", value="sex") built = LabelSet(labels=(Label(name="F"),), annotations=(annotation,)) From fb433ffd176dd420fb212b134fd02f0bea4312e6 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:34:48 -0800 Subject: [PATCH 06/15] refactor: build label sets from plain label names Pass labels positionally and use plain strings where a label has no description. Labels with a description stay Label objects. --- benchmarks/end_to_end/corpus.py | 9 +++--- docs/data-model/annotations.md | 4 +-- docs/data-model/records.md | 4 +-- docs/data-model/tasks.md | 17 ++++------ .../datasets/opentslm/har_cot/connector.py | 16 +++++----- .../datasets/opentslm/sleep_cot/connector.py | 4 +-- .../datasets/physionet/sleep_edfx/tasks.py | 16 +++++----- .../datasets/timenet/hello_world/connector.py | 4 +-- .../datasets/timenet/test_mean/connector.py | 4 +-- .../tsaqa/tsaqa_benchmark/connector.py | 2 +- .../src/timenet/format/control_reader.py | 2 +- .../timenet/tests/dataset/test_composition.py | 4 +-- .../timenet/tests/dataset/test_dataset.py | 25 +++------------ .../tests/format/test_label_set_storage.py | 32 +++++++++---------- packages/timenet/tests/registry/conftest.py | 8 +---- .../tests/registry/test_conformance.py | 8 +---- .../timenet/tests/registry/test_namespaced.py | 3 +- .../timenet/tests/types/test_label_sets.py | 24 +++++++------- .../timenet/tests/writer/test_id_encoding.py | 3 +- .../timenet/tests/writer/test_sharding.py | 8 +---- packages/timenet/tests/writer/test_writer.py | 3 +- 21 files changed, 79 insertions(+), 121 deletions(-) diff --git a/benchmarks/end_to_end/corpus.py b/benchmarks/end_to_end/corpus.py index 478119b36..963b0cb47 100644 --- a/benchmarks/end_to_end/corpus.py +++ b/benchmarks/end_to_end/corpus.py @@ -19,7 +19,6 @@ DatasetMetadata, Domain, ForecastingTask, - Label, LabelSet, License, TimeInterval, @@ -31,10 +30,10 @@ from timenet.types.wire import ValueDtype -CONDITION = LabelSet(labels=(Label(name="healthy"), Label(name="outer-race-fault"))) -RHYTHM = LabelSet(labels=(Label(name="normal-sinus"), Label(name="atrial-fibrillation"))) -SLEEP_STAGE = LabelSet(labels=tuple(Label(name=stage) for stage in ("W", "N1", "N2", "N3", "REM"))) -MEAN_SIGN = LabelSet(labels=(Label(name="above-zero"), Label(name="below-zero"))) +CONDITION = LabelSet(["healthy", "outer-race-fault"]) +RHYTHM = LabelSet(["normal-sinus", "atrial-fibrillation"]) +SLEEP_STAGE = LabelSet(["W", "N1", "N2", "N3", "REM"]) +MEAN_SIGN = LabelSet(["above-zero", "below-zero"]) @dataclass(frozen=True) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 21afac4da..fbb715672 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -122,10 +122,10 @@ A `LabelSet` takes annotations like the other owners. Pass them to the construct `annotate()` later. Use them for a name, a description, or other context. No key is required. ```python -from timenet.types import Annotation, Label, LabelSet +from timenet.types import Annotation, LabelSet SLEEP_STAGE = LabelSet( - labels=(Label(name="W"), Label(name="N1"), Label(name="R")), + ["W", "N1", "R"], annotations=(Annotation(key="name", value="sleep_stage"),), ) SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) diff --git a/docs/data-model/records.md b/docs/data-model/records.md index 412b77435..38cb2e366 100644 --- a/docs/data-model/records.md +++ b/docs/data-model/records.md @@ -119,9 +119,9 @@ A [Task](tasks.md) refers directly to its input Records. Registering a Task does Records. ```python -from timenet.types import ClassificationTask, Label, LabelSet +from timenet.types import ClassificationTask, LabelSet -CONDITION = LabelSet(labels=(Label(name="healthy"), Label(name="faulty"))) +CONDITION = LabelSet(["healthy", "faulty"]) task = ClassificationTask( inputs=(record,), diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index f6ce0c54f..f89eb85e5 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -12,9 +12,9 @@ A Task describes one modeling problem. It holds ordered input Records and one re expected output. ```python -from timenet.types import ClassificationTask, Label, LabelSet +from timenet.types import ClassificationTask, LabelSet -CONDITION = LabelSet(labels=(Label(name="healthy"), Label(name="faulty"))) +CONDITION = LabelSet(["healthy", "faulty"]) task = ClassificationTask( inputs=(record,), @@ -85,9 +85,9 @@ A dataset that ships a partition marks each Task with its `split`. Records carry own: one Record may serve Tasks of several splits. ```python -from timenet.types import ClassificationTask, Label, LabelSet, Split +from timenet.types import ClassificationTask, LabelSet, Split -ACTIVITY = LabelSet(labels=(Label(name="walking"), Label(name="sitting"))) +ACTIVITY = LabelSet(["walking", "sitting"]) dataset.add_task( ClassificationTask( @@ -221,13 +221,8 @@ dataset.add_task(age_question(record, 64)) ```python from timenet.types import Label, LabelSet -CONDITION = LabelSet( - labels=(Label(name="healthy"), Label(name="faulty", description="A bearing fault.")), -) -FINDINGS = LabelSet( - labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), - multi_label=True, -) +CONDITION = LabelSet(["healthy", Label(name="faulty", description="A bearing fault.")]) +FINDINGS = LabelSet(["AF", "PVC", "LBBB"], multi_label=True) ClassificationTask(inputs=(record,), targets="healthy", label_set=CONDITION) ClassificationTask(inputs=(record,), targets=("PVC", "AF"), label_set=FINDINGS) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py index 36d100dde..f8716bb7a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py @@ -35,16 +35,16 @@ # The label order is the order of the OpenTSLM training prompt. ACTIVITY = LabelSet( - labels=( - Label(name="biking"), - Label(name="lying"), - Label(name="running"), - Label(name="sitting"), - Label(name="standing"), - Label(name="walking"), + [ + "biking", + "lying", + "running", + "sitting", + "standing", + "walking", Label(name="walking_down", description="Walking down stairs."), Label(name="walking_up", description="Walking up stairs."), - ), + ], annotations=(Annotation(key="name", value="activity"),), ) # The release states no unit. A resting window has a magnitude of about 9.8, which is gravity in m/s². diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index 01f5fd0ea..a0b4f93e1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -35,14 +35,14 @@ # The release writes the stage codes of the Rechtschaffen and Kales manual. The prompts and the # rationales write the names in the descriptions. SLEEP_STAGE = LabelSet( - labels=( + [ Label(name="W", description="Wake"), Label(name="N1", description="Non-REM stage 1"), Label(name="N2", description="Non-REM stage 2"), Label(name="N3", description="Non-REM stage 3"), Label(name="N4", description="Non-REM stage 4"), Label(name="REM", description="REM sleep"), - ), + ], annotations=(Annotation(key="name", value="sleep_stage"),), ) # OpenTSLM converted the microvolts of Sleep-EDF to volts and kept every second sample of 100 Hz. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py index ccff401ad..970c7f0c4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py @@ -42,16 +42,16 @@ # technician left unscored. Each classification task draws its target from one closed set, whose # name annotation is the key whose values it holds, and each target is exactly one label. SLEEP_STAGES = LabelSet( - labels=( + [ Label(name="Sleep stage W", description="Wake."), - Label(name="Sleep stage 1"), - Label(name="Sleep stage 2"), - Label(name="Sleep stage 3"), - Label(name="Sleep stage 4"), + "Sleep stage 1", + "Sleep stage 2", + "Sleep stage 3", + "Sleep stage 4", Label(name="Sleep stage R", description="Rapid eye movement (REM) sleep."), Label(name="Movement time", description="An epoch that body movement obscures, so it has no stage."), Label(name="Sleep stage ?", description="An epoch with no sleep-stage score from data collection."), - ), + ], annotations=(Annotation(key="name", value=str(AnnotationKey.SLEEP_STAGE)),), ) STAGE_LABELS = SLEEP_STAGES.names @@ -62,7 +62,7 @@ # The sexes, after each sheet is decoded. The two sheets code the column with opposite # meanings, so a raw code never reaches a task. SEXES = LabelSet( - labels=(Label(name="F", description="Female."), Label(name="M", description="Male.")), + [Label(name="F", description="Female."), Label(name="M", description="Male.")], annotations=(Annotation(key="name", value=str(AnnotationKey.SEX)),), ) SEX_LABELS = SEXES.names @@ -70,7 +70,7 @@ # The two nights of the telemetry study. CONDITION_LABELS = tuple(Condition) CONDITIONS = LabelSet( - labels=[str(one) for one in Condition], + [str(one) for one in Condition], annotations=(Annotation(key="name", value=str(AnnotationKey.CONDITION)),), ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py index e40de0f96..61ea10f24 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py @@ -49,11 +49,11 @@ # Each classification task draws its target from a label set that every task with the same question # shares. _RHYTHM = LabelSet( - labels=(Label(name="normal", description="A regular wave."), Label(name="abnormal")), + [Label(name="normal", description="A regular wave."), "abnormal"], annotations=(Annotation(key="name", value="rhythm"),), ) _EVENT = LabelSet( - labels=(Label(name="onset", description="The moment the wave starts."), Label(name="offset")), + [Label(name="onset", description="The moment the wave starts."), "offset"], annotations=(Annotation(key="name", value="event"),), ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py index da086bcf8..84206c57c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py @@ -23,10 +23,10 @@ _NOISE_STD = 0.5 _SEED = 20260715 # fixed base seed keeps every build's values identical _MEAN_SIGN = LabelSet( - labels=( + [ Label(name="above_zero", description="The mean of the signal is above zero."), Label(name="below_zero", description="The mean of the signal is below zero."), - ), + ], annotations=(Annotation(key="name", value="mean_sign"),), ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index 0edd4cefc..5305ae07c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -74,7 +74,7 @@ class _Role(StrEnum): _TRUE_OR_FALSE_LABEL_SET = LabelSet( - labels=(Label(name="T", description="True."), Label(name="F", description="False.")), + [Label(name="T", description="True."), Label(name="F", description="False.")], annotations=(Annotation(key="name", value="true_or_false"),), ) """The label set that every true-or-false task shares.""" diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index 4df39ba92..ddbca0e82 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -1877,7 +1877,7 @@ def _read_label_sets(self) -> dict[int, LabelSet]: label_set_id = row["label_set_id"] try: label_sets[label_set_id] = LabelSet( - labels=tuple(Label.model_validate(label) for label in row["labels"]), + [Label.model_validate(label) for label in row["labels"]], multi_label=row["multi_label"], ordered=row["ordered"], annotations=annotations.get((ObjectKind.LABEL_SET, label_set_id), ()), diff --git a/packages/timenet/tests/dataset/test_composition.py b/packages/timenet/tests/dataset/test_composition.py index 80a9b17e2..0836c7043 100644 --- a/packages/timenet/tests/dataset/test_composition.py +++ b/packages/timenet/tests/dataset/test_composition.py @@ -58,12 +58,12 @@ from timenet.writer import TimeFWriter -ORDER = LabelSet(labels=(Label(name="late"),)) +ORDER = LabelSet(["late"]) _ALIAS = ParentAlias("base") _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) -SCP = LabelSet(labels=(Label(name="normal", description="No finding."), Label(name="abnormal"))) +SCP = LabelSet([Label(name="normal", description="No finding."), "abnormal"]) def store(root, dataset): diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 99740ef3d..be2724304 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -17,7 +17,6 @@ DatasetMetadata, ForecastingTask, InputModality, - Label, LabelSet, License, ScalarPredictionTask, @@ -34,21 +33,7 @@ ) -TEST_LABELS = LabelSet( - labels=( - Label(name="N2"), - Label(name="a"), - Label(name="afib"), - Label(name="b"), - Label(name="beat"), - Label(name="c"), - Label(name="d"), - Label(name="e"), - Label(name="event"), - Label(name="walking"), - Label(name="x"), - ), -) +TEST_LABELS = LabelSet(["N2", "a", "afib", "b", "beat", "c", "d", "e", "event", "walking", "x"]) def _classification_task(record: Record, label: str, **kwargs: Any) -> ClassificationTask: @@ -182,19 +167,19 @@ def test_derive_schema(make_series): def test_derive_schema_collects_label_set_annotations(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) - labels = (Label(name="a"), Label(name="b")) + labels = ("a", "b") ds.add_task( task=ClassificationTask( inputs=(record,), targets=("a",), - label_set=LabelSet(labels=labels, annotations=(Annotation(key="name", value="letters", unit="meter"),)), + label_set=LabelSet(labels, annotations=(Annotation(key="name", value="letters", unit="meter"),)), ) ) ds.add_task( task=ClassificationTask( inputs=(record,), targets=("b",), - label_set=LabelSet(labels=labels, annotations=(Annotation(key="note", value="extra"),)), + label_set=LabelSet(labels, annotations=(Annotation(key="note", value="extra"),)), ) ) @@ -214,7 +199,7 @@ def test_derive_schema_rejects_a_label_set_annotation_that_conflicts_with_a_reco task=ClassificationTask( inputs=(record,), targets=("a",), - label_set=LabelSet(labels=(Label(name="a"),), annotations=(Annotation(key="name", value="letters"),)), + label_set=LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)), ) ) with pytest.raises(ValueError, match="conflicting descriptors"): diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 7a23c0704..2a5ba78e6 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -30,10 +30,10 @@ SPEC = TimeSeriesSpec(spec_type="voltage", name="Voltage", unit_value=ureg.millivolt) STAGE_LABELS = LabelSet( - labels=(Label(name="W"), Label(name="N1"), Label(name="R", description="REM sleep.")), + ["W", "N1", Label(name="R", description="REM sleep.")], annotations=(Annotation(key="name", value="stage", description="Sleep stage"),), ) -FINDINGS = LabelSet(labels=(Label(name="AF"), Label(name="PVC")), multi_label=True) +FINDINGS = LabelSet(["AF", "PVC"], multi_label=True) def _record() -> Record: @@ -175,9 +175,9 @@ def _write_sets(tmp_path, *label_sets): def test_sets_are_stored_once_per_content_and_keep_the_first_label_order(tmp_path): - reordered = LabelSet(labels=tuple(reversed(STAGE_LABELS.labels))) - described = LabelSet(labels=(Label(name="W", description="Awake."), Label(name="N1"), Label(name="R"))) - multi = LabelSet(labels=STAGE_LABELS.labels, multi_label=True) + reordered = LabelSet(STAGE_LABELS.labels[::-1]) + described = LabelSet([Label(name="W", description="Awake."), "N1", "R"]) + multi = LabelSet(STAGE_LABELS.labels, multi_label=True) path = _write_sets(tmp_path, STAGE_LABELS, reordered, described, multi) assert [row[0] for row in _stored_sets(path)] == [1, 2, 3] @@ -208,12 +208,12 @@ def test_ordered_sets_in_another_order_are_stored_as_two_rows_in_their_own_order def test_equal_sets_with_different_annotations_are_stored_once_with_both(tmp_path): - first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) + first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) second = LabelSet( - labels=STAGE_LABELS.labels, + STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"), Annotation(key="source", value="manual")), ) - bare = LabelSet(labels=STAGE_LABELS.labels) + bare = LabelSet(STAGE_LABELS.labels) path = _write_sets(tmp_path, first, bare, second) assert [row[0] for row in _stored_sets(path)] == [1] @@ -228,10 +228,10 @@ def test_equal_sets_with_different_annotations_are_stored_once_with_both(tmp_pat def test_occurrences_that_differ_only_in_occurrence_fields_are_all_stored(tmp_path): - first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage", confidence=0.5),)) + first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage", confidence=0.5),)) first.annotate(Annotation(key="name", value="stage", occurrence_metadata={"k": 1})) second = LabelSet( - labels=STAGE_LABELS.labels, + STAGE_LABELS.labels, annotations=( Annotation(key="name", value="stage", confidence=0.5), Annotation(key="name", value="stage", source="rater", confidence=0.9), @@ -248,8 +248,8 @@ def test_occurrences_that_differ_only_in_occurrence_fields_are_all_stored(tmp_pa def test_writing_does_not_change_the_callers_label_sets(tmp_path): - first = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) - second = LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="source", value="manual"),)) + first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) + second = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="source", value="manual"),)) first_annotations, second_annotations = first.annotations, second.annotations _write_sets(tmp_path, first, second) @@ -261,8 +261,8 @@ def test_writing_does_not_change_the_callers_label_sets(tmp_path): def test_equal_annotations_of_two_sets_share_one_content_row(tmp_path): path = _write_sets( tmp_path, - LabelSet(labels=STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)), - LabelSet(labels=FINDINGS.labels, annotations=(Annotation(key="name", value="stage"),)), + LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)), + LabelSet(FINDINGS.labels, annotations=(Annotation(key="name", value="stage"),)), ) with connect_control(path, read_only=True) as connection: @@ -276,7 +276,7 @@ def test_the_writer_rejects_a_label_set_changed_after_add_task(tmp_path): dataset, record = _dataset() second = ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS) dataset.add_tasks(tasks=[ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), second]) - second.label_set = LabelSet(labels=(Label(name="X"),)) + second.label_set = LabelSet(["X"]) with pytest.raises(TimeFValidationError, match=r"ClassificationTask 2: labels not in label set \['X'\]"): _write(dataset, tmp_path / "control.duckdb") @@ -291,7 +291,7 @@ def test_the_writer_rejects_a_label_set_changed_after_add_task(tmp_path): ], ) def test_the_writer_checks_a_label_set_changed_after_it_was_built(tmp_path, field, value, message): - label_set = LabelSet(labels=(Label(name="W"),)) + label_set = LabelSet(["W"]) dataset, record = _dataset() dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=label_set)) setattr(label_set, field, value) diff --git a/packages/timenet/tests/registry/conftest.py b/packages/timenet/tests/registry/conftest.py index 0fbf47810..65ca63d2a 100644 --- a/packages/timenet/tests/registry/conftest.py +++ b/packages/timenet/tests/registry/conftest.py @@ -12,7 +12,6 @@ ClassificationTask, DatasetMetadata, Domain, - Label, LabelSet, License, TimeSeriesSpec, @@ -22,12 +21,7 @@ from timenet.writer import TimeFWriter -RHYTHM = LabelSet( - labels=( - Label(name="afib"), - Label(name="normal"), - ), -) +RHYTHM = LabelSet(["afib", "normal"]) def _ecg_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/registry/test_conformance.py b/packages/timenet/tests/registry/test_conformance.py index 6bb546928..7214ef099 100644 --- a/packages/timenet/tests/registry/test_conformance.py +++ b/packages/timenet/tests/registry/test_conformance.py @@ -24,7 +24,6 @@ ClassificationTask, DatasetMetadata, Domain, - Label, LabelSet, License, TimeSeriesSpec, @@ -34,12 +33,7 @@ from timenet.types.references import DatasetRef -RHYTHM = LabelSet( - labels=( - Label(name="afib"), - Label(name="normal"), - ), -) +RHYTHM = LabelSet(["afib", "normal"]) def _second_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/registry/test_namespaced.py b/packages/timenet/tests/registry/test_namespaced.py index 946790cc6..a3b86eb0b 100644 --- a/packages/timenet/tests/registry/test_namespaced.py +++ b/packages/timenet/tests/registry/test_namespaced.py @@ -6,7 +6,6 @@ from timenet.types import ( ClassificationTask, DatasetMetadata, - Label, LabelSet, License, TimeSeriesSpec, @@ -16,7 +15,7 @@ from timenet.writer import TimeFWriter -LABELS = LabelSet(labels=(Label(name="x"),)) +LABELS = LabelSet(["x"]) def _namespaced_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index e47a20039..ab8a1d095 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -6,8 +6,8 @@ from timenet.types import Annotation, AnswerTask, ClassificationTask, Label, LabelSet, TimePoint -STAGES = LabelSet(labels=(Label(name="W", description="Awake."), Label(name="N1"), Label(name="R"))) -FINDINGS = LabelSet(labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), multi_label=True) +STAGES = LabelSet([Label(name="W", description="Awake."), "N1", "R"]) +FINDINGS = LabelSet(["AF", "PVC", "LBBB"], multi_label=True) @pytest.mark.parametrize( @@ -65,13 +65,13 @@ def test_a_multi_label_target_rejects_a_repeated_label(): def test_equality_ignores_label_order_and_annotations_but_not_descriptions_or_multi_label(): - first = LabelSet(labels=(Label(name="F"), Label(name="M")), annotations=(Annotation(key="name", value="sex"),)) - second = LabelSet(labels=(Label(name="M"), Label(name="F"))) - described = LabelSet(labels=(Label(name="F", description="Female."), Label(name="M"))) - multi = LabelSet(labels=(Label(name="F"), Label(name="M")), multi_label=True) + first = LabelSet(["F", "M"], annotations=(Annotation(key="name", value="sex"),)) + second = LabelSet(["M", "F"]) + described = LabelSet([Label(name="F", description="Female."), "M"]) + multi = LabelSet(["F", "M"], multi_label=True) assert first._content_key() == second._content_key() assert len({first._content_key(), described._content_key(), multi._content_key()}) == 3 - assert first == LabelSet(labels=(Label(name="F"), Label(name="M"))) + assert first == LabelSet(["F", "M"]) assert first == second assert first != described assert first != multi @@ -91,17 +91,17 @@ def test_an_ordered_set_uses_its_label_order_for_equality(): def test_annotate_attaches_an_occurrence_and_rejects_a_span(): annotation = Annotation(key="name", value="sex") - built = LabelSet(labels=(Label(name="F"),), annotations=(annotation,)) + built = LabelSet(["F"], annotations=(annotation,)) assert built.annotations == (annotation,) assert built.annotations[0] is not annotation - label_set = LabelSet(labels=(Label(name="F"),)) + label_set = LabelSet(["F"]) attached = label_set.annotate(annotation) assert label_set.annotations == (attached,) assert attached is not annotation with pytest.raises(TimeFValidationError, match="has a span"): label_set.annotate(Annotation(key="x", span=TimePoint.seconds(1.0))) with pytest.raises(TimeFValidationError, match="has a span"): - LabelSet(labels=(Label(name="F"),), annotations=(Annotation(key="x", span=TimePoint.seconds(1.0)),)) + LabelSet(["F"], annotations=(Annotation(key="x", span=TimePoint.seconds(1.0)),)) def test_a_classification_task_walks_its_annotations_then_those_of_its_label_set(): @@ -116,7 +116,7 @@ def test_a_classification_task_walks_its_annotations_then_those_of_its_label_set def test_a_copied_task_gets_its_own_label_set(): - label_set = LabelSet(labels=(Label(name="W"),), annotations=(Annotation(key="name", value="stage"),)) + label_set = LabelSet(["W"], annotations=(Annotation(key="name", value="stage"),)) task = ClassificationTask(targets=("W",), label_set=label_set) copy = task.with_references({}, {}) assert copy.label_set == label_set @@ -129,7 +129,7 @@ def test_a_copied_task_gets_its_own_label_set(): def test_a_task_with_an_annotated_set_survives_a_pickle_round_trip(): label_set = LabelSet( - labels=(Label(name="W", description="Awake."),), + [Label(name="W", description="Awake.")], multi_label=True, annotations=(Annotation(key="name", value="stage"),), ) diff --git a/packages/timenet/tests/writer/test_id_encoding.py b/packages/timenet/tests/writer/test_id_encoding.py index 148f0f3c4..81de2645b 100644 --- a/packages/timenet/tests/writer/test_id_encoding.py +++ b/packages/timenet/tests/writer/test_id_encoding.py @@ -12,7 +12,6 @@ ClassificationTask, DatasetMetadata, ForecastingTask, - Label, LabelSet, License, StepInterval, @@ -27,7 +26,7 @@ from timenet.writer import TimeFWriter -LABELS = LabelSet(labels=(Label(name="x"),)) +LABELS = LabelSet(["x"]) def _spec(): diff --git a/packages/timenet/tests/writer/test_sharding.py b/packages/timenet/tests/writer/test_sharding.py index edbe605d2..1b43e56bc 100644 --- a/packages/timenet/tests/writer/test_sharding.py +++ b/packages/timenet/tests/writer/test_sharding.py @@ -16,7 +16,6 @@ Annotation, ClassificationTask, DatasetMetadata, - Label, LabelSet, License, TimeSeriesSpec, @@ -28,12 +27,7 @@ from timenet.writer import TimeFWriter -CLASSES = LabelSet( - labels=( - Label(name="c0"), - Label(name="c1"), - ), -) +CLASSES = LabelSet(["c0", "c1"]) # Small enough that a modest dataset shards every artifact type into several parts, tiny on disk. diff --git a/packages/timenet/tests/writer/test_writer.py b/packages/timenet/tests/writer/test_writer.py index adc9dca27..60a322f38 100644 --- a/packages/timenet/tests/writer/test_writer.py +++ b/packages/timenet/tests/writer/test_writer.py @@ -19,7 +19,6 @@ Annotation, ClassificationTask, DatasetMetadata, - Label, LabelSet, License, TimeSeriesSpec, @@ -31,7 +30,7 @@ from timenet.writer.value_encoding import ValueEncoding -LABELS = LabelSet(labels=(Label(name="x"),)) +LABELS = LabelSet(["x"]) def _written(tmp_path, dataset=None, **kwargs): From f49f1ede2adbd846e9d62f20671f240c42d68892 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:01:20 -0800 Subject: [PATCH 07/15] perf(format): read only the label sets that a task read needs A task read loaded every stored label set and all their annotations, once for each read. It now loads the sets that its tasks name. The audit keeps the check for an annotation of a missing label set. The docs explain ordered label sets once. --- docs/data-model/tasks.md | 11 ++--- .../src/timenet/format/control_reader.py | 41 +++++++++---------- .../tests/format/test_label_set_storage.py | 12 ++---- 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index f89eb85e5..0d648c293 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -236,10 +236,6 @@ If `multi_label=False`, the default, a target must contain exactly one label. If a target contains all labels that apply. Omitted labels do not apply. An empty target means that no label applies. -By default the order of labels does not matter. Set `ordered=True` when it does. Then the label order is -part of the identity of the set. Use it for ordinal labels, such as `low`, `medium` and `high`, or for -answer options whose order matters. - When the dataset adds or streams a task, it makes sure that the target follows the rules of its label set. Each label must be in the set. No label can repeat. A target with `multi_label=False` must contain exactly one label. @@ -251,9 +247,10 @@ A label set has no name. TimeF stores equal sets once. For this, two sets are eq same labels, label descriptions, `multi_label`, and `ordered`. The annotations do not matter. The order of labels matters only for an ordered set. `==` follows the same rule. -Use `ordered=True` when the label order has a meaning, for example for ordinal labels or for answer -options that a model sees in a fixed order. Then each order is its own set, and the stored set keeps -it. An unordered set keeps the label order of the first equal set that the writer meets. +By default the order of labels does not matter. Use `ordered=True` when it has a meaning, for example +for ordinal labels such as `low`, `medium` and `high`, or for answer options that a model sees in a +fixed order. Then each order is its own set, and the stored set keeps it. An unordered set keeps the +label order of the first equal set that the writer meets. The stored set gets one integer id, `label_set_id`. If equal sets carry different annotation occurrences, the stored set keeps each distinct occurrence. In one read, the reader gives tasks that diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index ddbca0e82..036896e49 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -23,7 +23,7 @@ from timenet.dataset.composition import ImportMap from timenet.errors import TimeFFormatError, TimeFNotFoundError, TimeFReaderError, TimeFValidationError from timenet.format.annotation_codec import decode_annotation_value -from timenet.format.control_schema import ANNOTATED_OBJECT_TYPES, LAST_ID_PREFIX, TABLES +from timenet.format.control_schema import ANNOTATED_OBJECT_TYPES, LAST_ID_PREFIX from timenet.format.duckdb import CATALOG_ERRORS, check_control_schema, connect_control from timenet.format.stored_array import ChunkLocation from timenet.format.stored_values import ( @@ -1853,8 +1853,8 @@ def _task_rows(self, task_ids: list[int] | None) -> list[_TaskRow]: _TaskRow._make, ) - def _read_label_sets(self) -> dict[int, LabelSet]: - """Read every stored label set with its annotations. + def _read_label_sets(self, label_set_ids: Collection[int]) -> dict[int, LabelSet]: + """Read the label sets with these ids, with their annotations. Returns: The label sets by id. @@ -1862,24 +1862,19 @@ def _read_label_sets(self) -> dict[int, LabelSet]: Raises: TimeFFormatError: If a stored label set or one of its annotations is not valid. """ - columns = ", ".join(TABLES["label_sets"].column_names) - # ruff: ignore[hardcoded-sql-expression] - fixed identifiers - rows = self._query_table(f"SELECT {columns} FROM label_sets").to_pylist() - annotations = self._read_annotations({ObjectKind.LABEL_SET: None}) - stored_ids = {row["label_set_id"] for row in rows} - for _, label_set_id in annotations: - if label_set_id not in stored_ids: - raise TimeFFormatError( - f"annotation occurrence refers to missing {ObjectKind.LABEL_SET} {label_set_id!r}" - ) + rows = self._query( + "SELECT label_set_id, labels, multi_label, ordered FROM label_sets " + "WHERE label_set_id IN (SELECT unnest(?))", + [sorted(label_set_ids)], + ) + annotations = self._read_annotations({ObjectKind.LABEL_SET: label_set_ids}) label_sets: dict[int, LabelSet] = {} - for row in rows: - label_set_id = row["label_set_id"] + for label_set_id, labels, multi_label, ordered in rows: try: label_sets[label_set_id] = LabelSet( - [Label.model_validate(label) for label in row["labels"]], - multi_label=row["multi_label"], - ordered=row["ordered"], + [Label.model_validate(label) for label in labels], + multi_label=multi_label, + ordered=ordered, annotations=annotations.get((ObjectKind.LABEL_SET, label_set_id), ()), ) except (ValidationError, TimeFValidationError) as exc: @@ -2301,7 +2296,7 @@ def __init__( # ruff: ignore[too-many-arguments] - one argument per task select self.dataset_occurrences: list[Annotation] = [] """The occurrences of the dataset itself that the scope read for its tasks' references.""" self.retained: _TasksById = {} - self._label_sets: dict[int, LabelSet] | None = None + self._label_sets: dict[int, LabelSet] = {} """The stored label sets by id, which the read builds once and all its tasks share.""" # The lookups of the caller's objects that a stream's batches share, or None for one pass. self._given: _Lookups | None = None @@ -2454,7 +2449,7 @@ def _build_batch(self, task_ids: list[EntityId] | None) -> list[Task]: def _label_set(self, task_id: int, label_set_id: int | None) -> LabelSet: """Return the stored label set that a classification task names. - The read builds every label set once. Tasks that use the same set share one object. + The read builds each label set once. Tasks that use the same set share one object. Returns: The shared label set. @@ -2465,8 +2460,6 @@ def _label_set(self, task_id: int, label_set_id: int | None) -> LabelSet: """ if label_set_id is None: raise TimeFFormatError(f"classification task {task_id!r} has no label set") - if self._label_sets is None: - self._label_sets = self.reader._read_label_sets() label_set = self._label_sets.get(label_set_id) if label_set is None: raise TimeFFormatError(f"classification task {task_id!r} refers to missing label set {label_set_id!r}") @@ -2509,6 +2502,10 @@ def _build(self, task_ids: list[EntityId] | None) -> _TasksById: if task_ids is None: reader._check_owners(task_annotations.keys(), {ObjectKind.TASK: {row.task_id for row in task_rows}}) + label_set_ids = {row.label_set_id for row in task_rows if row.label_set_id is not None} + label_set_ids -= self._label_sets.keys() + if label_set_ids: + self._label_sets.update(reader._read_label_sets(label_set_ids)) self._ensure_records( {record_id for refs in record_refs.values() for ids in refs.values() for record_id in ids} | {row.record_id for row in target_rows if row.record_id is not None} diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 2a5ba78e6..6447c6e3c 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -300,14 +300,6 @@ def test_the_writer_checks_a_label_set_changed_after_it_was_built(tmp_path, fiel _write(dataset, tmp_path / "control.duckdb") -def test_the_reader_rejects_an_annotation_of_a_missing_label_set(release_copy): - with connect_control(release_copy) as connection: - connection.execute("UPDATE annotation_occurrences SET object_id = 99 WHERE object_type = 'LabelSet'") - - with pytest.raises(TimeFFormatError, match="refers to missing LabelSet 99"): - _read_tasks(release_copy) - - def test_the_reader_rejects_a_task_that_names_a_missing_label_set(release_copy): with connect_control(release_copy) as connection: connection.execute("UPDATE tasks SET label_set_id = 99 WHERE task_id = 1") @@ -372,6 +364,10 @@ def test_a_stream_rejects_targets_that_do_not_fit_the_label_set(): ("UPDATE tasks SET label_set_id = 1 WHERE task_type = 'answer'", "tasks.label_set_id"), ("UPDATE tasks SET label_set_id = NULL WHERE task_id = 1", "tasks.label_set_id"), ("UPDATE tasks SET label_set_id = 99 WHERE task_id = 1", "label_sets.label_set_id"), + ( + "UPDATE annotation_occurrences SET object_id = 99 WHERE object_type = 'LabelSet'", + "refers to missing LabelSet 99", + ), ("UPDATE task_targets SET text_value = 'BOGUS' WHERE task_id = 1", "task 1 has a label that is not"), ( "INSERT INTO task_targets (task_id, position, target_kind, text_value) VALUES (1, 1, 'text', 'R')", From 0de54ee5c3fb8ddb612e206e10267fa162ca22d2 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:27:59 -0800 Subject: [PATCH 08/15] refactor(connectors): build label set name annotations positionally --- docs/data-model/annotations.md | 2 +- .../datasets/opentslm/har_cot/connector.py | 2 +- .../datasets/opentslm/sleep_cot/connector.py | 2 +- .../datasets/physionet/sleep_edfx/tasks.py | 6 +++--- .../datasets/timenet/hello_world/connector.py | 4 ++-- .../datasets/timenet/test_mean/connector.py | 2 +- .../datasets/tsaqa/tsaqa_benchmark/connector.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index fbb715672..851242d9e 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -126,7 +126,7 @@ from timenet.types import Annotation, LabelSet SLEEP_STAGE = LabelSet( ["W", "N1", "R"], - annotations=(Annotation(key="name", value="sleep_stage"),), + annotations=[Annotation("name", "sleep_stage")], ) SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) ``` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py index f8716bb7a..74007c0be 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py @@ -45,7 +45,7 @@ Label(name="walking_down", description="Walking down stairs."), Label(name="walking_up", description="Walking up stairs."), ], - annotations=(Annotation(key="name", value="activity"),), + annotations=[Annotation("name", "activity")], ) # The release states no unit. A resting window has a magnitude of about 9.8, which is gravity in m/s². ACCELERATION = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index a0b4f93e1..f438bddd3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -43,7 +43,7 @@ Label(name="N4", description="Non-REM stage 4"), Label(name="REM", description="REM sleep"), ], - annotations=(Annotation(key="name", value="sleep_stage"),), + annotations=[Annotation("name", "sleep_stage")], ) # OpenTSLM converted the microvolts of Sleep-EDF to volts and kept every second sample of 100 Hz. EEG = TimeSeriesSpec(spec_type="eeg", name="EEG", unit_value=ureg.volt, dtype="float64") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py index 970c7f0c4..b7648a704 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py @@ -52,7 +52,7 @@ Label(name="Movement time", description="An epoch that body movement obscures, so it has no stage."), Label(name="Sleep stage ?", description="An epoch with no sleep-stage score from data collection."), ], - annotations=(Annotation(key="name", value=str(AnnotationKey.SLEEP_STAGE)),), + annotations=[Annotation("name", str(AnnotationKey.SLEEP_STAGE))], ) STAGE_LABELS = SLEEP_STAGES.names @@ -63,7 +63,7 @@ # meanings, so a raw code never reaches a task. SEXES = LabelSet( [Label(name="F", description="Female."), Label(name="M", description="Male.")], - annotations=(Annotation(key="name", value=str(AnnotationKey.SEX)),), + annotations=[Annotation("name", str(AnnotationKey.SEX))], ) SEX_LABELS = SEXES.names @@ -71,7 +71,7 @@ CONDITION_LABELS = tuple(Condition) CONDITIONS = LabelSet( [str(one) for one in Condition], - annotations=(Annotation(key="name", value=str(AnnotationKey.CONDITION)),), + annotations=[Annotation("name", str(AnnotationKey.CONDITION))], ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py index 61ea10f24..6ada90658 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py @@ -50,11 +50,11 @@ # shares. _RHYTHM = LabelSet( [Label(name="normal", description="A regular wave."), "abnormal"], - annotations=(Annotation(key="name", value="rhythm"),), + annotations=[Annotation("name", "rhythm")], ) _EVENT = LabelSet( [Label(name="onset", description="The moment the wave starts."), "offset"], - annotations=(Annotation(key="name", value="event"),), + annotations=[Annotation("name", "event")], ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py index 84206c57c..87fc341ce 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py @@ -27,7 +27,7 @@ Label(name="above_zero", description="The mean of the signal is above zero."), Label(name="below_zero", description="The mean of the signal is below zero."), ], - annotations=(Annotation(key="name", value="mean_sign"),), + annotations=[Annotation("name", "mean_sign")], ) _SIGNAL = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index 5305ae07c..02ab007cf 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -75,7 +75,7 @@ class _Role(StrEnum): _TRUE_OR_FALSE_LABEL_SET = LabelSet( [Label(name="T", description="True."), Label(name="F", description="False.")], - annotations=(Annotation(key="name", value="true_or_false"),), + annotations=[Annotation("name", "true_or_false")], ) """The label set that every true-or-false task shares.""" _CONTEXT_KEYS = ("meta_info", "domain", "dataset") From 75189dfa27aaa07f1d4ed8c7c8ecc2e32237e4b5 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:53:58 -0800 Subject: [PATCH 09/15] refactor(types): reject a Python set for an ordered label set and tidy the label set code - LabelSet(..., ordered=True) rejects a set or frozenset, because its order is not fixed. - assert_datasets_equal also compares label set annotations. - The format doc says which owners a full read checks. - Smaller cleanups in the label set field, a docstring and two tests. --- docs/timef-format.md | 5 +++-- packages/timenet/src/timenet/format/control_reader.py | 3 +-- packages/timenet/src/timenet/testing/builders.py | 4 ++++ packages/timenet/src/timenet/types/label_sets.py | 8 +++++--- packages/timenet/tests/dataset/test_dataset.py | 2 +- packages/timenet/tests/format/test_label_set_storage.py | 9 +++++---- packages/timenet/tests/types/test_label_sets.py | 1 + 7 files changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/timef-format.md b/docs/timef-format.md index a8878ee87..b49177fea 100644 --- a/docs/timef-format.md +++ b/docs/timef-format.md @@ -176,8 +176,9 @@ An occurrence can annotate a Dataset, Task, Record, Source, Signal, or label set owner. For a Source, it is the storage ID of that row. `object_id` is `NULL` exactly when the owner is the Dataset. IDs are unique only within one kind, so the type is part of the reference. -A read of every Record or every Task rejects an occurrence with an unknown owner type or a missing -owner. It does not drop that occurrence. +A read of every Record or every Task rejects an occurrence with an unknown owner type. It also +rejects an occurrence whose Record, Source, Signal, or Task owner is missing. It does not drop that +occurrence. Only `audit_control_database()` reports an occurrence whose label set is missing. One content row can apply to many objects without copying a long value. Each attachment still has its own `occurrence_id` and temporal placement. diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index 036896e49..f18545149 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -2455,8 +2455,7 @@ def _label_set(self, task_id: int, label_set_id: int | None) -> LabelSet: The shared label set. Raises: - TimeFFormatError: If the task names no label set, no label set has this id, or a stored - label set is not valid. + TimeFFormatError: If the task names no label set, or no label set has this id. """ if label_set_id is None: raise TimeFFormatError(f"classification task {task_id!r} has no label set") diff --git a/packages/timenet/src/timenet/testing/builders.py b/packages/timenet/src/timenet/testing/builders.py index d76164c07..4ebc647e7 100644 --- a/packages/timenet/src/timenet/testing/builders.py +++ b/packages/timenet/src/timenet/testing/builders.py @@ -247,6 +247,10 @@ def assert_datasets_equal(expected: TimeFDataset, actual: TimeFDataset) -> None: assert replace(exp_task, scope=None, input_modalities=exp_task.resolved_input_modalities) == replace( act_task, scope=None, input_modalities=act_task.resolved_input_modalities ), f"task {task_id} differs" + if isinstance(exp_task, ClassificationTask) and isinstance(act_task, ClassificationTask): + assert exp_task.label_set.annotations == act_task.label_set.annotations, ( + f"task {task_id} label set annotations differ" + ) def _assert_sources_equal(record_id: EntityId, expected: Record, actual: Record) -> None: diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index b28e3f392..ea3a23651 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -1,7 +1,7 @@ """The allowed labels for a classification target.""" from collections.abc import Hashable, Iterable, Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TypeVar from timenet.errors import TimeFValidationError @@ -91,7 +91,7 @@ class LabelSet(SupportsAnnotate): """If ``True``, the label order is part of the identity of the set. If ``False``, the order does not matter. An ordered set needs labels in a fixed order, so pass a list or a tuple, not a Python set.""" - annotations: tuple[Annotation, ...] = field(default=(), compare=False) + annotations: tuple[Annotation, ...] = () """Annotations that describe the set, such as ``Annotation(key="name", value="sleep_stage")``. An annotation cannot have a span. The set holds its own occurrence of each annotation. @@ -115,7 +115,9 @@ def __init__( Raises: TimeFValidationError: If the set is not valid (see :meth:`_validate`). - """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it + """ + if ordered and isinstance(labels, (set, frozenset)): + raise TimeFValidationError("an ordered label set needs labels in a fixed order, got a set") self.labels = tuple(_as_label(label) for label in _as_tuple(labels, Label)) # ty: ignore[invalid-assignment] self.multi_label = multi_label self.ordered = ordered diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index be2724304..c34792922 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -202,7 +202,7 @@ def test_derive_schema_rejects_a_label_set_annotation_that_conflicts_with_a_reco label_set=LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)), ) ) - with pytest.raises(ValueError, match="conflicting descriptors"): + with pytest.raises(TimeFValidationError, match="conflicting descriptors"): ds.derive_schema() diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 6447c6e3c..75f4dc74a 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -156,9 +156,10 @@ def test_the_batches_of_a_task_stream_share_one_label_set_object(release): assert first is fourth -def _stored_sets(path): +def _stored_set_ids(path): with connect_control(path, read_only=True) as connection: - return connection.execute("SELECT label_set_id, labels, multi_label FROM label_sets ORDER BY 1").fetchall() + rows = connection.execute("SELECT label_set_id FROM label_sets ORDER BY label_set_id").fetchall() + return [row[0] for row in rows] def _write_sets(tmp_path, *label_sets): @@ -180,7 +181,7 @@ def test_sets_are_stored_once_per_content_and_keep_the_first_label_order(tmp_pat multi = LabelSet(STAGE_LABELS.labels, multi_label=True) path = _write_sets(tmp_path, STAGE_LABELS, reordered, described, multi) - assert [row[0] for row in _stored_sets(path)] == [1, 2, 3] + assert _stored_set_ids(path) == [1, 2, 3] first, second, *_ = _read_tasks(path) assert first.label_set is second.label_set assert first.label_set.names == ("W", "N1", "R") @@ -216,7 +217,7 @@ def test_equal_sets_with_different_annotations_are_stored_once_with_both(tmp_pat bare = LabelSet(STAGE_LABELS.labels) path = _write_sets(tmp_path, first, bare, second) - assert [row[0] for row in _stored_sets(path)] == [1] + assert _stored_set_ids(path) == [1] with connect_control(path, read_only=True) as connection: audit_control_database(connection, require_chunks=False) assert connection.execute( diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index ab8a1d095..506a270e4 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -27,6 +27,7 @@ ({"labels": (Label(name="W"),), "multi_label": 1}, "multi_label must be a bool, got int"), ({"labels": (Label(name="W"),), "ordered": 1}, "ordered must be a bool, got int"), ({"labels": (Label(name="W"),), "ordered": None}, "ordered must be a bool, got NoneType"), + ({"labels": {"W", "R"}, "ordered": True}, "needs labels in a fixed order, got a set"), ({"labels": (Label(name="W"),), "annotations": ("name",)}, "takes Annotation objects, got str"), ], ) From 1787dd05d4dd91355a37ce7b0a1802ed88e59c3f Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:40:01 -0800 Subject: [PATCH 10/15] refactor(connectors): pass the ARFBench task annotations to the constructor --- .../datasets/datadog/arfbench/questions.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py index ce42facad..b5cb9f953 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py @@ -214,7 +214,7 @@ def _create_task(self, inputs: list[Record], modality: InputModality) -> Classif Returns: The task, not yet added to a dataset. """ - task = ClassificationTask( + return ClassificationTask( inputs=inputs, prompt=self.question, targets=self.correct_answer, @@ -222,12 +222,13 @@ def _create_task(self, inputs: list[Record], modality: InputModality) -> Classif split=Split.TEST, metadata={"question_index": self.index}, label_set=LabelSet(self.answer_options, ordered=True), + annotations=[ + Annotation("task_category", value=self.task_category.value), + Annotation("difficulty", value=self.difficulty.value), + Annotation("interpolate_1", value=self.interpolate_1), + Annotation("interpolate_2", value=self.interpolate_2), + ], ) - task.annotate(Annotation("task_category", value=self.task_category.value)) - task.annotate(Annotation("difficulty", value=self.difficulty.value)) - task.annotate(Annotation("interpolate_1", value=self.interpolate_1)) - task.annotate(Annotation("interpolate_2", value=self.interpolate_2)) - return task def read_questions(qa_csv: Path) -> Iterator[ARFBenchQuestion]: From 5a0677fba2cad78b78bb03f17c268abee80fdbcc Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:44:58 -0800 Subject: [PATCH 11/15] refactor: pass the annotation value by keyword --- docs/data-model/annotations.md | 2 +- .../datasets/opentslm/har_cot/connector.py | 2 +- .../datasets/opentslm/sleep_cot/connector.py | 2 +- .../datasets/physionet/sleep_edfx/tasks.py | 6 +++--- .../datasets/timenet/hello_world/connector.py | 4 ++-- .../datasets/timenet/test_mean/connector.py | 2 +- .../datasets/tsaqa/tsaqa_benchmark/connector.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 851242d9e..97f65c3b9 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -126,7 +126,7 @@ from timenet.types import Annotation, LabelSet SLEEP_STAGE = LabelSet( ["W", "N1", "R"], - annotations=[Annotation("name", "sleep_stage")], + annotations=[Annotation("name", value="sleep_stage")], ) SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) ``` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py index 74007c0be..902330aa3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py @@ -45,7 +45,7 @@ Label(name="walking_down", description="Walking down stairs."), Label(name="walking_up", description="Walking up stairs."), ], - annotations=[Annotation("name", "activity")], + annotations=[Annotation("name", value="activity")], ) # The release states no unit. A resting window has a magnitude of about 9.8, which is gravity in m/s². ACCELERATION = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index f438bddd3..a93726eeb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -43,7 +43,7 @@ Label(name="N4", description="Non-REM stage 4"), Label(name="REM", description="REM sleep"), ], - annotations=[Annotation("name", "sleep_stage")], + annotations=[Annotation("name", value="sleep_stage")], ) # OpenTSLM converted the microvolts of Sleep-EDF to volts and kept every second sample of 100 Hz. EEG = TimeSeriesSpec(spec_type="eeg", name="EEG", unit_value=ureg.volt, dtype="float64") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py index b7648a704..50a097ad3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py @@ -52,7 +52,7 @@ Label(name="Movement time", description="An epoch that body movement obscures, so it has no stage."), Label(name="Sleep stage ?", description="An epoch with no sleep-stage score from data collection."), ], - annotations=[Annotation("name", str(AnnotationKey.SLEEP_STAGE))], + annotations=[Annotation("name", value=str(AnnotationKey.SLEEP_STAGE))], ) STAGE_LABELS = SLEEP_STAGES.names @@ -63,7 +63,7 @@ # meanings, so a raw code never reaches a task. SEXES = LabelSet( [Label(name="F", description="Female."), Label(name="M", description="Male.")], - annotations=[Annotation("name", str(AnnotationKey.SEX))], + annotations=[Annotation("name", value=str(AnnotationKey.SEX))], ) SEX_LABELS = SEXES.names @@ -71,7 +71,7 @@ CONDITION_LABELS = tuple(Condition) CONDITIONS = LabelSet( [str(one) for one in Condition], - annotations=[Annotation("name", str(AnnotationKey.CONDITION))], + annotations=[Annotation("name", value=str(AnnotationKey.CONDITION))], ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py index 6ada90658..4981fa0bb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py @@ -50,11 +50,11 @@ # shares. _RHYTHM = LabelSet( [Label(name="normal", description="A regular wave."), "abnormal"], - annotations=[Annotation("name", "rhythm")], + annotations=[Annotation("name", value="rhythm")], ) _EVENT = LabelSet( [Label(name="onset", description="The moment the wave starts."), "offset"], - annotations=[Annotation("name", "event")], + annotations=[Annotation("name", value="event")], ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py index 87fc341ce..358f8f4f1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py @@ -27,7 +27,7 @@ Label(name="above_zero", description="The mean of the signal is above zero."), Label(name="below_zero", description="The mean of the signal is below zero."), ], - annotations=[Annotation("name", "mean_sign")], + annotations=[Annotation("name", value="mean_sign")], ) _SIGNAL = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index 02ab007cf..97c09db23 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -75,7 +75,7 @@ class _Role(StrEnum): _TRUE_OR_FALSE_LABEL_SET = LabelSet( [Label(name="T", description="True."), Label(name="F", description="False.")], - annotations=[Annotation("name", "true_or_false")], + annotations=[Annotation("name", value="true_or_false")], ) """The label set that every true-or-false task shares.""" _CONTEXT_KEYS = ("meta_info", "domain", "dataset") From a0c77449f2d4ae863afa6a5f08eb8e711a8907ff Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:46:11 -0800 Subject: [PATCH 12/15] feat(types)!: make the annotations part of the identity of a label set Two label sets with the same labels and different annotations are two sets, and the writer stores each one with its own annotations. Sets with the same labels and the same annotations, in any order, are still stored once. The writer no longer merges the annotations of equal sets. --- docs/data-model/annotations.md | 5 +- docs/data-model/tasks.md | 17 ++- docs/timef-format.md | 7 +- .../src/timenet/format/control_schema.py | 4 +- .../src/timenet/format/control_writer.py | 47 ++----- .../timenet/src/timenet/testing/builders.py | 4 - .../timenet/src/timenet/types/label_sets.py | 46 +++++-- .../tests/format/test_label_set_storage.py | 125 ++++++++++++------ .../timenet/tests/types/test_label_sets.py | 39 +++++- 9 files changed, 190 insertions(+), 104 deletions(-) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 97f65c3b9..6407dcafa 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -131,8 +131,9 @@ SLEEP_STAGE = LabelSet( SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) ``` -A label set annotation has no span. Annotations are not part of the identity of a set. TimeF stores -equal label sets once. If equal sets carry different annotations, the stored set keeps all of them. +A label set annotation has no span. Annotations are part of the identity of a set. Two sets with the +same labels and different annotations are two sets, and TimeF stores each of them. TimeF stores sets +with the same labels and the same annotations once. ## Valid values diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index 0d648c293..ada7e2524 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -244,17 +244,24 @@ Labels come from the targets stored directly on the task or from the text of its annotations. A label set has no name. TimeF stores equal sets once. For this, two sets are equal if they have the -same labels, label descriptions, `multi_label`, and `ordered`. The annotations do not matter. The -order of labels matters only for an ordered set. `==` follows the same rule. +same labels, label descriptions, `multi_label`, `ordered`, and annotations. The order of labels +matters only for an ordered set. The order of annotations does not matter. `==` follows the same rule. + +Annotations are equal if all their fields are equal, the value included. So these two sets are two +stored sets, each with its own id: + +```python +LabelSet(["yes", "no"], annotations=[Annotation("name", value="afib_present")]) +LabelSet(["yes", "no"], annotations=[Annotation("name", value="signal_noisy")]) +``` By default the order of labels does not matter. Use `ordered=True` when it has a meaning, for example for ordinal labels such as `low`, `medium` and `high`, or for answer options that a model sees in a fixed order. Then each order is its own set, and the stored set keeps it. An unordered set keeps the label order of the first equal set that the writer meets. -The stored set gets one integer id, `label_set_id`. If equal sets carry different annotation -occurrences, the stored set keeps each distinct occurrence. In one read, the reader gives tasks that -use the same set a shared object. +The stored set gets one integer id, `label_set_id`. In one read, the reader gives tasks that use the +same set a shared object. ### Answer and rationale diff --git a/docs/timef-format.md b/docs/timef-format.md index b49177fea..8ca38265f 100644 --- a/docs/timef-format.md +++ b/docs/timef-format.md @@ -194,9 +194,10 @@ A column that a task type does not use is `NULL`. SQL can filter these columns w Every classification task has a label set, and `tasks.label_set_id` stores its ID. Each row in `label_sets` stores one set with the fields `label_set_id`, `labels`, `multi_label`, and `ordered`. -Sets with the same labels, label descriptions, `multi_label`, and `ordered` are one set, so a release -stores them once. If `ordered` is true, the label order is part of the set, so the same labels in -another order are a different set. The writer numbers the sets in the order that tasks first use them. +Sets with the same labels, label descriptions, `multi_label`, `ordered`, and annotations are one set, +so a release stores them once. If `ordered` is true, the label order is part of the set, so the same +labels in another order are a different set. The same labels with different annotations are also a +different set. The writer numbers the sets in the order that tasks first use them. The annotations of a set are rows of `annotation_occurrences` with `object_type` `LabelSet` and the `label_set_id` as `object_id`. diff --git a/packages/timenet/src/timenet/format/control_schema.py b/packages/timenet/src/timenet/format/control_schema.py index 882920a99..48414ae1c 100644 --- a/packages/timenet/src/timenet/format/control_schema.py +++ b/packages/timenet/src/timenet/format/control_schema.py @@ -299,7 +299,9 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column The ``labels`` column holds one ``{name, description}`` entry per label, in label order. ``ordered`` says whether that order is part of the identity of the set. If a label has no description, ``description`` is NULL. Tasks refer to a set by its id in -``tasks.label_set_id``. The writer gives each distinct set one id, in the order that tasks first use it.""" +``tasks.label_set_id``. The writer gives each distinct set one id, in the order that tasks first use it. +The annotations of a set are part of its identity, so two sets with the same labels and different +annotations are two rows.""" TASKS = Table( "tasks", diff --git a/packages/timenet/src/timenet/format/control_writer.py b/packages/timenet/src/timenet/format/control_writer.py index 92f6d78d3..1ba8bdbe0 100644 --- a/packages/timenet/src/timenet/format/control_writer.py +++ b/packages/timenet/src/timenet/format/control_writer.py @@ -8,7 +8,7 @@ import json from pathlib import Path from typing import TYPE_CHECKING, Any, NamedTuple, assert_never, cast -from weakref import WeakValueDictionary, ref +from weakref import WeakValueDictionary import duckdb import numpy as np @@ -212,9 +212,7 @@ class _LabelSetRows: """The ``label_sets`` rows of one write, one row for each distinct label set. Equal sets (see :meth:`LabelSet._content_key`) get one id, in the order that tasks first use them. - The row keeps the labels of the first set. The stored annotations are all the occurrences of the - first set, then each occurrence of a later equal set that differs from every stored one in a field. - A set object that many tasks share is checked and added once. The caller's sets do not change. + The row keeps the labels and the annotations of the first set. """ def __init__(self, ids: Iterator[int]) -> None: @@ -224,11 +222,10 @@ def __init__(self, ids: Iterator[int]) -> None: ids: The storage ids for new label sets. """ self._new_ids = ids - self._sets: dict[Hashable, tuple[dict[str, object], list[Annotation], set[Hashable]]] = {} - self._known: dict[int, tuple[ref[LabelSet], int]] = {} + self._sets: dict[Hashable, tuple[dict[str, object], tuple[Annotation, ...]]] = {} def register(self, label_set: LabelSet) -> int: - """Add a label set and its annotations. + """Add a label set. Returns: The stored id of the set. @@ -236,9 +233,6 @@ def register(self, label_set: LabelSet) -> int: Raises: TimeFValidationError: If the set is not valid. A set can change after a task gets it. """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it - known = self._known.get(id(label_set)) - if known is not None and known[0]() is label_set: - return known[1] label_set._validate() key = label_set._content_key() stored = self._sets.get(key) @@ -249,19 +243,8 @@ def register(self, label_set: LabelSet) -> int: "multi_label": label_set.multi_label, "ordered": label_set.ordered, } - occurrences = list(label_set.annotations) - self._sets[key] = (row, occurrences, {_occurrence_key(annotation) for annotation in occurrences}) - else: - row, occurrences, held = stored - for annotation in label_set.annotations: - occurrence = _occurrence_key(annotation) - if occurrence not in held: - held.add(occurrence) - occurrences.append(annotation) - label_set_id = cast("int", row["label_set_id"]) - object_id = id(label_set) - self._known[object_id] = (ref(label_set, lambda _: self._known.pop(object_id, None)), label_set_id) - return label_set_id + stored = self._sets[key] = (row, label_set.annotations) + return cast("int", stored[0]["label_set_id"]) def write( self, connection: duckdb.DuckDBPyConnection, annotations: list[tuple[ObjectKind, int | None, Annotation]] @@ -272,26 +255,12 @@ def write( connection: The open write transaction. annotations: The annotation triples that the writer collects. This method adds to them. """ - insert_rows(connection, TABLES["label_sets"], [row for row, _, _ in self._sets.values()]) - for row, occurrences, _ in self._sets.values(): + insert_rows(connection, TABLES["label_sets"], [row for row, _ in self._sets.values()]) + for row, occurrences in self._sets.values(): label_set_id = cast("int", row["label_set_id"]) annotations.extend((ObjectKind.LABEL_SET, label_set_id, annotation) for annotation in occurrences) -def _occurrence_key(annotation: Annotation) -> tuple[Hashable, ...]: - """Return a key that is equal for two occurrences only if every field of them is equal. - - Returns: - The hashable key. - """ - return ( - annotation._content_key, - annotation.source, - annotation.confidence, - json.dumps(annotation.occurrence_metadata, sort_keys=True), - ) - - class DuckDBControlWriter: """Serialize the relational part of one immutable TimeF version.""" diff --git a/packages/timenet/src/timenet/testing/builders.py b/packages/timenet/src/timenet/testing/builders.py index 4ebc647e7..d76164c07 100644 --- a/packages/timenet/src/timenet/testing/builders.py +++ b/packages/timenet/src/timenet/testing/builders.py @@ -247,10 +247,6 @@ def assert_datasets_equal(expected: TimeFDataset, actual: TimeFDataset) -> None: assert replace(exp_task, scope=None, input_modalities=exp_task.resolved_input_modalities) == replace( act_task, scope=None, input_modalities=act_task.resolved_input_modalities ), f"task {task_id} differs" - if isinstance(exp_task, ClassificationTask) and isinstance(act_task, ClassificationTask): - assert exp_task.label_set.annotations == act_task.label_set.annotations, ( - f"task {task_id} label set annotations differ" - ) def _assert_sources_equal(record_id: EntityId, expected: Record, actual: Record) -> None: diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index ea3a23651..83fd87c5a 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -1,7 +1,9 @@ """The allowed labels for a classification target.""" +from collections import Counter from collections.abc import Hashable, Iterable, Mapping, Sequence from dataclasses import dataclass +import json from typing import TypeVar from timenet.errors import TimeFValidationError @@ -72,11 +74,15 @@ class LabelSet(SupportsAnnotate): A plain string is a label without a description, so ``LabelSet(["yes", "no"])`` is enough when no label needs one. - A label set has no name. Two sets with the same labels (with the same descriptions), the same - ``multi_label`` and the same ``ordered`` are the same set, so ``==`` is true for them. If ``ordered`` is - ``False``, the labels can be in any order. If ``ordered`` is ``True``, the labels must be in the same order. - A dataset stores equal sets once, with the label order of the first of them that it writes. - Annotations are not part of that identity. + A label set has no name. Two sets are the same set if they have the same labels (with the same + descriptions), the same ``multi_label`` and the same ``ordered``. They must also have the same annotations + (with the same values). Then ``==`` is true for them. If ``ordered`` is ``False``, the labels can be in any + order. If ``ordered`` is ``True``, the labels must be in the same order. The annotations can be in any order. + A dataset stores equal sets once, with the label order and the annotation order of the first of them that + it writes. Two sets with the same labels and different annotations are two sets:: + + LabelSet(["yes", "no"], annotations=(Annotation(key="name", value="afib_present"),)) + LabelSet(["yes", "no"], annotations=(Annotation(key="name", value="signal_noisy"),)) """ labels: tuple[Label, ...] @@ -95,7 +101,7 @@ class LabelSet(SupportsAnnotate): """Annotations that describe the set, such as ``Annotation(key="name", value="sleep_stage")``. An annotation cannot have a span. The set holds its own occurrence of each annotation. - When a dataset holds several equal sets, it stores each distinct annotation occurrence of them once.""" + The annotations are part of the identity of the set.""" def __init__( self, @@ -111,7 +117,7 @@ def __init__( labels: The allowed labels in order. A plain string is a label without a description. multi_label: Whether a target can hold more than one label. ordered: Whether the label order is part of the identity of the set. - annotations: Annotations that describe the set. + annotations: Annotations that describe the set. They are part of the identity of the set. Raises: TimeFValidationError: If the set is not valid (see :meth:`_validate`). @@ -193,13 +199,20 @@ def __eq__(self, other: object) -> bool: def _content_key(self) -> tuple[Hashable, ...]: """Return the key that makes two sets equal. - The key ignores annotations. It ignores label order unless the set is ordered. + The key ignores label order unless the set is ordered. It ignores the order of the annotations. + It compares every field of each annotation. Returns: The hashable key. """ labels = ((label.name, label.description) for label in self.labels) - return (self.multi_label, self.ordered, tuple(labels) if self.ordered else frozenset(labels)) + annotations = Counter(_occurrence_key(annotation) for annotation in self.annotations) + return ( + self.multi_label, + self.ordered, + tuple(labels) if self.ordered else frozenset(labels), + frozenset(annotations.items()), + ) def check_targets(self, targets: Sequence[str]) -> None: """Make sure that the target labels follow the rules of this set. @@ -227,3 +240,18 @@ def check_targets(self, targets: Sequence[str]) -> None: repeated = sorted({label for label in targets if targets.count(label) > 1}) if repeated: raise TimeFValidationError(f"repeated labels of {self}: {repeated}") + + +def _occurrence_key(annotation: Annotation) -> tuple[Hashable, ...]: + """Return a key that is equal for two occurrences if every field of them is equal. + + Returns: + The hashable key. + """ + confidence = annotation.confidence + return ( + annotation._content_key, + annotation.source, + float.hex(confidence) if isinstance(confidence, float) else confidence, + json.dumps(annotation.occurrence_metadata, sort_keys=True), + ) diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 75f4dc74a..099fe7980 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -9,10 +9,13 @@ from timenet.errors import TimeFFormatError, TimeFValidationError from timenet.format.control_audit import audit_control_database from timenet.format.control_reader import DuckDBControlReader -from timenet.format.control_writer import DuckDBControlWriter +from timenet.format.control_writer import DuckDBControlWriter, _LabelSetRows from timenet.format.duckdb import connect_control from timenet.format.storage import StorageSources from timenet.format.stored_values import PackedValueWriter +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing.builders import assert_datasets_equal from timenet.types import ( Annotation, AnswerTask, @@ -176,7 +179,7 @@ def _write_sets(tmp_path, *label_sets): def test_sets_are_stored_once_per_content_and_keep_the_first_label_order(tmp_path): - reordered = LabelSet(STAGE_LABELS.labels[::-1]) + reordered = LabelSet(STAGE_LABELS.labels[::-1], annotations=STAGE_LABELS.annotations) described = LabelSet([Label(name="W", description="Awake."), "N1", "R"]) multi = LabelSet(STAGE_LABELS.labels, multi_label=True) path = _write_sets(tmp_path, STAGE_LABELS, reordered, described, multi) @@ -208,55 +211,101 @@ def test_ordered_sets_in_another_order_are_stored_as_two_rows_in_their_own_order assert fourth == unordered and not fourth.ordered -def test_equal_sets_with_different_annotations_are_stored_once_with_both(tmp_path): - first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) - second = LabelSet( - STAGE_LABELS.labels, - annotations=(Annotation(key="name", value="stage"), Annotation(key="source", value="manual")), - ) - bare = LabelSet(STAGE_LABELS.labels) - path = _write_sets(tmp_path, first, bare, second) +def _names(label_set): + return [annotation.value for annotation in label_set.annotations if annotation.key == "name"] - assert _stored_set_ids(path) == [1] + +def test_sets_with_the_same_labels_and_different_names_are_stored_as_two_rows(tmp_path): + afib = LabelSet(["yes", "no"], annotations=(Annotation("name", value="afib_present"),)) + noisy = LabelSet(["yes", "no"], annotations=(Annotation("name", value="signal_noisy"),)) + path = _write_sets(tmp_path, afib, noisy, LabelSet(["yes", "no"])) + + assert _stored_set_ids(path) == [1, 2, 3] with connect_control(path, read_only=True) as connection: audit_control_database(connection, require_chunks=False) + assert connection.execute( + "SELECT object_id FROM annotation_occurrences WHERE object_type = 'LabelSet' ORDER BY 1" + ).fetchall() == [(1,), (2,)] + first, second, bare = _label_sets(_read_tasks(path)) + assert _names(first) == ["afib_present"] + assert _names(second) == ["signal_noisy"] + assert bare.annotations == () + assert (first, second, bare) == (afib, noisy, LabelSet(["yes", "no"])) + + +def test_sets_with_the_same_labels_and_the_same_annotations_are_stored_once(tmp_path): + name = Annotation("name", value="stage") + note = Annotation("note", value="manual", source="rater", confidence=0.9, occurrence_metadata={"k": 1}) + first = LabelSet(STAGE_LABELS.labels, annotations=(name, note)) + second = LabelSet(STAGE_LABELS.labels[::-1], annotations=(note, name)) + path = _write_sets(tmp_path, first, second) + + assert _stored_set_ids(path) == [1] + with connect_control(path, read_only=True) as connection: assert connection.execute( "SELECT count(*) FROM annotation_occurrences WHERE object_type = 'LabelSet'" ).fetchone() == (2,) tasks = _read_tasks(path) - assert tasks[0].label_set is tasks[1].label_set is tasks[2].label_set - assert [(a.key, a.value) for a in tasks[0].label_set.annotations] == [("name", "stage"), ("source", "manual")] - - -def test_occurrences_that_differ_only_in_occurrence_fields_are_all_stored(tmp_path): - first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage", confidence=0.5),)) - first.annotate(Annotation(key="name", value="stage", occurrence_metadata={"k": 1})) - second = LabelSet( - STAGE_LABELS.labels, - annotations=( - Annotation(key="name", value="stage", confidence=0.5), - Annotation(key="name", value="stage", source="rater", confidence=0.9), - ), - ) + assert tasks[0].label_set is tasks[1].label_set + assert [(a.key, a.source, a.confidence, a.occurrence_metadata) for a in tasks[0].label_set.annotations] == [ + ("name", None, None, {}), + ("note", "rater", 0.9, {"k": 1}), + ] + + +@pytest.mark.parametrize( + "changes", + [{"source": "rater"}, {"confidence": 0.9}, {"occurrence_metadata": {"k": 1}}], +) +def test_sets_whose_annotations_differ_in_one_occurrence_field_are_stored_as_two_rows(tmp_path, changes): + first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation("name", value="stage"),)) + second = LabelSet(STAGE_LABELS.labels, annotations=(Annotation("name", value="stage", **changes),)) path = _write_sets(tmp_path, first, second) - (label_set, _) = _label_sets(_read_tasks(path)) - assert [(a.source, a.confidence, a.occurrence_metadata) for a in label_set.annotations] == [ - (None, 0.5, {}), - (None, None, {"k": 1}), - ("rater", 0.9, {}), - ] + assert _stored_set_ids(path) == [1, 2] + with connect_control(path, read_only=True) as connection: + assert connection.execute("SELECT count(*) FROM annotation_contents").fetchone() == (1,) + + +def _round_trip(dataset, tmp_path): + with TimeFReader(open_local_version(dataset.write(path=tmp_path / "out"))) as reader: + return reader.read() -def test_writing_does_not_change_the_callers_label_sets(tmp_path): - first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)) - second = LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="source", value="manual"),)) - first_annotations, second_annotations = first.annotations, second.annotations +def test_sets_with_their_annotations_in_another_order_round_trip_as_equal(tmp_path): + name = Annotation("name", value="stage") + note = Annotation("note", value="manual") + dataset, record = _dataset() + dataset.add_tasks( + tasks=[ + ClassificationTask(inputs=(record,), targets=("W",), label_set=LabelSet(["W"], annotations=(name, note))), + ClassificationTask(inputs=(record,), targets=("W",), label_set=LabelSet(["W"], annotations=(note, name))), + ] + ) - _write_sets(tmp_path, first, second) + assert_datasets_equal(dataset, _round_trip(dataset, tmp_path)) + + +def test_sets_with_a_nan_confidence_are_stored_once(tmp_path): + def nan_named(): + return LabelSet(["W"], annotations=(Annotation("name", value="stage", confidence=float("nan")),)) + + path = _write_sets(tmp_path, nan_named(), nan_named()) + + assert _stored_set_ids(path) == [1] + first, second = _label_sets(_read_tasks(path)) + assert first is second + assert first == nan_named() + + +def test_a_set_annotated_after_a_task_uses_it_is_another_set(): + label_set = LabelSet(["W"]) + rows = _LabelSetRows(iter((1, 2))) - assert first.annotations is first_annotations - assert second.annotations is second_annotations + assert rows.register(label_set) == 1 + label_set.annotate(Annotation("name", value="stage")) + assert rows.register(label_set) == 2 + assert rows.register(LabelSet(["W"])) == 1 def test_equal_annotations_of_two_sets_share_one_content_row(tmp_path): diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index 506a270e4..2af5890ff 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -65,19 +65,52 @@ def test_a_multi_label_target_rejects_a_repeated_label(): FINDINGS.check_targets(("AF", "PVC", "AF")) -def test_equality_ignores_label_order_and_annotations_but_not_descriptions_or_multi_label(): - first = LabelSet(["F", "M"], annotations=(Annotation(key="name", value="sex"),)) +def test_equality_ignores_label_order_but_not_descriptions_or_multi_label(): + first = LabelSet(["F", "M"]) second = LabelSet(["M", "F"]) described = LabelSet([Label(name="F", description="Female."), "M"]) multi = LabelSet(["F", "M"], multi_label=True) assert first._content_key() == second._content_key() assert len({first._content_key(), described._content_key(), multi._content_key()}) == 3 - assert first == LabelSet(["F", "M"]) assert first == second assert first != described assert first != multi +def _named(*annotations): + return LabelSet(["yes", "no"], annotations=annotations) + + +def test_equality_compares_the_annotations_and_ignores_their_order(): + name = Annotation(key="name", value="afib_present") + note = Annotation(key="note", value="manual") + assert _named(name) == _named(Annotation(key="name", value="afib_present")) + assert _named(name)._content_key() == _named(name)._content_key() + assert _named(name, note) == _named(note, name) + assert _named(name) != _named(Annotation(key="name", value="signal_noisy")) + assert _named(name) != _named(Annotation(key="title", value="afib_present")) + assert _named(name) != _named() + assert _named(name) != _named(name, note) + assert _named(name) != _named(name, name) + + +@pytest.mark.parametrize( + "changes", + [{"source": "rater"}, {"confidence": 0.9}, {"occurrence_metadata": {"k": 1}}, {"description": "The name."}], +) +def test_equality_compares_every_field_of_an_annotation(changes): + assert _named(Annotation(key="name", value="x")) != _named(Annotation(key="name", value="x", **changes)) + assert _named(Annotation(key="name", value="x", **changes)) == _named(Annotation(key="name", value="x", **changes)) + + +def test_sets_with_a_nan_confidence_are_equal(): + def nan_named(): + return _named(Annotation(key="name", value="x", confidence=float("nan"))) + + assert nan_named() == nan_named() + assert nan_named()._content_key() == nan_named()._content_key() + + def test_an_ordered_set_uses_its_label_order_for_equality(): low_high = LabelSet(["low", "high"], ordered=True) high_low = LabelSet(["high", "low"], ordered=True) From c8fe8719fac4ae324313120bea816f9ba2466fc4 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:47:34 -0800 Subject: [PATCH 13/15] refactor(types): use the shared as_tuple in LabelSet --- .../timenet/src/timenet/types/label_sets.py | 27 +++---------------- .../timenet/tests/types/test_label_sets.py | 15 ++++++----- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index 83fd87c5a..a7ffd95b0 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -1,12 +1,12 @@ """The allowed labels for a classification target.""" from collections import Counter -from collections.abc import Hashable, Iterable, Mapping, Sequence +from collections.abc import Hashable, Iterable, Sequence from dataclasses import dataclass import json -from typing import TypeVar from timenet.errors import TimeFValidationError +from timenet.types._fields import as_tuple from timenet.types._model import TimeFModel from timenet.types.annotations import Annotation, SupportsAnnotate from timenet.types.wire import NonEmptyString @@ -21,25 +21,6 @@ class Label(TimeFModel): """What the label means, such as ``The subject is awake.``.""" -_Item = TypeVar("_Item") - - -def _as_tuple(value: object, item_type: type[_Item]) -> tuple[_Item, ...]: - """Return the items of a label set field as a tuple. - - Returns: - The items. :meth:`LabelSet._validate` checks their type. - - Raises: - TimeFValidationError: If the value is one item, a string, a mapping, or not iterable. - """ - if isinstance(value, (item_type, str, Mapping)) or not isinstance(value, Iterable): - raise TimeFValidationError( - f"a label set takes a list of {item_type.__name__} items, got {type(value).__name__} {value!r}" - ) - return tuple(value) - - def _as_label(item: object) -> object: """Turn a plain string into a label without a description. Leave every other item as it is. @@ -124,10 +105,10 @@ def __init__( """ if ordered and isinstance(labels, (set, frozenset)): raise TimeFValidationError("an ordered label set needs labels in a fixed order, got a set") - self.labels = tuple(_as_label(label) for label in _as_tuple(labels, Label)) # ty: ignore[invalid-assignment] + self.labels = tuple(_as_label(label) for label in as_tuple(labels, item_type=Label)) # ty: ignore[invalid-assignment] self.multi_label = multi_label self.ordered = ordered - self.annotations = _as_tuple(annotations, Annotation) + self.annotations = as_tuple(annotations, item_type=Annotation) self._validate() self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index 2af5890ff..48565fb5e 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -14,13 +14,16 @@ ("kwargs", "message"), [ ({"labels": ()}, "at least one label"), - ({"labels": None}, "takes a list of Label items, got NoneType"), - ({"labels": 3}, "takes a list of Label items, got int"), - ({"labels": "WR"}, "takes a list of Label items, got str 'WR'"), - ({"labels": {"W": "Wake."}}, "takes a list of Label items, got dict"), - ({"labels": Label(name="W")}, "takes a list of Label items, got Label"), + ({"labels": None}, "iterable of items such as a list or a tuple, got NoneType"), + ({"labels": 3}, "iterable of items such as a list or a tuple, got int"), + ({"labels": "WR"}, "iterable of items such as a list or a tuple, got str"), + ({"labels": {"W": "Wake."}}, "iterable of items such as a list or a tuple, got dict"), + ({"labels": Label(name="W")}, "iterable of items such as a list or a tuple, got Label"), ({"labels": ("W", "")}, "cannot have an empty label"), - ({"labels": (Label(name="W"),), "annotations": None}, "takes a list of Annotation items, got NoneType"), + ( + {"labels": (Label(name="W"),), "annotations": None}, + "iterable of items such as a list or a tuple, got NoneType", + ), ({"labels": (Label(name="W"), Label(name="W", description="Awake."))}, "more than once"), ({"labels": (3,)}, "takes Label objects, got int"), ({"labels": ("W", "W")}, "more than once"), From 86e5933a12cae9fdc0f2b35e57698c97f4acc8a0 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:05:18 -0800 Subject: [PATCH 14/15] refactor(format): check each label set object once per write and simplify the key The writer keeps the id of each label set object for the write, so a set that many tasks share is checked and keyed once. The row and the id are stored together. The key uses the frozen Label objects directly. A label set rejects a Python set also when it is not ordered, because the writer keeps the order of the first equal set and a Python set has no fixed order. --- docs/data-model/tasks.md | 3 ++- .../src/timenet/format/control_writer.py | 21 ++++++++++++------- .../timenet/src/timenet/types/label_sets.py | 13 ++++++------ .../tests/format/test_label_set_storage.py | 6 +++--- .../timenet/tests/types/test_label_sets.py | 5 +++-- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index ada7e2524..c0e6819a3 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -258,7 +258,8 @@ LabelSet(["yes", "no"], annotations=[Annotation("name", value="signal_noisy")]) By default the order of labels does not matter. Use `ordered=True` when it has a meaning, for example for ordinal labels such as `low`, `medium` and `high`, or for answer options that a model sees in a fixed order. Then each order is its own set, and the stored set keeps it. An unordered set keeps the -label order of the first equal set that the writer meets. +label order of the first equal set that the writer meets. Pass the labels as a list or a tuple. A +label set rejects a Python set, because its order changes from one run to the next. The stored set gets one integer id, `label_set_id`. In one read, the reader gives tasks that use the same set a shared object. diff --git a/packages/timenet/src/timenet/format/control_writer.py b/packages/timenet/src/timenet/format/control_writer.py index 1ba8bdbe0..8b17d987d 100644 --- a/packages/timenet/src/timenet/format/control_writer.py +++ b/packages/timenet/src/timenet/format/control_writer.py @@ -212,7 +212,7 @@ class _LabelSetRows: """The ``label_sets`` rows of one write, one row for each distinct label set. Equal sets (see :meth:`LabelSet._content_key`) get one id, in the order that tasks first use them. - The row keeps the labels and the annotations of the first set. + The row keeps the labels and the annotations of the first set. Each set object is checked once per write. """ def __init__(self, ids: Iterator[int]) -> None: @@ -222,7 +222,8 @@ def __init__(self, ids: Iterator[int]) -> None: ids: The storage ids for new label sets. """ self._new_ids = ids - self._sets: dict[Hashable, tuple[dict[str, object], tuple[Annotation, ...]]] = {} + self._sets: dict[Hashable, tuple[int, dict[str, object], tuple[Annotation, ...]]] = {} + self._by_object: dict[int, tuple[LabelSet, int]] = {} def register(self, label_set: LabelSet) -> int: """Add a label set. @@ -233,18 +234,23 @@ def register(self, label_set: LabelSet) -> int: Raises: TimeFValidationError: If the set is not valid. A set can change after a task gets it. """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it + known = self._by_object.get(id(label_set)) + if known is not None: + return known[1] label_set._validate() key = label_set._content_key() stored = self._sets.get(key) if stored is None: + label_set_id = next(self._new_ids) row: dict[str, object] = { - "label_set_id": next(self._new_ids), + "label_set_id": label_set_id, "labels": [label.model_dump() for label in label_set.labels], "multi_label": label_set.multi_label, "ordered": label_set.ordered, } - stored = self._sets[key] = (row, label_set.annotations) - return cast("int", stored[0]["label_set_id"]) + stored = self._sets[key] = (label_set_id, row, label_set.annotations) + self._by_object[id(label_set)] = (label_set, stored[0]) + return stored[0] def write( self, connection: duckdb.DuckDBPyConnection, annotations: list[tuple[ObjectKind, int | None, Annotation]] @@ -255,9 +261,8 @@ def write( connection: The open write transaction. annotations: The annotation triples that the writer collects. This method adds to them. """ - insert_rows(connection, TABLES["label_sets"], [row for row, _ in self._sets.values()]) - for row, occurrences in self._sets.values(): - label_set_id = cast("int", row["label_set_id"]) + insert_rows(connection, TABLES["label_sets"], [row for _, row, _ in self._sets.values()]) + for label_set_id, _, occurrences in self._sets.values(): annotations.extend((ObjectKind.LABEL_SET, label_set_id, annotation) for annotation in occurrences) diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index a7ffd95b0..45157a24e 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -77,7 +77,7 @@ class LabelSet(SupportsAnnotate): ordered: bool = False """If ``True``, the label order is part of the identity of the set. If ``False``, the order does not matter. - An ordered set needs labels in a fixed order, so pass a list or a tuple, not a Python set.""" + Pass the labels as a list or a tuple, not a Python set, so that the stored order is fixed.""" annotations: tuple[Annotation, ...] = () """Annotations that describe the set, such as ``Annotation(key="name", value="sleep_stage")``. @@ -103,12 +103,12 @@ def __init__( Raises: TimeFValidationError: If the set is not valid (see :meth:`_validate`). """ - if ordered and isinstance(labels, (set, frozenset)): - raise TimeFValidationError("an ordered label set needs labels in a fixed order, got a set") - self.labels = tuple(_as_label(label) for label in as_tuple(labels, item_type=Label)) # ty: ignore[invalid-assignment] + if isinstance(labels, (set, frozenset)): + raise TimeFValidationError("a label set needs labels in a fixed order, got a set") + self.labels = tuple(_as_label(label) for label in as_tuple(labels)) # ty: ignore[invalid-assignment] self.multi_label = multi_label self.ordered = ordered - self.annotations = as_tuple(annotations, item_type=Annotation) + self.annotations = as_tuple(annotations) self._validate() self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) @@ -186,12 +186,11 @@ def _content_key(self) -> tuple[Hashable, ...]: Returns: The hashable key. """ - labels = ((label.name, label.description) for label in self.labels) annotations = Counter(_occurrence_key(annotation) for annotation in self.annotations) return ( self.multi_label, self.ordered, - tuple(labels) if self.ordered else frozenset(labels), + self.labels if self.ordered else frozenset(self.labels), frozenset(annotations.items()), ) diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index 099fe7980..9368109a1 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -298,14 +298,14 @@ def nan_named(): assert first == nan_named() -def test_a_set_annotated_after_a_task_uses_it_is_another_set(): +def test_one_write_gives_equal_sets_one_id_and_a_set_with_other_annotations_another(): label_set = LabelSet(["W"]) rows = _LabelSetRows(iter((1, 2))) assert rows.register(label_set) == 1 - label_set.annotate(Annotation("name", value="stage")) - assert rows.register(label_set) == 2 + assert rows.register(label_set) == 1 assert rows.register(LabelSet(["W"])) == 1 + assert rows.register(LabelSet(["W"], annotations=[Annotation("name", value="stage")])) == 2 def test_equal_annotations_of_two_sets_share_one_content_row(tmp_path): diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index 48565fb5e..b60c6ab1e 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -30,7 +30,8 @@ ({"labels": (Label(name="W"),), "multi_label": 1}, "multi_label must be a bool, got int"), ({"labels": (Label(name="W"),), "ordered": 1}, "ordered must be a bool, got int"), ({"labels": (Label(name="W"),), "ordered": None}, "ordered must be a bool, got NoneType"), - ({"labels": {"W", "R"}, "ordered": True}, "needs labels in a fixed order, got a set"), + ({"labels": {"W", "R"}}, "needs labels in a fixed order, got a set"), + ({"labels": frozenset({"W", "R"}), "ordered": True}, "needs labels in a fixed order, got a set"), ({"labels": (Label(name="W"),), "annotations": ("name",)}, "takes Annotation objects, got str"), ], ) @@ -122,7 +123,7 @@ def test_an_ordered_set_uses_its_label_order_for_equality(): assert low_high != high_low assert low_high._content_key() != high_low._content_key() assert low_high != LabelSet(["low", "high"]) - assert LabelSet(["low", "high"]) == LabelSet({"high", "low"}) + assert LabelSet(["low", "high"]) == LabelSet(["high", "low"]) assert LabelSet(["low"]).ordered is False From fc7cf3ddfed37aae3e1350ecb4e1b5221f2ea8ba Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:41:36 -0800 Subject: [PATCH 15/15] feat(format)!: bump the control schema version to 3 Version 3 stores label sets by id, with their annotations. A reader of version 2 misreads it, and this reader misreads version 2, so both the written and the oldest readable version are 3. --- .../timenet/src/timenet/format/control_schema.py | 2 +- packages/timenet/src/timenet/format/duckdb.py | 7 ++++--- packages/timenet/tests/format/test_duckdb.py | 12 ++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/timenet/src/timenet/format/control_schema.py b/packages/timenet/src/timenet/format/control_schema.py index 48414ae1c..fd6448d17 100644 --- a/packages/timenet/src/timenet/format/control_schema.py +++ b/packages/timenet/src/timenet/format/control_schema.py @@ -3,7 +3,7 @@ The SQL table definitions and the writer's Arrow batch schemas come from :data:`CONTROL_TABLES`. The audit also uses it to examine keys and references. Define schema changes there. -This is version 2 of the control schema. Every table uses ids as keys. +This is version 3 of the control schema. Every table uses ids as keys. Records, Signals, and Tasks use the public ids that the caller sees. All other objects, label sets included, use the writer's storage ids. A foreign key stores the key of the row that it refers to. diff --git a/packages/timenet/src/timenet/format/duckdb.py b/packages/timenet/src/timenet/format/duckdb.py index 74640e588..b689fe8eb 100644 --- a/packages/timenet/src/timenet/format/duckdb.py +++ b/packages/timenet/src/timenet/format/duckdb.py @@ -14,13 +14,14 @@ CONTROL_FILE = "control.duckdb" """Name of the relational control-plane database in a TimeF version.""" -CONTROL_SCHEMA_VERSION = 2 +CONTROL_SCHEMA_VERSION = 3 """Schema version written into :data:`CONTROL_FILE`, and the newest that this TimeNet reads.""" -OLDEST_READABLE_CONTROL_SCHEMA_VERSION = 2 +OLDEST_READABLE_CONTROL_SCHEMA_VERSION = 3 """The oldest schema version of :data:`CONTROL_FILE` that this TimeNet reads. -Version 2 is the schema of TimeNet 0.12. Version 1 names layouts used during early development, and no TimeNet reads them. +Version 3 stores label sets by id, with their annotations. Version 2 is the schema of TimeNet 0.12. Version 1 names +layouts used during early development. This TimeNet reads neither of them. """ _IN_MEMORY_DATABASE = "an in-memory control database" diff --git a/packages/timenet/tests/format/test_duckdb.py b/packages/timenet/tests/format/test_duckdb.py index 6c05ea44a..83268ce1c 100644 --- a/packages/timenet/tests/format/test_duckdb.py +++ b/packages/timenet/tests/format/test_duckdb.py @@ -68,22 +68,22 @@ def test_a_newer_control_schema_names_the_file_and_both_versions(tmp_path): path = tmp_path.joinpath("control.duckdb") with connect_control(path) as connection: create_control_schema(connection) - connection.execute("UPDATE control_metadata SET value = '3'") + connection.execute("UPDATE control_metadata SET value = '4'") with pytest.raises(TimeFFormatVersionError) as caught: check_control_schema(connection) - assert str(caught.value) == f"{path} uses control schema version 3, but the newest that this timenet reads is 2." + assert str(caught.value) == f"{path} uses control schema version 4, but the newest that this timenet reads is 3." assert caught.value.subject is FormatVersionSubject.CONTROL_SCHEMA def test_a_newer_in_memory_control_schema_names_no_file(): with duckdb.connect() as connection: create_control_schema(connection) - connection.execute("UPDATE control_metadata SET value = '3'") + connection.execute("UPDATE control_metadata SET value = '4'") with pytest.raises( - TimeFFormatVersionError, match=r"^an in-memory control database uses control schema version 3, but" + TimeFFormatVersionError, match=r"^an in-memory control database uses control schema version 4, but" ): check_control_schema(connection) @@ -92,12 +92,12 @@ def test_an_older_control_schema_names_the_oldest_version_that_is_read(tmp_path) path = tmp_path.joinpath("control.duckdb") with connect_control(path) as connection: create_control_schema(connection) - connection.execute("UPDATE control_metadata SET value = '1'") + connection.execute("UPDATE control_metadata SET value = '2'") with pytest.raises(TimeFFormatVersionError) as caught: check_control_schema(connection) - assert str(caught.value) == f"{path} uses control schema version 1, but the oldest that this timenet reads is 2." + assert str(caught.value) == f"{path} uses control schema version 2, but the oldest that this timenet reads is 3." assert caught.value.path == path