diff --git a/benchmarks/end_to_end/corpus.py b/benchmarks/end_to_end/corpus.py index 2dbe348a6..963b0cb47 100644 --- a/benchmarks/end_to_end/corpus.py +++ b/benchmarks/end_to_end/corpus.py @@ -19,7 +19,6 @@ DatasetMetadata, Domain, ForecastingTask, - Label, LabelSet, License, TimeInterval, @@ -31,10 +30,10 @@ from timenet.types.wire import ValueDtype -CONDITION = LabelSet(name="condition", labels=(Label(name="healthy"), Label(name="outer-race-fault"))) -RHYTHM = LabelSet(name="rhythm", labels=(Label(name="normal-sinus"), Label(name="atrial-fibrillation"))) -SLEEP_STAGE = LabelSet(name="sleep-stage", labels=tuple(Label(name=stage) for stage in ("W", "N1", "N2", "N3", "REM"))) -MEAN_SIGN = LabelSet(name="mean-sign", labels=(Label(name="above-zero"), Label(name="below-zero"))) +CONDITION = LabelSet(["healthy", "outer-race-fault"]) +RHYTHM = LabelSet(["normal-sinus", "atrial-fibrillation"]) +SLEEP_STAGE = LabelSet(["W", "N1", "N2", "N3", "REM"]) +MEAN_SIGN = LabelSet(["above-zero", "below-zero"]) @dataclass(frozen=True) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 27a7b5c35..6407dcafa 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -9,7 +9,7 @@ tags: # Annotations An `Annotation` stores reusable content. An annotation occurrence attaches that content to a Dataset, -Task, Record, Source, or Signal. +Task, Record, Source, Signal, or label set. ```python from timenet.types import Annotation @@ -116,6 +116,25 @@ a separate occurrence with shared content. Other Signals remain unchanged. Time spans use whole microseconds on the Record timeline. Step spans are reserved for Tasks on ordinal Signals. +## Label set annotations + +A `LabelSet` takes annotations like the other owners. Pass them to the constructor, or call +`annotate()` later. Use them for a name, a description, or other context. No key is required. + +```python +from timenet.types import Annotation, LabelSet + +SLEEP_STAGE = LabelSet( + ["W", "N1", "R"], + annotations=[Annotation("name", value="sleep_stage")], +) +SLEEP_STAGE.annotate(Annotation(key="description", value="Sleep stage of one epoch.")) +``` + +A label set annotation has no span. Annotations are part of the identity of a set. Two sets with the +same labels and different annotations are two sets, and TimeF stores each of them. TimeF stores sets +with the same labels and the same annotations once. + ## Valid values An annotation value can be a string, integer, float, Boolean, or list of strings. For example, a diff --git a/docs/data-model/records.md b/docs/data-model/records.md index 76717a056..38cb2e366 100644 --- a/docs/data-model/records.md +++ b/docs/data-model/records.md @@ -119,12 +119,9 @@ A [Task](tasks.md) refers directly to its input Records. Registering a Task does Records. ```python -from timenet.types import ClassificationTask, Label, LabelSet +from timenet.types import ClassificationTask, LabelSet -CONDITION = LabelSet( - name="condition", - labels=(Label(name="healthy"), Label(name="faulty")), -) +CONDITION = LabelSet(["healthy", "faulty"]) task = ClassificationTask( inputs=(record,), diff --git a/docs/data-model/tasks.md b/docs/data-model/tasks.md index fa7f17039..c0e6819a3 100644 --- a/docs/data-model/tasks.md +++ b/docs/data-model/tasks.md @@ -12,9 +12,9 @@ A Task describes one modeling problem. It holds ordered input Records and one re expected output. ```python -from timenet.types import ClassificationTask, Label, LabelSet +from timenet.types import ClassificationTask, LabelSet -CONDITION = LabelSet(name="condition", labels=(Label(name="healthy"), Label(name="faulty"))) +CONDITION = LabelSet(["healthy", "faulty"]) task = ClassificationTask( inputs=(record,), @@ -85,12 +85,9 @@ A dataset that ships a partition marks each Task with its `split`. Records carry own: one Record may serve Tasks of several splits. ```python -from timenet.types import ClassificationTask, Label, LabelSet, Split +from timenet.types import ClassificationTask, LabelSet, Split -ACTIVITY = LabelSet( - name="activity", - labels=(Label(name="walking"), Label(name="sitting")), -) +ACTIVITY = LabelSet(["walking", "sitting"]) dataset.add_task( ClassificationTask( @@ -224,26 +221,20 @@ dataset.add_task(age_question(record, 64)) ```python from timenet.types import Label, LabelSet -CONDITION = LabelSet( - name="condition", - labels=(Label(name="healthy"), Label(name="faulty", description="A bearing fault.")), -) -FINDINGS = LabelSet( - name="findings", - labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), - multi_label=True, -) +CONDITION = LabelSet(["healthy", Label(name="faulty", description="A bearing fault.")]) +FINDINGS = LabelSet(["AF", "PVC", "LBBB"], multi_label=True) ClassificationTask(inputs=(record,), targets="healthy", label_set=CONDITION) ClassificationTask(inputs=(record,), targets=("PVC", "AF"), label_set=FINDINGS) ``` -A `LabelSet` defines the allowed labels for a classification target. The `multi_label` flag controls -how many labels a target can contain. +A `LabelSet` defines the allowed labels for a classification target. Like other owners, it can carry +[annotations](annotations.md#label-set-annotations). The `multi_label` flag controls how many labels +a target can contain. If `multi_label=False`, the default, a target must contain exactly one label. If `multi_label=True`, a target contains all labels that apply. Omitted labels do not apply. An empty target means that no -label applies. The order of labels does not matter. +label applies. When the dataset adds or streams a task, it makes sure that the target follows the rules of its label set. Each label must be in the set. No label can repeat. A target with `multi_label=False` @@ -252,9 +243,26 @@ must contain exactly one label. Labels come from the targets stored directly on the task or from the text of its target annotations. -Tasks can share the same label set object. Each name identifies one label set within a dataset. -If two different sets have the same name, `add_task()`, the task stream, and the writer raise an -error. TimeF stores each set once. The reader gives tasks that use the same set a shared object. +A label set has no name. TimeF stores equal sets once. For this, two sets are equal if they have the +same labels, label descriptions, `multi_label`, `ordered`, and annotations. The order of labels +matters only for an ordered set. The order of annotations does not matter. `==` follows the same rule. + +Annotations are equal if all their fields are equal, the value included. So these two sets are two +stored sets, each with its own id: + +```python +LabelSet(["yes", "no"], annotations=[Annotation("name", value="afib_present")]) +LabelSet(["yes", "no"], annotations=[Annotation("name", value="signal_noisy")]) +``` + +By default the order of labels does not matter. Use `ordered=True` when it has a meaning, for example +for ordinal labels such as `low`, `medium` and `high`, or for answer options that a model sees in a +fixed order. Then each order is its own set, and the stored set keeps it. An unordered set keeps the +label order of the first equal set that the writer meets. Pass the labels as a list or a tuple. A +label set rejects a Python set, because its order changes from one run to the next. + +The stored set gets one integer id, `label_set_id`. In one read, the reader gives tasks that use the +same set a shared object. ### Answer and rationale diff --git a/docs/timef-dataset.md b/docs/timef-dataset.md index 407b08a4d..056379fa0 100644 --- a/docs/timef-dataset.md +++ b/docs/timef-dataset.md @@ -213,7 +213,7 @@ inserts that stream once without retaining every Task in memory. ## Annotations -The same `Annotation` type can annotate a Dataset, Task, Record, Source, or Signal. Annotation +The same `Annotation` type can annotate a Dataset, Task, Record, Source, Signal, or label set. Annotation content is immutable and reusable. Every call to `annotate()` creates a separate occurrence, so a long payload can be stored once while its occurrences carry independent placement, confidence, and provenance. `Annotation` is final: reuse content through a constant, or a builder function for @@ -238,8 +238,8 @@ lead_i.annotate( `derive_schema()` collects the time-series specifications, annotation descriptors, and concrete Task types found in the dataset. Annotation descriptors come from every owner: the Dataset, Records, -Sources, Signals, and Tasks. The Tasks of a task stream are not read here. The writer adds their -keys to the schema while it writes them. It does not load Signal values. +Sources, Signals, Tasks, and label sets. The Tasks of a task stream are not read here. The writer +adds their keys to the schema while it writes them. It does not load Signal values. Write a version with the declarative helper: diff --git a/docs/timef-format.md b/docs/timef-format.md index c3b939d79..8ca38265f 100644 --- a/docs/timef-format.md +++ b/docs/timef-format.md @@ -171,13 +171,14 @@ The `annotation_occurrences` table records where the content applies. It points the content through `content_id`. It also carries the span as `span_type`, `start_us`, and `end_us`, plus provenance, confidence, and occurrence metadata. -An occurrence can annotate a Dataset, Task, Record, Source, or Signal. It names its owner with -`object_type` and `object_id`. For a Record, Signal, or Task, `object_id` is the entity ID of the -owner. For a Source, it is the storage ID of the Source. `object_id` is `NULL` exactly when the +An occurrence can annotate a Dataset, Task, Record, Source, Signal, or label set. It names its owner with +`object_type` and `object_id`. For a Record, Signal, Task, or label set, `object_id` is the ID of the +owner. For a Source, it is the storage ID of that row. `object_id` is `NULL` exactly when the owner is the Dataset. IDs are unique only within one kind, so the type is part of the reference. -A read of every Record or every Task rejects an occurrence with an unknown owner type or a missing -owner. It does not drop that occurrence. +A read of every Record or every Task rejects an occurrence with an unknown owner type. It also +rejects an occurrence whose Record, Source, Signal, or Task owner is missing. It does not drop that +occurrence. Only `audit_control_database()` reports an occurrence whose label set is missing. One content row can apply to many objects without copying a long value. Each attachment still has its own `occurrence_id` and temporal placement. @@ -188,15 +189,20 @@ its own `occurrence_id` and temporal placement. Python subclass. Four typed columns store the scope: `scope_type`, `scope_start`, `scope_end`, and `scope_signal_ids`. -Configuration for each task type uses the columns `label_set`, `unit`, `target_name`, and `mode`. +Configuration for each task type uses the columns `label_set_id`, `unit`, `target_name`, and `mode`. A column that a task type does not use is `NULL`. SQL can filter these columns without parsing JSON. -Every classification task has a label set, and `tasks.label_set` stores its name. Each row in -`label_sets` stores one set with the fields `name`, `labels`, and `multi_label`. The name is unique -within a release. - -The `labels` field holds one `{name, description}` entry per label, in label order. If a label has -no description, its `description` is `NULL`. +Every classification task has a label set, and `tasks.label_set_id` stores its ID. Each row in +`label_sets` stores one set with the fields `label_set_id`, `labels`, `multi_label`, and `ordered`. +Sets with the same labels, label descriptions, `multi_label`, `ordered`, and annotations are one set, +so a release stores them once. If `ordered` is true, the label order is part of the set, so the same +labels in another order are a different set. The same labels with different annotations are also a +different set. The writer numbers the sets in the order that tasks first use them. +The annotations of a set are rows of `annotation_occurrences` with `object_type` `LabelSet` and the +`label_set_id` as `object_id`. + +The `labels` field holds one `{name, description}` entry per label, in the label order of the first +equal set that the writer meets. If a label has no description, its `description` is `NULL`. Each inline target is one row in `task_targets`. Its `target_kind` names the typed value column that is set. A target stores entity IDs directly: `record_id` for a Record, `signal_id` for a diff --git a/docs/timef-reader.md b/docs/timef-reader.md index a22afcc54..594669d3e 100644 --- a/docs/timef-reader.md +++ b/docs/timef-reader.md @@ -112,8 +112,8 @@ Tasks have four useful read paths: The Record, Signal, and Task columns hold the ids that the dataset assigned, for example `task_id`, `input_record_ids`, `from_task_ids`, and `span_signal_ids`. `occurrence_id` and `content_id` are storage row numbers of this version. The next write numbers them again. In - `read_annotation_table()`, `object_id` is a Record, Signal, or Task id, a storage row number for - a Source, or `NULL` for the dataset. + `read_annotation_table()`, `object_id` is a Record, Signal, or Task id, a label set id, a storage + row number for a Source, or `NULL` for the dataset. One call returns one consistent set of objects. A Record that two tasks use is one object, and the Signals on one irregular axis share one handle to its time offsets. `read_tasks(inputs=records)` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py index 6d80560cc..b5cb9f953 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/questions.py @@ -5,12 +5,11 @@ import csv from dataclasses import dataclass from enum import Enum -import hashlib import json from typing import TYPE_CHECKING, Self, TypeVar from timenet.errors import TimeNetConnectorError -from timenet.types import Annotation, ClassificationTask, InputModality, Label, LabelSet, Split +from timenet.types import Annotation, ClassificationTask, InputModality, LabelSet, Split if TYPE_CHECKING: @@ -108,15 +107,6 @@ def _parse_answer_options(cell: str) -> tuple[str, ...]: return tuple(options) -def _label_set_name(options: tuple[str, ...]) -> str: - """Name a label set by its options, so that equal option lists share one set. - - Returns: - ``options_`` and the first 8 hex digits of the SHA-256 of the options as a JSON list. - """ - return "options_" + hashlib.sha256(json.dumps(options).encode()).hexdigest()[:8] - - @dataclass(frozen=True) class ARFBenchQuestion: """One row of the question table: a multiple-choice question about one or two metrics.""" @@ -216,31 +206,29 @@ def create_chart_task(self, chart_records: Mapping[str, Record]) -> Classificati def _create_task(self, inputs: list[Record], modality: InputModality) -> ClassificationTask: """Create a test task that asks for the correct answer from the options of the question. - The options are the label set of the task. The set is named after a hash of the options, so - questions with the same options share one set. The row index in the metadata pairs the - series task with the chart task of ``datadog/arfbench-charts``. The interpolation flags - describe what an engineer saw. This method does not interpolate. + The options are the label set of the task. The set is ordered, because the order of the options is + part of the question. Questions with the same options in the same order share one set. The row + index in the metadata pairs the series task with the chart task of ``datadog/arfbench-charts``. + The interpolation flags describe what an engineer saw. This method does not interpolate. Returns: The task, not yet added to a dataset. """ - task = ClassificationTask( + return ClassificationTask( inputs=inputs, prompt=self.question, targets=self.correct_answer, input_modalities=frozenset({InputModality.TEXT, modality}), split=Split.TEST, metadata={"question_index": self.index}, - label_set=LabelSet( - name=_label_set_name(self.answer_options), - labels=tuple(Label(name=option) for option in self.answer_options), - ), + label_set=LabelSet(self.answer_options, ordered=True), + annotations=[ + Annotation("task_category", value=self.task_category.value), + Annotation("difficulty", value=self.difficulty.value), + Annotation("interpolate_1", value=self.interpolate_1), + Annotation("interpolate_2", value=self.interpolate_2), + ], ) - task.annotate(Annotation("task_category", value=self.task_category.value)) - task.annotate(Annotation("difficulty", value=self.difficulty.value)) - task.annotate(Annotation("interpolate_1", value=self.interpolate_1)) - task.annotate(Annotation("interpolate_2", value=self.interpolate_2)) - return task def read_questions(qa_csv: Path) -> Iterator[ARFBenchQuestion]: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py index f75e74c8d..6b598e234 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/tests/test_connector.py @@ -107,16 +107,20 @@ def test_each_question_is_a_classification_task_over_its_options(release): } -def test_questions_with_equal_options_share_a_label_set(release): +def test_questions_with_equal_options_in_equal_order_share_a_label_set(release): up_down = {**PRESENCE_ROW, "Unnamed: 0": "2", "options_str": '["Up", "Down"]', "correct_answer": "Up"} - write_qa_csv(_path(release, "questions").parent, [*ROWS, up_down]) - presence, correlation, up_down_task = ARFBenchConnector().convert(release).get_all() + no_yes = {**PRESENCE_ROW, "Unnamed: 0": "3", "options_str": '["No", "Yes"]', "correct_answer": "No"} + write_qa_csv(_path(release, "questions").parent, [*ROWS, up_down, no_yes]) + presence, correlation, up_down_task, no_yes_task = ARFBenchConnector().convert(release).get_all() assert isinstance(presence, ClassificationTask) and isinstance(correlation, ClassificationTask) - assert isinstance(up_down_task, ClassificationTask) + assert isinstance(up_down_task, ClassificationTask) and isinstance(no_yes_task, ClassificationTask) + assert presence.label_set.ordered assert presence.label_set == correlation.label_set - assert presence.label_set.name.startswith("options_") - assert up_down_task.label_set.name != presence.label_set.name + assert presence.label_set.annotations == () + assert up_down_task.label_set != presence.label_set + assert no_yes_task.label_set != presence.label_set + assert no_yes_task.label_set.names == ("No", "Yes") def test_the_dataset_round_trips_through_parquet_with_its_missing_timesteps(release, tmp_path): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py index 9ab4d7136..902330aa3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/har_cot/connector.py @@ -16,7 +16,7 @@ from timenet.connectors import BaseConnector from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset from timenet.errors import TimeNetConnectorError -from timenet.types import ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg +from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeSeriesSpec, ureg from timenet_connectors.download import ensure_archive from timenet_connectors.sources.readers import CsvDataReader, DataRow @@ -35,17 +35,17 @@ # The label order is the order of the OpenTSLM training prompt. ACTIVITY = LabelSet( - name="activity", - labels=( - Label(name="biking"), - Label(name="lying"), - Label(name="running"), - Label(name="sitting"), - Label(name="standing"), - Label(name="walking"), + [ + "biking", + "lying", + "running", + "sitting", + "standing", + "walking", Label(name="walking_down", description="Walking down stairs."), Label(name="walking_up", description="Walking up stairs."), - ), + ], + annotations=[Annotation("name", value="activity")], ) # The release states no unit. A resting window has a magnitude of about 9.8, which is gravity in m/s². ACCELERATION = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py index b2ecf5f96..a93726eeb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/opentslm/sleep_cot/connector.py @@ -35,15 +35,15 @@ # The release writes the stage codes of the Rechtschaffen and Kales manual. The prompts and the # rationales write the names in the descriptions. SLEEP_STAGE = LabelSet( - name="sleep_stage", - labels=( + [ Label(name="W", description="Wake"), Label(name="N1", description="Non-REM stage 1"), Label(name="N2", description="Non-REM stage 2"), Label(name="N3", description="Non-REM stage 3"), Label(name="N4", description="Non-REM stage 4"), Label(name="REM", description="REM sleep"), - ), + ], + annotations=[Annotation("name", value="sleep_stage")], ) # OpenTSLM converted the microvolts of Sleep-EDF to volts and kept every second sample of 100 Hz. EEG = TimeSeriesSpec(spec_type="eeg", name="EEG", unit_value=ureg.volt, dtype="float64") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md index c2e49a27b..a7a7bf5c1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/README.md @@ -122,8 +122,8 @@ where the recording came from. ### How the connector writes them Each classification task has a set of allowed target labels. The three sets contain eight sleep -stages, two sexes, and two study conditions. Each `LabelSet` uses its annotation key as its name: -`sleep_stage`, `sex`, or `condition`. +stages, two sexes, and two study conditions. Each `LabelSet` carries a `name` annotation with its +annotation key: `sleep_stage`, `sex`, or `condition`. Tasks that ask the same question share one label set. Each target must contain exactly one label from that set. TimeNet enforces this requirement when it writes the dataset. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py index ded52053a..50a097ad3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tasks.py @@ -6,8 +6,9 @@ No task names its own id. Nothing resolves one by id: each task holds its record in ``inputs``, and no task here states a lineage. Every task takes the id the dataset gives it. -Each classification task draws its target from one closed set, a ``LabelSet`` named after the -annotation key whose values it holds. The tasks that ask one question share one set. +Each classification task draws its target from one closed set. The ``name`` annotation of each +``LabelSet`` holds the annotation key whose values the set lists. The tasks that ask one question +share one set. No task carries a prompt. The release states no question in words. """ @@ -38,20 +39,20 @@ # The labels the release writes for a scored epoch. Five name a stage of sleep. `Sleep stage W` # is wake, `Movement time` marks a disturbed epoch, and `Sleep stage ?` marks one the -# technician left unscored. Each classification task draws its target from one closed set, named -# after the key whose values it holds, and each target is exactly one label. +# technician left unscored. Each classification task draws its target from one closed set, whose +# name annotation is the key whose values it holds, and each target is exactly one label. SLEEP_STAGES = LabelSet( - name=str(AnnotationKey.SLEEP_STAGE), - labels=( + [ Label(name="Sleep stage W", description="Wake."), - Label(name="Sleep stage 1"), - Label(name="Sleep stage 2"), - Label(name="Sleep stage 3"), - Label(name="Sleep stage 4"), + "Sleep stage 1", + "Sleep stage 2", + "Sleep stage 3", + "Sleep stage 4", Label(name="Sleep stage R", description="Rapid eye movement (REM) sleep."), Label(name="Movement time", description="An epoch that body movement obscures, so it has no stage."), Label(name="Sleep stage ?", description="An epoch with no sleep-stage score from data collection."), - ), + ], + annotations=[Annotation("name", value=str(AnnotationKey.SLEEP_STAGE))], ) STAGE_LABELS = SLEEP_STAGES.names @@ -61,14 +62,17 @@ # The sexes, after each sheet is decoded. The two sheets code the column with opposite # meanings, so a raw code never reaches a task. SEXES = LabelSet( - name=str(AnnotationKey.SEX), - labels=(Label(name="F", description="Female."), Label(name="M", description="Male.")), + [Label(name="F", description="Female."), Label(name="M", description="Male.")], + annotations=[Annotation("name", value=str(AnnotationKey.SEX))], ) SEX_LABELS = SEXES.names # The two nights of the telemetry study. CONDITION_LABELS = tuple(Condition) -CONDITIONS = LabelSet(name=str(AnnotationKey.CONDITION), labels=tuple(Label(name=str(one)) for one in Condition)) +CONDITIONS = LabelSet( + [str(one) for one in Condition], + annotations=[Annotation("name", value=str(AnnotationKey.CONDITION))], +) def build_epoch_tasks(recording_id: str, span_annotations: Sequence[Annotation]) -> Iterator[ClassificationTask]: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py index cdac61de4..7a4b0ecb4 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/tests/test_tasks.py @@ -11,6 +11,7 @@ TimeInterval, TimePoint, ) +from timenet.types.text import read_value from timenet_connectors.datasets.physionet.sleep_edfx import tasks from timenet_connectors.datasets.physionet.sleep_edfx.keys import AnnotationKey @@ -153,7 +154,8 @@ def _pick(built, kind, label_set=None): def _label_sets(built): - return {one.label_set.name for one in built if isinstance(one, ClassificationTask)} + tasks = [one for one in built if isinstance(one, ClassificationTask)] + return {read_value(a.value) for one in tasks for a in one.label_set.annotations if a.key == "name"} def test_age_is_a_scalar_with_a_unit(): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py index 781e71d64..4981fa0bb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/hello_world/connector.py @@ -49,12 +49,12 @@ # Each classification task draws its target from a label set that every task with the same question # shares. _RHYTHM = LabelSet( - name="rhythm", - labels=(Label(name="normal", description="A regular wave."), Label(name="abnormal")), + [Label(name="normal", description="A regular wave."), "abnormal"], + annotations=[Annotation("name", value="rhythm")], ) _EVENT = LabelSet( - name="event", - labels=(Label(name="onset", description="The moment the wave starts."), Label(name="offset")), + [Label(name="onset", description="The moment the wave starts."), "offset"], + annotations=[Annotation("name", value="event")], ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py index 3e84a3fef..358f8f4f1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/connector.py @@ -14,7 +14,7 @@ from timenet.connectors import BaseConnector from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset -from timenet.types import ClassificationTask, Label, LabelSet, TimeSeriesSpec, ureg +from timenet.types import Annotation, ClassificationTask, Label, LabelSet, TimeSeriesSpec, ureg _N_RECORDS = 1000 @@ -23,11 +23,11 @@ _NOISE_STD = 0.5 _SEED = 20260715 # fixed base seed keeps every build's values identical _MEAN_SIGN = LabelSet( - name="mean_sign", - labels=( + [ Label(name="above_zero", description="The mean of the signal is above zero."), Label(name="below_zero", description="The mean of the signal is below zero."), - ), + ], + annotations=[Annotation("name", value="mean_sign")], ) _SIGNAL = TimeSeriesSpec( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py index 4c0cef688..47ae40737 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/timenet/test_mean/tests/test_connector.py @@ -46,7 +46,8 @@ def test_has_balanced_binary_labels(): def test_every_task_shares_the_mean_sign_label_set(): label_sets = [task.label_set for task in _convert().tasks_of(ClassificationTask)] - assert {label_set.name for label_set in label_sets} == {"mean_sign"} + names = {read_value(a.value) for label_set in label_sets for a in label_set.annotations if a.key == "name"} + assert names == {"mean_sign"} assert len({id(label_set) for label_set in label_sets}) == 1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md index edfb7bed6..90fe18238 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/README.md @@ -156,8 +156,8 @@ metadata. | `puzzling` | `AnswerTask` | Four letters, such as `C,B,D,A` | 20,932 | 2,979 | 5,978 | - **A true-or-false question** is a classification, since `T` and `F` mean the same in every - question. Every true-or-false task shares the `true_or_false` label set: `T` (True.) and `F` - (False.). + question. Every true-or-false task shares one label set, with a `name` annotation of `true_or_false`: `T` + (True.) and `F` (False.). - **A multiple-choice question or a puzzle** is an answer task. Its letters name choices that differ from one question to the next. A classification question's choices are its UCR dataset's class names, such as `normal process`. The `answer_text` metadata holds the right choice's text when the diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index dfa5c70e4..97c09db23 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -74,8 +74,8 @@ class _Role(StrEnum): _TRUE_OR_FALSE_LABEL_SET = LabelSet( - name="true_or_false", - labels=(Label(name="T", description="True."), Label(name="F", description="False.")), + [Label(name="T", description="True."), Label(name="F", description="False.")], + annotations=[Annotation("name", value="true_or_false")], ) """The label set that every true-or-false task shares.""" _CONTEXT_KEYS = ("meta_info", "domain", "dataset") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py index a5d29c807..112151423 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/tests/test_connector.py @@ -109,7 +109,8 @@ def _check_task(task, row: dict[str, Any]) -> None: assert task.metadata["question_type"] == row["question_type"] assert task.metadata["source_issues"] == _EXPECTED_ISSUES.get(_row_of(task), []) if isinstance(task, ClassificationTask): - assert (task.label_set.name, task.label_set.names) == ("true_or_false", ("T", "F")) + assert task.label_set.names == ("T", "F") + assert [(a.key, a.value) for a in task.label_set.annotations] == [("name", "true_or_false")] def test_every_task_holds_its_published_row(): diff --git a/packages/timenet/src/timenet/dataset/composition.py b/packages/timenet/src/timenet/dataset/composition.py index 430ff4099..b58c55756 100644 --- a/packages/timenet/src/timenet/dataset/composition.py +++ b/packages/timenet/src/timenet/dataset/composition.py @@ -3,7 +3,7 @@ from collections import deque from collections.abc import Container, Hashable, Iterable, Iterator, Mapping from copy import deepcopy -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypeAlias from timenet.dataset.record import Record @@ -18,6 +18,7 @@ TimePoint, TSCorrespondenceTask, ) +from timenet.types.annotations import _copy_annotation from timenet.types.clock import TimeOrigin from timenet.types.ids import EntityId, check_id from timenet.types.load_plan import LoadPlan @@ -163,23 +164,6 @@ def _fits(annotation: Annotation, signals: tuple[Signal, ...], owner: str, time_ return True -def _copy_annotation(original: Annotation) -> Annotation: - """Copy an annotation occurrence as a value, with the handle of a stored value. - - The copy has deep copies of the value, the metadata, and the occurrence metadata. So a change to - one of them in the copy leaves the original as it is. Imports and edits copy annotations this way. - - Returns: - The copy. - """ - return replace( - original, - value=deepcopy(original.value), - metadata=deepcopy(original.metadata), - occurrence_metadata=deepcopy(original.occurrence_metadata), - ) - - def _hierarchy(record: Record) -> list[tuple[Record | Source, dict[str, tuple]]]: """Return the fields that an import changes when it adds to the copy of a Record. diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index 9a4532450..c019ed46c 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -23,7 +23,6 @@ DatasetSchema, EntityId, InputModality, - LabelSetRegistry, ObjectKind, StepSpan, SupportsAnnotate, @@ -92,8 +91,6 @@ def __init__(self, *, metadata: DatasetMetadata) -> None: """The Record that holds each registered Signal, by Signal ID.""" self._tasks: list[Task] = [] self._tasks_by_id: dict[EntityId, Task] = {} - self._label_sets = LabelSetRegistry() - """The label sets that the registered tasks use, one per name.""" self._ids = IdCounters() self._streamed_task_base = 0 self._annotations: list[Annotation] = [] @@ -475,8 +472,8 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: the cross-task checks (``from_tasks`` derivations) that need every task at once do not run for a stream. - If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks are - added to it. After the last task, the schema lists them. + If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks and + their label sets are added to it. After the last task, the schema lists them. Yields: Each validated task, in the source's order. @@ -487,12 +484,11 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: """ # ruff: ignore[docstring-extraneous-exception] (raised by _validate_streamed_task, not directly here) index = _TaskValidationIndex(self) declared = set(self._streamed_task_types) - label_sets = LabelSetRegistry() schema = self._schema descriptors = {} if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations} for position, task in enumerate(self.iter_tasks(), start=1): task._id = self._ids.claim(ObjectKind.TASK, self._streamed_task_base + position) - self._validate_streamed_task(task, index, declared, label_sets) + self._validate_streamed_task(task, index, declared) if schema is not None: self._add_descriptors(descriptors, task.walk_annotations()) yield task @@ -504,7 +500,6 @@ def _validate_streamed_task( task: Task, index: _TaskValidationIndex, declared: set[type[Task]], - label_sets: LabelSetRegistry, ) -> None: """Make sure that each streamed task follows the dataset rules. @@ -512,8 +507,6 @@ def _validate_streamed_task( task: The streamed task. index: The shared index of dataset objects and annotations. declared: The task types :meth:`set_task_stream` declared. - label_sets: The label sets used so far in the stream. - This method adds the label set of this task. Raises: TimeFValidationError: If the task's type is not declared, or the task fails a check of @@ -524,9 +517,9 @@ def _validate_streamed_task( f"streamed task {task.id!r} has type {type(task).__name__}, not one of the declared " f"{sorted(t.__name__ for t in declared)}" ) - self._check_task(task, index, label_sets) + self._check_task(task, index) - def _check_task(self, task: Task, index: _TaskValidationIndex, label_sets: LabelSetRegistry) -> None: + def _check_task(self, task: Task, index: _TaskValidationIndex) -> None: """Make sure that a task follows the dataset rules. This method handles tasks added alone, in a batch, or in a stream. @@ -534,7 +527,6 @@ def _check_task(self, task: Task, index: _TaskValidationIndex, label_sets: Label Args: task: The task to check. index: The shared index of dataset objects and annotations. - label_sets: The label sets used so far. This method adds the label set of this task. Raises: TimeFValidationError: If the task's record attachment, references, answer, labels, label @@ -544,7 +536,6 @@ def _check_task(self, task: Task, index: _TaskValidationIndex, label_sets: Label task.check_against_scope() self._check_task_answer(task) task.check_targets() - task.register_label_set(label_sets) self._check_signal_refs(task, index) self._validate_task_spans(task, index) self._check_annotation_refs(task, self._referenced_records(task), index) @@ -582,7 +573,7 @@ def _register_batch(self, batch: tuple[Task, ...]) -> tuple[Task, ...]: for offset, task in enumerate(batch): task._id = first + offset try: - label_sets = self._validate_task_batch(batch) + self._validate_task_batch(batch) except BaseException: for task, task_id in zip(batch, previous, strict=True): task._id = task_id @@ -591,14 +582,13 @@ def _register_batch(self, batch: tuple[Task, ...]) -> tuple[Task, ...]: self._ids.claim(ObjectKind.TASK, task.id) self._tasks.append(task) self._tasks_by_id[task.id] = task - self._label_sets = label_sets return batch def _is_registered(self, task: Task) -> bool: """Return whether this exact task object is in the dataset.""" return task._id is not None and self._tasks_by_id.get(task._id) is task - def _validate_task_batch(self, batch: tuple[Task, ...]) -> LabelSetRegistry: + def _validate_task_batch(self, batch: tuple[Task, ...]) -> None: """Make sure that a batch follows the dataset rules before attaching its tasks. This method examines each task and the relationships between tasks. @@ -608,9 +598,6 @@ def _validate_task_batch(self, batch: tuple[Task, ...]) -> LabelSetRegistry: Args: batch: The tasks to register together. - Returns: - A copy of the registry that includes the label sets from this batch. - Raises: TimeFValidationError: This error occurs under the conditions documented on :meth:`add_tasks`. @@ -627,10 +614,8 @@ def _validate_task_batch(self, batch: tuple[Task, ...]) -> LabelSetRegistry: ) self._check_no_derivation_cycle(batch) index = _TaskValidationIndex(self) - label_sets = self._label_sets.copy() for task in batch: - self._check_task(task, index, label_sets) - return label_sets + self._check_task(task, index) @classmethod def _validate_task_spans(cls, task: Task, index: _TaskValidationIndex) -> None: @@ -787,7 +772,6 @@ def from_parts( # ruff: ignore[too-many-arguments] - every part of a stored dat dataset._tasks = list(tasks) for task in dataset._tasks: dataset._tasks_by_id[dataset._ids.claim(ObjectKind.TASK, task.id)] = task - task.register_label_set(dataset._label_sets) dataset._annotations = list(annotations) dataset._schema = schema if import_state is not None: diff --git a/packages/timenet/src/timenet/dataset/edit.py b/packages/timenet/src/timenet/dataset/edit.py index c51fba7eb..7967907ef 100644 --- a/packages/timenet/src/timenet/dataset/edit.py +++ b/packages/timenet/src/timenet/dataset/edit.py @@ -13,7 +13,7 @@ from dataclasses import replace from pathlib import Path -from timenet.dataset.composition import _copy_annotation, _fits +from timenet.dataset.composition import _fits from timenet.dataset.dataset import TimeFDataset from timenet.dataset.record import Record from timenet.dataset.source import Source @@ -24,6 +24,7 @@ from timenet.registry._paths import release_dir, split_release_dir from timenet.registry.dependencies import open_local_version from timenet.types import Annotation, DatasetSchema, EntityId, Task, TimeInterval, TimeSpan, Version +from timenet.types.annotations import _copy_annotation from timenet.types.object_kind import ObjectKind from timenet.types.references import DatasetRef from timenet.writer import TimeFWriter @@ -242,7 +243,7 @@ def reachable(task: Task, annotation: Annotation) -> bool: def _copied(annotations: tuple[Annotation, ...], copies: dict[int, Annotation]) -> tuple[Annotation, ...]: """Return a copy of each annotation, and keep each copy in ``copies``. - The copies have their own value and metadata (see ``composition._copy_annotation``). An + The copies have their own value and metadata (see ``annotations._copy_annotation``). An annotation that ``copies`` already holds keeps that copy, so an annotation object that two owners share gets one copy. diff --git a/packages/timenet/src/timenet/format/control_audit.py b/packages/timenet/src/timenet/format/control_audit.py index bb41882f2..faa99bfc0 100644 --- a/packages/timenet/src/timenet/format/control_audit.py +++ b/packages/timenet/src/timenet/format/control_audit.py @@ -56,6 +56,7 @@ def _sql_in(values: Iterable[str]) -> str: ("signals", "signal_id"), ("annotation_contents", "content_id"), ("annotation_occurrences", "occurrence_id"), + ("label_sets", "label_set_id"), ("tasks", "task_id"), ) """The primary id column of each entity table.""" @@ -112,8 +113,8 @@ def _sql_in(values: Iterable[str]) -> str: OR list_contains([label.name FOR label IN labels], '') OR list_contains([label.description FOR label IN labels], '') UNION ALL - SELECT 'tasks.label_set' FROM tasks - WHERE (label_set IS NOT NULL) <> (task_type = '{TaskType.CLASSIFICATION}') + SELECT 'tasks.label_set_id' FROM tasks + WHERE (label_set_id IS NOT NULL) <> (task_type = '{TaskType.CLASSIFICATION}') UNION ALL {_ENTITY_ID_CHECKS} LIMIT 1 @@ -122,7 +123,7 @@ def _sql_in(values: Iterable[str]) -> str: _LABELED_TARGETS = f""" WITH labeled AS ( SELECT t.task_id, [label.name FOR label IN s.labels] AS labels, s.multi_label - FROM tasks t JOIN label_sets s ON s.name = t.label_set + FROM tasks t JOIN label_sets s ON s.label_set_id = t.label_set_id ), targets AS ( SELECT x.task_id, x.target_kind = '{TaskTargetKind.TEXT}' AS is_text, x.text_value AS label, diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index d8bf92455..f18545149 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -6,7 +6,7 @@ from collections import ChainMap, Counter, defaultdict from collections.abc import Callable, Collection, Container, Iterable, Iterator, Mapping, MutableMapping from fractions import Fraction -from functools import cached_property, partial +from functools import partial from graphlib import CycleError, TopologicalSorter import json import os @@ -23,7 +23,7 @@ from timenet.dataset.composition import ImportMap from timenet.errors import TimeFFormatError, TimeFNotFoundError, TimeFReaderError, TimeFValidationError from timenet.format.annotation_codec import decode_annotation_value -from timenet.format.control_schema import ANNOTATED_OBJECT_TYPES, LAST_ID_PREFIX, TABLES +from timenet.format.control_schema import ANNOTATED_OBJECT_TYPES, LAST_ID_PREFIX from timenet.format.duckdb import CATALOG_ERRORS, check_control_schema, connect_control from timenet.format.stored_array import ChunkLocation from timenet.format.stored_values import ( @@ -50,6 +50,7 @@ Annotation, ArrayHandle, InputModality, + Label, LabelSet, MissingValues, Task, @@ -73,7 +74,8 @@ ValueLoaderFactory = Callable[[int, TimeSeriesSpec, int], ArrayHandle] _OwnerId: TypeAlias = int -"""The ID of an annotation's owner: the public ID of a Record, Signal, or Task, or the catalog row ID of a Source.""" +"""The ID of an annotation's owner: the key of its row in the table that +:data:`~timenet.format.control_schema.ANNOTATED_OWNER_TABLES` names for its kind.""" _AnnotationOwner: TypeAlias = tuple[ObjectKind, _OwnerId | None] """An annotation's owner: its kind and its ID. The ID is ``None`` for the Dataset itself.""" _OccurrenceId: TypeAlias = int @@ -251,7 +253,7 @@ class _TaskRow(NamedTuple): scope_end: int | None scope_signal_ids: list[int] | None has_inline_targets: bool - label_set: str | None + label_set_id: int | None unit: str | None target_name: str | None mode: str | None @@ -790,8 +792,6 @@ def close(self) -> None: self._disconnect() self._clock_cache.clear() self._task_facts = None - # Drop the cached label sets, so that a reopened reader reads them again. - self.__dict__.pop("_label_sets_by_name", None) self._known_record_ids.clear() self._chunk_strings.clear() @@ -1629,7 +1629,7 @@ def read_task_table( ) SELECT t.task_id, t.task_type, t.prompt, t.rationale, t.scope_type, t.scope_start, t.scope_end, t.scope_signal_ids, - t.label_set, t.unit, t.target_name, t.mode, t.has_inline_targets, + t.label_set_id, t.unit, t.target_name, t.mode, t.has_inline_targets, t.input_modalities, t.split, coalesce(inputs.input_record_ids, []) AS input_record_ids, coalesce(parents.from_task_ids, []) AS from_task_ids, @@ -1853,44 +1853,34 @@ def _task_rows(self, task_ids: list[int] | None) -> list[_TaskRow]: _TaskRow._make, ) - @cached_property - def _label_sets_by_name(self) -> dict[str, LabelSet]: - """The stored label sets by name. + def _read_label_sets(self, label_set_ids: Collection[int]) -> dict[int, LabelSet]: + """Read the label sets with these ids, with their annotations. - The reader reads each set once. Tasks that use the same set share one object. + Returns: + The label sets by id. Raises: - TimeFFormatError: If a stored label set is not valid. + TimeFFormatError: If a stored label set or one of its annotations is not valid. """ - columns = ", ".join(TABLES["label_sets"].column_names) - # ruff: ignore[hardcoded-sql-expression] - fixed identifiers - rows = self._query_table(f"SELECT {columns} FROM label_sets").to_pylist() - label_sets: dict[str, LabelSet] = {} - for row in rows: + rows = self._query( + "SELECT label_set_id, labels, multi_label, ordered FROM label_sets " + "WHERE label_set_id IN (SELECT unnest(?))", + [sorted(label_set_ids)], + ) + annotations = self._read_annotations({ObjectKind.LABEL_SET: label_set_ids}) + label_sets: dict[int, LabelSet] = {} + for label_set_id, labels, multi_label, ordered in rows: try: - label_set = LabelSet.model_validate(row) - except ValidationError as exc: - raise TimeFFormatError(f"the stored label set {row['name']!r} is not valid: {exc}") from exc - label_sets[label_set.name] = label_set + label_sets[label_set_id] = LabelSet( + [Label.model_validate(label) for label in labels], + multi_label=multi_label, + ordered=ordered, + annotations=annotations.get((ObjectKind.LABEL_SET, label_set_id), ()), + ) + except (ValidationError, TimeFValidationError) as exc: + raise TimeFFormatError(f"the stored label set {label_set_id!r} is not valid: {exc}") from exc return label_sets - def _label_set(self, task_id: int, name: str | None) -> LabelSet: - """Return the stored label set that a classification task names. - - Returns: - The shared label set. - - Raises: - TimeFFormatError: If the task names no label set, no label set has this name, or a stored - label set is not valid. - """ - if name is None: - raise TimeFFormatError(f"classification task {task_id!r} has no label set") - label_set = self._label_sets_by_name.get(name) - if label_set is None: - raise TimeFFormatError(f"tasks refer to a missing label set {name!r}") - return label_set - def _target_rows(self, task_ids: list[int] | None) -> list[_TargetRow]: """Read the target rows for one batch of tasks, or every task, in task and position order. @@ -2268,7 +2258,7 @@ class _TaskBuilder: Records, Signals, and annotations that tasks refer to outside that set once, and keeps every task that another task derives from until its dependants are built. A one-pass read is one scope. A stream starts a new scope for each batch, so its batches share only the caller's - objects. + objects and the label sets, which the builder reads once. """ def __init__( # ruff: ignore[too-many-arguments] - one argument per task selection axis @@ -2306,6 +2296,8 @@ def __init__( # ruff: ignore[too-many-arguments] - one argument per task select self.dataset_occurrences: list[Annotation] = [] """The occurrences of the dataset itself that the scope read for its tasks' references.""" self.retained: _TasksById = {} + self._label_sets: dict[int, LabelSet] = {} + """The stored label sets by id, which the read builds once and all its tasks share.""" # The lookups of the caller's objects that a stream's batches share, or None for one pass. self._given: _Lookups | None = None self._dataset_annotations_indexed = dataset_annotations is not None @@ -2326,8 +2318,8 @@ def iter_batches(self) -> Iterator[list[Task]]: """Yield the selected tasks in task ID order, one bounded batch at a time, each in its own scope. The tasks of a batch share the objects the batch builds, and no batch shares them with - another. The builder forgets a batch's objects before it yields them, so it holds nothing - of a batch it has handed out. + another, apart from the label sets. The builder forgets a batch's other objects before it + yields them. Yields: Each batch's built tasks. @@ -2454,6 +2446,24 @@ def _build_batch(self, task_ids: list[EntityId] | None) -> list[Task]: self.retained[task_id] = task return [self._task(task_id, built) for task_id in task_ids] + def _label_set(self, task_id: int, label_set_id: int | None) -> LabelSet: + """Return the stored label set that a classification task names. + + The read builds each label set once. Tasks that use the same set share one object. + + Returns: + The shared label set. + + Raises: + TimeFFormatError: If the task names no label set, or no label set has this id. + """ + if label_set_id is None: + raise TimeFFormatError(f"classification task {task_id!r} has no label set") + label_set = self._label_sets.get(label_set_id) + if label_set is None: + raise TimeFFormatError(f"classification task {task_id!r} refers to missing label set {label_set_id!r}") + return label_set + def _task(self, task_id: EntityId, built: _TasksById) -> Task: """Return a task from this batch or the retained parents. @@ -2491,6 +2501,10 @@ def _build(self, task_ids: list[EntityId] | None) -> _TasksById: if task_ids is None: reader._check_owners(task_annotations.keys(), {ObjectKind.TASK: {row.task_id for row in task_rows}}) + label_set_ids = {row.label_set_id for row in task_rows if row.label_set_id is not None} + label_set_ids -= self._label_sets.keys() + if label_set_ids: + self._label_sets.update(reader._read_label_sets(label_set_ids)) self._ensure_records( {record_id for refs in record_refs.values() for ids in refs.values() for record_id in ids} | {row.record_id for row in target_rows if row.record_id is not None} @@ -2556,8 +2570,8 @@ def _build(self, task_ids: list[EntityId] | None) -> _TasksById: **decode_task_payload(task_type, row.payload_columns), } if task_type is TaskType.CLASSIFICATION: - kwargs["label_set"] = reader._label_set(row.task_id, row.label_set) - elif row.label_set is not None: + kwargs["label_set"] = self._label_set(row.task_id, row.label_set_id) + elif row.label_set_id is not None: raise TimeFFormatError( f"{task_type} task {row.task_id!r} has a label set, but only a classification task can have one" ) diff --git a/packages/timenet/src/timenet/format/control_schema.py b/packages/timenet/src/timenet/format/control_schema.py index d4b417fe2..fd6448d17 100644 --- a/packages/timenet/src/timenet/format/control_schema.py +++ b/packages/timenet/src/timenet/format/control_schema.py @@ -3,11 +3,9 @@ The SQL table definitions and the writer's Arrow batch schemas come from :data:`CONTROL_TABLES`. The audit also uses it to examine keys and references. Define schema changes there. -This is version 2 of the control schema. Every table except ``label_sets`` uses ids as keys. -Records, Signals, and Tasks use the public ids that the caller sees. All other objects use the -writer's storage ids. - -Each label set has a unique name in a dataset. The ``label_sets`` table uses this name as its key. A foreign key stores the key of the row that it refers to. +This is version 3 of the control schema. Every table uses ids as keys. +Records, Signals, and Tasks use the public ids that the caller sees. All other objects, label sets +included, use the writer's storage ids. A foreign key stores the key of the row that it refers to. Fixed-shape fields are typed columns, so SQL can filter them. Only the free-form ``metadata`` dictionaries are JSON. The database has no persisted constraints. The writer keeps @@ -124,6 +122,7 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column ObjectKind.SOURCE: ("sources", "source_id"), ObjectKind.SIGNAL: ("signals", "signal_id"), ObjectKind.TASK: ("tasks", "task_id"), + ObjectKind.LABEL_SET: ("label_sets", "label_set_id"), } """The owner table and key column of each annotated object type. The Dataset has no owner row.""" @@ -281,24 +280,28 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column unique=(("occurrence_id",),), foreign_keys=(ForeignKey("content_id", "annotation_contents", "content_id"),), ) -"""The owner is ``(object_type, object_id)``. ``object_id`` is the public id of a Record, Signal, -or Task, or the storage id of a Source. Ids are unique only within one kind, so the type is part of -the reference. ``object_id`` is NULL exactly when the owner is the Dataset.""" +"""The owner is ``(object_type, object_id)``. ``object_id`` is the key of the owner row in the table +that :data:`ANNOTATED_OWNER_TABLES` names for its type. Ids are unique only within one kind, so the type +is part of the reference. ``object_id`` is NULL exactly when the owner is the Dataset.""" LABEL_SETS = Table( "label_sets", ( - _required("name"), + _key("label_set_id"), _required("labels", ColumnType.LABEL_LIST), _required("multi_label", ColumnType.BOOLEAN), + _required("ordered", ColumnType.BOOLEAN), ), - unique=(("name",),), + unique=(("label_set_id",),), ) """Each row stores one label set that a classification task uses. The ``labels`` column holds one ``{name, description}`` entry per label, in label order. -If a label has no description, ``description`` is NULL. Tasks refer to a set by its name in -``tasks.label_set``.""" +``ordered`` says whether that order is part of the identity of the set. +If a label has no description, ``description`` is NULL. Tasks refer to a set by its id in +``tasks.label_set_id``. The writer gives each distinct set one id, in the order that tasks first use it. +The annotations of a set are part of its identity, so two sets with the same labels and different +annotations are two rows.""" TASKS = Table( "tasks", @@ -313,7 +316,7 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column Column("scope_end", ColumnType.BIGINT), Column("scope_signal_ids", ColumnType.BIGINT_LIST), _required("has_inline_targets", ColumnType.BOOLEAN), - Column("label_set", ColumnType.VARCHAR), + Column("label_set_id", ColumnType.BIGINT), Column("unit", ColumnType.VARCHAR), Column("target_name", ColumnType.VARCHAR), Column("mode", ColumnType.VARCHAR), @@ -323,7 +326,7 @@ def _required(name: str, column_type: ColumnType = ColumnType.VARCHAR) -> Column unique=(("task_id",),), foreign_keys=( ForeignKey("scope_signal_ids", "signals", "signal_id", many=True), - ForeignKey("label_set", "label_sets", "name"), + ForeignKey("label_set_id", "label_sets", "label_set_id"), ), ) diff --git a/packages/timenet/src/timenet/format/control_writer.py b/packages/timenet/src/timenet/format/control_writer.py index 51d05a10a..8b17d987d 100644 --- a/packages/timenet/src/timenet/format/control_writer.py +++ b/packages/timenet/src/timenet/format/control_writer.py @@ -39,7 +39,8 @@ from timenet.types import ( Annotation, ArrayHandle, - LabelSetRegistry, + ClassificationTask, + LabelSet, ObjectKind, Task, TSCorrespondenceTask, @@ -207,6 +208,64 @@ def flush(self) -> None: self.signals.flush() +class _LabelSetRows: + """The ``label_sets`` rows of one write, one row for each distinct label set. + + Equal sets (see :meth:`LabelSet._content_key`) get one id, in the order that tasks first use them. + The row keeps the labels and the annotations of the first set. Each set object is checked once per write. + """ + + def __init__(self, ids: Iterator[int]) -> None: + """Start with no label sets. + + Args: + ids: The storage ids for new label sets. + """ + self._new_ids = ids + self._sets: dict[Hashable, tuple[int, dict[str, object], tuple[Annotation, ...]]] = {} + self._by_object: dict[int, tuple[LabelSet, int]] = {} + + def register(self, label_set: LabelSet) -> int: + """Add a label set. + + Returns: + The stored id of the set. + + Raises: + TimeFValidationError: If the set is not valid. A set can change after a task gets it. + """ # ruff: ignore[docstring-extraneous-exception] - _validate raises it + known = self._by_object.get(id(label_set)) + if known is not None: + return known[1] + label_set._validate() + key = label_set._content_key() + stored = self._sets.get(key) + if stored is None: + label_set_id = next(self._new_ids) + row: dict[str, object] = { + "label_set_id": label_set_id, + "labels": [label.model_dump() for label in label_set.labels], + "multi_label": label_set.multi_label, + "ordered": label_set.ordered, + } + stored = self._sets[key] = (label_set_id, row, label_set.annotations) + self._by_object[id(label_set)] = (label_set, stored[0]) + return stored[0] + + def write( + self, connection: duckdb.DuckDBPyConnection, annotations: list[tuple[ObjectKind, int | None, Annotation]] + ) -> None: + """Insert the rows and queue the annotations of each set. + + Args: + connection: The open write transaction. + annotations: The annotation triples that the writer collects. This method adds to them. + """ + insert_rows(connection, TABLES["label_sets"], [row for _, row, _ in self._sets.values()]) + for label_set_id, _, occurrences in self._sets.values(): + annotations.extend((ObjectKind.LABEL_SET, label_set_id, annotation) for annotation in occurrences) + + class DuckDBControlWriter: """Serialize the relational part of one immutable TimeF version.""" @@ -493,14 +552,13 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume TimeFValidationError: If a task refers to a Record or Signal outside the dataset. Also if the dataset registered a different Record or Signal object under that id. Also if two tasks share an id, or a task derives from itself or a task outside the dataset. - Also if two tasks use different label sets with the same name. """ # Weak: if no other task names a streamed task, Python can free it after the writer writes it. written: WeakValueDictionary[int, Task] = WeakValueDictionary() dependencies: list[tuple[int, int, Task]] = [] annotation_refs: list[_AnnotationRef] = [] task_counts: dict[str, int] = {} - label_sets = LabelSetRegistry() + label_sets = _LabelSetRows(self._storage_ids["label_set"]) index = _TaskValidationIndex(dataset) for batch in _task_batches(tasks_source): task_rows: list[dict[str, object]] = [] @@ -511,7 +569,6 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume TimeFDataset._check_signal_refs(task, index) # A task can change after add_task, so check its labels and its label set again. task.check_targets() - task.register_label_set(label_sets) task_id = task.id written[task_id] = task task_type = str(task.task_type) @@ -526,6 +583,9 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume **self._scope_columns(task, signal_ids), "has_inline_targets": task.targets is not None, **encode_task_payload(task), + "label_set_id": ( + label_sets.register(task.label_set) if isinstance(task, ClassificationTask) else None + ), "rationale": self._pack(ObjectKind.TASK, task_id, PackedField.RATIONALE, task.rationale), "metadata": _json(task.metadata), } @@ -561,8 +621,8 @@ def _write_tasks( # ruff: ignore[too-many-arguments, too-many-positional-argume insert_rows(connection, TABLES["task_record_refs"], record_ref_rows) # Store this batch's locators now, so a streamed task source never holds them all. self.external.write_locators(connection) - # Tasks refer to a label set by its name, so the sets can follow the tasks that use them. - insert_rows(connection, TABLES["label_sets"], [label_set.model_dump() for label_set in label_sets]) + # Tasks refer to a label set by its id, so the sets can follow the tasks that use them. + label_sets.write(connection, annotations) duplicate = connection.execute( "SELECT task_id FROM tasks GROUP BY task_id HAVING count(*) > 1 LIMIT 1" ).fetchone() diff --git a/packages/timenet/src/timenet/format/duckdb.py b/packages/timenet/src/timenet/format/duckdb.py index 74640e588..b689fe8eb 100644 --- a/packages/timenet/src/timenet/format/duckdb.py +++ b/packages/timenet/src/timenet/format/duckdb.py @@ -14,13 +14,14 @@ CONTROL_FILE = "control.duckdb" """Name of the relational control-plane database in a TimeF version.""" -CONTROL_SCHEMA_VERSION = 2 +CONTROL_SCHEMA_VERSION = 3 """Schema version written into :data:`CONTROL_FILE`, and the newest that this TimeNet reads.""" -OLDEST_READABLE_CONTROL_SCHEMA_VERSION = 2 +OLDEST_READABLE_CONTROL_SCHEMA_VERSION = 3 """The oldest schema version of :data:`CONTROL_FILE` that this TimeNet reads. -Version 2 is the schema of TimeNet 0.12. Version 1 names layouts used during early development, and no TimeNet reads them. +Version 3 stores label sets by id, with their annotations. Version 2 is the schema of TimeNet 0.12. Version 1 names +layouts used during early development. This TimeNet reads neither of them. """ _IN_MEMORY_DATABASE = "an in-memory control database" diff --git a/packages/timenet/src/timenet/format/task_codec.py b/packages/timenet/src/timenet/format/task_codec.py index 400335be1..97f11aa30 100644 --- a/packages/timenet/src/timenet/format/task_codec.py +++ b/packages/timenet/src/timenet/format/task_codec.py @@ -20,7 +20,6 @@ from timenet.format.task_target_kind import SPAN_KINDS, TaskTargetKind from timenet.format.task_target_row import TargetValueRow from timenet.types import ( - ClassificationTask, EntityId, InputModality, LocalizationMode, @@ -57,11 +56,11 @@ class TaskRelationshipField(StrEnum): """The fields stored in ``task_annotation_refs``.""" -PAYLOAD_COLUMNS = ("label_set", "unit", "target_name", "mode") +PAYLOAD_COLUMNS = ("unit", "target_name", "mode") """The ``tasks`` columns that hold configuration for each task type. -Unused columns are NULL. A classification task stores its label set name in ``label_set``. -The ``label_sets`` table stores the set itself.""" +Unused columns are NULL. The writer sets the ``label_set_id`` column of a classification task itself. +The ``label_sets`` table stores the set.""" def decode_input_modalities(values: list[str] | None) -> frozenset[InputModality]: @@ -182,9 +181,7 @@ def encode_task_payload(task: Task) -> dict[str, str | None]: One value per :data:`PAYLOAD_COLUMNS`, ``None`` where the task type has no such field. """ columns: dict[str, str | None] = dict.fromkeys(PAYLOAD_COLUMNS) - if isinstance(task, ClassificationTask): - columns["label_set"] = task.label_set.name - elif isinstance(task, ScalarPredictionTask): + if isinstance(task, ScalarPredictionTask): columns["unit"] = None if task.unit is None else str(task.unit) columns["target_name"] = task.target_name elif isinstance(task, TemporalLocalizationTask): diff --git a/packages/timenet/src/timenet/reader/load_plan.py b/packages/timenet/src/timenet/reader/load_plan.py index 3b2e8db01..d32ae7367 100644 --- a/packages/timenet/src/timenet/reader/load_plan.py +++ b/packages/timenet/src/timenet/reader/load_plan.py @@ -67,7 +67,7 @@ def _task_children(item: Task) -> tuple[object, ...]: item.prompt, item.rationale, *(item.targets or ()), - *item.annotations, + *item.walk_annotations(), *item.inputs, *getattr(item, "candidate_records", ()), *item.input_annotations, diff --git a/packages/timenet/src/timenet/testing/builders.py b/packages/timenet/src/timenet/testing/builders.py index 2a9c81cdb..d76164c07 100644 --- a/packages/timenet/src/timenet/testing/builders.py +++ b/packages/timenet/src/timenet/testing/builders.py @@ -16,7 +16,6 @@ DatasetMetadata, Domain, EntityId, - Label, LabelSet, License, LocalizationMode, @@ -31,9 +30,9 @@ ) -_RHYTHM_LABELS = LabelSet(name="rhythm", labels=(Label(name="normal"), Label(name="abnormal"))) +_RHYTHM_LABELS = LabelSet(["normal", "abnormal"]) """The label set of the fixture's rhythm classification task.""" -_EVENT_LABELS = LabelSet(name="event", labels=(Label(name="onset"), Label(name="offset"))) +_EVENT_LABELS = LabelSet(["onset", "offset"]) """The label set of the fixture's scoped event classification task.""" diff --git a/packages/timenet/src/timenet/types/__init__.py b/packages/timenet/src/timenet/types/__init__.py index 7fcface81..343bf76d6 100644 --- a/packages/timenet/src/timenet/types/__init__.py +++ b/packages/timenet/src/timenet/types/__init__.py @@ -13,7 +13,7 @@ from timenet.types.domains import Domain from timenet.types.handles import ArrayHandle, ValueHandle from timenet.types.ids import EntityId, check_id -from timenet.types.label_sets import Label, LabelSet, LabelSetRegistry +from timenet.types.label_sets import Label, LabelSet from timenet.types.licenses import License from timenet.types.metadata import DatasetFilter, DatasetMetadata, DatasetSchema, ListedRelease from timenet.types.modalities import InputModality @@ -68,7 +68,6 @@ "InputModality", "Label", "LabelSet", - "LabelSetRegistry", "LazyText", "LazyTextList", "License", diff --git a/packages/timenet/src/timenet/types/annotations.py b/packages/timenet/src/timenet/types/annotations.py index 417540a5a..fece3bd8c 100644 --- a/packages/timenet/src/timenet/types/annotations.py +++ b/packages/timenet/src/timenet/types/annotations.py @@ -13,6 +13,7 @@ class synthesis. :class:`AnnotationDescriptor` is the type-level projection stor """ from collections.abc import Hashable +from copy import deepcopy from dataclasses import KW_ONLY, dataclass, field, replace from enum import StrEnum, unique import json @@ -184,6 +185,23 @@ def _new_occurrence(self) -> "Annotation": return replace(self) +def _copy_annotation(original: Annotation) -> Annotation: + """Copy an annotation occurrence as a value, with the handle of a stored value. + + The copy has deep copies of the value, the metadata, and the occurrence metadata. So a change to + one of them in the copy leaves the original as it is. Imports and edits copy annotations this way. + + Returns: + The copy. + """ + return replace( + original, + value=deepcopy(original.value), + metadata=deepcopy(original.metadata), + occurrence_metadata=deepcopy(original.occurrence_metadata), + ) + + def annotation_type_of(annotation: Annotation) -> AnnotationType: """Return the :class:`AnnotationType` shape of an annotation instance. @@ -203,8 +221,8 @@ class SupportsAnnotate(Protocol): ``annotate`` binds a fresh occurrence of ``annotation`` to this object and returns it; the returned occurrence is what a Task names in ``input_annotations`` or ``target_annotations``. - ``annotations`` lists every occurrence attached so far. Records, Sources, Signals, Tasks, and - the dataset itself implement it. A :class:`~timenet.dataset.SignalSelection` does not: it fans + ``annotations`` lists every occurrence attached so far. Records, Sources, Signals, Tasks, label + sets, and the dataset itself implement it. A :class:`~timenet.dataset.SignalSelection` does not: it fans one annotation out over several Signals and returns every occurrence it created. The contract is checked statically; there is no runtime ``isinstance`` support because ``annotations`` is a property. diff --git a/packages/timenet/src/timenet/types/label_sets.py b/packages/timenet/src/timenet/types/label_sets.py index 0a33f469d..45157a24e 100644 --- a/packages/timenet/src/timenet/types/label_sets.py +++ b/packages/timenet/src/timenet/types/label_sets.py @@ -1,12 +1,14 @@ """The allowed labels for a classification target.""" -from collections.abc import Iterator, Sequence -from typing import Self - -from pydantic import StrictBool, model_validator +from collections import Counter +from collections.abc import Hashable, Iterable, Sequence +from dataclasses import dataclass +import json from timenet.errors import TimeFValidationError +from timenet.types._fields import as_tuple from timenet.types._model import TimeFModel +from timenet.types.annotations import Annotation, SupportsAnnotate from timenet.types.wire import NonEmptyString @@ -19,49 +21,179 @@ class Label(TimeFModel): """What the label means, such as ``The subject is awake.``.""" -class LabelSet(TimeFModel): +def _as_label(item: object) -> object: + """Turn a plain string into a label without a description. Leave every other item as it is. + + Returns: + The label, or the item. :meth:`LabelSet._validate` checks its type. + + Raises: + TimeFValidationError: If the string is empty. + """ + if not isinstance(item, str): + return item + if not item: + raise TimeFValidationError("a label set cannot have an empty label") + return Label(name=item) + + +@dataclass(init=False, eq=False) +class LabelSet(SupportsAnnotate): """The allowed labels for a classification target. - A target must contain one label unless ``multi_label=True``. + A target must contain one label unless ``multi_label=True``. By default the label order does not matter. + Use ``ordered=True`` for ordinal labels or for options whose order matters. Tasks that ask the same kind of question share one label set:: SLEEP_STAGE = LabelSet( - name="sleep_stage", - labels=( - Label(name="awake", description="The subject is awake."), - Label(name="non_rem_sleep"), - Label(name="rem_sleep"), - ), + [Label(name="awake", description="The subject is awake."), "non_rem_sleep", "rem_sleep"], + annotations=(Annotation(key="name", value="sleep_stage"),), ) ClassificationTask(inputs=(record,), targets=("awake",), label_set=SLEEP_STAGE) - Each name identifies one label set in a dataset. + A plain string is a label without a description, so ``LabelSet(["yes", "no"])`` is enough when no label + needs one. + + A label set has no name. Two sets are the same set if they have the same labels (with the same + descriptions), the same ``multi_label`` and the same ``ordered``. They must also have the same annotations + (with the same values). Then ``==`` is true for them. If ``ordered`` is ``False``, the labels can be in any + order. If ``ordered`` is ``True``, the labels must be in the same order. The annotations can be in any order. + A dataset stores equal sets once, with the label order and the annotation order of the first of them that + it writes. Two sets with the same labels and different annotations are two sets:: + + LabelSet(["yes", "no"], annotations=(Annotation(key="name", value="afib_present"),)) + LabelSet(["yes", "no"], annotations=(Annotation(key="name", value="signal_noisy"),)) """ - name: NonEmptyString - """The name of the set within a dataset, such as ``sleep_stage``.""" labels: tuple[Label, ...] - """The allowed labels in a fixed order.""" - multi_label: StrictBool = False + """The allowed labels in order. For an unordered set, a dataset keeps the order of the first equal set that + it writes.""" + multi_label: bool = False """If ``False``, a target holds exactly one label from the set. If ``True``, a target holds all labels from the set that apply. An empty target means that no label applies.""" + ordered: bool = False + """If ``True``, the label order is part of the identity of the set. If ``False``, the order does not matter. + + Pass the labels as a list or a tuple, not a Python set, so that the stored order is fixed.""" + annotations: tuple[Annotation, ...] = () + """Annotations that describe the set, such as ``Annotation(key="name", value="sleep_stage")``. + + An annotation cannot have a span. The set holds its own occurrence of each annotation. + The annotations are part of the identity of the set.""" + + def __init__( + self, + labels: Iterable[Label | str], + *, + multi_label: bool = False, + ordered: bool = False, + annotations: Iterable[Annotation] = (), + ) -> None: + """Build a label set, make sure that it is valid, and attach the annotations. + + Args: + labels: The allowed labels in order. A plain string is a label without a description. + multi_label: Whether a target can hold more than one label. + ordered: Whether the label order is part of the identity of the set. + annotations: Annotations that describe the set. They are part of the identity of the set. + + Raises: + TimeFValidationError: If the set is not valid (see :meth:`_validate`). + """ + if isinstance(labels, (set, frozenset)): + raise TimeFValidationError("a label set needs labels in a fixed order, got a set") + self.labels = tuple(_as_label(label) for label in as_tuple(labels)) # ty: ignore[invalid-assignment] + self.multi_label = multi_label + self.ordered = ordered + self.annotations = as_tuple(annotations) + self._validate() + self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) + + def _validate(self) -> None: + """Make sure that the set is valid. The writer calls this again, because a set can change after it is built. - @model_validator(mode="after") - def _valid_set(self) -> Self: + Raises: + TimeFValidationError: If a label is not a Label, the set has no label or repeats a name, + ``multi_label`` or ``ordered`` is not a bool, or an annotation is not an Annotation or has a span. + """ + for label in self.labels: + if not isinstance(label, Label): + raise TimeFValidationError(f"a label set takes Label objects, got {type(label).__name__}") if not self.labels: - raise ValueError(f"label set {self.name!r} must list at least one label") + raise TimeFValidationError("a label set must list at least one label") if len(set(self.names)) != len(self.labels): - raise ValueError(f"label set {self.name!r} lists a label more than once") - return self + raise TimeFValidationError(f"{self} lists a label more than once") + if not isinstance(self.multi_label, bool): + raise TimeFValidationError(f"{self} multi_label must be a bool, got {type(self.multi_label).__name__}") + if not isinstance(self.ordered, bool): + raise TimeFValidationError(f"{self} ordered must be a bool, got {type(self.ordered).__name__}") + for annotation in self.annotations: + self._check_annotation(annotation) + + def _check_annotation(self, annotation: object) -> None: + """Make sure that an annotation can describe the set. + + Raises: + TimeFValidationError: If the annotation is not an Annotation or has a span. + """ + if not isinstance(annotation, Annotation): + raise TimeFValidationError(f"{self} takes Annotation objects, got {type(annotation).__name__}") + if annotation.span is not None: + raise TimeFValidationError( + f"annotation {annotation.key!r} has a span, but {self} takes annotations without a span" + ) + + def annotate(self, annotation: Annotation) -> Annotation: + """Attach one annotation occurrence and return it. + + Returns: + The attached occurrence. + + Raises: + TimeFValidationError: If the annotation has a span. A label set has no timeline. + """ # ruff: ignore[docstring-extraneous-exception] - _check_annotation raises it + self._check_annotation(annotation) + attached = annotation._new_occurrence() + self.annotations = (*self.annotations, attached) + return attached @property def names(self) -> tuple[str, ...]: """The name of each label, in label order.""" return tuple(label.name for label in self.labels) + def __str__(self) -> str: + """Return the labels of the set, for error messages.""" + return f"label set {list(self.names)}" + + def __eq__(self, other: object) -> bool: + """Return whether the sets have the same content key (see :meth:`_content_key`).""" + if not isinstance(other, LabelSet): + return NotImplemented + return self._content_key() == other._content_key() + + __hash__ = None + + def _content_key(self) -> tuple[Hashable, ...]: + """Return the key that makes two sets equal. + + The key ignores label order unless the set is ordered. It ignores the order of the annotations. + It compares every field of each annotation. + + Returns: + The hashable key. + """ + annotations = Counter(_occurrence_key(annotation) for annotation in self.annotations) + return ( + self.multi_label, + self.ordered, + self.labels if self.ordered else frozenset(self.labels), + frozenset(annotations.items()), + ) + def check_targets(self, targets: Sequence[str]) -> None: """Make sure that the target labels follow the rules of this set. @@ -82,60 +214,24 @@ def check_targets(self, targets: Sequence[str]) -> None: names = set(self.names) unknown = sorted({label for label in targets if label not in names}) if unknown: - raise TimeFValidationError(f"labels not in label set {self.name!r}: {unknown}") + raise TimeFValidationError(f"labels not in {self}: {unknown}") if not self.multi_label and len(targets) != 1: - raise TimeFValidationError(f"{len(targets)} labels, but label set {self.name!r} takes exactly one") + raise TimeFValidationError(f"{len(targets)} labels, but {self} takes exactly one") repeated = sorted({label for label in targets if targets.count(label) > 1}) if repeated: - raise TimeFValidationError(f"repeated labels of label set {self.name!r}: {repeated}") + raise TimeFValidationError(f"repeated labels of {self}: {repeated}") -class LabelSetRegistry: - """The label sets of a dataset, with one set per name. +def _occurrence_key(annotation: Annotation) -> tuple[Hashable, ...]: + """Return a key that is equal for two occurrences if every field of them is equal. - The sets stay in the order in which tasks first added them. + Returns: + The hashable key. """ - - def __init__(self) -> None: - """Start with no label sets.""" - self._by_name: dict[str, LabelSet] = {} - - def register_or_reuse_matching_label_set(self, label_set: LabelSet) -> LabelSet: - """Store a label set by name. - - If the registry already holds an equal set with this name, return that set. - - Returns: - The set that the registry holds under the name. - - Raises: - TimeFValidationError: If a different set already has the same name. - """ - known = self._by_name.setdefault(label_set.name, label_set) - if known is not label_set and known != label_set: - raise TimeFValidationError( - f"label set {label_set.name!r} differs from another set with that name. " - "A name stands for one label set in a dataset" - ) - return known - - def copy(self) -> "LabelSetRegistry": - """Return a separate registry with the same label sets. - - The dataset uses the copy to check a batch before adding its tasks. - If a task fails, the dataset discards the copy and keeps its original registry. - - Returns: - The copy. - """ - copy = LabelSetRegistry() - copy._by_name = dict(self._by_name) - return copy - - def __iter__(self) -> Iterator[LabelSet]: - """Iterate over the sets in the order in which tasks first added them. - - Returns: - The iterator. - """ - return iter(self._by_name.values()) + confidence = annotation.confidence + return ( + annotation._content_key, + annotation.source, + float.hex(confidence) if isinstance(confidence, float) else confidence, + json.dumps(annotation.occurrence_metadata, sort_keys=True), + ) diff --git a/packages/timenet/src/timenet/types/object_kind.py b/packages/timenet/src/timenet/types/object_kind.py index 03a975777..19b3cb11c 100644 --- a/packages/timenet/src/timenet/types/object_kind.py +++ b/packages/timenet/src/timenet/types/object_kind.py @@ -12,6 +12,7 @@ class ObjectKind(StrEnum): SOURCE = "Source" SIGNAL = "Signal" TASK = "Task" + LABEL_SET = "LabelSet" ANNOTATION = "Annotation" ANNOTATION_CONTENT = "Content" AXIS = "Axis" diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 4d1754cdb..899c0ac1e 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -1,6 +1,7 @@ """Declarative modelling tasks over records and signals.""" from collections.abc import Iterable, Iterator, Mapping +from copy import copy as shallow_copy from dataclasses import dataclass, field, replace from enum import StrEnum, unique from functools import lru_cache @@ -11,9 +12,9 @@ from timenet.errors import TimeFValidationError from timenet.json import JsonMapping, validate_json_mapping from timenet.types._fields import TupleField, ValueField, as_tuple -from timenet.types.annotations import Annotation, SupportsAnnotate, value_type_of +from timenet.types.annotations import Annotation, SupportsAnnotate, _copy_annotation, value_type_of from timenet.types.ids import EntityId, assigned_id -from timenet.types.label_sets import LabelSet, LabelSetRegistry +from timenet.types.label_sets import LabelSet from timenet.types.modalities import InputModality from timenet.types.spans import Span, StepSpan, TimeSpan from timenet.types.splits import Split, parse_split @@ -360,15 +361,6 @@ def check_against_scope(self) -> None: def check_targets(self) -> None: """Make sure that the final targets follow the rules of this task class.""" - def register_label_set(self, label_sets: LabelSetRegistry) -> None: - """Add the label set of the task to the dataset's label sets. - - Only a classification task has a label set. - - Args: - label_sets: The label sets of the dataset so far. - """ - @final @dataclass(kw_only=True) @@ -408,8 +400,7 @@ def check_targets(self) -> None: for value in values: if not isinstance(value, TextHandle): raise TimeFValidationError( - f"{name} has label set {self.label_set.name!r}, so each label must be text, " - f"not {type(value).__name__}" + f"{name} has {self.label_set}, so each label must be text, not {type(value).__name__}" ) labels.append(value.read()) try: @@ -417,19 +408,32 @@ def check_targets(self) -> None: except TimeFValidationError as exc: raise TimeFValidationError(f"{name}: {exc}") from exc - def register_label_set(self, label_sets: LabelSetRegistry) -> None: - """Add the label set of the task to the dataset's label sets. + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of the task, then those of its label set. + + Yields: + Each annotation occurrence, in the order above. + """ + yield from self.annotations + yield from self.label_set.annotations + + def with_references(self, records: Mapping[EntityId, "Record"], signals: Mapping[EntityId, "Signal"]) -> Self: + """Return a copy of the task that references the given Records and Signals, by ID. + + See :meth:`Task.with_references`. The copy also gets its own copy of the label set and of its + annotations, so a change to the set of the copy leaves the set of the task as it is. Args: - label_sets: The label sets of the dataset so far. + records: The Records for the copy, by ID. + signals: The Signals for the copy, by ID. - Raises: - TimeFValidationError: If a different set already has the same name. + Returns: + The copy. """ - try: - label_sets.register_or_reuse_matching_label_set(self.label_set) - except TimeFValidationError as exc: - raise TimeFValidationError(f"ClassificationTask {self.id!r}: {exc}") from exc + copy = super().with_references(records, signals) + copy.label_set = shallow_copy(self.label_set) + copy.label_set.annotations = tuple(_copy_annotation(annotation) for annotation in self.label_set.annotations) + return copy @final diff --git a/packages/timenet/tests/dataset/test_composition.py b/packages/timenet/tests/dataset/test_composition.py index 9c541d8ca..0836c7043 100644 --- a/packages/timenet/tests/dataset/test_composition.py +++ b/packages/timenet/tests/dataset/test_composition.py @@ -58,12 +58,12 @@ from timenet.writer import TimeFWriter -ORDER = LabelSet(name="order", labels=(Label(name="late"),)) +ORDER = LabelSet(["late"]) _ALIAS = ParentAlias("base") _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) -SCP = LabelSet(name="scp", labels=(Label(name="normal", description="No finding."), Label(name="abnormal"))) +SCP = LabelSet([Label(name="normal", description="No finding."), "abnormal"]) def store(root, dataset): diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 7c2f24426..c34792922 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -1,3 +1,4 @@ +from dataclasses import replace from typing import Any import numpy as np @@ -16,7 +17,6 @@ DatasetMetadata, ForecastingTask, InputModality, - Label, LabelSet, License, ScalarPredictionTask, @@ -33,22 +33,7 @@ ) -TEST_LABELS = LabelSet( - name="labels", - labels=( - Label(name="N2"), - Label(name="a"), - Label(name="afib"), - Label(name="b"), - Label(name="beat"), - Label(name="c"), - Label(name="d"), - Label(name="e"), - Label(name="event"), - Label(name="walking"), - Label(name="x"), - ), -) +TEST_LABELS = LabelSet(["N2", "a", "afib", "b", "beat", "c", "d", "e", "event", "walking", "x"]) def _classification_task(record: Record, label: str, **kwargs: Any) -> ClassificationTask: @@ -179,25 +164,44 @@ def test_derive_schema(make_series): assert ds.schema is schema -def test_derive_schema_collects_dataset_and_task_annotations(make_series): +def test_derive_schema_collects_label_set_annotations(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) - record.annotate(Annotation(key="age", value=64)) - ds.annotate(Annotation(key="cohort", value="adult")) - task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - task.annotate(Annotation(key="difficulty", value=2)) + labels = ("a", "b") + ds.add_task( + task=ClassificationTask( + inputs=(record,), + targets=("a",), + label_set=LabelSet(labels, annotations=(Annotation(key="name", value="letters", unit="meter"),)), + ) + ) + ds.add_task( + task=ClassificationTask( + inputs=(record,), + targets=("b",), + label_set=LabelSet(labels, annotations=(Annotation(key="note", value="extra"),)), + ) + ) schema = ds.derive_schema() - assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"] + assert [descriptor.key for descriptor in schema.annotations] == ["name", "note"] + assert schema.annotations[0] == AnnotationDescriptor( + key="name", annotation_type=AnnotationType.STATIC, value_type="str", unit="meter" + ) -def test_derive_schema_rejects_a_task_annotation_that_conflicts_with_a_record_annotation(make_series): +def test_derive_schema_rejects_a_label_set_annotation_that_conflicts_with_a_record_annotation(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) record.annotate(Annotation(key="name", value=1)) - task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - task.annotate(Annotation(key="name", value="letters")) + ds.add_task( + task=ClassificationTask( + inputs=(record,), + targets=("a",), + label_set=LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)), + ) + ) with pytest.raises(TimeFValidationError, match="conflicting descriptors"): ds.derive_schema() @@ -206,7 +210,8 @@ def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) record.annotate(Annotation(key="age", value=64)) - task = _classification_task(record, "a") + label_set = LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)) + task = ClassificationTask(inputs=(record,), targets=("a",), label_set=label_set) task.annotate(Annotation(key="difficulty", value=2)) ds.set_task_stream([ClassificationTask], lambda: iter((task,))) assert [descriptor.key for descriptor in ds.derive_schema().annotations] == ["age"] @@ -214,15 +219,15 @@ def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series): list(ds.iter_streamed_tasks_validated()) assert ds.schema is not None - assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty"] + assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty", "name"] def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema(make_series): ds = _dataset() record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) record.annotate(Annotation(key="name", value=1)) - task = _classification_task(record, "a") - task.annotate(Annotation(key="name", value="letters")) + label_set = LabelSet(["a"], annotations=(Annotation(key="name", value="letters"),)) + task = ClassificationTask(inputs=(record,), targets=("a",), label_set=label_set) ds.set_task_stream([ClassificationTask], lambda: iter((task,))) ds.derive_schema() @@ -230,6 +235,19 @@ def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema( list(ds.iter_streamed_tasks_validated()) +def test_derive_schema_collects_dataset_and_task_annotations(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="age", value=64)) + ds.annotate(Annotation(key="cohort", value="adult")) + task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + task.annotate(Annotation(key="difficulty", value=2)) + + schema = ds.derive_schema() + + assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"] + + def test_derive_schema_dedupes_specs(make_series): ds = _dataset() # Two signals of the same modality => one distinct spec. @@ -888,10 +906,13 @@ def test_walk_annotations_goes_from_the_dataset_to_records_to_tasks(make_series) record = ds.add_record(record=Record(sources=(source,))) task = ds.add_task(task=_classification_task(record, "a")) answer = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - for owner, key in ((answer, "answer"), (task, "task"), (signal, "signal"), (source, "source"), (record, "record")): + task.label_set = replace(task.label_set) + owners = ((answer, "answer"), (task.label_set, "label_set"), (task, "task"), (signal, "signal")) + for owner, key in (*owners, (source, "source"), (record, "record")): owner.annotate(Annotation(key=key, value=key)) ds.annotate(Annotation(key="dataset", value="dataset")) - assert [a.key for a in ds.walk_annotations()] == ["dataset", "record", "source", "signal", "task", "answer"] + keys = [a.key for a in ds.walk_annotations()] + assert keys == ["dataset", "record", "source", "signal", "task", "label_set", "answer"] def test_add_tasks_allows_deriving_from_another_task_in_the_same_batch(make_series): diff --git a/packages/timenet/tests/format/test_control_writer.py b/packages/timenet/tests/format/test_control_writer.py index 764024477..6c5902e6e 100644 --- a/packages/timenet/tests/format/test_control_writer.py +++ b/packages/timenet/tests/format/test_control_writer.py @@ -99,7 +99,7 @@ def test_control_writer_serializes_recursive_hierarchy_and_shared_axis(tmp_path, assert connection.execute("SELECT count(*) FROM signals").fetchone() == (2,) assert connection.execute("SELECT count(*) FROM axes").fetchone() == (1,) assert connection.execute( - "SELECT task_type, prompt, scope_type, label_set, unit, target_name, mode FROM tasks" + "SELECT task_type, prompt, scope_type, label_set_id, unit, target_name, mode FROM tasks" ).fetchone() == ("answer", "Alive?", None, None, None, None, None) assert connection.execute( "SELECT position, target_kind, text_value, integer_value, record_id FROM task_targets" diff --git a/packages/timenet/tests/format/test_duckdb.py b/packages/timenet/tests/format/test_duckdb.py index 18ef89646..83268ce1c 100644 --- a/packages/timenet/tests/format/test_duckdb.py +++ b/packages/timenet/tests/format/test_duckdb.py @@ -59,6 +59,7 @@ def test_entity_id_columns_are_bigint_and_dataset_id_stays_varchar(tmp_path): "signal_id", "content_id", "occurrence_id", + "label_set_id", "task_id", } @@ -67,22 +68,22 @@ def test_a_newer_control_schema_names_the_file_and_both_versions(tmp_path): path = tmp_path.joinpath("control.duckdb") with connect_control(path) as connection: create_control_schema(connection) - connection.execute("UPDATE control_metadata SET value = '3'") + connection.execute("UPDATE control_metadata SET value = '4'") with pytest.raises(TimeFFormatVersionError) as caught: check_control_schema(connection) - assert str(caught.value) == f"{path} uses control schema version 3, but the newest that this timenet reads is 2." + assert str(caught.value) == f"{path} uses control schema version 4, but the newest that this timenet reads is 3." assert caught.value.subject is FormatVersionSubject.CONTROL_SCHEMA def test_a_newer_in_memory_control_schema_names_no_file(): with duckdb.connect() as connection: create_control_schema(connection) - connection.execute("UPDATE control_metadata SET value = '3'") + connection.execute("UPDATE control_metadata SET value = '4'") with pytest.raises( - TimeFFormatVersionError, match=r"^an in-memory control database uses control schema version 3, but" + TimeFFormatVersionError, match=r"^an in-memory control database uses control schema version 4, but" ): check_control_schema(connection) @@ -91,12 +92,12 @@ def test_an_older_control_schema_names_the_oldest_version_that_is_read(tmp_path) path = tmp_path.joinpath("control.duckdb") with connect_control(path) as connection: create_control_schema(connection) - connection.execute("UPDATE control_metadata SET value = '1'") + connection.execute("UPDATE control_metadata SET value = '2'") with pytest.raises(TimeFFormatVersionError) as caught: check_control_schema(connection) - assert str(caught.value) == f"{path} uses control schema version 1, but the oldest that this timenet reads is 2." + assert str(caught.value) == f"{path} uses control schema version 2, but the oldest that this timenet reads is 3." assert caught.value.path == path diff --git a/packages/timenet/tests/format/test_label_set_storage.py b/packages/timenet/tests/format/test_label_set_storage.py index c8ef67ffa..9368109a1 100644 --- a/packages/timenet/tests/format/test_label_set_storage.py +++ b/packages/timenet/tests/format/test_label_set_storage.py @@ -9,10 +9,13 @@ from timenet.errors import TimeFFormatError, TimeFValidationError from timenet.format.control_audit import audit_control_database from timenet.format.control_reader import DuckDBControlReader -from timenet.format.control_writer import DuckDBControlWriter +from timenet.format.control_writer import DuckDBControlWriter, _LabelSetRows from timenet.format.duckdb import connect_control from timenet.format.storage import StorageSources from timenet.format.stored_values import PackedValueWriter +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing.builders import assert_datasets_equal from timenet.types import ( Annotation, AnswerTask, @@ -21,6 +24,7 @@ Label, LabelSet, License, + TimePoint, TimeSeriesSpec, Version, ureg, @@ -29,9 +33,10 @@ SPEC = TimeSeriesSpec(spec_type="voltage", name="Voltage", unit_value=ureg.millivolt) STAGE_LABELS = LabelSet( - name="stage", labels=(Label(name="W"), Label(name="N1"), Label(name="R", description="REM sleep.")) + ["W", "N1", Label(name="R", description="REM sleep.")], + annotations=(Annotation(key="name", value="stage", description="Sleep stage"),), ) -FINDINGS = LabelSet(name="findings", labels=(Label(name="AF"), Label(name="PVC")), multi_label=True) +FINDINGS = LabelSet(["AF", "PVC"], multi_label=True) def _record() -> Record: @@ -62,6 +67,10 @@ def _write(dataset: TimeFDataset, path): DuckDBControlWriter(path, external=PackedValueWriter(path.parent, StorageSources())).write_hierarchy(dataset) +def _label_sets(tasks) -> list[LabelSet]: + return [task.label_set for task in tasks if isinstance(task, ClassificationTask)] + + def _read_tasks(path): with DuckDBControlReader(path) as reader: return reader.read_tasks(reader.read_records()) @@ -83,7 +92,7 @@ def release(tmp_path_factory) -> Path: ClassificationTask(inputs=(record,), targets=("PVC", "AF"), label_set=FINDINGS), # 3 ClassificationTask(inputs=(record,), target_annotations=(stage,), label_set=STAGE_LABELS), # 4 # An equal set built separately is the same set. - ClassificationTask(inputs=(record,), targets=("R",), label_set=STAGE_LABELS.model_copy()), # 5 + ClassificationTask(inputs=(record,), targets=("R",), label_set=replace(STAGE_LABELS)), # 5 # A multi-label target with no label says that none applies. ClassificationTask(inputs=(record,), targets=(), label_set=FINDINGS), # 6 ] @@ -107,17 +116,20 @@ def release_copy(release, tmp_path) -> Path: def test_each_label_set_is_stored_once_and_read_as_one_object(release): with connect_control(release, read_only=True) as connection: audit_control_database(connection, require_chunks=False) - assert connection.execute("SELECT name, labels, multi_label FROM label_sets ORDER BY name").fetchall() == [ - ("findings", [{"name": "AF", "description": None}, {"name": "PVC", "description": None}], True), + assert connection.execute( + "SELECT label_set_id, labels, multi_label, ordered FROM label_sets ORDER BY 1" + ).fetchall() == [ ( - "stage", + 1, [ {"name": "W", "description": None}, {"name": "N1", "description": None}, {"name": "R", "description": "REM sleep."}, ], False, + False, ), + (2, [{"name": "AF", "description": None}, {"name": "PVC", "description": None}], True, False), ] stage_w, _, findings, stage_n1, stage_r, none_apply = _read_tasks(release) @@ -130,84 +142,228 @@ def test_each_label_set_is_stored_once_and_read_as_one_object(release): assert none_apply.targets == () -def test_the_task_table_shows_the_label_set_name(release): +def test_a_filtered_read_still_loads_the_label_set_annotations(release): with DuckDBControlReader(release) as reader: - assert reader.read_task_table()["label_set"].to_pylist() == [ - "stage", - None, - "findings", - "stage", - "stage", - "findings", - ] + tasks = reader.read_tasks_by_id([4, 5]) + first, second = _label_sets(tasks.values()) + assert first is second + assert [(annotation.key, annotation.value) for annotation in first.annotations] == [("name", "stage")] -OTHER_STAGE_LABELS = LabelSet(name="stage", labels=(Label(name="W"), Label(name="R"))) -SAME_NAME = r"ClassificationTask [0-9]+: label set 'stage' differs from another set" +def test_the_batches_of_a_task_stream_share_one_label_set_object(release): + with DuckDBControlReader(release) as reader: + batches = list(reader.iter_task_batches(batch_size=1)) + + first, fourth = _label_sets(batches[0] + batches[3]) + assert first is fourth -def test_add_task_rejects_a_different_set_with_a_used_name(): - dataset, record = _dataset() - dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS)) - with pytest.raises(TimeFValidationError, match=SAME_NAME): - dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS)) - assert len(dataset.tasks) == 1 +def _stored_set_ids(path): + with connect_control(path, read_only=True) as connection: + rows = connection.execute("SELECT label_set_id FROM label_sets ORDER BY label_set_id").fetchall() + return [row[0] for row in rows] -def test_a_rejected_batch_leaves_no_label_set_behind(): +def _write_sets(tmp_path, *label_sets): dataset, record = _dataset() + dataset.add_tasks( + tasks=[ + ClassificationTask(inputs=(record,), targets=(label_set.labels[0].name,), label_set=label_set) + for label_set in label_sets + ] + ) + path = tmp_path / "control.duckdb" + _write(dataset, path) + return path + + +def test_sets_are_stored_once_per_content_and_keep_the_first_label_order(tmp_path): + reordered = LabelSet(STAGE_LABELS.labels[::-1], annotations=STAGE_LABELS.annotations) + described = LabelSet([Label(name="W", description="Awake."), "N1", "R"]) + multi = LabelSet(STAGE_LABELS.labels, multi_label=True) + path = _write_sets(tmp_path, STAGE_LABELS, reordered, described, multi) + + assert _stored_set_ids(path) == [1, 2, 3] + first, second, *_ = _read_tasks(path) + assert first.label_set is second.label_set + assert first.label_set.names == ("W", "N1", "R") - with pytest.raises(TimeFValidationError, match=SAME_NAME): - dataset.add_tasks( - tasks=[ - ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), - ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS), - ] - ) - assert dataset.tasks == () - dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS)) +def test_ordered_sets_in_another_order_are_stored_as_two_rows_in_their_own_order(tmp_path): + low_high = LabelSet(["low", "high"], ordered=True) + high_low = LabelSet(["high", "low"], ordered=True) + unordered = LabelSet(["low", "high"]) + path = _write_sets(tmp_path, low_high, high_low, LabelSet(["low", "high"], ordered=True), unordered) -def test_a_stream_rejects_a_different_set_with_a_used_name(): + with connect_control(path, read_only=True) as connection: + audit_control_database(connection, require_chunks=False) + assert connection.execute("SELECT label_set_id, ordered FROM label_sets ORDER BY 1").fetchall() == [ + (1, True), + (2, True), + (3, False), + ] + first, second, third, fourth = _label_sets(_read_tasks(path)) + assert first == low_high and first.ordered + assert first.names == ("low", "high") + assert second.names == ("high", "low") + assert first is third + assert fourth == unordered and not fourth.ordered + + +def _names(label_set): + return [annotation.value for annotation in label_set.annotations if annotation.key == "name"] + + +def test_sets_with_the_same_labels_and_different_names_are_stored_as_two_rows(tmp_path): + afib = LabelSet(["yes", "no"], annotations=(Annotation("name", value="afib_present"),)) + noisy = LabelSet(["yes", "no"], annotations=(Annotation("name", value="signal_noisy"),)) + path = _write_sets(tmp_path, afib, noisy, LabelSet(["yes", "no"])) + + assert _stored_set_ids(path) == [1, 2, 3] + with connect_control(path, read_only=True) as connection: + audit_control_database(connection, require_chunks=False) + assert connection.execute( + "SELECT object_id FROM annotation_occurrences WHERE object_type = 'LabelSet' ORDER BY 1" + ).fetchall() == [(1,), (2,)] + first, second, bare = _label_sets(_read_tasks(path)) + assert _names(first) == ["afib_present"] + assert _names(second) == ["signal_noisy"] + assert bare.annotations == () + assert (first, second, bare) == (afib, noisy, LabelSet(["yes", "no"])) + + +def test_sets_with_the_same_labels_and_the_same_annotations_are_stored_once(tmp_path): + name = Annotation("name", value="stage") + note = Annotation("note", value="manual", source="rater", confidence=0.9, occurrence_metadata={"k": 1}) + first = LabelSet(STAGE_LABELS.labels, annotations=(name, note)) + second = LabelSet(STAGE_LABELS.labels[::-1], annotations=(note, name)) + path = _write_sets(tmp_path, first, second) + + assert _stored_set_ids(path) == [1] + with connect_control(path, read_only=True) as connection: + assert connection.execute( + "SELECT count(*) FROM annotation_occurrences WHERE object_type = 'LabelSet'" + ).fetchone() == (2,) + tasks = _read_tasks(path) + assert tasks[0].label_set is tasks[1].label_set + assert [(a.key, a.source, a.confidence, a.occurrence_metadata) for a in tasks[0].label_set.annotations] == [ + ("name", None, None, {}), + ("note", "rater", 0.9, {"k": 1}), + ] + + +@pytest.mark.parametrize( + "changes", + [{"source": "rater"}, {"confidence": 0.9}, {"occurrence_metadata": {"k": 1}}], +) +def test_sets_whose_annotations_differ_in_one_occurrence_field_are_stored_as_two_rows(tmp_path, changes): + first = LabelSet(STAGE_LABELS.labels, annotations=(Annotation("name", value="stage"),)) + second = LabelSet(STAGE_LABELS.labels, annotations=(Annotation("name", value="stage", **changes),)) + path = _write_sets(tmp_path, first, second) + + assert _stored_set_ids(path) == [1, 2] + with connect_control(path, read_only=True) as connection: + assert connection.execute("SELECT count(*) FROM annotation_contents").fetchone() == (1,) + + +def _round_trip(dataset, tmp_path): + with TimeFReader(open_local_version(dataset.write(path=tmp_path / "out"))) as reader: + return reader.read() + + +def test_sets_with_their_annotations_in_another_order_round_trip_as_equal(tmp_path): + name = Annotation("name", value="stage") + note = Annotation("note", value="manual") dataset, record = _dataset() - dataset.set_task_stream( - (ClassificationTask,), - lambda: iter( - [ - ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), - ClassificationTask(inputs=(record,), targets=("W",), label_set=OTHER_STAGE_LABELS), - ] - ), + dataset.add_tasks( + tasks=[ + ClassificationTask(inputs=(record,), targets=("W",), label_set=LabelSet(["W"], annotations=(name, note))), + ClassificationTask(inputs=(record,), targets=("W",), label_set=LabelSet(["W"], annotations=(note, name))), + ] ) - with pytest.raises(TimeFValidationError, match=SAME_NAME): - list(dataset.iter_streamed_tasks_validated()) + assert_datasets_equal(dataset, _round_trip(dataset, tmp_path)) + + +def test_sets_with_a_nan_confidence_are_stored_once(tmp_path): + def nan_named(): + return LabelSet(["W"], annotations=(Annotation("name", value="stage", confidence=float("nan")),)) + + path = _write_sets(tmp_path, nan_named(), nan_named()) + + assert _stored_set_ids(path) == [1] + first, second = _label_sets(_read_tasks(path)) + assert first is second + assert first == nan_named() + + +def test_one_write_gives_equal_sets_one_id_and_a_set_with_other_annotations_another(): + label_set = LabelSet(["W"]) + rows = _LabelSetRows(iter((1, 2))) + + assert rows.register(label_set) == 1 + assert rows.register(label_set) == 1 + assert rows.register(LabelSet(["W"])) == 1 + assert rows.register(LabelSet(["W"], annotations=[Annotation("name", value="stage")])) == 2 + + +def test_equal_annotations_of_two_sets_share_one_content_row(tmp_path): + path = _write_sets( + tmp_path, + LabelSet(STAGE_LABELS.labels, annotations=(Annotation(key="name", value="stage"),)), + LabelSet(FINDINGS.labels, annotations=(Annotation(key="name", value="stage"),)), + ) + + with connect_control(path, read_only=True) as connection: + assert connection.execute("SELECT count(*) FROM annotation_contents").fetchone() == (1,) + assert connection.execute( + "SELECT object_id FROM annotation_occurrences WHERE object_type = 'LabelSet' ORDER BY 1" + ).fetchall() == [(1,), (2,)] + + +def test_the_writer_rejects_a_label_set_changed_after_add_task(tmp_path): + dataset, record = _dataset() + second = ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS) + dataset.add_tasks(tasks=[ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), second]) + second.label_set = LabelSet(["X"]) + + with pytest.raises(TimeFValidationError, match=r"ClassificationTask 2: labels not in label set \['X'\]"): + _write(dataset, tmp_path / "control.duckdb") @pytest.mark.parametrize( - ("changed", "message"), + ("field", "value", "message"), [ - (OTHER_STAGE_LABELS, SAME_NAME), - (LabelSet(name="other", labels=(Label(name="X"),)), "ClassificationTask 2: labels not in label set 'other'"), + ("labels", (Label(name="W"), Label(name="W")), "lists a label more than once"), + ("multi_label", 1, "multi_label must be a bool, got int"), + ("annotations", (Annotation(key="x", span=TimePoint.seconds(1.0)),), "has a span"), ], ) -def test_the_writer_rejects_a_label_set_changed_after_add_task(tmp_path, changed, message): +def test_the_writer_checks_a_label_set_changed_after_it_was_built(tmp_path, field, value, message): + label_set = LabelSet(["W"]) dataset, record = _dataset() - second = ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS) - dataset.add_tasks(tasks=[ClassificationTask(inputs=(record,), targets=("W",), label_set=STAGE_LABELS), second]) - second.label_set = changed + dataset.add_task(task=ClassificationTask(inputs=(record,), targets=("W",), label_set=label_set)) + setattr(label_set, field, value) with pytest.raises(TimeFValidationError, match=message): _write(dataset, tmp_path / "control.duckdb") +def test_the_reader_rejects_a_task_that_names_a_missing_label_set(release_copy): + with connect_control(release_copy) as connection: + connection.execute("UPDATE tasks SET label_set_id = 99 WHERE task_id = 1") + + with pytest.raises(TimeFFormatError, match="classification task 1 refers to missing label set 99"): + _read_tasks(release_copy) + + @pytest.mark.parametrize( ("task", "message"), [ ( ClassificationTask(targets=("X",), label_set=STAGE_LABELS), - "^ClassificationTask 1: labels not in label set 'stage': \\['X'\\]", + "^ClassificationTask 1: labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]", ), (ClassificationTask(targets=(1,), label_set=STAGE_LABELS), "each label must be text, not int"), ], @@ -226,7 +382,7 @@ def test_add_task_checks_target_annotations_against_the_label_set(): unknown = record.annotate(Annotation(key="stage", value="X")) dataset.add_task(task=ClassificationTask(inputs=(record,), target_annotations=(awake,), label_set=STAGE_LABELS)) - with pytest.raises(TimeFValidationError, match="labels not in label set 'stage': \\['X'\\]"): + with pytest.raises(TimeFValidationError, match="labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]"): dataset.add_task( task=ClassificationTask(inputs=(record,), target_annotations=(unknown,), label_set=STAGE_LABELS) ) @@ -239,22 +395,29 @@ def test_a_stream_rejects_targets_that_do_not_fit_the_label_set(): lambda: iter([ClassificationTask(inputs=(record,), targets=("X",), label_set=STAGE_LABELS)]), ) - with pytest.raises(TimeFValidationError, match="labels not in label set 'stage': \\['X'\\]"): + with pytest.raises(TimeFValidationError, match="labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]"): list(dataset.iter_streamed_tasks_validated()) @pytest.mark.parametrize( ("corruption", "field"), [ - ("UPDATE label_sets SET labels = [] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': 'W'}] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': ''}] WHERE name = 'stage'", "label_sets.labels"), + ("UPDATE label_sets SET labels = [] WHERE label_set_id = 1", "label_sets.labels"), + ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': 'W'}] WHERE label_set_id = 1", "label_sets.labels"), + ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': ''}] WHERE label_set_id = 1", "label_sets.labels"), # A NULL name drops out of list_distinct, so the check for distinct names finds it. - ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': NULL}] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE label_sets SET labels = [{'name': 'W', 'description': ''}] WHERE name = 'stage'", "label_sets.labels"), - ("UPDATE tasks SET label_set = 'stage' WHERE task_type = 'answer'", "tasks.label_set"), - ("UPDATE tasks SET label_set = NULL WHERE task_id = 1", "tasks.label_set"), - ("UPDATE tasks SET label_set = 'missing' WHERE task_id = 1", "label_sets.name"), + ("UPDATE label_sets SET labels = [{'name': 'W'}, {'name': NULL}] WHERE label_set_id = 1", "label_sets.labels"), + ( + "UPDATE label_sets SET labels = [{'name': 'W', 'description': ''}] WHERE label_set_id = 1", + "label_sets.labels", + ), + ("UPDATE tasks SET label_set_id = 1 WHERE task_type = 'answer'", "tasks.label_set_id"), + ("UPDATE tasks SET label_set_id = NULL WHERE task_id = 1", "tasks.label_set_id"), + ("UPDATE tasks SET label_set_id = 99 WHERE task_id = 1", "label_sets.label_set_id"), + ( + "UPDATE annotation_occurrences SET object_id = 99 WHERE object_type = 'LabelSet'", + "refers to missing LabelSet 99", + ), ("UPDATE task_targets SET text_value = 'BOGUS' WHERE task_id = 1", "task 1 has a label that is not"), ( "INSERT INTO task_targets (task_id, position, target_kind, text_value) VALUES (1, 1, 'text', 'R')", @@ -280,7 +443,7 @@ def test_the_audit_rejects_a_corrupt_label_set(release_copy, corruption, field): def test_the_reader_rejects_a_classification_task_without_a_label_set(release_copy): with connect_control(release_copy) as connection: - connection.execute("UPDATE tasks SET label_set = NULL WHERE task_id = 1") + connection.execute("UPDATE tasks SET label_set_id = NULL WHERE task_id = 1") with pytest.raises(TimeFFormatError, match="classification task 1 has no label set"): _read_tasks(release_copy) @@ -288,7 +451,7 @@ def test_the_reader_rejects_a_classification_task_without_a_label_set(release_co def test_the_reader_rejects_a_label_set_on_another_task_type(release_copy): with connect_control(release_copy) as connection: - connection.execute("UPDATE tasks SET label_set = 'stage' WHERE task_type = 'answer'") + connection.execute("UPDATE tasks SET label_set_id = 1 WHERE task_type = 'answer'") with pytest.raises(TimeFFormatError, match="answer task 2 has a label set"): _read_tasks(release_copy) diff --git a/packages/timenet/tests/format/test_task_codec.py b/packages/timenet/tests/format/test_task_codec.py index 562a95828..e0aa3432c 100644 --- a/packages/timenet/tests/format/test_task_codec.py +++ b/packages/timenet/tests/format/test_task_codec.py @@ -29,7 +29,6 @@ def test_task_payload_keeps_targets_out_of_configuration_columns(): task = AnswerTask(prompt="Alive?", targets=("Yes", 1)) assert encode_task_payload(task) == { - "label_set": None, "unit": None, "target_name": None, "mode": None, diff --git a/packages/timenet/tests/registry/conftest.py b/packages/timenet/tests/registry/conftest.py index 8f8c51258..65ca63d2a 100644 --- a/packages/timenet/tests/registry/conftest.py +++ b/packages/timenet/tests/registry/conftest.py @@ -12,7 +12,6 @@ ClassificationTask, DatasetMetadata, Domain, - Label, LabelSet, License, TimeSeriesSpec, @@ -22,13 +21,7 @@ from timenet.writer import TimeFWriter -RHYTHM = LabelSet( - name="rhythm", - labels=( - Label(name="afib"), - Label(name="normal"), - ), -) +RHYTHM = LabelSet(["afib", "normal"]) def _ecg_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/registry/test_conformance.py b/packages/timenet/tests/registry/test_conformance.py index 1fd42dc36..7214ef099 100644 --- a/packages/timenet/tests/registry/test_conformance.py +++ b/packages/timenet/tests/registry/test_conformance.py @@ -24,7 +24,6 @@ ClassificationTask, DatasetMetadata, Domain, - Label, LabelSet, License, TimeSeriesSpec, @@ -34,13 +33,7 @@ from timenet.types.references import DatasetRef -RHYTHM = LabelSet( - name="rhythm", - labels=( - Label(name="afib"), - Label(name="normal"), - ), -) +RHYTHM = LabelSet(["afib", "normal"]) def _second_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/registry/test_namespaced.py b/packages/timenet/tests/registry/test_namespaced.py index 78db80630..a3b86eb0b 100644 --- a/packages/timenet/tests/registry/test_namespaced.py +++ b/packages/timenet/tests/registry/test_namespaced.py @@ -6,7 +6,6 @@ from timenet.types import ( ClassificationTask, DatasetMetadata, - Label, LabelSet, License, TimeSeriesSpec, @@ -16,7 +15,7 @@ from timenet.writer import TimeFWriter -LABELS = LabelSet(name="labels", labels=(Label(name="x"),)) +LABELS = LabelSet(["x"]) def _namespaced_dataset() -> TimeFDataset: diff --git a/packages/timenet/tests/types/test_label_sets.py b/packages/timenet/tests/types/test_label_sets.py index 7087e619b..b60c6ab1e 100644 --- a/packages/timenet/tests/types/test_label_sets.py +++ b/packages/timenet/tests/types/test_label_sets.py @@ -1,24 +1,43 @@ -from pydantic import ValidationError +import pickle + import pytest from timenet.errors import TimeFValidationError -from timenet.types import ClassificationTask, Label, LabelSet +from timenet.types import Annotation, AnswerTask, ClassificationTask, Label, LabelSet, TimePoint -STAGES = LabelSet(name="stage", labels=(Label(name="W", description="Awake."), Label(name="N1"), Label(name="R"))) -FINDINGS = LabelSet(name="findings", labels=(Label(name="AF"), Label(name="PVC"), Label(name="LBBB")), multi_label=True) +STAGES = LabelSet([Label(name="W", description="Awake."), "N1", "R"]) +FINDINGS = LabelSet(["AF", "PVC", "LBBB"], multi_label=True) @pytest.mark.parametrize( - ("labels", "message"), + ("kwargs", "message"), [ - ((), "at least one label"), - ((Label(name="W"), Label(name="W", description="Awake.")), "more than once"), + ({"labels": ()}, "at least one label"), + ({"labels": None}, "iterable of items such as a list or a tuple, got NoneType"), + ({"labels": 3}, "iterable of items such as a list or a tuple, got int"), + ({"labels": "WR"}, "iterable of items such as a list or a tuple, got str"), + ({"labels": {"W": "Wake."}}, "iterable of items such as a list or a tuple, got dict"), + ({"labels": Label(name="W")}, "iterable of items such as a list or a tuple, got Label"), + ({"labels": ("W", "")}, "cannot have an empty label"), + ( + {"labels": (Label(name="W"),), "annotations": None}, + "iterable of items such as a list or a tuple, got NoneType", + ), + ({"labels": (Label(name="W"), Label(name="W", description="Awake."))}, "more than once"), + ({"labels": (3,)}, "takes Label objects, got int"), + ({"labels": ("W", "W")}, "more than once"), + ({"labels": (Label(name="W"),), "multi_label": 1}, "multi_label must be a bool, got int"), + ({"labels": (Label(name="W"),), "ordered": 1}, "ordered must be a bool, got int"), + ({"labels": (Label(name="W"),), "ordered": None}, "ordered must be a bool, got NoneType"), + ({"labels": {"W", "R"}}, "needs labels in a fixed order, got a set"), + ({"labels": frozenset({"W", "R"}), "ordered": True}, "needs labels in a fixed order, got a set"), + ({"labels": (Label(name="W"),), "annotations": ("name",)}, "takes Annotation objects, got str"), ], ) -def test_a_label_set_needs_at_least_one_label_and_distinct_names(labels, message): - with pytest.raises(ValidationError, match=message): - LabelSet(name="stage", labels=labels) +def test_a_label_set_rejects_invalid_fields(kwargs, message): + with pytest.raises(TimeFValidationError, match=message): + LabelSet(**kwargs) @pytest.mark.parametrize(("label_set", "type_name"), [(None, "NoneType"), ("stage", "str")]) @@ -30,8 +49,8 @@ def test_a_classification_task_requires_a_label_set(label_set, type_name): @pytest.mark.parametrize( ("targets", "message"), [ - (("X",), "labels not in label set 'stage': \\['X'\\]"), - ((), "0 labels, but label set 'stage' takes exactly one"), + (("X",), "labels not in label set \\['W', 'N1', 'R'\\]: \\['X'\\]"), + ((), "0 labels, but label set \\['W', 'N1', 'R'\\] takes exactly one"), (("W", "R"), "2 labels"), ], ) @@ -46,5 +65,120 @@ def test_a_multi_label_target_accepts_any_distinct_listed_labels(targets): def test_a_multi_label_target_rejects_a_repeated_label(): - with pytest.raises(TimeFValidationError, match="repeated labels of label set 'findings': \\['AF'\\]"): + with pytest.raises(TimeFValidationError, match=r"repeated labels of label set \['AF', 'PVC', 'LBBB'\]: \['AF'\]"): FINDINGS.check_targets(("AF", "PVC", "AF")) + + +def test_equality_ignores_label_order_but_not_descriptions_or_multi_label(): + first = LabelSet(["F", "M"]) + second = LabelSet(["M", "F"]) + described = LabelSet([Label(name="F", description="Female."), "M"]) + multi = LabelSet(["F", "M"], multi_label=True) + assert first._content_key() == second._content_key() + assert len({first._content_key(), described._content_key(), multi._content_key()}) == 3 + assert first == second + assert first != described + assert first != multi + + +def _named(*annotations): + return LabelSet(["yes", "no"], annotations=annotations) + + +def test_equality_compares_the_annotations_and_ignores_their_order(): + name = Annotation(key="name", value="afib_present") + note = Annotation(key="note", value="manual") + assert _named(name) == _named(Annotation(key="name", value="afib_present")) + assert _named(name)._content_key() == _named(name)._content_key() + assert _named(name, note) == _named(note, name) + assert _named(name) != _named(Annotation(key="name", value="signal_noisy")) + assert _named(name) != _named(Annotation(key="title", value="afib_present")) + assert _named(name) != _named() + assert _named(name) != _named(name, note) + assert _named(name) != _named(name, name) + + +@pytest.mark.parametrize( + "changes", + [{"source": "rater"}, {"confidence": 0.9}, {"occurrence_metadata": {"k": 1}}, {"description": "The name."}], +) +def test_equality_compares_every_field_of_an_annotation(changes): + assert _named(Annotation(key="name", value="x")) != _named(Annotation(key="name", value="x", **changes)) + assert _named(Annotation(key="name", value="x", **changes)) == _named(Annotation(key="name", value="x", **changes)) + + +def test_sets_with_a_nan_confidence_are_equal(): + def nan_named(): + return _named(Annotation(key="name", value="x", confidence=float("nan"))) + + assert nan_named() == nan_named() + assert nan_named()._content_key() == nan_named()._content_key() + + +def test_an_ordered_set_uses_its_label_order_for_equality(): + low_high = LabelSet(["low", "high"], ordered=True) + high_low = LabelSet(["high", "low"], ordered=True) + assert low_high == LabelSet(["low", "high"], ordered=True) + assert low_high._content_key() == LabelSet(["low", "high"], ordered=True)._content_key() + assert low_high != high_low + assert low_high._content_key() != high_low._content_key() + assert low_high != LabelSet(["low", "high"]) + assert LabelSet(["low", "high"]) == LabelSet(["high", "low"]) + assert LabelSet(["low"]).ordered is False + + +def test_annotate_attaches_an_occurrence_and_rejects_a_span(): + annotation = Annotation(key="name", value="sex") + built = LabelSet(["F"], annotations=(annotation,)) + assert built.annotations == (annotation,) + assert built.annotations[0] is not annotation + label_set = LabelSet(["F"]) + attached = label_set.annotate(annotation) + assert label_set.annotations == (attached,) + assert attached is not annotation + with pytest.raises(TimeFValidationError, match="has a span"): + label_set.annotate(Annotation(key="x", span=TimePoint.seconds(1.0))) + with pytest.raises(TimeFValidationError, match="has a span"): + LabelSet(["F"], annotations=(Annotation(key="x", span=TimePoint.seconds(1.0)),)) + + +def test_a_classification_task_walks_its_annotations_then_those_of_its_label_set(): + task = ClassificationTask( + targets=("W",), label_set=LabelSet(["W", "R"], annotations=(Annotation(key="set", value="s"),)) + ) + task.annotate(Annotation(key="task", value="t")) + assert [annotation.key for annotation in task.walk_annotations()] == ["task", "set"] + answer = AnswerTask(targets=("x",)) + answer.annotate(Annotation(key="answer", value="a")) + assert [annotation.key for annotation in answer.walk_annotations()] == ["answer"] + + +def test_a_copied_task_gets_its_own_label_set(): + label_set = LabelSet(["W"], annotations=(Annotation(key="name", value="stage"),)) + task = ClassificationTask(targets=("W",), label_set=label_set) + copy = task.with_references({}, {}) + assert copy.label_set == label_set + assert copy.label_set is not label_set + assert copy.label_set.annotations == label_set.annotations + assert copy.label_set.annotations[0] is not label_set.annotations[0] + copy.label_set.annotate(Annotation(key="note", value="child")) + assert [a.key for a in label_set.annotations] == ["name"] + + +def test_a_task_with_an_annotated_set_survives_a_pickle_round_trip(): + label_set = LabelSet( + [Label(name="W", description="Awake.")], + multi_label=True, + annotations=(Annotation(key="name", value="stage"),), + ) + task = ClassificationTask(targets=("W",), label_set=label_set) + restored = pickle.loads(pickle.dumps(task)).label_set + assert restored == label_set + assert [(a.key, a.value) for a in restored.annotations] == [("name", "stage")] + + +def test_a_set_takes_its_labels_as_plain_strings_or_label_objects(): + label_set = LabelSet(["W", Label(name="R", description="REM sleep.")], multi_label=True) + assert label_set.labels == (Label(name="W"), Label(name="R", description="REM sleep.")) + assert label_set.multi_label is True + assert LabelSet(labels=("W", "R")) == LabelSet([Label(name="W"), Label(name="R")]) diff --git a/packages/timenet/tests/writer/test_id_encoding.py b/packages/timenet/tests/writer/test_id_encoding.py index b97597dfe..81de2645b 100644 --- a/packages/timenet/tests/writer/test_id_encoding.py +++ b/packages/timenet/tests/writer/test_id_encoding.py @@ -12,7 +12,6 @@ ClassificationTask, DatasetMetadata, ForecastingTask, - Label, LabelSet, License, StepInterval, @@ -27,7 +26,7 @@ from timenet.writer import TimeFWriter -LABELS = LabelSet(name="labels", labels=(Label(name="x"),)) +LABELS = LabelSet(["x"]) def _spec(): diff --git a/packages/timenet/tests/writer/test_sharding.py b/packages/timenet/tests/writer/test_sharding.py index 4ec1840f6..1b43e56bc 100644 --- a/packages/timenet/tests/writer/test_sharding.py +++ b/packages/timenet/tests/writer/test_sharding.py @@ -16,7 +16,6 @@ Annotation, ClassificationTask, DatasetMetadata, - Label, LabelSet, License, TimeSeriesSpec, @@ -28,13 +27,7 @@ from timenet.writer import TimeFWriter -CLASSES = LabelSet( - name="classes", - labels=( - Label(name="c0"), - Label(name="c1"), - ), -) +CLASSES = LabelSet(["c0", "c1"]) # Small enough that a modest dataset shards every artifact type into several parts, tiny on disk. diff --git a/packages/timenet/tests/writer/test_writer.py b/packages/timenet/tests/writer/test_writer.py index 7759d9442..60a322f38 100644 --- a/packages/timenet/tests/writer/test_writer.py +++ b/packages/timenet/tests/writer/test_writer.py @@ -19,7 +19,6 @@ Annotation, ClassificationTask, DatasetMetadata, - Label, LabelSet, License, TimeSeriesSpec, @@ -31,7 +30,7 @@ from timenet.writer.value_encoding import ValueEncoding -LABELS = LabelSet(name="labels", labels=(Label(name="x"),)) +LABELS = LabelSet(["x"]) def _written(tmp_path, dataset=None, **kwargs): @@ -108,7 +107,7 @@ def test_manifest_is_valid_and_matches_dataset(tmp_path): assert (version_dir / rel).exists() -def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_path): +def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_label_sets(tmp_path): dataset = make_dataset() record = dataset.records[0] streamed = TimeFDataset.from_parts( @@ -118,14 +117,14 @@ def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_pat schema=dataset.derive_schema(), last_ids=dataset.last_ids, ) - task = ClassificationTask(inputs=(record,), targets=("x",), label_set=LABELS) - task.annotate(Annotation(key="task_note", value="letters")) + label_set = LabelSet(["x"], annotations=(Annotation(key="label_set_note", value="letters"),)) + task = ClassificationTask(inputs=(record,), targets=("x",), label_set=label_set) streamed.set_task_stream([ClassificationTask], lambda: iter((task,))) version_dir = _written(tmp_path, streamed) manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text()) - assert "task_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations] + assert "label_set_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations] def test_manifest_counts_match_the_control_database(tmp_path):