From 270ffea15f3afd6630c9ab05476dfd1b81bccdf7 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:40:51 -0800 Subject: [PATCH 1/7] feat(dataset): walk every annotation owner and list all keys in the schema Add annotation_owners() to Record, Task and the dataset. The dataset yields itself, then each Record with its Sources and Signals, then each task. walk_annotations() on Record, Task and the dataset goes through it, and every owner gets annotation(key), which returns the first annotation with a key. derive_schema and describe read every owner through the walk, so the schema now also lists the annotation keys of the dataset and of tasks. The keys of streamed tasks are added while the writer streams them, and a key with conflicting descriptors raises before the manifest is written. --- docs/data-model/annotations.md | 3 +- docs/timef-dataset.md | 4 +- .../timenet/src/timenet/dataset/dataset.py | 96 +++++++++++++------ .../timenet/src/timenet/dataset/describe.py | 2 +- .../timenet/src/timenet/dataset/record.py | 19 ++-- .../timenet/src/timenet/types/annotations.py | 11 +++ packages/timenet/src/timenet/types/tasks.py | 14 ++- .../timenet/tests/dataset/test_dataset.py | 82 ++++++++++++++++ .../timenet/tests/dataset/test_describe.py | 13 ++- packages/timenet/tests/writer/test_writer.py | 20 ++++ 10 files changed, 226 insertions(+), 38 deletions(-) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 27a7b5c35..6a9014e3a 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -25,7 +25,8 @@ occurrence = record.annotate(condition) `key` can be positional: `Annotation("condition", value="healthy")`. All other fields are keyword-only. `annotate()` returns the attached occurrence. A Task must refer to this occurrence, not the unattached -content object. +content object. Every owner also has `annotation(key)`. It returns the first annotation with that key, +or `None`. ## Content and occurrence fields diff --git a/docs/timef-dataset.md b/docs/timef-dataset.md index 0f75e82aa..407b08a4d 100644 --- a/docs/timef-dataset.md +++ b/docs/timef-dataset.md @@ -237,7 +237,9 @@ lead_i.annotate( ## Schema and persistence `derive_schema()` collects the time-series specifications, annotation descriptors, and concrete Task -types found in the dataset. It does not load Signal values. +types found in the dataset. Annotation descriptors come from every owner: the Dataset, Records, +Sources, Signals, and Tasks. The Tasks of a task stream are not read here. The writer adds their +keys to the schema while it writes them. It does not load Signal values. Write a version with the declarative helper: diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index fd5788d71..97f0cc407 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -385,6 +385,26 @@ def annotate(self, annotation: Annotation) -> Annotation: self._annotations.append(attached) return attached + def annotation_owners(self) -> Iterator[SupportsAnnotate]: + """Yield every object of the dataset that can hold annotations. + + The order is the dataset, then the owners of each Record (see :meth:`Record.annotation_owners`), + then the owners of each task (see :meth:`Task.annotation_owners`). The tasks of a streamed dataset + (see :meth:`set_task_stream`) are not walked, so their owners do not come. + + Yields: + Each owner, in the order above. + """ + yield self + for record in self._records: + yield from record.annotation_owners() + for task in self._tasks: + yield from task.annotation_owners() + + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of each object that :meth:`annotation_owners` yields.""" + return (annotation for owner in self.annotation_owners() for annotation in owner.annotations) + def set_task_stream(self, task_types: Sequence[type[Task]], source: Callable[[], Iterator[Task]]) -> None: """Provide tasks as a re-iterable stream instead of materializing them in the dataset. @@ -459,19 +479,29 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: the cross-task checks (``from_tasks`` derivations) that need every task at once do not run for a stream. + If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks are + added to it. After the last task, the schema lists them. + Yields: Each validated task, in the source's order. Raises: - TimeFValidationError: If a streamed task fails one of the per-task checks. + TimeFValidationError: If a streamed task fails one of the per-task checks, or one of its + annotation keys has a descriptor that conflicts with the schema. """ # ruff: ignore[docstring-extraneous-exception] (raised by _validate_streamed_task, not directly here) index = _TaskValidationIndex(self) declared = set(self._streamed_task_types) label_sets = LabelSetRegistry() + schema = self._schema + descriptors = None if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations} for position, task in enumerate(self.iter_tasks(), start=1): task._id = self._ids.claim(ObjectKind.TASK, self._streamed_task_base + position) self._validate_streamed_task(task, index, declared, label_sets) + if descriptors is not None: + self._add_descriptors(descriptors, task.walk_annotations()) yield task + if schema is not None and descriptors is not None: + self._schema = schema.model_copy(update={"annotations": tuple(descriptors.values())}) def _validate_streamed_task( self, @@ -686,15 +716,19 @@ def _check_no_derivation_cycle(batch: tuple[Task, ...]) -> None: def derive_schema(self) -> DatasetSchema: """Walk the dataset's instances and build its :class:`DatasetSchema`. - This method collects the distinct spec, annotation, and task types. It stores the result - on the dataset, and it returns the result. + This method collects the distinct spec, annotation, and task types. Annotations come from every + owner that :meth:`annotation_owners` yields. It stores the result on the dataset, and it returns + the result. + + The tasks of a streamed dataset (see :meth:`set_task_stream`) are not walked here. + :meth:`iter_streamed_tasks_validated` adds their annotation keys while the writer writes them. Returns: The derived :class:`DatasetSchema`. Raises: TimeFValidationError: If one spec type or annotation key yields conflicting descriptors - across records. + across the dataset. """ specs = self._ordered_unique(signal.spec for record in self._records for signal in record.signals) by_spec_type: dict[str, object] = {} @@ -705,35 +739,15 @@ def derive_schema(self) -> DatasetSchema: f"spec_type {spec.spec_type!r} has conflicting TimeSeriesSpec contracts: {existing!r} and {spec!r}" ) by_spec_type[spec.spec_type] = spec - record_annotations = (annotation for record in self._records for annotation in record.walk_annotations()) - annotations = self._ordered_unique( - AnnotationDescriptor( - key=annotation.key, - annotation_type=annotation_type_of(annotation), - value_type=value_type_of(annotation.value), - # The __post_init__ method normalizes unit to a plain string or None, so this - # value is always str | None. - unit=cast("str | None", annotation.unit), - description=annotation.description, - ) - for annotation in record_annotations - ) - by_key: dict[str, AnnotationDescriptor] = {} - for descriptor in annotations: - existing = by_key.get(descriptor.key) - if existing is not None: - raise TimeFValidationError( - f"annotation {descriptor.key!r} has conflicting descriptors across records: " - f"{existing!r} and {descriptor!r}" - ) - by_key[descriptor.key] = descriptor + annotations: dict[str, AnnotationDescriptor] = {} + self._add_descriptors(annotations, self.walk_annotations()) # A streamed dataset declares its task types up front, so the schema records them without # draining the stream just to collect them. streamed = self._ordered_unique(self._streamed_task_types) tasks = streamed if self._task_stream is not None else self._ordered_unique(type(task) for task in self._tasks) self._schema = DatasetSchema( time_series_specs=tuple(specs), - annotations=tuple(annotations), + annotations=tuple(annotations.values()), tasks=tuple(tasks), ) return self._schema @@ -886,6 +900,34 @@ def _referenced_records(self, task: Task) -> tuple[Record, ...]: found.setdefault(id(record), record) return tuple(found.values()) + @staticmethod + def _add_descriptors(descriptors: dict[str, AnnotationDescriptor], annotations: Iterable[Annotation]) -> None: + """Add the descriptor of each annotation key that ``descriptors`` does not hold yet. + + Args: + descriptors: The descriptors so far, by annotation key, in the order of first use. + annotations: The annotations to add. + + Raises: + TimeFValidationError: If an annotation gives a key a different descriptor than ``descriptors`` holds. + """ + for annotation in annotations: + descriptor = AnnotationDescriptor( + key=annotation.key, + annotation_type=annotation_type_of(annotation), + value_type=value_type_of(annotation.value), + # The __post_init__ method normalizes unit to a plain string or None, so this + # value is always str | None. + unit=cast("str | None", annotation.unit), + description=annotation.description, + ) + existing = descriptors.setdefault(descriptor.key, descriptor) + if existing != descriptor: + raise TimeFValidationError( + f"annotation {descriptor.key!r} has conflicting descriptors across the dataset: " + f"{existing!r} and {descriptor!r}" + ) + @staticmethod def _ordered_unique(items: Iterable[T]) -> list[T]: """Return items with duplicates removed, preserving first-seen order.""" diff --git a/packages/timenet/src/timenet/dataset/describe.py b/packages/timenet/src/timenet/dataset/describe.py index 9e5a287dd..43aa17066 100644 --- a/packages/timenet/src/timenet/dataset/describe.py +++ b/packages/timenet/src/timenet/dataset/describe.py @@ -68,7 +68,7 @@ def _identity(dataset: TimeFDataset) -> str: def _counts(dataset: TimeFDataset, unique_series: dict[EntityId, Signal]) -> str: task_counts = Counter(str(task.task_type) for task in dataset.tasks) - annotation_ids = {ann._content_key for record in dataset.records for ann in record.walk_annotations()} + annotation_ids = {ann._content_key for ann in dataset.walk_annotations()} spec_counts = Counter(ts.spec.spec_type for ts in unique_series.values()) fields = { "records": str(len(dataset.records)), diff --git a/packages/timenet/src/timenet/dataset/record.py b/packages/timenet/src/timenet/dataset/record.py index 4a15a6bc0..b94885374 100644 --- a/packages/timenet/src/timenet/dataset/record.py +++ b/packages/timenet/src/timenet/dataset/record.py @@ -1,6 +1,6 @@ """The :class:`Record` type: one logical unit of time-series data.""" -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from dataclasses import dataclass, field from datetime import datetime @@ -164,13 +164,20 @@ def walk_signals(self) -> Iterable[Signal]: for source in self.walk_sources(): yield from sorted(source.signals, key=lambda signal: signal.name) + def annotation_owners(self) -> Iterator[SupportsAnnotate]: + """Yield this Record, then each Source followed by its Signals. + + Yields: + Each object of the Record hierarchy that can hold annotations. + """ + yield self + for source in self.walk_sources(): + yield source + yield from source.signals + def walk_annotations(self) -> Iterable[Annotation]: """Yield annotation occurrences from this Record, its Sources, and its Signals.""" - yield from self.annotations - for source in self.walk_sources(): - yield from source.annotations - for signal in source.signals: - yield from signal.annotations + return (annotation for owner in self.annotation_owners() for annotation in owner.annotations) @property def has_absolute_time(self) -> bool: diff --git a/packages/timenet/src/timenet/types/annotations.py b/packages/timenet/src/timenet/types/annotations.py index 417540a5a..478f0367b 100644 --- a/packages/timenet/src/timenet/types/annotations.py +++ b/packages/timenet/src/timenet/types/annotations.py @@ -219,6 +219,17 @@ def annotate(self, annotation: Annotation) -> Annotation: """Attach one occurrence of ``annotation`` and return it.""" ... + def annotation(self, key: str) -> Annotation | None: + """Look up an annotation of this object by its key. + + Args: + key: The annotation key, such as ``name``. + + Returns: + The first annotation with this key, or ``None`` if there is none. + """ + return next((annotation for annotation in self.annotations if annotation.key == key), None) + class AnnotationDescriptor(TimeFModel): """Type-level projection of an annotation key, stored in the schema and manifest.""" diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 0a000ac41..57659caf5 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -1,6 +1,6 @@ """Declarative modelling tasks over records and signals.""" -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field, replace from enum import StrEnum, unique from functools import lru_cache @@ -278,6 +278,18 @@ def annotate(self, annotation: Annotation) -> Annotation: self.annotations = (*self.annotations, attached) return attached + def annotation_owners(self) -> Iterator[SupportsAnnotate]: + """Yield the task and each object that it owns that can hold annotations. + + Yields: + The task. A task class with an owned object yields it after the task. + """ + yield self + + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of each object that :meth:`annotation_owners` yields.""" + return (annotation for owner in self.annotation_owners() for annotation in owner.annotations) + def spans(self) -> tuple[Span, ...]: """Return the task scope followed by every span-valued target.""" spans = [] if self.scope is None else [self.scope] diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 6aa31a4c5..81216898b 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -179,6 +179,57 @@ def test_derive_schema(make_series): assert ds.schema is schema +def test_derive_schema_collects_dataset_and_task_annotations(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="age", value=64)) + ds.annotate(Annotation(key="cohort", value="adult")) + task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + task.annotate(Annotation(key="difficulty", value=2)) + + schema = ds.derive_schema() + + assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"] + + +def test_derive_schema_rejects_a_task_annotation_that_conflicts_with_a_record_annotation(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="name", value=1)) + task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + task.annotate(Annotation(key="name", value="letters")) + with pytest.raises(TimeFValidationError, match="conflicting descriptors"): + ds.derive_schema() + + +def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="age", value=64)) + task = _classification_task(record, "a") + task.annotate(Annotation(key="difficulty", value=2)) + ds.set_task_stream([ClassificationTask], lambda: iter((task,))) + assert [descriptor.key for descriptor in ds.derive_schema().annotations] == ["age"] + + list(ds.iter_streamed_tasks_validated()) + + assert ds.schema is not None + assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty"] + + +def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="name", value=1)) + task = _classification_task(record, "a") + task.annotate(Annotation(key="name", value="letters")) + ds.set_task_stream([ClassificationTask], lambda: iter((task,))) + ds.derive_schema() + + with pytest.raises(TimeFValidationError, match="conflicting descriptors"): + list(ds.iter_streamed_tasks_validated()) + + def test_derive_schema_dedupes_specs(make_series): ds = _dataset() # Two signals of the same modality => one distinct spec. @@ -830,6 +881,37 @@ def test_add_tasks_rejects_the_whole_batch_when_one_is_invalid(make_series): assert ds.tasks == () # all-or-nothing: the valid one is not registered either +def test_annotation_owners_walks_the_dataset_then_records_then_tasks(make_series): + ds = _dataset() + signal = make_series() + source = Source(name="Source", signals=(signal,)) + record = ds.add_record(record=Record(sources=(source,))) + task = ds.add_task(task=_classification_task(record, "a")) + answer = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + assert list(ds.annotation_owners()) == [ds, record, source, signal, task, answer] + + +def test_annotation_finds_the_first_match_on_every_owner(make_series): + ds = _dataset() + signal = make_series() + source = Source(name="Source", signals=(signal,)) + record = ds.add_record(record=Record(sources=(source,))) + task = _classification_task(record, "a") + owners = (ds, record, source, signal, task) + for owner in owners: + owner.annotate(Annotation(key="name", value=type(owner).__name__)) + owner.annotate(Annotation(key="name", value="second")) + found = [owner.annotation("name") for owner in owners] + assert [None if one is None else one.value for one in found] == [ + "TimeFDataset", + "Record", + "Source", + "Signal", + "ClassificationTask", + ] + assert ds.annotation("other") is None + + def test_add_tasks_allows_deriving_from_another_task_in_the_same_batch(make_series): # The deriving task is listed before its parent, so this only passes because the batch is validated # as a unit rather than task by task. diff --git a/packages/timenet/tests/dataset/test_describe.py b/packages/timenet/tests/dataset/test_describe.py index adb17cf0a..3a5103f59 100644 --- a/packages/timenet/tests/dataset/test_describe.py +++ b/packages/timenet/tests/dataset/test_describe.py @@ -3,7 +3,7 @@ from timenet.dataset import TimeFDataset from timenet.dataset.describe import describe_text from timenet.testing import make_dataset -from timenet.types import DatasetMetadata, License, Version +from timenet.types import Annotation, DatasetMetadata, License, Version def _previews(text: str, record_id: int) -> bool: @@ -58,3 +58,14 @@ def test_describe_empty_dataset_does_not_crash(): text = describe_text(empty, rows=5) assert "demo/empty @ 1.0.0" in text assert "records 0" in text + + +def test_describe_counts_annotations_of_every_owner(): + dataset = make_dataset() + before = describe_text(dataset, rows=0) + dataset.annotate(Annotation(key="describe_dataset_note", value="a")) + dataset.tasks[0].annotate(Annotation(key="describe_task_note", value="b")) + match = re.search(r"^ annotations\s+(\d+)$", before, re.MULTILINE) + assert match is not None + count = int(match[1]) + assert re.search(rf"^ annotations\s+{count + 2}$", describe_text(dataset, rows=0), re.MULTILINE) diff --git a/packages/timenet/tests/writer/test_writer.py b/packages/timenet/tests/writer/test_writer.py index 06c357bf1..7759d9442 100644 --- a/packages/timenet/tests/writer/test_writer.py +++ b/packages/timenet/tests/writer/test_writer.py @@ -108,6 +108,26 @@ def test_manifest_is_valid_and_matches_dataset(tmp_path): assert (version_dir / rel).exists() +def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_path): + dataset = make_dataset() + record = dataset.records[0] + streamed = TimeFDataset.from_parts( + metadata=dataset.metadata, + records=dataset.records, + tasks=(), + schema=dataset.derive_schema(), + last_ids=dataset.last_ids, + ) + task = ClassificationTask(inputs=(record,), targets=("x",), label_set=LABELS) + task.annotate(Annotation(key="task_note", value="letters")) + streamed.set_task_stream([ClassificationTask], lambda: iter((task,))) + + version_dir = _written(tmp_path, streamed) + + manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text()) + assert "task_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations] + + def test_manifest_counts_match_the_control_database(tmp_path): version_dir = _written(tmp_path) manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text()) From a97ccf1563e9c140b054c4ab440f31fd3006c5ed Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:39:12 -0800 Subject: [PATCH 2/7] refactor(dataset): walk the dataset annotations without an owner list TimeFDataset.annotation_owners() had no caller apart from walk_annotations(). walk_annotations() now yields the dataset occurrences, then those of each Record, then those of each task. Record and Task keep annotation_owners(), which the load plan uses. --- .../timenet/src/timenet/dataset/dataset.py | 24 ++++++++----------- .../timenet/tests/dataset/test_dataset.py | 7 ++++-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index 97f0cc407..75db138a4 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -385,25 +385,21 @@ def annotate(self, annotation: Annotation) -> Annotation: self._annotations.append(attached) return attached - def annotation_owners(self) -> Iterator[SupportsAnnotate]: - """Yield every object of the dataset that can hold annotations. + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of the dataset, then of each Record, then of each task. - The order is the dataset, then the owners of each Record (see :meth:`Record.annotation_owners`), - then the owners of each task (see :meth:`Task.annotation_owners`). The tasks of a streamed dataset - (see :meth:`set_task_stream`) are not walked, so their owners do not come. + Each Record and task gives the occurrences of the objects it holds (see :meth:`Record.walk_annotations` + and :meth:`Task.walk_annotations`). The tasks of a streamed dataset (see :meth:`set_task_stream`) are + not walked, so their occurrences do not come. Yields: - Each owner, in the order above. + Each annotation occurrence, in the order above. """ - yield self + yield from self.annotations for record in self._records: - yield from record.annotation_owners() + yield from record.walk_annotations() for task in self._tasks: - yield from task.annotation_owners() - - def walk_annotations(self) -> Iterator[Annotation]: - """Yield the annotation occurrences of each object that :meth:`annotation_owners` yields.""" - return (annotation for owner in self.annotation_owners() for annotation in owner.annotations) + yield from task.walk_annotations() def set_task_stream(self, task_types: Sequence[type[Task]], source: Callable[[], Iterator[Task]]) -> None: """Provide tasks as a re-iterable stream instead of materializing them in the dataset. @@ -717,7 +713,7 @@ def derive_schema(self) -> DatasetSchema: """Walk the dataset's instances and build its :class:`DatasetSchema`. This method collects the distinct spec, annotation, and task types. Annotations come from every - owner that :meth:`annotation_owners` yields. It stores the result on the dataset, and it returns + occurrence that :meth:`walk_annotations` yields. It stores the result on the dataset, and it returns the result. The tasks of a streamed dataset (see :meth:`set_task_stream`) are not walked here. diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 81216898b..4b5d705d6 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -881,14 +881,17 @@ def test_add_tasks_rejects_the_whole_batch_when_one_is_invalid(make_series): assert ds.tasks == () # all-or-nothing: the valid one is not registered either -def test_annotation_owners_walks_the_dataset_then_records_then_tasks(make_series): +def test_walk_annotations_goes_from_the_dataset_to_records_to_tasks(make_series): ds = _dataset() signal = make_series() source = Source(name="Source", signals=(signal,)) record = ds.add_record(record=Record(sources=(source,))) task = ds.add_task(task=_classification_task(record, "a")) answer = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) - assert list(ds.annotation_owners()) == [ds, record, source, signal, task, answer] + for owner, key in ((answer, "answer"), (task, "task"), (signal, "signal"), (source, "source"), (record, "record")): + owner.annotate(Annotation(key=key, value=key)) + ds.annotate(Annotation(key="dataset", value="dataset")) + assert [a.key for a in ds.walk_annotations()] == ["dataset", "record", "source", "signal", "task", "answer"] def test_annotation_finds_the_first_match_on_every_owner(make_series): From 959525756dde6dd6d6ced7089b56a54c966d91d6 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:57:51 -0800 Subject: [PATCH 3/7] refactor(types): rename annotation_owners to annotatable_entities The name says what the method gives: the entities of a Record or a task that can carry annotations. --- packages/timenet/src/timenet/dataset/record.py | 8 ++++---- packages/timenet/src/timenet/types/tasks.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/timenet/src/timenet/dataset/record.py b/packages/timenet/src/timenet/dataset/record.py index b94885374..8175c0294 100644 --- a/packages/timenet/src/timenet/dataset/record.py +++ b/packages/timenet/src/timenet/dataset/record.py @@ -164,11 +164,11 @@ def walk_signals(self) -> Iterable[Signal]: for source in self.walk_sources(): yield from sorted(source.signals, key=lambda signal: signal.name) - def annotation_owners(self) -> Iterator[SupportsAnnotate]: - """Yield this Record, then each Source followed by its Signals. + def annotatable_entities(self) -> Iterator[SupportsAnnotate]: + """Yield the entities of this Record that can carry annotations: the Record, then each Source and its Signals. Yields: - Each object of the Record hierarchy that can hold annotations. + Each entity, in the order above. """ yield self for source in self.walk_sources(): @@ -177,7 +177,7 @@ def annotation_owners(self) -> Iterator[SupportsAnnotate]: def walk_annotations(self) -> Iterable[Annotation]: """Yield annotation occurrences from this Record, its Sources, and its Signals.""" - return (annotation for owner in self.annotation_owners() for annotation in owner.annotations) + return (annotation for entity in self.annotatable_entities() for annotation in entity.annotations) @property def has_absolute_time(self) -> bool: diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 57659caf5..2fdadbcd2 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -278,17 +278,17 @@ def annotate(self, annotation: Annotation) -> Annotation: self.annotations = (*self.annotations, attached) return attached - def annotation_owners(self) -> Iterator[SupportsAnnotate]: - """Yield the task and each object that it owns that can hold annotations. + def annotatable_entities(self) -> Iterator[SupportsAnnotate]: + """Yield the entities of this task that can carry annotations: the task, then any entity that it holds. Yields: - The task. A task class with an owned object yields it after the task. + The task. A task class that holds another annotatable entity yields it after the task. """ yield self def walk_annotations(self) -> Iterator[Annotation]: - """Yield the annotation occurrences of each object that :meth:`annotation_owners` yields.""" - return (annotation for owner in self.annotation_owners() for annotation in owner.annotations) + """Yield the annotation occurrences of each entity that :meth:`annotatable_entities` yields.""" + return (annotation for entity in self.annotatable_entities() for annotation in entity.annotations) def spans(self) -> tuple[Span, ...]: """Return the task scope followed by every span-valued target.""" From 9cb970284462c4bf0803f36e4a52039e728d1ff2 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:06:53 -0800 Subject: [PATCH 4/7] refactor(dataset): keep Record.walk_annotations as on main Record.annotatable_entities() had no caller apart from Record.walk_annotations(), so it is gone and walk_annotations() walks the Sources and Signals itself again. Task keeps annotatable_entities(), because a task class can hold another annotatable entity. --- .../timenet/src/timenet/dataset/record.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/packages/timenet/src/timenet/dataset/record.py b/packages/timenet/src/timenet/dataset/record.py index 8175c0294..4a15a6bc0 100644 --- a/packages/timenet/src/timenet/dataset/record.py +++ b/packages/timenet/src/timenet/dataset/record.py @@ -1,6 +1,6 @@ """The :class:`Record` type: one logical unit of time-series data.""" -from collections.abc import Iterable, Iterator +from collections.abc import Iterable from dataclasses import dataclass, field from datetime import datetime @@ -164,20 +164,13 @@ def walk_signals(self) -> Iterable[Signal]: for source in self.walk_sources(): yield from sorted(source.signals, key=lambda signal: signal.name) - def annotatable_entities(self) -> Iterator[SupportsAnnotate]: - """Yield the entities of this Record that can carry annotations: the Record, then each Source and its Signals. - - Yields: - Each entity, in the order above. - """ - yield self - for source in self.walk_sources(): - yield source - yield from source.signals - def walk_annotations(self) -> Iterable[Annotation]: """Yield annotation occurrences from this Record, its Sources, and its Signals.""" - return (annotation for entity in self.annotatable_entities() for annotation in entity.annotations) + yield from self.annotations + for source in self.walk_sources(): + yield from source.annotations + for signal in source.signals: + yield from signal.annotations @property def has_absolute_time(self) -> bool: From 8a103b542618304f9a169109adff8bcc802436a3 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:58:23 -0800 Subject: [PATCH 5/7] refactor(types): drop the annotation(key) lookup Nothing in the source needs a lookup by key. Callers read the annotations directly. --- docs/data-model/annotations.md | 3 +-- .../timenet/src/timenet/types/annotations.py | 11 ---------- .../timenet/tests/dataset/test_dataset.py | 21 ------------------- 3 files changed, 1 insertion(+), 34 deletions(-) diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 6a9014e3a..27a7b5c35 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -25,8 +25,7 @@ occurrence = record.annotate(condition) `key` can be positional: `Annotation("condition", value="healthy")`. All other fields are keyword-only. `annotate()` returns the attached occurrence. A Task must refer to this occurrence, not the unattached -content object. Every owner also has `annotation(key)`. It returns the first annotation with that key, -or `None`. +content object. ## Content and occurrence fields diff --git a/packages/timenet/src/timenet/types/annotations.py b/packages/timenet/src/timenet/types/annotations.py index 478f0367b..417540a5a 100644 --- a/packages/timenet/src/timenet/types/annotations.py +++ b/packages/timenet/src/timenet/types/annotations.py @@ -219,17 +219,6 @@ def annotate(self, annotation: Annotation) -> Annotation: """Attach one occurrence of ``annotation`` and return it.""" ... - def annotation(self, key: str) -> Annotation | None: - """Look up an annotation of this object by its key. - - Args: - key: The annotation key, such as ``name``. - - Returns: - The first annotation with this key, or ``None`` if there is none. - """ - return next((annotation for annotation in self.annotations if annotation.key == key), None) - class AnnotationDescriptor(TimeFModel): """Type-level projection of an annotation key, stored in the schema and manifest.""" diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 4b5d705d6..7c2f24426 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -894,27 +894,6 @@ def test_walk_annotations_goes_from_the_dataset_to_records_to_tasks(make_series) assert [a.key for a in ds.walk_annotations()] == ["dataset", "record", "source", "signal", "task", "answer"] -def test_annotation_finds_the_first_match_on_every_owner(make_series): - ds = _dataset() - signal = make_series() - source = Source(name="Source", signals=(signal,)) - record = ds.add_record(record=Record(sources=(source,))) - task = _classification_task(record, "a") - owners = (ds, record, source, signal, task) - for owner in owners: - owner.annotate(Annotation(key="name", value=type(owner).__name__)) - owner.annotate(Annotation(key="name", value="second")) - found = [owner.annotation("name") for owner in owners] - assert [None if one is None else one.value for one in found] == [ - "TimeFDataset", - "Record", - "Source", - "Signal", - "ClassificationTask", - ] - assert ds.annotation("other") is None - - def test_add_tasks_allows_deriving_from_another_task_in_the_same_batch(make_series): # The deriving task is listed before its parent, so this only passes because the batch is validated # as a unit rather than task by task. From 0a35df57e1dbf2db3edbb6aac3158eb95f80c02c Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:01:00 -0800 Subject: [PATCH 6/7] refactor(types): let Task.walk_annotations yield its own annotations Task.annotatable_entities() only yielded the task. walk_annotations() now returns the task's annotations, and a task class that holds another annotated object can add its occurrences there. --- packages/timenet/src/timenet/types/tasks.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 2fdadbcd2..4d1754cdb 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -278,17 +278,9 @@ def annotate(self, annotation: Annotation) -> Annotation: self.annotations = (*self.annotations, attached) return attached - def annotatable_entities(self) -> Iterator[SupportsAnnotate]: - """Yield the entities of this task that can carry annotations: the task, then any entity that it holds. - - Yields: - The task. A task class that holds another annotatable entity yields it after the task. - """ - yield self - def walk_annotations(self) -> Iterator[Annotation]: - """Yield the annotation occurrences of each entity that :meth:`annotatable_entities` yields.""" - return (annotation for entity in self.annotatable_entities() for annotation in entity.annotations) + """Yield the annotation occurrences of the task. A task class that holds another annotated object adds its own.""" + return iter(self.annotations) def spans(self) -> tuple[Span, ...]: """Return the task scope followed by every span-valued target.""" From d4a95de23cad484886f5060ef98a9444d12d58ba Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:49:53 -0800 Subject: [PATCH 7/7] refactor(dataset): guard the streamed task check on the schema only --- packages/timenet/src/timenet/dataset/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index 75db138a4..9a4532450 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -489,14 +489,14 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: declared = set(self._streamed_task_types) label_sets = LabelSetRegistry() schema = self._schema - descriptors = None if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations} + descriptors = {} if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations} for position, task in enumerate(self.iter_tasks(), start=1): task._id = self._ids.claim(ObjectKind.TASK, self._streamed_task_base + position) self._validate_streamed_task(task, index, declared, label_sets) - if descriptors is not None: + if schema is not None: self._add_descriptors(descriptors, task.walk_annotations()) yield task - if schema is not None and descriptors is not None: + if schema is not None: self._schema = schema.model_copy(update={"annotations": tuple(descriptors.values())}) def _validate_streamed_task(