diff --git a/docs/timef-dataset.md b/docs/timef-dataset.md index 0f75e82aa..407b08a4d 100644 --- a/docs/timef-dataset.md +++ b/docs/timef-dataset.md @@ -237,7 +237,9 @@ lead_i.annotate( ## Schema and persistence `derive_schema()` collects the time-series specifications, annotation descriptors, and concrete Task -types found in the dataset. It does not load Signal values. +types found in the dataset. Annotation descriptors come from every owner: the Dataset, Records, +Sources, Signals, and Tasks. The Tasks of a task stream are not read here. The writer adds their +keys to the schema while it writes them. It does not load Signal values. Write a version with the declarative helper: diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index fd5788d71..9a4532450 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -385,6 +385,22 @@ def annotate(self, annotation: Annotation) -> Annotation: self._annotations.append(attached) return attached + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of the dataset, then of each Record, then of each task. + + Each Record and task gives the occurrences of the objects it holds (see :meth:`Record.walk_annotations` + and :meth:`Task.walk_annotations`). The tasks of a streamed dataset (see :meth:`set_task_stream`) are + not walked, so their occurrences do not come. + + Yields: + Each annotation occurrence, in the order above. + """ + yield from self.annotations + for record in self._records: + yield from record.walk_annotations() + for task in self._tasks: + yield from task.walk_annotations() + def set_task_stream(self, task_types: Sequence[type[Task]], source: Callable[[], Iterator[Task]]) -> None: """Provide tasks as a re-iterable stream instead of materializing them in the dataset. @@ -459,19 +475,29 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]: the cross-task checks (``from_tasks`` derivations) that need every task at once do not run for a stream. + If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks are + added to it. After the last task, the schema lists them. + Yields: Each validated task, in the source's order. Raises: - TimeFValidationError: If a streamed task fails one of the per-task checks. + TimeFValidationError: If a streamed task fails one of the per-task checks, or one of its + annotation keys has a descriptor that conflicts with the schema. """ # ruff: ignore[docstring-extraneous-exception] (raised by _validate_streamed_task, not directly here) index = _TaskValidationIndex(self) declared = set(self._streamed_task_types) label_sets = LabelSetRegistry() + schema = self._schema + descriptors = {} if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations} for position, task in enumerate(self.iter_tasks(), start=1): task._id = self._ids.claim(ObjectKind.TASK, self._streamed_task_base + position) self._validate_streamed_task(task, index, declared, label_sets) + if schema is not None: + self._add_descriptors(descriptors, task.walk_annotations()) yield task + if schema is not None: + self._schema = schema.model_copy(update={"annotations": tuple(descriptors.values())}) def _validate_streamed_task( self, @@ -686,15 +712,19 @@ def _check_no_derivation_cycle(batch: tuple[Task, ...]) -> None: def derive_schema(self) -> DatasetSchema: """Walk the dataset's instances and build its :class:`DatasetSchema`. - This method collects the distinct spec, annotation, and task types. It stores the result - on the dataset, and it returns the result. + This method collects the distinct spec, annotation, and task types. Annotations come from every + occurrence that :meth:`walk_annotations` yields. It stores the result on the dataset, and it returns + the result. + + The tasks of a streamed dataset (see :meth:`set_task_stream`) are not walked here. + :meth:`iter_streamed_tasks_validated` adds their annotation keys while the writer writes them. Returns: The derived :class:`DatasetSchema`. Raises: TimeFValidationError: If one spec type or annotation key yields conflicting descriptors - across records. + across the dataset. """ specs = self._ordered_unique(signal.spec for record in self._records for signal in record.signals) by_spec_type: dict[str, object] = {} @@ -705,35 +735,15 @@ def derive_schema(self) -> DatasetSchema: f"spec_type {spec.spec_type!r} has conflicting TimeSeriesSpec contracts: {existing!r} and {spec!r}" ) by_spec_type[spec.spec_type] = spec - record_annotations = (annotation for record in self._records for annotation in record.walk_annotations()) - annotations = self._ordered_unique( - AnnotationDescriptor( - key=annotation.key, - annotation_type=annotation_type_of(annotation), - value_type=value_type_of(annotation.value), - # The __post_init__ method normalizes unit to a plain string or None, so this - # value is always str | None. - unit=cast("str | None", annotation.unit), - description=annotation.description, - ) - for annotation in record_annotations - ) - by_key: dict[str, AnnotationDescriptor] = {} - for descriptor in annotations: - existing = by_key.get(descriptor.key) - if existing is not None: - raise TimeFValidationError( - f"annotation {descriptor.key!r} has conflicting descriptors across records: " - f"{existing!r} and {descriptor!r}" - ) - by_key[descriptor.key] = descriptor + annotations: dict[str, AnnotationDescriptor] = {} + self._add_descriptors(annotations, self.walk_annotations()) # A streamed dataset declares its task types up front, so the schema records them without # draining the stream just to collect them. streamed = self._ordered_unique(self._streamed_task_types) tasks = streamed if self._task_stream is not None else self._ordered_unique(type(task) for task in self._tasks) self._schema = DatasetSchema( time_series_specs=tuple(specs), - annotations=tuple(annotations), + annotations=tuple(annotations.values()), tasks=tuple(tasks), ) return self._schema @@ -886,6 +896,34 @@ def _referenced_records(self, task: Task) -> tuple[Record, ...]: found.setdefault(id(record), record) return tuple(found.values()) + @staticmethod + def _add_descriptors(descriptors: dict[str, AnnotationDescriptor], annotations: Iterable[Annotation]) -> None: + """Add the descriptor of each annotation key that ``descriptors`` does not hold yet. + + Args: + descriptors: The descriptors so far, by annotation key, in the order of first use. + annotations: The annotations to add. + + Raises: + TimeFValidationError: If an annotation gives a key a different descriptor than ``descriptors`` holds. + """ + for annotation in annotations: + descriptor = AnnotationDescriptor( + key=annotation.key, + annotation_type=annotation_type_of(annotation), + value_type=value_type_of(annotation.value), + # The __post_init__ method normalizes unit to a plain string or None, so this + # value is always str | None. + unit=cast("str | None", annotation.unit), + description=annotation.description, + ) + existing = descriptors.setdefault(descriptor.key, descriptor) + if existing != descriptor: + raise TimeFValidationError( + f"annotation {descriptor.key!r} has conflicting descriptors across the dataset: " + f"{existing!r} and {descriptor!r}" + ) + @staticmethod def _ordered_unique(items: Iterable[T]) -> list[T]: """Return items with duplicates removed, preserving first-seen order.""" diff --git a/packages/timenet/src/timenet/dataset/describe.py b/packages/timenet/src/timenet/dataset/describe.py index 9e5a287dd..43aa17066 100644 --- a/packages/timenet/src/timenet/dataset/describe.py +++ b/packages/timenet/src/timenet/dataset/describe.py @@ -68,7 +68,7 @@ def _identity(dataset: TimeFDataset) -> str: def _counts(dataset: TimeFDataset, unique_series: dict[EntityId, Signal]) -> str: task_counts = Counter(str(task.task_type) for task in dataset.tasks) - annotation_ids = {ann._content_key for record in dataset.records for ann in record.walk_annotations()} + annotation_ids = {ann._content_key for ann in dataset.walk_annotations()} spec_counts = Counter(ts.spec.spec_type for ts in unique_series.values()) fields = { "records": str(len(dataset.records)), diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 0a000ac41..4d1754cdb 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -1,6 +1,6 @@ """Declarative modelling tasks over records and signals.""" -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field, replace from enum import StrEnum, unique from functools import lru_cache @@ -278,6 +278,10 @@ def annotate(self, annotation: Annotation) -> Annotation: self.annotations = (*self.annotations, attached) return attached + def walk_annotations(self) -> Iterator[Annotation]: + """Yield the annotation occurrences of the task. A task class that holds another annotated object adds its own.""" + return iter(self.annotations) + def spans(self) -> tuple[Span, ...]: """Return the task scope followed by every span-valued target.""" spans = [] if self.scope is None else [self.scope] diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index 6aa31a4c5..7c2f24426 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -179,6 +179,57 @@ def test_derive_schema(make_series): assert ds.schema is schema +def test_derive_schema_collects_dataset_and_task_annotations(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="age", value=64)) + ds.annotate(Annotation(key="cohort", value="adult")) + task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + task.annotate(Annotation(key="difficulty", value=2)) + + schema = ds.derive_schema() + + assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"] + + +def test_derive_schema_rejects_a_task_annotation_that_conflicts_with_a_record_annotation(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="name", value=1)) + task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + task.annotate(Annotation(key="name", value="letters")) + with pytest.raises(TimeFValidationError, match="conflicting descriptors"): + ds.derive_schema() + + +def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="age", value=64)) + task = _classification_task(record, "a") + task.annotate(Annotation(key="difficulty", value=2)) + ds.set_task_stream([ClassificationTask], lambda: iter((task,))) + assert [descriptor.key for descriptor in ds.derive_schema().annotations] == ["age"] + + list(ds.iter_streamed_tasks_validated()) + + assert ds.schema is not None + assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty"] + + +def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema(make_series): + ds = _dataset() + record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),))) + record.annotate(Annotation(key="name", value=1)) + task = _classification_task(record, "a") + task.annotate(Annotation(key="name", value="letters")) + ds.set_task_stream([ClassificationTask], lambda: iter((task,))) + ds.derive_schema() + + with pytest.raises(TimeFValidationError, match="conflicting descriptors"): + list(ds.iter_streamed_tasks_validated()) + + def test_derive_schema_dedupes_specs(make_series): ds = _dataset() # Two signals of the same modality => one distinct spec. @@ -830,6 +881,19 @@ def test_add_tasks_rejects_the_whole_batch_when_one_is_invalid(make_series): assert ds.tasks == () # all-or-nothing: the valid one is not registered either +def test_walk_annotations_goes_from_the_dataset_to_records_to_tasks(make_series): + ds = _dataset() + signal = make_series() + source = Source(name="Source", signals=(signal,)) + record = ds.add_record(record=Record(sources=(source,))) + task = ds.add_task(task=_classification_task(record, "a")) + answer = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",))) + for owner, key in ((answer, "answer"), (task, "task"), (signal, "signal"), (source, "source"), (record, "record")): + owner.annotate(Annotation(key=key, value=key)) + ds.annotate(Annotation(key="dataset", value="dataset")) + assert [a.key for a in ds.walk_annotations()] == ["dataset", "record", "source", "signal", "task", "answer"] + + def test_add_tasks_allows_deriving_from_another_task_in_the_same_batch(make_series): # The deriving task is listed before its parent, so this only passes because the batch is validated # as a unit rather than task by task. diff --git a/packages/timenet/tests/dataset/test_describe.py b/packages/timenet/tests/dataset/test_describe.py index adb17cf0a..3a5103f59 100644 --- a/packages/timenet/tests/dataset/test_describe.py +++ b/packages/timenet/tests/dataset/test_describe.py @@ -3,7 +3,7 @@ from timenet.dataset import TimeFDataset from timenet.dataset.describe import describe_text from timenet.testing import make_dataset -from timenet.types import DatasetMetadata, License, Version +from timenet.types import Annotation, DatasetMetadata, License, Version def _previews(text: str, record_id: int) -> bool: @@ -58,3 +58,14 @@ def test_describe_empty_dataset_does_not_crash(): text = describe_text(empty, rows=5) assert "demo/empty @ 1.0.0" in text assert "records 0" in text + + +def test_describe_counts_annotations_of_every_owner(): + dataset = make_dataset() + before = describe_text(dataset, rows=0) + dataset.annotate(Annotation(key="describe_dataset_note", value="a")) + dataset.tasks[0].annotate(Annotation(key="describe_task_note", value="b")) + match = re.search(r"^ annotations\s+(\d+)$", before, re.MULTILINE) + assert match is not None + count = int(match[1]) + assert re.search(rf"^ annotations\s+{count + 2}$", describe_text(dataset, rows=0), re.MULTILINE) diff --git a/packages/timenet/tests/writer/test_writer.py b/packages/timenet/tests/writer/test_writer.py index 06c357bf1..7759d9442 100644 --- a/packages/timenet/tests/writer/test_writer.py +++ b/packages/timenet/tests/writer/test_writer.py @@ -108,6 +108,26 @@ def test_manifest_is_valid_and_matches_dataset(tmp_path): assert (version_dir / rel).exists() +def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_path): + dataset = make_dataset() + record = dataset.records[0] + streamed = TimeFDataset.from_parts( + metadata=dataset.metadata, + records=dataset.records, + tasks=(), + schema=dataset.derive_schema(), + last_ids=dataset.last_ids, + ) + task = ClassificationTask(inputs=(record,), targets=("x",), label_set=LABELS) + task.annotate(Annotation(key="task_note", value="letters")) + streamed.set_task_stream([ClassificationTask], lambda: iter((task,))) + + version_dir = _written(tmp_path, streamed) + + manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text()) + assert "task_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations] + + def test_manifest_counts_match_the_control_database(tmp_path): version_dir = _written(tmp_path) manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text())