Skip to content
4 changes: 3 additions & 1 deletion docs/timef-dataset.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,9 @@ lead_i.annotate(
## Schema and persistence

`derive_schema()` collects the time-series specifications, annotation descriptors, and concrete Task
types found in the dataset. It does not load Signal values.
types found in the dataset. Annotation descriptors come from every owner: the Dataset, Records,
Sources, Signals, and Tasks. The Tasks of a task stream are not read here. The writer adds their
keys to the schema while it writes them. It does not load Signal values.

Write a version with the declarative helper:

Expand Down
92 changes: 65 additions & 27 deletions packages/timenet/src/timenet/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,22 @@ def annotate(self, annotation: Annotation) -> Annotation:
self._annotations.append(attached)
return attached

def walk_annotations(self) -> Iterator[Annotation]:
"""Yield the annotation occurrences of the dataset, then of each Record, then of each task.

Each Record and task gives the occurrences of the objects it holds (see :meth:`Record.walk_annotations`
and :meth:`Task.walk_annotations`). The tasks of a streamed dataset (see :meth:`set_task_stream`) are
not walked, so their occurrences do not come.

Yields:
Each annotation occurrence, in the order above.
"""
yield from self.annotations
for record in self._records:
yield from record.walk_annotations()
for task in self._tasks:
yield from task.walk_annotations()

def set_task_stream(self, task_types: Sequence[type[Task]], source: Callable[[], Iterator[Task]]) -> None:
"""Provide tasks as a re-iterable stream instead of materializing them in the dataset.

Expand Down Expand Up @@ -459,19 +475,29 @@ def iter_streamed_tasks_validated(self) -> Iterator[Task]:
the cross-task checks (``from_tasks`` derivations) that need every task at once do not run
for a stream.

If the dataset has a schema (see :meth:`derive_schema`), the annotation keys of the tasks are
added to it. After the last task, the schema lists them.

Yields:
Each validated task, in the source's order.

Raises:
TimeFValidationError: If a streamed task fails one of the per-task checks.
TimeFValidationError: If a streamed task fails one of the per-task checks, or one of its
annotation keys has a descriptor that conflicts with the schema.
""" # ruff: ignore[docstring-extraneous-exception] (raised by _validate_streamed_task, not directly here)
index = _TaskValidationIndex(self)
declared = set(self._streamed_task_types)
label_sets = LabelSetRegistry()
schema = self._schema
descriptors = {} if schema is None else {descriptor.key: descriptor for descriptor in schema.annotations}
for position, task in enumerate(self.iter_tasks(), start=1):
task._id = self._ids.claim(ObjectKind.TASK, self._streamed_task_base + position)
self._validate_streamed_task(task, index, declared, label_sets)
if schema is not None:
self._add_descriptors(descriptors, task.walk_annotations())
yield task
if schema is not None:
self._schema = schema.model_copy(update={"annotations": tuple(descriptors.values())})

def _validate_streamed_task(
self,
Expand Down Expand Up @@ -686,15 +712,19 @@ def _check_no_derivation_cycle(batch: tuple[Task, ...]) -> None:
def derive_schema(self) -> DatasetSchema:
"""Walk the dataset's instances and build its :class:`DatasetSchema`.

This method collects the distinct spec, annotation, and task types. It stores the result
on the dataset, and it returns the result.
This method collects the distinct spec, annotation, and task types. Annotations come from every
occurrence that :meth:`walk_annotations` yields. It stores the result on the dataset, and it returns
the result.

The tasks of a streamed dataset (see :meth:`set_task_stream`) are not walked here.
:meth:`iter_streamed_tasks_validated` adds their annotation keys while the writer writes them.

Returns:
The derived :class:`DatasetSchema`.

Raises:
TimeFValidationError: If one spec type or annotation key yields conflicting descriptors
across records.
across the dataset.
"""
specs = self._ordered_unique(signal.spec for record in self._records for signal in record.signals)
by_spec_type: dict[str, object] = {}
Expand All @@ -705,35 +735,15 @@ def derive_schema(self) -> DatasetSchema:
f"spec_type {spec.spec_type!r} has conflicting TimeSeriesSpec contracts: {existing!r} and {spec!r}"
)
by_spec_type[spec.spec_type] = spec
record_annotations = (annotation for record in self._records for annotation in record.walk_annotations())
annotations = self._ordered_unique(
AnnotationDescriptor(
key=annotation.key,
annotation_type=annotation_type_of(annotation),
value_type=value_type_of(annotation.value),
# The __post_init__ method normalizes unit to a plain string or None, so this
# value is always str | None.
unit=cast("str | None", annotation.unit),
description=annotation.description,
)
for annotation in record_annotations
)
by_key: dict[str, AnnotationDescriptor] = {}
for descriptor in annotations:
existing = by_key.get(descriptor.key)
if existing is not None:
raise TimeFValidationError(
f"annotation {descriptor.key!r} has conflicting descriptors across records: "
f"{existing!r} and {descriptor!r}"
)
by_key[descriptor.key] = descriptor
annotations: dict[str, AnnotationDescriptor] = {}
self._add_descriptors(annotations, self.walk_annotations())
# A streamed dataset declares its task types up front, so the schema records them without
# draining the stream just to collect them.
streamed = self._ordered_unique(self._streamed_task_types)
tasks = streamed if self._task_stream is not None else self._ordered_unique(type(task) for task in self._tasks)
self._schema = DatasetSchema(
time_series_specs=tuple(specs),
annotations=tuple(annotations),
annotations=tuple(annotations.values()),
tasks=tuple(tasks),
)
return self._schema
Expand Down Expand Up @@ -886,6 +896,34 @@ def _referenced_records(self, task: Task) -> tuple[Record, ...]:
found.setdefault(id(record), record)
return tuple(found.values())

@staticmethod
def _add_descriptors(descriptors: dict[str, AnnotationDescriptor], annotations: Iterable[Annotation]) -> None:
"""Add the descriptor of each annotation key that ``descriptors`` does not hold yet.

Args:
descriptors: The descriptors so far, by annotation key, in the order of first use.
annotations: The annotations to add.

Raises:
TimeFValidationError: If an annotation gives a key a different descriptor than ``descriptors`` holds.
"""
for annotation in annotations:
descriptor = AnnotationDescriptor(
key=annotation.key,
annotation_type=annotation_type_of(annotation),
value_type=value_type_of(annotation.value),
# The __post_init__ method normalizes unit to a plain string or None, so this
# value is always str | None.
unit=cast("str | None", annotation.unit),
description=annotation.description,
)
existing = descriptors.setdefault(descriptor.key, descriptor)
if existing != descriptor:
raise TimeFValidationError(
f"annotation {descriptor.key!r} has conflicting descriptors across the dataset: "
f"{existing!r} and {descriptor!r}"
)

@staticmethod
def _ordered_unique(items: Iterable[T]) -> list[T]:
"""Return items with duplicates removed, preserving first-seen order."""
Expand Down
2 changes: 1 addition & 1 deletion packages/timenet/src/timenet/dataset/describe.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def _identity(dataset: TimeFDataset) -> str:

def _counts(dataset: TimeFDataset, unique_series: dict[EntityId, Signal]) -> str:
task_counts = Counter(str(task.task_type) for task in dataset.tasks)
annotation_ids = {ann._content_key for record in dataset.records for ann in record.walk_annotations()}
annotation_ids = {ann._content_key for ann in dataset.walk_annotations()}
spec_counts = Counter(ts.spec.spec_type for ts in unique_series.values())
fields = {
"records": str(len(dataset.records)),
Expand Down
6 changes: 5 additions & 1 deletion packages/timenet/src/timenet/types/tasks.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Declarative modelling tasks over records and signals."""

from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass, field, replace
from enum import StrEnum, unique
from functools import lru_cache
Expand Down Expand Up @@ -278,6 +278,10 @@ def annotate(self, annotation: Annotation) -> Annotation:
self.annotations = (*self.annotations, attached)
return attached

def walk_annotations(self) -> Iterator[Annotation]:
"""Yield the annotation occurrences of the task. A task class that holds another annotated object adds its own."""
return iter(self.annotations)

def spans(self) -> tuple[Span, ...]:
"""Return the task scope followed by every span-valued target."""
spans = [] if self.scope is None else [self.scope]
Expand Down
64 changes: 64 additions & 0 deletions packages/timenet/tests/dataset/test_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,57 @@ def test_derive_schema(make_series):
assert ds.schema is schema


def test_derive_schema_collects_dataset_and_task_annotations(make_series):
ds = _dataset()
record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),)))
record.annotate(Annotation(key="age", value=64))
ds.annotate(Annotation(key="cohort", value="adult"))
task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",)))
task.annotate(Annotation(key="difficulty", value=2))

schema = ds.derive_schema()

assert [descriptor.key for descriptor in schema.annotations] == ["cohort", "age", "difficulty"]


def test_derive_schema_rejects_a_task_annotation_that_conflicts_with_a_record_annotation(make_series):
ds = _dataset()
record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),)))
record.annotate(Annotation(key="name", value=1))
task = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",)))
task.annotate(Annotation(key="name", value="letters"))
with pytest.raises(TimeFValidationError, match="conflicting descriptors"):
ds.derive_schema()


def test_a_task_stream_adds_its_annotation_keys_to_the_schema(make_series):
ds = _dataset()
record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),)))
record.annotate(Annotation(key="age", value=64))
task = _classification_task(record, "a")
task.annotate(Annotation(key="difficulty", value=2))
ds.set_task_stream([ClassificationTask], lambda: iter((task,)))
assert [descriptor.key for descriptor in ds.derive_schema().annotations] == ["age"]

list(ds.iter_streamed_tasks_validated())

assert ds.schema is not None
assert [descriptor.key for descriptor in ds.schema.annotations] == ["age", "difficulty"]


def test_a_task_stream_rejects_an_annotation_key_that_conflicts_with_the_schema(make_series):
ds = _dataset()
record = ds.add_record(record=Record(sources=(Source(name="Source", signals=(make_series(),)),)))
record.annotate(Annotation(key="name", value=1))
task = _classification_task(record, "a")
task.annotate(Annotation(key="name", value="letters"))
ds.set_task_stream([ClassificationTask], lambda: iter((task,)))
ds.derive_schema()

with pytest.raises(TimeFValidationError, match="conflicting descriptors"):
list(ds.iter_streamed_tasks_validated())


def test_derive_schema_dedupes_specs(make_series):
ds = _dataset()
# Two signals of the same modality => one distinct spec.
Expand Down Expand Up @@ -830,6 +881,19 @@ def test_add_tasks_rejects_the_whole_batch_when_one_is_invalid(make_series):
assert ds.tasks == () # all-or-nothing: the valid one is not registered either


def test_walk_annotations_goes_from_the_dataset_to_records_to_tasks(make_series):
ds = _dataset()
signal = make_series()
source = Source(name="Source", signals=(signal,))
record = ds.add_record(record=Record(sources=(source,)))
task = ds.add_task(task=_classification_task(record, "a"))
answer = ds.add_task(task=AnswerTask(inputs=(record,), targets=("x",)))
for owner, key in ((answer, "answer"), (task, "task"), (signal, "signal"), (source, "source"), (record, "record")):
owner.annotate(Annotation(key=key, value=key))
ds.annotate(Annotation(key="dataset", value="dataset"))
assert [a.key for a in ds.walk_annotations()] == ["dataset", "record", "source", "signal", "task", "answer"]


def test_add_tasks_allows_deriving_from_another_task_in_the_same_batch(make_series):
# The deriving task is listed before its parent, so this only passes because the batch is validated
# as a unit rather than task by task.
Expand Down
13 changes: 12 additions & 1 deletion packages/timenet/tests/dataset/test_describe.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from timenet.dataset import TimeFDataset
from timenet.dataset.describe import describe_text
from timenet.testing import make_dataset
from timenet.types import DatasetMetadata, License, Version
from timenet.types import Annotation, DatasetMetadata, License, Version


def _previews(text: str, record_id: int) -> bool:
Expand Down Expand Up @@ -58,3 +58,14 @@ def test_describe_empty_dataset_does_not_crash():
text = describe_text(empty, rows=5)
assert "demo/empty @ 1.0.0" in text
assert "records 0" in text


def test_describe_counts_annotations_of_every_owner():
dataset = make_dataset()
before = describe_text(dataset, rows=0)
dataset.annotate(Annotation(key="describe_dataset_note", value="a"))
dataset.tasks[0].annotate(Annotation(key="describe_task_note", value="b"))
match = re.search(r"^ annotations\s+(\d+)$", before, re.MULTILINE)
assert match is not None
count = int(match[1])
assert re.search(rf"^ annotations\s+{count + 2}$", describe_text(dataset, rows=0), re.MULTILINE)
20 changes: 20 additions & 0 deletions packages/timenet/tests/writer/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,26 @@ def test_manifest_is_valid_and_matches_dataset(tmp_path):
assert (version_dir / rel).exists()


def test_the_manifest_schema_lists_the_annotation_keys_of_streamed_tasks(tmp_path):
dataset = make_dataset()
record = dataset.records[0]
streamed = TimeFDataset.from_parts(
metadata=dataset.metadata,
records=dataset.records,
tasks=(),
schema=dataset.derive_schema(),
last_ids=dataset.last_ids,
)
task = ClassificationTask(inputs=(record,), targets=("x",), label_set=LABELS)
task.annotate(Annotation(key="task_note", value="letters"))
streamed.set_task_stream([ClassificationTask], lambda: iter((task,)))

version_dir = _written(tmp_path, streamed)

manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text())
assert "task_note" in [descriptor.key for descriptor in manifest.dataset_schema.annotations]


def test_manifest_counts_match_the_control_database(tmp_path):
version_dir = _written(tmp_path)
manifest = Manifest.model_validate_json((version_dir / "manifest.json").read_text())
Expand Down
Loading