diff --git a/.agents/skills/add-dataset-connector/SKILL.md b/.agents/skills/add-dataset-connector/SKILL.md index 0347e5ea5..95e53089d 100644 --- a/.agents/skills/add-dataset-connector/SKILL.md +++ b/.agents/skills/add-dataset-connector/SKILL.md @@ -87,6 +87,10 @@ Add the org namespace `__init__.py` if the org is new. Keep `download` I/O-only with lazy value loaders (never materialize arrays in `convert`). See the worked example in `references/connector-anatomy.md`. +Attach annotations when you build the owner: pass `annotations=[Annotation("key", value=value)]` to +`TimeFDataset`, `Record`, `Source`, `Signal`, or the task. Use `annotate()` only when the owner +exists already, or when a task refers to the occurrence that the call returns. + You cannot pass ids. Records, signals, and tasks get a read-only int `id` when you add them to the dataset. Sources, axes, and annotations have no id. Put upstream identifiers in `Record.metadata` (for example `{"ecg_id": 17}`) or in an annotation. Refer to other objects by the Python object, diff --git a/.agents/skills/add-dataset-connector/references/connector-anatomy.md b/.agents/skills/add-dataset-connector/references/connector-anatomy.md index 754d40bec..1d63a25bc 100644 --- a/.agents/skills/add-dataset-connector/references/connector-anatomy.md +++ b/.agents/skills/add-dataset-connector/references/connector-anatomy.md @@ -132,9 +132,19 @@ Populate a `TimeFDataset` (`from timenet.dataset import Record, Signal, Source, field such as `sources`, `signals`, or `inputs` takes any iterable and stores a tuple. A windowed record says so through its axis: `RegularAxis.at_index(...)` moves the origin into the recording. -- `record.annotate(Annotation("key", value=value))` attaches one annotation and returns it. - `record.add_annotations([...])` takes an iterable and returns a tuple. There is one annotation - class, and its `span` gives its shape. No span means the whole record. +- Attach annotations when you build the owner. `TimeFDataset`, `Record`, `Source`, `Signal`, and + every task take `annotations=[...]`, a list of `Annotation` objects: + `Record(sources=[source], annotations=[Annotation("key", value=value)])`. `Annotation` takes + only `key` by position. For an annotation that is not always present, build the list first and + append to it. +- Use `owner.annotate(annotation)` only in these cases. The owner exists already, for example a + Record that you import from a parent. A task refers to the annotation in `input_annotations` or + `target_annotations`, so you need the occurrence that the call returns. + `record.add_annotations([...])` does the same for a batch and returns a tuple. A constructor + attaches a new occurrence of each annotation, so one `Annotation` object can go to several + owners. A task can also refer to an occurrence that it takes from the owner, for example + `record.annotations[0]`. +- There is one annotation class, and its `span` gives its shape. No span means the whole record. `span=TimePoint.seconds(...)` means a time offset. `span=TimeInterval.seconds(...)` means a region. - For text that the connector reads from a source file, use `LazyText(function)` as the annotation value. Import it from `timenet.types`. The function returns the string. If you know @@ -242,11 +252,11 @@ class TSQAConnector( ) for signal, values in enumerate(signals) ] - record = Record(sources=[Source("TSQA series", signals=time_series)]) - dataset.add_record(record) - record.annotate(Annotation("task", value=row.task)) + annotations = [Annotation("task", value=row.task)] if row.label: - record.annotate(Annotation("label", value=row.label)) + annotations.append(Annotation("label", value=row.label)) + record = Record(sources=[Source("TSQA series", signals=time_series)], annotations=annotations) + dataset.add_record(record) dataset.add_task(AnswerTask(inputs=[record], prompt=row.question, targets=row.answer)) return dataset diff --git a/docs/connectors.md b/docs/connectors.md index e0c2922b1..446baa57a 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -80,7 +80,12 @@ Each Signal has exactly one owning Source. Do not attach one Signal to several S TimeNet rejects that ambiguous ownership. Signals can share immutable `TimeAxis` and `TimeSeriesSpec` objects. Annotation content can also be -reused. Each call to `annotate()` creates a separate occurrence for the selected owner. +reused. Each call to `annotate()`, and each owner that you build with `annotations=[...]`, creates a +separate occurrence for the owner. + +Attach annotations when you build the owner: pass `annotations=[...]` to the dataset, Record, Source, +Signal, or Task. Use `annotate()` only for the cases that +[Attach annotations](data-model/annotations.md#attach-annotations) lists. `Annotation` and the Task classes are final. To reuse an annotation, keep a constant or write a builder function, as [Reuse content](data-model/annotations.md#reuse-content) shows. To reuse a diff --git a/docs/data-model/annotations.md b/docs/data-model/annotations.md index 6407dcafa..8aa7a31dd 100644 --- a/docs/data-model/annotations.md +++ b/docs/data-model/annotations.md @@ -12,6 +12,7 @@ An `Annotation` stores reusable content. An annotation occurrence attaches that Task, Record, Source, Signal, or label set. ```python +from timenet.dataset import Record, TimeFDataset from timenet.types import Annotation condition = Annotation( @@ -19,13 +20,42 @@ condition = Annotation( value="healthy", description="Condition assigned by the source dataset.", ) -occurrence = record.annotate(condition) +record = Record(sources=(source,), annotations=[condition]) ``` `key` can be positional: `Annotation("condition", value="healthy")`. All other fields are keyword-only. +## Attach annotations + +Pass `annotations=[...]` when you build the owner. A dataset, a Task, a Record, a Source, and a +Signal all take this argument. It is a list, or any other iterable, of `Annotation` objects: + +```python +dataset = TimeFDataset( + metadata=metadata, + annotations=[ + Annotation("declared_sampling_rate_hz", value=20), + Annotation("collection_site", value="clinic-a"), + ], +) +``` + +The dataset attaches each annotation in order, as `dataset.annotate()` does. A dataset annotation +cannot have a `span`. Every owner attaches a new occurrence of each annotation that you pass, as +`annotate()` does. So you can pass one `Annotation` object to several owners. + +Use `annotate()` in these cases: + +- The owner exists already, for example a Record that you import from a parent dataset. +- A Task refers to the annotation. Keep the occurrence that `annotate()` returns. + +```python +occurrence = record.annotate(condition) +``` + `annotate()` returns the attached occurrence. A Task must refer to this occurrence, not the unattached -content object. +content object. For an annotation that you passed to the constructor, take the occurrence from the +owner, for example `record.annotations[0]`. ## Content and occurrence fields @@ -71,8 +101,13 @@ row are stored as one row again, but stored text and the same text in memory bec An Annotation without a `span` describes its complete owner: ```python -record.annotate(Annotation("age", value=54, unit="years")) -record.annotate(Annotation("condition", value="healthy")) +record = Record( + sources=(source,), + annotations=[ + Annotation("age", value=54, unit="years"), + Annotation("condition", value="healthy"), + ], +) ``` Use `metadata` for untyped implementation details. Use an Annotation when consumers need a named, diff --git a/docs/data-model/records.md b/docs/data-model/records.md index 38cb2e366..5e59a75b7 100644 --- a/docs/data-model/records.md +++ b/docs/data-model/records.md @@ -74,12 +74,17 @@ Use an [Annotation](annotations.md) for a typed, searchable fact. Examples inclu firmware version, or an event on the timeline. ```python +from timenet.dataset import Record from timenet.types import Annotation -record.annotate(Annotation("condition", value="healthy")) +record = Record( + sources=(source,), + annotations=[Annotation("condition", value="healthy")], +) ``` -Sources and Signals also support `metadata` and `annotate()`. +Sources and Signals also take `metadata` and `annotations`. To annotate an owner that exists +already, call `annotate()`, as [Attach annotations](annotations.md#attach-annotations) shows. ## Add time information diff --git a/docs/data-model/sources.md b/docs/data-model/sources.md index 4e4c4f244..fb6c1dcde 100644 --- a/docs/data-model/sources.md +++ b/docs/data-model/sources.md @@ -115,7 +115,8 @@ Names do not have to be unique. If two Signals below one Source have the same na ## Annotate a Source -`source.annotate(annotation)` attaches one occurrence and returns it. A static annotation +Pass `annotations=[...]` to the constructor to attach annotations when you build the Source. +`source.annotate(annotation)` attaches one occurrence to a Source that exists and returns it. A static annotation describes the whole device, for example its firmware version. A timed annotation covers an interval, for example a time when an electrode lost contact with the skin: diff --git a/docs/timef-dataset.md b/docs/timef-dataset.md index 056379fa0..75519917a 100644 --- a/docs/timef-dataset.md +++ b/docs/timef-dataset.md @@ -76,7 +76,8 @@ An irregular Signal also needs its time offsets. Pass `time_offsets_us` to `time_offsets_us()`. Regular axes calculate their offsets from their cadence, and ordinal axes describe order without a clock. -Call `signal.annotate(annotation)` to attach one occurrence to a Signal. +Pass `annotations=[...]` to the constructor to attach annotations when you build the Signal. Call +`signal.annotate(annotation)` to attach an occurrence to a Signal that exists. ## Source @@ -110,8 +111,9 @@ with two parents, and a Signal with two owners. Sources group Signals. All timed Signals in a Record use its common relative origin. -Call `source.annotate(annotation)` for a Source annotation. A selection attaches annotation content -to several Signals in one operation: +Pass `annotations=[...]` to the constructor to attach annotations when you build the Source. Call +`source.annotate(annotation)` to attach an occurrence to a Source that exists. A selection attaches +annotation content to several Signals in one operation: ```python source.select(signals=(lead_i, lead_ii)).annotate(annotation) @@ -128,9 +130,11 @@ annotations and metadata: ```python from timenet.dataset import Record +from timenet.types import Annotation record = Record( sources=(imu,), + annotations=[Annotation("condition", value="healthy")], metadata={"site": "lab-a", "session": "record-123"}, ) ``` @@ -147,16 +151,19 @@ record = Record( `record.signals` flattens every Signal in the hierarchy. `walk_sources()` and `walk_signals()` expose the same deterministic traversal used by the writer. -Call `record.annotate(annotation)` to attach an occurrence. `time_point()` and `time_interval()` +Pass `annotations=[...]` to the constructor to attach annotations when you build the Record. Call +`record.annotate(annotation)` to attach an occurrence to a Record that exists. `time_point()` and `time_interval()` convert wall-clock values against the Record origin. They require a known absolute timestamp. Store subject identifiers as a list-valued record annotation. Omit it for a record with no subjects: ```python +from timenet.dataset import Record from timenet.types import Annotation -record.annotate( - Annotation("subject_ids", value=["patient-1", "patient-2"]) +record = Record( + sources=(imu,), + annotations=[Annotation("subject_ids", value=["patient-1", "patient-2"])], ) ``` @@ -197,12 +204,19 @@ Register complete Records independently of Tasks. This supports datasets with or ```python from timenet.dataset import TimeFDataset +from timenet.types import Annotation -dataset = TimeFDataset(metadata=metadata) +dataset = TimeFDataset( + metadata=metadata, + annotations=[Annotation("source_revision", value="4f2a9c1")], +) dataset.add_record(record) dataset.add_task(task) ``` +`annotations` is optional. It takes an iterable of `Annotation` objects and attaches each one to the +dataset in order, as `dataset.annotate()` does. A dataset annotation cannot have a `span`. + `add_record()` validates hierarchy ownership. It rejects a Record that is already in a dataset, and a Source or Signal that another Record owns. It then gives the Record and its Signals their IDs. `add_task()` validates referenced objects, the answer representation, and temporal scopes. A @@ -213,17 +227,23 @@ inserts that stream once without retaining every Task in memory. ## Annotations -The same `Annotation` type can annotate a Dataset, Task, Record, Source, Signal, or label set. Annotation -content is immutable and reusable. Every call to `annotate()` creates a separate occurrence, so a -long payload can be stored once while its occurrences carry independent placement, confidence, and -provenance. `Annotation` is final: reuse content through a constant, or a builder function for -content that varies, as [Annotations](data-model/annotations.md#reuse-content) shows. +The same `Annotation` type can annotate a Dataset, Task, Record, Source, Signal, or label set. Each of +them takes `annotations=[...]` at construction. This is the usual way to attach annotations. Use +`annotate()` when the owner exists already, or when a Task refers to the occurrence that the call +returns. + +Annotation content is immutable and reusable. Every call to `annotate()` and every constructor creates +a separate occurrence, so a long payload can be stored once while its occurrences carry independent +placement, confidence, and provenance. `Annotation` is final: reuse content through a constant, or a +builder function for content that varies, as [Annotations](data-model/annotations.md#reuse-content) +shows. ```python +from timenet.dataset import Record from timenet.types import Annotation, TimePoint male = Annotation("patient_sex", value="male") -record.annotate(male) +record = Record(sources=(imu,), annotations=[male]) lead_i.annotate( Annotation( diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py index 690cf98ce..ec76c7ffe 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/connector.py @@ -286,14 +286,18 @@ def convert(self, files: Mapping[str, Path]) -> TimeFDataset: Raises: TimeFValidationError: If source rows violate the SensorCaps mapping. """ - dataset = TimeFDataset(metadata=self.metadata()) - for key, value in ( - ("declared_sampling_rate_hz", DECLARED_SAMPLING_RATE_HZ), - ("source_sampling_rate_hz", _SOURCE_SAMPLING_RATE_HZ), - ("source_window_samples", _SOURCE_WINDOW_SAMPLES), - ("time_axis_reconstruction", "round(linspace(0, 119, 60)) / 20 seconds relative to window start"), - ): - dataset.annotate(Annotation(key, value=value)) + dataset = TimeFDataset( + metadata=self.metadata(), + annotations=[ + Annotation("declared_sampling_rate_hz", value=DECLARED_SAMPLING_RATE_HZ), + Annotation("source_sampling_rate_hz", value=_SOURCE_SAMPLING_RATE_HZ), + Annotation("source_window_samples", value=_SOURCE_WINDOW_SAMPLES), + Annotation( + "time_axis_reconstruction", + value="round(linspace(0, 119, 60)) / 20 seconds relative to window start", + ), + ], + ) records: dict[tuple[str, int], Record] = {} for part, path in files.items(): count = 0 @@ -326,7 +330,6 @@ def _make_record(self, ref: _RowRef, row: _ParsedRow) -> Record: ) for spec, name, sensor, dimension in channels ] - record = Record(sources=[Source("IMU", signals=signals)]) source = ref.part.rsplit("_", 1)[0] metadata: dict[str, str | int] = { "source_dataset": source, @@ -337,9 +340,10 @@ def _make_record(self, ref: _RowRef, row: _ParsedRow) -> Record: } if row.location is not None: metadata["sensor_location"] = row.location - for key, value in metadata.items(): - record.annotate(Annotation(key, value=value)) - return record + return Record( + sources=[Source("IMU", signals=signals)], + annotations=[Annotation(key, value=value) for key, value in metadata.items()], + ) CONNECTOR = SensorCapsConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py index 12acffa4c..258b02172 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/chengsenwang/tsqa/connector.py @@ -94,11 +94,10 @@ def _make_record(sample: TSQASample) -> Record: time_axis=OrdinalAxis(), ) signals.append(signal) - record = Record(sources=[Source("TSQA series", signals=signals)]) - record.annotate(Annotation("task", value=sample.category)) + annotations = [Annotation("task", value=sample.category)] if sample.label: - record.annotate(Annotation("label", value=sample.label)) - return record + annotations.append(Annotation("label", value=sample.label)) + return Record(sources=[Source("TSQA series", signals=signals)], annotations=annotations) @staticmethod def _make_task(sample: TSQASample, record: Record, *, split: Split) -> AnswerTask: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/series.py b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/series.py index 2bb540c07..81a22bd44 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/series.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/datadog/arfbench/series.py @@ -220,17 +220,14 @@ def create_record(self) -> Record: if label in groups ] sources.append(Source(label or UNTAGGED, signals=signals)) - record = Record( + return Record( metadata={"source_record_id": f"arfbench-{self.metric}"}, start_time=TimeOrigin(origin_us), sources=sources, - ) - record.add_annotations( - [ + annotations=[ Annotation("metric_id", value=self.metric), Annotation("incident_id", value=self.metric.partition("_")[0]), Annotation("query_name", value=self.finest.read_query_name()), Annotation("finest_interval_s", value=self.finest.interval_s, unit="second"), - ] + ], ) - return record diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py index 69d3a7922..d00237c07 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/ptb_xl/connector.py @@ -166,10 +166,13 @@ def convert(self, source: PtbXlSource) -> TimeFDataset: ) for position, name in enumerate(header.sig_name) ] - record = dataset.add_record( - Record(metadata={"ecg_id": identity}, sources=[Source("PTB-XL ECG", signals=signals)]) + dataset.add_record( + Record( + metadata={"ecg_id": identity}, + sources=[Source("PTB-XL ECG", signals=signals)], + annotations=annotations, + ) ) - record.add_annotations(annotations) except (KeyError, ValueError, SyntaxError, OSError, TimeNetConnectorError) as exc: raise TimeNetConnectorError( f"invalid PTB-XL row {line} (ecg_id {row.get('ecg_id')}) in {source.database_csv}: {exc}" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py index 3822c322a..7611f20e6 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/physionet/sleep_edfx/connector.py @@ -299,10 +299,10 @@ def convert(self, source: SleepEdfxSource) -> TimeFDataset: record = Record( sources=[Source("Sleep-EDF polysomnograph", signals=series)], time_span=TimeInterval.micros(0, session_end), + annotations=[Annotation("subject_ids", value=[recording.subject_id])], metadata={"recording_id": recording.recording_id}, ) dataset.add_record(record) - record.annotate(Annotation("subject_ids", value=[recording.subject_id])) annotations.attach(recording.recording_id, series, sleep_stages) record.add_annotations(recording_metadata) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py index a4263f400..76607c09f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/tsaqa/tsaqa_benchmark/connector.py @@ -669,11 +669,11 @@ def _records(row: _Row, loader: _SeriesLoader) -> tuple[Record, ...]: loader=partial(loader, row, position), ) metadata: dict[str, JsonValue] = {"file": row.file, "row": row.index, "role": str(role)} - record = Record(sources=[Source("series", signals=[signal])], metadata=metadata) - if position == 0: - for key, value in row.contexts.on_input.items(): - record.annotate(Annotation(key, value=value if isinstance(value, str) else list(value))) - records.append(record) + contexts = row.contexts.on_input if position == 0 else {} + annotations = [ + Annotation(key, value=value if isinstance(value, str) else list(value)) for key, value in contexts.items() + ] + records.append(Record(sources=[Source("series", signals=[signal])], annotations=annotations, metadata=metadata)) return tuple(records) diff --git a/packages/timenet/src/timenet/dataset/composition.py b/packages/timenet/src/timenet/dataset/composition.py index b58c55756..1d9c5e23f 100644 --- a/packages/timenet/src/timenet/dataset/composition.py +++ b/packages/timenet/src/timenet/dataset/composition.py @@ -262,7 +262,8 @@ def copy_signal(self, original: Signal) -> Signal: The copy, with copies of its annotations. The child gives the copy an ID when it indexes the copy. """ - return Signal.from_loader( + annotations = tuple(self.copy_annotation(annotation) for annotation in original.annotations) + copy = Signal.from_loader( spec=original.spec, time_axis=original.time_axis, name=original.name, @@ -270,9 +271,11 @@ def copy_signal(self, original: Signal) -> Signal: n_values=original.n_values, null_count=original.null_count, time_offsets_loader=original._time_offsets, - annotations=tuple(self.copy_annotation(annotation) for annotation in original.annotations), + annotations=annotations, metadata=deepcopy(original.metadata), ) + _set_annotations(copy, annotations) + return copy def copy_annotation(self, original: Annotation) -> Annotation: """Copy a parent annotation occurrence as a value (see :func:`_copy_annotation`). diff --git a/packages/timenet/src/timenet/dataset/dataset.py b/packages/timenet/src/timenet/dataset/dataset.py index c019ed46c..056552b09 100644 --- a/packages/timenet/src/timenet/dataset/dataset.py +++ b/packages/timenet/src/timenet/dataset/dataset.py @@ -33,6 +33,7 @@ read_value, value_type_of, ) +from timenet.types._fields import as_tuple from timenet.types.ids import IdCounters from timenet.types.load_plan import LoadPlan from timenet.types.parent_dataset_card import ParentDatasetCard @@ -71,12 +72,17 @@ def annotations_for(self, record: Record) -> set[int]: class TimeFDataset(SupportsAnnotate): # ruff: ignore[too-many-public-methods] """Holds records and their tasks as Python objects. It does not do I/O. The writer handles persistence.""" - def __init__(self, *, metadata: DatasetMetadata) -> None: - """Create an empty dataset. + def __init__(self, *, metadata: DatasetMetadata, annotations: Iterable[Annotation] = ()) -> None: + """Create a dataset with no records and no tasks. Args: metadata: The dataset's descriptive identity. - """ + annotations: The annotations to attach to the dataset, in order, as :meth:`annotate` does. + + Raises: + TimeFValidationError: If ``annotations`` is a string, a mapping, or not iterable, or an + annotation has a time span. + """ # ruff: ignore[docstring-extraneous-exception] - raised by as_tuple and annotate self._metadata = metadata self._import_state = ImportState() """The parent cards, the lineage, and the import map of this dataset. :meth:`import_from` @@ -99,6 +105,8 @@ def __init__(self, *, metadata: DatasetMetadata) -> None: self._task_stream: Callable[[], Iterator[Task]] | None = None self._streamed_task_types: tuple[type[Task], ...] = () self._schema: DatasetSchema | None = None + for annotation in as_tuple(annotations): + self.annotate(annotation) def import_from( # ruff: ignore[too-many-arguments] - one argument per selection self, diff --git a/packages/timenet/src/timenet/dataset/edit.py b/packages/timenet/src/timenet/dataset/edit.py index 7967907ef..8fc138180 100644 --- a/packages/timenet/src/timenet/dataset/edit.py +++ b/packages/timenet/src/timenet/dataset/edit.py @@ -483,9 +483,9 @@ def _rebuild_tasks( inputs=tuple(record for record in task.inputs if record.id in records_by_id), input_annotations=tuple(copies[id(annotation)] for annotation in input_annotations), target_annotations=tuple(copies[id(annotation)] for annotation in task.target_annotations), - annotations=_copied(task.annotations, copies), from_tasks=(), ) + kept.annotations = _copied(task.annotations, copies) copy = kept.with_references(records_by_id, signals_by_id) copy._id = task.id rebuilt[task.id] = copy diff --git a/packages/timenet/src/timenet/dataset/record.py b/packages/timenet/src/timenet/dataset/record.py index 4a15a6bc0..743dda13a 100644 --- a/packages/timenet/src/timenet/dataset/record.py +++ b/packages/timenet/src/timenet/dataset/record.py @@ -36,7 +36,8 @@ class Record(SupportsAnnotate): start_time: TimeOrigin = field(default_factory=TimeOrigin) """The common relative zero, with an optional absolute timestamp.""" annotations: TupleField[Annotation] = field(default=ValueField(as_tuple, ())) - """Annotations attached to the record. Give any iterable. The record stores a tuple.""" + """Annotations attached to the record. Give any iterable. The record stores a tuple. It attaches a new + occurrence of each annotation, as :meth:`annotate` does.""" metadata: JsonMapping = field(default_factory=dict) """Optional JSON-compatible recording metadata.""" time_span: TimeInterval | None = None @@ -61,6 +62,7 @@ def __post_init__(self) -> None: self._check_times() for annotation in self.annotations: self._validate_annotation(annotation) + self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) def _check_times(self) -> None: """Check the time of each Signal and the ``time_span``, as construction does. diff --git a/packages/timenet/src/timenet/dataset/source.py b/packages/timenet/src/timenet/dataset/source.py index 57fa8a86a..1986d2840 100644 --- a/packages/timenet/src/timenet/dataset/source.py +++ b/packages/timenet/src/timenet/dataset/source.py @@ -55,7 +55,8 @@ class Source(SupportsAnnotate): signals: TupleField[Signal] = field(default=ValueField(as_tuple, ())) """Signals produced directly by this source.""" annotations: TupleField[Annotation] = field(default=ValueField(as_tuple, ())) - """Annotations attached to this source.""" + """Annotations attached to this source. The source attaches a new occurrence of each annotation, as + :meth:`annotate` does.""" metadata: JsonMapping = field(default_factory=dict) """Optional JSON-compatible source metadata.""" @@ -72,13 +73,15 @@ def __post_init__(self) -> None: self.metadata = validate_json_mapping(self.metadata) for annotation in self.annotations: - if annotation.span is not None: - check_span_within_window( - f"annotation {annotation.key!r}", - annotation.span, - tuple(self.walk_signals()), - f"source {self.name!r}", - ) + self._validate_annotation(annotation) + self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) + + def _validate_annotation(self, annotation: Annotation) -> None: + """Validate an annotation without changing this source.""" + if annotation.span is not None: + check_span_within_window( + f"annotation {annotation.key!r}", annotation.span, tuple(self.walk_signals()), f"source {self.name!r}" + ) def walk_sources(self) -> Iterator["Source"]: """Yield this source and all descendants in deterministic depth-first order. diff --git a/packages/timenet/src/timenet/dataset/time_series.py b/packages/timenet/src/timenet/dataset/time_series.py index 822d1ec00..4ba7fb199 100644 --- a/packages/timenet/src/timenet/dataset/time_series.py +++ b/packages/timenet/src/timenet/dataset/time_series.py @@ -256,7 +256,8 @@ class Signal(SupportsAnnotate): The writer counts missing timesteps and rejects a supplied count that differs from the values. """ annotations: tuple[Annotation, ...] = () - """Annotations attached directly to this signal.""" + """Annotations attached directly to this signal. The constructors attach a new occurrence of each + annotation, as :meth:`annotate` does.""" metadata: JsonMapping = field(default_factory=dict) """Optional JSON-compatible signal metadata.""" _id: EntityId | None = field(default=None, init=False, repr=False) @@ -349,6 +350,7 @@ def _initialize( # ruff: ignore[too-many-arguments] - shared eager and lazy fie set_field(self, "annotations", as_tuple(annotations)) set_field(self, "metadata", {} if metadata is None else metadata) self.__post_init__() + set_field(self, "annotations", tuple(annotation._new_occurrence() for annotation in self.annotations)) def annotate(self, annotation: Annotation) -> Annotation: """Attach one annotation and return it. diff --git a/packages/timenet/src/timenet/format/control_reader.py b/packages/timenet/src/timenet/format/control_reader.py index f18545149..a16e2c720 100644 --- a/packages/timenet/src/timenet/format/control_reader.py +++ b/packages/timenet/src/timenet/format/control_reader.py @@ -1109,9 +1109,12 @@ def _read_signals( null_count=row.null_count, loader=signal.values, time_offsets_loader=signal.time_offsets, - annotations=annotations.get((ObjectKind.SIGNAL, row.signal_id), ()), metadata=_decode_json(row.metadata, default={}), ) + stored = annotations.get((ObjectKind.SIGNAL, row.signal_id), ()) + for annotation in stored: + built._validate_annotation(annotation) + object.__setattr__(built, "annotations", stored) # ruff: ignore[unnecessary-dunder-call] (Signal is frozen) object.__setattr__(built, "_id", row.signal_id) # ruff: ignore[unnecessary-dunder-call] (Signal is frozen) signals_by_source[row.source_id].append(built) return signals_by_source @@ -1155,13 +1158,17 @@ def _assemble_records( def build_source(source_id: EntityId) -> Source: source_row = sources_by_id[source_id] - return Source( + source = Source( name=source_row.name, sources=tuple(build_source(child) for child in children[source_id]), signals=tuple(signals_by_source[source_id]), - annotations=annotations.get((ObjectKind.SOURCE, source_id), ()), metadata=_decode_json(source_row.metadata, default={}), ) + stored = annotations.get((ObjectKind.SOURCE, source_id), ()) + for annotation in stored: + source._validate_annotation(annotation) + source.annotations = stored + return source records: list[Record] = [] for record_row in hierarchy.records: @@ -1175,10 +1182,13 @@ def build_source(source_id: EntityId) -> Source: record = Record( sources=tuple(build_source(source_id) for source_id in hierarchy.roots_by_record[record_id]), start_time=self._clock_cache[record_row.clock_id], - annotations=annotations.get((ObjectKind.RECORD, record_id), ()), time_span=span, metadata=_decode_json(record_row.metadata, default={}), ) + stored = annotations.get((ObjectKind.RECORD, record_id), ()) + for annotation in stored: + record._validate_annotation(annotation) + record.annotations = stored record._id = record_id records.append(record) self._record_occurrences[id(record)] = { @@ -2475,7 +2485,7 @@ def _task(self, task_id: EntityId, built: _TasksById) -> Task: raise TimeFFormatError(f"task dependency refers to missing task {task_id!r}") return task - # ruff: ignore[too-many-locals] - one batch joins several tables + # ruff: ignore[too-many-locals, too-many-statements] - one batch joins several tables def _build(self, task_ids: list[EntityId] | None) -> _TasksById: """Read every table for one batch of task IDs and construct the tasks. @@ -2565,7 +2575,6 @@ def _build(self, task_ids: list[EntityId] | None) -> _TasksById: ), row.rationale, ), - "annotations": task_annotations.get((ObjectKind.TASK, row.task_id), ()), "metadata": _decode_json(row.metadata, default={}), **decode_task_payload(task_type, row.payload_columns), } @@ -2595,6 +2604,7 @@ def _build(self, task_ids: list[EntityId] | None) -> _TasksById: "annotation occurrence", ) task = cls(**kwargs) + task.annotations = task_annotations.get((ObjectKind.TASK, row.task_id), ()) task._id = row.task_id try: TimeFDataset._check_task_record_scope(task) diff --git a/packages/timenet/src/timenet/types/tasks.py b/packages/timenet/src/timenet/types/tasks.py index 899c0ac1e..9cdf76862 100644 --- a/packages/timenet/src/timenet/types/tasks.py +++ b/packages/timenet/src/timenet/types/tasks.py @@ -185,7 +185,8 @@ def age_question(record: Record, age: int) -> AnswerTask: from_tasks: TupleField["Task"] = field(default=ValueField(as_tuple, ()), compare=False) """Tasks from which this task was derived.""" annotations: TupleField[Annotation] = field(default=ValueField(as_tuple, ())) - """Annotations describing this task itself.""" + """Annotations describing this task itself. The task attaches a new occurrence of each annotation, as + :meth:`annotate` does.""" metadata: JsonMapping = field(default_factory=dict) """Optional JSON-compatible task metadata.""" _id: EntityId | None = field(default=None, init=False, repr=False, compare=False) @@ -223,6 +224,7 @@ def __post_init__(self) -> None: for target in self.targets or (): if not _is_target_item(target): raise TimeFValidationError(f"{type(self).__name__} target has unsupported type {type(target).__name__}") + self.annotations = tuple(annotation._new_occurrence() for annotation in self.annotations) @property def id(self) -> EntityId: @@ -333,7 +335,9 @@ def with_references(self, records: Mapping[EntityId, "Record"], signals: Mapping TimeFValidationError: If a Record or Signal that the task references has no ID, or if ``records`` or ``signals`` has no object for its ID. """ # ruff: ignore[docstring-extraneous-exception] (raised by _by_id and assigned_id) - return replace(self, **self._references_by_id(records, signals)) + copy = replace(self, **self._references_by_id(records, signals)) + copy.annotations = self.annotations + return copy def _references_by_id( self, records: Mapping[EntityId, "Record"], signals: Mapping[EntityId, "Signal"] diff --git a/packages/timenet/tests/dataset/test_dataset.py b/packages/timenet/tests/dataset/test_dataset.py index c34792922..8d450a01a 100644 --- a/packages/timenet/tests/dataset/test_dataset.py +++ b/packages/timenet/tests/dataset/test_dataset.py @@ -55,6 +55,25 @@ def _dataset(): return TimeFDataset(metadata=_metadata()) +def test_the_constructor_attaches_annotations_in_order_as_annotate_does(): + site = Annotation("site", value="lab") + dataset = TimeFDataset(metadata=_metadata(), annotations=[site, Annotation("revision", value=3)]) + assert [(one.key, one.value) for one in dataset.annotations] == [("site", "lab"), ("revision", 3)] + assert dataset.annotations[0] is not site + assert _dataset().annotations == () + + +@pytest.mark.parametrize("annotations", ["ab", {"a": 1}, Annotation("site", value="lab")]) +def test_the_constructor_rejects_annotations_that_are_not_an_iterable_of_items(annotations): + with pytest.raises(TimeFValidationError, match="takes an iterable"): + TimeFDataset(metadata=_metadata(), annotations=annotations) + + +def test_the_constructor_rejects_an_annotation_with_a_time_span(): + with pytest.raises(TimeFValidationError, match="cannot have a time span"): + TimeFDataset(metadata=_metadata(), annotations=[Annotation("mark", span=TimePoint.seconds(1.0))]) + + def test_add_record_registers_and_returns(make_series): ds = _dataset() record = Record(sources=(Source(name="Source", signals=(make_series(),)),)) diff --git a/packages/timenet/tests/reader/test_reader.py b/packages/timenet/tests/reader/test_reader.py index 1d780e15d..a55855105 100644 --- a/packages/timenet/tests/reader/test_reader.py +++ b/packages/timenet/tests/reader/test_reader.py @@ -302,6 +302,64 @@ def test_a_read_back_annotation_equals_the_constant_it_was_written_from(tmp_path assert stored == (1, 2) +def _shared_annotation_dataset(shared: Annotation) -> TimeFDataset: + dataset = _referenced_annotation_dataset(with_task=False) + spec = TimeSeriesSpec(spec_type="ecg", name="lead", unit_value=ureg.millivolt) + axis = RegularAxis.from_rate_hz(Fraction(500)) + for name in ("first", "second"): + eager = Signal(spec=spec, name="I", time_axis=axis, data=[0.0, 1.0], annotations=[shared]) + lazy = Signal.from_values([2.0, 3.0], spec=spec, name="II", time_axis=axis) + lazy = Signal.from_loader( + spec=spec, + name="III", + time_axis=axis, + loader=lazy.to_arrow, + n_values=2, + annotations=[shared], + ) + source = Source(name, signals=[eager, lazy], annotations=[shared]) + record = dataset.add_record(Record(sources=[source], annotations=[shared])) + dataset.add_task( + AnswerTask( + prompt="Condition?", + targets=("healthy",), + inputs=(record,), + input_annotations=(record.annotations[0], eager.annotations[0]), + annotations=[shared], + ) + ) + return dataset + + +@_PLANS +def test_owners_built_with_one_annotation_object_each_attach_a_new_occurrence(tmp_path, load): + shared = Annotation("condition", value="healthy") + dataset = _shared_annotation_dataset(shared) + owners = [ + *dataset.tasks, + *(owner for record in dataset.records[1:] for owner in (record, *record.walk_sources(), *record.signals)), + ] + attached = [annotation for owner in owners for annotation in owner.annotations] + assert all(annotation == shared and annotation is not shared for annotation in attached) + assert len({id(annotation) for annotation in attached}) == len(attached) + + version_dir = _write(tmp_path, dataset=dataset) + restored = _read(version_dir, load) + + assert_datasets_equal(dataset, restored) + for task in restored.tasks: + (record,) = task.inputs + assert task.input_annotations[0] is record.annotations[0] + assert task.input_annotations[1] is record.signals[0].annotations[0] + with connect_control(version_dir / "control.duckdb", read_only=True) as connection: + stored = connection.execute( + """SELECT count(DISTINCT content_id), count(*) + FROM annotation_occurrences JOIN annotation_contents USING (content_id) + WHERE name = 'condition'""" + ).fetchone() + assert stored == (1, 10) + + def _tasks_dataset(*, streaming: bool) -> TimeFDataset: dataset = _referenced_annotation_dataset(with_task=False) record = dataset.records[0] diff --git a/packages/timenet/tests/types/test_tasks.py b/packages/timenet/tests/types/test_tasks.py index 95bcb7b42..b21f56147 100644 --- a/packages/timenet/tests/types/test_tasks.py +++ b/packages/timenet/tests/types/test_tasks.py @@ -157,6 +157,26 @@ def test_a_task_references_its_signal_targets_then_the_signals_of_its_spans_each assert AnswerTask(targets=("a",), scope=TimeInterval.seconds(0.0, 1.0)).referenced_signals() == () +def test_a_task_attaches_a_new_occurrence_of_its_own_annotations_and_keeps_its_references(): + (record,) = _registered("a") + context = record.annotate(Annotation("context", value="ctx")) + quality = Annotation("quality", value="good") + + task = AnswerTask( + inputs=(record,), + targets=("a",), + input_annotations=(context,), + target_annotations=(context,), + annotations=[quality], + ) + + (own,) = task.annotations + assert own == quality + assert own is not quality + assert task.input_annotations[0] is context + assert task.target_annotations[0] is context + + def test_a_copy_with_references_points_at_the_records_and_signals_of_the_same_ids(): record, target, holder = _registered("a", "b", "c") records, signals = _replacements((record, target, holder))