From 378532a3f446f76784f1a7ecc1359d0d4deba78d Mon Sep 17 00:00:00 2001 From: Othman Sbai Date: Sat, 7 Nov 2026 00:22:32 +0100 Subject: [PATCH] feat(connectors): add EngineMT-QA with shared source validation and loading Port the pandalin98/enginemt-qa snapshot from 0f448be2 without unrelated repository changes. Preserve the published question stages, label sets, split auditing, 33-channel signals and repeatable lazy task stream. Use Pydantic SourceRow models, connector error conventions, the core bounded shared channel loader and numerical signal validation. Adopt assert_round_trips and verify lazy channel sharing and eviction with focused tests; 48 connector tests pass. --- .../datasets/pandalin98/__init__.py | 4 + .../datasets/pandalin98/enginemt_qa/README.md | 93 ++++ .../pandalin98/enginemt_qa/__init__.py | 6 + .../pandalin98/enginemt_qa/connector.py | 91 ++++ .../pandalin98/enginemt_qa/dataset.yaml | 35 ++ .../pandalin98/enginemt_qa/records.py | 41 ++ .../pandalin98/enginemt_qa/requirements.txt | 4 + .../datasets/pandalin98/enginemt_qa/source.py | 414 ++++++++++++++++ .../datasets/pandalin98/enginemt_qa/tasks.py | 64 +++ .../pandalin98/enginemt_qa/tests/README.md | 23 + .../enginemt_qa/tests/fixtures/test_qa.jsonl | 3 + .../enginemt_qa/tests/fixtures/train_qa.jsonl | 4 + .../enginemt_qa/tests/test_connector.py | 450 ++++++++++++++++++ .../pandalin98/enginemt_qa/vocabulary.py | 350 ++++++++++++++ 14 files changed, 1582 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/source.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/vocabulary.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py new file mode 100644 index 000000000..2c96ddf0d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py @@ -0,0 +1,4 @@ +"""Connectors for datasets released by Yilin Wang (``pandalin98``) and co-authors. + +Keep this regular package so pkgutil.walk_packages can discover its connector children. +""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md new file mode 100644 index 000000000..81b5117e3 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md @@ -0,0 +1,93 @@ +# EngineMT-QA + +`pandalin98/enginemt-qa` converts questions about simulated aero-engine signals from the +[ITFormer paper][paper] and [authors' code][code]. The [Hugging Face release][card] is pinned to +`ea2051a15512212976c22c1a075ec549c7d48eef`. + +The Hub metadata declares MIT, but its license section is an unfilled placeholder and the paper +states no dataset license. The local card follows that metadata; confirmation from the authors +remains outstanding. + +## Source and records + +| File | Content | +| --- | --- | +| `time_series_data.h5` | `seq_data`: float64, shape `(118921, 600, 33)`; `data_ID`: IDs 1–118,921 | +| `train_qa.jsonl` | 13,291 lines, 101,565 question-answer pairs | +| `test_qa.jsonl` | 5,539 lines, 42,477 question-answer pairs | + +The connector creates one Record for each row referenced by a labelled pair: 7,755 rows in the +pinned release. Tasks referring to the same row share its Record. Unreferenced rows, which the +authors also use for encoder pretraining, are not included. + +Each Record contains 33 float64 signals named `channel_00` through `channel_32`, with 600 values +and unknown units. Values are unchanged. The 1 Hz regular axis follows the paper's description of +600-point, ten-minute segments; the HDF5 file itself states no rate. Record metadata retains the +published `row_id`. + +The release supplies no channel names, engine identity, flight identity or scaling parameters. +The [authors' channel audit][audit] does not establish which of the paper's 32 channels corresponds +to each of the 33 published columns. The connector therefore retains all 33 without assigning +physical names. + +## Tasks and splits + +Each labelled pair becomes one task, in source order. Train and test follow the two QA files. + +| Stage | TimeNet task | Inputs | Target | +| --- | --- | --- | --- | +| Understanding | `AnswerTask` | 1 Record | Published answer | +| Perception | `ClassificationTask` | 1 Record | Published option letters | +| Reasoning | `ClassificationTask` | 10 Records | Published option letter | +| Decision-Making | `AnswerTask` | 10 Records | Published answer | + +Closed-question label sets preserve the printed options. Fault-cause questions allow several +letters; other closed questions require one. Published question text supplies the prompt. + +The authors take the first 60 steps of each of ten Records for multi-cycle questions. Tasks keep +the complete Records and store `steps_per_cycle: 60`; consumers reproducing that input must apply +this truncation themselves. The selected IDs are not necessarily consecutive cycles. + +Task metadata retains `stage`, the closed-question name where applicable, `source_file`, +`source_line`, `qa_offset`, and `source_issues`. `source_line` and `qa_offset` are zero-based to +match the authors' evaluation; error messages use one-based physical file lines. + +## Missing labels and source overlap + +The connector omits tasks for 1,769 empty closed answers and records their locations in the +`unlabelled_pairs` dataset annotation. An empty fault-cause answer is not interpreted as “no +fault.” The authors trained on these pairs, so this is a deliberate conversion difference. + +Duplicate pairs remain in the dataset, with flags in `source_issues`: + +| Flag | Meaning | +| --- | --- | +| `repeated_across_splits` | Both files contain the same rows, question and answer | +| `repeated_question` | The same file repeats those rows, question and answer | + +In the pinned QA files, 27,955 of 41,977 labelled test tasks also appear in train. Scores on this +split therefore include substantial overlap. Consumers evaluating unseen pairs can exclude the +flagged test tasks and should report that choice. + +Other limitations remain: component-health and fault-cause answers can disagree, generated open +answers vary in wording, and published counts and channel descriptions differ between the files, +card and paper. The connector follows the pinned files and does not repair their answers. + +## Building and verification + +```bash +uv run timenet-build build pandalin98/enginemt-qa --out .timenet-registry +``` + +The download includes an 18.8 GB signals file; the referenced rows contain about 1.2 GB of values. +Keep source files available until writing finishes. Conversion checks the HDF5 structure, QA +schema, option vocabulary and pinned QA inventory. Task rereads verify the QA line fingerprints. +Invalid sources raise `TimeNetConnectorError` with context. + +The QA parser has been run on the pinned QA files and the HDF5 header inspected. A complete build +and an end-to-end scan of the signals file have not been verified. Tests use small fixtures. + +[paper]: https://arxiv.org/abs/2506.20093 +[code]: https://github.com/Pandalin98/ITFormer-ICML25 +[card]: https://huggingface.co/datasets/pandalin98/EngineMT-QA +[audit]: https://github.com/Pandalin98/ITFormer-ICML25/blob/b80142f/docs/ISSUE_23_CHANNEL_AUDIT.md diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py new file mode 100644 index 000000000..0f217e8c8 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py @@ -0,0 +1,6 @@ +"""EngineMT-QA: aero-engine question answering over simulated N-CMAPSS cycles, in four tasks.""" + +from timenet_connectors.datasets.pandalin98.enginemt_qa.connector import ( + CONNECTOR as CONNECTOR, + EngineMTQAConnector as EngineMTQAConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py new file mode 100644 index 000000000..c46cf1194 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py @@ -0,0 +1,91 @@ +"""Orchestrate EngineMT-QA source validation, records and repeatable tasks.""" + +from collections.abc import Mapping +from typing import ClassVar + +from timenet.dataset import TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ( + AnswerTask, + ClassificationTask, +) +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.datasets.pandalin98.enginemt_qa.records import build_record, unlabelled_annotation +from timenet_connectors.datasets.pandalin98.enginemt_qa.source import ( + FileCounts, + QAFile, + make_channel_loader, + read_recording_count, + scan_questions, +) +from timenet_connectors.datasets.pandalin98.enginemt_qa.tasks import iter_tasks +from timenet_connectors.datasets.pandalin98.enginemt_qa.vocabulary import QA_PARTS, ClosedQuestion, Part, Stage + + +class EngineMTQAConnector(BaseHuggingFaceConnector): + """Build EngineMT-QA from the pinned files of its Hugging Face release.""" + + HF_REPO = "pandalin98/EngineMT-QA" + SOURCE_REVISION = "ea2051a15512212976c22c1a075ec549c7d48eef" + SOURCE_FILES: ClassVar[Mapping[str, str]] = {part.value: part.source_file for part in Part} + # Counted on the pinned files. They match the authors' README: 101,565 train and 42,477 test pairs. + SOURCE_COUNTS: ClassVar[Mapping[Part, FileCounts]] = { + Part.TRAIN: FileCounts( + lines=13_291, + pairs={ + Stage.UNDERSTANDING: 41_128, + Stage.PERCEPTION: 35_987, + Stage.REASONING: 16_300, + Stage.DECISION_MAKING: 8_150, + }, + unlabelled={ClosedQuestion.FAULT_CAUSE: 1_054, ClosedQuestion.HEALTH_STATE: 215}, + ), + Part.TEST: FileCounts( + lines=5_539, + pairs={ + Stage.UNDERSTANDING: 17_240, + Stage.PERCEPTION: 15_085, + Stage.REASONING: 6_768, + Stage.DECISION_MAKING: 3_384, + }, + unlabelled={ClosedQuestion.FAULT_CAUSE: 422, ClosedQuestion.HEALTH_STATE: 78}, + ), + } + + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Convert the files into one lazy Record per named row, and a stream of one task per labelled pair. + + Conversion checks the signals file's layout and every QA line once, and compares each QA file's + counts with the pinned ones. The values stay in the HDF5 file until written, so the files must stay + available until then. A closed pair with an empty answer is a missing label and gets no task. + + Args: + raw_refs: The downloaded files, one per part. + + Returns: + A dataset with the Records, the unlabelled pairs as an annotation, and a stream of every task. + + Raises: + TimeNetConnectorError: If a part is missing, or a file is malformed or has other counts. + TimeNetConnectorError: If a file cannot be read. + """ + paths = {Part(ref.part): ref.path for ref in raw_refs} + if paths.keys() != set(Part): + raise TimeNetConnectorError(f"EngineMT-QA needs the parts {[part.value for part in Part]}") + signals_path = paths[Part.SIGNALS] + n_rows = read_recording_count(signals_path) + qa_files = [QAFile(paths[part], part, n_rows) for part in QA_PARTS] + scan = scan_questions(qa_files, self.SOURCE_COUNTS) + reader = make_channel_loader(signals_path) + records = {row_id: build_record(row_id, reader) for row_id in sorted(scan.named_rows)} + + dataset = TimeFDataset(metadata=self.metadata()) + dataset.annotate(unlabelled_annotation(scan.unlabelled)) + for record in records.values(): + dataset.add_record(record=record) + + dataset.set_task_stream((AnswerTask, ClassificationTask), lambda: iter_tasks(qa_files, records, scan)) + return dataset + + +CONNECTOR = EngineMTQAConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml new file mode 100644 index 000000000..37caae758 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# Dataset Card for the pandalin98/enginemt-qa connector: the single source for this dataset's metadata. +# +# Training convention: the tasks are the authors' four stages, one task per published QA pair, split as +# the authors train and test (train_qa.jsonl, test_qa.jsonl). Closed questions are classifications over +# their option letters, as the authors' model answers and their metric scores. 1,769 closed pairs with an +# empty answer have no task. Signals are the release's z-scored values at 1 Hz, 33 unnamed channels per +# 600-step cycle. Tasks that the other split repeats name repeated_across_splits in their +# source_issues metadata. +yaml_schema_version: 1 +dataset_id: pandalin98/enginemt-qa +dataset_version: 1.0.0 +name: EngineMT-QA +description: >- + Aero-engine time-series question answering over simulated N-CMAPSS flight cycles: 142,273 labelled + QA pairs (100,296 train, 41,977 test) in four tasks. Understanding and Perception ask about one + 600-step cycle of 33 z-scored channels; Reasoning and Decision-Making ask about 10 cycles. + Perception and Reasoning are multiple choice; Understanding and Decision-Making are open-ended + answers written with LLMs and checked by experts. +license: MIT +source_url: https://huggingface.co/datasets/pandalin98/EngineMT-QA +source_revision: ea2051a15512212976c22c1a075ec549c7d48eef +citation: >- + Wang et al. ITFormer: Bridging Time Series and Natural Language for Multi-Modal QA with + Large-Scale Multitask Dataset. ICML 2025. https://arxiv.org/abs/2506.20093 +domains: + - industrial +tags: + - aero-engine + - n-cmapss + - simulated + - question-answering + - classification + - generated-supervision + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py new file mode 100644 index 000000000..44b60c2a6 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/records.py @@ -0,0 +1,41 @@ +"""Build EngineMT-QA records and dataset provenance from checked source facts.""" + +from collections.abc import Sequence + +from timenet.dataset import Record, Signal, SignalChannel, Source +from timenet.json import JsonValue +from timenet.types import ( + Annotation, +) +from timenet.types.shared_loading import SharedChannelLoader +from timenet_connectors.datasets.pandalin98.enginemt_qa.source import PairLocation +from timenet_connectors.datasets.pandalin98.enginemt_qa.vocabulary import ( + AXIS, + CHANNELS, + N_STEPS, + SOURCE_NAME, + SPEC, + AnnotationKey, + MetadataKey, +) + + +def build_record(row_id: int, reader: SharedChannelLoader[int]) -> Record: + """Return a row's Record with 33 lazy signals and its published row ID.""" + signals = Signal.from_shared_loader( + loader=reader, + recording_key=row_id, + channels=tuple(SignalChannel(name=name, spec=SPEC, time_axis=AXIS, n_values=N_STEPS) for name in CHANNELS), + ) + return Record(sources=(Source(name=SOURCE_NAME, signals=signals),), metadata={MetadataKey.ROW_ID: row_id}) + + +def unlabelled_annotation(places: Sequence[PairLocation]) -> Annotation: + """Return the source locations of published pairs that have no answer label.""" + pairs: list[JsonValue] = [place.metadata for place in places] + return Annotation( + key=AnnotationKey.UNLABELLED_PAIRS, + value=len(places), + description="How many closed pairs have an empty published answer. They are missing labels and have no task.", + metadata={"pairs": pairs}, + ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt new file mode 100644 index 000000000..ef2960923 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt @@ -0,0 +1,4 @@ +# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class. +huggingface_hub>=1.1 +# h5py reads the signals file, time_series_data.h5. +h5py>=3.10 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/source.py new file mode 100644 index 000000000..22c39133a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/source.py @@ -0,0 +1,414 @@ +"""Read and validate EngineMT-QA files and retain checked source facts.""" + +from collections import Counter +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass, field +import hashlib +import json +import logging +from pathlib import Path +from typing import Any, Literal, NamedTuple + +import numpy as np +import pyarrow as pa +from pydantic import Field, field_validator +from pydantic_core import PydanticCustomError + +from timenet.errors import TimeNetConnectorError +from timenet.json import JsonValue +from timenet.types.shared_loading import SharedChannelLoader +from timenet_connectors.datasets.pandalin98.enginemt_qa.vocabulary import ( + CHANNELS, + DATA_ID, + DIGEST_BYTES, + MULTI_CYCLE_ROWS, + N_STEPS, + QA_PARTS, + ROW_ID_PATTERN, + ROW_SHAPE, + SEQ_DATA, + SPEC, + Attribute, + ClosedQuestion, + MetadataKey, + Part, + SourceIssue, + Stage, +) +from timenet_connectors.sources.jsonl import iter_jsonl_rows +from timenet_connectors.sources.rows import NonBlankText, SourceRow, validate_row +from timenet_connectors.sources.signals import numeric_signal + + +_LOGGER = logging.getLogger(__name__) + + +class FileCounts(NamedTuple): + """The pinned size of a QA file, which conversion checks so that a short or changed file fails.""" + + lines: int + pairs: Mapping[Stage, int] + """The pairs of each stage, empty answers included.""" + unlabelled: Mapping[ClosedQuestion, int] + """The closed pairs with an empty answer, for each question that has any.""" + + +class PairLocation(NamedTuple): + """Where a pair was published.""" + + part: Part + line_index: int + """The 0-based line in the file, which the authors call ``source_line``.""" + pair_index: int + """The 0-based place in the line, which the authors call ``qa_offset``.""" + + @property + def metadata(self) -> dict[str, JsonValue]: + """The place as the authors' evaluation names it.""" + return { + MetadataKey.SOURCE_FILE: self.part.source_file, + MetadataKey.SOURCE_LINE: self.line_index, + MetadataKey.QA_OFFSET: self.pair_index, + } + + +@dataclass(frozen=True) +class Pair: + """One published question with its answer, and the rows it is about.""" + + place: PairLocation + rows: tuple[int, ...] + """The row ids in published order: one for a single-cycle question and ten for a multi-cycle one.""" + stage: Stage + question: str + """The question as published, with its ```` marker.""" + answer: str + """The answer as published: free text, or letters for a closed question.""" + closed: ClosedQuestion | None + """The closed question, or ``None`` for an open one.""" + + @property + def missing_label(self) -> ClosedQuestion | None: + """The closed question, if its published answer is empty. Such a pair is a missing label and has no task.""" + return None if self.answer else self.closed + + @property + def has_label(self) -> bool: + """Whether the pair has a task.""" + return self.missing_label is None + + @property + def digest(self) -> bytes: + """A digest of the rows, the question and the answer, which match for a repeated pair.""" + content = json.dumps([self.rows, self.question, self.answer]).encode() + return hashlib.blake2b(content, digest_size=DIGEST_BYTES).digest() + + +class _Line(SourceRow): + """One published JSON line, before source-specific ID and conversation checks.""" + + id: str | list[str] + name: str | list[str] + conversations: list[dict[str, object]] + + +class _Question(SourceRow): + """A published question turn with its verbatim text.""" + + role: Literal["human"] = Field(alias="from") + stage: Stage + attribute: Attribute = Field(strict=False) + value: NonBlankText + + @field_validator("stage", mode="before") + @classmethod + def _published_stage(cls, value: object) -> Stage: + if not isinstance(value, str) or value not in {str(stage.value) for stage in Stage}: + raise PydanticCustomError("stage", "expected a stage from '1' to '4'") + return Stage(int(value)) + + +class _Answer(SourceRow): + """An answer turn; blank text represents a published missing label.""" + + role: Literal["gpt"] = Field(alias="from") + value: str + + +def _as_list(value: object) -> list[object]: + return value if isinstance(value, list) else [value] + + +def _row_ids(line: _Line, n_rows: int) -> tuple[int, ...]: + """Read the row ids of a line, and check that its ``name`` repeats them. + + Returns: + One id for a single-cycle line, ten for a multi-cycle line, in published order. + + Raises: + TimeNetConnectorError: If an id is not a row of the file, or the ids repeat or are not one or ten. + """ + raw_ids, raw_names = line.id, line.name + is_list = isinstance(raw_ids, list) + ids = _as_list(raw_ids) + names_match = isinstance(raw_names, list) == is_list and _as_list(raw_names) == [f"{i}.csv" for i in ids] + if not names_match: + raise TimeNetConnectorError(f"name {raw_names!r} doesn't repeat id {raw_ids!r}") + rows: list[int] = [] + for i in ids: + if not isinstance(i, str) or not ROW_ID_PATTERN.fullmatch(i): + raise TimeNetConnectorError(f"id {raw_ids!r} is not decimal row ids") + rows.append(int(i)) + if is_list and len(rows) != MULTI_CYCLE_ROWS: + raise TimeNetConnectorError(f"a multi-cycle id names {len(rows)} rows, not {MULTI_CYCLE_ROWS}") + if len(set(rows)) != len(rows): + raise TimeNetConnectorError(f"id {raw_ids!r} repeats a row") + if max(rows) > n_rows: + raise TimeNetConnectorError(f"id {raw_ids!r} names a row past the file's {n_rows} rows") + return tuple(rows) + + +def _parse_pair(place: PairLocation, rows: tuple[int, ...], question_turn: object, answer_turn: object) -> Pair: + """Check one question and its answer against its stage, and build the pair. + + Returns: + The pair. + + Raises: + TimeNetConnectorError: If the stage, the format and the number of rows disagree, a closed question is unknown, or + an answer is malformed. + """ + question, answer = ( + validate_row(question_turn, _Question, location="question"), + validate_row(answer_turn, _Answer, location="answer"), + ) + stage, attribute = question.stage, question.attribute + if (attribute is Attribute.CLOSE) != stage.is_closed: + raise TimeNetConnectorError(f"stage {stage} questions are not {attribute.value!r}") + if (len(rows) > 1) != stage.is_multi_cycle: + raise TimeNetConnectorError(f"a stage {stage} question names {len(rows)} rows") + text, reply = question.value, answer.value + if text.count("") != 1: + raise TimeNetConnectorError("a question must hold one marker") + closed = ClosedQuestion.of(stage, text) if stage.is_closed else None + if closed is None and not reply.strip(): + raise TimeNetConnectorError("an open question has an empty answer") + if closed is not None and reply: + # Parse here, so a malformed answer fails the read and not the task stream. + closed.parse_answer(reply) + return Pair(place, rows, stage, text, reply, closed) + + +@dataclass +class QAFile: + """One QA file. It is read twice, and the second read must see the file the first one did.""" + + path: Path + part: Part + n_rows: int + line_digests: list[bytes] | None = field(default=None, init=False) + """A digest of each line, kept by the first complete read.""" + + def pairs(self) -> Iterator[Pair]: + """Read every line of the file and flatten it into pairs, in file order. + + The first complete read keeps a digest of each line. A later read checks each line against its + digest before it builds pairs. + + Yields: + Each published pair, empty answers included. + + Raises: + TimeNetConnectorError: If a line or a pair is malformed, with its file, line and pair. + TimeNetConnectorError: If the file cannot be read, or changed since the first complete read. + """ + expected = self.line_digests + digests: list[bytes] = [] + try: + for located in iter_jsonl_rows(self.path, _Line): + number = located.location.line - 1 + digest = hashlib.blake2b(located.raw, digest_size=DIGEST_BYTES).digest() + if expected is not None and (number >= len(expected) or digest != expected[number]): + raise TimeNetConnectorError(f"{located.location.context}: the line changed since conversion") + digests.append(digest) + yield from self._line_pairs(number, located.value) + except TimeNetConnectorError as exc: + if expected is not None: + raise TimeNetConnectorError(f"{exc}; the file changed since conversion") from exc + raise + if expected is None: + self.line_digests = digests + elif len(digests) != len(expected): + raise TimeNetConnectorError(f"{self.path}: lines were removed since conversion") + + def _line_pairs(self, number: int, line: _Line) -> Iterator[Pair]: + where = f"{self.path} (line {number + 1})" + try: + rows = _row_ids(line, self.n_rows) + turns = line.conversations + if not turns or len(turns) % 2: + raise TimeNetConnectorError("conversations must alternate a human question and a gpt answer") + for offset in range(len(turns) // 2): + where = f"{self.path} (line {number + 1}), pair {offset}" + place = PairLocation(self.part, number, offset) + yield _parse_pair(place, rows, turns[2 * offset], turns[2 * offset + 1]) + except (ValueError, TimeNetConnectorError) as exc: + raise TimeNetConnectorError(f"{where}: {exc}") from exc + + +def _import_h5py() -> Any: + """Import ``h5py`` when a file is read, so the connector can load without it. + + Returns: + The ``h5py`` module. + + Raises: + ImportError: If ``h5py``, declared in this connector's requirements, is not installed. + """ + try: + import h5py # ruff: ignore[import-outside-top-level] + except ImportError as exc: + raise ImportError("reading EngineMT-QA needs h5py, declared in this connector's requirements.txt") from exc + return h5py + + +def _open_signals(path: Path) -> Any: + """Open the signals file for reading. + + Returns: + The open ``h5py.File``, to use as a context manager. + + Raises: + TimeNetConnectorError: If the file cannot be opened as HDF5, as with a truncated download. + """ + h5py = _import_h5py() + try: + return h5py.File(path, "r") + except OSError as exc: + raise TimeNetConnectorError(f"{path}: cannot open the signals file: {exc}") from exc + + +def read_recording_count(path: Path) -> int: + """Check that the signals file holds exactly ``seq_data`` and ``data_ID``, with the ids ``1..n``. + + Returns: + The number of rows ``n``. + + Raises: + TimeNetConnectorError: If the layout, the shape, the dtype or the ids differ from the published file's. + TimeNetConnectorError: If the file cannot be opened or read as HDF5. + """ + h5py = _import_h5py() + with _open_signals(path) as file: + if set(file) != {SEQ_DATA, DATA_ID}: + raise TimeNetConnectorError(f"{path}: expected {SEQ_DATA} and {DATA_ID}, got {sorted(file)}") + values, ids = file[SEQ_DATA], file[DATA_ID] + for name, node in ((SEQ_DATA, values), (DATA_ID, ids)): + if not isinstance(node, h5py.Dataset): + raise TimeNetConnectorError(f"{path}: {name} is a {type(node).__name__}, not an HDF5 dataset") + if values.shape[1:] != ROW_SHAPE or values.dtype != np.float64: + raise TimeNetConnectorError(f"{path}: {SEQ_DATA} is {values.dtype} {values.shape}") + n_rows = values.shape[0] + if ids.shape != (n_rows,) or not np.issubdtype(ids.dtype, np.integer): + raise TimeNetConnectorError(f"{path}: {DATA_ID} is {ids.dtype} {ids.shape}, not {n_rows} integer ids") + try: + published_ids = ids[:] + except OSError as exc: + raise TimeNetConnectorError(f"{path}: cannot read {DATA_ID}: {exc}") from exc + if not np.array_equal(published_ids, np.arange(1, n_rows + 1)): + raise TimeNetConnectorError(f"{path}: {DATA_ID} is not the row ids 1 to {n_rows}") + return n_rows + + +def _read_row(file: Any, path: Path, row_id: int) -> np.ndarray: + """Read one row's values from an open signals file. + + Returns: + The values, shaped ``(600, 33)``: steps, then channels. + + Raises: + TimeNetConnectorError: If the file no longer holds the row, as when it changed since conversion, or a + block of the file cannot be read. + TimeNetConnectorError: If the row holds a NaN or an infinite value. z-scored values are finite. + """ + try: + values = np.asarray(file[SEQ_DATA][row_id - 1], dtype=np.float64) + except (KeyError, IndexError, ValueError, OSError) as exc: + raise TimeNetConnectorError(f"{path}: cannot read row id {row_id}: {exc}") from exc + if values.shape != ROW_SHAPE: + raise TimeNetConnectorError(f"{path}: row id {row_id} is shaped {values.shape}; the file changed") + if not np.isfinite(values).all(): + raise TimeNetConnectorError(f"{path}: row id {row_id} holds a NaN or an infinite value") + return values + + +def make_channel_loader(path: Path) -> SharedChannelLoader[int]: + """Return a conversion-wide bounded cache of decoded HDF5 rows.""" + + def read_channels(row_id: int) -> tuple[pa.Array, ...]: + with _open_signals(path) as file: + values = _read_row(file, path, row_id) + return tuple( + numeric_signal(values[:, channel], f"{path}: row {row_id}, {name}", spec=SPEC, n_values=N_STEPS) + for channel, name in enumerate(CHANNELS) + ) + + return SharedChannelLoader(read_channels) + + +@dataclass +class QAInventory: + """What one read of every QA pair finds.""" + + named_rows: set[int] = field(default_factory=set) + """The rows that the pairs with a label name.""" + unlabelled: list[PairLocation] = field(default_factory=list) + """The places of the closed pairs with an empty answer.""" + digests: dict[Part, Counter[bytes]] = field(default_factory=dict) + """How often each labelled pair's digest occurs in each file.""" + + def issues(self, pair: Pair) -> list[SourceIssue]: + """Name the source issues of a labelled pair. + + Returns: + The issues, in ``SourceIssue`` order. + """ + digest = pair.digest + found = [] + if self.digests[pair.place.part][digest] > 1: + found.append(SourceIssue.REPEATED_QUESTION) + if any(self.digests[part][digest] for part in QA_PARTS if part is not pair.place.part): + found.append(SourceIssue.REPEATED_ACROSS_SPLITS) + return found + + +def scan_questions(qa_files: Sequence[QAFile], counts: Mapping[Part, FileCounts]) -> QAInventory: + """Check every pair once, and collect what the dataset needs from all of them. + + Returns: + The rows that labelled pairs name, the unlabelled pairs, and the digests of the labelled pairs. + + Raises: + TimeNetConnectorError: If a file's counts differ from the pinned ones. + """ + scan = QAInventory() + for qa in qa_files: + digests: Counter[bytes] = Counter() + pairs: Counter[Stage] = Counter() + unlabelled: Counter[ClosedQuestion] = Counter() + for pair in qa.pairs(): + pairs[pair.stage] += 1 + question = pair.missing_label + if question is None: + scan.named_rows.update(pair.rows) + digests[pair.digest] += 1 + else: + scan.unlabelled.append(pair.place) + unlabelled[question] += 1 + found = FileCounts(len(qa.line_digests or ()), dict(pairs), dict(unlabelled)) + if found != counts[qa.part]: + raise TimeNetConnectorError(f"{qa.path}: expected {counts[qa.part]}, found {found}") + scan.digests[qa.part] = digests + if scan.unlabelled: + _LOGGER.warning("%d closed pairs have an empty answer and get no task", len(scan.unlabelled)) + return scan diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py new file mode 100644 index 000000000..a00216306 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tasks.py @@ -0,0 +1,64 @@ +"""Build repeatable EngineMT-QA tasks from checked pairs and records.""" + +from collections.abc import Iterator, Mapping, Sequence + +from timenet.dataset import Record +from timenet.errors import TimeNetConnectorError +from timenet.json import JsonValue +from timenet.types import ( + AnswerTask, + ClassificationTask, + Task, +) +from timenet_connectors.datasets.pandalin98.enginemt_qa.source import Pair, QAFile, QAInventory +from timenet_connectors.datasets.pandalin98.enginemt_qa.vocabulary import STEPS_PER_CYCLE, MetadataKey, SourceIssue + + +def _inputs(pair: Pair, records: Mapping[int, Record]) -> tuple[Record, ...]: + """Return the Records of the pair's rows, in published order. + + Raises: + TimeNetConnectorError: If a row has no Record, as when a QA file changed since conversion. + """ + missing = [row for row in pair.rows if row not in records] + if missing: + raise TimeNetConnectorError( + f"{pair.place.part.source_file} line {pair.place.line_index + 1}: no Record for rows {missing}" + ) + return tuple(records[row] for row in pair.rows) + + +def _task(pair: Pair, inputs: tuple[Record, ...], issues: Sequence[SourceIssue]) -> AnswerTask | ClassificationTask: + """Build the task of a pair that has a label. + + Args: + inputs: The Records of the pair's rows, in published order. + issues: The pair's source issues, in ``SourceIssue`` order. + + Returns: + An answer task for an open question, with the published answer as target. A classification task + for a closed question, with the published letters as targets over its label set. + """ + issue_names: list[JsonValue] = [issue.value for issue in issues] + metadata = pair.place.metadata | {MetadataKey.STAGE: pair.stage.value, MetadataKey.SOURCE_ISSUES: issue_names} + if pair.stage.is_multi_cycle: + metadata[MetadataKey.STEPS_PER_CYCLE] = STEPS_PER_CYCLE + split = pair.place.part.timef_split + if pair.closed is None: + return AnswerTask(inputs=inputs, prompt=pair.question, targets=(pair.answer,), split=split, metadata=metadata) + return ClassificationTask( + inputs=inputs, + prompt=pair.question, + targets=pair.closed.parse_answer(pair.answer), + label_set=pair.closed.label_set, + split=split, + metadata=metadata | {MetadataKey.CLOSED_QUESTION: pair.closed.value}, + ) + + +def iter_tasks(qa_files: Sequence[QAFile], records: Mapping[int, Record], scan: QAInventory) -> Iterator[Task]: + """Yield labelled QA tasks in source order on each pass.""" + for qa in qa_files: + for pair in qa.pairs(): + if pair.has_label: + yield _task(pair, _inputs(pair, records), scan.issues(pair)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md new file mode 100644 index 000000000..6aa23ee20 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md @@ -0,0 +1,23 @@ +# Test fixtures + +`fixtures/` holds real lines of the pinned release, copied byte for byte. The tests write the +signals file themselves, because the real one is 18.8 GB. Its values encode the row, the step and +the channel, so a misplaced value cannot pass. + +| File | Release lines (0-based), in fixture order | +| --- | --- | +| `train_qa.jsonl` | 316, 92, 11228, 3503 | +| `test_qa.jsonl` | 2786, 456, 1082 | + +- Train line 316 (row 278) has an empty health state. +- Train line 92 (row 8435) has the fault cause `a, b, e`. +- Train line 11228 (row 20) is the same line as test line 2786. +- Train line 3503 and test line 456 are multi-cycle lines. They share five of their ten rows. +- Test line 1082 (row 37669) has an empty fault cause. + +To make a file again, download the release's QA file and copy the lines above, in this order: + +```python +lines = open("train_qa.jsonl", "rb").read().splitlines(keepends=True) +open("fixtures/train_qa.jsonl", "wb").write(b"".join(lines[n] for n in (316, 92, 11228, 3503))) +``` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl new file mode 100644 index 000000000..7ac9b44ae --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl @@ -0,0 +1,3 @@ +{"id": "20", "name": "20.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal over one cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude reflects consistent engine performance, indicating that the engine operates efficiently without experiencing additional strain or requiring adjustments."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the specific representation of the change in Throttle Resolver Angle within the provided engine signal over the course of a single cycle?\n "}, {"from": "gpt", "value": "The Throttle Resolver Angle exhibits a steady and controlled increase. This signifies a deliberate and balanced augmentation of engine power, ensuring an optimal equilibrium between thrust generation and fuel consumption efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "So, what does the change in total temperature at the fan inlet tell us about the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature stays the same throughout the cycle. This shows that the engine is running smoothly without any ups and downs in temperature, which is a good sign of stable operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise representation and analytical interpretation of the change in total pressure at the fan inlet observed over the duration of a single cycle?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet is maintained at a constant level. This indicates the presence of stable airflow and consistent engine performance, with no significant pressure fluctuations occurring during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal during one cycle, what does the variation in Total Temperature at the LPC outlet signify?\n "}, {"from": "gpt", "value": "The LPC outlet temperature exhibits a slow and steady rise. This indicates a gradual adaptation in the compressor's workload, which contributes to maintaining stable and efficient engine performance."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPC outlet means for this engine signal we're analyzing in one cycle?\n "}, {"from": "gpt", "value": "Well, the HPC outlet temperature is increasing slowly over time. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the fuel is being used as efficiently as possible."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPT outlet means for the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "Well, the HPT outlet temperature is increasing slowly. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the engine is using fuel efficiently."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal during a single cycle, what does the variation in the Total Temperature at the LPT outlet signify?\n "}, {"from": "gpt", "value": "The LPT outlet temperature exhibits a gradual rise. This indicates that the engine is making incremental adjustments to its power output, which supports stable operational conditions and enhances fuel efficiency."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} +{"id": ["279", "287", "295", "304", "309", "319", "326", "334", "339", "346"], "name": ["279.csv", "287.csv", "295.csv", "304.csv", "309.csv", "319.csv", "326.csv", "334.csv", "339.csv", "346.csv"], "conversations": [{"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the qualitative condition of the engine? a: Good Condition\n b: Moderate Condition\n c: Poor Condition\n d: Bad Condition\n e: Extremely Poor Condition\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the probability range of engine failure? a: 1%-10%\n b: 10%-30%\n c: 30%-50%\n d: 50%-70%\n e: 70%-100%\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "4", "attribute": "open", "value": "With respect to the engine signal provided across a span of 10 cycles, what specific actions should be undertaken to ensure optimal performance and safety?\n "}, {"from": "gpt", "value": "It is imperative to closely monitor the engine's condition and proactively schedule maintenance in the near future to mitigate potential risks and maintain operational reliability."}]} +{"id": "37669", "name": "37669.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal over one operational cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude reflects consistent engine performance, ensuring that the engine operates smoothly without experiencing any additional strain or the need for adjustments."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise interpretation and operational significance of the observed change in the throttle resolver angle during a single cycle?\n "}, {"from": "gpt", "value": "The throttle resolver angle exhibits a slow and gradual increase. This indicates that the engine is undergoing incremental power adjustments, which facilitates smoother engine operation and ensures optimal fuel efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total temperature at fan inlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature appears to remain constant. This observation might indicate stable engine operation without significant temperature fluctuations."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal over a single cycle, what does the variation in total pressure at the fan inlet indicate?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet exhibits a slow decline. This gradual drop could indicate either a controlled reduction in throttle settings or minor inefficiencies in compressor performance. Despite these changes, the engine maintains stable operational performance throughout the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total temperature at LPC outlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The LPC outlet temperature appears to be rising slowly. This could reflect gradual adjustments in the compressor workload, which may contribute to promoting stable engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total temperature at HPC outlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The HPC outlet temperature appears to remain constant. This observation might suggest stable compressor performance, with no significant temperature fluctuations detected during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal over a single cycle, what does the variation in the Total Temperature at the High-Pressure Turbine (HPT) outlet signify?\n "}, {"from": "gpt", "value": "The HPT outlet temperature exhibits a slow and steady rise. This gradual increase indicates controlled adjustments in engine power output, which contributes to maintaining stable engine operation and ensures optimal fuel efficiency throughout the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the significance of the variation in Total temperature at the LPT outlet as it pertains to the engine signal provided within a single cycle?\n "}, {"from": "gpt", "value": "The LPT outlet temperature exhibits a gradual increase. This reflects incremental adjustments in engine power, which contribute to stable operation and the efficient utilization of fuel resources."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": ""}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl new file mode 100644 index 000000000..69b8967fd --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl @@ -0,0 +1,4 @@ +{"id": "278", "name": "278.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the precise interpretation of the fluctuation in altitude with regard to the engine signal that has been presented within the confines of a singular cycle?\n "}, {"from": "gpt", "value": "The altitude is observed to be dropping rapidly. A swift descent reduces the engine's workload; however, it may necessitate rapid adjustments to maintain stability and operational efficiency, which could potentially lead to increased wear on critical components over time."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Throttle resolver angle represent in the given engine signal over one cycle?\n "}, {"from": "gpt", "value": "The Throttle Resolver Angle appears to be decreasing slowly. This gradual reduction may suggest a controlled decrease in power, which could contribute to smoother engine operation and potentially support optimal fuel conservation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the precise interpretation of the alteration in the total temperature at the fan inlet with regard to the engine signal that has been presented within the confines of a singular cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature remains unchanged. This indicates stable engine operation, as there are no significant temperature fluctuations observed during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the precise interpretation of the variation in total pressure at the fan inlet with regard to the engine signal that has been presented within the confines of a singular cycle?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet remains unchanged. This indicates a stable airflow and consistent engine performance, with no significant pressure fluctuations observed."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal data over a single cycle, what does the variation in Total Temperature at the LPC outlet signify?\n "}, {"from": "gpt", "value": "The Total Temperature at the LPC outlet remains constant throughout the cycle. This stability indicates that the compressor is performing consistently, with no significant temperature fluctuations, which reflects reliable and efficient compressor operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPC outlet means in relation to the engine signal we're analyzing for this one cycle?\n "}, {"from": "gpt", "value": "Well, the HPC outlet temperature stays steady. That tells us the compressor is performing consistently without any ups and downs in temperature, which is a good sign for stable engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, could you explain what the change in Total Temperature at the High-Pressure Turbine (HPT) outlet signifies in the context of the engine signal we're analyzing for this single cycle?\n "}, {"from": "gpt", "value": "Well, the HPT outlet temperature stays constant throughout the cycle. This suggests that the turbine is performing steadily, with no significant temperature fluctuations, which is a good sign of stable operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise representation and analytical significance of the change in Total Temperature observed at the Low-Pressure Turbine (LPT) outlet during the duration of a single cycle?\n "}, {"from": "gpt", "value": "The Total Temperature at the LPT outlet is maintained at a constant level. This stability signifies consistent compressor performance, reflecting the absence of temperature fluctuations and ensuring reliable engine operation."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": ""}]} +{"id": "8435", "name": "8435.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal for a single cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude suggests that the engine is performing consistently without encountering any additional strain or operational challenges."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal, what is the precise interpretation and operational significance of the observed change in the throttle resolver angle over the duration of a single cycle?\n "}, {"from": "gpt", "value": "The throttle resolver angle exhibits a gradual and steady increase. This indicates that the engine is undergoing incremental power adjustments, which facilitate smooth and controlled engine performance while simultaneously optimizing fuel efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the significance of the variation in the total temperature at the fan inlet with respect to the engine signal provided within a single cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature remains unchanged. This indicates stable engine operation, demonstrating the absence of any temperature fluctuations during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total pressure at fan inlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The fan inlet pressure appears to remain constant. This could indicate stable airflow and consistent engine performance, suggesting the absence of significant pressure fluctuations during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the specific significance of the variation in the total temperature at the LPC outlet within the context of the provided engine signal over a single cycle?\n "}, {"from": "gpt", "value": "The LPC outlet temperature remains unchanged. This indicates stable compressor performance, with no significant temperature fluctuations observed."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Regarding the provided engine signal within a single cycle, what is the significance of the variation in the total temperature at the high-pressure compressor (HPC) outlet?\n "}, {"from": "gpt", "value": "The total temperature at the HPC outlet remains consistent. This stability signifies reliable compressor performance, devoid of any temperature fluctuations, ensuring optimal engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal, what is the precise interpretation and operational significance of the change in total temperature observed at the high-pressure turbine (HPT) outlet over the course of a single cycle?\n "}, {"from": "gpt", "value": "The total temperature at the HPT outlet is maintained at a constant level. This stability signifies consistent turbine performance, highlighting the absence of temperature fluctuations and ensuring reliable engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "With respect to the engine signal provided within a single cycle, what does the variation in the Total Temperature at the Low-Pressure Turbine (LPT) outlet signify?\n "}, {"from": "gpt", "value": "The temperature at the LPT outlet remains consistent. This indicates stable compressor performance, with no significant temperature fluctuations observed during the cycle."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a, b, e"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} +{"id": "20", "name": "20.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal over one cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude reflects consistent engine performance, indicating that the engine operates efficiently without experiencing additional strain or requiring adjustments."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the specific representation of the change in Throttle Resolver Angle within the provided engine signal over the course of a single cycle?\n "}, {"from": "gpt", "value": "The Throttle Resolver Angle exhibits a steady and controlled increase. This signifies a deliberate and balanced augmentation of engine power, ensuring an optimal equilibrium between thrust generation and fuel consumption efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "So, what does the change in total temperature at the fan inlet tell us about the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature stays the same throughout the cycle. This shows that the engine is running smoothly without any ups and downs in temperature, which is a good sign of stable operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise representation and analytical interpretation of the change in total pressure at the fan inlet observed over the duration of a single cycle?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet is maintained at a constant level. This indicates the presence of stable airflow and consistent engine performance, with no significant pressure fluctuations occurring during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal during one cycle, what does the variation in Total Temperature at the LPC outlet signify?\n "}, {"from": "gpt", "value": "The LPC outlet temperature exhibits a slow and steady rise. This indicates a gradual adaptation in the compressor's workload, which contributes to maintaining stable and efficient engine performance."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPC outlet means for this engine signal we're analyzing in one cycle?\n "}, {"from": "gpt", "value": "Well, the HPC outlet temperature is increasing slowly over time. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the fuel is being used as efficiently as possible."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPT outlet means for the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "Well, the HPT outlet temperature is increasing slowly. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the engine is using fuel efficiently."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal during a single cycle, what does the variation in the Total Temperature at the LPT outlet signify?\n "}, {"from": "gpt", "value": "The LPT outlet temperature exhibits a gradual rise. This indicates that the engine is making incremental adjustments to its power output, which supports stable operational conditions and enhances fuel efficiency."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} +{"id": ["279", "287", "295", "305", "310", "319", "327", "335", "340", "346"], "name": ["279.csv", "287.csv", "295.csv", "305.csv", "310.csv", "319.csv", "327.csv", "335.csv", "340.csv", "346.csv"], "conversations": [{"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the qualitative condition of the engine? a: Good Condition\n b: Moderate Condition\n c: Poor Condition\n d: Bad Condition\n e: Extremely Poor Condition\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the probability range of engine failure? a: 1%-10%\n b: 10%-30%\n c: 30%-50%\n d: 50%-70%\n e: 70%-100%\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "4", "attribute": "open", "value": "How should we approach the analysis and management of the provided engine signal, which spans across 10 operational cycles?\n "}, {"from": "gpt", "value": "It is essential to closely monitor the engine's condition throughout these cycles and proactively schedule maintenance in the near future to ensure optimal performance and prevent potential issues."}]} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py new file mode 100644 index 000000000..b7259b268 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py @@ -0,0 +1,450 @@ +import json +from pathlib import Path +import shutil +from typing import Any, NamedTuple + +import h5py +import numpy as np +import pytest + +from timenet.dataset import Record, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.testing import assert_round_trips +from timenet.types import AnswerTask, ClassificationTask, Task +from timenet.types.text import read_value +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.datasets.pandalin98.enginemt_qa import EngineMTQAConnector, source +from timenet_connectors.datasets.pandalin98.enginemt_qa.source import FileCounts +from timenet_connectors.datasets.pandalin98.enginemt_qa.vocabulary import ClosedQuestion, Part, Stage + + +_FIXTURES = Path(__file__).parent / "fixtures" +_TRAIN, _TEST, _SIGNALS = "train_qa.jsonl", "test_qa.jsonl", "time_series_data.h5" +_STEPS = 600 +CHANNELS = 33 +_N_ROWS = 37_669 +"""The largest row id that a fixture line names, so the fixture file has just enough rows.""" +_MULTI_TRAIN = (279, 287, 295, 305, 310, 319, 327, 335, 340, 346) +_MULTI_TEST = (279, 287, 295, 304, 309, 319, 326, 334, 339, 346) +_NAMED_ROWS = sorted({278, 8435, 20, 37669, *_MULTI_TRAIN, *_MULTI_TEST}) + +# See tests/README.md for where each fixture line comes from. +_COUNTS = { + Part.TRAIN: FileCounts( + lines=4, + pairs={Stage.UNDERSTANDING: 24, Stage.PERCEPTION: 21, Stage.REASONING: 2, Stage.DECISION_MAKING: 1}, + unlabelled={ClosedQuestion.HEALTH_STATE: 1}, + ), + Part.TEST: FileCounts( + lines=3, + pairs={Stage.UNDERSTANDING: 16, Stage.PERCEPTION: 14, Stage.REASONING: 2, Stage.DECISION_MAKING: 1}, + unlabelled={ClosedQuestion.FAULT_CAUSE: 1}, + ), +} + + +def _values(row_id: int) -> np.ndarray: + """Every value names its row, step and channel, so a misplaced value can't pass.""" + steps = np.arange(_STEPS, dtype=np.float64)[:, None] + channels = np.arange(CHANNELS, dtype=np.float64)[None, :] + return row_id * 1000 + steps + channels / 100 + + +def _write_signals(path: Path, rows: int = _N_ROWS) -> None: + """Write the signals file. Only the named rows hold values, so the chunked file stays small.""" + with h5py.File(path, "w") as file: + file["data_ID"] = np.arange(1, rows + 1, dtype=np.int64) + values = file.create_dataset("seq_data", shape=(rows, _STEPS, CHANNELS), dtype=np.float64, chunks=True) + for row_id in _NAMED_ROWS: + if row_id <= rows: + values[row_id - 1] = _values(row_id) + + +def _lines(name: str) -> list[dict[str, Any]]: + return [json.loads(line) for line in (_FIXTURES / name).read_text(encoding="utf-8").splitlines()] + + +def _write_qa(path: Path, lines: list[dict[str, Any]]) -> None: + path.write_text("".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8") + + +@pytest.fixture(autouse=True) +def _fixture_counts(monkeypatch): + monkeypatch.setattr(EngineMTQAConnector, "SOURCE_COUNTS", _COUNTS) + + +@pytest.fixture +def refs(tmp_path) -> list[HubFileRef]: + """The release in ``tmp_path``: real QA lines and synthetic signals.""" + _write_signals(tmp_path / _SIGNALS) + for name in (_TRAIN, _TEST): + shutil.copyfile(_FIXTURES / name, tmp_path / name) + return [HubFileRef(tmp_path / name, part) for part, name in EngineMTQAConnector.SOURCE_FILES.items()] + + +@pytest.fixture +def dataset(refs) -> TimeFDataset: + return EngineMTQAConnector().convert(refs) + + +def _row_id(record: Record) -> int: + value = record.metadata["row_id"] + assert isinstance(value, int) + return value + + +class _View(NamedTuple): + """A task as plain values.""" + + kind: str + split: str + rows: tuple[int, ...] + prompt: str + targets: tuple[str, ...] + label_set: str | None + metadata: dict[str, Any] + + +def _view(task: Task) -> _View: + assert isinstance(task, AnswerTask | ClassificationTask) + assert task.split is not None + return _View( + kind=type(task).__name__, + split=task.split.value, + rows=tuple(_row_id(record) for record in task.inputs), + prompt=str(read_value(task.prompt)), + targets=tuple(str(read_value(target)) for target in task.targets or ()), + label_set=task.label_set.name if isinstance(task, ClassificationTask) and task.label_set else None, + metadata=dict(task.metadata), + ) + + +def _views(dataset: TimeFDataset) -> list[_View]: + return [_view(task) for task in dataset.iter_tasks()] + + +def _find(views: list[_View], split: str, line: int, offset: int) -> _View: + return next( + view + for view in views + if view.split == split and view.metadata["source_line"] == line and view.metadata["qa_offset"] == offset + ) + + +def test_converts_the_fixture(dataset): + # Each named row once, though rows 20, 279, 287, 295, 319 and 346 are named in both files. + assert [_row_id(record) for record in dataset.records] == _NAMED_ROWS + views = _views(dataset) + # Per single-cycle line: 8 answer and 7 classification tasks, less one for an empty answer. + # Per multi-cycle line: 2 classification tasks and 1 answer task. + single = ["AnswerTask"] * 8 + ["ClassificationTask"] * 7 + train = single[:-1] + single + single + ["ClassificationTask"] * 2 + ["AnswerTask"] + test = single + ["ClassificationTask"] * 2 + ["AnswerTask"] + single[:-2] + single[-1:] + assert [view.kind for view in views] == train + test + assert [view.split for view in views] == ["train"] * len(train) + ["test"] * len(test) + + +def test_fault_cause_targets_its_letters_over_a_multi_label_set(dataset): + tasks = [task for task in dataset.iter_tasks() if task.metadata.get("closed_question") == "fault_cause"] + letters = {_view(task).rows: _view(task).targets for task in tasks} + assert letters[8435,] == ("a", "b", "e") + assert letters[278,] == ("a",) + label_set = next(task.label_set for task in tasks if isinstance(task, ClassificationTask)) + assert label_set is not None + assert label_set.multi_label + assert [(label.name, label.description) for label in label_set.labels] == [ + ("a", "HPT efficiency modifier"), + ("b", "LPT efficiency modifier"), + ("c", "Fan efficiency modifier"), + ("d", "HPT flow modifier"), + ("e", "LPT flow modifier"), + ("f", "Fan flow modifier"), + ] + + +def test_closed_questions_keep_their_place_and_label_set(dataset): + view = _find(_views(dataset), "train", line=1, offset=13) + published = _lines(_TRAIN)[1]["conversations"][26]["value"] + assert view == _View( + kind="ClassificationTask", + split="train", + rows=(8435,), + prompt=published, + targets=("a", "b", "e"), + label_set="fault_cause", + metadata={ + "source_file": _TRAIN, + "source_line": 1, + "qa_offset": 13, + "stage": 2, + "source_issues": [], + "closed_question": "fault_cause", + }, + ) + health = _find(_views(dataset), "train", line=0, offset=8) + assert (health.label_set, health.metadata["closed_question"]) == ("component_health", "fan_health") + + +def test_open_questions_keep_the_published_text(dataset): + lines = _lines(_TEST) + view = _find(_views(dataset), "test", line=1, offset=2) + assert view == _View( + kind="AnswerTask", + split="test", + rows=_MULTI_TEST, + prompt=lines[1]["conversations"][4]["value"], + targets=(lines[1]["conversations"][5]["value"],), + label_set=None, + metadata={ + "source_file": _TEST, + "source_line": 1, + "qa_offset": 2, + "stage": 4, + "source_issues": [], + "steps_per_cycle": 60, + }, + ) + + +def test_reasoning_takes_the_ten_rows_in_published_order(dataset): + view = _find(_views(dataset), "train", line=3, offset=0) + assert view.rows == _MULTI_TRAIN + assert (view.metadata["stage"], view.metadata["closed_question"], view.targets) == (3, "engine_condition", ("b",)) + + +def test_empty_closed_answers_get_no_task_and_are_listed(dataset): + views = _views(dataset) + with pytest.raises(StopIteration): + _find(views, "train", line=0, offset=14) + with pytest.raises(StopIteration): + _find(views, "test", line=2, offset=13) + unlabelled = next(a for a in dataset.annotations if a.key == "unlabelled_pairs") + assert unlabelled.value == 2 + assert unlabelled.metadata["pairs"] == [ + {"source_file": _TRAIN, "source_line": 0, "qa_offset": 14}, + {"source_file": _TEST, "source_line": 2, "qa_offset": 13}, + ] + + +def test_pairs_repeated_across_files_are_flagged_on_both_sides(dataset): + # Test line 0 is a byte copy of train line 2. The multi-cycle lines name other rows. + issues = {(view.split, view.metadata["source_line"]): view.metadata["source_issues"] for view in _views(dataset)} + assert issues["train", 2] == ["repeated_across_splits"] + assert issues["test", 0] == ["repeated_across_splits"] + assert issues["train", 3] == issues["test", 1] == [] + + +def test_a_pair_repeated_in_its_file_is_flagged(refs, tmp_path, monkeypatch): + train = _lines(_TRAIN) + _write_qa(tmp_path / _TRAIN, [*train, train[3]]) + counts = _COUNTS[Part.TRAIN] + pairs = {**counts.pairs, Stage.REASONING: 4, Stage.DECISION_MAKING: 2} + monkeypatch.setitem(_COUNTS, Part.TRAIN, counts._replace(lines=5, pairs=pairs)) + views = _views(EngineMTQAConnector().convert(refs)) + assert {tuple(v.metadata["source_issues"]) for v in views if v.rows == _MULTI_TRAIN} == {("repeated_question",)} + + +def test_every_signal_of_a_record_holds_its_row(dataset): + record = dataset.records[0] + assert [source.name for source in record.sources] == ["N-CMAPSS engine"] + assert [signal.name for signal in record.signals] == [f"channel_{c:02d}" for c in range(CHANNELS)] + for channel, signal in enumerate(record.signals): + assert isinstance(signal.time_axis, RegularAxis) + assert signal.time_axis.period_us == 1_000_000 + np.testing.assert_array_equal(signal.to_numpy(), _values(_row_id(record))[:, channel]) + + +def test_storage_round_trip_keeps_every_task_and_value(dataset, tmp_path): + restored = assert_round_trips(dataset, tmp_path / "registry") + assert _views(restored) == _views(dataset) + for record in restored.records: + for channel, signal in enumerate(record.signals): + np.testing.assert_array_equal(signal.to_numpy(), _values(_row_id(record))[:, channel]) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + (lambda lines: lines[:-1], "expected"), + (lambda lines: [], "expected"), + ], + ids=["a-line-short", "empty"], +) +def test_a_qa_file_with_other_counts_fails(refs, tmp_path, change, message): + _write_qa(tmp_path / _TRAIN, change(_lines(_TRAIN))) + with pytest.raises(TimeNetConnectorError, match=message) as raised: + EngineMTQAConnector().convert(refs) + assert _TRAIN in str(raised.value) + + +def test_a_shortened_signals_file_fails_when_read(dataset, tmp_path): + _write_signals(tmp_path / _SIGNALS, rows=4) + with pytest.raises(TimeNetConnectorError, match="cannot read row id"): + dataset.records[-1].signals[0].to_numpy() + + +def test_a_missing_signals_file_fails_when_read(dataset, tmp_path): + (tmp_path / _SIGNALS).unlink() + with pytest.raises(TimeNetConnectorError, match="cannot open the signals file"): + dataset.records[0].signals[0].to_numpy() + + +def test_a_row_with_a_nan_fails_when_read(dataset, tmp_path): + with h5py.File(tmp_path / _SIGNALS, "a") as file: + file["seq_data"][19, 5, 3] = np.nan + with pytest.raises(TimeNetConnectorError, match="row id 20 holds a NaN"): + dataset.records[0].signals[0].to_numpy() + + +@pytest.mark.parametrize( + ("name", "change"), + [ + ("changed", lambda lines: [{**lines[0], "id": "8435", "name": "8435.csv"}, *lines[1:]]), + ("removed", lambda lines: lines[:-1]), + ], +) +def test_a_qa_file_changed_after_conversion_fails_when_the_tasks_are_read(dataset, tmp_path, name, change): + _write_qa(tmp_path / _TRAIN, change(_lines(_TRAIN))) + message = r"line 1\): the line changed since conversion" if name == "changed" else "lines were removed" + with pytest.raises(TimeNetConnectorError, match=message): + list(dataset.iter_tasks()) + + +def test_a_missing_qa_file_fails_when_the_tasks_are_read(dataset, tmp_path): + (tmp_path / _TRAIN).unlink() + with pytest.raises(TimeNetConnectorError, match="cannot read source file"): + list(dataset.iter_tasks()) + + +def test_a_file_that_is_not_hdf5_fails_with_its_path(refs, tmp_path): + (tmp_path / _SIGNALS).write_bytes(b"not hdf5") + with pytest.raises(TimeNetConnectorError, match="cannot open the signals file"): + EngineMTQAConnector().convert(refs) + + +def _rewrite(path: Path, **datasets: Any) -> None: + with h5py.File(path, "w") as file: + for name, value in datasets.items(): + if value is None: + file.create_group(name) + else: + file[name] = value + + +_IDS = np.arange(1, 4) +_ZEROS = np.zeros((3, _STEPS, CHANNELS)) + + +@pytest.mark.parametrize( + ("datasets", "message"), + [ + ({"data_ID": _IDS, "seq_data": _ZEROS, "extra": 1}, "expected seq_data and data_ID"), + ({"data_ID": _IDS, "seq_data": np.zeros((3, _STEPS, CHANNELS - 1))}, r"seq_data is float64 \(3, 600, 32\)"), + ({"data_ID": _IDS, "seq_data": _ZEROS.astype(np.float32)}, "seq_data is float32"), + ({"data_ID": _IDS, "seq_data": None}, "seq_data is a Group, not an HDF5 dataset"), + ({"data_ID": _IDS.astype(np.float64), "seq_data": _ZEROS}, "data_ID is float64"), + ({"data_ID": np.array([2, 2, 3]), "seq_data": _ZEROS}, "data_ID is not the row ids 1 to 3"), + ], + ids=["extra-key", "32-channels", "float32", "group", "float-ids", "ids-not-1-to-n"], +) +def test_a_signals_file_with_another_layout_fails(refs, tmp_path, datasets, message): + _rewrite(tmp_path / _SIGNALS, **datasets) + with pytest.raises(TimeNetConnectorError, match=message): + EngineMTQAConnector().convert(refs) + + +def _with_pair(offset: int, **changes: Any) -> dict[str, Any]: + """Train line 0 with one pair changed: ``answer`` sets the answer, other keys the question.""" + line = _lines(_TRAIN)[0] + conversations = [dict(turn) for turn in line["conversations"]] + question, answer = conversations[2 * offset], conversations[2 * offset + 1] + answer["value"] = changes.pop("answer", answer["value"]) + question.update(changes) + return {**line, "conversations": conversations} + + +def _with(**changes: Any) -> dict[str, Any]: + """A train line with its own keys changed: the multi-cycle line 3 if ``multi``, else line 0.""" + line = _lines(_TRAIN)[3 if changes.pop("multi", False) else 0] + return {**line, **changes} + + +def _health_state(text: str) -> str: + return _lines(_TRAIN)[0]["conversations"][28]["value"].replace("Imminent", text) + + +# In a single-cycle line, pairs 8 to 14 are the Perception questions: 13 is the fault cause and 14 the +# health state. +@pytest.mark.parametrize( + ("line", "message"), + [ + (_with_pair(13, answer="a, g"), "labels not in label set 'fault_cause'"), + (_with_pair(13, answer="a,b"), "labels not in label set 'fault_cause'"), + (_with_pair(13, answer="a, a"), "repeated labels"), + (_with_pair(14, answer="a, b"), "takes exactly one"), + (_with_pair(8, value="What is the health status of the fan?\n "), "unknown stage 2"), + (_with_pair(14, value=_health_state("Near")), "prints options other than"), + (_with_pair(0, answer=""), "open question has an empty answer"), + (_with_pair(0, attribute="close"), "stage 1 questions are not 'close'"), + (_with_pair(0, stage="5"), "question: stage:"), + (_with_pair(0, stage=1), "question: stage:"), + (_with_pair(0, stage="01"), "question: stage:"), + (_with_pair(0, attribute="closed"), "question: attribute:"), + (_with_pair(0, stage=[2]), r"question: stage:"), + (_with_pair(0, attribute={}), "question: attribute:"), + (_with_pair(0, value="Describe the signal."), "one marker"), + (_with_pair(0, **{"from": "gpt"}), "question: from:"), + (_with(name="4.csv"), "doesn't repeat id"), + (_with(multi=True, id=["1"] * 10, name=["1.csv"] * 10), "repeats a row"), + (_with(multi=True, id=["1"] * 9, name=["1.csv"] * 9), "names 9 rows"), + (_with(id="0278", name="0278.csv"), "not decimal row ids"), + (_with(id="37670", name="37670.csv"), "past the file's 37669 rows"), + (_with(extra=1), "extra: Extra inputs"), + (_with(conversations=[]), "alternate"), + ], +) +def test_a_malformed_line_fails_with_its_place(refs, tmp_path, line, message): + _write_qa(tmp_path / _TRAIN, [_lines(_TRAIN)[0], line]) + with pytest.raises(TimeNetConnectorError, match=message) as raised: + EngineMTQAConnector().convert(refs) + assert "train_qa.jsonl (line 2)" in str(raised.value) + + +def test_a_line_that_is_not_json_fails_with_its_place(refs, tmp_path): + with (tmp_path / _TRAIN).open("a", encoding="utf-8") as file: + file.write("not json\n") + with pytest.raises(TimeNetConnectorError, match=r"train_qa.jsonl \(line 5\)"): + EngineMTQAConnector().convert(refs) + + +def test_a_missing_part_fails(refs): + with pytest.raises(TimeNetConnectorError, match="needs the parts"): + EngineMTQAConnector().convert(refs[1:]) + + +def test_lazy_channels_share_reads_and_eviction(refs, monkeypatch): + original = source._read_row + reads = [] + + def read(file, path, row_id): + reads.append(row_id) + return original(file, path, row_id) + + monkeypatch.setattr(source, "_read_row", read) + converted = EngineMTQAConnector().convert(refs) + first, second = converted.records[:2] + assert reads == [] + for signal in first.signals: + signal.to_arrow() + assert reads == [_row_id(first)] + second.signals[0].to_arrow() + first.signals[-1].to_arrow() + assert reads == [_row_id(first), _row_id(second), _row_id(first)] + + +def test_duplicate_json_field_is_rejected_with_source_context(refs, tmp_path): + path = tmp_path / _TRAIN + path.write_text(path.read_text().replace('"id":', '"id": "1", "id":', 1)) + with pytest.raises(TimeNetConnectorError, match=r"train_qa.jsonl \(line 1\): duplicate key 'id'"): + EngineMTQAConnector().convert(refs) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/vocabulary.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/vocabulary.py new file mode 100644 index 000000000..aeba7bb08 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/vocabulary.py @@ -0,0 +1,350 @@ +"""EngineMT-QA release constants, channel contracts and closed-question vocabulary.""" + +from collections.abc import Sequence +from enum import IntEnum, StrEnum, unique +import re +import string +from typing import NamedTuple + +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import ( + Label, + LabelSet, + Split, + TimeSeriesSpec, +) + + +N_STEPS = 600 + + +_N_CHANNELS = 33 + + +"""The released channel count. The paper says 32. The authors' audit confirms 33 and names none of them.""" + + +MULTI_CYCLE_ROWS = 10 + + +"""How many rows a multi-cycle QA names.""" + + +STEPS_PER_CYCLE = N_STEPS // MULTI_CYCLE_ROWS + + +"""The steps of each cycle that the authors' model reads in a multi-cycle task: the first 60 of 600.""" + + +_SAMPLE_RATE_HZ = 1 + + +"""The paper's "10-minute (600 points) time-series segments" (Section 5.2). The HDF5 file states no rate.""" + + +ROW_SHAPE = (N_STEPS, _N_CHANNELS) + + +SEQ_DATA = "seq_data" + + +"""The HDF5 dataset of every row's values, shaped ``(rows, steps, channels)``.""" + + +DATA_ID = "data_ID" + + +"""The HDF5 dataset of each row's id, which a QA's ``id`` names. Row ``i`` has id ``i + 1``.""" + + +SOURCE_NAME = "N-CMAPSS engine" + + +CHANNELS = tuple(f"channel_{index:02d}" for index in range(_N_CHANNELS)) + + +"""The release names no channel. The zero-padded names sort in the source's order.""" + + +# In 24 sampled rows, every channel has a mean near 0 and a standard deviation near 1, so the values +# look z-scored. The release states neither the units nor the statistics. +SPEC = TimeSeriesSpec( + spec_type="enginemt_qa_channel", name="EngineMT-QA channel, z-scored", unit_value=None, dtype="float64" +) + + +AXIS = RegularAxis.from_rate_hz(_SAMPLE_RATE_HZ) + + +# A published row id: a decimal number with no sign and no leading zero, such as "278". +ROW_ID_PATTERN = re.compile(r"[1-9][0-9]*") + + +# One printed option of a closed question: its letter, then its text, such as "a: Health\n". +_OPTION = re.compile(r"([a-z]):\s+([^\n]+)\n") + + +DIGEST_BYTES = 16 + + +"""The size of a line or pair digest. 16 bytes make an accidental match between two of the release's +144,042 pairs practically impossible, and keep the digests of a file small.""" + + +@unique +class MetadataKey(StrEnum): + """Keys stored in record, source or task metadata.""" + + ROW_ID = "row_id" + SOURCE_FILE = "source_file" + SOURCE_LINE = "source_line" + QA_OFFSET = "qa_offset" + STAGE = "stage" + STEPS_PER_CYCLE = "steps_per_cycle" + CLOSED_QUESTION = "closed_question" + SOURCE_ISSUES = "source_issues" + + +@unique +class AnnotationKey(StrEnum): + """Keys of attached annotations.""" + + UNLABELLED_PAIRS = "unlabelled_pairs" + + +@unique +class SourceIssue(StrEnum): + """A problem in a published pair, which the connector keeps as published and names in the task's metadata. + + The names match the TSAQA connector's, so one filter can serve both datasets. + """ + + REPEATED_QUESTION = "repeated_question" + """Another pair of the same file has the same rows, question and answer.""" + REPEATED_ACROSS_SPLITS = "repeated_across_splits" + """The other file has a pair with the same rows, question and answer.""" + + +@unique +class Part(StrEnum): + """The release's three files: the signals, and the train and test QA.""" + + SIGNALS = "signals" + TRAIN = "train" + TEST = "test" + + @property + def source_file(self) -> str: + """The file's name in the Hub repository, such as ``train_qa.jsonl``.""" + return "time_series_data.h5" if self is Part.SIGNALS else f"{self.value}_qa.jsonl" + + @property + def timef_split(self) -> Split: + """The split of a QA file: the authors train on ``train_qa`` and test on ``test_qa``. + + Raises: + TimeNetConnectorError: If the part is not a QA file. + """ + if self not in _SPLITS: + raise TimeNetConnectorError(f"{self} is not a QA file") + return _SPLITS[self] + + +_SPLITS = {Part.TRAIN: Split.TRAIN, Part.TEST: Split.TEST} + + +QA_PARTS = tuple(_SPLITS) + + +@unique +class Stage(IntEnum): + """The release's ``stage``: one of the paper's four tasks.""" + + UNDERSTANDING = 1 + PERCEPTION = 2 + REASONING = 3 + DECISION_MAKING = 4 + + @property + def is_closed(self) -> bool: + """Whether the stage asks closed questions, which the release marks ``close``.""" + return self in {Stage.PERCEPTION, Stage.REASONING} + + @property + def is_multi_cycle(self) -> bool: + """Whether the stage's questions name 10 cycles rather than one.""" + return self in {Stage.REASONING, Stage.DECISION_MAKING} + + +@unique +class Attribute(StrEnum): + """The release's ``attribute`` of a question.""" + + OPEN = "open" + CLOSE = "close" + + +def _option_letters(name: str, options: Sequence[str], *, multi_label: bool = False) -> LabelSet: + """Build the label set of a closed question: one label per option letter, from ``a``. + + Returns: + The label set. Each label's name is a letter, and its description is the option the letter + stands for. + """ + labels = tuple( + Label(name=letter, description=option) + for letter, option in zip(string.ascii_lowercase[: len(options)], options, strict=True) + ) + return LabelSet(name=name, labels=labels, multi_label=multi_label) + + +_COMPONENT_HEALTH = _option_letters("component_health", ("Health", "Fault")) + + +_FAULT_CAUSE = _option_letters( + "fault_cause", + ( + "HPT efficiency modifier", + "LPT efficiency modifier", + "Fan efficiency modifier", + "HPT flow modifier", + "LPT flow modifier", + "Fan flow modifier", + ), + # The authors' answers list every cause that applies, such as "a, b, e". + multi_label=True, +) + + +_HEALTH_STATE = _option_letters("health_state", ("Operating Normally", "Abnormal Degradation", "Imminent Failure")) + + +_ENGINE_CONDITION = _option_letters( + "engine_condition", + ("Good Condition", "Moderate Condition", "Poor Condition", "Bad Condition", "Extremely Poor Condition"), +) + + +_FAILURE_PROBABILITY = _option_letters("failure_probability", ("1%-10%", "10%-30%", "30%-50%", "50%-70%", "70%-100%")) + + +class _ClosedSpec(NamedTuple): + stage: Stage + label_set: LabelSet + stem: str + """The question's text up to its ``?``. The release publishes each closed question with one stem.""" + + +_REASONING_STEM = ( + "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different " + "health states and performing temporal inference to predict the health decline trend, " +) + + +_ONE_CYCLE = "of the given engine signal in one cycle" + + +def _prints_options(options: str, label_set: LabelSet) -> bool: + """Tell whether the printed options of a question are the label set's, in letter order. + + Returns: + ``True`` if each printed letter and text is a label's name and description, in label order. + """ + printed = _OPTION.findall(options) + return printed == [(label.name, label.description) for label in label_set.labels] + + +@unique +class ClosedQuestion(StrEnum): + """A closed question, which the release always publishes with the same stem and options.""" + + FAN_HEALTH = "fan_health" + LPC_HEALTH = "lpc_health" + HPC_HEALTH = "hpc_health" + HPT_HEALTH = "hpt_health" + LPT_HEALTH = "lpt_health" + FAULT_CAUSE = "fault_cause" + HEALTH_STATE = "health_state" + ENGINE_CONDITION = "engine_condition" + FAILURE_PROBABILITY = "failure_probability" + + @property + def stage(self) -> Stage: + """The question's stage.""" + return _CLOSED[self].stage + + @property + def label_set(self) -> LabelSet: + """The question's option letters.""" + return _CLOSED[self].label_set + + @classmethod + def of(cls, stage: Stage, text: str) -> "ClosedQuestion": + """Identify a published closed question by its stem, and check that it prints its label set's options. + + Args: + stage: The stage that the release gives the question. + text: The published question. + + Returns: + The question. + + Raises: + TimeNetConnectorError: If no closed question of the stage has this stem, or its printed options differ from + the label set's, which would change what each letter means. + """ + stem, _, options = text.partition("?") + question = _BY_STEM.get(stem) + if question is None or question.stage is not stage: + raise TimeNetConnectorError(f"unknown stage {stage} closed question {text[:80]!r}") + if not _prints_options(options, question.label_set): + raise TimeNetConnectorError(f"{question} prints options other than its label set's: {options!r}") + return question + + def parse_answer(self, answer: str) -> tuple[str, ...]: + """Parse a published answer, such as ``a`` or ``a, b, e``, into its letters. + + Returns: + The letters, in published order. + + Raises: + TimeNetConnectorError: If the answer isn't distinct letters of the question's options, joined by ``", "``, + or holds several letters where one is expected. + """ + letters = tuple(answer.split(", ")) + try: + self.label_set.check_targets(letters) + except TimeNetConnectorError as exc: + raise TimeNetConnectorError(f"{self}: answer {answer!r}: {exc}") from exc + return letters + + +def _health_spec(component: str) -> _ClosedSpec: + return _ClosedSpec(Stage.PERCEPTION, _COMPONENT_HEALTH, f"What is the health status of {component} {_ONE_CYCLE}") + + +_CLOSED = { + ClosedQuestion.FAN_HEALTH: _health_spec("fan"), + ClosedQuestion.LPC_HEALTH: _health_spec("LPC"), + ClosedQuestion.HPC_HEALTH: _health_spec("HPC"), + ClosedQuestion.HPT_HEALTH: _health_spec("HPT"), + ClosedQuestion.LPT_HEALTH: _health_spec("LPT"), + ClosedQuestion.FAULT_CAUSE: _ClosedSpec( + Stage.PERCEPTION, + _FAULT_CAUSE, + f"Was the timing issue in the operation {_ONE_CYCLE} caused by efficiency or flow problems", + ), + ClosedQuestion.HEALTH_STATE: _ClosedSpec( + Stage.PERCEPTION, _HEALTH_STATE, f"What is the potential health state {_ONE_CYCLE}" + ), + ClosedQuestion.ENGINE_CONDITION: _ClosedSpec( + Stage.REASONING, _ENGINE_CONDITION, _REASONING_STEM + "what is the qualitative condition of the engine" + ), + ClosedQuestion.FAILURE_PROBABILITY: _ClosedSpec( + Stage.REASONING, _FAILURE_PROBABILITY, _REASONING_STEM + "what is the probability range of engine failure" + ), +} + + +_BY_STEM = {spec.stem: question for question, spec in _CLOSED.items()}