From d6a0d451e9c3b9e80287fc3127221a8ae08741c7 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 20 Nov 2026 12:08:25 -0800 Subject: [PATCH 1/2] feat(connectors): add the EngineMT-QA question answering dataset The connector pandalin98/enginemt-qa builds one task per labelled question-answer pair of the EngineMT-QA release on the Hugging Face Hub. A single-row question reads the Record of its row. A multi-row question reads one joined Record with the first 60 steps of each of its 10 rows on an ordinal axis. The authors' code gives its model the same input. When a flight ends inside a row, the release fills the rest of the row with zeros. A row Record stops at its last real step. In a joined Record, the padded steps are null. The train file copies 4,039 lines of the test file. Tasks that the other file repeats carry the source issue repeated_across_splits, so that a user can rebuild the paper's split. A pair with an empty answer gets no task. The annotation unlabelled_pairs lists each one. Co-authored-by: Othman Sbai <212133622+osbai@users.noreply.github.com> --- .../datasets/pandalin98/__init__.py | 4 + .../datasets/pandalin98/enginemt_qa/README.md | 323 ++++++ .../pandalin98/enginemt_qa/__init__.py | 6 + .../pandalin98/enginemt_qa/connector.py | 916 ++++++++++++++++++ .../pandalin98/enginemt_qa/dataset.yaml | 36 + .../pandalin98/enginemt_qa/requirements.txt | 4 + .../pandalin98/enginemt_qa/tests/README.md | 25 + .../enginemt_qa/tests/fixtures/test_qa.jsonl | 3 + .../enginemt_qa/tests/fixtures/train_qa.jsonl | 4 + .../enginemt_qa/tests/test_connector.py | 447 +++++++++ pyproject.toml | 1 + uv.lock | 53 + 12 files changed, 1822 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py new file mode 100644 index 000000000..2c96ddf0d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/__init__.py @@ -0,0 +1,4 @@ +"""Connectors for datasets released by Yilin Wang (``pandalin98``) and co-authors. + +Keep this regular package so pkgutil.walk_packages can discover its connector children. +""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md new file mode 100644 index 000000000..2bc0c87ee --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md @@ -0,0 +1,323 @@ +# EngineMT-QA + +EngineMT-QA asks questions about simulated aero-engine flights from NASA's N-CMAPSS simulator. Each +question is about one 600-step row of 33 signals, or about a group of 10 rows. The authors define four +tasks, one per stage: Understanding, Perception, Reasoning and Decision-Making. The dataset comes +with the ITFormer paper (Wang et al., ICML 2025). +The paper is [arXiv 2506.20093](https://arxiv.org/abs/2506.20093). +The authors' code is [ITFormer-ICML25](https://github.com/Pandalin98/ITFormer-ICML25). + +The connector builds one task per labelled question-answer pair. A single-row question reads the +Record of its row. A multi-row question reads one joined Record: the first 60 steps of each of its +10 rows. The authors' code gives its model the same input. The connector lives in one file, +`connector.py`. + +## The source data + +The release is the Hub dataset +[pandalin98/EngineMT-QA](https://huggingface.co/datasets/pandalin98/EngineMT-QA), pinned at +`ea2051a15512212976c22c1a075ec549c7d48eef`. The Hub metadata declares the MIT license. It holds three +files: + +| File | Holds | +| --- | --- | +| `time_series_data.h5` | `seq_data`, float64 shaped (118,921, 600, 33), and `data_ID`, the ids 1 to 118,921 | +| `train_qa.jsonl` | 13,291 lines, 101,565 QA pairs | +| `test_qa.jsonl` | 5,539 lines, 42,477 QA pairs | + +### Signals + +All signal values are in the one HDF5 file. A QA line names its rows by id, stored as a string such +as `"633"`. Row `i` of `seq_data` has id `i + 1`. So a QA's id `n` is row `n - 1`, which is how the +authors' `TsQaDataset` reads it. The HDF5 file has no attributes and names no channel. The authors' +channel audit +([`docs/ISSUE_23_CHANNEL_AUDIT.md`](https://github.com/Pandalin98/ITFormer-ICML25/blob/b80142f/docs/ISSUE_23_CHANNEL_AUDIT.md)) +says not to name the 33 channels without a mapping, so the connector does not. + +The release does not say what a row is. We read about 400 rows over HTTP and found: + +- A row is a window of one flight. Consecutive ids are consecutive windows: the last step of a row + joins the first step of the next one. So a row of 600 steps at 1 Hz is the paper's "10-minute (600 + points) time-series segment", and N-CMAPSS records its flights at 1 Hz. +- A flight is 10 to 21 consecutive rows. When a flight ends inside a row, the release fills the rest + of that row with zeros in all 33 channels. In 171 consecutive rows, 11 rows ended this way, one per + flight. +- The release does not mark where a flight starts. The zeros at the end of a flight are the only + sign of a flight boundary. +- The values look standardized, with statistics that the release does not give. Channel 32 is + constant within each row, and its values do not follow the health answers. + +The QA files name only 7,755 of the 118,921 rows. The single-row lines name 5,945 rows. The 8,845 +distinct lists of the multi-row lines use 1,900 rows, and 90 rows occur in both. No question names +the other rows. + +### QA + +Each line has an `id`, a `name` and a `conversations` list of `human` questions. A `gpt` answer +follows each question. A pair is one question and its answer. Each question has a `stage`, 1 to 4, +and an `attribute`, `open` or `close`. A line has one of two shapes: + +| Line | `id` | Pairs | +| --- | --- | --- | +| Single-row | one row id | 8 Understanding, then 7 Perception | +| Multi-row | a list of 10 row ids | 2 Reasoning, then 1 Decision-Making | + +Every question holds one `` marker where the series goes. + +### Train and test files + +The train file is the paper's training set, followed by copies of most test lines. The copies are +one block at the end of the train file. A labelled pair is a pair whose answer is not empty (see +[Empty answers](#empty-answers)). Line numbers count from 1: + +| Part | Lines | Pairs | Labelled pairs | +| --- | --- | --- | --- | +| `train_qa.jsonl` lines 1 to 9,252: the paper's training set | 9,252 | 73,248 | 72,341 | +| `train_qa.jsonl` lines 9,253 to 13,291: copies of test lines | 4,039 | 28,317 | 27,955 | +| `test_qa.jsonl`, lines that the train file copies | 4,039 | 28,317 | 27,955 | +| `test_qa.jsonl`, lines only in the test file | 1,500 | 14,160 | 14,022 | + +The copies are not a random sample of the test file. They hold 2,689 of the 3,384 multi-row test +lines and 1,350 of the 2,155 single-row test lines. [Source issues](#source-issues) gives the +counts per stage and compares them with the paper. + +### The four tasks + +The paper defines the tasks in Section 5.1 and Appendix C: + +| Stage | Task | Input | Format | Authors' metric | +| --- | --- | --- | --- | --- | +| 1 | Understanding | 1 row | open | BLEU, ROUGE | +| 2 | Perception | 1 row | multiple choice | accuracy, F1 | +| 3 | Reasoning | 10 rows | multiple choice | accuracy, F1 | +| 4 | Decision-Making | 10 rows | open | BLEU, ROUGE | + +Understanding asks what a change in one named signal means, such as the altitude or the total +temperature at the LPT outlet. The closed questions use fixed wording. There are nine, and the +release prints each one with one option list: + +| Stage | Question | Options | +| --- | --- | --- | +| 2 | Health status of the fan, LPC, HPC, HPT, LPT (5 questions) | a: Health, b: Fault | +| 2 | Fault cause | a to f: HPT, LPT, Fan efficiency modifier, then HPT, LPT, Fan flow modifier | +| 2 | Potential health state | a: Operating Normally, b: Abnormal Degradation, c: Imminent Failure | +| 3 | Qualitative condition of the engine | a to e: Good, Moderate, Poor, Bad, Extremely Poor Condition | +| 3 | Probability range of engine failure | a to e: 1%-10%, 10%-30%, 30%-50%, 50%-70%, 70%-100% | + +An answer is one letter. The fault cause is the exception. It can hold several letters joined by +`", "`, such as `a, b, e`. + +### The multi-row questions + +The paper says that Reasoning and Decision-Making look at "10 consecutive cycles" to judge the +long-term health of the engine. In the one list that we traced, the 10 rows are windows in the middle +of 9 different flights, over about one day of flying. The ids of a list span a median of 145 rows. + +The authors' code does not give the model the 10 whole rows. It takes the first 60 steps of each row +and joins them into one (600, 33) array (`TsQaDataset.__getitem__` at `b80142f`). The paper does not +describe this cut. It describes encoding each segment separately. + +The three answers of a multi-row line carry about one label. The condition grade sets it. Grade e +never occurs. The failure probability follows the grade: a goes with a or b, b with b, c with c or e, +and d with d. The Decision-Making answer is maintenance advice in varied wording. Its content +follows the grade: preventive maintenance at grade a, urgent repair or replacement at grade d. + +## The TimeNet dataset + +### Records + +The dataset holds two kinds of Record. Both have one Source, `N-CMAPSS engine`, with 33 float64 +signals, `channel_00` to `channel_32`, in `seq_data` column order. The values are the published +values, unchanged. They have no unit. + +- A row Record holds one row that a labelled single-row pair names: 5,945 of the 118,921 rows. Its + signals are on a regular 1 Hz axis. If the flight ended inside the row, the signals stop at the last + real step. The zeros that the release added are not stored. Its metadata holds `row_id`, the row's + `data_ID`, and `padded_steps`, the number of zero steps left out. In the pinned release, 317 row + Records end in padding, with fewer than 500 padded steps each. +- A joined Record holds the input of one multi-row question: the first 60 steps of each of its 10 + rows, in published order, 600 steps in all. There is one for each distinct list of 10 ids, 8,845 + in all. The 10 blocks come from different flights, so the signals use an ordinal axis: the steps + have an order but no clock. If a flight ended within the first 60 steps of a row, the padded steps + are null, with the missing-value reason `alignment_padding`. In the pinned release, no joined Record + has a null. The metadata holds `row_ids` and `steps_per_row`, which is 60. + +A row that both files or several lines name is one Record, and their tasks share it. To pad a row +Record back to the authors' array, add `padded_steps` zero steps at its end. + +### Tasks + +Each published QA pair with a label becomes one task of its stage, with one input Record. Its split is +the split of its file: `train_qa.jsonl` is train and `test_qa.jsonl` is test. The tasks come in file +order, train file first. + +| Stage | TimeNet type | Input | Targets | +| --- | --- | --- | --- | +| Understanding | `AnswerTask` | row Record | the published answer | +| Perception | `ClassificationTask` | row Record | the published letters | +| Reasoning | `ClassificationTask` | joined Record | the published letter | +| Decision-Making | `AnswerTask` | joined Record | the published answer | + +That gives 142,273 tasks: 100,296 train and 41,977 test. + +The prompt is the published question, unchanged, with its `` marker and its options. The connector +adds no wording. + +A closed question is a classification over letters. It always offers the same options in the same +order, so a letter has one meaning per question. The authors' model answers with the letters, and +their `closed_question_metrics` scores the set of letters. So the targets are the published letters, +one item per letter, in published order, such as `("a", "b", "e")`. + +Each closed task has a label set: the allowed letters, with the option text of each letter as its +description. A `name` annotation names each label set. The five health-status questions share the +label set `component_health`. The other label sets are `fault_cause`, `health_state`, +`engine_condition` and `failure_probability`. Only `fault_cause` is multi-label. + +### Task metadata + +| Key | Value | +| --- | --- | +| `source_file`, `source_line`, `qa_offset` | where the pair is published. The line and offset are 0-based, as in the authors' evaluation | +| `stage` | the published stage, 1 to 4 | +| `closed_question` | the closed question, such as `hpc_health` or `fault_cause`. Closed tasks only | +| `source_issues` | a list of issue names, empty for most tasks. See [Source issues](#source-issues) | + +Error messages count lines from 1, as editors do. `source_line` counts from 0, as the authors do. + +### Dataset annotations + +The annotation `unlabelled_pairs` holds the count of closed pairs with an empty answer, 1,769. Its +metadata `pairs` lists the place of each one. See [Empty answers](#empty-answers). + +## Source issues + +The connector keeps every labelled pair as published. It names two problems in the task metadata +`source_issues`, with the same names as the TSAQA connector: + +| Issue | Meaning | Tasks | +| --- | --- | --- | +| `repeated_across_splits` | the other file has a pair with the same rows, question and answer | 27,961 train, 27,955 test | +| `repeated_question` | another pair in the same file has the same rows, question and answer | 12 train | + +4,039 of the 5,539 test lines are byte copies of a train line. The test file matches the per-stage +counts in the paper's Fig. 8 exactly. The train file holds the paper's train counts plus those 4,039 +copied lines: + +| Stage | Train file | Copied from test | Paper's train (Fig. 8) | +| --- | --- | --- | --- | +| 1 Understanding | 41,128 | 10,800 | 30,328 | +| 2 Perception | 35,987 | 9,450 | 26,537 | +| 3 Reasoning | 16,300 | 5,378 | 10,922 | +| 4 Decision-Making | 8,150 | 2,689 | 5,461 | + +These counts include the empty answers. So the release adds test lines to the paper's training set. +With the released train file, a model reaches a Reasoning test accuracy of 99.3% to 99.8%, as the +authors' issue [#34](https://github.com/Pandalin98/ITFormer-ICML25/issues/34) reports. The connector keeps the +released split. To train on the paper's split, leave out the train tasks whose `source_issues` holds +`repeated_across_splits`. + +## Empty answers + +1,769 closed answers are empty: the fault cause 1,054 times in train and 422 times in test, the +health state 215 times in train and 78 times in test. The connector treats an empty answer as a +missing label, and the pair gets no task. The authors' current metric raises an error on an empty +reference answer (`utils/metrics.py` at `b80142f`). + +An empty fault cause does not mean "no fault". The single-row ids fall into 12 contiguous blocks. In +each block, every row has the same fault cause, whether its components are answered healthy or not. +The blocks with an empty fault cause are those where only HPC, or LPC and HPC, are answered `b: +Fault`. None of the six options names an HPC or LPC modifier, so these rows have no correct option. +The empty health states have no such pattern. + +The answers also disagree in one place. In the blocks whose fault cause is `b, e` (LPT efficiency and +flow), the component answered `b: Fault` is the LPC, never the LPT. The connector keeps both answers +as published. + +## What the connector checks + +If the content is wrong, conversion fails. For a QA line, the message names the file and the line. +Conversion fails in these cases: + +- The signals file does not hold exactly the HDF5 datasets `seq_data` and `data_ID`, with `seq_data` + as float64 shaped (rows, 600, 33) and `data_ID` as the integers 1 to the number of rows. +- A QA file has another number of lines, pairs per stage or empty answers per question than the + pinned release. This catches a truncated or empty file. +- A QA line is not one JSON object with exactly the keys `id`, `name` and `conversations`, or a turn + has other keys, values or roles than the release uses. The validated JSON Lines reader raises + `TimeNetConnectorError` for these. +- A `name` does not repeat its `id`, an id is not a row of the file, or a list does not hold 10 + distinct ids. +- The conversation does not alternate a `human` question and a `gpt` answer. +- The stage, the `open` or `close` attribute and the number of rows disagree, or a question does not + hold one `` marker. +- A closed question is not one of the nine, or prints other options than its label set. Other options + change what a letter means. +- A closed answer is not distinct letters of its options, or holds several letters for a single-label + question. +- An open answer is empty. +- A named row holds only zeros, or a step of 33 zeros before its last real step. Padding is only at + the end of a row. + +Conversion reads the last step of each named row. If all 33 values are zero, it reads the whole row to +find where the padding starts. The other values stay in the HDF5 file until the writer reads them. +When a signal reads its rows, it can fail. If the file cannot be opened, a row cannot be read, or a +row's padding changed since conversion, it raises `TimeFFormatError`. If a row holds a NaN or an +infinite value, it raises `TimeFValidationError`. + +## How the connector follows the authors + +| The authors (code at `b80142f`) | The connector | +| --- | --- | +| `_build_index` flattens each line into one sample per pair | one task per labelled pair | +| `seq_data[int(id) - 1]` | the row Record of row id `id` | +| first 60 steps of each of 10 rows, joined | the joined Record of the 10 ids | +| SFT target: the published answer | the target: the answer, or its letters | +| `closed_question_metrics`: the set of letters a to f | a `ClassificationTask` over a label set of letters | +| train on `train_qa.jsonl`, evaluate on `test_qa.jsonl` | `Split.TRAIN`, `Split.TEST` | +| `source_line`, `qa_offset` | `source_line`, `qa_offset` in task metadata | + +The connector differs from the authors' input in one way: it does not store the padding zeros. A row +Record stops at the last real step. A joined Record has nulls where the authors' array has zeros. + +## Comparing with the paper and the card + +- Channels: the paper says 32 channels. The release has 33. The authors' audit cannot tell which + column is extra, so a dropped column is a guess. The connector keeps all 33. +- Pairs: the paper gives both 11k and 110k pairs, and the Hub card gives 87k train and 25k test + pairs. The files hold 101,565 train and 42,477 test pairs. See [Source issues](#source-issues) for + how they relate to Fig. 8. +- Reasoning: the paper says Reasoning predicts "the probability of future failures and the remaining + useful life" (Section 5.1). The release has no remaining-useful-life question. Reasoning asks for + the condition and the failure probability. +- Cycles: the paper calls a row a "cycle" and also a "10-minute (600 points)" segment. A row is a + window of a flight, not a whole flight. The paper says a multi-row question is about "10 + consecutive cycles". Every one of the 11,534 multi-row lists has gaps between its ids, and 256 of + them mix two ranges of ids. +- Multi-row input: the paper encodes each segment separately. The code joins the first 60 steps of + each row. The connector follows the code. + +## Data limitations + +- The train split holds copies of most of the test split. See [Source issues](#source-issues). +- The signals come from a simulator, are scaled, and have no channel names. The questions name + signals such as altitude, but no Record says which channel is which. +- The paper says an LLM rewrote the questions and answers, and experts reviewed them (Appendix A). The + Understanding and Decision-Making answers vary in wording for the same content. +- The fault cause describes the fault mode of a whole block of rows, not of one row. See + [Empty answers](#empty-answers). +- The paper does not say how the authors derived the closed answers from the N-CMAPSS health labels. + +## Open questions + +- Channel names: which N-CMAPSS variable each of the 33 channels holds, and which one the paper's 32 + leaves out. The authors promise a mapping in issue + [#9](https://github.com/Pandalin98/ITFormer-ICML25/issues/9). +- Flights: the release does not mark where a flight or an engine unit starts. + +## Local build + +```bash +uv run timenet-build build pandalin98/enginemt-qa --local-registry .timenet-registry +``` + +The build downloads 18.8 GB. After the download, it takes about two minutes and writes 1.47 GB. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py new file mode 100644 index 000000000..0f217e8c8 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/__init__.py @@ -0,0 +1,6 @@ +"""EngineMT-QA: aero-engine question answering over simulated N-CMAPSS cycles, in four tasks.""" + +from timenet_connectors.datasets.pandalin98.enginemt_qa.connector import ( + CONNECTOR as CONNECTOR, + EngineMTQAConnector as EngineMTQAConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py new file mode 100644 index 000000000..ef68fc9e6 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py @@ -0,0 +1,916 @@ +"""Convert the pinned EngineMT-QA files into lazy Records and one task per labelled QA pair. + +A single-row question gets the Record of its row. A multi-row question gets one Record that joins the +first 60 steps of its 10 rows, as the authors' model reads them. README.md explains the choices. +""" + +from collections import Counter +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from enum import IntEnum, StrEnum, unique +from functools import lru_cache, partial +import hashlib +import json +import logging +from pathlib import Path +import re +import string +from typing import Any, ClassVar, Literal, NamedTuple + +import numpy as np +import pyarrow as pa +from pydantic import BaseModel, ConfigDict, Field + +from timenet.dataset import OrdinalAxis, Record, RegularAxis, Signal, Source, TimeFDataset +from timenet.errors import TimeFFormatError, TimeFValidationError +from timenet.json import JsonValue +from timenet.types import ( + Annotation, + AnswerTask, + ClassificationTask, + Label, + LabelSet, + MissingReason, + MissingValues, + Split, + Task, + TimeSeriesSpec, +) +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector +from timenet_connectors.sources.readers import DataRow, JsonlDataReader + + +_LOGGER = logging.getLogger(__name__) + +_N_STEPS = 600 +_N_CHANNELS = 33 +"""The released channel count. The paper says 32, but the published file holds 33 and names none of them. + +See docs/ISSUE_23_CHANNEL_AUDIT.md in the ITFormer repository. +""" +_MULTI_ROWS = 10 +"""How many rows a multi-row question names.""" +_STEPS_PER_ROW = _N_STEPS // _MULTI_ROWS +"""The steps of each row that the authors' code joins for a multi-row question: the first 60 of 600.""" +_SAMPLE_RATE_HZ = 1 +"""Consecutive rows continue each other, and the paper calls a row a "10-minute (600 points)" segment.""" +_ROW_SHAPE = (_N_STEPS, _N_CHANNELS) +_SEQ_DATA = "seq_data" +"""The HDF5 dataset of every row's values, shaped ``(rows, steps, channels)``.""" +_DATA_ID = "data_ID" +"""The HDF5 dataset of each row's id, which a QA's ``id`` names. Row ``i`` has id ``i + 1``.""" +_SOURCE_NAME = "N-CMAPSS engine" +_CHANNELS = tuple(f"channel_{index:02d}" for index in range(_N_CHANNELS)) +"""The release names no channel. The zero-padded names sort in the source's order.""" +_ROW_SPEC = TimeSeriesSpec( + spec_type="enginemt_qa_channel", name="EngineMT-QA channel, as published", unit_value=None, dtype="float64" +) +_JOINED_SPEC = TimeSeriesSpec( + spec_type="enginemt_qa_joined_channel", + name="EngineMT-QA channel, first 60 steps of 10 rows", + unit_value=None, + dtype="float64", + missing=MissingValues( + reason=MissingReason.ALIGNMENT_PADDING, + description="The flight ended within the first 60 steps of a row, and the release filled the rest with zeros.", + ), +) +_ROW_AXIS = RegularAxis.from_rate_hz(_SAMPLE_RATE_HZ) +_JOINED_AXIS = OrdinalAxis() +"""The 10 blocks come from different times, so the joined steps have an order but no clock.""" +# A published row id: a decimal number with no sign and no leading zero, such as "278". +_ROW_ID = re.compile(r"[1-9][0-9]*") +# One printed option of a closed question: its letter, then its text, such as "a: Health\n". +_OPTION = re.compile(r"([a-z]):\s+([^\n]+)\n") +_DIGEST_BYTES = 16 +"""The size of a pair digest, which makes an accidental match between two of the 144,042 pairs practically +impossible.""" + + +@unique +class _Key(StrEnum): + """The keys of the annotations and the metadata that the connector writes.""" + + ROW_ID = "row_id" + ROW_IDS = "row_ids" + PADDED_STEPS = "padded_steps" + STEPS_PER_ROW = "steps_per_row" + SOURCE_FILE = "source_file" + SOURCE_LINE = "source_line" + QA_OFFSET = "qa_offset" + STAGE = "stage" + CLOSED_QUESTION = "closed_question" + SOURCE_ISSUES = "source_issues" + UNLABELLED_PAIRS = "unlabelled_pairs" + + +@unique +class SourceIssue(StrEnum): + """A problem in a published pair, which the connector keeps and names in the task's metadata. + + The names match the TSAQA connector's, so one filter can serve both datasets. + """ + + REPEATED_QUESTION = "repeated_question" + """Another pair of the same file has the same rows, question and answer.""" + REPEATED_ACROSS_SPLITS = "repeated_across_splits" + """The other file has a pair with the same rows, question and answer.""" + + +@unique +class _Part(StrEnum): + """The release's three files: the signals, and the train and test QA.""" + + SIGNALS = "signals" + TRAIN = "train" + TEST = "test" + + @property + def source_file(self) -> str: + """The file's name in the Hub repository, such as ``train_qa.jsonl``.""" + return "time_series_data.h5" if self is _Part.SIGNALS else f"{self.value}_qa.jsonl" + + @property + def task_split(self) -> Split: + """The split of a QA file: the authors train on ``train_qa`` and test on ``test_qa``. + + Raises: + TimeFValidationError: If the part is not a QA file. + """ + if self not in _SPLITS: + raise TimeFValidationError(f"{self} is not a QA file") + return _SPLITS[self] + + +_SPLITS = {_Part.TRAIN: Split.TRAIN, _Part.TEST: Split.TEST} +_QA_PARTS = tuple(_SPLITS) + + +@unique +class _Stage(IntEnum): + """The release's ``stage``: one of the paper's four tasks.""" + + UNDERSTANDING = 1 + PERCEPTION = 2 + REASONING = 3 + DECISION_MAKING = 4 + + @property + def is_closed(self) -> bool: + """Whether the stage asks closed questions, which the release marks ``close``.""" + return self in {_Stage.PERCEPTION, _Stage.REASONING} + + @property + def is_multi_row(self) -> bool: + """Whether the stage's questions name 10 rows rather than one.""" + return self in {_Stage.REASONING, _Stage.DECISION_MAKING} + + +def _option_letters(name: str, options: Sequence[str], *, multi_label: bool = False) -> LabelSet: + """Build the label set of a closed question: one label per option letter from ``a``, described by its option. + + Returns: + The label set. + """ + labels = [ + Label(name=letter, description=option) + for letter, option in zip(string.ascii_lowercase[: len(options)], options, strict=True) + ] + return LabelSet(labels, multi_label=multi_label, annotations=[Annotation("name", value=name)]) + + +_COMPONENT_HEALTH = _option_letters("component_health", ("Health", "Fault")) +_FAULT_CAUSE = _option_letters( + "fault_cause", + ( + "HPT efficiency modifier", + "LPT efficiency modifier", + "Fan efficiency modifier", + "HPT flow modifier", + "LPT flow modifier", + "Fan flow modifier", + ), + # The authors' answers list every cause that applies, such as "a, b, e". + multi_label=True, +) +_HEALTH_STATE = _option_letters("health_state", ("Operating Normally", "Abnormal Degradation", "Imminent Failure")) +_ENGINE_CONDITION = _option_letters( + "engine_condition", + ("Good Condition", "Moderate Condition", "Poor Condition", "Bad Condition", "Extremely Poor Condition"), +) +_FAILURE_PROBABILITY = _option_letters("failure_probability", ("1%-10%", "10%-30%", "30%-50%", "50%-70%", "70%-100%")) + + +class _ClosedSpec(NamedTuple): + stage: _Stage + label_set: LabelSet + stem: str + """The question's text up to its ``?``. The release publishes each closed question with one stem.""" + + +_REASONING_STEM = ( + "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different " + "health states and performing temporal inference to predict the health decline trend, " +) +_ONE_CYCLE = "of the given engine signal in one cycle" + + +def _prints_options(options: str, label_set: LabelSet) -> bool: + """Tell whether a question prints exactly the label set's letters and descriptions, in order. + + Returns: + ``True`` if the printed options are the label set's. + """ + printed = _OPTION.findall(options) + return printed == [(label.name, label.description) for label in label_set.labels] + + +@unique +class _ClosedQuestion(StrEnum): + """A closed question, which the release always publishes with the same stem and options.""" + + FAN_HEALTH = "fan_health" + LPC_HEALTH = "lpc_health" + HPC_HEALTH = "hpc_health" + HPT_HEALTH = "hpt_health" + LPT_HEALTH = "lpt_health" + FAULT_CAUSE = "fault_cause" + HEALTH_STATE = "health_state" + ENGINE_CONDITION = "engine_condition" + FAILURE_PROBABILITY = "failure_probability" + + @property + def stage(self) -> _Stage: + """The question's stage.""" + return _CLOSED[self].stage + + @property + def label_set(self) -> LabelSet: + """The question's option letters.""" + return _CLOSED[self].label_set + + @classmethod + def of(cls, stage: _Stage, text: str) -> "_ClosedQuestion": + """Identify a published closed question by its stem, and check that it prints its label set's options. + + Returns: + The question. + + Raises: + ValueError: If no closed question of the stage has this stem, or it prints other options. + """ + stem, _, options = text.partition("?") + question = _BY_STEM.get(stem) + if question is None or question.stage is not stage: + raise ValueError(f"unknown stage {stage} closed question {text[:80]!r}") + if not _prints_options(options, question.label_set): + raise ValueError(f"{question} prints options other than its label set's: {options!r}") + return question + + def parse_answer(self, answer: str) -> tuple[str, ...]: + """Parse a published answer, such as ``a`` or ``a, b, e``, into its letters in published order. + + Returns: + The letters. + + Raises: + ValueError: If the answer is not distinct option letters joined by ``", "``, or has too many. + """ + letters = tuple(answer.split(", ")) + try: + self.label_set.check_targets(letters) + except TimeFValidationError as exc: + raise ValueError(f"{self}: answer {answer!r}: {exc}") from exc + return letters + + +def _health_spec(component: str) -> _ClosedSpec: + return _ClosedSpec(_Stage.PERCEPTION, _COMPONENT_HEALTH, f"What is the health status of {component} {_ONE_CYCLE}") + + +_CLOSED = { + _ClosedQuestion.FAN_HEALTH: _health_spec("fan"), + _ClosedQuestion.LPC_HEALTH: _health_spec("LPC"), + _ClosedQuestion.HPC_HEALTH: _health_spec("HPC"), + _ClosedQuestion.HPT_HEALTH: _health_spec("HPT"), + _ClosedQuestion.LPT_HEALTH: _health_spec("LPT"), + _ClosedQuestion.FAULT_CAUSE: _ClosedSpec( + _Stage.PERCEPTION, + _FAULT_CAUSE, + f"Was the timing issue in the operation {_ONE_CYCLE} caused by efficiency or flow problems", + ), + _ClosedQuestion.HEALTH_STATE: _ClosedSpec( + _Stage.PERCEPTION, _HEALTH_STATE, f"What is the potential health state {_ONE_CYCLE}" + ), + _ClosedQuestion.ENGINE_CONDITION: _ClosedSpec( + _Stage.REASONING, _ENGINE_CONDITION, _REASONING_STEM + "what is the qualitative condition of the engine" + ), + _ClosedQuestion.FAILURE_PROBABILITY: _ClosedSpec( + _Stage.REASONING, _FAILURE_PROBABILITY, _REASONING_STEM + "what is the probability range of engine failure" + ), +} +_BY_STEM = {spec.stem: question for question, spec in _CLOSED.items()} + + +class _FileCounts(NamedTuple): + """The pinned size of a QA file, which conversion checks so that a short or changed file fails.""" + + lines: int + pairs: Mapping[_Stage, int] + """The pairs of each stage, empty answers included.""" + unlabelled: Mapping[_ClosedQuestion, int] + """The closed pairs with an empty answer, for each question that has any.""" + + +class _Place(NamedTuple): + """Where a pair was published.""" + + part: _Part + line: int + """The 0-based line in the file, which the authors call ``source_line``.""" + offset: int + """The 0-based place in the line, which the authors call ``qa_offset``.""" + + @property + def metadata(self) -> dict[str, JsonValue]: + """The place as the authors' evaluation names it.""" + return {_Key.SOURCE_FILE: self.part.source_file, _Key.SOURCE_LINE: self.line, _Key.QA_OFFSET: self.offset} + + +@dataclass(frozen=True) +class _Pair: + """One published question with its answer, and the rows it is about.""" + + place: _Place + rows: tuple[int, ...] + """The row ids in published order: one for a single-row question and ten for a multi-row one.""" + stage: _Stage + question: str + """The question as published, with its ```` marker.""" + answer: str + """The answer as published: free text, or letters for a closed question.""" + closed: _ClosedQuestion | None + """The closed question, or ``None`` for an open one.""" + + @property + def missing_label(self) -> _ClosedQuestion | None: + """The closed question, if its published answer is empty. Such a pair is a missing label and has no task.""" + return None if self.answer else self.closed + + @property + def has_label(self) -> bool: + """Whether the pair has a task.""" + return self.missing_label is None + + @property + def digest(self) -> bytes: + """A digest of the rows, the question and the answer, which match for a repeated pair.""" + content = json.dumps([self.rows, self.question, self.answer]).encode() + return hashlib.blake2b(content, digest_size=_DIGEST_BYTES).digest() + + def task(self, record: Record, issues: Sequence[SourceIssue]) -> AnswerTask | ClassificationTask: + """Build the task of a labelled pair over the Record of its rows. + + Returns: + An answer task with the published answer for an open question, or a classification task with the + published letters for a closed one. + """ + issue_names: list[JsonValue] = [issue.value for issue in issues] + metadata = self.place.metadata | {_Key.STAGE: self.stage.value, _Key.SOURCE_ISSUES: issue_names} + split = self.place.part.task_split + if self.closed is None: + return AnswerTask( + inputs=(record,), prompt=self.question, targets=(self.answer,), split=split, metadata=metadata + ) + return ClassificationTask( + inputs=(record,), + prompt=self.question, + targets=self.closed.parse_answer(self.answer), + label_set=self.closed.label_set, + split=split, + metadata=metadata | {_Key.CLOSED_QUESTION: self.closed.value}, + ) + + +class _Turn(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, frozen=True) + + +class _QuestionTurn(_Turn): + role: Literal["human"] = Field(alias="from") + stage: Literal["1", "2", "3", "4"] + attribute: Literal["open", "close"] + value: str + + +class _AnswerTurn(_Turn): + role: Literal["gpt"] = Field(alias="from") + value: str + + +class _Line(DataRow): + """One published QA line: the rows, their file names, and the alternating question and answer turns.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + id: str | list[str] + name: str | list[str] + conversations: list[_QuestionTurn | _AnswerTurn] + + +def _row_ids(line: _Line, n_rows: int) -> tuple[int, ...]: + """Read the row ids of a line, and check that its ``name`` repeats them. + + Returns: + One id for a single-row line, ten for a multi-row line, in published order. + + Raises: + ValueError: If an id is not a row of the file, or the ids repeat or are not one or ten. + """ + is_list = isinstance(line.id, list) + ids = line.id if isinstance(line.id, list) else [line.id] + names = line.name if isinstance(line.name, list) else [line.name] + if isinstance(line.name, list) != is_list or names != [f"{i}.csv" for i in ids]: + raise ValueError(f"name {line.name!r} doesn't repeat id {line.id!r}") + if not all(_ROW_ID.fullmatch(i) for i in ids): + raise ValueError(f"id {line.id!r} is not decimal row ids") + rows = tuple(int(i) for i in ids) + if is_list and len(rows) != _MULTI_ROWS: + raise ValueError(f"a multi-row id names {len(rows)} rows, not {_MULTI_ROWS}") + if len(set(rows)) != len(rows): + raise ValueError(f"id {line.id!r} repeats a row") + if max(rows) > n_rows: + raise ValueError(f"id {line.id!r} names a row past the file's {n_rows} rows") + return rows + + +def _parse_pair(place: _Place, rows: tuple[int, ...], question: object, answer: object) -> _Pair: + """Check one question and its answer against its stage, and build the pair. + + Returns: + The pair. + + Raises: + ValueError: If the turns don't alternate, the stage, format and rows disagree, or an answer is malformed. + """ + if not isinstance(question, _QuestionTurn) or not isinstance(answer, _AnswerTurn): + raise ValueError("conversations must alternate a human question and a gpt answer") + stage = _Stage(int(question.stage)) + if (question.attribute == "close") != stage.is_closed: + raise ValueError(f"stage {stage} questions are not {question.attribute!r}") + if (len(rows) > 1) != stage.is_multi_row: + raise ValueError(f"a stage {stage} question names {len(rows)} rows") + text, reply = question.value, answer.value + if not text.strip() or text.count("") != 1: + raise ValueError("a question must hold one marker") + closed = _ClosedQuestion.of(stage, text) if stage.is_closed else None + if closed is None and not reply.strip(): + raise ValueError("an open question has an empty answer") + if closed is not None and reply: + # Parse here, so a malformed answer fails the read and not the task stream. + closed.parse_answer(reply) + return _Pair(place, rows, stage, text, reply, closed) + + +@dataclass(frozen=True) +class _QAFile: + """One QA file and the number of rows its ids may name.""" + + path: Path + part: _Part + n_rows: int + + def lines(self) -> Iterator[tuple[int, list[_Pair]]]: + """Read every line of the file and flatten it into pairs, in file order. + + Yields: + The 0-based line number and the line's pairs, empty answers included. + + Raises: + TimeFValidationError: If a line breaks a rule of the release, with its file, line and pair. + """ + for number, line in enumerate(JsonlDataReader(self.path, _Line).iter_rows()): + try: + yield number, self._line_pairs(number, line) + except ValueError as exc: + raise TimeFValidationError(f"{self.part.source_file} line {number + 1}: {exc}") from exc + + def _line_pairs(self, number: int, line: _Line) -> list[_Pair]: + """Split a line into its pairs. + + Returns: + The pairs, in published order. + + Raises: + ValueError: If the line breaks a rule of the release. The message names the pair, if any. + """ + rows = _row_ids(line, self.n_rows) + turns = line.conversations + if not turns or len(turns) % 2: + raise ValueError("conversations must alternate a human question and a gpt answer") + pairs = [] + for offset in range(len(turns) // 2): + try: + pairs.append(_parse_pair(_Place(self.part, number, offset), rows, *turns[2 * offset : 2 * offset + 2])) + except ValueError as exc: + raise ValueError(f"pair {offset}: {exc}") from exc + return pairs + + def pairs(self) -> Iterator[_Pair]: + """Yield every pair of the file, in file order.""" + for _, pairs in self.lines(): + yield from pairs + + +def _import_h5py() -> Any: + """Import ``h5py`` when a file is read, so the connector can load without it. + + Returns: + The ``h5py`` module. + + Raises: + ImportError: If ``h5py``, declared in this connector's requirements, is not installed. + """ + try: + import h5py # ruff: ignore[import-outside-top-level] + except ImportError as exc: + raise ImportError("reading EngineMT-QA needs h5py, declared in this connector's requirements.txt") from exc + return h5py + + +def _open_signals(path: Path) -> Any: + """Open the signals file for reading, as a context manager. + + Returns: + The open ``h5py.File``. + + Raises: + TimeFFormatError: If the file cannot be opened as HDF5, as with a truncated download. + """ + h5py = _import_h5py() + try: + return h5py.File(path, "r") + except OSError as exc: + raise TimeFFormatError(f"{path}: cannot open the signals file: {exc}") from exc + + +def _check_layout(path: Path) -> int: + """Check that the signals file holds exactly ``seq_data`` and ``data_ID``, with the ids ``1..n``. + + Returns: + The number of rows ``n``. + + Raises: + TimeFValidationError: If the layout, the shape, the dtype or the ids differ from the published file's. + TimeFFormatError: If the file cannot be opened or read as HDF5. + """ + h5py = _import_h5py() + with _open_signals(path) as file: + if set(file) != {_SEQ_DATA, _DATA_ID}: + raise TimeFValidationError(f"{path}: expected {_SEQ_DATA} and {_DATA_ID}, got {sorted(file)}") + values, ids = file[_SEQ_DATA], file[_DATA_ID] + for name, node in ((_SEQ_DATA, values), (_DATA_ID, ids)): + if not isinstance(node, h5py.Dataset): + raise TimeFValidationError(f"{path}: {name} is a {type(node).__name__}, not an HDF5 dataset") + if values.shape[1:] != _ROW_SHAPE or values.dtype != np.float64: + raise TimeFValidationError(f"{path}: {_SEQ_DATA} is {values.dtype} {values.shape}") + n_rows = values.shape[0] + if ids.shape != (n_rows,) or not np.issubdtype(ids.dtype, np.integer): + raise TimeFValidationError(f"{path}: {_DATA_ID} is {ids.dtype} {ids.shape}, not {n_rows} integer ids") + try: + published_ids = ids[:] + except OSError as exc: + raise TimeFFormatError(f"{path}: cannot read {_DATA_ID}: {exc}") from exc + if not np.array_equal(published_ids, np.arange(1, n_rows + 1)): + raise TimeFValidationError(f"{path}: {_DATA_ID} is not the row ids 1 to {n_rows}") + return n_rows + + +def _read(file: Any, path: Path, row_id: int, steps: slice) -> np.ndarray: + """Read steps of one row from an open signals file, and check that they are finite. + + Returns: + The values, shaped ``(steps, 33)``. + + Raises: + TimeFFormatError: If the file no longer holds the row, or a block of it cannot be read. + TimeFValidationError: If a value is NaN or infinite. + """ + try: + values = np.asarray(file[_SEQ_DATA][row_id - 1, steps], dtype=np.float64) + except (KeyError, IndexError, ValueError, OSError) as exc: + raise TimeFFormatError(f"{path}: cannot read row id {row_id}: {exc}") from exc + if values.ndim != len(_ROW_SHAPE) or values.shape[1:] != _ROW_SHAPE[1:]: + raise TimeFFormatError(f"{path}: row id {row_id} is shaped {values.shape}; the file changed") + if not np.isfinite(values).all(): + raise TimeFValidationError(f"{path}: row id {row_id} holds a NaN or an infinite value") + return values + + +def _real_steps(values: np.ndarray, where: str) -> int: + """Count the steps before a row's padding: the release fills the end of a flight's last row with zeros. + + Returns: + The number of steps before the trailing steps whose 33 values are all exactly zero. + + Raises: + TimeFValidationError: If the row holds only zeros, or an all-zero step before its last real step. + """ + real = ~(values == 0).all(axis=1) + if not real.any(): + raise TimeFValidationError(f"{where} holds only zeros") + n_real = int(np.flatnonzero(real)[-1]) + 1 + if not real[:n_real].all(): + raise TimeFValidationError(f"{where} has a step of 33 zeros before its last real step") + return n_real + + +def _row_lengths(path: Path, rows: Iterable[int]) -> dict[int, int]: + """Find how many real steps each row has, reading a whole row only when its last step is all zeros. + + Returns: + The number of real steps of each row. + """ + lengths = {} + with _open_signals(path) as file: + for row in sorted(rows): + last = _read(file, path, row, slice(_N_STEPS - 1, _N_STEPS)) + if (last == 0).all(): + lengths[row] = _real_steps(_read(file, path, row, slice(None)), f"{path}: row id {row}") + else: + lengths[row] = _N_STEPS + return lengths + + +class _RowReader: + """Read the real steps of one row when a signal of its Record asks for them. + + The writer loads a Record's signals one after another, so caching the last row reads each row once. + """ + + def __init__(self, path: Path, lengths: Mapping[int, int]) -> None: + self._path = path + self._lengths = lengths + self._row = lru_cache(maxsize=1)(self._read) + + def _read(self, row_id: int) -> np.ndarray: + with _open_signals(self._path) as file: + values = _read(file, self._path, row_id, slice(None)) + if values.shape != _ROW_SHAPE: + raise TimeFFormatError(f"{self._path}: row id {row_id} is shaped {values.shape}; the file changed") + n_real = _real_steps(values, f"{self._path}: row id {row_id}") + if n_real != self._lengths[row_id]: + raise TimeFFormatError(f"{self._path}: row id {row_id} has {n_real} real steps; the file changed") + return values[:n_real] + + def __call__(self, row_id: int, channel: int) -> pa.Array: + """Return one channel of a row's real steps. + + Returns: + The channel's float64 values. + """ + return pa.array(self._row(row_id)[:, channel], type=pa.float64()) + + +class _JoinedReader: + """Join the first 60 steps of 10 rows when a signal of their Record asks for them. + + A step that is padding in its row becomes null. The cache reads each group of rows once for all 33 channels. + """ + + def __init__(self, path: Path, lengths: Mapping[int, int]) -> None: + self._path = path + self._lengths = lengths + self._joined = lru_cache(maxsize=1)(self._read) + + def _read(self, rows: tuple[int, ...]) -> tuple[np.ndarray, np.ndarray]: + blocks = [] + with _open_signals(self._path) as file: + for row in rows: + values = _read(file, self._path, row, slice(0, _STEPS_PER_ROW)) + n_real = min(self._lengths[row], _STEPS_PER_ROW) + where = f"{self._path}: row id {row}" + if n_real < _STEPS_PER_ROW and not (values[n_real:] == 0).all(): + raise TimeFFormatError(f"{where} has values after its padding started; the file changed") + if n_real != _real_steps(values[:n_real], where): + raise TimeFFormatError(f"{where} has padding before step {n_real}; the file changed") + blocks.append(values) + missing = np.concatenate([_block_missing(self._lengths[row]) for row in rows]) + return np.concatenate(blocks), missing + + def __call__(self, rows: tuple[int, ...], channel: int) -> pa.Array: + """Return one channel of the joined rows. + + Returns: + The channel's 600 float64 values, with a null for each padded step. + """ + values, missing = self._joined(rows) + return pa.array(values[:, channel], mask=missing, type=pa.float64()) + + +def _block_missing(length: int) -> np.ndarray: + """Mark the padded steps among the first 60 steps of a row with ``length`` real steps. + + Returns: + One flag per step, ``True`` for padding. + """ + return np.arange(_STEPS_PER_ROW) >= length + + +def _row_record(row_id: int, length: int, reader: _RowReader) -> Record: + """Build the Record of one row: 33 lazy signals over its real steps, at 1 Hz. + + Returns: + The Record, with the row id and the number of padded steps the connector left out as metadata. + """ + signals = tuple( + Signal.from_loader( + name=name, + spec=_ROW_SPEC, + time_axis=_ROW_AXIS, + n_values=length, + null_count=0, + loader=partial(reader, row_id, channel), + ) + for channel, name in enumerate(_CHANNELS) + ) + metadata: dict[str, JsonValue] = {_Key.ROW_ID: row_id, _Key.PADDED_STEPS: _N_STEPS - length} + return Record(sources=[Source(_SOURCE_NAME, signals=signals)], metadata=metadata) + + +def _joined_record(rows: tuple[int, ...], lengths: Mapping[int, int], reader: _JoinedReader) -> Record: + """Build the Record of a multi-row question: the first 60 steps of each of its 10 rows, joined in order. + + Returns: + The Record, with the row ids and the steps taken from each row as metadata. + """ + null_count = int(sum(_block_missing(lengths[row]).sum() for row in rows)) + signals = tuple( + Signal.from_loader( + name=name, + spec=_JOINED_SPEC, + time_axis=_JOINED_AXIS, + n_values=_N_STEPS, + null_count=null_count, + loader=partial(reader, rows, channel), + ) + for channel, name in enumerate(_CHANNELS) + ) + metadata: dict[str, JsonValue] = {_Key.ROW_IDS: list(rows), _Key.STEPS_PER_ROW: _STEPS_PER_ROW} + return Record(sources=[Source(_SOURCE_NAME, signals=signals)], metadata=metadata) + + +@dataclass +class _Scan: + """What one read of every QA pair finds.""" + + single_rows: set[int] = field(default_factory=set) + """The rows that labelled single-row pairs name.""" + row_groups: dict[tuple[int, ...], None] = field(default_factory=dict) + """The row lists that labelled multi-row pairs name, in first-seen order.""" + unlabelled: list[_Place] = field(default_factory=list) + """The places of the closed pairs with an empty answer.""" + digests: dict[_Part, Counter[bytes]] = field(default_factory=dict) + """How often each labelled pair's digest occurs in each file.""" + + def issues(self, pair: _Pair) -> list[SourceIssue]: + """Name the source issues of a labelled pair. + + Returns: + The issues, in ``SourceIssue`` order. + """ + digest = pair.digest + found = [] + if self.digests[pair.place.part][digest] > 1: + found.append(SourceIssue.REPEATED_QUESTION) + if any(self.digests[part][digest] for part in _QA_PARTS if part is not pair.place.part): + found.append(SourceIssue.REPEATED_ACROSS_SPLITS) + return found + + +def _scan(qa_files: Sequence[_QAFile], counts: Mapping[_Part, _FileCounts]) -> _Scan: + """Check every pair once, and collect what the dataset needs from all of them. + + Returns: + The rows that labelled pairs name, the unlabelled pairs, and the digests of the labelled pairs. + + Raises: + TimeFValidationError: If a file's counts differ from the pinned ones. + """ + scan = _Scan() + for qa in qa_files: + digests: Counter[bytes] = Counter() + pairs: Counter[_Stage] = Counter() + unlabelled: Counter[_ClosedQuestion] = Counter() + n_lines = 0 + for _, line_pairs in qa.lines(): + n_lines += 1 + for pair in line_pairs: + pairs[pair.stage] += 1 + question = pair.missing_label + if question is not None: + scan.unlabelled.append(pair.place) + unlabelled[question] += 1 + continue + digests[pair.digest] += 1 + if pair.stage.is_multi_row: + scan.row_groups[pair.rows] = None + else: + scan.single_rows.update(pair.rows) + found = _FileCounts(n_lines, dict(pairs), dict(unlabelled)) + if found != counts[qa.part]: + raise TimeFValidationError(f"{qa.path}: expected {counts[qa.part]}, found {found}") + scan.digests[qa.part] = digests + if scan.unlabelled: + _LOGGER.warning("%d closed pairs have an empty answer and get no task", len(scan.unlabelled)) + return scan + + +def _unlabelled_annotation(places: Sequence[_Place]) -> Annotation: + pairs: list[JsonValue] = [place.metadata for place in places] + return Annotation( + _Key.UNLABELLED_PAIRS, + value=len(places), + description="How many closed pairs have an empty published answer. They are missing labels and have no task.", + metadata={"pairs": pairs}, + ) + + +class EngineMTQAConnector( + BaseHuggingFaceConnector, + hf_repo="pandalin98/EngineMT-QA", + source_revision="ea2051a15512212976c22c1a075ec549c7d48eef", + source_files={part.value: part.source_file for part in _Part}, +): + """Build EngineMT-QA from the pinned files of its Hugging Face release.""" + + # Counted on the pinned files. They match the authors' README: 101,565 train and 42,477 test pairs. + SOURCE_COUNTS: ClassVar[Mapping[_Part, _FileCounts]] = { + _Part.TRAIN: _FileCounts( + lines=13_291, + pairs={ + _Stage.UNDERSTANDING: 41_128, + _Stage.PERCEPTION: 35_987, + _Stage.REASONING: 16_300, + _Stage.DECISION_MAKING: 8_150, + }, + unlabelled={_ClosedQuestion.FAULT_CAUSE: 1_054, _ClosedQuestion.HEALTH_STATE: 215}, + ), + _Part.TEST: _FileCounts( + lines=5_539, + pairs={ + _Stage.UNDERSTANDING: 17_240, + _Stage.PERCEPTION: 15_085, + _Stage.REASONING: 6_768, + _Stage.DECISION_MAKING: 3_384, + }, + unlabelled={_ClosedQuestion.FAULT_CAUSE: 422, _ClosedQuestion.HEALTH_STATE: 78}, + ), + } + + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: + """Convert the files into lazy Records and a stream of one task per labelled pair. + + Conversion checks the signals file's layout, every QA line, and each QA file's pinned counts. It reads + the last step of each named row to find padded rows. The other values stay in the HDF5 file until + written, so the files must stay available until then. + + Args: + files: The downloaded file of each part. + + Returns: + A dataset with the Records, the unlabelled pairs as an annotation, and a stream of every task. + + Raises: + TimeFValidationError: If a file is malformed or has other counts. + """ # ruff: ignore[docstring-extraneous-exception] - raised by _check_layout and _scan + signals_path = files[_Part.SIGNALS] + n_rows = _check_layout(signals_path) + qa_files = [_QAFile(files[part], part, n_rows) for part in _QA_PARTS] + scan = _scan(qa_files, self.SOURCE_COUNTS) + grouped_rows = {row for rows in scan.row_groups for row in rows} + lengths = _row_lengths(signals_path, scan.single_rows | grouped_rows) + row_reader, joined_reader = _RowReader(signals_path, lengths), _JoinedReader(signals_path, lengths) + records: dict[tuple[int, ...], Record] = { + (row,): _row_record(row, lengths[row], row_reader) for row in sorted(scan.single_rows) + } + records |= {rows: _joined_record(rows, lengths, joined_reader) for rows in scan.row_groups} + + dataset = TimeFDataset(metadata=self.metadata(), annotations=[_unlabelled_annotation(scan.unlabelled)]) + for record in records.values(): + dataset.add_record(record) + + def tasks() -> Iterator[Task]: + for qa in qa_files: + for pair in qa.pairs(): + if not pair.has_label: + continue + if pair.rows not in records: + raise TimeFFormatError(f"{qa.path}: {pair.place} names rows {pair.rows} without a Record") + yield pair.task(records[pair.rows], scan.issues(pair)) + + dataset.set_task_stream((AnswerTask, ClassificationTask), tasks) + return dataset + + +CONNECTOR = EngineMTQAConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml new file mode 100644 index 000000000..1b6d287fb --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# Dataset Card for the pandalin98/enginemt-qa connector: the single source for this dataset's metadata. +# +# Training convention: the tasks are the authors' four stages, one task per labelled QA pair, split as +# the authors' files are (train_qa.jsonl, test_qa.jsonl). Closed questions are classifications over +# their option letters. 1,769 closed pairs with an empty answer have no task. A single-row question +# reads its row at 1 Hz. A multi-row question reads the first 60 steps of its 10 rows, joined on an +# ordinal axis, as the authors' code does. The padding zeros at the end of a flight are not stored. +# The train file holds copies of 4,039 test lines. Their tasks name repeated_across_splits in their +# source_issues metadata. +yaml_schema_version: 1 +dataset_id: pandalin98/enginemt-qa +dataset_version: 1.0.0 +name: EngineMT-QA +description: >- + Aero-engine time-series question answering over simulated N-CMAPSS flights: 142,273 labelled + QA pairs (100,296 train, 41,977 test) in four tasks. Understanding and Perception ask about one + 10-minute window of 33 unnamed, scaled channels. Reasoning and Decision-Making ask about 10 + windows from successive flights. Perception and Reasoning are multiple choice. Understanding and + Decision-Making are open-ended answers written with LLMs and checked by experts. +license: MIT +source_url: https://huggingface.co/datasets/pandalin98/EngineMT-QA +source_revision: ea2051a15512212976c22c1a075ec549c7d48eef +citation: >- + Wang et al. ITFormer: Bridging Time Series and Natural Language for Multi-Modal QA with + Large-Scale Multitask Dataset. ICML 2025. https://arxiv.org/abs/2506.20093 +domains: + - industrial +tags: + - aero-engine + - n-cmapss + - simulated + - question-answering + - classification + - generated-supervision + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt new file mode 100644 index 000000000..ef2960923 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/requirements.txt @@ -0,0 +1,4 @@ +# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class. +huggingface_hub>=1.1 +# h5py reads the signals file, time_series_data.h5. +h5py>=3.10 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md new file mode 100644 index 000000000..182e6df12 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/README.md @@ -0,0 +1,25 @@ +# Test fixtures + +`fixtures/` holds real lines of the pinned release, copied byte for byte. The tests write the +signals file themselves, because the real one is 18.8 GB. Its values encode the row, the step and +the channel, so a misplaced value cannot pass. Two rows end in zero padding, as a flight's last row +does in the release: row 278 keeps 500 steps, and row 287 keeps 40, fewer than the 60 steps that a +multi-row question takes. + +| File | Release lines (0-based), in fixture order | +| --- | --- | +| `train_qa.jsonl` | 316, 92, 11228, 3503 | +| `test_qa.jsonl` | 2786, 456, 1082 | + +- Train line 316 (row 278) has an empty health state. +- Train line 92 (row 8435) has the fault cause `a, b, e`. +- Train line 11228 (row 20) is the same line as test line 2786. +- Train line 3503 and test line 456 are multi-row lines. They share five of their ten rows. +- Test line 1082 (row 37669) has an empty fault cause. + +To make a file again, download the release's QA file and copy the lines above, in this order: + +```python +lines = open("train_qa.jsonl", "rb").read().splitlines(keepends=True) +open("fixtures/train_qa.jsonl", "wb").write(b"".join(lines[n] for n in (316, 92, 11228, 3503))) +``` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl new file mode 100644 index 000000000..7ac9b44ae --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/test_qa.jsonl @@ -0,0 +1,3 @@ +{"id": "20", "name": "20.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal over one cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude reflects consistent engine performance, indicating that the engine operates efficiently without experiencing additional strain or requiring adjustments."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the specific representation of the change in Throttle Resolver Angle within the provided engine signal over the course of a single cycle?\n "}, {"from": "gpt", "value": "The Throttle Resolver Angle exhibits a steady and controlled increase. This signifies a deliberate and balanced augmentation of engine power, ensuring an optimal equilibrium between thrust generation and fuel consumption efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "So, what does the change in total temperature at the fan inlet tell us about the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature stays the same throughout the cycle. This shows that the engine is running smoothly without any ups and downs in temperature, which is a good sign of stable operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise representation and analytical interpretation of the change in total pressure at the fan inlet observed over the duration of a single cycle?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet is maintained at a constant level. This indicates the presence of stable airflow and consistent engine performance, with no significant pressure fluctuations occurring during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal during one cycle, what does the variation in Total Temperature at the LPC outlet signify?\n "}, {"from": "gpt", "value": "The LPC outlet temperature exhibits a slow and steady rise. This indicates a gradual adaptation in the compressor's workload, which contributes to maintaining stable and efficient engine performance."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPC outlet means for this engine signal we're analyzing in one cycle?\n "}, {"from": "gpt", "value": "Well, the HPC outlet temperature is increasing slowly over time. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the fuel is being used as efficiently as possible."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPT outlet means for the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "Well, the HPT outlet temperature is increasing slowly. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the engine is using fuel efficiently."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal during a single cycle, what does the variation in the Total Temperature at the LPT outlet signify?\n "}, {"from": "gpt", "value": "The LPT outlet temperature exhibits a gradual rise. This indicates that the engine is making incremental adjustments to its power output, which supports stable operational conditions and enhances fuel efficiency."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} +{"id": ["279", "287", "295", "304", "309", "319", "326", "334", "339", "346"], "name": ["279.csv", "287.csv", "295.csv", "304.csv", "309.csv", "319.csv", "326.csv", "334.csv", "339.csv", "346.csv"], "conversations": [{"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the qualitative condition of the engine? a: Good Condition\n b: Moderate Condition\n c: Poor Condition\n d: Bad Condition\n e: Extremely Poor Condition\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the probability range of engine failure? a: 1%-10%\n b: 10%-30%\n c: 30%-50%\n d: 50%-70%\n e: 70%-100%\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "4", "attribute": "open", "value": "With respect to the engine signal provided across a span of 10 cycles, what specific actions should be undertaken to ensure optimal performance and safety?\n "}, {"from": "gpt", "value": "It is imperative to closely monitor the engine's condition and proactively schedule maintenance in the near future to mitigate potential risks and maintain operational reliability."}]} +{"id": "37669", "name": "37669.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal over one operational cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude reflects consistent engine performance, ensuring that the engine operates smoothly without experiencing any additional strain or the need for adjustments."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise interpretation and operational significance of the observed change in the throttle resolver angle during a single cycle?\n "}, {"from": "gpt", "value": "The throttle resolver angle exhibits a slow and gradual increase. This indicates that the engine is undergoing incremental power adjustments, which facilitates smoother engine operation and ensures optimal fuel efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total temperature at fan inlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature appears to remain constant. This observation might indicate stable engine operation without significant temperature fluctuations."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal over a single cycle, what does the variation in total pressure at the fan inlet indicate?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet exhibits a slow decline. This gradual drop could indicate either a controlled reduction in throttle settings or minor inefficiencies in compressor performance. Despite these changes, the engine maintains stable operational performance throughout the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total temperature at LPC outlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The LPC outlet temperature appears to be rising slowly. This could reflect gradual adjustments in the compressor workload, which may contribute to promoting stable engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total temperature at HPC outlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The HPC outlet temperature appears to remain constant. This observation might suggest stable compressor performance, with no significant temperature fluctuations detected during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal over a single cycle, what does the variation in the Total Temperature at the High-Pressure Turbine (HPT) outlet signify?\n "}, {"from": "gpt", "value": "The HPT outlet temperature exhibits a slow and steady rise. This gradual increase indicates controlled adjustments in engine power output, which contributes to maintaining stable engine operation and ensures optimal fuel efficiency throughout the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the significance of the variation in Total temperature at the LPT outlet as it pertains to the engine signal provided within a single cycle?\n "}, {"from": "gpt", "value": "The LPT outlet temperature exhibits a gradual increase. This reflects incremental adjustments in engine power, which contribute to stable operation and the efficient utilization of fuel resources."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": ""}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl new file mode 100644 index 000000000..69b8967fd --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/fixtures/train_qa.jsonl @@ -0,0 +1,4 @@ +{"id": "278", "name": "278.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the precise interpretation of the fluctuation in altitude with regard to the engine signal that has been presented within the confines of a singular cycle?\n "}, {"from": "gpt", "value": "The altitude is observed to be dropping rapidly. A swift descent reduces the engine's workload; however, it may necessitate rapid adjustments to maintain stability and operational efficiency, which could potentially lead to increased wear on critical components over time."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Throttle resolver angle represent in the given engine signal over one cycle?\n "}, {"from": "gpt", "value": "The Throttle Resolver Angle appears to be decreasing slowly. This gradual reduction may suggest a controlled decrease in power, which could contribute to smoother engine operation and potentially support optimal fuel conservation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the precise interpretation of the alteration in the total temperature at the fan inlet with regard to the engine signal that has been presented within the confines of a singular cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature remains unchanged. This indicates stable engine operation, as there are no significant temperature fluctuations observed during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the precise interpretation of the variation in total pressure at the fan inlet with regard to the engine signal that has been presented within the confines of a singular cycle?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet remains unchanged. This indicates a stable airflow and consistent engine performance, with no significant pressure fluctuations observed."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal data over a single cycle, what does the variation in Total Temperature at the LPC outlet signify?\n "}, {"from": "gpt", "value": "The Total Temperature at the LPC outlet remains constant throughout the cycle. This stability indicates that the compressor is performing consistently, with no significant temperature fluctuations, which reflects reliable and efficient compressor operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPC outlet means in relation to the engine signal we're analyzing for this one cycle?\n "}, {"from": "gpt", "value": "Well, the HPC outlet temperature stays steady. That tells us the compressor is performing consistently without any ups and downs in temperature, which is a good sign for stable engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, could you explain what the change in Total Temperature at the High-Pressure Turbine (HPT) outlet signifies in the context of the engine signal we're analyzing for this single cycle?\n "}, {"from": "gpt", "value": "Well, the HPT outlet temperature stays constant throughout the cycle. This suggests that the turbine is performing steadily, with no significant temperature fluctuations, which is a good sign of stable operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise representation and analytical significance of the change in Total Temperature observed at the Low-Pressure Turbine (LPT) outlet during the duration of a single cycle?\n "}, {"from": "gpt", "value": "The Total Temperature at the LPT outlet is maintained at a constant level. This stability signifies consistent compressor performance, reflecting the absence of temperature fluctuations and ensuring reliable engine operation."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": ""}]} +{"id": "8435", "name": "8435.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal for a single cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude suggests that the engine is performing consistently without encountering any additional strain or operational challenges."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal, what is the precise interpretation and operational significance of the observed change in the throttle resolver angle over the duration of a single cycle?\n "}, {"from": "gpt", "value": "The throttle resolver angle exhibits a gradual and steady increase. This indicates that the engine is undergoing incremental power adjustments, which facilitate smooth and controlled engine performance while simultaneously optimizing fuel efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the significance of the variation in the total temperature at the fan inlet with respect to the engine signal provided within a single cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature remains unchanged. This indicates stable engine operation, demonstrating the absence of any temperature fluctuations during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "What does the change in Total pressure at fan inlet represent of the given engine signal in one cycle?\n "}, {"from": "gpt", "value": "The fan inlet pressure appears to remain constant. This could indicate stable airflow and consistent engine performance, suggesting the absence of significant pressure fluctuations during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the specific significance of the variation in the total temperature at the LPC outlet within the context of the provided engine signal over a single cycle?\n "}, {"from": "gpt", "value": "The LPC outlet temperature remains unchanged. This indicates stable compressor performance, with no significant temperature fluctuations observed."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Regarding the provided engine signal within a single cycle, what is the significance of the variation in the total temperature at the high-pressure compressor (HPC) outlet?\n "}, {"from": "gpt", "value": "The total temperature at the HPC outlet remains consistent. This stability signifies reliable compressor performance, devoid of any temperature fluctuations, ensuring optimal engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal, what is the precise interpretation and operational significance of the change in total temperature observed at the high-pressure turbine (HPT) outlet over the course of a single cycle?\n "}, {"from": "gpt", "value": "The total temperature at the HPT outlet is maintained at a constant level. This stability signifies consistent turbine performance, highlighting the absence of temperature fluctuations and ensuring reliable engine operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "With respect to the engine signal provided within a single cycle, what does the variation in the Total Temperature at the Low-Pressure Turbine (LPT) outlet signify?\n "}, {"from": "gpt", "value": "The temperature at the LPT outlet remains consistent. This indicates stable compressor performance, with no significant temperature fluctuations observed during the cycle."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a, b, e"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} +{"id": "20", "name": "20.csv", "conversations": [{"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal over one cycle, what does the change in altitude signify?\n "}, {"from": "gpt", "value": "The altitude remains constant throughout the cycle. This stable altitude reflects consistent engine performance, indicating that the engine operates efficiently without experiencing additional strain or requiring adjustments."}, {"from": "human", "stage": "1", "attribute": "open", "value": "May I inquire as to the specific representation of the change in Throttle Resolver Angle within the provided engine signal over the course of a single cycle?\n "}, {"from": "gpt", "value": "The Throttle Resolver Angle exhibits a steady and controlled increase. This signifies a deliberate and balanced augmentation of engine power, ensuring an optimal equilibrium between thrust generation and fuel consumption efficiency."}, {"from": "human", "stage": "1", "attribute": "open", "value": "So, what does the change in total temperature at the fan inlet tell us about the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "The fan inlet temperature stays the same throughout the cycle. This shows that the engine is running smoothly without any ups and downs in temperature, which is a good sign of stable operation."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the engine signal provided, what is the precise representation and analytical interpretation of the change in total pressure at the fan inlet observed over the duration of a single cycle?\n "}, {"from": "gpt", "value": "The total pressure at the fan inlet is maintained at a constant level. This indicates the presence of stable airflow and consistent engine performance, with no significant pressure fluctuations occurring during the cycle."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the given engine signal during one cycle, what does the variation in Total Temperature at the LPC outlet signify?\n "}, {"from": "gpt", "value": "The LPC outlet temperature exhibits a slow and steady rise. This indicates a gradual adaptation in the compressor's workload, which contributes to maintaining stable and efficient engine performance."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPC outlet means for this engine signal we're analyzing in one cycle?\n "}, {"from": "gpt", "value": "Well, the HPC outlet temperature is increasing slowly over time. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the fuel is being used as efficiently as possible."}, {"from": "human", "stage": "1", "attribute": "open", "value": "Hey, can you explain what the change in Total temperature at the HPT outlet means for the engine signal we're analyzing in this one cycle?\n "}, {"from": "gpt", "value": "Well, the HPT outlet temperature is increasing slowly. This indicates that the engine is making gradual adjustments to its power output, which helps maintain stable operation and ensures the engine is using fuel efficiently."}, {"from": "human", "stage": "1", "attribute": "open", "value": "In the context of the provided engine signal during a single cycle, what does the variation in the Total Temperature at the LPT outlet signify?\n "}, {"from": "gpt", "value": "The LPT outlet temperature exhibits a gradual rise. This indicates that the engine is making incremental adjustments to its power output, which supports stable operational conditions and enhances fuel efficiency."}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of fan of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPC of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of HPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the health status of LPT of the given engine signal in one cycle?\n a: Health\n b: Fault\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "Was the timing issue in the operation of the given engine signal in one cycle caused by efficiency or flow problems?\n a: HPT efficiency modifier\n b: LPT efficiency modifier\n c: Fan efficiency modifier\n d: HPT flow modifier\n e: LPT flow modifier\n f: Fan flow modifier\n"}, {"from": "gpt", "value": "a"}, {"from": "human", "stage": "2", "attribute": "close", "value": "What is the potential health state of the given engine signal in one cycle?\n a: Operating Normally\n b: Abnormal Degradation\n c: Imminent Failure\n"}, {"from": "gpt", "value": "a"}]} +{"id": ["279", "287", "295", "305", "310", "319", "327", "335", "340", "346"], "name": ["279.csv", "287.csv", "295.csv", "305.csv", "310.csv", "319.csv", "327.csv", "335.csv", "340.csv", "346.csv"], "conversations": [{"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the qualitative condition of the engine? a: Good Condition\n b: Moderate Condition\n c: Poor Condition\n d: Bad Condition\n e: Extremely Poor Condition\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "3", "attribute": "close", "value": "Given the time series signal , by perceiving the engine signal across 10 cycles to reflect different health states and performing temporal inference to predict the health decline trend, what is the probability range of engine failure? a: 1%-10%\n b: 10%-30%\n c: 30%-50%\n d: 50%-70%\n e: 70%-100%\n"}, {"from": "gpt", "value": "b"}, {"from": "human", "stage": "4", "attribute": "open", "value": "How should we approach the analysis and management of the provided engine signal, which spans across 10 operational cycles?\n "}, {"from": "gpt", "value": "It is essential to closely monitor the engine's condition throughout these cycles and proactively schedule maintenance in the near future to ensure optimal performance and prevent potential issues."}]} diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py new file mode 100644 index 000000000..42d142361 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py @@ -0,0 +1,447 @@ +import json +from pathlib import Path +import shutil +from typing import Any, NamedTuple, cast + +import h5py +import numpy as np +import pytest + +from timenet.dataset import OrdinalAxis, Record, RegularAxis, TimeFDataset +from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, open_local_version +from timenet.types import AnswerTask, ClassificationTask, MissingReason, Task +from timenet.types.text import read_value +from timenet_connectors.datasets.pandalin98.enginemt_qa import EngineMTQAConnector +from timenet_connectors.datasets.pandalin98.enginemt_qa.connector import _ClosedQuestion, _FileCounts, _Part, _Stage + + +_FIXTURES = Path(__file__).parent / "fixtures" +_TRAIN, _TEST, _SIGNALS = "train_qa.jsonl", "test_qa.jsonl", "time_series_data.h5" +_STEPS = 600 +_CHANNELS = 33 +_N_ROWS = 37_669 +"""The largest row id that a fixture line names, so the fixture file has just enough rows.""" +_SINGLE_ROWS = [20, 278, 8435, 37669] +_MULTI_TRAIN = (279, 287, 295, 305, 310, 319, 327, 335, 340, 346) +_MULTI_TEST = (279, 287, 295, 304, 309, 319, 326, 334, 339, 346) +_NAMED_ROWS = sorted({*_SINGLE_ROWS, *_MULTI_TRAIN, *_MULTI_TEST}) +_PADDED = {278: 100, 287: 560} +"""Rows whose flight ends inside them: the trailing steps the release fills with zeros. Row 287 keeps 40 steps, +fewer than the 60 that a multi-row question takes.""" + +# See tests/README.md for where each fixture line comes from. +_COUNTS = { + _Part.TRAIN: _FileCounts( + lines=4, + pairs={_Stage.UNDERSTANDING: 24, _Stage.PERCEPTION: 21, _Stage.REASONING: 2, _Stage.DECISION_MAKING: 1}, + unlabelled={_ClosedQuestion.HEALTH_STATE: 1}, + ), + _Part.TEST: _FileCounts( + lines=3, + pairs={_Stage.UNDERSTANDING: 16, _Stage.PERCEPTION: 14, _Stage.REASONING: 2, _Stage.DECISION_MAKING: 1}, + unlabelled={_ClosedQuestion.FAULT_CAUSE: 1}, + ), +} + + +def _values(row_id: int) -> np.ndarray: + """Every value names its row, step and channel, so a misplaced value can't pass.""" + steps = np.arange(_STEPS, dtype=np.float64)[:, None] + channels = np.arange(_CHANNELS, dtype=np.float64)[None, :] + values = row_id * 1000 + steps + channels / 100 + 0.5 + values[_STEPS - _PADDED.get(row_id, 0) :] = 0 + return values + + +def _write_signals(path: Path, rows: int = _N_ROWS) -> None: + """Write the signals file. Only the named rows hold values, so the chunked file stays small.""" + with h5py.File(path, "w") as file: + file["data_ID"] = np.arange(1, rows + 1, dtype=np.int64) + values = file.create_dataset("seq_data", shape=(rows, _STEPS, _CHANNELS), dtype=np.float64, chunks=True) + for row_id in _NAMED_ROWS: + if row_id <= rows: + values[row_id - 1] = _values(row_id) + + +def _lines(name: str) -> list[dict[str, Any]]: + return [json.loads(line) for line in (_FIXTURES / name).read_text(encoding="utf-8").splitlines()] + + +def _write_qa(path: Path, lines: list[dict[str, Any]]) -> None: + path.write_text("".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8") + + +@pytest.fixture(autouse=True) +def _fixture_counts(monkeypatch): + monkeypatch.setattr(EngineMTQAConnector, "SOURCE_COUNTS", _COUNTS) + + +@pytest.fixture +def files(tmp_path) -> dict[str, Path]: + """The release in ``tmp_path``: real QA lines and synthetic signals.""" + _write_signals(tmp_path / _SIGNALS) + for name in (_TRAIN, _TEST): + shutil.copyfile(_FIXTURES / name, tmp_path / name) + return {part: tmp_path / name for part, name in EngineMTQAConnector.SOURCE_FILES.items()} + + +@pytest.fixture +def dataset(files) -> TimeFDataset: + return EngineMTQAConnector().convert(files) + + +def _rows(record: Record) -> tuple[int, ...]: + rows = record.metadata["row_ids"] if "row_ids" in record.metadata else [record.metadata["row_id"]] + assert isinstance(rows, list) and all(isinstance(row, int) for row in rows) + return tuple(cast(list[int], rows)) + + +def _record(dataset: TimeFDataset, rows: tuple[int, ...]) -> Record: + return next(record for record in dataset.records if _rows(record) == rows) + + +def _label_set_name(task: Task) -> str | None: + if not isinstance(task, ClassificationTask) or task.label_set is None: + return None + return next(str(read_value(a.value)) for a in task.label_set.annotations if a.key == "name") + + +class _View(NamedTuple): + """A task as plain values.""" + + kind: str + split: str + rows: tuple[int, ...] + prompt: str + targets: tuple[str, ...] + label_set: str | None + metadata: dict[str, Any] + + +def _view(task: Task) -> _View: + assert isinstance(task, AnswerTask | ClassificationTask) + assert task.split is not None + (record,) = task.inputs + return _View( + kind=type(task).__name__, + split=task.split.value, + rows=_rows(record), + prompt=str(read_value(task.prompt)), + targets=tuple(str(read_value(target)) for target in task.targets or ()), + label_set=_label_set_name(task), + metadata=dict(task.metadata), + ) + + +def _views(dataset: TimeFDataset) -> list[_View]: + return [_view(task) for task in dataset.iter_tasks()] + + +def _find(views: list[_View], split: str, line: int, offset: int) -> _View: + return next( + view + for view in views + if view.split == split and view.metadata["source_line"] == line and view.metadata["qa_offset"] == offset + ) + + +def _joined(rows: tuple[int, ...], channel: int) -> list[float | None]: + """The authors' input for one channel: the first 60 steps of each row, with padding as None.""" + blocks = [_values(row)[:60, channel].tolist() for row in rows] + return [ + None if row in _PADDED and step >= _STEPS - _PADDED[row] else value + for row, block in zip(rows, blocks, strict=True) + for step, value in enumerate(block) + ] + + +def test_converts_the_fixture(dataset): + # One Record per single row, then one per 10-row list. Row 20 is named in both files. + assert [_rows(record) for record in dataset.records] == [(row,) for row in _SINGLE_ROWS] + [ + _MULTI_TRAIN, + _MULTI_TEST, + ] + views = _views(dataset) + # Per single-row line: 8 answer and 7 classification tasks, less one for an empty answer. + # Per multi-row line: 2 classification tasks and 1 answer task. + single = ["AnswerTask"] * 8 + ["ClassificationTask"] * 7 + train = single[:-1] + single + single + ["ClassificationTask"] * 2 + ["AnswerTask"] + test = single + ["ClassificationTask"] * 2 + ["AnswerTask"] + single[:-2] + single[-1:] + assert [view.kind for view in views] == train + test + assert [view.split for view in views] == ["train"] * len(train) + ["test"] * len(test) + + +def test_fault_cause_targets_its_letters_over_a_multi_label_set(dataset): + tasks = [task for task in dataset.iter_tasks() if task.metadata.get("closed_question") == "fault_cause"] + letters = {_view(task).rows: _view(task).targets for task in tasks} + assert letters[8435,] == ("a", "b", "e") + assert letters[278,] == ("a",) + label_set = next(task.label_set for task in tasks if isinstance(task, ClassificationTask)) + assert label_set is not None + assert label_set.multi_label + assert [(label.name, label.description) for label in label_set.labels] == [ + ("a", "HPT efficiency modifier"), + ("b", "LPT efficiency modifier"), + ("c", "Fan efficiency modifier"), + ("d", "HPT flow modifier"), + ("e", "LPT flow modifier"), + ("f", "Fan flow modifier"), + ] + + +def test_closed_questions_keep_their_place_and_label_set(dataset): + view = _find(_views(dataset), "train", line=1, offset=13) + published = _lines(_TRAIN)[1]["conversations"][26]["value"] + assert view == _View( + kind="ClassificationTask", + split="train", + rows=(8435,), + prompt=published, + targets=("a", "b", "e"), + label_set="fault_cause", + metadata={ + "source_file": _TRAIN, + "source_line": 1, + "qa_offset": 13, + "stage": 2, + "source_issues": [], + "closed_question": "fault_cause", + }, + ) + health = _find(_views(dataset), "train", line=0, offset=8) + assert (health.label_set, health.metadata["closed_question"]) == ("component_health", "fan_health") + + +def test_open_questions_keep_the_published_text(dataset): + lines = _lines(_TEST) + view = _find(_views(dataset), "test", line=1, offset=2) + assert view == _View( + kind="AnswerTask", + split="test", + rows=_MULTI_TEST, + prompt=lines[1]["conversations"][4]["value"], + targets=(lines[1]["conversations"][5]["value"],), + label_set=None, + metadata={"source_file": _TEST, "source_line": 1, "qa_offset": 2, "stage": 4, "source_issues": []}, + ) + + +def test_reasoning_reads_the_joined_record_of_its_rows_in_published_order(dataset): + view = _find(_views(dataset), "train", line=3, offset=0) + assert view.rows == _MULTI_TRAIN + assert (view.metadata["stage"], view.metadata["closed_question"], view.targets) == (3, "engine_condition", ("b",)) + + +def test_empty_closed_answers_get_no_task_and_are_listed(dataset): + views = _views(dataset) + with pytest.raises(StopIteration): + _find(views, "train", line=0, offset=14) + with pytest.raises(StopIteration): + _find(views, "test", line=2, offset=13) + unlabelled = next(a for a in dataset.annotations if a.key == "unlabelled_pairs") + assert unlabelled.value == 2 + assert unlabelled.metadata["pairs"] == [ + {"source_file": _TRAIN, "source_line": 0, "qa_offset": 14}, + {"source_file": _TEST, "source_line": 2, "qa_offset": 13}, + ] + + +def test_pairs_repeated_across_files_are_flagged_on_both_sides(dataset): + # Test line 0 is a byte copy of train line 2. The multi-row lines name other rows. + issues = {(view.split, view.metadata["source_line"]): view.metadata["source_issues"] for view in _views(dataset)} + assert issues["train", 2] == ["repeated_across_splits"] + assert issues["test", 0] == ["repeated_across_splits"] + assert issues["train", 3] == issues["test", 1] == [] + + +def test_a_pair_repeated_in_its_file_is_flagged(files, tmp_path, monkeypatch): + train = _lines(_TRAIN) + _write_qa(tmp_path / _TRAIN, [*train, train[3]]) + counts = _COUNTS[_Part.TRAIN] + pairs = {**counts.pairs, _Stage.REASONING: 4, _Stage.DECISION_MAKING: 2} + monkeypatch.setitem(_COUNTS, _Part.TRAIN, counts._replace(lines=5, pairs=pairs)) + views = _views(EngineMTQAConnector().convert(files)) + assert {tuple(v.metadata["source_issues"]) for v in views if v.rows == _MULTI_TRAIN} == {("repeated_question",)} + + +def test_a_row_record_holds_the_rows_real_steps_at_1_hz(dataset): + for row in (20, 278): + record = _record(dataset, (row,)) + assert [source.name for source in record.sources] == ["N-CMAPSS engine"] + assert [signal.name for signal in record.signals] == [f"channel_{c:02d}" for c in range(_CHANNELS)] + assert record.metadata["padded_steps"] == _PADDED.get(row, 0) + for channel, signal in enumerate(record.signals): + assert isinstance(signal.time_axis, RegularAxis) + assert signal.time_axis.period_us == 1_000_000 + np.testing.assert_array_equal(signal.to_numpy(), _values(row)[: _STEPS - _PADDED.get(row, 0), channel]) + + +def test_a_joined_record_holds_the_first_60_steps_of_each_row_with_padding_as_null(dataset): + record = _record(dataset, _MULTI_TRAIN) + assert record.metadata["steps_per_row"] == 60 + for channel, signal in enumerate(record.signals): + assert isinstance(signal.time_axis, OrdinalAxis) + assert signal.spec.missing is not None and signal.spec.missing.reason is MissingReason.ALIGNMENT_PADDING + assert signal.to_arrow().to_pylist() == _joined(_MULTI_TRAIN, channel) + assert signal.null_count == 20 + + +def test_storage_round_trip_keeps_every_task_and_value(dataset, tmp_path): + registry = LocalRegistry(tmp_path / "registry") + registry.put_dataset(dataset) + with TimeFReader(open_local_version(registry.root / "pandalin98/enginemt-qa/1.0.0")) as reader: + restored = reader.read() + assert _views(restored) == _views(dataset) + for original, record in zip(dataset.records, restored.records, strict=True): + assert record.metadata == original.metadata + for before, after in zip(original.signals, record.signals, strict=True): + assert after.to_arrow().equals(before.to_arrow()) + assert after.null_count == before.null_count + + +def test_a_qa_file_with_other_counts_fails(files, tmp_path): + _write_qa(tmp_path / _TRAIN, _lines(_TRAIN)[:-1]) + with pytest.raises(TimeFValidationError, match="expected") as raised: + EngineMTQAConnector().convert(files) + assert _TRAIN in str(raised.value) + + +def test_a_shortened_signals_file_fails_when_read(dataset, tmp_path): + _write_signals(tmp_path / _SIGNALS, rows=4) + with pytest.raises(TimeFFormatError, match="cannot read row id"): + _record(dataset, (8435,)).signals[0].to_numpy() + + +def test_a_missing_signals_file_fails_when_read(dataset, tmp_path): + (tmp_path / _SIGNALS).unlink() + with pytest.raises(TimeFFormatError, match="cannot open the signals file"): + dataset.records[0].signals[0].to_numpy() + + +def test_a_row_with_a_nan_fails_when_read(dataset, tmp_path): + with h5py.File(tmp_path / _SIGNALS, "a") as file: + file["seq_data"][19, 5, 3] = np.nan + with pytest.raises(TimeFValidationError, match="row id 20 holds a NaN"): + _record(dataset, (20,)).signals[0].to_numpy() + + +def test_padding_that_changed_after_conversion_fails_when_read(dataset, tmp_path): + with h5py.File(tmp_path / _SIGNALS, "a") as file: + file["seq_data"][277, 450:] = 0 + with pytest.raises(TimeFFormatError, match="row id 278 has 450 real steps; the file changed"): + _record(dataset, (278,)).signals[0].to_numpy() + + +def test_a_zero_step_inside_a_row_fails(files, tmp_path): + with h5py.File(tmp_path / _SIGNALS, "a") as file: + file["seq_data"][277, 10] = 0 + with pytest.raises(TimeFValidationError, match="row id 278 has a step of 33 zeros before its last real step"): + EngineMTQAConnector().convert(files) + + +def test_a_missing_qa_file_fails_when_the_tasks_are_read(dataset, tmp_path): + (tmp_path / _TRAIN).unlink() + with pytest.raises(TimeNetConnectorError, match=_TRAIN): + list(dataset.iter_tasks()) + + +def test_a_file_that_is_not_hdf5_fails_with_its_path(files, tmp_path): + (tmp_path / _SIGNALS).write_bytes(b"not hdf5") + with pytest.raises(TimeFFormatError, match="cannot open the signals file"): + EngineMTQAConnector().convert(files) + + +def _rewrite(path: Path, **datasets: Any) -> None: + with h5py.File(path, "w") as file: + for name, value in datasets.items(): + if value is None: + file.create_group(name) + else: + file[name] = value + + +_IDS = np.arange(1, 4) +_ZEROS = np.zeros((3, _STEPS, _CHANNELS)) + + +@pytest.mark.parametrize( + ("datasets", "message"), + [ + ({"data_ID": _IDS, "seq_data": _ZEROS, "extra": 1}, "expected seq_data and data_ID"), + ({"data_ID": _IDS, "seq_data": np.zeros((3, _STEPS, _CHANNELS - 1))}, r"seq_data is float64 \(3, 600, 32\)"), + ({"data_ID": _IDS, "seq_data": _ZEROS.astype(np.float32)}, "seq_data is float32"), + ({"data_ID": _IDS, "seq_data": None}, "seq_data is a Group, not an HDF5 dataset"), + ({"data_ID": _IDS.astype(np.float64), "seq_data": _ZEROS}, "data_ID is float64"), + ({"data_ID": np.array([2, 2, 3]), "seq_data": _ZEROS}, "data_ID is not the row ids 1 to 3"), + ], + ids=["extra-key", "32-channels", "float32", "group", "float-ids", "ids-not-1-to-n"], +) +def test_a_signals_file_with_another_layout_fails(files, tmp_path, datasets, message): + _rewrite(tmp_path / _SIGNALS, **datasets) + with pytest.raises(TimeFValidationError, match=message): + EngineMTQAConnector().convert(files) + + +def _with_pair(offset: int, **changes: Any) -> dict[str, Any]: + """Train line 0 with one pair changed: ``answer`` sets the answer, other keys the question.""" + line = _lines(_TRAIN)[0] + conversations = [dict(turn) for turn in line["conversations"]] + question, answer = conversations[2 * offset], conversations[2 * offset + 1] + answer["value"] = changes.pop("answer", answer["value"]) + question.update(changes) + return {**line, "conversations": conversations} + + +def _with(**changes: Any) -> dict[str, Any]: + """A train line with its own keys changed: the multi-row line 3 if ``multi``, else line 0.""" + line = _lines(_TRAIN)[3 if changes.pop("multi", False) else 0] + return {**line, **changes} + + +def _health_state(text: str) -> str: + return _lines(_TRAIN)[0]["conversations"][28]["value"].replace("Imminent", text) + + +# In a single-row line, pairs 8 to 14 are the Perception questions: 13 is the fault cause and 14 the +# health state. +@pytest.mark.parametrize( + ("line", "message"), + [ + (_with_pair(13, answer="a, g"), r"fault_cause: answer 'a, g': labels not in label set \['a',"), + (_with_pair(13, answer="a, a"), "repeated labels"), + (_with_pair(14, answer="a, b"), "takes exactly one"), + (_with_pair(8, value="What is the health status of the fan?\n "), "unknown stage 2"), + (_with_pair(14, value=_health_state("Near")), "prints options other than"), + (_with_pair(0, answer=""), "open question has an empty answer"), + (_with_pair(0, attribute="close"), "stage 1 questions are not 'close'"), + (_with_pair(0, value="Describe the signal."), "one marker"), + (_with(name="4.csv"), "doesn't repeat id"), + (_with(multi=True, id=["1"] * 10, name=["1.csv"] * 10), "repeats a row"), + (_with(multi=True, id=["1"] * 9, name=["1.csv"] * 9), "names 9 rows"), + (_with(id="0278", name="0278.csv"), "not decimal row ids"), + (_with(id="37670", name="37670.csv"), "past the file's 37669 rows"), + (_with(conversations=[]), "alternate"), + ], +) +def test_a_line_that_breaks_a_release_rule_fails_with_its_place(files, tmp_path, line, message): + _write_qa(tmp_path / _TRAIN, [_lines(_TRAIN)[0], line]) + with pytest.raises(TimeFValidationError, match=message) as raised: + EngineMTQAConnector().convert(files) + assert "train_qa.jsonl line 2" in str(raised.value) + + +@pytest.mark.parametrize( + "line", + [ + _with_pair(0, stage="5"), + _with_pair(0, **{"from": "gpt"}), + _with(extra=1), + ], + ids=["unknown-stage", "answer-first", "extra-key"], +) +def test_a_line_with_another_shape_fails_with_its_place(files, tmp_path, line): + _write_qa(tmp_path / _TRAIN, [_lines(_TRAIN)[0], line]) + with pytest.raises(TimeNetConnectorError, match=f"{_TRAIN}: line 2"): + EngineMTQAConnector().convert(files) diff --git a/pyproject.toml b/pyproject.toml index 1b9ddbfe8..99b31d220 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dev = [ "ruff>=0.17.3", "ty>=0.0.92", "edfio>=0.4.14", + "h5py>=3.10", "wfdb>=4.1", "xlrd>=2.0", ] diff --git a/uv.lock b/uv.lock index 3d9b91921..f4b1820e9 100644 --- a/uv.lock +++ b/uv.lock @@ -25,6 +25,7 @@ members = [ [manifest.dependency-groups] dev = [ { name = "edfio", specifier = ">=0.4.14" }, + { name = "h5py", specifier = ">=3.10" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "hypothesis", specifier = ">=6.140" }, { name = "jsonschema", specifier = ">=4.18" }, @@ -1116,6 +1117,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h5py" +version = "3.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/95/a825894f3e45cbac7554c4e97314ce886b233a20033787eda755ca8fecc7/h5py-3.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:719439d14b83f74eeb080e9650a6c7aa6d0d9ea0ca7f804347b05fac6fbf18af", size = 3721663, upload-time = "2026-03-06T13:47:49.599Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3b/38ff88b347c3e346cda1d3fc1b65a7aa75d40632228d8b8a5d7b58508c24/h5py-3.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3f0a0e136f2e95dd0b67146abb6668af4f1a69c81ef8651a2d316e8e01de447", size = 3087630, upload-time = "2026-03-06T13:47:51.249Z" }, + { url = "https://files.pythonhosted.org/packages/98/a8/2594cef906aee761601eff842c7dc598bea2b394a3e1c00966832b8eeb7c/h5py-3.16.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a6fbc5367d4046801f9b7db9191b31895f22f1c6df1f9987d667854cac493538", size = 4823472, upload-time = "2026-03-06T13:47:53.085Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/c1f604538ff6db22a0690be2dc44ab59178e115f63c917794e529356ab23/h5py-3.16.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fb1720028d99040792bb2fb31facb8da44a6f29df7697e0b84f0d79aff2e9bd3", size = 5027150, upload-time = "2026-03-06T13:47:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fd/301739083c2fc4fd89950f9bcfce75d6e14b40b0ca3d40e48a8993d1722c/h5py-3.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:314b6054fe0b1051c2b0cb2df5cbdab15622fb05e80f202e3b6a5eee0d6fe365", size = 4814544, upload-time = "2026-03-06T13:47:56.893Z" }, + { url = "https://files.pythonhosted.org/packages/4c/42/2193ed41ccee78baba8fcc0cff2c925b8b9ee3793305b23e1f22c20bf4c7/h5py-3.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ffbab2fedd6581f6aa31cf1639ca2cb86e02779de525667892ebf4cc9fd26434", size = 5034013, upload-time = "2026-03-06T13:47:59.01Z" }, + { url = "https://files.pythonhosted.org/packages/f7/20/e6c0ff62ca2ad1a396a34f4380bafccaaf8791ff8fccf3d995a1fc12d417/h5py-3.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:17d1f1630f92ad74494a9a7392ab25982ce2b469fc62da6074c0ce48366a2999", size = 3191673, upload-time = "2026-03-06T13:48:00.626Z" }, + { url = "https://files.pythonhosted.org/packages/f2/48/239cbe352ac4f2b8243a8e620fa1a2034635f633731493a7ff1ed71e8658/h5py-3.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:85b9c49dd58dc44cf70af944784e2c2038b6f799665d0dcbbc812a26e0faa859", size = 2673834, upload-time = "2026-03-06T13:48:02.579Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c0/5d4119dba94093bbafede500d3defd2f5eab7897732998c04b54021e530b/h5py-3.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c5313566f4643121a78503a473f0fb1e6dcc541d5115c44f05e037609c565c4d", size = 3685604, upload-time = "2026-03-06T13:48:04.198Z" }, + { url = "https://files.pythonhosted.org/packages/b0/42/c84efcc1d4caebafb1ecd8be4643f39c85c47a80fe254d92b8b43b1eadaf/h5py-3.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42b012933a83e1a558c673176676a10ce2fd3759976a0fedee1e672d1e04fc9d", size = 3061940, upload-time = "2026-03-06T13:48:05.783Z" }, + { url = "https://files.pythonhosted.org/packages/89/84/06281c82d4d1686fde1ac6b0f307c50918f1c0151062445ab3b6fa5a921d/h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527", size = 5198852, upload-time = "2026-03-06T13:48:07.482Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e9/1a19e42cd43cc1365e127db6aae85e1c671da1d9a5d746f4d34a50edb577/h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e", size = 5405250, upload-time = "2026-03-06T13:48:09.628Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8e/9790c1655eabeb85b92b1ecab7d7e62a2069e53baefd58c98f0909c7a948/h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794", size = 5190108, upload-time = "2026-03-06T13:48:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/51/d7/ab693274f1bd7e8c5f9fdd6c7003a88d59bedeaf8752716a55f532924fbb/h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074", size = 5419216, upload-time = "2026-03-06T13:48:13.322Z" }, + { url = "https://files.pythonhosted.org/packages/03/c1/0976b235cf29ead553e22f2fb6385a8252b533715e00d0ae52ed7b900582/h5py-3.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:96b422019a1c8975c2d5dadcf61d4ba6f01c31f92bbde6e4649607885fe502d6", size = 3182868, upload-time = "2026-03-06T13:48:15.759Z" }, + { url = "https://files.pythonhosted.org/packages/14/d9/866b7e570b39070f92d47b0ff1800f0f8239b6f9e45f02363d7112336c1f/h5py-3.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:39c2838fb1e8d97bcf1755e60ad1f3dd76a7b2a475928dc321672752678b96db", size = 2653286, upload-time = "2026-03-06T13:48:17.279Z" }, + { url = "https://files.pythonhosted.org/packages/0f/9e/6142ebfda0cb6e9349c091eae73c2e01a770b7659255248d637bec54a88b/h5py-3.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:370a845f432c2c9619db8eed334d1e610c6015796122b0e57aa46312c22617d9", size = 3671808, upload-time = "2026-03-06T13:48:19.737Z" }, + { url = "https://files.pythonhosted.org/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42108e93326c50c2810025aade9eac9d6827524cdccc7d4b75a546e5ab308edb", size = 3045837, upload-time = "2026-03-06T13:48:21.854Z" }, + { url = "https://files.pythonhosted.org/packages/da/1e/6172269e18cc5a484e2913ced33339aad588e02ba407fafd00d369e22ef3/h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524", size = 5193860, upload-time = "2026-03-06T13:48:24.071Z" }, + { url = "https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402", size = 5400417, upload-time = "2026-03-06T13:48:25.728Z" }, + { url = "https://files.pythonhosted.org/packages/bc/81/5b62d760039eed64348c98129d17061fdfc7839fc9c04eaaad6dee1004e4/h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7", size = 5185214, upload-time = "2026-03-06T13:48:27.436Z" }, + { url = "https://files.pythonhosted.org/packages/28/c4/532123bcd9080e250696779c927f2cb906c8bf3447df98f5ceb8dcded539/h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff", size = 5414598, upload-time = "2026-03-06T13:48:29.49Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/a27997f84341fc0dfcdd1fe4179b6ba6c32a7aa880fdb8c514d4dad6fba3/h5py-3.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:18f2bbcd545e6991412253b98727374c356d67caa920e68dc79eab36bf5fedad", size = 3175509, upload-time = "2026-03-06T13:48:31.131Z" }, + { url = "https://files.pythonhosted.org/packages/a5/23/bb8647521d4fd770c30a76cfc6cb6a2f5495868904054e92f2394c5a78ff/h5py-3.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:656f00e4d903199a1d58df06b711cf3ca632b874b4207b7dbec86185b5c8c7d4", size = 2647362, upload-time = "2026-03-06T13:48:33.411Z" }, + { url = "https://files.pythonhosted.org/packages/48/3c/7fcd9b4c9eed82e91fb15568992561019ae7a829d1f696b2c844355d95dd/h5py-3.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9c9d307c0ef862d1cd5714f72ecfafe0a5d7529c44845afa8de9f46e5ba8bd65", size = 3678608, upload-time = "2026-03-06T13:48:35.183Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b7/9366ed44ced9b7ef357ab48c94205280276db9d7f064aa3012a97227e966/h5py-3.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c1eff849cdd53cbc73c214c30ebdb6f1bb8b64790b4b4fc36acdb5e43570210", size = 3054773, upload-time = "2026-03-06T13:48:37.139Z" }, + { url = "https://files.pythonhosted.org/packages/58/a5/4964bc0e91e86340c2bbda83420225b2f770dcf1eb8a39464871ad769436/h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965", size = 5198886, upload-time = "2026-03-06T13:48:38.879Z" }, + { url = "https://files.pythonhosted.org/packages/f1/16/d905e7f53e661ce2c24686c38048d8e2b750ffc4350009d41c4e6c6c9826/h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd", size = 5404883, upload-time = "2026-03-06T13:48:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/58f34cb74af46d39f4cd18ea20909a8514960c5a3e5b92fd06a28161e0a8/h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c", size = 5192039, upload-time = "2026-03-06T13:48:43.117Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ca/934a39c24ce2e2db017268c08da0537c20fa0be7e1549be3e977313fc8f5/h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc", size = 5421526, upload-time = "2026-03-06T13:48:44.838Z" }, + { url = "https://files.pythonhosted.org/packages/3e/14/615a450205e1b56d16c6783f5ccd116cde05550faad70ae077c955654a75/h5py-3.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:fa48993a0b799737ba7fd21e2350fa0a60701e58180fae9f2de834bc39a147ab", size = 3183263, upload-time = "2026-03-06T13:48:47.117Z" }, + { url = "https://files.pythonhosted.org/packages/7b/48/a6faef5ed632cae0c65ac6b214a6614a0b510c3183532c521bdb0055e117/h5py-3.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:1897a771a7f40d05c262fc8f37376ec37873218544b70216872876c627640f63", size = 2663450, upload-time = "2026-03-06T13:48:48.707Z" }, + { url = "https://files.pythonhosted.org/packages/5d/32/0c8bb8aedb62c772cf7c1d427c7d1951477e8c2835f872bc0a13d1f85f86/h5py-3.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15922e485844f77c0b9d275396d435db3baa58292a9c2176a386e072e0cf2491", size = 3760693, upload-time = "2026-03-06T13:48:50.453Z" }, + { url = "https://files.pythonhosted.org/packages/1d/1f/fcc5977d32d6387c5c9a694afee716a5e20658ac08b3ff24fdec79fb05f2/h5py-3.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:df02dd29bd247f98674634dfe41f89fd7c16ba3d7de8695ec958f58404a4e618", size = 3181305, upload-time = "2026-03-06T13:48:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a1/af87f64b9f986889884243643621ebbd4ac72472ba8ec8cec891ac8e2ca1/h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242", size = 5074061, upload-time = "2026-03-06T13:48:54.089Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d0/146f5eaff3dc246a9c7f6e5e4f42bd45cc613bce16693bcd4d1f7c958bf5/h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16", size = 5279216, upload-time = "2026-03-06T13:48:56.75Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/12a13424f1e604fc7df9497b73c0356fb78c2fb206abd7465ce47226e8fd/h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7", size = 5070068, upload-time = "2026-03-06T13:48:59.169Z" }, + { url = "https://files.pythonhosted.org/packages/41/8c/bbe98f813722b4873818a8db3e15aa3e625b59278566905ac439725e8070/h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725", size = 5300253, upload-time = "2026-03-06T13:49:02.033Z" }, + { url = "https://files.pythonhosted.org/packages/32/9e/87e6705b4d6890e7cecdf876e2a7d3e40654a2ae37482d79a6f1b87f7b92/h5py-3.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4c6ab014ab704b4feaa719ae783b86522ed0bf1f82184704ed3c9e4e3228796e", size = 3381671, upload-time = "2026-03-06T13:49:04.351Z" }, + { url = "https://files.pythonhosted.org/packages/96/91/9fad90cfc5f9b2489c7c26ad897157bce82f0e9534a986a221b99760b23b/h5py-3.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:faca8fb4e4319c09d83337adc80b2ca7d5c5a343c2d6f1b6388f32cfecca13c1", size = 2740706, upload-time = "2026-03-06T13:49:06.347Z" }, +] + [[package]] name = "hf-xet" version = "1.6.0" From 620d3efc2aef7c114ab0b01e1baf3873d47aa701 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 20 Nov 2026 14:50:42 -0800 Subject: [PATCH 2/2] fix(connectors): leave out EngineMT-QA train pairs that repeat a test pair The released train file holds 4,039 byte copies of test lines. The paper does not mention them. Before, their tasks stayed in the train split with the source issue repeated_across_splits, so every user had to know about the flag to avoid training on test pairs. Now a labelled train pair that has the same rows, question and answer as a test pair gets no task. That leaves out 27,961 train pairs: the labelled pairs of the copied lines, and 6 pairs of a train-only line about a row that a test line also names. The annotation test_pairs_in_train lists each one. The test split keeps every labelled pair, and the train split keeps 72,335 tasks. The tasks no longer have the source_issues metadata key. After the change, no pair repeats another pair in its own split. The README now describes the rows from a read of the whole file. A flight is a median of 16 rows long, and channel 32 is constant within a flight. Most multi-row lists take about one window from each of 10 consecutive flights. 651 lists mix two runs of channel 32, which are likely two engine units. --- .../datasets/pandalin98/enginemt_qa/README.md | 110 +++++++++++------- .../pandalin98/enginemt_qa/connector.py | 84 ++++++------- .../pandalin98/enginemt_qa/dataset.yaml | 14 +-- .../enginemt_qa/tests/test_connector.py | 32 ++--- 4 files changed, 123 insertions(+), 117 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md index 2bc0c87ee..24c688448 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/README.md @@ -7,8 +7,8 @@ with the ITFormer paper (Wang et al., ICML 2025). The paper is [arXiv 2506.20093](https://arxiv.org/abs/2506.20093). The authors' code is [ITFormer-ICML25](https://github.com/Pandalin98/ITFormer-ICML25). -The connector builds one task per labelled question-answer pair. A single-row question reads the -Record of its row. A multi-row question reads one joined Record: the first 60 steps of each of its +The connector builds one task per labelled question-answer pair. It leaves out the train pairs that +repeat a test pair. A single-row question reads the Record of its row. A multi-row question reads one joined Record: the first 60 steps of each of its 10 rows. The authors' code gives its model the same input. The connector lives in one file, `connector.py`. @@ -34,18 +34,20 @@ channel audit ([`docs/ISSUE_23_CHANNEL_AUDIT.md`](https://github.com/Pandalin98/ITFormer-ICML25/blob/b80142f/docs/ISSUE_23_CHANNEL_AUDIT.md)) says not to name the 33 channels without a mapping, so the connector does not. -The release does not say what a row is. We read about 400 rows over HTTP and found: +The release does not say what a row is. We read the first and last steps of every row and found: - A row is a window of one flight. Consecutive ids are consecutive windows: the last step of a row joins the first step of the next one. So a row of 600 steps at 1 Hz is the paper's "10-minute (600 points) time-series segment", and N-CMAPSS records its flights at 1 Hz. -- A flight is 10 to 21 consecutive rows. When a flight ends inside a row, the release fills the rest - of that row with zeros in all 33 channels. In 171 consecutive rows, 11 rows ended this way, one per - flight. +- When a flight ends inside a row, the release fills the rest of that row with zeros in all 33 + channels. 6,380 rows end this way. Counted from one such row to the next, a flight is a median of + 16 rows long. - The release does not mark where a flight starts. The zeros at the end of a flight are the only - sign of a flight boundary. -- The values look standardized, with statistics that the release does not give. Channel 32 is - constant within each row, and its values do not follow the health answers. + sign of a flight boundary. A flight that ends exactly at the end of a row leaves no sign. +- The values look standardized, with statistics that the release does not give. +- Channel 32 is constant within each flight. It changes only 72 times in the whole file, so it splits + the rows into 73 runs of 34 to 359 flights. We read each run as one engine unit, but the release + does not say so. The QA files name only 7,755 of the 118,921 rows. The single-row lines name 5,945 rows. The 8,845 distinct lists of the multi-row lines use 1,900 rows, and 90 rows occur in both. No question names @@ -78,8 +80,9 @@ one block at the end of the train file. A labelled pair is a pair whose answer i | `test_qa.jsonl`, lines only in the test file | 1,500 | 14,160 | 14,022 | The copies are not a random sample of the test file. They hold 2,689 of the 3,384 multi-row test -lines and 1,350 of the 2,155 single-row test lines. [Source issues](#source-issues) gives the -counts per stage and compares them with the paper. +lines and 1,350 of the 2,155 single-row test lines. [Train and test overlap](#train-and-test-overlap) +gives the counts per stage, compares them with the paper, and says which pairs the connector leaves +out. ### The four tasks @@ -110,8 +113,14 @@ An answer is one letter. The fault cause is the exception. It can hold several l ### The multi-row questions The paper says that Reasoning and Decision-Making look at "10 consecutive cycles" to judge the -long-term health of the engine. In the one list that we traced, the 10 rows are windows in the middle -of 9 different flights, over about one day of flying. The ids of a list span a median of 145 rows. +long-term health of the engine. In N-CMAPSS, a cycle is a flight. A list of 10 rows takes about one +window from each of 10 consecutive flights, at a random place in the flight: + +- 8,194 of the 8,845 distinct lists stay within one run of channel 32. Their ids ascend. Two + successive rows come from the same flight or from flights at most 2 apart. A list covers 1 to 11 + flights. +- 651 lists mix two adjacent runs of channel 32, which we read as two engine units. In 186 of them, + the ids of the two runs interleave. The authors' code does not give the model the 10 whole rows. It takes the first 60 steps of each row and joins them into one (600, 33) array (`TsQaDataset.__getitem__` at `b80142f`). The paper does not @@ -147,8 +156,10 @@ Record back to the authors' array, add `padded_steps` zero steps at its end. ### Tasks -Each published QA pair with a label becomes one task of its stage, with one input Record. Its split is -the split of its file: `train_qa.jsonl` is train and `test_qa.jsonl` is test. The tasks come in file +Each published QA pair with a label becomes one task of its stage, with one input Record. A train pair +that repeats a test pair is the exception. It gets no task, see +[Train and test overlap](#train-and-test-overlap). The split of a task is the split of its file: +`train_qa.jsonl` is train and `test_qa.jsonl` is test. The tasks come in file order, train file first. | Stage | TimeNet type | Input | Targets | @@ -158,7 +169,7 @@ order, train file first. | Reasoning | `ClassificationTask` | joined Record | the published letter | | Decision-Making | `AnswerTask` | joined Record | the published answer | -That gives 142,273 tasks: 100,296 train and 41,977 test. +That gives 114,312 tasks: 72,335 train and 41,977 test. The prompt is the published question, unchanged, with its `` marker and its options. The connector adds no wording. @@ -180,24 +191,20 @@ label set `component_health`. The other label sets are `fault_cause`, `health_st | `source_file`, `source_line`, `qa_offset` | where the pair is published. The line and offset are 0-based, as in the authors' evaluation | | `stage` | the published stage, 1 to 4 | | `closed_question` | the closed question, such as `hpc_health` or `fault_cause`. Closed tasks only | -| `source_issues` | a list of issue names, empty for most tasks. See [Source issues](#source-issues) | Error messages count lines from 1, as editors do. `source_line` counts from 0, as the authors do. ### Dataset annotations -The annotation `unlabelled_pairs` holds the count of closed pairs with an empty answer, 1,769. Its -metadata `pairs` lists the place of each one. See [Empty answers](#empty-answers). - -## Source issues +Two annotations list the pairs that get no task. Each one holds a count, and its metadata `pairs` +lists the place of each pair: -The connector keeps every labelled pair as published. It names two problems in the task metadata -`source_issues`, with the same names as the TSAQA connector: - -| Issue | Meaning | Tasks | +| Annotation | Pairs | Count | | --- | --- | --- | -| `repeated_across_splits` | the other file has a pair with the same rows, question and answer | 27,961 train, 27,955 test | -| `repeated_question` | another pair in the same file has the same rows, question and answer | 12 train | +| `unlabelled_pairs` | closed pairs with an empty answer. See [Empty answers](#empty-answers) | 1,769 | +| `test_pairs_in_train` | train pairs that repeat a test pair. See [Train and test overlap](#train-and-test-overlap) | 27,961 | + +## Train and test overlap 4,039 of the 5,539 test lines are byte copies of a train line. The test file matches the per-stage counts in the paper's Fig. 8 exactly. The train file holds the paper's train counts plus those 4,039 @@ -211,10 +218,24 @@ copied lines: | 4 Decision-Making | 8,150 | 2,689 | 5,461 | These counts include the empty answers. So the release adds test lines to the paper's training set. -With the released train file, a model reaches a Reasoning test accuracy of 99.3% to 99.8%, as the -authors' issue [#34](https://github.com/Pandalin98/ITFormer-ICML25/issues/34) reports. The connector keeps the -released split. To train on the paper's split, leave out the train tasks whose `source_issues` holds -`repeated_across_splits`. +The paper does not mention the copies. With the released train file, a model reaches a Reasoning +test accuracy of 99.3% to 99.8%, as the authors' issue +[#34](https://github.com/Pandalin98/ITFormer-ICML25/issues/34) reports. + +The connector keeps every labelled test pair. It leaves out each labelled train pair that repeats a +test pair: the same rows, the same question and the same answer. That is 27,961 pairs: + +- The 27,955 labelled pairs of the 4,039 copied lines. +- 6 Perception pairs of train line 2,003, about row 40034. Test line 1,765 names the same row and + asks the same 6 questions with the same answers. Its Understanding answers have other wording, so + the lines are not copies. + +The train split keeps 72,335 tasks. The paper's training set has 72,341 labelled pairs, which +include those 6 pairs. + +The two files still share rows. Row 40034 is the only row that a single-row line of each file names. +The multi-row lines of both files draw on the same 1,900 rows, in other combinations. No list of 10 +rows is in both files. ## Empty answers @@ -273,32 +294,36 @@ infinite value, it raises `TimeFValidationError`. | first 60 steps of each of 10 rows, joined | the joined Record of the 10 ids | | SFT target: the published answer | the target: the answer, or its letters | | `closed_question_metrics`: the set of letters a to f | a `ClassificationTask` over a label set of letters | -| train on `train_qa.jsonl`, evaluate on `test_qa.jsonl` | `Split.TRAIN`, `Split.TEST` | +| train on `train_qa.jsonl`, evaluate on `test_qa.jsonl` | `Split.TRAIN` without the repeated test pairs, `Split.TEST` | | `source_line`, `qa_offset` | `source_line`, `qa_offset` in task metadata | -The connector differs from the authors' input in one way: it does not store the padding zeros. A row -Record stops at the last real step. A joined Record has nulls where the authors' array has zeros. +The connector differs from the authors in two ways. It does not store the padding zeros: a row +Record stops at the last real step, and a joined Record has nulls where the authors' array has zeros. +It also leaves out the train pairs that repeat a test pair. ## Comparing with the paper and the card - Channels: the paper says 32 channels. The release has 33. The authors' audit cannot tell which column is extra, so a dropped column is a guess. The connector keeps all 33. - Pairs: the paper gives both 11k and 110k pairs, and the Hub card gives 87k train and 25k test - pairs. The files hold 101,565 train and 42,477 test pairs. See [Source issues](#source-issues) for - how they relate to Fig. 8. + pairs. The files hold 101,565 train and 42,477 test pairs. See + [Train and test overlap](#train-and-test-overlap) for how they relate to Fig. 8. - Reasoning: the paper says Reasoning predicts "the probability of future failures and the remaining useful life" (Section 5.1). The release has no remaining-useful-life question. Reasoning asks for the condition and the failure probability. -- Cycles: the paper calls a row a "cycle" and also a "10-minute (600 points)" segment. A row is a - window of a flight, not a whole flight. The paper says a multi-row question is about "10 - consecutive cycles". Every one of the 11,534 multi-row lists has gaps between its ids, and 256 of - them mix two ranges of ids. +- Cycles: the paper calls a row a "cycle" and also a "10-minute (600 points)" segment. In N-CMAPSS, + a cycle is a flight, and a row is a 10-minute window of a flight. A multi-row list takes about one + window from each of 10 consecutive flights. 651 lists mix two engine units, by our reading of + channel 32. See [The multi-row questions](#the-multi-row-questions). - Multi-row input: the paper encodes each segment separately. The code joins the first 60 steps of each row. The connector follows the code. ## Data limitations -- The train split holds copies of most of the test split. See [Source issues](#source-issues). +- The released train file holds copies of most test lines. The connector leaves them out, but the + multi-row lines of both splits still draw on the same rows. See + [Train and test overlap](#train-and-test-overlap). +- 651 multi-row lists mix two engine units, by our reading of channel 32. The connector keeps them. - The signals come from a simulator, are scaled, and have no channel names. The questions name signals such as altitude, but no Record says which channel is which. - The paper says an LLM rewrote the questions and answers, and experts reviewed them (Appendix A). The @@ -312,7 +337,8 @@ Record stops at the last real step. A joined Record has nulls where the authors' - Channel names: which N-CMAPSS variable each of the 33 channels holds, and which one the paper's 32 leaves out. The authors promise a mapping in issue [#9](https://github.com/Pandalin98/ITFormer-ICML25/issues/9). -- Flights: the release does not mark where a flight or an engine unit starts. +- Engine units: we read each run of channel 32 as one engine unit. The release does not mark where a + flight or an engine unit starts. ## Local build diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py index ef68fc9e6..4f21acc26 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/connector.py @@ -100,21 +100,8 @@ class _Key(StrEnum): QA_OFFSET = "qa_offset" STAGE = "stage" CLOSED_QUESTION = "closed_question" - SOURCE_ISSUES = "source_issues" UNLABELLED_PAIRS = "unlabelled_pairs" - - -@unique -class SourceIssue(StrEnum): - """A problem in a published pair, which the connector keeps and names in the task's metadata. - - The names match the TSAQA connector's, so one filter can serve both datasets. - """ - - REPEATED_QUESTION = "repeated_question" - """Another pair of the same file has the same rows, question and answer.""" - REPEATED_ACROSS_SPLITS = "repeated_across_splits" - """The other file has a pair with the same rows, question and answer.""" + TEST_PAIRS_IN_TRAIN = "test_pairs_in_train" @unique @@ -364,19 +351,18 @@ def has_label(self) -> bool: @property def digest(self) -> bytes: - """A digest of the rows, the question and the answer, which match for a repeated pair.""" + """A digest of the rows, the question and the answer, which match for a train pair that repeats a test pair.""" content = json.dumps([self.rows, self.question, self.answer]).encode() return hashlib.blake2b(content, digest_size=_DIGEST_BYTES).digest() - def task(self, record: Record, issues: Sequence[SourceIssue]) -> AnswerTask | ClassificationTask: + def task(self, record: Record) -> AnswerTask | ClassificationTask: """Build the task of a labelled pair over the Record of its rows. Returns: An answer task with the published answer for an open question, or a classification task with the published letters for a closed one. """ - issue_names: list[JsonValue] = [issue.value for issue in issues] - metadata = self.place.metadata | {_Key.STAGE: self.stage.value, _Key.SOURCE_ISSUES: issue_names} + metadata = self.place.metadata | {_Key.STAGE: self.stage.value} split = self.place.part.task_split if self.closed is None: return AnswerTask( @@ -770,36 +756,23 @@ class _Scan: """The row lists that labelled multi-row pairs name, in first-seen order.""" unlabelled: list[_Place] = field(default_factory=list) """The places of the closed pairs with an empty answer.""" - digests: dict[_Part, Counter[bytes]] = field(default_factory=dict) - """How often each labelled pair's digest occurs in each file.""" - - def issues(self, pair: _Pair) -> list[SourceIssue]: - """Name the source issues of a labelled pair. - - Returns: - The issues, in ``SourceIssue`` order. - """ - digest = pair.digest - found = [] - if self.digests[pair.place.part][digest] > 1: - found.append(SourceIssue.REPEATED_QUESTION) - if any(self.digests[part][digest] for part in _QA_PARTS if part is not pair.place.part): - found.append(SourceIssue.REPEATED_ACROSS_SPLITS) - return found + test_pairs_in_train: list[_Place] = field(default_factory=list) + """The places of the labelled train pairs that repeat a test pair. They get no task.""" def _scan(qa_files: Sequence[_QAFile], counts: Mapping[_Part, _FileCounts]) -> _Scan: """Check every pair once, and collect what the dataset needs from all of them. Returns: - The rows that labelled pairs name, the unlabelled pairs, and the digests of the labelled pairs. + The rows that labelled pairs name, the unlabelled pairs, and the train pairs that repeat a test pair. Raises: TimeFValidationError: If a file's counts differ from the pinned ones. """ scan = _Scan() + labelled: dict[_Part, list[tuple[bytes, _Place]]] = {} for qa in qa_files: - digests: Counter[bytes] = Counter() + digests: list[tuple[bytes, _Place]] = [] pairs: Counter[_Stage] = Counter() unlabelled: Counter[_ClosedQuestion] = Counter() n_lines = 0 @@ -812,7 +785,7 @@ def _scan(qa_files: Sequence[_QAFile], counts: Mapping[_Part, _FileCounts]) -> _ scan.unlabelled.append(pair.place) unlabelled[question] += 1 continue - digests[pair.digest] += 1 + digests.append((pair.digest, pair.place)) if pair.stage.is_multi_row: scan.row_groups[pair.rows] = None else: @@ -820,20 +793,19 @@ def _scan(qa_files: Sequence[_QAFile], counts: Mapping[_Part, _FileCounts]) -> _ found = _FileCounts(n_lines, dict(pairs), dict(unlabelled)) if found != counts[qa.part]: raise TimeFValidationError(f"{qa.path}: expected {counts[qa.part]}, found {found}") - scan.digests[qa.part] = digests + labelled[qa.part] = digests + test_digests = {digest for digest, _ in labelled[_Part.TEST]} + scan.test_pairs_in_train = [place for digest, place in labelled[_Part.TRAIN] if digest in test_digests] if scan.unlabelled: _LOGGER.warning("%d closed pairs have an empty answer and get no task", len(scan.unlabelled)) + if scan.test_pairs_in_train: + _LOGGER.warning("%d train pairs repeat a test pair and get no task", len(scan.test_pairs_in_train)) return scan -def _unlabelled_annotation(places: Sequence[_Place]) -> Annotation: +def _places_annotation(key: _Key, places: Sequence[_Place], description: str) -> Annotation: pairs: list[JsonValue] = [place.metadata for place in places] - return Annotation( - _Key.UNLABELLED_PAIRS, - value=len(places), - description="How many closed pairs have an empty published answer. They are missing labels and have no task.", - metadata={"pairs": pairs}, - ) + return Annotation(key, value=len(places), description=description, metadata={"pairs": pairs}) class EngineMTQAConnector( @@ -879,7 +851,7 @@ def convert(self, files: Mapping[str, Path]) -> TimeFDataset: files: The downloaded file of each part. Returns: - A dataset with the Records, the unlabelled pairs as an annotation, and a stream of every task. + A dataset with the Records, the pairs without a task as annotations, and a stream of the other tasks. Raises: TimeFValidationError: If a file is malformed or has other counts. @@ -896,18 +868,32 @@ def convert(self, files: Mapping[str, Path]) -> TimeFDataset: } records |= {rows: _joined_record(rows, lengths, joined_reader) for rows in scan.row_groups} - dataset = TimeFDataset(metadata=self.metadata(), annotations=[_unlabelled_annotation(scan.unlabelled)]) + annotations = [ + _places_annotation( + _Key.UNLABELLED_PAIRS, + scan.unlabelled, + "How many closed pairs have an empty published answer. They are missing labels and have no task.", + ), + _places_annotation( + _Key.TEST_PAIRS_IN_TRAIN, + scan.test_pairs_in_train, + "How many train pairs repeat a test pair with the same rows, question and answer. They have no task.", + ), + ] + dataset = TimeFDataset(metadata=self.metadata(), annotations=annotations) for record in records.values(): dataset.add_record(record) + left_out = set(scan.test_pairs_in_train) + def tasks() -> Iterator[Task]: for qa in qa_files: for pair in qa.pairs(): - if not pair.has_label: + if not pair.has_label or pair.place in left_out: continue if pair.rows not in records: raise TimeFFormatError(f"{qa.path}: {pair.place} names rows {pair.rows} without a Record") - yield pair.task(records[pair.rows], scan.issues(pair)) + yield pair.task(records[pair.rows]) dataset.set_task_stream((AnswerTask, ClassificationTask), tasks) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml index 1b6d287fb..fa7a8a564 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/dataset.yaml @@ -1,23 +1,17 @@ # yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json # Dataset Card for the pandalin98/enginemt-qa connector: the single source for this dataset's metadata. -# -# Training convention: the tasks are the authors' four stages, one task per labelled QA pair, split as -# the authors' files are (train_qa.jsonl, test_qa.jsonl). Closed questions are classifications over -# their option letters. 1,769 closed pairs with an empty answer have no task. A single-row question -# reads its row at 1 Hz. A multi-row question reads the first 60 steps of its 10 rows, joined on an -# ordinal axis, as the authors' code does. The padding zeros at the end of a flight are not stored. -# The train file holds copies of 4,039 test lines. Their tasks name repeated_across_splits in their -# source_issues metadata. yaml_schema_version: 1 dataset_id: pandalin98/enginemt-qa dataset_version: 1.0.0 name: EngineMT-QA description: >- - Aero-engine time-series question answering over simulated N-CMAPSS flights: 142,273 labelled - QA pairs (100,296 train, 41,977 test) in four tasks. Understanding and Perception ask about one + Aero-engine time-series question answering over simulated N-CMAPSS flights: 114,312 labelled + QA pairs (72,335 train, 41,977 test) in four tasks. Understanding and Perception ask about one 10-minute window of 33 unnamed, scaled channels. Reasoning and Decision-Making ask about 10 windows from successive flights. Perception and Reasoning are multiple choice. Understanding and Decision-Making are open-ended answers written with LLMs and checked by experts. + The released train file repeats 4,039 test lines. The train split leaves out every pair that + repeats a test pair. license: MIT source_url: https://huggingface.co/datasets/pandalin98/EngineMT-QA source_revision: ea2051a15512212976c22c1a075ec549c7d48eef diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py index 42d142361..aa8e04215 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/pandalin98/enginemt_qa/tests/test_connector.py @@ -165,9 +165,10 @@ def test_converts_the_fixture(dataset): ] views = _views(dataset) # Per single-row line: 8 answer and 7 classification tasks, less one for an empty answer. - # Per multi-row line: 2 classification tasks and 1 answer task. + # Per multi-row line: 2 classification tasks and 1 answer task. Train line 2 repeats test line 0 + # and has none. single = ["AnswerTask"] * 8 + ["ClassificationTask"] * 7 - train = single[:-1] + single + single + ["ClassificationTask"] * 2 + ["AnswerTask"] + train = single[:-1] + single + ["ClassificationTask"] * 2 + ["AnswerTask"] test = single + ["ClassificationTask"] * 2 + ["AnswerTask"] + single[:-2] + single[-1:] assert [view.kind for view in views] == train + test assert [view.split for view in views] == ["train"] * len(train) + ["test"] * len(test) @@ -206,7 +207,6 @@ def test_closed_questions_keep_their_place_and_label_set(dataset): "source_line": 1, "qa_offset": 13, "stage": 2, - "source_issues": [], "closed_question": "fault_cause", }, ) @@ -224,7 +224,7 @@ def test_open_questions_keep_the_published_text(dataset): prompt=lines[1]["conversations"][4]["value"], targets=(lines[1]["conversations"][5]["value"],), label_set=None, - metadata={"source_file": _TEST, "source_line": 1, "qa_offset": 2, "stage": 4, "source_issues": []}, + metadata={"source_file": _TEST, "source_line": 1, "qa_offset": 2, "stage": 4}, ) @@ -248,22 +248,22 @@ def test_empty_closed_answers_get_no_task_and_are_listed(dataset): ] -def test_pairs_repeated_across_files_are_flagged_on_both_sides(dataset): - # Test line 0 is a byte copy of train line 2. The multi-row lines name other rows. - issues = {(view.split, view.metadata["source_line"]): view.metadata["source_issues"] for view in _views(dataset)} - assert issues["train", 2] == ["repeated_across_splits"] - assert issues["test", 0] == ["repeated_across_splits"] - assert issues["train", 3] == issues["test", 1] == [] +def test_train_pairs_that_repeat_a_test_pair_are_listed(dataset): + # Train line 2 is a byte copy of test line 0. + repeats = next(a for a in dataset.annotations if a.key == "test_pairs_in_train") + assert repeats.value == 15 + assert repeats.metadata["pairs"] == [ + {"source_file": _TRAIN, "source_line": 2, "qa_offset": offset} for offset in range(15) + ] -def test_a_pair_repeated_in_its_file_is_flagged(files, tmp_path, monkeypatch): +def test_a_train_pair_with_another_answer_than_the_test_pair_keeps_its_task(files, tmp_path): train = _lines(_TRAIN) - _write_qa(tmp_path / _TRAIN, [*train, train[3]]) - counts = _COUNTS[_Part.TRAIN] - pairs = {**counts.pairs, _Stage.REASONING: 4, _Stage.DECISION_MAKING: 2} - monkeypatch.setitem(_COUNTS, _Part.TRAIN, counts._replace(lines=5, pairs=pairs)) + conversations = [dict(turn) for turn in train[2]["conversations"]] + conversations[1]["value"] += " It stays steady." + _write_qa(tmp_path / _TRAIN, [*train[:2], {**train[2], "conversations": conversations}, train[3]]) views = _views(EngineMTQAConnector().convert(files)) - assert {tuple(v.metadata["source_issues"]) for v in views if v.rows == _MULTI_TRAIN} == {("repeated_question",)} + assert [v.metadata["qa_offset"] for v in views if v.split == "train" and v.metadata["source_line"] == 2] == [0] def test_a_row_record_holds_the_rows_real_steps_at_1_hz(dataset):