diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/__init__.py new file mode 100644 index 000000000..cf5292cb5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/__init__.py @@ -0,0 +1 @@ +"""Connectors for datasets released by Fraunhofer IEE.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/__init__.py new file mode 100644 index 000000000..a136a3289 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/__init__.py @@ -0,0 +1,6 @@ +"""The CARE to Compare connector package: its connector implementation and dataset card.""" + +from timenet_connectors.datasets.fraunhofer_iee.care_to_compare.connector import ( + CONNECTOR as CONNECTOR, + CareToCompareConnector as CareToCompareConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/connector.py new file mode 100644 index 000000000..b1254e130 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/connector.py @@ -0,0 +1,374 @@ +"""The CARE to Compare connector: wind turbine SCADA data with labeled pre-fault anomaly events. + +Each event file of the release becomes one Record. The Record holds one turbine's 10-minute SCADA +signals (one Signal per sensor statistic) and its operating status, over about a year of training +data followed by a prediction period. The event table becomes annotations: the training and +prediction periods, the event window, and the event's label and root-cause note. Each Record gets a +ClassificationTask that asks whether the prediction period holds an anomaly. Each anomaly event also +gets a TemporalLocalizationTask for its window and, for wind farms A and B, a ClassificationTask for +the faulty component. + +:mod:`~timenet_connectors.datasets.fraunhofer_iee.care_to_compare.source` parses the files. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path +import re +from typing import Self + +import numpy as np +import pyarrow as pa + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ( + Annotation, + ClassificationTask, + Label, + LabelSet, + LocalizationMode, + MissingReason, + MissingValues, + TemporalLocalizationTask, + TimeInterval, + TimeOrigin, + TimeSeriesSpec, + ureg, +) +from timenet_connectors.datasets.fraunhofer_iee.care_to_compare.source import ( + PERIOD_US, + EventInfo, + SensorDescription, + read_event_info, + read_event_table, + read_sensor_descriptions, + split_feature_column, +) +from timenet_connectors.download import ensure_archive + + +ZENODO_URL = "https://zenodo.org/records/15846963/files/CARE_To_Compare.zip?download=1" +"""Version 6 of the Zenodo record, the release with the corrected labels.""" +ARCHIVE_NAME = "CARE_To_Compare.zip" +ARCHIVE_SHA256 = "ca61379e98956d891041ad45c885109bd8a14199fde0688d0184a11c2d4194f1" +"""The SHA-256 hash of the archive, from a download whose MD5 matches the +``2547b58c21ac8c242d13232860cf500c`` that Zenodo publishes.""" + +PERIOD_S = PERIOD_US / 1_000_000 +_FARM_DIRECTORY = re.compile(r"^Wind Farm ([A-Z])$") + +EVENT_LABEL = LabelSet( + [ + Label( + name="anomaly", + description="The prediction period holds an anomaly that leads up to a documented turbine fault.", + ), + Label(name="normal", description="The prediction period shows normal behavior."), + ], + annotations=[Annotation("name", value="event_label")], +) +FAULT_COMPONENT = LabelSet( + [ + Label(name="hydraulic", description="The hydraulic group."), + Label(name="gearbox", description="The gearbox or its bearings."), + Label(name="generator_bearing", description="A generator bearing."), + Label(name="transformer", description="The transformer or its cell."), + Label(name="rotor_bearing", description="The rotor (main) bearing."), + ], + annotations=[Annotation("name", value="fault_component")], +) + +DETECTION_PROMPT = ( + "Does the prediction period of this wind turbine's SCADA data show an anomaly that leads up to a turbine fault?" +) +LOCALIZATION_PROMPT = "Locate the anomaly in the prediction period that leads up to the turbine fault." +COMPONENT_PROMPT = "Which component does the anomaly in the prediction period lead up to a fault of?" +EVENT_WINDOW = ( + "The event of the prediction period, labeled anomaly or normal. For an anomaly event, the window starts " + "where the authors estimate the anomaly starts and ends where the turbine fault starts." +) + +SCADA_MISSING = MissingValues( + reason=MissingReason.NOT_RECORDED, + description=( + "The SCADA system recorded no row for this 10-minute step, or the row left the cell empty. " + "Wind farms B and C were delivered with every missing value replaced by 0, so a run of zeros " + "there can also be a missing value." + ), +) +STATUS = TimeSeriesSpec( + spec_type="care_status_type_id", + name="Turbine operating status (0 normal, 1 derated, 2 idling, 3 service, 4 downtime, 5 other)", + unit_value=None, + dtype="int8", + missing=MissingValues( + reason=MissingReason.NOT_RECORDED, + description="The SCADA system recorded no row for this 10-minute step.", + ), +) +STATISTIC_NAMES = { + "average": "10-minute mean", + "minimum": "10-minute minimum", + "maximum": "10-minute maximum", + "std_dev": "10-minute standard deviation", +} + + +def _load_column(cache_path: Path, index: int) -> pa.Array: + """Read one feature row of a cached event matrix. + + Returns: + The values as a float32 Arrow array, with a null for each NaN. + """ + values = np.ascontiguousarray(np.load(cache_path, mmap_mode="r")[index]) + return pa.array(values, type=pa.float32(), mask=np.isnan(values)) + + +def _column_loader(cache_path: Path, index: int) -> Callable[[], pa.Array]: + return lambda: _load_column(cache_path, index) + + +def _interval(first_step: int, last_step: int) -> TimeInterval: + """The interval from one grid step through another, on the Record timeline. + + Returns: + The half-open interval that holds both steps. + """ + return TimeInterval.seconds(first_step * PERIOD_S, (last_step + 1) * PERIOD_S) + + +@dataclass(frozen=True) +class CareWindFarm: + """One wind farm directory of the release.""" + + name: str + """The wind farm's letter: ``A``, ``B``, or ``C``.""" + directory: Path + + @classmethod + def discover(cls, root: Path) -> list[Self]: + """Find the wind farm directories under an extraction root. + + Returns: + The wind farms in letter order. + + Raises: + TimeNetConnectorError: If the root holds no wind farm directory. + """ + farms = [] + for info in sorted(root.rglob("event_info.csv")): + match = _FARM_DIRECTORY.match(info.parent.name) + if match: + farms.append(cls(name=match.group(1), directory=info.parent)) + if not farms: + raise TimeNetConnectorError(f"no 'Wind Farm /event_info.csv' under {root}") + return sorted(farms, key=lambda farm: farm.name) + + @property + def event_info_path(self) -> Path: + """The wind farm's event table.""" + return self.directory / "event_info.csv" + + @property + def feature_description_path(self) -> Path: + """The wind farm's sensor descriptions.""" + return self.directory / "feature_description.csv" + + def event_path(self, event_id: int) -> Path: + """The SCADA file of one event. + + Returns: + The path of ``datasets/.csv``. + """ + return self.directory / "datasets" / f"{event_id}.csv" + + +class _FarmConverter: + """Turns the events of one wind farm into Records and tasks.""" + + def __init__(self, farm: CareWindFarm) -> None: + self.farm = farm + self.sensors = read_sensor_descriptions(farm.feature_description_path) + self.events = read_event_info(farm.event_info_path) + self._specs: dict[str, TimeSeriesSpec] = {} + self._axis = RegularAxis(period_us=Fraction(PERIOD_US), offset_us=0) + files = {int(path.stem) for path in (farm.directory / "datasets").glob("*.csv") if path.stem.isdigit()} + if files != set(self.events): + raise TimeNetConnectorError( + f"Wind Farm {farm.name}: event files {sorted(files ^ set(self.events))} do not match event_info.csv" + ) + + def _sensor(self, column: str) -> tuple[SensorDescription, str]: + # Wind farm A names its energy counters (sensor_44 to sensor_51) without a statistic suffix. + # Each of them has one statistic in the feature description. + bare = self.sensors.get(column) + if bare is not None and len(bare.statistics) == 1: + return bare, bare.statistics[0] + sensor_name, statistic = split_feature_column(column) + sensor = self.sensors.get(sensor_name) + if sensor is None: + raise TimeNetConnectorError(f"Wind Farm {self.farm.name}: column {column} has no feature description") + return sensor, statistic + + def _spec(self, column: str) -> TimeSeriesSpec: + """The spec of one sensor statistic, shared by every Record of the wind farm.""" + spec = self._specs.get(column) + if spec is None: + sensor, statistic = self._sensor(column) + unit = sensor.unit_expression + spec = TimeSeriesSpec( + spec_type=f"care_{self.farm.name.lower()}_{column}", + name=f"{sensor.description} ({STATISTIC_NAMES[statistic]})", + unit_value=ureg.parse_units(unit) if unit else None, + dtype="float32", + missing=SCADA_MISSING, + ) + self._specs[column] = spec + return spec + + def _signal_annotations(self, column: str) -> list[Annotation]: + sensor, statistic = self._sensor(column) + annotations = [Annotation("statistic", value=statistic)] + if sensor.unit and sensor.unit_expression is None: + annotations.append(Annotation("source_unit", value=sensor.unit)) + if sensor.is_angle: + annotations.append(Annotation("is_angle", value=True)) + if sensor.is_counter: + annotations.append(Annotation("is_counter", value=True)) + return annotations + + def add_event(self, dataset: TimeFDataset, event: EventInfo) -> None: + """Read one event file and add its Record and tasks. + + Raises: + TimeNetConnectorError: If the file disagrees with its event_info row. + """ + path = self.farm.event_path(event.event_id) + table = read_event_table(path) + where = f"Wind Farm {self.farm.name} event {event.event_id}" + if table.asset_id != event.asset_id: + raise TimeNetConnectorError(f"{where}: file asset {table.asset_id} is not {event.asset_id}") + try: + window = (table.step_of(event.start_us), table.step_of(event.end_us)) + except TimeNetConnectorError as exc: + raise TimeNetConnectorError(f"{where}: event window: {exc}") from exc + + # Keep the values on disk, one contiguous row per feature, so each Signal loads only its own. + cache_path = path.with_suffix(".values.npy") + np.save(cache_path, table.values) + signals = [ + Signal.from_loader( + spec=self._spec(column), + name=column, + time_axis=self._axis, + loader=_column_loader(cache_path, index), + n_values=table.n_steps, + annotations=self._signal_annotations(column), + ) + for index, column in enumerate(table.feature_columns) + ] + status = Signal.from_values( + pa.array(table.status, type=pa.int8(), mask=table.status < 0), + spec=STATUS, + name="status_type_id", + time_axis=self._axis, + ) + prediction = _interval(*table.prediction_steps) + facts = [ + Annotation("wind_farm", value=self.farm.name), + Annotation("event_id", value=event.event_id), + Annotation("asset_id", value=event.asset_id), + Annotation("event_label", value=event.label), + Annotation("training_period", span=_interval(*table.train_steps)), + Annotation("prediction_period", span=prediction), + ] + if event.description is not None: + facts.append(Annotation("event_description", value=event.description)) + component = event.fault_component(self.farm.name) + if component is not None: + facts.append(Annotation("fault_component", value=component)) + record = Record( + metadata={"source_record_id": f"care-{self.farm.name.lower()}-{event.event_id}"}, + start_time=TimeOrigin(table.start_us), + sources=[ + Source("SCADA", signals=signals), + Source("Turbine controller", signals=[status]), + ], + annotations=facts, + ) + dataset.add_record(record) + event_window = record.annotate( + Annotation("event_window", value=event.label, description=EVENT_WINDOW, span=_interval(*window)) + ) + + detection = ClassificationTask( + inputs=[record], + prompt=DETECTION_PROMPT, + targets=event.label, + label_set=EVENT_LABEL, + scope=prediction, + ) + tasks: list[ClassificationTask | TemporalLocalizationTask] = [detection] + if event.label == "anomaly": + tasks.append( + TemporalLocalizationTask( + inputs=[record], + prompt=LOCALIZATION_PROMPT, + mode=LocalizationMode.SPARSE, + scope=prediction, + target_annotations=(event_window,), + from_tasks=(detection,), + ) + ) + if component is not None: + tasks.append( + ClassificationTask( + inputs=[record], + prompt=COMPONENT_PROMPT, + targets=component, + label_set=FAULT_COMPONENT, + scope=prediction, + from_tasks=(detection,), + ) + ) + dataset.add_tasks(tasks) + + +class CareToCompareConnector(BaseConnector[list[CareWindFarm]]): + """Connector for CARE to Compare, version 6 of the Zenodo record.""" + + async def download_async(self, cache_dir: Path) -> list[CareWindFarm]: # ruff: ignore[no-self-use] + """Download and extract the release archive once. + + Args: + cache_dir: The directory that holds the extracted files. + + Returns: + The wind farm directories. + """ + root = await ensure_archive(ZENODO_URL, cache_dir, filename=ARCHIVE_NAME, sha256=ARCHIVE_SHA256) + return CareWindFarm.discover(Path(root)) + + def convert(self, farms: list[CareWindFarm]) -> TimeFDataset: + """Create one Record and its tasks for each event of each wind farm. + + Args: + farms: The wind farm directories from :meth:`download_async`. + + Returns: + The populated dataset. + """ + dataset = TimeFDataset(metadata=self.metadata()) + for farm in farms: + converter = _FarmConverter(farm) + for event in sorted(converter.events.values(), key=lambda event: event.event_id): + converter.add_event(dataset, event) + return dataset + + +CONNECTOR = CareToCompareConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/dataset.yaml new file mode 100644 index 000000000..5e4605bf5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/dataset.yaml @@ -0,0 +1,56 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# Dataset Card for the fraunhofer-iee/care-to-compare connector: the single source for this +# dataset's metadata. +# +# Scope: all three wind farms of version 6 of the Zenodo record. Each event file becomes one Record +# with one Signal per sensor statistic, as released (no resampling, no cleaning, no unit conversion). +# The release has no split across events. Within each Record, the `training_period` annotation marks +# the rows meant for fitting a normal-behavior model and `prediction_period` the rows to evaluate. +yaml_schema_version: 1 +dataset_id: fraunhofer-iee/care-to-compare +dataset_version: 1.0.0 +source_revision: zenodo-15846963-v6 +name: CARE to Compare +description: >- + Real-world SCADA data of 36 wind turbines from three wind farms (A: 5 onshore turbines in + Portugal, from the EDP open data platform; B: 9 and C: 22 offshore turbines in Germany), released + by Fraunhofer IEE as a benchmark for early fault detection. There are 95 Records, one per event + file: 45 anomaly events that lead up to a documented turbine fault and 50 normal events. Each + Record holds one turbine's 10-minute SCADA signals over about one year of training data followed by + 4 to 98 days of prediction data that contain the event window. Wind farm A has 81 sensor signals, + B 252, and C 952: the 10-minute mean of every sensor and, for some sensors, the 10-minute minimum, + maximum and standard deviation. A further signal holds the turbine's operating status (0 normal, + 1 derated, 2 idling, 3 service, 4 downtime, 5 other); for wind farm A the release derived it from a + fault logbook (14 days before each fault marked as downtime, 3 days after as service), so it is + meant for filtering training data, not for evaluation. A skipped 10-minute step or an empty cell is + a null. Wind farms B and C were delivered with missing values replaced by 0, and the release notes + that many of their minimum, maximum and standard deviation values are implausible; the 10-minute + means are reliable. Sensor names are anonymized; each Signal's spec carries the release's sensor + description and unit. Reactive power, reactive energy, mG and DL units have no pint form and are + kept in a `source_unit` annotation. Timestamps were shifted by whole years so that each file starts + in 2022. Annotations give the wind farm, asset, event label, the training and prediction periods, + the event window and the operator's root-cause note. Each Record has a ClassificationTask for the + event label over the prediction period. Each anomaly Record also has a TemporalLocalizationTask + for the event window. For the 18 anomaly events of wind farms A and B, a ClassificationTask gives + the faulty component (hydraulic, gearbox, generator_bearing, transformer or rotor_bearing); TimeNet + derived these labels from the release's root-cause notes. The event windows' starts were estimated + by the authors and can be later than the true onset of an anomaly. The release defines the CARE + score (coverage, accuracy, reliability, earliness) for evaluation. +license: CC-BY-SA-4.0 +license_url: https://creativecommons.org/licenses/by-sa/4.0/ +source_url: https://doi.org/10.5281/zenodo.15846963 +citation: >- + Gück, C., Roelofs, C. M. A., & Faulstich, S. (2024). CARE to Compare: A Real-World Benchmark + Dataset for Early Fault Detection in Wind Turbine Data. Data, 9(12), 138. + https://doi.org/10.3390/data9120138 +domains: + - energy + - industrial +tags: + - wind-turbine + - scada + - anomaly-detection + - fault-detection + - predictive-maintenance + - condition-monitoring + - multivariate diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/source.py new file mode 100644 index 000000000..ebbfe4de6 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/source.py @@ -0,0 +1,492 @@ +"""Read the files of the CARE to Compare release into plain NumPy arrays. + +The release has one directory per wind farm. Each directory holds ``feature_description.csv``, +``event_info.csv``, and one ``datasets/.csv`` per event. All three are semicolon-separated. +An event file holds one turbine's SCADA rows at a 10-minute resolution: about a year of training rows +followed by the prediction rows that contain the event. The rows are sorted and lie on the 10-minute +grid, with a few skipped steps. + +This module only parses and validates. It imports no TimeNet type, so the connector can turn its +output into Records and the tests can check the parsing on its own. +""" + +from __future__ import annotations + +from collections.abc import Iterator +import csv +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np + +from timenet.errors import TimeNetConnectorError + + +PERIOD_US = 600_000_000 +"""One step of the SCADA grid: 10 minutes, in microseconds.""" + +DESCRIPTIVE_COLUMNS = ("time_stamp", "asset_id", "id", "train_test", "status_type_id") +"""The leading columns of every event file. The sensor columns follow them.""" + +TRAIN = "train" +PREDICTION = "prediction" + +STATISTIC_SUFFIXES = {"average": "avg", "minimum": "min", "maximum": "max", "std_dev": "std"} +"""The statistic names of ``feature_description.csv`` and their column suffixes in the event files.""" +SUFFIX_STATISTICS = {suffix: statistic for statistic, suffix in STATISTIC_SUFFIXES.items()} + +STATUS_MISSING = -1 +"""The status value of a grid step without a row. The release uses the IDs 0 to 5.""" + +TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S" + +# The unit strings of the three feature descriptions, as pint expressions. Wind farm A's file stores +# the degree sign as U+FFFD, so both spellings map to the same unit. The release states reactive +# power and reactive energy (kVAr, kvar, VArh, kvarh), a vibration unit (mG), and "DL" without a +# definition that pint can express without guessing, so the connector keeps their source string +# as an annotation instead. +UNIT_EXPRESSIONS: dict[str, str] = { + "°C": "degree_Celsius", + "�C": "degree_Celsius", + "Celsius": "degree_Celsius", + "°": "degree", + "�": "degree", + "deg": "degree", + "m/s": "meter / second", + "m/s^2": "meter / second ** 2", + "rpm": "revolution / minute", + "1/min": "revolution / minute", + "rpm/s": "revolution / minute / second", + "rad/s": "radian / second", + "A": "ampere", + "V": "volt", + "Hz": "hertz", + "mHz": "millihertz", + "kW": "kilowatt", + "Wh": "watt * hour", + "kWh": "kilowatt * hour", + "kVA": "kilovolt * ampere", + "Nm": "newton * meter", + "kNm": "kilonewton * meter", + "%": "percent", + "bar": "bar", + "hPa": "hectopascal", + "Pa": "pascal", + "L/min": "liter / minute", + "l/h": "liter / hour", + "L": "liter", +} + +# The fault component of each anomaly event whose free-text description names one unambiguously. +# The keys are the lowercased descriptions of wind farms A and B. Wind farm C's descriptions are +# operator notes that often name several components, so the connector derives no component for them. +FAULT_COMPONENTS: dict[tuple[str, str], str] = { + ("A", "transformer failure"): "transformer", + ("A", "hydraulic group"): "hydraulic", + ("A", "gearbox failure"): "gearbox", + ("A", "gearbox bearings damaged"): "gearbox", + ("A", "generator bearing failure"): "generator_bearing", + ("B", "high temperature in transformer cell"): "transformer", + ("B", "rotor bearing 2 - damage"): "rotor_bearing", + ("B", "turbine is stopped due to a main bearing damage"): "rotor_bearing", + ("B", "turbine is in standstill since 01.08 due to rotorbearing damage."): "rotor_bearing", +} + + +def parse_timestamp_us(text: str) -> int: + """Parse a release timestamp such as ``2023-08-12 09:50:00`` into microseconds since the epoch. + + The release states no time zone and shifts every year for anonymization, so the value is read + as UTC and stands only for the position in time, not a true calendar date. + + Returns: + The timestamp in microseconds since 1970-01-01 UTC. + + Raises: + TimeNetConnectorError: If the text is not in the release format. + """ + try: + moment = datetime.strptime(text.strip(), TIMESTAMP_FORMAT).replace(tzinfo=UTC) + except ValueError as exc: + raise TimeNetConnectorError(f"timestamp {text!r} is not in the format {TIMESTAMP_FORMAT}") from exc + return int(moment.timestamp()) * 1_000_000 + + +def _parse_bool(text: str, *, field: str) -> bool: + if text == "True": + return True + if text == "False": + return False + raise TimeNetConnectorError(f"{field} must be True or False, not {text!r}") + + +def _read_rows(path: Path) -> tuple[list[str], list[list[str]]]: + """Read a semicolon-separated file into its header and rows. + + Returns: + The header and the data rows. + + Raises: + TimeNetConnectorError: If the file is empty. + """ + with path.open(newline="", encoding="utf-8") as stream: + rows = list(csv.reader(stream, delimiter=";")) + if not rows: + raise TimeNetConnectorError(f"{path.name} is empty") + return rows[0], [row for row in rows[1:] if row] + + +@dataclass(frozen=True) +class SensorDescription: + """One row of ``feature_description.csv``.""" + + sensor_name: str + """The anonymized sensor name, such as ``sensor_11`` or ``wind_speed_3``.""" + statistics: tuple[str, ...] + """The 10-minute statistics that the event files hold for the sensor, such as ``average``.""" + description: str + """The release's description of the sensor.""" + unit: str + """The unit string of the release. It can be empty.""" + is_angle: bool + is_counter: bool + + @property + def unit_expression(self) -> str | None: + """The unit as a pint expression, or ``None`` when the release unit has no safe mapping.""" + return UNIT_EXPRESSIONS.get(self.unit) + + +FEATURE_DESCRIPTION_COLUMNS = ("sensor_name", "statistics_type", "description", "unit", "is_angle", "is_counter") + + +def read_sensor_descriptions(path: Path) -> dict[str, SensorDescription]: + """Read a wind farm's ``feature_description.csv``. + + Returns: + Each sensor name mapped to its description, in file order. + + Raises: + TimeNetConnectorError: If the header is not the expected one, a row is malformed, a statistic + is unknown, or a sensor repeats. + """ + header, rows = _read_rows(path) + if tuple(header) != FEATURE_DESCRIPTION_COLUMNS: + raise TimeNetConnectorError(f"{path.name} header {header} is not {list(FEATURE_DESCRIPTION_COLUMNS)}") + sensors: dict[str, SensorDescription] = {} + for number, row in enumerate(rows, start=1): + if len(row) != len(FEATURE_DESCRIPTION_COLUMNS): + raise TimeNetConnectorError(f"{path.name} row {number} has {len(row)} fields") + name, statistics_text, description, unit, is_angle, is_counter = (cell.strip() for cell in row) + statistics = tuple(statistic.strip() for statistic in statistics_text.split(",")) + unknown = [statistic for statistic in statistics if statistic not in STATISTIC_SUFFIXES] + if unknown: + raise TimeNetConnectorError(f"{path.name} row {number}: unknown statistics {unknown}") + if name in sensors: + raise TimeNetConnectorError(f"{path.name} row {number}: sensor {name} repeats") + sensors[name] = SensorDescription( + sensor_name=name, + statistics=statistics, + description=description, + unit=unit, + is_angle=_parse_bool(is_angle, field=f"{path.name} row {number} is_angle"), + is_counter=_parse_bool(is_counter, field=f"{path.name} row {number} is_counter"), + ) + return sensors + + +def split_feature_column(column: str) -> tuple[str, str]: + """Split an event-file column such as ``wind_speed_3_avg`` into the sensor and the statistic. + + Returns: + The sensor name and the statistic name, such as ``("wind_speed_3", "average")``. + + Raises: + TimeNetConnectorError: If the column does not end in a known statistic suffix. + """ + sensor, _, suffix = column.rpartition("_") + if not sensor or suffix not in SUFFIX_STATISTICS: + raise TimeNetConnectorError(f"column {column!r} does not end in one of {sorted(SUFFIX_STATISTICS)}") + return sensor, SUFFIX_STATISTICS[suffix] + + +@dataclass(frozen=True) +class EventInfo: + """One row of ``event_info.csv``.""" + + event_id: int + asset_id: int + label: str + """``anomaly`` or ``normal``.""" + start_us: int + """The first timestamp of the event window.""" + end_us: int + """The last timestamp of the event window. For an anomaly event, the turbine fault starts here.""" + description: str | None + """The operator's note on the root cause of an anomaly event, if the release gives one.""" + + def fault_component(self, wind_farm: str) -> str | None: + """The component that the description names, for wind farms A and B. + + Returns: + A label of the fault component label set, or ``None`` when the event is normal or the + description names no single component. + """ + if self.label != "anomaly" or self.description is None: + return None + return FAULT_COMPONENTS.get((wind_farm, self.description.strip().lower())) + + +EVENT_LABELS = ("anomaly", "normal") + + +def read_event_info(path: Path) -> dict[int, EventInfo]: + """Read a wind farm's ``event_info.csv``. + + Wind farm A names the asset column ``asset``. Wind farms B and C name it ``asset_id``. + + Returns: + Each event ID mapped to its row, in file order. + + Raises: + TimeNetConnectorError: If a column is missing, a row is malformed, or an event ID repeats. + """ + header, rows = _read_rows(path) + asset_column = "asset_id" if "asset_id" in header else "asset" + required = (asset_column, "event_id", "event_label", "event_start", "event_end", "event_description") + missing = [column for column in required if column not in header] + if missing: + raise TimeNetConnectorError(f"{path.name} lacks the columns {missing}") + index = {column: header.index(column) for column in required} + events: dict[int, EventInfo] = {} + for number, row in enumerate(rows, start=1): + if len(row) != len(header): + raise TimeNetConnectorError(f"{path.name} row {number} has {len(row)} fields, not {len(header)}") + try: + event_id = int(row[index["event_id"]]) + asset_id = int(row[index[asset_column]]) + except ValueError as exc: + raise TimeNetConnectorError(f"{path.name} row {number}: {exc}") from exc + label = row[index["event_label"]].strip() + if label not in EVENT_LABELS: + raise TimeNetConnectorError(f"{path.name} row {number}: event_label {label!r} is not one of {EVENT_LABELS}") + start_us = parse_timestamp_us(row[index["event_start"]]) + end_us = parse_timestamp_us(row[index["event_end"]]) + if end_us < start_us: + raise TimeNetConnectorError(f"{path.name} row {number}: the event ends before it starts") + if event_id in events: + raise TimeNetConnectorError(f"{path.name} row {number}: event {event_id} repeats") + description = row[index["event_description"]].strip() or None + events[event_id] = EventInfo( + event_id=event_id, + asset_id=asset_id, + label=label, + start_us=start_us, + end_us=end_us, + description=description, + ) + return events + + +@dataclass(frozen=True) +class EventTable: + """One event file placed on its 10-minute grid. + + Grid step ``i`` is the timestamp ``start_us + i * PERIOD_US``. A step that the file skips holds + NaN in ``values`` and :data:`STATUS_MISSING` in ``status``. + """ + + asset_id: int + start_us: int + """The first timestamp of the file.""" + feature_columns: tuple[str, ...] + """The sensor columns, in file order.""" + values: np.ndarray + """float32, shape ``(n_features, n_steps)``. NaN marks an empty cell or a skipped step.""" + status: np.ndarray + """int8, shape ``(n_steps,)``: the status ID of each step, or :data:`STATUS_MISSING`.""" + train_steps: tuple[int, int] + """The grid steps ``[first, last]`` of the training rows.""" + prediction_steps: tuple[int, int] + """The grid steps ``[first, last]`` of the prediction rows.""" + + @property + def n_steps(self) -> int: + """The number of 10-minute steps from the first to the last row.""" + return int(self.status.shape[0]) + + def step_of(self, timestamp_us: int) -> int: + """The grid step of a timestamp. + + Returns: + The step index. + + Raises: + TimeNetConnectorError: If the timestamp is off the grid or outside the file. + """ + offset = timestamp_us - self.start_us + if offset % PERIOD_US or not 0 <= offset // PERIOD_US < self.n_steps: + raise TimeNetConnectorError(f"timestamp {timestamp_us} us is off the grid or outside the file") + return offset // PERIOD_US + + +def _to_float32(cells: list[list[str]], path: Path, first_row: int) -> np.ndarray: + """Convert a block of sensor cells to float32, with NaN for empty cells. + + Returns: + The block, shape ``(n_rows, n_features)``. + + Raises: + TimeNetConnectorError: If a cell is not a number. + """ + try: + return np.array([[cell if cell else "nan" for cell in row] for row in cells], dtype=np.float32) + except ValueError as exc: + raise TimeNetConnectorError(f"{path.name} rows {first_row}+: {exc}") from exc + + +def _read_header(reader: Iterator[list[str]], path: Path) -> tuple[str, ...]: + """Read and check the header of an event file. + + Returns: + The sensor columns that follow the descriptive columns. + + Raises: + TimeNetConnectorError: If the file is empty, the descriptive columns do not lead, or a column + repeats. + """ + header = next(reader, None) + if header is None: + raise TimeNetConnectorError(f"{path.name} is empty") + if tuple(header[: len(DESCRIPTIVE_COLUMNS)]) != DESCRIPTIVE_COLUMNS: + raise TimeNetConnectorError(f"{path.name} does not start with the columns {list(DESCRIPTIVE_COLUMNS)}") + feature_columns = tuple(header[len(DESCRIPTIVE_COLUMNS) :]) + if len(set(feature_columns)) != len(feature_columns): + raise TimeNetConnectorError(f"{path.name} repeats a column") + return feature_columns + + +@dataclass +class _EventRows: + """The rows of an event file, before they are placed on the grid.""" + + times: list[int] = field(default_factory=list) + statuses: list[int] = field(default_factory=list) + is_train: list[bool] = field(default_factory=list) + assets: set[str] = field(default_factory=set) + blocks: list[np.ndarray] = field(default_factory=list) + + def add_descriptive(self, row: list[str], number: int, path: Path) -> None: + """Check and keep the descriptive cells of one row. + + Raises: + TimeNetConnectorError: If the row ID is not the row index, or a cell is malformed. + """ + if row[2] != str(number): + raise TimeNetConnectorError(f"{path.name} row {number}: id {row[2]!r} is not the row index") + if row[3] not in {TRAIN, PREDICTION}: + raise TimeNetConnectorError(f"{path.name} row {number}: train_test {row[3]!r} is unknown") + try: + status = int(row[4]) + except ValueError as exc: + raise TimeNetConnectorError(f"{path.name} row {number}: status_type_id {row[4]!r}") from exc + self.times.append(parse_timestamp_us(row[0])) + self.assets.add(row[1]) + self.is_train.append(row[3] == TRAIN) + self.statuses.append(status) + + +def _read_event_rows(path: Path, block_rows: int) -> tuple[tuple[str, ...], _EventRows]: + """Read the header and the rows of an event file. + + Returns: + The sensor columns and the rows. + + Raises: + TimeNetConnectorError: If the header or a row is malformed. + """ + rows = _EventRows() + with path.open(newline="", encoding="utf-8") as stream: + reader = csv.reader(stream, delimiter=";") + feature_columns = _read_header(reader, path) + width = len(DESCRIPTIVE_COLUMNS) + len(feature_columns) + cells: list[list[str]] = [] + for number, row in enumerate(reader): + if not row: + continue + if len(row) != width: + raise TimeNetConnectorError(f"{path.name} row {number}: {len(row)} fields, not {width}") + rows.add_descriptive(row, number, path) + cells.append(row[len(DESCRIPTIVE_COLUMNS) :]) + if len(cells) == block_rows: + rows.blocks.append(_to_float32(cells, path, number - block_rows + 1)) + cells = [] + if cells: + rows.blocks.append(_to_float32(cells, path, len(rows.times) - len(cells))) + if not rows.times: + raise TimeNetConnectorError(f"{path.name} has no rows") + if len(rows.assets) != 1: + raise TimeNetConnectorError(f"{path.name} holds the assets {sorted(rows.assets)}, not one") + return feature_columns, rows + + +def _grid_steps(rows: _EventRows, path: Path) -> tuple[np.ndarray, tuple[int, int], tuple[int, int]]: + """Map each row to its step on the 10-minute grid. + + Returns: + The step of each row, and the first and last steps of the training and prediction rows. + + Raises: + TimeNetConnectorError: If the timestamps are off the grid or not rising, or the training rows + do not all precede the prediction rows. + """ + epochs = np.array(rows.times, dtype=np.int64) + offsets = epochs - epochs[0] + if np.any(offsets % PERIOD_US) or np.any(np.diff(offsets) <= 0): + raise TimeNetConnectorError(f"{path.name}: timestamps are off the 10-minute grid or not rising") + steps = offsets // PERIOD_US + train = np.array(rows.is_train, dtype=bool) + train_rows = np.flatnonzero(train) + prediction_rows = np.flatnonzero(~train) + if not len(train_rows) or not len(prediction_rows): + raise TimeNetConnectorError(f"{path.name} lacks training or prediction rows") + if train_rows[-1] > prediction_rows[0]: + raise TimeNetConnectorError(f"{path.name}: a training row follows a prediction row") + return ( + steps, + (int(steps[train_rows[0]]), int(steps[train_rows[-1]])), + (int(steps[prediction_rows[0]]), int(steps[prediction_rows[-1]])), + ) + + +def read_event_table(path: Path, *, block_rows: int = 4096) -> EventTable: + """Read one event file and place its rows on the 10-minute grid. + + The parser checks the properties the connector relies on: the descriptive columns lead, the row + IDs count up from zero, the timestamps rise on the 10-minute grid, every row has one asset, and the + training rows precede the prediction rows. + + Args: + path: The event file. + block_rows: How many rows to convert to float at a time, to bound memory. + + Returns: + The file's values on the grid. + """ + feature_columns, rows = _read_event_rows(path, block_rows) + steps, train_steps, prediction_steps = _grid_steps(rows, path) + n_steps = int(steps[-1]) + 1 + values = np.full((len(feature_columns), n_steps), np.nan, dtype=np.float32) + values[:, steps] = np.concatenate(rows.blocks, axis=0).T + status = np.full(n_steps, STATUS_MISSING, dtype=np.int8) + status[steps] = np.array(rows.statuses, dtype=np.int8) + return EventTable( + asset_id=int(next(iter(rows.assets))), + start_us=int(rows.times[0]), + feature_columns=feature_columns, + values=values, + status=status, + train_steps=train_steps, + prediction_steps=prediction_steps, + ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/fixtures.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/fixtures.py new file mode 100644 index 000000000..883a42c2b --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/fixtures.py @@ -0,0 +1,82 @@ +"""Write a small synthetic wind farm directory in the shape of the CARE to Compare release.""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from pathlib import Path + + +FEATURE_DESCRIPTION = ( + "sensor_name;statistics_type;description;unit;is_angle;is_counter\r\n" + "sensor_0;average;Ambient temperature;�C;False;False\r\n" + "wind_speed_3;maximum,minimum,average,std_dev;Windspeed;m/s;False;False\r\n" + "sensor_5;average;Pitch angle;�;True;False\r\n" + "sensor_31;average;Grid reactive power;kVAr;False;False\r\n" + "sensor_44;average;Active power - generator disconnected;Wh;False;False\r\n" +) +FEATURE_COLUMNS = ( + "sensor_0_avg", + "wind_speed_3_avg", + "wind_speed_3_max", + "wind_speed_3_min", + "wind_speed_3_std", + "sensor_5_avg", + "sensor_31_avg", + "sensor_44", +) +"""The release's column layout, including a counter that wind farm A names without a statistic suffix.""" +START = datetime(2022, 8, 12, 9, 50) +STEP = timedelta(minutes=10) +N_TRAIN = 20 +N_PREDICTION = 10 +SKIPPED_STEP = 7 +"""A grid step that the event files leave out, like the gaps of the release.""" +EMPTY_CELL = (3, 1) +"""A (row, feature) cell that the event files leave empty.""" +ANOMALY_EVENT = 22 +NORMAL_EVENT = 24 +ANOMALY_START_STEP = N_TRAIN + 2 +ANOMALY_END_STEP = N_TRAIN + 6 + + +def timestamp(step: int) -> str: + return (START + step * STEP).strftime("%Y-%m-%d %H:%M:%S") + + +def value(step: int, feature: int) -> float: + return round(step + feature / 10, 1) + + +def write_event_file(path: Path, *, asset: int, steps: list[int] | None = None) -> None: + """Write one event file. The rows follow the grid except at :data:`SKIPPED_STEP`.""" + if steps is None: + steps = [step for step in range(N_TRAIN + N_PREDICTION) if step != SKIPPED_STEP] + lines = [";".join(("time_stamp", "asset_id", "id", "train_test", "status_type_id", *FEATURE_COLUMNS))] + for row, step in enumerate(steps): + cells = [ + "" if (row, feature) == EMPTY_CELL else str(value(step, feature)) for feature in range(len(FEATURE_COLUMNS)) + ] + phase = "train" if step < N_TRAIN else "prediction" + status = "4" if step == ANOMALY_END_STEP else "0" + lines.append(";".join((timestamp(step), str(asset), str(row), phase, status, *cells))) + path.write_text("\r\n".join(lines) + "\r\n", encoding="utf-8") + + +def write_wind_farm(root: Path, name: str = "A") -> Path: + """Write a wind farm with one anomaly event and one normal event. + + Returns: + The wind farm directory. + """ + directory = root / f"Wind Farm {name}" + (directory / "datasets").mkdir(parents=True) + (directory / "feature_description.csv").write_text(FEATURE_DESCRIPTION, encoding="utf-8") + (directory / "event_info.csv").write_text( + "asset;event_id;event_label;event_start;event_start_id;event_end;event_end_id;event_description\r\n" + f"21;{ANOMALY_EVENT};anomaly;{timestamp(ANOMALY_START_STEP)};0;{timestamp(ANOMALY_END_STEP)};0;Hydraulic group\r\n" + f"0;{NORMAL_EVENT};normal;{timestamp(N_TRAIN)};0;{timestamp(N_TRAIN + 9)};0;\r\n", + encoding="utf-8", + ) + write_event_file(directory / "datasets" / f"{ANOMALY_EVENT}.csv", asset=21) + write_event_file(directory / "datasets" / f"{NORMAL_EVENT}.csv", asset=0) + return directory diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/test_connector.py new file mode 100644 index 000000000..b5ed5b28e --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/test_connector.py @@ -0,0 +1,100 @@ +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing import assert_datasets_equal +from timenet.types import ClassificationTask, TemporalLocalizationTask, TimeInterval +from timenet.types.text import read_value +from timenet_connectors.datasets.fraunhofer_iee.care_to_compare.connector import ( + EVENT_LABEL, + FAULT_COMPONENT, + CareToCompareConnector, + CareWindFarm, +) +from timenet_connectors.datasets.fraunhofer_iee.care_to_compare.tests import fixtures + + +def _convert(root): + fixtures.write_wind_farm(root) + return CareToCompareConnector().convert(CareWindFarm.discover(root)) + + +def _records(dataset): + # The detection task of each event, in event order, names its Record. + return [task.inputs[0] for task in dataset.tasks_of(ClassificationTask) if task.label_set == EVENT_LABEL] + + +def _annotation(owner, key): + (annotation,) = [annotation for annotation in owner.annotations if annotation.key == key] + return annotation + + +def test_the_wind_farm_directories_are_found_under_the_extraction_root(tmp_path): + fixtures.write_wind_farm(tmp_path / "CARE_To_Compare", "B") + fixtures.write_wind_farm(tmp_path / "CARE_To_Compare", "A") + farms = CareWindFarm.discover(tmp_path) + assert [farm.name for farm in farms] == ["A", "B"] + with pytest.raises(TimeNetConnectorError, match="no 'Wind Farm"): + CareWindFarm.discover(tmp_path / "CARE_To_Compare" / "Wind Farm A" / "datasets") + + +def test_each_event_becomes_a_record_with_one_signal_per_column(tmp_path): + records = _records(_convert(tmp_path)) + assert len(records) == 2 + record = records[0] + signals = {signal.name: signal for signal in record.signals} + assert sorted(signals) == sorted([*fixtures.FEATURE_COLUMNS, "status_type_id"]) + n_steps = fixtures.N_TRAIN + fixtures.N_PREDICTION + wind_speed = signals["wind_speed_3_avg"].to_arrow() + assert len(wind_speed) == n_steps + # The skipped step and the empty cell are nulls; the other values keep the release's digits. + assert not wind_speed.is_valid()[fixtures.SKIPPED_STEP].as_py() + assert not wind_speed.is_valid()[fixtures.EMPTY_CELL[0]].as_py() + assert wind_speed[0].as_py() == pytest.approx(fixtures.value(0, 1), abs=1e-5) + # The counter without a statistic suffix reads as its sensor's one statistic. + assert [read_value(a.value) for a in signals["sensor_44"].annotations if a.key == "statistic"] == ["average"] + status = signals["status_type_id"].to_arrow() + assert status[fixtures.ANOMALY_END_STEP].as_py() == 4 + assert not status.is_valid()[fixtures.SKIPPED_STEP].as_py() + + +def test_the_event_table_becomes_annotations(tmp_path): + record = _records(_convert(tmp_path))[0] + assert read_value(_annotation(record, "wind_farm").value) == "A" + assert read_value(_annotation(record, "event_id").value) == fixtures.ANOMALY_EVENT + assert read_value(_annotation(record, "asset_id").value) == 21 + assert read_value(_annotation(record, "event_label").value) == "anomaly" + assert read_value(_annotation(record, "fault_component").value) == "hydraulic" + period_s = 600 + assert _annotation(record, "event_window").span == TimeInterval.seconds( + fixtures.ANOMALY_START_STEP * period_s, (fixtures.ANOMALY_END_STEP + 1) * period_s + ) + + +def test_each_event_gets_its_tasks(tmp_path): + dataset = _convert(tmp_path) + classifications = dataset.tasks_of(ClassificationTask) + localizations = dataset.tasks_of(TemporalLocalizationTask) + # Detection for both events, and a component label for the anomaly. + assert [read_value(task.targets[0]) for task in classifications] == ["anomaly", "hydraulic", "normal"] + assert [task.label_set for task in classifications] == [EVENT_LABEL, FAULT_COMPONENT, EVENT_LABEL] + (localization,) = localizations + assert [annotation.key for annotation in localization.target_annotations] == ["event_window"] + + +def test_an_event_file_without_an_event_info_row_raises(tmp_path): + farm = fixtures.write_wind_farm(tmp_path) + fixtures.write_event_file(farm / "datasets" / "99.csv", asset=3) + with pytest.raises(TimeNetConnectorError, match=r"event files \[99\] do not match"): + CareToCompareConnector().convert(CareWindFarm.discover(tmp_path)) + + +def test_the_dataset_round_trips_through_storage(tmp_path): + source = tmp_path / "source" + source.mkdir() + dataset = _convert(source) + version_dir = dataset.write(path=tmp_path / "registry") + with TimeFReader(open_local_version(version_dir)) as reader: + reader.verify() + assert_datasets_equal(dataset, reader.read()) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/test_source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/test_source.py new file mode 100644 index 000000000..36942f72f --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/fraunhofer_iee/care_to_compare/tests/test_source.py @@ -0,0 +1,110 @@ +import numpy as np +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet_connectors.datasets.fraunhofer_iee.care_to_compare.source import ( + PERIOD_US, + STATUS_MISSING, + parse_timestamp_us, + read_event_info, + read_event_table, + read_sensor_descriptions, + split_feature_column, +) +from timenet_connectors.datasets.fraunhofer_iee.care_to_compare.tests import fixtures + + +def test_the_sensor_descriptions_keep_the_release_units(tmp_path): + sensors = read_sensor_descriptions(fixtures.write_wind_farm(tmp_path) / "feature_description.csv") + assert list(sensors) == ["sensor_0", "wind_speed_3", "sensor_5", "sensor_31", "sensor_44"] + assert sensors["sensor_0"].unit_expression == "degree_Celsius" + assert sensors["wind_speed_3"].statistics == ("maximum", "minimum", "average", "std_dev") + assert sensors["sensor_5"].is_angle + assert sensors["sensor_5"].unit_expression == "degree" + # Reactive power has no safe pint form, so the connector keeps the source string. + assert sensors["sensor_31"].unit == "kVAr" + assert sensors["sensor_31"].unit_expression is None + + +def test_a_column_splits_into_its_sensor_and_statistic(): + assert split_feature_column("wind_speed_3_avg") == ("wind_speed_3", "average") + assert split_feature_column("sensor_52_std") == ("sensor_52", "std_dev") + with pytest.raises(TimeNetConnectorError, match="does not end in"): + split_feature_column("sensor_52") + + +def test_the_event_info_reads_both_asset_column_names(tmp_path): + events = read_event_info(fixtures.write_wind_farm(tmp_path) / "event_info.csv") + anomaly = events[fixtures.ANOMALY_EVENT] + assert anomaly.asset_id == 21 + assert anomaly.label == "anomaly" + assert anomaly.description == "Hydraulic group" + assert anomaly.fault_component("A") == "hydraulic" + assert anomaly.fault_component("C") is None + normal = events[fixtures.NORMAL_EVENT] + assert normal.description is None + assert normal.fault_component("A") is None + + renamed = tmp_path / "renamed.csv" + renamed.write_text( + "asset_id;event_id;event_label;event_start;event_start_id;event_end;event_end_id;event_description\r\n" + '53;35;anomaly;2023-09-24 00:00:00;1;2023-09-30 09:00:00;2;"Several stops; failure ""Level 1"""\r\n', + encoding="utf-8", + ) + (event,) = read_event_info(renamed).values() + assert event.asset_id == 53 + # A quoted description keeps its semicolon and its doubled quotes. + assert event.description == 'Several stops; failure "Level 1"' + + +def test_the_event_table_lies_on_the_grid_with_nulls_for_gaps(tmp_path): + farm = fixtures.write_wind_farm(tmp_path) + table = read_event_table(farm / "datasets" / f"{fixtures.ANOMALY_EVENT}.csv", block_rows=4) + n_steps = fixtures.N_TRAIN + fixtures.N_PREDICTION + assert table.asset_id == 21 + assert table.feature_columns == fixtures.FEATURE_COLUMNS + assert table.values.shape == (len(fixtures.FEATURE_COLUMNS), n_steps) + assert table.values.dtype == np.float32 + assert table.start_us == parse_timestamp_us(fixtures.timestamp(0)) + # The skipped step is a null in every signal and in the status. + assert np.isnan(table.values[:, fixtures.SKIPPED_STEP]).all() + assert table.status[fixtures.SKIPPED_STEP] == STATUS_MISSING + # The empty cell is a null in its own signal only. Row 3 is grid step 3, before the gap. + row, feature = fixtures.EMPTY_CELL + assert np.isnan(table.values[feature, row]) + assert table.values[feature + 1, row] == np.float32(fixtures.value(row, feature + 1)) + # Every other cell keeps its value, and rows after the gap land on their own step. + assert table.values[2, 15] == np.float32(fixtures.value(15, 2)) + assert table.status[fixtures.ANOMALY_END_STEP] == 4 + assert table.train_steps == (0, fixtures.N_TRAIN - 1) + assert table.prediction_steps == (fixtures.N_TRAIN, n_steps - 1) + assert table.step_of(table.start_us + 5 * PERIOD_US) == 5 + with pytest.raises(TimeNetConnectorError, match="off the grid"): + table.step_of(table.start_us + PERIOD_US // 2) + + +@pytest.mark.parametrize( + ("steps", "message"), + [ + ([0, 1, 1, 2, 25], "not rising"), + ([0, 25, 3], "not rising"), + ], + ids=["a repeated timestamp", "time goes back"], +) +def test_an_event_table_off_the_grid_raises(tmp_path, steps, message): + path = tmp_path / "1.csv" + fixtures.write_event_file(path, asset=1, steps=steps) + with pytest.raises(TimeNetConnectorError, match=message): + read_event_table(path) + + +def test_a_wrong_row_id_raises_with_its_row(tmp_path): + path = tmp_path / "1.csv" + fixtures.write_event_file(path, asset=1) + lines = path.read_text(encoding="utf-8").splitlines() + cells = lines[3].split(";") + cells[2] = "9" + lines[3] = ";".join(cells) + path.write_text("\n".join(lines), encoding="utf-8") + with pytest.raises(TimeNetConnectorError, match=r"1\.csv row 2: id '9'"): + read_event_table(path)