From 0fdfc7c00c0ca86bd60ab141a58f39deeb030d65 Mon Sep 17 00:00:00 2001 From: Glody Figueiredo Date: Sat, 21 Nov 2026 08:02:52 +0100 Subject: [PATCH] feat(connectors): add Kelmarsh, Penmanshiel and Hill of Towie wind farm connectors Add three open wind-farm SCADA datasets (CC-BY-4.0), the first wind data in the catalog. One Record is one turbine for one UTC day: 144 ten-minute steps of active power, wind speed, blade pitch angle, rotor speed, nacelle direction and ambient temperature. - cubico/kelmarsh (6 Senvion MM92) and cubico/penmanshiel (14 Senvion MM82), 2019-2022: labels from the operator's status log; each stop is annotated with its IEC category and message, curtailments too. Tasks: stopped on the day, stopped minutes, stop intervals, main stop reason. - res/hill_of_towie (21 Siemens SWT-2.3-VS-82), January 2016 to August 2024: labels from the shutdown durations RES published. Tasks: shut down on the day, minutes, ten-minute periods, and at least 60 minutes of downtime at wind of 4 m/s or more. Each connector has fixture-based tests that call convert() without network access. Checked with make check, make check-ci, make test, make test-connectors and make license-check. Co-Authored-By: Claude Opus 5.5 --- .../datasets/cubico/__init__.py | 1 + .../datasets/cubico/kelmarsh/__init__.py | 6 + .../datasets/cubico/kelmarsh/connector.py | 24 ++ .../datasets/cubico/kelmarsh/dataset.yaml | 44 ++ .../datasets/cubico/kelmarsh/farm.py | 226 ++++++++++ .../datasets/cubico/kelmarsh/greenbyte.py | 181 ++++++++ .../datasets/cubico/kelmarsh/tasks.py | 316 ++++++++++++++ .../cubico/kelmarsh/tests/__init__.py | 0 .../cubico/kelmarsh/tests/test_connector.py | 300 +++++++++++++ .../datasets/cubico/penmanshiel/__init__.py | 6 + .../datasets/cubico/penmanshiel/connector.py | 61 +++ .../datasets/cubico/penmanshiel/dataset.yaml | 45 ++ .../cubico/penmanshiel/tests/__init__.py | 0 .../penmanshiel/tests/test_connector.py | 70 ++++ .../datasets/res/__init__.py | 1 + .../datasets/res/hill_of_towie/__init__.py | 6 + .../datasets/res/hill_of_towie/connector.py | 394 ++++++++++++++++++ .../datasets/res/hill_of_towie/dataset.yaml | 42 ++ .../datasets/res/hill_of_towie/scada.py | 196 +++++++++ .../res/hill_of_towie/tests/__init__.py | 0 .../res/hill_of_towie/tests/test_connector.py | 290 +++++++++++++ 21 files changed, 2209 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/farm.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/greenbyte.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tasks.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/test_connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/test_connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/scada.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/__init__.py new file mode 100644 index 000000000..79e38e48d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/__init__.py @@ -0,0 +1 @@ +"""Connectors for the open wind farm datasets of Cubico Sustainable Investments.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/__init__.py new file mode 100644 index 000000000..a68c912e0 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/__init__.py @@ -0,0 +1,6 @@ +"""Kelmarsh wind farm: daily turbine SCADA records with operator stop and curtailment labels.""" + +from timenet_connectors.datasets.cubico.kelmarsh.connector import ( + CONNECTOR as CONNECTOR, + KelmarshConnector as KelmarshConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/connector.py new file mode 100644 index 000000000..b554b62e4 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/connector.py @@ -0,0 +1,24 @@ +"""The Kelmarsh connector: one Record for each turbine and UTC day, with labels from the status log. + +The release has one zip archive for each year, with the files of the six turbines. This connector +reads 2019 to 2022. The later years use another table layout. +""" + +from timenet_connectors.datasets.cubico.kelmarsh.farm import GreenbyteFarmConnector + + +class KelmarshConnector(GreenbyteFarmConnector): + """Connector for the Kelmarsh wind farm release on Zenodo.""" + + FILES_URL = "https://zenodo.org/records/16807551/files" + STATIC_FILE = ("Kelmarsh_WT_static.csv", "a249919471580244d41a547e407ef371d45f2d5347a59bc23cf3c980f607ee93") + # Zenodo states an MD5 hash for each file. These SHA-256 hashes are of the files with those MD5 hashes. + YEAR_ARCHIVES = { # ruff: ignore[mutable-class-default] + 2019: (("Kelmarsh_SCADA_2019_3085.zip", "c2f11578b9a1678be198fd7883a7dfe1b923c5b0565f806bdda5dd9d0097f4dc"),), + 2020: (("Kelmarsh_SCADA_2020_3086.zip", "1cf3ad08aaed0952055b3668e39f73bbea8e09654d28b104ad2c4041b256452a"),), + 2021: (("Kelmarsh_SCADA_2021_4456.zip", "9b9aed27db92ede492d83d59083c2d4463d74716746606adcb08b2b3bb393ea6"),), + 2022: (("Kelmarsh_SCADA_2022_4457.zip", "c897b1178f1a12e18b1b718c9eada52b58775a1345945265014817be3b2be79b"),), + } + + +CONNECTOR = KelmarshConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/dataset.yaml new file mode 100644 index 000000000..c237008e7 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/dataset.yaml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: cubico/kelmarsh +dataset_version: 1.1.0 +name: Kelmarsh wind farm +description: >- + Ten-minute SCADA data of the six Senvion MM92 turbines (2,050 kW each) of the Kelmarsh wind + farm in the United Kingdom, for 2019 to 2022. One Record is one turbine for one UTC day, with + 144 steps of six signals: active power, wind speed, blade pitch angle, rotor speed, nacelle + direction, and ambient temperature. Each value is the mean of a ten-minute period, at the + start time of the period. A step that the source leaves empty stays missing. The labels come + from the status log of the operator, not from the signals. Each stop event of the log is an + annotation with the IEC category that the operator gave it, as one of six reasons, and a + second annotation with the message of the event. Each curtailment event is an annotation + too. The tasks ask whether the turbine stopped on the day, for how many minutes, in which + intervals, and for which main reason. A day gets the reason task only if every stop of the + day has a category. A stop entry without an end time would be a point annotation, and the + days from that stop to the next System OK entry of the log would get no task. These years + have no such entry. The log records curtailment only in 2022, so only the records of that + year have the task that asks whether the operator limited the output. A further task asks whether the turbine had at least 60 minutes of downtime on the day, where + downtime is stop time in the log. A day with no stop and less than 60 minutes of curtailment + is no downtime. A day with 1 to 59 minutes of stop, or with no stop and 60 minutes or more of + curtailment, gets no such task: a short stop is another question, and a power limit is not a + stop. The metadata of each Record states how many of the 144 steps have no power value, so + that a user can keep the days with full coverage. + The split follows + dates, the same for all turbines: days before July 2021 are train, the rest of 2021 is + validation, and 2022 is test. If a stop or curtailment runs across one of the two dates, the + days of that turbine that it touches get no task. The power setpoint and the alarm codes of the source are not + signals, because they state the answer. +license: CC-BY-4.0 +source_url: https://doi.org/10.5281/zenodo.16807551 +citation: >- + Plumley, C., & Takeuchi, R. (2025). Kelmarsh wind farm data. Zenodo. + https://doi.org/10.5281/zenodo.16807551 +domains: + - energy + - industrial +tags: + - wind-turbine + - scada + - classification + - temporal-localization + - scalar-prediction diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/farm.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/farm.py new file mode 100644 index 000000000..55422a6e0 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/farm.py @@ -0,0 +1,226 @@ +"""The connector base for a wind farm that Cubico released as Greenbyte exports. + +A release has one or more zip archives for each year. An archive holds two files for each +turbine: a table of ten-minute SCADA statistics and a status log. A subclass states the files of +its release. One Record is one turbine for one UTC day. +""" + +from collections import defaultdict +from collections.abc import Mapping +import csv +from dataclasses import dataclass +from datetime import UTC, datetime +from fractions import Fraction +from pathlib import Path +from typing import ClassVar + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.json import JsonMapping +from timenet.types import MissingReason, MissingValues, TimeInterval, TimeOrigin, TimeSeriesSpec, ureg +from timenet_connectors.datasets.cubico.kelmarsh import tasks +from timenet_connectors.datasets.cubico.kelmarsh.greenbyte import ( + DAY_US, + STEP_US, + EventKind, + OpenStop, + StatusEvent, + StatusLog, + TurbineDay, + read_status_log, + read_turbine_days, + read_turbine_name, +) +from timenet_connectors.download import Artifact, download_files, ensure_archive + + +DATA_PREFIX = "Turbine_Data_" +STATUS_PREFIX = "Status_" + +_GAPS = MissingValues( + reason=MissingReason.NOT_RECORDED, + description="The source has no value for the ten-minute period, for example after a communication loss.", +) + + +def _spec(spec_type: str, name: str, unit: object) -> TimeSeriesSpec: + return TimeSeriesSpec(spec_type=spec_type, name=name, unit_value=unit, dtype="float64", missing=_GAPS) + + +# The signal name, the source column, and the spec. Each value is the mean of a ten-minute period. +SIGNALS: tuple[tuple[str, str, TimeSeriesSpec], ...] = ( + ("power", "Power (kW)", _spec("active_power", "Active power", ureg.kilowatt)), + ("wind_speed", "Wind speed (m/s)", _spec("wind_speed", "Wind speed", ureg.meter / ureg.second)), + ( + "pitch_angle", + "Blade angle (pitch position) A (°)", + _spec("blade_pitch_angle", "Blade pitch angle", ureg.degree), + ), + ("rotor_speed", "Rotor speed (RPM)", _spec("rotor_speed", "Rotor speed", ureg.rpm)), + ("nacelle_direction", "Nacelle position (°)", _spec("nacelle_direction", "Nacelle direction", ureg.degree)), + ( + "ambient_temperature", + "Nacelle ambient temperature (°C)", + _spec("ambient_temperature", "Ambient temperature", ureg.degC), + ), +) +SOURCE_COLUMNS = tuple(column for _, column, _ in SIGNALS) +# A value is at the start of its period, as in the source. +EVERY_TEN_MINUTES = RegularAxis(period_us=Fraction(STEP_US)) + + +@dataclass(frozen=True) +class TurbineYearFiles: + """The files of one turbine for one year, and the table that describes all turbines.""" + + data_path: Path + status_path: Path + static_path: Path + + +@dataclass +class _TurbineLog: + """The status entries of one turbine, from the logs of every year.""" + + events: set[StatusEvent] + open_stops: set[OpenStop] + ok_times_us: set[int] + + def add(self, log: StatusLog) -> None: + self.events.update(log.events) + self.open_stops.update(log.open_stops) + self.ok_times_us.update(log.ok_times_us) + + +class GreenbyteFarmConnector(BaseConnector[list[TurbineYearFiles]]): + """Connector for one wind farm release of Greenbyte exports on Zenodo.""" + + FILES_URL: ClassVar[str] + """The URL of the folder that holds the files of the Zenodo record.""" + STATIC_FILE: ClassVar[tuple[str, str]] + """The name and the SHA-256 hash of the table that describes the turbines.""" + YEAR_ARCHIVES: ClassVar[Mapping[int, tuple[tuple[str, str], ...]]] + """The name and the SHA-256 hash of each archive of a year.""" + SPLIT_CUTOFFS: ClassVar[tasks.SplitCutoffs] = tasks.SplitCutoffs( + validation=datetime(2021, 7, 1, tzinfo=UTC), test=datetime(2022, 1, 1, tzinfo=UTC) + ) + """Of the four years, the first two and a half are train, the next half is validation, and the last is test.""" + + async def download_async(self, cache_dir: Path) -> list[TurbineYearFiles]: + """Download the turbine table and the archives of each year, and extract each archive once. + + Args: + cache_dir: The directory that holds the downloaded and extracted files. + + Returns: + The files of each turbine and year, in year and turbine order. + + Raises: + TimeNetConnectorError: If a turbine table of an archive has no status log. + """ + static_name, static_sha256 = self.STATIC_FILE + static_path = cache_dir / static_name + await download_files( + [Artifact(f"{self.FILES_URL}/{static_name}?download=1", static_path, sha256=static_sha256)] + ) + refs: list[TurbineYearFiles] = [] + for year, archives in self.YEAR_ARCHIVES.items(): + for name, sha256 in archives: + root = await ensure_archive( + f"{self.FILES_URL}/{name}?download=1", cache_dir / str(year), filename=name, sha256=sha256 + ) + for data_path in sorted(root.glob(f"{DATA_PREFIX}*.csv")): + status_path = root / f"{STATUS_PREFIX}{data_path.name.removeprefix(DATA_PREFIX)}" + if not status_path.is_file(): + raise TimeNetConnectorError(f"{year}: {data_path.name} has no status log") + refs.append(TurbineYearFiles(data_path=data_path, status_path=status_path, static_path=static_path)) + return refs + + def convert(self, raw_refs: list[TurbineYearFiles], /) -> TimeFDataset: + """Create one Record for each turbine and day, with its annotations and tasks. + + A day gets no task if a stop without an end time leaves the state of the turbine unknown, + or if a stop or curtailment of the day runs across a split cutoff. + + Args: + raw_refs: The files from :meth:`download_async`. + + Returns: + The populated dataset. + """ + dataset = TimeFDataset(metadata=self.metadata()) + # A stop can run past the end of a year, so each day sees the entries of every year. + names = [read_turbine_name(ref.data_path) for ref in raw_refs] + logs: dict[str, _TurbineLog] = defaultdict(lambda: _TurbineLog(set(), set(), set())) + logs_curtailment: list[bool] = [] + for ref, name in zip(raw_refs, names, strict=True): + log = read_status_log(ref.status_path) + logs[name].add(log) + logs_curtailment.append(any(event.kind is EventKind.CURTAILMENT for event in log.events)) + for ref, name, curtailment in zip(raw_refs, names, logs_curtailment, strict=True): + turbine = _read_turbine_metadata(ref.static_path, name) + events = sorted(logs[name].events, key=lambda event: (event.start_us, event.end_us)) + open_stops = sorted(logs[name].open_stops, key=lambda stop: (stop.start_us, stop.message)) + unknown = tasks.unknown_windows(open_stops, sorted(logs[name].ok_times_us)) + crossing = self.SPLIT_CUTOFFS.crossing_days([(event.start_us, event.end_us) for event in events]) + for day in read_turbine_days(ref.data_path, SOURCE_COLUMNS): + record = dataset.add_record(record=_create_record(day, turbine)) + day_events = tasks.events_of_day(events, day.start_us) + tasks.annotate_day(record, day_events) + tasks.annotate_open_stops(record, open_stops, day.start_us) + if any(start < day.start_us + DAY_US and (end is None or end > day.start_us) for start, end in unknown): + continue + if day.start_us in crossing: + continue + dataset.add_tasks( + tasks=list( + tasks.iter_day_tasks( + record, + day_events, + split=self.SPLIT_CUTOFFS.split_of_day(day.start_us), + curtailment=curtailment, + ) + ) + ) + return dataset + + +def _create_record(day: TurbineDay, turbine: JsonMapping) -> Record: + signals = tuple( + Signal.from_values(day.values[column].to_pylist(), spec=spec, name=name, time_axis=EVERY_TEN_MINUTES) + for name, column, spec in SIGNALS + ) + return Record( + sources=(Source(name="SCADA", signals=signals),), + start_time=TimeOrigin(day.start_us), + # The number of steps without a power value, so that a user can keep days with full coverage. + metadata={**turbine, "power_missing_steps": int(day.values[SIGNALS[0][1]].null_count)}, + time_span=TimeInterval.micros(0, DAY_US), + ) + + +def _read_turbine_metadata(static_path: Path, name: str) -> JsonMapping: + """Read the row of one turbine from the table that describes all turbines. + + Returns: + The facts of the turbine, as the metadata of each of its Records. + + Raises: + TimeNetConnectorError: If the table has no row for the turbine. + """ + with static_path.open(encoding="utf-8-sig", newline="") as stream: + for row in csv.DictReader(stream): + if row["Title"] == name: + return { + "wind_farm": row["Wind Farm"], + "turbine": name, + "manufacturer": row["Manufacturer"], + "model": row["Model"], + "rated_power_kw": float(row["Rated power (kW)"]), + "hub_height_m": float(row["Hub Height (m)"]), + "rotor_diameter_m": float(row["Rotor Diameter (m)"]), + "latitude": float(row["Latitude"]), + "longitude": float(row["Longitude"]), + } + raise TimeNetConnectorError(f"{static_path.name}: no row for turbine {name!r}") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/greenbyte.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/greenbyte.py new file mode 100644 index 000000000..97f554abb --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/greenbyte.py @@ -0,0 +1,181 @@ +"""Readers for the Greenbyte exports of the release: the ten-minute turbine table and the status log. + +Both files start with nine comment lines. The third one names the turbine. All times are UTC. +""" + +from collections.abc import Iterator, Mapping +import csv +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import StrEnum +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.csv as pv + +from timenet.errors import TimeNetConnectorError +from timenet.types import US_PER_S, unix_us + + +COMMENT_LINES = 9 +TURBINE_PREFIX = "# Turbine: " +TIMESTAMP_COLUMN = "# Date and time" +STEP_US = 600 * US_PER_S +STEPS_PER_DAY = 144 +DAY_US = STEPS_PER_DAY * STEP_US +MISSING_END = "-" +SYSTEM_OK = "System OK" + + +class EventKind(StrEnum): + """The two values of the ``Status`` column that this connector keeps.""" + + STOP = "Stop" + CURTAILMENT = "Curtailment" + + +@dataclass(frozen=True) +class StatusEvent: + """One stop or curtailment entry of the status log, with Unix times in microseconds.""" + + kind: EventKind + start_us: int + end_us: int + message: str + iec_category: str + """The IEC category that the operator gave the entry, or an empty string.""" + + +@dataclass(frozen=True) +class OpenStop: + """A stop entry without an end time. The log does not state when the turbine ran again.""" + + start_us: int + message: str + + +@dataclass(frozen=True) +class StatusLog: + """What this connector keeps of one status log, with Unix times in microseconds.""" + + events: tuple[StatusEvent, ...] + open_stops: tuple[OpenStop, ...] + ok_times_us: tuple[int, ...] + """The start of each ``System OK`` entry.""" + + +@dataclass(frozen=True) +class TurbineDay: + """The values of one turbine for one UTC day: 144 steps for each column.""" + + start_us: int + values: Mapping[str, pa.Array] + + +def read_turbine_name(path: Path) -> str: + """Read the turbine name from the comment lines of an export. + + Args: + path: A turbine table or a status log. + + Returns: + The name, such as ``Kelmarsh 1``. + + Raises: + TimeNetConnectorError: If no comment line names the turbine. + """ + with path.open(encoding="utf-8") as stream: + for _, line in zip(range(COMMENT_LINES), stream, strict=False): + if line.startswith(TURBINE_PREFIX): + return line.removeprefix(TURBINE_PREFIX).strip() + raise TimeNetConnectorError(f"{path.name}: no comment line starts with {TURBINE_PREFIX!r}") + + +def read_turbine_days(path: Path, columns: tuple[str, ...]) -> Iterator[TurbineDay]: + """Read the columns of a turbine table and cut them into UTC days. + + Args: + path: The turbine table. + columns: The source names of the value columns to read. + + Yields: + Each day of the table, in time order. An empty cell of the source is a null. + + Raises: + TimeNetConnectorError: If a column is absent, the table does not start at midnight, the + timestamps are not ten minutes apart, or the last day is not complete. + """ + try: + table = pv.read_csv( + path, + read_options=pv.ReadOptions(skip_rows=COMMENT_LINES), + convert_options=pv.ConvertOptions( + include_columns=[TIMESTAMP_COLUMN, *columns], + column_types={TIMESTAMP_COLUMN: pa.timestamp("us"), **dict.fromkeys(columns, pa.float64())}, + ), + ).combine_chunks() + except (pa.ArrowInvalid, KeyError) as exc: + raise TimeNetConnectorError(f"{path.name}: {exc}") from exc + times = table[TIMESTAMP_COLUMN].to_numpy().astype("datetime64[us]").astype(np.int64) + if times.size == 0 or times.size % STEPS_PER_DAY != 0 or times[0] % DAY_US != 0: + raise TimeNetConnectorError(f"{path.name}: the table must hold whole UTC days, got {times.size} rows") + if np.any(np.diff(times) != STEP_US): + raise TimeNetConnectorError(f"{path.name}: the timestamps are not ten minutes apart") + for first in range(0, times.size, STEPS_PER_DAY): + yield TurbineDay( + start_us=int(times[first]), + values={column: table[column].chunk(0).slice(first, STEPS_PER_DAY) for column in columns}, + ) + + +def read_status_log(path: Path) -> StatusLog: + """Read the stop and curtailment entries of a status log. + + The log also holds informational entries, warnings, and communication losses. This function + skips them, except that it keeps the time of each ``System OK`` entry. + + Args: + path: The status log. + + Returns: + The entries with an end time, the stops without one, and the ``System OK`` times. + + Raises: + TimeNetConnectorError: If an entry ends before it starts, or a curtailment has no end time. + """ + kinds = {kind.value for kind in EventKind} + events: list[StatusEvent] = [] + open_stops: list[OpenStop] = [] + ok_times_us: list[int] = [] + with path.open(encoding="utf-8", newline="") as stream: + for _ in range(COMMENT_LINES): + next(stream, None) + for number, row in enumerate(csv.DictReader(stream), start=1): + if row["Message"] == SYSTEM_OK: + ok_times_us.append(_parse_utc(row["Timestamp start"])) + if row["Status"] not in kinds: + continue + kind, start_us = EventKind(row["Status"]), _parse_utc(row["Timestamp start"]) + if row["Timestamp end"] == MISSING_END: + if kind is not EventKind.STOP: + raise TimeNetConnectorError(f"{path.name} data row {number}: a {kind} entry has no end time") + open_stops.append(OpenStop(start_us=start_us, message=row["Message"])) + continue + end_us = _parse_utc(row["Timestamp end"]) + if end_us < start_us: + raise TimeNetConnectorError(f"{path.name} data row {number}: the entry ends before it starts") + events.append( + StatusEvent( + kind=kind, + start_us=start_us, + end_us=end_us, + message=row["Message"], + iec_category=row["IEC category"], + ) + ) + return StatusLog(events=tuple(events), open_stops=tuple(open_stops), ok_times_us=tuple(ok_times_us)) + + +def _parse_utc(text: str) -> int: + return unix_us(datetime.fromisoformat(text).replace(tzinfo=UTC)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tasks.py new file mode 100644 index 000000000..74c36039d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tasks.py @@ -0,0 +1,316 @@ +"""The annotations and tasks of one turbine day, built from the status log of the operator.""" + +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from datetime import datetime +from itertools import groupby + +from timenet.dataset import Record +from timenet.types import ( + US_PER_S, + Annotation, + ClassificationTask, + Label, + LabelSet, + ScalarPredictionTask, + Split, + Task, + TemporalLocalizationTask, + TimeInterval, + TimePoint, + unix_us, +) +from timenet_connectors.datasets.cubico.kelmarsh.greenbyte import DAY_US, EventKind, OpenStop, StatusEvent + + +STOP_KEY = "stop" +STOP_MESSAGE_KEY = "stop_message" +CURTAILMENT_KEY = "curtailment" +OPEN_STOP_KEY = "stop_without_end" +UNKNOWN_REASON = "unknown" + +# The reasons are the IEC categories that the operator gives a stop. The category is the only +# reason field that means the same for every message of the log. +STOP_REASONS = { + "Forced outage": "fault", + "Scheduled Maintenance": "maintenance", + "Out of Environmental Specification": "environment", + "Out of Electrical Specification": "grid", + "Requested Shutdown": "requested", + "Technical Standby": "standby", +} +STOPPED = LabelSet( + annotations=[Annotation("name", value="stopped")], + labels=( + Label(name="stopped", description="The status log has a stop on this day."), + Label(name="running", description="The status log has no stop on this day."), + ), +) +STOP_REASON = LabelSet( + annotations=[Annotation("name", value="stop_reason")], + labels=( + Label(name="fault", description="IEC category Forced outage: a fault stopped the turbine."), + Label(name="maintenance", description="IEC category Scheduled Maintenance."), + Label( + name="environment", + description="IEC category Out of Environmental Specification, such as icing or the wind limits.", + ), + Label(name="grid", description="IEC category Out of Electrical Specification, such as a grid loss."), + Label(name="requested", description="IEC category Requested Shutdown: a remote or park stop."), + Label(name="standby", description="IEC category Technical Standby, such as a battery test or a cable unwind."), + ), +) +CURTAILED = LabelSet( + annotations=[Annotation("name", value="curtailed")], + labels=( + Label(name="curtailed", description="The status log has a curtailment on this day."), + Label(name="not_curtailed", description="The status log has no curtailment on this day."), + ), +) +DOWNTIME_60 = LabelSet( + annotations=[Annotation("name", value="downtime_60")], + labels=( + Label(name="downtime", description="The status log has 60 minutes or more of stop time on this day."), + Label( + name="no_downtime", + description="The status log has no stop on this day and less than 60 minutes of curtailment.", + ), + ), +) +DOWNTIME_MINUTES = 60 +DOWNTIME_60_PROMPT = ( + "Did the turbine have at least 60 minutes of downtime on this day? " + "Calm-wind waiting and power limiting do not count." +) +STOPPED_PROMPT = "Did the turbine stop at any time on this day?" +STOPPED_MINUTES_PROMPT = "For how many minutes was the turbine stopped on this day?" +STOP_INTERVALS_PROMPT = "In which intervals was the turbine stopped on this day?" +STOP_REASON_PROMPT = "What is the main reason that the turbine was stopped on this day?" +CURTAILED_PROMPT = "Did the operator limit the power output of the turbine at any time on this day?" + +DayEvent = tuple[StatusEvent, int, int] +"""An event and the part of it that lies in one day, as microseconds from the start of the day.""" + + +@dataclass(frozen=True) +class SplitCutoffs: + """The dates that divide a dataset into train, validation and test, in that order in time. + + All turbines share the dates, because they share the weather. No train day comes after a + validation day, and no validation day comes after a test day. + """ + + validation: datetime + """The first UTC day of validation. Earlier days are train.""" + test: datetime + """The first UTC day of test.""" + + def split_of_day(self, day_start_us: int) -> Split: + """Give the split of a day. + + Args: + day_start_us: The start of the UTC day, in Unix microseconds. + + Returns: + The split that holds the day. + """ + if day_start_us < unix_us(self.validation): + return Split.TRAIN + return Split.VALIDATION if day_start_us < unix_us(self.test) else Split.TEST + + def crossing_days(self, intervals: Sequence[tuple[int, int]]) -> set[int]: + """Give the days of each incident that runs across a cutoff. + + An incident is a set of intervals that overlap or touch. Its days would be in two splits, + with one event in both, so they get no task. + + Args: + intervals: The start and the end of each event of one turbine, in Unix microseconds. + + Returns: + The start of each such day, in Unix microseconds. + """ + cuts = (unix_us(self.validation), unix_us(self.test)) + days: set[int] = set() + for start, end in _merge(intervals): + if any(start < cut < end for cut in cuts): + days.update(range(start // DAY_US * DAY_US, end, DAY_US)) + return days + + +def events_of_day(events: Sequence[StatusEvent], day_start_us: int) -> list[DayEvent]: + """Clip the events to one day. + + Args: + events: The events of the turbine. + day_start_us: The start of the UTC day, in Unix microseconds. + + Returns: + Each event that covers a part of the day, with that part, in start order. + """ + clipped = [ + (event, max(event.start_us - day_start_us, 0), min(event.end_us - day_start_us, DAY_US)) for event in events + ] + return sorted((item for item in clipped if item[1] < item[2]), key=lambda item: (item[1], item[2])) + + +def annotate_day(record: Record, day_events: Sequence[DayEvent]) -> None: + """Add one annotation for each curtailment of the day, and two for each stop. + + Args: + record: The Record of the day. + day_events: The clipped events of the day, from :func:`events_of_day`. + """ + for event, start, end in day_events: + span = TimeInterval.micros(start, end) + if event.kind is EventKind.CURTAILMENT: + record.annotate(Annotation(key=CURTAILMENT_KEY, value=event.message, span=span)) + continue + reason = STOP_REASONS.get(event.iec_category, UNKNOWN_REASON) + record.annotate(Annotation(key=STOP_KEY, value=reason, span=span)) + record.annotate(Annotation(key=STOP_MESSAGE_KEY, value=event.message, span=span)) + + +def unknown_windows(open_stops: Sequence[OpenStop], ok_times_us: Sequence[int]) -> list[tuple[int, int | None]]: + """Give the time in which the state of the turbine is not known, for each stop without an end. + + The window runs from the start of the stop to the next ``System OK`` entry of the log. That + entry is not taken as the end of the stop. It only shows that the labels can be trusted again. + + Args: + open_stops: The stops without an end time. + ok_times_us: The ``System OK`` times of the same turbine. + + Returns: + The start and the end of each window, in Unix microseconds. The end is ``None`` if no + ``System OK`` entry follows the stop. + """ + ok_times = sorted(ok_times_us) + return [(stop.start_us, next((time for time in ok_times if time > stop.start_us), None)) for stop in open_stops] + + +def annotate_open_stops(record: Record, open_stops: Sequence[OpenStop], day_start_us: int) -> None: + """Add one point annotation for each stop without an end that starts on the day. + + Args: + record: The Record of the day. + open_stops: The stops without an end time, of the turbine. + day_start_us: The start of the UTC day, in Unix microseconds. + """ + for stop in open_stops: + if 0 <= stop.start_us - day_start_us < DAY_US: + record.annotate( + Annotation(key=OPEN_STOP_KEY, value=stop.message, span=TimePoint.micros(stop.start_us - day_start_us)) + ) + + +def iter_day_tasks( + record: Record, day_events: Sequence[DayEvent], *, split: Split, curtailment: bool +) -> Iterator[Task]: + """Create the tasks of one turbine day. + + Args: + record: The Record of the day. + day_events: The clipped events of the day, from :func:`events_of_day`. + split: The split of the day. + curtailment: Whether the status log of this turbine and year records curtailment. Without + such a log, a day without an entry is not known to be free of curtailment. + + Yields: + The tasks, not yet added to a dataset. + """ + stops = [item for item in day_events if item[0].kind is EventKind.STOP] + intervals = _merge([(start, end) for _, start, end in stops]) + stopped_us = sum(end - start for start, end in intervals) + yield ClassificationTask( + inputs=(record,), + targets=("stopped" if stops else "running",), + label_set=STOPPED, + prompt=STOPPED_PROMPT, + split=split, + ) + yield ScalarPredictionTask( + inputs=(record,), + targets=(stopped_us / (60 * US_PER_S),), + unit="minute", + target_name="stopped_minutes", + prompt=STOPPED_MINUTES_PROMPT, + split=split, + ) + if stops: + yield TemporalLocalizationTask( + inputs=(record,), + targets=tuple(TimeInterval.micros(start, end) for start, end in intervals), + prompt=STOP_INTERVALS_PROMPT, + split=split, + ) + curtailed_us = sum( + end - start + for start, end in _merge( + [(start, end) for event, start, end in day_events if event.kind is EventKind.CURTAILMENT] + ) + ) + downtime = _downtime_60(stopped_us, curtailed_us) + reason = _main_reason(stops) + if reason is not None: + yield ClassificationTask( + inputs=(record,), targets=(reason,), label_set=STOP_REASON, prompt=STOP_REASON_PROMPT, split=split + ) + if downtime is not None: + yield ClassificationTask( + inputs=(record,), targets=(downtime,), label_set=DOWNTIME_60, prompt=DOWNTIME_60_PROMPT, split=split + ) + if curtailment: + curtailed = any(event.kind is EventKind.CURTAILMENT for event, _, _ in day_events) + yield ClassificationTask( + inputs=(record,), + targets=("curtailed" if curtailed else "not_curtailed",), + label_set=CURTAILED, + prompt=CURTAILED_PROMPT, + split=split, + ) + + +def _downtime_60(stopped_us: int, curtailed_us: int) -> str | None: + """Give the downtime label of a day. + + A day with 60 minutes or more of stop is ``downtime``. A day with no stop and less than 60 + minutes of curtailment is ``no_downtime``. A day with 1 to 59 minutes of stop, or with no stop + and a long curtailment, gets no label: a short stop is a different question, and a power limit + is not a stop. + + Args: + stopped_us: The merged stop time of the day, in microseconds. + curtailed_us: The merged curtailment time of the day, in microseconds. + + Returns: + ``downtime``, ``no_downtime``, or ``None`` if the day is in between. + """ + limit_us = DOWNTIME_MINUTES * 60 * US_PER_S + if stopped_us >= limit_us: + return "downtime" + return "no_downtime" if stopped_us == 0 and curtailed_us < limit_us else None + + +def _merge(intervals: Sequence[tuple[int, int]]) -> list[tuple[int, int]]: + """Join intervals that overlap or touch, so that no stopped time counts twice.""" + merged: list[tuple[int, int]] = [] + for start, end in sorted(intervals): + if merged and start <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + return merged + + +def _main_reason(stops: Sequence[DayEvent]) -> str | None: + """Give the reason with the most stopped time, or ``None`` if a stop of the day has no category.""" + if not stops or any(event.iec_category not in STOP_REASONS for event, _, _ in stops): + return None + by_reason = sorted(stops, key=lambda item: STOP_REASONS[item[0].iec_category]) + totals = { + reason: sum(end - start for _, start, end in group) + for reason, group in groupby(by_reason, key=lambda item: STOP_REASONS[item[0].iec_category]) + } + # A tie goes to the reason that comes first in the alphabet, so that a rebuild gives the same label. + return max(sorted(totals), key=lambda reason: totals[reason]) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/test_connector.py new file mode 100644 index 000000000..bcbd1498a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/kelmarsh/tests/test_connector.py @@ -0,0 +1,300 @@ +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pytest + +from timenet.dataset import TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ( + ClassificationTask, + ScalarPredictionTask, + Split, + Task, + TemporalLocalizationTask, + TimeInterval, + TimePoint, + read_value, + unix_us, +) +from timenet_connectors.datasets.cubico.kelmarsh import connector as kelmarsh_connector, tasks +from timenet_connectors.datasets.cubico.kelmarsh.connector import KelmarshConnector +from timenet_connectors.datasets.cubico.kelmarsh.farm import SOURCE_COLUMNS, TurbineYearFiles +from timenet_connectors.datasets.cubico.kelmarsh.greenbyte import STEPS_PER_DAY + + +# A synthetic export of one invented turbine for two days. It has the shape of the release and +# holds no value of it. +_TURBINE = "Testfarm 1" +_START = datetime(2022, 3, 1, tzinfo=UTC) +_DAYS = 2 +_COMMENTS = f"# Synthetic export.\n#\n# Turbine: {_TURBINE}\n# Time zone: UTC\n#\n#\n#\n#\n#\n" +_GAP_STEP = 5 +"""The step of the first day at which every signal is empty.""" +_STATIC = ( + "Wind Farm,Title,Alternative Title,Identity,Manufacturer,Model,Rated power (kW),Hub Height (m)," + "Rotor Diameter (m),Latitude,Longitude,Elevation (m),Country,Commercial Operations Date\n" + f"Testfarm,{_TURBINE},TF1,X 1,Maker,M1,2000,80.5,90,1.5,-2.5,100,UK,01/01/2016\n" +) +_STATUS_HEADER = ( + "Timestamp start,Timestamp end,Duration,Status,Code,Message,Comment,Service contract category,IEC category\n" +) +# Day 1 has two stops that overlap from 01:00 to 03:00, and a stop that runs into day 2 from 23:00 +# to 00:30. Day 2 also has a curtailment and a stop without a category. +_STATUS_ROWS = ( + "2022-03-01 00:10:00,-,-,Informational,0,System OK,,System OK (32),Full Performance\n" + "2022-03-01 01:00:00,2022-03-01 02:30:00,01:30:00,Stop,1,Manual stop,,Manual stop (service) (9)," + "Scheduled Maintenance\n" + "2022-03-01 02:00:00,2022-03-01 03:00:00,01:00:00,Stop,2,Grid loss,,External stop (grid) (4)," + "Out of Electrical Specification\n" + "2022-03-01 23:00:00,2022-03-02 00:30:00,01:30:00,Stop,3,Low hydraulic pressure,,Mechanical error (23)," + "Forced outage\n" + "2022-03-02 05:00:00,2022-03-02 05:10:00,00:10:00,Stop,4,Check vane!,,Sensor error (21),\n" + "2022-03-02 10:00:00,2022-03-02 12:00:00,02:00:00,Curtailment,108,Technical curtailment,,,Partial Performance\n" +) +_HOUR_US = 3_600_000_000 + + +def _value(column: int, step: int) -> float: + return column * 1000 + step / 3 + + +def _write_data(path: Path, *, skip_step: int | None = None) -> None: + lines = [_COMMENTS, ",".join(["# Date and time", "Unused (x)", *SOURCE_COLUMNS]) + "\n"] + for step in range(_DAYS * STEPS_PER_DAY): + if step == skip_step: + continue + cells = ["" if step == _GAP_STEP else repr(_value(column, step)) for column in range(len(SOURCE_COLUMNS))] + lines.append(",".join([f"{_START + timedelta(minutes=10 * step):%Y-%m-%d %H:%M:%S}", "7", *cells]) + "\n") + path.write_text("".join(lines), encoding="utf-8") + + +def _write_release( + directory: Path, *, status_rows: str = _STATUS_ROWS, skip_step: int | None = None +) -> TurbineYearFiles: + ref = TurbineYearFiles( + data_path=directory / "Turbine_Data_Testfarm_1.csv", + status_path=directory / "Status_Testfarm_1.csv", + static_path=directory / "static.csv", + ) + _write_data(ref.data_path, skip_step=skip_step) + ref.status_path.write_text(_COMMENTS + _STATUS_HEADER + status_rows, encoding="utf-8") + ref.static_path.write_text(_STATIC, encoding="utf-8") + return ref + + +@pytest.fixture +def dataset(tmp_path) -> TimeFDataset: + return KelmarshConnector().convert([_write_release(tmp_path)]) + + +def _targets(dataset: TimeFDataset, task_type: type[Task], prompt: str) -> list[object]: + found = [task for task in dataset.tasks_of(task_type) if read_value(task.prompt) == prompt] + return [tuple(read_value(target) for target in task.targets or ()) for task in found] + + +def test_each_day_becomes_a_record_with_the_source_values(dataset): + assert len(dataset.records) == _DAYS + for day, record in enumerate(dataset.records): + assert record.start_time.timestamp == unix_us(_START + timedelta(days=day)) + assert record.metadata["turbine"] == _TURBINE + assert record.metadata["hub_height_m"] == pytest.approx(80.5) + by_name = {signal.name: signal for signal in record.signals} + assert set(by_name) == { + "power", + "wind_speed", + "pitch_angle", + "rotor_speed", + "nacelle_direction", + "ambient_temperature", + } + steps = range(day * STEPS_PER_DAY, (day + 1) * STEPS_PER_DAY) + expected = np.array([_value(0, step) for step in steps]) + values = by_name["power"].to_arrow() + if day == 0: + # The empty cell stays missing. The connector does not fill it. + assert values.null_count == 1 + assert values[_GAP_STEP].as_py() is None + expected[_GAP_STEP] = np.nan + else: + assert values.null_count == 0 + np.testing.assert_array_equal(values.to_numpy(zero_copy_only=False), expected) + + +def test_stops_and_curtailments_become_annotations_clipped_to_the_day(dataset): + first, second = ([(a.key, read_value(a.value), a.span) for a in record.annotations] for record in dataset.records) + assert (tasks.STOP_KEY, "maintenance", TimeInterval.micros(_HOUR_US, _HOUR_US * 5 // 2)) in first + assert (tasks.STOP_MESSAGE_KEY, "Grid loss", TimeInterval.micros(2 * _HOUR_US, 3 * _HOUR_US)) in first + assert (tasks.STOP_KEY, "fault", TimeInterval.micros(23 * _HOUR_US, 24 * _HOUR_US)) in first + assert (tasks.STOP_KEY, "fault", TimeInterval.micros(0, _HOUR_US // 2)) in second + assert ( + tasks.STOP_KEY, + tasks.UNKNOWN_REASON, + TimeInterval.micros(5 * _HOUR_US, 5 * _HOUR_US + _HOUR_US // 6), + ) in second + assert (tasks.CURTAILMENT_KEY, "Technical curtailment", TimeInterval.micros(10 * _HOUR_US, 12 * _HOUR_US)) in second + assert len(first) == 6 + assert len(second) == 5 + + +def test_tasks_state_the_stops_of_each_day(dataset): + assert _targets(dataset, ClassificationTask, tasks.STOPPED_PROMPT) == [("stopped",), ("stopped",)] + # Day 1: 01:00 to 03:00 counts once, plus 23:00 to 24:00. Day 2: 00:00 to 00:30, plus 10 minutes. + assert _targets(dataset, ScalarPredictionTask, tasks.STOPPED_MINUTES_PROMPT) == [(180.0,), (40.0,)] + assert _targets(dataset, TemporalLocalizationTask, tasks.STOP_INTERVALS_PROMPT) == [ + (TimeInterval.micros(_HOUR_US, 3 * _HOUR_US), TimeInterval.micros(23 * _HOUR_US, 24 * _HOUR_US)), + (TimeInterval.micros(0, _HOUR_US // 2), TimeInterval.micros(5 * _HOUR_US, 5 * _HOUR_US + _HOUR_US // 6)), + ] + # Day 1: maintenance has 90 minutes, grid and fault have 60 each. Day 2 has a stop without a category. + assert _targets(dataset, ClassificationTask, tasks.STOP_REASON_PROMPT) == [("maintenance",)] + assert _targets(dataset, ClassificationTask, tasks.CURTAILED_PROMPT) == [("not_curtailed",), ("curtailed",)] + assert {task.split for task in dataset.tasks} == {Split.TEST} + + +def test_a_day_without_events_is_running_and_has_no_curtailment_task(tmp_path): + dataset = KelmarshConnector().convert([_write_release(tmp_path, status_rows="")]) + assert _targets(dataset, ClassificationTask, tasks.STOPPED_PROMPT) == [("running",), ("running",)] + assert _targets(dataset, ScalarPredictionTask, tasks.STOPPED_MINUTES_PROMPT) == [(0.0,), (0.0,)] + assert dataset.tasks_of(TemporalLocalizationTask) == () + # The log records no curtailment, so a day is not known to be free of it. + assert _targets(dataset, ClassificationTask, tasks.CURTAILED_PROMPT) == [] + assert all(record.annotations == () for record in dataset.records) + + +class _CutoffAtDayTwo(KelmarshConnector): + CARD = Path(kelmarsh_connector.__file__).with_name("dataset.yaml") + SPLIT_CUTOFFS = tasks.SplitCutoffs(validation=_START, test=_START + timedelta(days=1)) + + +def test_split_follows_the_dates_and_no_train_day_is_after_a_test_day(): + cutoffs = KelmarshConnector.SPLIT_CUTOFFS + days = [unix_us(datetime(2019, 1, 1, tzinfo=UTC) + timedelta(days=day)) for day in range(4 * 365 + 1)] + splits = [cutoffs.split_of_day(day) for day in days] + assert splits == sorted(splits, key=[Split.TRAIN, Split.VALIDATION, Split.TEST].index) + assert splits[0] == Split.TRAIN + assert cutoffs.split_of_day(unix_us(datetime(2021, 6, 30, tzinfo=UTC))) == Split.TRAIN + assert cutoffs.split_of_day(unix_us(datetime(2021, 7, 1, tzinfo=UTC))) == Split.VALIDATION + assert cutoffs.split_of_day(unix_us(datetime(2021, 12, 31, tzinfo=UTC))) == Split.VALIDATION + assert cutoffs.split_of_day(unix_us(datetime(2022, 1, 1, tzinfo=UTC))) == Split.TEST + + +def test_no_event_of_the_log_has_tasks_in_two_splits(tmp_path): + # The cutoff to test is at the start of day 2. The stop from 23:00 to 00:30 runs across it. + dataset = _CutoffAtDayTwo().convert([_write_release(tmp_path)]) + first, second = dataset.records + # The Records keep their annotations. Only the tasks are left out. + assert len(first.annotations) == 6 + assert len(second.annotations) == 5 + assert dataset.tasks == () + + +def test_an_event_that_ends_at_a_cutoff_keeps_its_tasks(tmp_path): + rows = ( + "2022-03-01 23:00:00,2022-03-02 00:00:00,01:00:00,Stop,3,Low hydraulic pressure,,Mechanical error (23)," + "Forced outage\n" + "2022-03-02 05:00:00,2022-03-02 05:10:00,00:10:00,Stop,4,Check vane!,,Sensor error (21),Forced outage\n" + ) + dataset = _CutoffAtDayTwo().convert([_write_release(tmp_path, status_rows=rows)]) + _assert_no_event_in_two_splits(dataset, rows) + assert [task.split for task in dataset.tasks_of(ScalarPredictionTask)] == [Split.VALIDATION, Split.TEST] + + +def test_stops_that_touch_across_a_cutoff_are_one_incident(tmp_path): + # Two entries of the log: the first ends at the cutoff, and the second starts there. + rows = ( + "2022-03-01 23:00:00,2022-03-02 00:00:00,01:00:00,Stop,3,Low hydraulic pressure,,Mechanical error (23)," + "Forced outage\n" + "2022-03-02 00:00:00,2022-03-02 00:30:00,00:30:00,Stop,3,Low hydraulic pressure,,Mechanical error (23)," + "Forced outage\n" + ) + dataset = _CutoffAtDayTwo().convert([_write_release(tmp_path, status_rows=rows)]) + assert dataset.tasks == () + + +def _assert_no_event_in_two_splits(dataset: TimeFDataset, status_rows: str) -> None: + """Check the intervals of the source log, not the clipped annotations, against the splits.""" + split_of_day = {task.inputs[0].start_time.timestamp: task.split for task in dataset.tasks} + day_us = 86_400_000_000 + for row in status_rows.splitlines(): + start, end = (unix_us(datetime.fromisoformat(cell).replace(tzinfo=UTC)) for cell in row.split(",")[:2]) + days = range(start // day_us * day_us, end, day_us) + assert len({split_of_day[day] for day in days if day in split_of_day}) <= 1 + + +def test_a_table_with_a_missing_row_is_rejected(tmp_path): + ref = _write_release(tmp_path, skip_step=STEPS_PER_DAY) + with pytest.raises(TimeNetConnectorError, match="whole UTC days"): + KelmarshConnector().convert([ref]) + + +def test_a_stop_without_an_end_time_leaves_its_days_without_tasks(tmp_path): + # The log does not state when the turbine ran again. The next System OK entry is on the same day. + rows = ( + "2022-03-01 01:00:00,-,-,Stop,1,Manual stop,,Manual stop (service) (9),Scheduled Maintenance\n" + "2022-03-01 02:00:00,-,-,Informational,0,System OK,,System OK (32),Full Performance\n" + ) + dataset = KelmarshConnector().convert([_write_release(tmp_path, status_rows=rows)]) + first, second = dataset.records + assert [(a.key, read_value(a.value), a.span) for a in first.annotations] == [ + (tasks.OPEN_STOP_KEY, "Manual stop", TimePoint.micros(_HOUR_US)) + ] + assert second.annotations == () + assert {task.inputs[0].start_time for task in dataset.tasks} == {second.start_time} + assert _targets(dataset, ClassificationTask, tasks.STOPPED_PROMPT) == [("running",)] + + +def test_a_stop_without_an_end_time_and_no_later_system_ok_leaves_every_later_day_without_tasks(tmp_path): + row = "2022-03-01 01:00:00,-,-,Stop,1,Manual stop,,Manual stop (service) (9),Scheduled Maintenance\n" + dataset = KelmarshConnector().convert([_write_release(tmp_path, status_rows=row)]) + assert dataset.tasks == () + + +def test_a_curtailment_without_an_end_time_is_rejected(tmp_path): + row = "2022-03-01 01:00:00,-,-,Curtailment,108,Technical curtailment,,,Partial Performance\n" + ref = _write_release(tmp_path, status_rows=row) + with pytest.raises(TimeNetConnectorError, match="no end time"): + KelmarshConnector().convert([ref]) + + +def _downtime_targets(tmp_path: Path, status_rows: str) -> list[object]: + dataset = KelmarshConnector().convert([_write_release(tmp_path, status_rows=status_rows)]) + return _targets(dataset, ClassificationTask, tasks.DOWNTIME_60_PROMPT) + + +def test_the_downtime_task_asks_for_an_hour_of_stop(dataset): + # Day 1 has 180 minutes of stop. Day 2 has 40 minutes of stop and a long curtailment: no task. + assert _targets(dataset, ClassificationTask, tasks.DOWNTIME_60_PROMPT) == [("downtime",)] + + +def test_downtime_starts_at_exactly_sixty_minutes_of_stop(tmp_path): + hour = "2022-03-01 01:00:00,2022-03-01 02:00:00,01:00:00,Stop,1,Manual stop,,Manual stop (service) (9),Scheduled Maintenance\n" + short = hour.replace("02:00:00,01:00:00", "01:59:00,00:59:00") + assert _downtime_targets(tmp_path, hour) == [("downtime",), ("no_downtime",)] + # A stop of 59 minutes is a short stop: no label for that day, and the next day has no stop. + assert _downtime_targets(tmp_path, short) == [("no_downtime",)] + + +def test_a_power_limit_is_not_downtime_and_a_long_one_leaves_the_day_without_the_task(tmp_path): + def curtailment(end: str) -> str: + return f"2022-03-02 10:00:00,2022-03-02 {end},00:00:00,Curtailment,108,Technical curtailment,,,Partial Performance\n" + + assert _downtime_targets(tmp_path, curtailment("10:30:00")) == [("no_downtime",), ("no_downtime",)] + assert _downtime_targets(tmp_path, curtailment("11:00:00")) == [("no_downtime",)] + + +def test_the_downtime_rule_by_minutes(): + minute_us = 60_000_000 + assert tasks._downtime_60(60 * minute_us, 0) == "downtime" + assert tasks._downtime_60(60 * minute_us, 600 * minute_us) == "downtime" + assert tasks._downtime_60(59 * minute_us, 0) is None + assert tasks._downtime_60(0, 59 * minute_us) == "no_downtime" + assert tasks._downtime_60(0, 60 * minute_us) is None + + +def test_a_record_states_how_many_power_values_are_missing(dataset): + # The first day has one step without any value. The second day has none. + assert [record.metadata["power_missing_steps"] for record in dataset.records] == [1, 0] + + +def test_the_card_has_the_new_version(): + assert str(KelmarshConnector().metadata().dataset_version) == "1.1.0" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/__init__.py new file mode 100644 index 000000000..bb257caf0 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/__init__.py @@ -0,0 +1,6 @@ +"""Penmanshiel wind farm: daily turbine SCADA records with operator stop and curtailment labels.""" + +from timenet_connectors.datasets.cubico.penmanshiel.connector import ( + CONNECTOR as CONNECTOR, + PenmanshielConnector as PenmanshielConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/connector.py new file mode 100644 index 000000000..94a546cb5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/connector.py @@ -0,0 +1,61 @@ +"""The Penmanshiel connector: one Record for each turbine and UTC day, with labels from the status log. + +The release has two zip archives for each year: one for turbines 01 to 10 and one for turbines 11 +to 15. The farm has no turbine 03. This connector reads 2019 to 2022. The later years use another +table layout. The files have the format of the Kelmarsh release, so the Kelmarsh modules read them. +""" + +from timenet_connectors.datasets.cubico.kelmarsh.farm import GreenbyteFarmConnector + + +class PenmanshielConnector(GreenbyteFarmConnector): + """Connector for the Penmanshiel wind farm release on Zenodo.""" + + FILES_URL = "https://zenodo.org/records/16807304/files" + STATIC_FILE = ("Penmanshiel_WT_static.csv", "2a742dfd5c8486bf54c1afb647bd803954f4c56aea31635e5aa93828f56ebd7b") + # Zenodo states an MD5 hash for each file. These SHA-256 hashes are of the files with those MD5 hashes. + YEAR_ARCHIVES = { # ruff: ignore[mutable-class-default] + 2019: ( + ( + "Penmanshiel_SCADA_2019_WT01-10_3112.zip", + "c611f4d04fa9f42ce093db643dcb11d96a995810ca52a9b5fda76dc8f26281d7", + ), + ( + "Penmanshiel_SCADA_2019_WT11-15_3117.zip", + "54523b9bffe717566f8ef056af0836e91ae2aed3e1a4083533f659a154a60eaf", + ), + ), + 2020: ( + ( + "Penmanshiel_SCADA_2020_WT01-10_3109.zip", + "f2bbf7f4efa24c9bdcc98cb987b17830272d77067a43958cd62ee5e6c490efe1", + ), + ( + "Penmanshiel_SCADA_2020_WT11-15_3118.zip", + "6c41960df290326831634aab2162c020a340f6ad439ed6eba2daa381ba707f31", + ), + ), + 2021: ( + ( + "Penmanshiel_SCADA_2021_WT01-10_4460.zip", + "aa66084e5d06ee41242ff53c2eae990edf2ae271072ee70c67def448becfb97d", + ), + ( + "Penmanshiel_SCADA_2021_WT11-15_4461.zip", + "7896dd7d314963efe4db589b6294a129d38a35773fc7b83c44ee23de73802cf6", + ), + ), + 2022: ( + ( + "Penmanshiel_SCADA_2022_WT01-10_4462.zip", + "94df920978c8eded7f3c6e300943108b27454312b3172639053c0ba21a7dafcd", + ), + ( + "Penmanshiel_SCADA_2022_WT11-15_4463.zip", + "d88e0e9d2af2409e23ef1c7a6de893f3bde53520068dcf36a92f402422ee03ad", + ), + ), + } + + +CONNECTOR = PenmanshielConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/dataset.yaml new file mode 100644 index 000000000..78a794590 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/dataset.yaml @@ -0,0 +1,45 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: cubico/penmanshiel +dataset_version: 1.1.0 +name: Penmanshiel wind farm +description: >- + Ten-minute SCADA data of the fourteen Senvion MM82 turbines (2,050 kW each) of the Penmanshiel + wind farm in the United Kingdom, for 2019 to 2022. One Record is one turbine for one UTC day, with + 144 steps of six signals: active power, wind speed, blade pitch angle, rotor speed, nacelle + direction, and ambient temperature. Each value is the mean of a ten-minute period, at the + start time of the period. A step that the source leaves empty stays missing. The labels come + from the status log of the operator, not from the signals. Each stop event of the log is an + annotation with the IEC category that the operator gave it, as one of six reasons, and a + second annotation with the message of the event. Each curtailment event is an annotation + too. The tasks ask whether the turbine stopped on the day, for how many minutes, in which + intervals, and for which main reason. A day gets the reason task only if every stop of the + day has a category. A stop entry without an end time is a point annotation. The days from + that stop to the next System OK entry of the log get no task, because the state of the + turbine is not known. This applies to one turbine, for 30 April and 1 May 2021. The logs + record curtailment only in 2021 and 2022, so only a turbine and year with a curtailment + entry has the task that asks whether the operator limited the output. A further task asks whether the turbine had at least 60 minutes of downtime on the day, where + downtime is stop time in the log. A day with no stop and less than 60 minutes of curtailment + is no downtime. A day with 1 to 59 minutes of stop, or with no stop and 60 minutes or more of + curtailment, gets no such task: a short stop is another question, and a power limit is not a + stop. The metadata of each Record states how many of the 144 steps have no power value, so + that a user can keep the days with full coverage. + The split follows + dates, the same for all turbines: days before July 2021 are train, the rest of 2021 is + validation, and 2022 is test. If a stop or curtailment runs across one of the two dates, the + days of that turbine that it touches get no task. The power setpoint and the alarm codes of the source are not + signals, because they state the answer. +license: CC-BY-4.0 +source_url: https://doi.org/10.5281/zenodo.16807304 +citation: >- + Plumley, C., & Takeuchi, R. (2025). Penmanshiel wind farm data. Zenodo. + https://doi.org/10.5281/zenodo.16807304 +domains: + - energy + - industrial +tags: + - wind-turbine + - scada + - classification + - temporal-localization + - scalar-prediction diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/test_connector.py new file mode 100644 index 000000000..08a4eb90f --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/cubico/penmanshiel/tests/test_connector.py @@ -0,0 +1,70 @@ +from pathlib import Path + +from timenet.types import ClassificationTask, read_value +from timenet_connectors.datasets.cubico.kelmarsh import tasks +from timenet_connectors.datasets.cubico.kelmarsh.farm import SOURCE_COLUMNS, TurbineYearFiles +from timenet_connectors.datasets.cubico.kelmarsh.greenbyte import STEPS_PER_DAY +from timenet_connectors.datasets.cubico.penmanshiel.connector import PenmanshielConnector + + +# A synthetic export of one invented turbine for one day. It has the shape of the release and +# holds no value of it. The turbine table of the release ends each line with a comma and has +# rows without a turbine. +_COMMENTS = "# Synthetic export.\n#\n# Turbine: Testfarm 02\n# Time zone: UTC\n#\n#\n#\n#\n#\n" +_STATIC = ( + "\ufeffWind Farm,Title,Alternative Title,Identity,Manufacturer,Model,Rated power (kW),Hub Height (m)," + "Rotor Diameter (m),Latitude,Longitude,Elevation (m),Country,Commercial Operations Date,\n" + "Testfarm,Testfarm 01,T01,X 1,Maker,M1,2000,59,82,1.5,-2.5,100,UK,01/09/2016,\n" + ",,,,,,,,,,,,,,\n" + "Testfarm,Testfarm 02,T02,X 2,Maker,M1,2000,60,82,1.6,-2.6,100,UK,01/09/2016,\n" + ",,,,,,,,,,,,,,\n" +) +_STATUS = ( + "Timestamp start,Timestamp end,Duration,Status,Code,Message,Comment,Service contract category,IEC category," + "Global contract category,Custom contract category\n" + "2022-03-01 10:00:00,2022-03-01 10:30:00,00:30:00,Curtailment,108,Grid constraint curtailment,,," + "Partial Performance,,\n" +) + + +def _write_release(directory: Path) -> TurbineYearFiles: + ref = TurbineYearFiles( + data_path=directory / "Turbine_Data_Testfarm_02.csv", + status_path=directory / "Status_Testfarm_02.csv", + static_path=directory / "static.csv", + ) + rows = [ + f"2022-03-01 {step // 6:02d}:{step % 6}0:00," + ",".join(["1.5"] * len(SOURCE_COLUMNS)) + for step in range(STEPS_PER_DAY) + ] + header = ",".join(["# Date and time", *SOURCE_COLUMNS]) + ref.data_path.write_text(_COMMENTS + "\n".join([header, *rows]) + "\n", encoding="utf-8") + ref.status_path.write_text(_COMMENTS + _STATUS, encoding="utf-8") + ref.static_path.write_text(_STATIC, encoding="utf-8") + return ref + + +def test_the_card_names_the_dataset(): + assert PenmanshielConnector().metadata().dataset_id == "cubico/penmanshiel" + + +def test_each_year_has_an_archive_for_each_group_of_turbines(): + for year, archives in PenmanshielConnector.YEAR_ARCHIVES.items(): + assert [name.split("_")[2:4] for name, _ in archives] == [[str(year), "WT01-10"], [str(year), "WT11-15"]] + + +def test_a_day_of_the_release_format_becomes_a_record_with_tasks(tmp_path): + dataset = PenmanshielConnector().convert([_write_release(tmp_path)]) + (record,) = dataset.records + assert record.metadata["turbine"] == "Testfarm 02" + assert record.metadata["hub_height_m"] == 60 + assert [(a.key, read_value(a.value)) for a in record.annotations] == [ + (tasks.CURTAILMENT_KEY, "Grid constraint curtailment") + ] + labels = {read_value(task.prompt): task.targets for task in dataset.tasks_of(ClassificationTask)} + assert labels[tasks.STOPPED_PROMPT] == ("running",) + assert labels[tasks.CURTAILED_PROMPT] == ("curtailed",) + + +def test_the_card_has_the_new_version(): + assert str(PenmanshielConnector().metadata().dataset_version) == "1.1.0" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py new file mode 100644 index 000000000..66557bdbe --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py @@ -0,0 +1 @@ +"""Connectors for the open wind farm datasets of RES (Renewable Energy Systems).""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/__init__.py new file mode 100644 index 000000000..bc02ad84b --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/__init__.py @@ -0,0 +1,6 @@ +"""Hill of Towie wind farm: daily turbine SCADA records with shutdown duration labels.""" + +from timenet_connectors.datasets.res.hill_of_towie.connector import ( + CONNECTOR as CONNECTOR, + HillOfTowieConnector as HillOfTowieConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/connector.py new file mode 100644 index 000000000..ea6ef3ef8 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/connector.py @@ -0,0 +1,394 @@ +"""The Hill of Towie connector: one Record for each turbine and UTC day, with shutdown labels. + +The release has one zip archive of ten-minute SCADA tables for each year, and a file with the +seconds of shutdown of each turbine and ten-minute period. RES computed those durations from the +SCADA data and the logs of its operators. They cover January 2016 to August 2024, so this +connector reads those months. The archives stay zipped, because a year is several gigabytes of +tables and the connector reads three of them. +""" + +from collections.abc import Iterator, Mapping +import csv +from dataclasses import dataclass +from datetime import UTC, datetime +from fractions import Fraction +from pathlib import Path +from typing import ClassVar + +import numpy as np + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.json import JsonMapping +from timenet.types import ( + US_PER_S, + Annotation, + ClassificationTask, + Label, + LabelSet, + MissingReason, + MissingValues, + ScalarPredictionTask, + Split, + Task, + TemporalLocalizationTask, + TimeInterval, + TimeOrigin, + TimeSeriesSpec, + unix_us, + ureg, +) +from timenet_connectors.datasets.cubico.kelmarsh.tasks import ( + DOWNTIME_60_PROMPT, + STOPPED_MINUTES_PROMPT, + STOPPED_PROMPT, + SplitCutoffs, +) +from timenet_connectors.datasets.res.hill_of_towie.scada import ( + DAY_S, + STEP_S, + STEPS_PER_DAY, + Grid, + read_shutdown_seconds, + read_signals, +) +from timenet_connectors.download import Artifact, download_files + + +SHUTDOWN_KEY = "shutdown" +STOPPED = LabelSet( + annotations=[Annotation("name", value="stopped")], + labels=( + Label(name="stopped", description="The turbine was shut down for some of this day."), + Label(name="running", description="The turbine was not shut down on this day."), + ), +) +DOWNTIME_60 = LabelSet( + annotations=[Annotation("name", value="downtime_60")], + labels=( + Label( + name="downtime", + description="The turbine was shut down for 60 minutes or more of this day while the wind was 4 m/s or more.", + ), + Label( + name="no_downtime", + description="The turbine had no shutdown time at wind of 4 m/s or more on this day, and almost no shutdown " + "time without a wind value.", + ), + ), +) +# Shutdown at less wind is not counted: the signals cannot tell it from waiting for wind. A day with some +# of it, or with shutdown time in periods that have no wind value, is in between and gets no label. +DOWNTIME_WIND_MS = 4.0 +DOWNTIME_S = 3600 +UNREAD_WIND_S = 600 +SHUTDOWN_PERIODS_PROMPT = ( + "In which ten-minute periods was the turbine shut down for some or all of the time on this day?" +) + +_GAPS = MissingValues( + reason=MissingReason.NOT_RECORDED, + description="The source has no row or no value for the ten-minute period.", +) + + +def _spec(spec_type: str, name: str, unit: object) -> TimeSeriesSpec: + return TimeSeriesSpec(spec_type=spec_type, name=name, unit_value=unit, dtype="float64", missing=_GAPS) + + +# The signal name, the source table and column, and the spec. Each value is the mean of a +# ten-minute period. The names and units are those of the Kelmarsh and Penmanshiel connectors. +SIGNALS: tuple[tuple[str, str, str, TimeSeriesSpec], ...] = ( + ("power", "tblSCTurGrid", "wtc_ActPower_mean", _spec("active_power", "Active power", ureg.kilowatt)), + ( + "wind_speed", + "tblSCTurbine", + "wtc_AcWindSp_mean", + _spec("wind_speed", "Wind speed", ureg.meter / ureg.second), + ), + ( + "pitch_angle", + "tblSCTurbine", + "wtc_PitcPosA_mean", + _spec("blade_pitch_angle", "Blade pitch angle", ureg.degree), + ), + ("rotor_speed", "tblSCTurbine", "wtc_MainSRpm_mean", _spec("rotor_speed", "Rotor speed", ureg.rpm)), + ( + "nacelle_direction", + "tblSCTurbine", + "wtc_NacelPos_mean", + _spec("nacelle_direction", "Nacelle direction", ureg.degree), + ), + ( + "ambient_temperature", + "tblSCTurTemp", + "wtc_AmbieTmp_mean", + _spec("ambient_temperature", "Ambient temperature", ureg.degC), + ), +) +# The source states the end of each period. A value is at the start of its period here. +EVERY_TEN_MINUTES = RegularAxis(period_us=Fraction(STEP_S * US_PER_S)) +DAY_US = DAY_S * US_PER_S + + +def _tables() -> dict[str, list[str]]: + tables: dict[str, list[str]] = {} + for _, table, column, _ in SIGNALS: + tables.setdefault(table, []).append(column) + return tables + + +TABLES = _tables() +# The order of the signals in the values that the reader gives. +_VALUE_ORDER = [name for table in TABLES for name, signal_table, _, _ in SIGNALS if signal_table == table] + + +@dataclass(frozen=True) +class HillOfTowieFiles: + """The downloaded files of the release.""" + + archives: tuple[Path, ...] + shutdown_path: Path + metadata_path: Path + + +class HillOfTowieConnector(BaseConnector[list[HillOfTowieFiles]]): + """Connector for the Hill of Towie wind farm release on Zenodo.""" + + FILES_URL: ClassVar[str] = "https://zenodo.org/records/22662930/files" + # Zenodo states an MD5 hash for each file. These SHA-256 hashes are of the files with those MD5 hashes. + METADATA_FILE: ClassVar[tuple[str, str]] = ( + "Hill_of_Towie_turbine_metadata.csv", + "dd40789c2bf2268da9d77890cc48617266f3ed9d13c5af2be6d6535671f261bc", + ) + SHUTDOWN_FILE: ClassVar[tuple[str, str]] = ( + "Hill_of_Towie_ShutdownDuration.zip", + "b69c5cb02e3c3e804b16ecdd024cead76edaba51a49a73ac12cc33f13170cab2", + ) + YEAR_ARCHIVES: ClassVar[Mapping[int, tuple[str, str]]] = { + 2016: ("2016.zip", "800bb3d399e6b79278758bbd95e234e364b062a9ff7643a7f9c045b0db9e9738"), + 2017: ("2017.zip", "a49c11def85623b7f0537988debfa499cca15fc6ec971256051597d91224ab1f"), + 2018: ("2018.zip", "4d59191bd12ad0e85eb4de44f2f8f7d8e2556452341e89cd194a816debc48761"), + 2019: ("2019.zip", "24a9b9bfdf32558d80c518a11954d7230254f5ba384671d3df6ba3e53deb5b26"), + 2020: ("2020.zip", "3e55de08be27e6a6e702a99bfcccd57e20abfa69e7c65d5901caf581f128f020"), + 2021: ("2021.zip", "2284741792b4bfc7ca08234f49440dad4820bac5e73098011745a3f69e295c19"), + 2022: ("2022.zip", "1b0f8657273ddbb8c4f84042a0bcb1c5eb1fb7833849b4c491f2b9f10604700d"), + 2023: ("2023.zip", "e65f7acd65664584e5b72e4ad7ccd4c0f0af4ad68f618b8cf0b4e5c1f4a69521"), + 2024: ("2024.zip", "b6bb540e3d2fcc2cff8756d96ef4155037b1cc41b284e6249b41054f4e50fbc3"), + } + FIRST_DAY: ClassVar[datetime] = datetime(2016, 1, 1, tzinfo=UTC) + END_DAY: ClassVar[datetime] = datetime(2024, 9, 1, tzinfo=UTC) + """The day after the last day. The shutdown durations end with August 2024.""" + SPLIT_CUTOFFS: ClassVar[SplitCutoffs] = SplitCutoffs( + validation=datetime(2022, 1, 1, tzinfo=UTC), test=datetime(2023, 1, 1, tzinfo=UTC) + ) + """The first six years are train, 2022 is validation, and 2023 to August 2024 is test.""" + + async def download_async(self, cache_dir: Path) -> list[HillOfTowieFiles]: + """Download the turbine table, the shutdown durations, and the archive of each year. + + Args: + cache_dir: The directory that holds the downloaded files. + + Returns: + One entry with the paths of the files. + """ + names = [self.METADATA_FILE, self.SHUTDOWN_FILE, *self.YEAR_ARCHIVES.values()] + await download_files( + [Artifact(f"{self.FILES_URL}/{name}?download=1", cache_dir / name, sha256=sha256) for name, sha256 in names] + ) + return [ + HillOfTowieFiles( + archives=tuple(cache_dir / name for name, _ in self.YEAR_ARCHIVES.values()), + shutdown_path=cache_dir / self.SHUTDOWN_FILE[0], + metadata_path=cache_dir / self.METADATA_FILE[0], + ) + ] + + def convert(self, raw_refs: list[HillOfTowieFiles], /) -> TimeFDataset: + """Create one Record for each turbine and day, with its annotations and tasks. + + A day without a row in any of the tables gets no Record. A day gets no task if a run of + shutdown of the day runs across a split cutoff. + + Args: + raw_refs: The entry from :meth:`download_async`. + + Returns: + The populated dataset. + """ + (files,) = raw_refs + dataset = TimeFDataset(metadata=self.metadata()) + turbines = _read_turbines(files.metadata_path) + start_s = int(self.FIRST_DAY.timestamp()) + days = (int(self.END_DAY.timestamp()) - start_s) // DAY_S + grid = Grid( + station_ids=tuple(station for station, _ in turbines), + turbine_names=tuple(str(turbine["turbine"]) for _, turbine in turbines), + start_s=start_s, + steps=days * STEPS_PER_DAY, + ) + shutdown = read_shutdown_seconds(files.shutdown_path, grid) + values, present = read_signals(files.archives, TABLES, grid) + any_row = present.any(axis=0) + for index, (_, turbine) in enumerate(turbines): + crossing = self._crossing_days(shutdown[index]) + for day in range(days): + steps = slice(day * STEPS_PER_DAY, (day + 1) * STEPS_PER_DAY) + if not any_row[index, steps].any(): + continue + day_start_us = unix_us(self.FIRST_DAY) + day * DAY_US + record = dataset.add_record(record=_create_record(values[index, steps], day_start_us, turbine)) + split = None if day_start_us in crossing else self.SPLIT_CUTOFFS.split_of_day(day_start_us) + _label_day( + dataset, + record, + shutdown[index, steps], + values[index, steps, _VALUE_ORDER.index("wind_speed")], + split, + ) + return dataset + + def _crossing_days(self, seconds: np.ndarray) -> set[int]: + """Give the days of one turbine that a run of shutdown across a split cutoff touches.""" + first_us, step_us = unix_us(self.FIRST_DAY), STEP_S * US_PER_S + runs = [(first_us + first * step_us, first_us + end * step_us) for first, end in _shutdown_intervals(seconds)] + return self.SPLIT_CUTOFFS.crossing_days(runs) + + +def _create_record(values: np.ndarray, day_start_us: int, turbine: JsonMapping) -> Record: + signals = tuple( + Signal.from_values( + [None if np.isnan(value) else float(value) for value in values[:, _VALUE_ORDER.index(name)]], + spec=spec, + name=name, + time_axis=EVERY_TEN_MINUTES, + ) + for name, _, _, spec in SIGNALS + ) + return Record( + sources=(Source(name="SCADA", signals=signals),), + start_time=TimeOrigin(day_start_us), + # The number of steps without a power value, so that a user can keep days with full coverage. + metadata={**turbine, "power_missing_steps": int(np.isnan(values[:, _VALUE_ORDER.index("power")]).sum())}, + time_span=TimeInterval.micros(0, DAY_US), + ) + + +def _shutdown_intervals(seconds: np.ndarray) -> list[tuple[int, int]]: + """Give the runs of periods with shutdown time, as first step and step after the last.""" + edges = np.flatnonzero(np.diff(np.concatenate([[False], seconds > 0, [False]]))) + return [(int(first), int(end)) for first, end in zip(edges[::2], edges[1::2], strict=True)] + + +def _label_day( + dataset: TimeFDataset, record: Record, seconds: np.ndarray, wind: np.ndarray, split: Split | None +) -> None: + """Add the shutdown annotations of one turbine day, and its tasks if the day has a split.""" + intervals = _shutdown_intervals(seconds) + for first, end in intervals: + record.annotate( + Annotation(key=SHUTDOWN_KEY, value=int(seconds[first:end].sum()), span=_span_of_steps(first, end)) + ) + if split is not None: + dataset.add_tasks(tasks=list(_iter_day_tasks(record, seconds, wind, intervals, split))) + + +def _span_of_steps(first: int, end: int) -> TimeInterval: + return TimeInterval.micros(first * STEP_S * US_PER_S, end * STEP_S * US_PER_S) + + +def _downtime_60(seconds: np.ndarray, wind: np.ndarray) -> str | None: + """Give the downtime label of a day. + + Only the shutdown at wind of 4 m/s or more counts. A day with an hour of it is ``downtime``. A + day with none of it, and less than ten minutes of shutdown in periods without a wind value, is + ``no_downtime``. + + Args: + seconds: The seconds of shutdown of each of the 144 periods of the day. + wind: The wind speed of each period, with NaN where the source has no value. + + Returns: + ``downtime``, ``no_downtime``, or ``None`` if the day is in between. + """ + windy_s = float(seconds[wind >= DOWNTIME_WIND_MS].sum()) + unread_s = float(seconds[np.isnan(wind)].sum()) + if windy_s >= DOWNTIME_S: + return "downtime" + return "no_downtime" if windy_s == 0 and unread_s < UNREAD_WIND_S else None + + +def _iter_day_tasks( + record: Record, seconds: np.ndarray, wind: np.ndarray, intervals: list[tuple[int, int]], split: Split +) -> Iterator[Task]: + """Create the tasks of one turbine day. + + Yields: + The tasks, not yet added to a dataset. + """ + yield ClassificationTask( + inputs=(record,), + targets=("stopped" if intervals else "running",), + label_set=STOPPED, + prompt=STOPPED_PROMPT, + split=split, + ) + yield ScalarPredictionTask( + inputs=(record,), + targets=(float(seconds.sum()) / 60,), + unit="minute", + target_name="stopped_minutes", + prompt=STOPPED_MINUTES_PROMPT, + split=split, + ) + if intervals: + yield TemporalLocalizationTask( + inputs=(record,), + targets=tuple(_span_of_steps(first, end) for first, end in intervals), + prompt=SHUTDOWN_PERIODS_PROMPT, + split=split, + ) + downtime = _downtime_60(seconds, wind) + if downtime is not None: + yield ClassificationTask( + inputs=(record,), targets=(downtime,), label_set=DOWNTIME_60, prompt=DOWNTIME_60_PROMPT, split=split + ) + + +def _read_turbines(metadata_path: Path) -> list[tuple[int, JsonMapping]]: + """Read the table that describes the turbines. + + Returns: + The station id and the facts of each turbine, in station order. The facts are the metadata + of each Record of the turbine. + + Raises: + TimeNetConnectorError: If the table has no turbine. + """ + turbines: list[tuple[int, JsonMapping]] = [] + with metadata_path.open(encoding="utf-8-sig", newline="") as stream: + for row in csv.DictReader(stream): + turbines.append( + ( + int(row["Station ID"]), + { + "wind_farm": row["Wind Farm"], + "turbine": row["Turbine Name"], + "manufacturer": row["Manufacturer"], + "model": row["Model"], + "rated_power_kw": float(row["Rated power (kW)"]), + "hub_height_m": float(row["Hub Height (m)"]), + "rotor_diameter_m": float(row["Rotor Diameter (m)"]), + "latitude": float(row["Latitude"]), + "longitude": float(row["Longitude"]), + }, + ) + ) + if not turbines: + raise TimeNetConnectorError(f"{metadata_path.name}: no turbine") + return sorted(turbines, key=lambda item: item[0]) + + +CONNECTOR = HillOfTowieConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/dataset.yaml new file mode 100644 index 000000000..382387f74 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/dataset.yaml @@ -0,0 +1,42 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: res/hill_of_towie +dataset_version: 1.1.0 +name: Hill of Towie wind farm +description: >- + Ten-minute SCADA data of the 21 Siemens SWT-2.3-VS-82 turbines (2,300 kW each) of the Hill of + Towie wind farm in the United Kingdom, for January 2016 to August 2024. One Record is one + turbine for one UTC day, with 144 steps of six signals: active power, wind speed, blade pitch + angle, rotor speed, nacelle direction, and ambient temperature. Each value is the mean of a + ten-minute period. The source states the end of each period. Here a value is at the start of + its period. A period that the source leaves empty or has no row for stays missing. A day + without a row gets no Record. The labels are the shutdown durations of the release: the + seconds of shutdown of each turbine and period, which RES computed from the SCADA data and + the logs of its operators. Each run of periods with shutdown time is an annotation with the + seconds of shutdown in the run. The tasks ask whether the turbine was shut down on the day, + for how many minutes, and in which ten-minute periods. + A further task asks whether the turbine had at least 60 minutes of downtime on the day. Only + shutdown time in periods with wind of 4 m/s or more counts, because shutdown in lighter wind + cannot be told from waiting for wind in the signals. A day with such an hour is downtime. A + day with none of it, and less than ten minutes of shutdown in periods without a wind value, + is no downtime. Other days get no such task. The metadata of each Record states how many of + the 144 steps have no power value, so that a user can keep the days with full coverage. The release gives no reason for a + shutdown and no curtailment label. The split follows dates, the same for all + turbines: days before 2022 are train, 2022 is validation, and 2023 to August 2024 is test. If + a run of shutdown runs across one of the two dates, the days of that turbine that it touches + get no task. The power setpoint, the status + counters, and the alarm log of the source are not signals, because they state the answer. +license: CC-BY-4.0 +source_url: https://doi.org/10.5281/zenodo.22662930 +citation: >- + Clerc, A., & Lingkan, E. (2026). Hill of Towie wind farm open dataset (2.1.0). Zenodo. + https://doi.org/10.5281/zenodo.22662930 +domains: + - energy + - industrial +tags: + - wind-turbine + - scada + - classification + - temporal-localization + - scalar-prediction diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/scada.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/scada.py new file mode 100644 index 000000000..7ac8b8c08 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/scada.py @@ -0,0 +1,196 @@ +"""Readers for the ten-minute SCADA tables and the shutdown durations of the Hill of Towie release. + +A year archive holds one CSV file for each table and month. A row is one turbine for one +ten-minute period, and its ``TimeStamp`` is the end of the period in UTC. The shutdown file gives +the seconds of shutdown of each turbine and period, with the start of the period in UTC. The +readers put both on one grid of period starts. +""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +import re +import zipfile + +import numpy as np +import pyarrow as pa +from pyarrow import csv as pa_csv + +from timenet.errors import TimeNetConnectorError + + +STEP_S = 600 +STEPS_PER_DAY = 144 +DAY_S = STEP_S * STEPS_PER_DAY + +TIMESTAMP_COLUMN = "TimeStamp" +STATION_COLUMN = "StationId" +SHUTDOWN_MEMBER = "ShutdownDuration.csv" +SHUTDOWN_START_COLUMN = "TimeStamp_StartFormat" +SHUTDOWN_TURBINE_COLUMN = "TurbineName" +SHUTDOWN_SECONDS_COLUMN = "ShutdownDuration" + + +@dataclass(frozen=True) +class Grid: + """The turbines and the ten-minute periods that the connector reads.""" + + station_ids: tuple[int, ...] + turbine_names: tuple[str, ...] + start_s: int + """The start of the first period, in Unix seconds.""" + steps: int + + def __post_init__(self) -> None: + if len(self.station_ids) != len(self.turbine_names): + raise TimeNetConnectorError("The grid needs one name for each station.") + + +def read_signals( + archives: Sequence[Path], tables: Mapping[str, Sequence[str]], grid: Grid +) -> tuple[np.ndarray, np.ndarray]: + """Read the columns of the tables onto the grid. + + A period without a row in the source stays ``NaN``. The first period of a month is in the file + of that month and in the file of the month before. Such a repeated row must state the same + values. + + Args: + archives: The year archives. + tables: The columns to read from each table, in the order of the result. + grid: The turbines and periods to keep. A row of another period is skipped. + + Returns: + The values with shape ``(turbines, steps, columns)``, and whether each table has a row for + each turbine and period, with shape ``(tables, turbines, steps)``. + + Raises: + TimeNetConnectorError: If an archive has no file for a table. + """ + widths = [len(columns) for columns in tables.values()] + values = np.full((len(grid.station_ids), grid.steps, sum(widths)), np.nan) + present = np.zeros((len(tables), len(grid.station_ids), grid.steps), dtype=bool) + offsets = np.cumsum([0, *widths]) + for archive in archives: + with zipfile.ZipFile(archive) as opened: + for index, (table, columns) in enumerate(tables.items()): + pattern = re.compile(rf"(?:.*/)?{re.escape(table)}_\d{{4}}_\d{{2}}\.csv") + members = sorted(name for name in opened.namelist() if pattern.fullmatch(name)) + if not members: + raise TimeNetConnectorError(f"{archive}: no file for table {table}") + for member in members: + _read_member( + opened, + member, + columns, + grid, + values=values[:, :, offsets[index] : offsets[index + 1]], + present=present[index], + ) + return values, present + + +def _read_member( # ruff: ignore[too-many-arguments] + opened: zipfile.ZipFile, member: str, columns: Sequence[str], grid: Grid, *, values: np.ndarray, present: np.ndarray +) -> None: + """Put the rows of one table file onto the grid. + + Raises: + TimeNetConnectorError: If a row is off the ten-minute grid, has an unknown station, or + repeats another row with other values. + """ + types = {TIMESTAMP_COLUMN: pa.string(), STATION_COLUMN: pa.int64(), **dict.fromkeys(columns, pa.float64())} + try: + with opened.open(member) as stream: + table = pa_csv.read_csv( + stream, convert_options=pa_csv.ConvertOptions(include_columns=list(types), column_types=types) + ) + except (pa.ArrowInvalid, KeyError) as error: + raise TimeNetConnectorError(f"{member}: {error}") from error + end_s = _parse_utc_seconds(table[TIMESTAMP_COLUMN].to_numpy(zero_copy_only=False), member) + offset_s = end_s - STEP_S - grid.start_s + if (offset_s % STEP_S).any(): + raise TimeNetConnectorError(f"{member}: a timestamp is not on the ten-minute grid") + steps = offset_s // STEP_S + stations = table[STATION_COLUMN].to_numpy(zero_copy_only=False) + turbines = np.searchsorted(grid.station_ids, stations) + known = turbines < len(grid.station_ids) + known[known] = np.asarray(grid.station_ids)[turbines[known]] == stations[known] + if not known.all(): + raise TimeNetConnectorError(f"{member}: unknown station {stations[~known][0]}") + keep = (steps >= 0) & (steps < grid.steps) + turbines, steps = turbines[keep], steps[keep] + rows = np.column_stack([table[column].to_numpy(zero_copy_only=False) for column in columns])[keep] + cells = turbines * grid.steps + steps + if len(np.unique(cells)) != len(cells): + raise TimeNetConnectorError(f"{member}: two rows for the same station and timestamp") + repeated = present[turbines, steps] + if not np.array_equal(values[turbines[repeated], steps[repeated]], rows[repeated], equal_nan=True): + raise TimeNetConnectorError(f"{member}: a row repeats a row of an earlier file with other values") + values[turbines, steps] = rows + present[turbines, steps] = True + + +def read_shutdown_seconds(archive: Path, grid: Grid) -> np.ndarray: + """Read the seconds of shutdown of each turbine and period of the grid. + + Args: + archive: The archive with the shutdown durations. + grid: The turbines and periods to keep. A row of another period is skipped. + + Returns: + The seconds with shape ``(turbines, steps)``. + + Raises: + TimeNetConnectorError: If the file cannot be read, a row is off the grid, names an unknown + turbine, or has a duration outside a period, or a turbine and period of the grid has + no row or two rows. + """ + types = { + SHUTDOWN_START_COLUMN: pa.timestamp("s", tz="UTC"), + SHUTDOWN_TURBINE_COLUMN: pa.string(), + SHUTDOWN_SECONDS_COLUMN: pa.int64(), + } + try: + with zipfile.ZipFile(archive) as opened, opened.open(SHUTDOWN_MEMBER) as stream: + table = pa_csv.read_csv( + stream, convert_options=pa_csv.ConvertOptions(include_columns=list(types), column_types=types) + ) + except (pa.ArrowInvalid, KeyError) as error: + raise TimeNetConnectorError(f"{SHUTDOWN_MEMBER}: {error}") from error + offset_s = table[SHUTDOWN_START_COLUMN].cast(pa.int64()).to_numpy() - grid.start_s + if (offset_s % STEP_S).any(): + raise TimeNetConnectorError(f"{SHUTDOWN_MEMBER}: a timestamp is not on the ten-minute grid") + names = table[SHUTDOWN_TURBINE_COLUMN].dictionary_encode().combine_chunks() + index_of_name = {name: index for index, name in enumerate(grid.turbine_names)} + unknown = [name for name in names.dictionary.to_pylist() if name not in index_of_name] + if unknown: + raise TimeNetConnectorError(f"{SHUTDOWN_MEMBER}: unknown turbine {unknown[0]!r}") + turbines = np.array([index_of_name[name] for name in names.dictionary.to_pylist()])[names.indices.to_numpy()] + seconds = table[SHUTDOWN_SECONDS_COLUMN].to_numpy() + if ((seconds < 0) | (seconds > STEP_S)).any(): + raise TimeNetConnectorError(f"{SHUTDOWN_MEMBER}: a duration is outside 0 to {STEP_S} seconds") + steps = offset_s // STEP_S + keep = (steps >= 0) & (steps < grid.steps) + counts = np.zeros((len(grid.station_ids), grid.steps), dtype=np.int64) + np.add.at(counts, (turbines[keep], steps[keep]), 1) + if (counts != 1).any(): + raise TimeNetConnectorError(f"{SHUTDOWN_MEMBER}: a turbine and period has no row or more than one row") + result = np.zeros((len(grid.station_ids), grid.steps), dtype=np.int64) + result[turbines[keep], steps[keep]] = seconds[keep] + return result + + +def _parse_utc_seconds(texts: np.ndarray, member: str) -> np.ndarray: + """Parse UTC timestamps. The source writes midnight with or without the time of day. + + Returns: + The timestamps in Unix seconds. + + Raises: + TimeNetConnectorError: If a text is not a timestamp. + """ + try: + return texts.astype("datetime64[s]").astype(np.int64) + except ValueError as error: + raise TimeNetConnectorError(f"{member}: {error}") from error diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/test_connector.py new file mode 100644 index 000000000..4cc2447f4 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie/tests/test_connector.py @@ -0,0 +1,290 @@ +from datetime import UTC, datetime, timedelta +from pathlib import Path +import zipfile + +import numpy as np +import pytest + +from timenet.dataset import TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ( + ClassificationTask, + ScalarPredictionTask, + Split, + Task, + TemporalLocalizationTask, + TimeInterval, + read_value, + unix_us, +) +from timenet_connectors.datasets.cubico.kelmarsh.tasks import STOPPED_MINUTES_PROMPT, STOPPED_PROMPT, SplitCutoffs +from timenet_connectors.datasets.res.hill_of_towie import connector as hill_of_towie +from timenet_connectors.datasets.res.hill_of_towie.connector import TABLES, HillOfTowieConnector, HillOfTowieFiles +from timenet_connectors.datasets.res.hill_of_towie.scada import STEPS_PER_DAY + + +# A synthetic release of two invented turbines for three days in two months. It has the shape of +# the release and holds no value of it. +_START = datetime(2020, 1, 30, tzinfo=UTC) +_DAYS = 3 +_STATIONS = {"T01": 2304510, "T02": 2304511} +_METADATA = ( + "Wind Farm,Turbine Name,Latitude,Longitude,Station ID,Manufacturer,Model,Rated power (kW)," + "Hub Height (m),Rotor Diameter (m),Country,Commercial Operations Date\n" + "Testfarm,T02,1.6,-2.6,2304511,Maker,M1,2300,59,82,UK,2012-05-12\n" + "Testfarm,T01,1.5,-2.5,2304510,Maker,M1,2300,59,82,UK,2012-05-12\n" +) +_ABSENT_STEP = 7 +"""The step of the first day for which T01 has no row in any table.""" +_EMPTY_DAY = 1 +"""The day for which T02 has no row in any table.""" +_TEN_MINUTES_US = 600_000_000 +# Seconds of shutdown of T01 by step: a run on day 1 that ends partly stopped, and a run that +# crosses from day 1 into day 2. +_SHUTDOWN = {10: 600, 11: 600, 12: 120, 143: 30, 144: 600} + + +class _TestRelease(HillOfTowieConnector): + CARD = Path(hill_of_towie.__file__).with_name("dataset.yaml") + FIRST_DAY = _START + END_DAY = _START + timedelta(days=_DAYS) + + +def _value(turbine: int, column: int, step: int) -> float: + return turbine * 10_000 + column * 1000 + step / 3 + + +def _has_row(name: str, step: int) -> bool: + if name == "T01": + return step != _ABSENT_STEP + return step // STEPS_PER_DAY != _EMPTY_DAY + + +def _table_files(table: str, columns: list[str], *, conflict: bool) -> dict[str, str]: + """Write a table as one file for each month, with the first period of a month in both files.""" + column_offset = [name for names in TABLES.values() for name in names].index(columns[0]) + header = ",".join(["TimeStamp", "StationId", "wtc_Other_mean", *columns]) + "\n" + months: dict[str, list[str]] = {"2020_01": [], "2020_02": []} + # The step before the first one ends at the start of the window. The reader skips it. + for step in range(-1, _DAYS * STEPS_PER_DAY): + end = _START + timedelta(minutes=10 * (step + 1)) + # The source writes midnight with or without the time of day. + stamp = f"{end:%Y-%m-%d}" if (end.hour, end.minute, step % 2) == (0, 0, 0) else f"{end:%Y-%m-%d %H:%M:%S}" + for turbine, (name, station) in enumerate(_STATIONS.items()): + if not _has_row(name, step): + continue + cells = [repr(_value(turbine, column_offset + column, step)) for column in range(len(columns))] + if step == 0 and table == "tblSCTurGrid": + cells[0] = "" + row = ",".join([stamp, str(station), "7", *cells]) + "\n" + february = datetime(2020, 2, 1, tzinfo=UTC) + if end <= february: + months["2020_01"].append(row) + if end >= february: + other = conflict and end == february and table == "tblSCTurGrid" + months["2020_02"].append(",".join([stamp, str(station), "7", "1"]) + "\n" if other else row) + return {f"{table}_{month}.csv": header + "".join(rows) for month, rows in months.items()} + + +def _write_release(directory: Path, *, conflict: bool = False, shutdown_rows: int | None = None) -> HillOfTowieFiles: + files = HillOfTowieFiles( + archives=(directory / "2020.zip",), + shutdown_path=directory / "shutdown.zip", + metadata_path=directory / "metadata.csv", + ) + with zipfile.ZipFile(files.archives[0], "w") as archive: + for table, columns in TABLES.items(): + for name, text in _table_files(table, columns, conflict=conflict).items(): + archive.writestr(name, text) + archive.writestr("tblAlarmLog_2020_01.csv", "TimeStamp,StationId\n") + rows = [ + f"{_START + timedelta(minutes=10 * step):%Y-%m-%d %H:%M:%S}+00:00,{name}," + f"{_SHUTDOWN.get(step, 0) if name == 'T01' else 0}\n" + for step in range(_DAYS * STEPS_PER_DAY + 1) + for name in _STATIONS + ] + with zipfile.ZipFile(files.shutdown_path, "w") as archive: + archive.writestr( + "ShutdownDuration.csv", + "TimeStamp_StartFormat,TurbineName,ShutdownDuration\n" + "".join(rows[:shutdown_rows]), + ) + files.metadata_path.write_text(_METADATA, encoding="utf-8") + return files + + +@pytest.fixture +def dataset(tmp_path) -> TimeFDataset: + return _TestRelease().convert([_write_release(tmp_path)]) + + +def _targets(dataset: TimeFDataset, task_type: type[Task], prompt: str) -> list[object]: + found = [task for task in dataset.tasks_of(task_type) if read_value(task.prompt) == prompt] + return [tuple(read_value(target) for target in task.targets or ()) for task in found] + + +def test_each_day_with_a_row_becomes_a_record_at_the_start_of_each_period(dataset): + days = [(record.metadata["turbine"], record.start_time.timestamp) for record in dataset.records] + assert days == [ + *[("T01", unix_us(_START + timedelta(days=day))) for day in range(_DAYS)], + *[("T02", unix_us(_START + timedelta(days=day))) for day in range(_DAYS) if day != _EMPTY_DAY], + ] + first = dataset.records[0] + assert first.metadata["rated_power_kw"] == pytest.approx(2300) + by_name = {signal.name: signal for signal in first.signals} + assert set(by_name) == { + "power", + "wind_speed", + "pitch_angle", + "rotor_speed", + "nacelle_direction", + "ambient_temperature", + } + # The row that ends at 00:10 holds the first period of the day. Power is column 0, and + # ambient temperature is the last column. + for name, column in (("power", 0), ("ambient_temperature", 5)): + expected = np.array([_value(0, column, step) for step in range(STEPS_PER_DAY)]) + expected[_ABSENT_STEP] = np.nan + if name == "power": + expected[0] = np.nan + values = by_name[name].to_arrow() + # A period without a row and an empty cell stay missing. The connector does not fill them. + assert values[_ABSENT_STEP].as_py() is None + np.testing.assert_array_equal(values.to_numpy(zero_copy_only=False), expected) + + +def test_the_signal_columns_of_a_table_keep_their_order(dataset): + record = dataset.records[_DAYS] # the first day of T02 + by_name = {signal.name: signal.to_arrow()[3].as_py() for signal in record.signals} + columns = [name for names in TABLES.values() for name in names] + assert by_name["wind_speed"] == pytest.approx(_value(1, columns.index("wtc_AcWindSp_mean"), 3)) + assert by_name["rotor_speed"] == pytest.approx(_value(1, columns.index("wtc_MainSRpm_mean"), 3)) + assert by_name["nacelle_direction"] == pytest.approx(_value(1, columns.index("wtc_NacelPos_mean"), 3)) + + +def test_runs_of_shutdown_become_annotations_and_tasks(dataset): + first, second, third = dataset.records[:_DAYS] + spans = [[(a.key, a.value, a.span) for a in record.annotations] for record in (first, second, third)] + assert spans == [ + [ + ("shutdown", 1320, TimeInterval.micros(10 * _TEN_MINUTES_US, 13 * _TEN_MINUTES_US)), + ("shutdown", 30, TimeInterval.micros(143 * _TEN_MINUTES_US, 144 * _TEN_MINUTES_US)), + ], + [("shutdown", 600, TimeInterval.micros(0, _TEN_MINUTES_US))], + [], + ] + assert _targets(dataset, ClassificationTask, STOPPED_PROMPT) == [ + ("stopped",), + ("stopped",), + ("running",), + ("running",), + ("running",), + ] + assert _targets(dataset, ScalarPredictionTask, STOPPED_MINUTES_PROMPT) == [(22.5,), (10.0,), (0.0,), (0.0,), (0.0,)] + assert _targets(dataset, TemporalLocalizationTask, hill_of_towie.SHUTDOWN_PERIODS_PROMPT) == [ + ( + TimeInterval.micros(10 * _TEN_MINUTES_US, 13 * _TEN_MINUTES_US), + TimeInterval.micros(143 * _TEN_MINUTES_US, 144 * _TEN_MINUTES_US), + ), + (TimeInterval.micros(0, _TEN_MINUTES_US),), + ] + assert {task.split for task in dataset.tasks} == {Split.TRAIN} + + +def test_a_run_of_shutdown_across_a_cutoff_leaves_its_days_without_tasks(tmp_path): + # The run of T01 from step 143 to step 144 runs across the start of day 2. + class CutoffAtDayTwo(_TestRelease): + SPLIT_CUTOFFS = SplitCutoffs(validation=_START + timedelta(days=1), test=_START + timedelta(days=2)) + + dataset = CutoffAtDayTwo().convert([_write_release(tmp_path)]) + first, second = dataset.records[:2] + assert len(first.annotations) == 2 + assert len(second.annotations) == 1 + splits = [(task.inputs[0].metadata["turbine"], task.split) for task in dataset.tasks_of(ScalarPredictionTask)] + assert splits == [("T01", Split.TEST), ("T02", Split.TRAIN), ("T02", Split.TEST)] + + +def test_split_follows_the_dates(): + cutoffs = HillOfTowieConnector.SPLIT_CUTOFFS + assert cutoffs.split_of_day(unix_us(datetime(2021, 12, 31, tzinfo=UTC))) == Split.TRAIN + assert cutoffs.split_of_day(unix_us(datetime(2022, 1, 1, tzinfo=UTC))) == Split.VALIDATION + assert cutoffs.split_of_day(unix_us(datetime(2022, 12, 31, tzinfo=UTC))) == Split.VALIDATION + assert cutoffs.split_of_day(unix_us(datetime(2023, 1, 1, tzinfo=UTC))) == Split.TEST + + +def test_a_repeated_row_with_other_values_is_rejected(tmp_path): + files = _write_release(tmp_path, conflict=True) + with pytest.raises(TimeNetConnectorError, match="repeats a row"): + _TestRelease().convert([files]) + + +def test_a_period_without_a_shutdown_duration_is_rejected(tmp_path): + files = _write_release(tmp_path, shutdown_rows=-4) + with pytest.raises(TimeNetConnectorError, match="no row or more than one row"): + _TestRelease().convert([files]) + + +def test_an_archive_without_a_table_is_rejected(tmp_path): + files = _write_release(tmp_path) + with zipfile.ZipFile(files.archives[0], "w") as archive: + archive.writestr("tblSCTurGrid_2020_01.csv", "TimeStamp,StationId,wtc_ActPower_mean\n") + with pytest.raises(TimeNetConnectorError, match="no file for table"): + _TestRelease().convert([files]) + + +def test_the_downtime_task_counts_only_shutdown_at_wind_of_4_ms_or_more(dataset): + # No day of the release has an hour of shutdown, and the wind values are high. A day with a shutdown + # of 22.5 or 10 minutes is in between and has no task; a day with none is no downtime. + assert _targets(dataset, ClassificationTask, hill_of_towie.DOWNTIME_60_PROMPT) == [ + ("no_downtime",), + ("no_downtime",), + ("no_downtime",), + ] + + +def test_an_hour_of_shutdown_in_wind_is_downtime(tmp_path, monkeypatch): + monkeypatch.setitem(globals(), "_SHUTDOWN", dict.fromkeys(range(20, 26), 600)) + dataset = _TestRelease().convert([_write_release(tmp_path)]) + labels = [] + for task in dataset.tasks_of(ClassificationTask): + if read_value(task.prompt) == hill_of_towie.DOWNTIME_60_PROMPT: + (record,) = task.inputs + assert task.targets is not None + labels.append((record.metadata["turbine"], read_value(task.targets[0]))) + assert labels == [ + ("T01", "downtime"), + ("T01", "no_downtime"), + ("T01", "no_downtime"), + ("T02", "no_downtime"), + ("T02", "no_downtime"), + ] + + +def test_the_downtime_rule_by_wind(): + seconds = np.zeros(144) + wind = np.full(144, 8.0) + assert hill_of_towie._downtime_60(seconds, wind) == "no_downtime" + seconds[:6] = 600 + assert hill_of_towie._downtime_60(seconds, wind) == "downtime" + seconds[5] = 599 + assert hill_of_towie._downtime_60(seconds, wind) is None + # Shutdown in light wind does not count and does not block the label. + assert hill_of_towie._downtime_60(seconds, np.full(144, 3.9)) == "no_downtime" + # Shutdown in periods without a wind value: under ten minutes is fine, ten minutes or more is not known. + unread = np.full(144, 8.0) + unread[:1] = np.nan + seconds = np.zeros(144) + seconds[0] = 599 + assert hill_of_towie._downtime_60(seconds, unread) == "no_downtime" + seconds[0] = 600 + assert hill_of_towie._downtime_60(seconds, unread) is None + + +def test_a_record_states_how_many_power_values_are_missing(dataset): + for record in dataset.records: + power = next(signal for signal in record.signals if signal.name == "power") + assert record.metadata["power_missing_steps"] == power.null_count + assert dataset.records[0].metadata["power_missing_steps"] >= 1 + + +def test_the_card_has_the_new_version(): + assert str(HillOfTowieConnector().metadata().dataset_version) == "1.1.0"