From c76ed1502732d51efea66c30ac636b65054d6bdb Mon Sep 17 00:00:00 2001 From: "M. Kalk" <44619887+ZMakschZ@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:21:55 +0100 Subject: [PATCH 1/2] feat(connectors): add Hill of Towie SCADA forecasting dataset --- .../datasets/res/__init__.py | 1 + .../res/hill_of_towie_scada/__init__.py | 6 + .../res/hill_of_towie_scada/connector.py | 229 ++++++++++++++++++ .../res/hill_of_towie_scada/dataset.yaml | 25 ++ .../res/hill_of_towie_scada/tests/__init__.py | 1 + .../tests/test_connector.py | 99 ++++++++ 6 files changed, 361 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py new file mode 100644 index 000000000..6af311826 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py @@ -0,0 +1 @@ +"""Renewable Energy Systems datasets.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py new file mode 100644 index 000000000..2f14b3731 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py @@ -0,0 +1,6 @@ +"""Hill of Towie SCADA forecasting connector.""" + +from timenet_connectors.datasets.res.hill_of_towie_scada.connector import ( + CONNECTOR as CONNECTOR, + HillOfTowieSCADAConnector as HillOfTowieSCADAConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py new file mode 100644 index 000000000..f0a8e9856 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py @@ -0,0 +1,229 @@ +"""Convert Hill of Towie ten-minute SCADA into turbine power forecasts.""" + +from collections.abc import Iterator, Sequence +import csv +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from fractions import Fraction +from io import TextIOWrapper +from math import isfinite +from pathlib import Path +from zipfile import BadZipFile, ZipFile + +import pyarrow as pa + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import ForecastingTask, Split, TimeOrigin, TimeSeriesSpec, ureg +from timenet_connectors.download import Artifact, download_files + + +_FILES_URL = "https://zenodo.org/records/22662930/files" +_ARCHIVE = "2026.zip" +_ARCHIVE_SHA256 = "4bcfd706769369eacde59ea4f3a45bc3f9c0fb334c67fea75fcf3aa0c424eaea" +_METADATA = "Hill_of_Towie_turbine_metadata.csv" +_METADATA_SHA256 = "dd40789c2bf2268da9d77890cc48617266f3ed9d13c5af2be6d6535671f261bc" + +_SPLITS = {1: Split.TRAIN, 2: Split.TRAIN, 3: Split.VALIDATION, 4: Split.TEST} +_INTERVAL = timedelta(minutes=10) +_AXIS = RegularAxis.from_rate_hz(Fraction(1, 600)) +_CONTEXT_STEPS = 36 +_FORECAST_STEPS = 6 +_POWER = TimeSeriesSpec(spec_type="active_power", name="Active power", unit_value=ureg.kilowatt, dtype="float64") +_WIND = TimeSeriesSpec( + spec_type="wind_speed", name="Active wind speed", unit_value=ureg.meter / ureg.second, dtype="float64" +) +_RPM = TimeSeriesSpec(spec_type="generator_speed", name="Generator speed", unit_value=ureg.rpm, dtype="float64") + +_ScadaKey = tuple[int, datetime] + + +@dataclass(frozen=True) +class HillOfTowieFiles: + """Local source archive and turbine metadata.""" + + archive: Path + metadata: Path + + +@dataclass(frozen=True) +class _Turbine: + name: str + rated_power_kw: int + + +def _read_turbines(path: Path) -> dict[int, _Turbine]: + """Read station IDs and static turbine metadata from the published lookup.""" + turbines: dict[int, _Turbine] = {} + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + for row in reader: + try: + station = int(row["Station ID"]) + turbine = _Turbine(row["Turbine Name"], int(row["Rated power (kW)"])) + except (KeyError, TypeError, ValueError) as exc: + raise TimeNetConnectorError(f"{path.name}:{reader.line_num}: invalid turbine metadata") from exc + if station in turbines or not turbine.name or turbine.rated_power_kw <= 0: + raise TimeNetConnectorError(f"{path.name}:{reader.line_num}: invalid or duplicate station {station}") + turbines[station] = turbine + if not turbines: + raise TimeNetConnectorError(f"{path.name}: no turbine metadata") + return turbines + + +def _read_measurements( + archive: ZipFile, month: int, table: str, columns: tuple[str, ...] +) -> dict[_ScadaKey, tuple[float, ...]]: + """Read complete measurements for the month owning each UTC sampling interval.""" + member = f"{table}_2026_{month:02}.csv" + values: dict[_ScadaKey, tuple[float, ...]] = {} + seen: set[_ScadaKey] = set() + with archive.open(member) as raw, TextIOWrapper(raw, encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + required = {"StationId", "TimeStamp", *columns} + if missing := required.difference(reader.fieldnames or ()): + raise TimeNetConnectorError(f"{member}: missing columns {sorted(missing)}") + for row in reader: + try: + station = int(row["StationId"]) + timestamp = datetime.fromisoformat(row["TimeStamp"]) + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=UTC) + timestamp = timestamp.astimezone(UTC) + # The publisher assigns a boundary interval to the month in which it starts. + interval_start = timestamp - _INTERVAL + if (interval_start.year, interval_start.month) != (2026, month): + continue + if timestamp.minute % 10 or timestamp.second or timestamp.microsecond: + raise ValueError("timestamp is outside the ten-minute grid") + fields = tuple(row[column].strip() for column in columns) + if any(field.casefold() in {"", "nan"} for field in fields): + measurement = None + else: + measurement = tuple(float(field) for field in fields) + if not all(isfinite(value) for value in measurement): + raise ValueError("non-finite measurement") + except (AttributeError, TypeError, ValueError) as exc: + raise TimeNetConnectorError(f"{member}:{reader.line_num}: invalid SCADA row") from exc + key = (station, timestamp) + if key in seen: + raise TimeNetConnectorError(f"{member}:{reader.line_num}: duplicate measurement for {key}") + seen.add(key) + if measurement is not None: + values[key] = measurement + return values + + +@dataclass(frozen=True) +class _ForecastWindow: + station: int + cutoff: datetime + history: tuple[tuple[float, ...], ...] + future_power: tuple[float, ...] + + +def _windows( + power: dict[_ScadaKey, tuple[float, ...]], telemetry: dict[_ScadaKey, tuple[float, ...]] +) -> Iterator[_ForecastWindow]: + """Yield hourly cutoffs with uninterrupted, complete inputs and future power.""" + for station, cutoff in sorted(power): + if cutoff.minute: + continue + history_keys = [(station, cutoff - step * _INTERVAL) for step in range(_CONTEXT_STEPS - 1, -1, -1)] + future_keys = [(station, cutoff + step * _INTERVAL) for step in range(1, _FORECAST_STEPS + 1)] + if all(key in power and key in telemetry for key in history_keys) and all(key in power for key in future_keys): + history = tuple(power[key] + telemetry[key] for key in history_keys) + future = tuple(power[key][0] for key in future_keys) + yield _ForecastWindow(station, cutoff, history, future) + + +def _signal(spec: TimeSeriesSpec, values: Sequence[float]) -> Signal: + """Attach the imported values as a lazily materialized regular series.""" + return Signal.from_loader( + spec=spec, + name=spec.spec_type, + time_axis=_AXIS, + loader=lambda: pa.array(values, type=pa.float64()), + n_values=len(values), + null_count=0, + ) + + +def _add_forecast(dataset: TimeFDataset, window: _ForecastWindow, turbine: _Turbine, split: Split) -> None: + """Register one history record and its separate power-only forecast target.""" + first_context_end = window.cutoff - (_CONTEXT_STEPS - 1) * _INTERVAL + first_target_end = window.cutoff + _INTERVAL + signals = tuple( + _signal(spec, values) + for spec, values in zip((_POWER, _WIND, _RPM), zip(*window.history, strict=True), strict=True) + ) + context = dataset.add_record( + record=Record( + start_time=TimeOrigin(first_context_end), + metadata={ + "turbine_id": turbine.name, + "station_id": window.station, + "rated_power_kw": turbine.rated_power_kw, + "window_start_utc": (first_context_end - _INTERVAL).isoformat(), + "forecast_cutoff_utc": window.cutoff.isoformat(), + }, + sources=(Source(name="SCADA", signals=signals),), + ), + ) + target = dataset.add_record( + record=Record( + start_time=TimeOrigin(first_target_end), + metadata={"turbine_id": turbine.name, "station_id": window.station}, + sources=(Source(name="SCADA", signals=(_signal(_POWER, window.future_power),)),), + ), + ) + dataset.add_task( + task=ForecastingTask( + inputs=(context,), + targets=(target,), + prompt="Forecast mean active power in kW for the next six ten-minute intervals.", + split=split, + ), + ) + + +class HillOfTowieSCADAConnector(BaseConnector[HillOfTowieFiles]): + """Build hourly turbine power forecasting tasks from the pinned 2026 SCADA archive.""" + + # Override: the source release and artifact paths are fixed. + # ruff: ignore[no-self-use] + async def download_async(self, cache_dir: Path) -> HillOfTowieFiles: + """Cache the two version-pinned Zenodo artifacts.""" + archive, metadata = await download_files( + [ + Artifact(f"{_FILES_URL}/{_ARCHIVE}?download=1", cache_dir / _ARCHIVE, sha256=_ARCHIVE_SHA256), + Artifact(f"{_FILES_URL}/{_METADATA}?download=1", cache_dir / _METADATA, sha256=_METADATA_SHA256), + ], + ) + return HillOfTowieFiles(archive, metadata) + + def convert(self, raw: HillOfTowieFiles, /) -> TimeFDataset: + """Join monthly SCADA tables by station and time and build chronological splits.""" + dataset = TimeFDataset(metadata=self.metadata()) + try: + turbines = _read_turbines(raw.metadata) + with ZipFile(raw.archive) as archive: + for month, split in _SPLITS.items(): + power = _read_measurements(archive, month, "tblSCTurGrid", ("wtc_ActPower_mean",)) + telemetry = _read_measurements( + archive, month, "tblSCTurbine", ("wtc_AcWindSp_mean", "wtc_GenRpm_mean") + ) + if unknown := {station for station, _ in power.keys() | telemetry.keys()}.difference(turbines): + raise TimeNetConnectorError(f"SCADA 2026-{month:02}: unknown station IDs {sorted(unknown)}") + for window in _windows(power, telemetry): + _add_forecast(dataset, window, turbines[window.station], split) + except (BadZipFile, OSError, KeyError, UnicodeError, csv.Error) as exc: + raise TimeNetConnectorError(f"Unable to read Hill of Towie source files: {exc}") from exc + if not all((dataset.get_train(), dataset.get_validation(), dataset.get_test())): + raise TimeNetConnectorError("Hill of Towie SCADA must contain complete windows in all three splits") + return dataset + + +CONNECTOR = HillOfTowieSCADAConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml new file mode 100644 index 000000000..3d143b0a7 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: res/hill-of-towie-scada +dataset_version: 1.0.0 +name: Hill of Towie SCADA +description: >- + Ten-minute SCADA means for the Hill of Towie wind farm from January through April 2026. + Forecast six active-power readings from the preceding 36 readings of active power, active + wind speed, and generator speed. Forecasts start at hourly UTC cutoffs. Each example uses one + turbine and one source month and excludes windows with missing required readings. Timestamps + label the end of each ten-minute measurement interval. January and February are for training, + March for validation, and April for testing. This subset uses the 2026 SCADA archive and static + turbine metadata from source release 2.1.0. +license: CC-BY-4.0 +access: open +source_url: https://zenodo.org/records/22662930 +citation: >- + Clerc, A., and Lingkan, E. (2026). Hill of Towie wind farm open dataset (Version 2.1.0). + Zenodo. https://doi.org/10.5281/zenodo.22662930. Released by RES on behalf of TRIG. +domains: + - energy +tags: + - wind-energy + - scada + - forecasting diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py new file mode 100644 index 000000000..af5d44209 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the Hill of Towie SCADA connector.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py new file mode 100644 index 000000000..fd2c7b935 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py @@ -0,0 +1,99 @@ +"""Test the Hill of Towie connector against a small SCADA archive.""" + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pytest + +from timenet import TimeNet +from timenet.dataset import Record +from timenet.dataset.axis import RegularAxis +from timenet.engine import store_dataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ForecastingTask, unix_us, ureg +from timenet_connectors.datasets.res.hill_of_towie_scada.connector import ( + HillOfTowieFiles, + HillOfTowieSCADAConnector, +) + + +def _source(tmp_path: Path, *, january_gap: bool = False, duplicate: bool = False) -> HillOfTowieFiles: + metadata = tmp_path / "metadata.csv" + metadata.write_text( + "\ufeffStation ID,Turbine Name,Rated power (kW)\n2304510,T01,2300\n2304511,T02,2300\n", + encoding="utf-8", + ) + path = tmp_path / "2026.zip" + with ZipFile(path, "w") as archive: + for month in range(1, 5): + origin = datetime(2026, month, 1) + power = ["TimeStamp,StationId,wtc_ActPower_mean"] + telemetry = ["TimeStamp,StationId,wtc_AcWindSp_mean,wtc_GenRpm_mean"] + for step in range(43): + stamp = (origin + timedelta(minutes=step * 10)).isoformat(sep=" ") + for station, offset in ((2304510, 0), (2304511, 100)): + value = month * 100 + step + offset + power.append(f"{stamp},{station},{value}") + if step <= 36: # Future telemetry is intentionally absent. + wind = "" if january_gap and month == 1 and station == 2304510 and step == 24 else 8 + offset + telemetry.append(f"{stamp},{station},{wind},{900 + offset}") + if duplicate and month == 1: + power.append(power[-1]) + archive.writestr(f"tblSCTurGrid_2026_{month:02}.csv", "\n".join(power)) + archive.writestr(f"tblSCTurbine_2026_{month:02}.csv", "\n".join([telemetry[0], *reversed(telemetry[1:])])) + return HillOfTowieFiles(path, metadata) + + +def test_forecasts_roundtrip_preserves_station_time_units_and_splits(tmp_path): + converted = HillOfTowieSCADAConnector().convert(_source(tmp_path)) + registry = tmp_path / "registry" + store_dataset(converted, registry) + dataset = TimeNet(sources=[registry]).load("res/hill-of-towie-scada@1.0.0") + assert [len(dataset.get_train()), len(dataset.get_validation()), len(dataset.get_test())] == [4, 2, 2] + for tasks, months in ( + (dataset.get_train(), [1, 1, 2, 2]), + (dataset.get_validation(), [3, 3]), + (dataset.get_test(), [4, 4]), + ): + for index, (task, month) in enumerate(zip(tasks, months, strict=True)): + assert isinstance(task, ForecastingTask) + context = task.inputs[0] + assert task.targets is not None + target = task.targets[0] + assert isinstance(target, Record) + turbine = index % 2 + 1 + offset = 100 * (turbine - 1) + assert context.metadata["turbine_id"] == target.metadata["turbine_id"] == f"T{turbine:02}" + assert context.metadata["window_start_utc"] == f"2026-{month:02}-01T00:00:00+00:00" + assert context.start_time.timestamp == unix_us(datetime(2026, month, 1, 0, 10, tzinfo=UTC)) + assert target.start_time.timestamp == unix_us(datetime(2026, month, 1, 6, 10, tzinfo=UTC)) + + channels = {signal.name: signal for signal in context.signals} + assert channels.keys() == {"active_power", "wind_speed", "generator_speed"} + power = channels["active_power"] + wind = channels["wind_speed"] + rpm = channels["generator_speed"] + (future_power,) = target.signals + assert power.spec.unit_value == future_power.spec.unit_value == ureg.kilowatt + assert wind.spec.unit_value == ureg.meter / ureg.second + assert rpm.spec.unit_value == ureg.rpm + for signal in (power, wind, rpm, future_power): + assert isinstance(signal.time_axis, RegularAxis) + assert signal.time_axis.period_us == 600_000_000 + np.testing.assert_array_equal(power.to_numpy(), month * 100 + offset + np.arange(1, 37)) + np.testing.assert_array_equal(wind.to_numpy(), np.full(36, 8 + offset)) + np.testing.assert_array_equal(rpm.to_numpy(), np.full(36, 900 + offset)) + np.testing.assert_array_equal(future_power.to_numpy(), month * 100 + offset + np.arange(37, 43)) + + +def test_missing_history_excludes_only_affected_windows(tmp_path): + dataset = HillOfTowieSCADAConnector().convert(_source(tmp_path, january_gap=True)) + assert len(dataset.get_train()) == 3 + assert dataset.get_train()[0].inputs[0].metadata["turbine_id"] == "T02" + + +def test_duplicate_measurement_fails(tmp_path): + with pytest.raises(TimeNetConnectorError, match="duplicate measurement"): + HillOfTowieSCADAConnector().convert(_source(tmp_path, duplicate=True)) From 263b0dcda172b52db4a9ac4da1d78481eb73dcd3 Mon Sep 17 00:00:00 2001 From: "M. Kalk" <44619887+ZMakschZ@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:25:38 +0100 Subject: [PATCH 2/2] feat(connectors): use full-history Hill of Towie forecasting release --- .../res/hill_of_towie_scada/README.md | 94 ++++++ .../res/hill_of_towie_scada/connector.py | 313 ++++++++---------- .../res/hill_of_towie_scada/dataset.yaml | 16 +- .../tests/test_connector.py | 173 ++++++---- .../res/hill_of_towie_scada/windows.py | 147 ++++++++ 5 files changed, 497 insertions(+), 246 deletions(-) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md new file mode 100644 index 000000000..6053d4eda --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md @@ -0,0 +1,94 @@ +# Hill of Towie SCADA Forecasting + +`res/hill-of-towie-scada@2.0.0` covers all 21 turbines from January 2016 through April 2026. +It uses annual ten-minute SCADA archives from Zenodo release 2.1.0. Each task predicts six measured +active-power values in kW from the previous 36 ten-minute readings of active power, active wind +speed, and generator RPM. [Source release and CC BY 4.0 attribution](https://zenodo.org/records/22662930) + +## Dataset split + +All boundaries are UTC, with an inclusive start and exclusive end for physical measurement +intervals. Source timestamps label the **end** of each ten-minute interval, so a reading stamped +at a split boundary belongs to the preceding interval. An hourly task is eligible when its entire +six-hour input and one-hour target fit in one calendar month and one row of the table below. The +same boundaries apply to every turbine. An input needs all three channels, and a target needs +only power. Source gaps and blanks remain nulls in the shared records. + +| TimeF split | `evaluation_cohort` | Start UTC | End UTC | Windows | Rationale | +| --- | --- | --- | --- | ---: | --- | +| Train | `historical_train` | 2016-01-01 00:00 | 2024-10-01 00:00 | 1,564,202 | Retains several seasons, the AeroUp rollout, and TuneUp. Ends after the documented late-summer 2024 curtailment period. | +| Validation | `post_tuneup_validation` | 2024-10-01 00:00 | 2025-01-01 00:00 | 45,511 | Tunes on later data following the TuneUp campaign, ahead of the 2025 holdout. | +| Test | `pre_commissioning` | 2025-01-01 00:00 | 2025-10-28 13:00 | 141,619 | Main score for forecasting before documented Dynamic Yaw commissioning. | +| Test | `dynamic_yaw_commissioning` | 2025-10-28 13:00 | 2026-01-07 13:00 | 34,857 | Covers commissioning, tests, and changes to turbine roles. | +| Test | `dynamic_yaw_west_trial` | 2026-01-07 13:00 | 2026-02-03 18:00 | 13,434 | Starts at the formal west trial boundary in the RES configuration. | +| Test | `dynamic_yaw_west_and_east_trial` | 2026-02-03 18:00 | 2026-05-01 00:00 | 42,614 | Starts at the formal east cohort boundary and ends with the archive. | + +Use the main test cohort for the headline score and report the three later cohorts separately. +September 2025 lost 44.34% of expected turbine intervals, including a 12.73-day gap across every +turbine. Report a main-test sensitivity score with September omitted to show how that gap affects +coverage. The Dynamic Yaw schedule alternates 50 minutes on and 50 minutes off. The cohort labels +describe dates. Per-turbine exposure, controller state and wake effects require the RES role +definitions and controller telemetry. [Trial configuration](https://github.com/resgroup/hill-of-towie-open-source-analysis/blob/main/scripts/wfc_analysis_2026/wind_up_config/HOT_dynamic_yaw.yaml) +· [Commissioning and curtailment dates](https://github.com/resgroup/hill-of-towie-open-source-analysis/blob/main/scripts/wfc_analysis_2026/wind_up_config/exclusion_periods/HOT_DY.yaml) + +## Measurement and operating changes + +| When | What changed | +| --- | --- | +| August 2016 | The turbine SCADA schema removed the low-speed power reduction fields. | +| July 2017 | The schema added boost availability. | +| August 1, 2017 and April 10–11, 2018 | The secondary anemometer correction started on T14 and T17 first. Most other turbines followed on April 10, 2018, with T01 and T06 on April 11. The historical audit found a median +11.91 percentage-point step in raw secondary wind speed relative to primary. The active-to-primary ratio was unchanged to two decimal places. | +| September 23–29, 2021 | T13 underwent the first AeroUp retrofit. | +| April 2022 | The schema added actual wind direction, boost activation, and turbine operating state. The three forecast channels span the earlier and later schemas. | +| July 14, 2022–May 26, 2023 | RES rolled out AeroUp to the rest of the fleet. Use the per-turbine first and last work dates to distinguish pre-upgrade, installation, and post-upgrade windows. | +| March 14–August 31, 2024 | TuneUp began on T02, T03, T06, T08, T09, T13, T16, and T20 on March 14. T21 joined on May 2. RES also flags heavy curtailment from August 9 through September 28. | +| October 28, 2025 onward | Dynamic Yaw commissioning began. RES lists turbine role changes and controller configuration tests in October, November, and December. | +| January 7 and February 3, 2026 | The formal west and east trial boundaries were at 13:00 and 18:00 UTC, respectively. Dynamic Yaw changes the relationship between inflow, yaw, and measured power. | + +RES records farm-wide pre-campaign tests from October 28 at 13:00 to November 17 at 13:45 UTC. +It records a deliberate north-offset test on T15 and T18 on November 19–20, a change in T06's +role on December 2, and changes in T19's and T21's roles on December 11. These dates are why the +main test stops in October 2025. [RES commissioning log](https://github.com/resgroup/hill-of-towie-open-source-analysis/blob/main/scripts/wfc_analysis_2026/wind_up_config/exclusion_periods/HOT_DY.yaml) + +The dataset targets actual delivered power, including valid readings during curtailment and +changes in operation. The connector uses the active wind-speed channel throughout. Use the same +validation period to compare full-history training with a June 2023 onward training run, after +the AeroUp rollout. [SCADA data dictionary](https://zenodo.org/records/22662930/preview/Hill_of_Towie_data_dictionary.md?include_deleted=0) +· [AeroUp dates by turbine](https://zenodo.org/records/22662930/files/Hill_of_Towie_AeroUp_install_dates.csv?download=1) +· [RES campaign summary](https://zenodo.org/records/22662930) + +Zenodo release 2.0.0 extended SCADA from August 2024 to April 2026 and added separate fastlog, +LiDAR, and controller telemetry. Release 2.1.0 updated the documentation. The forecasting +records here use the ten-minute SCADA tables. [Source changelog](https://zenodo.org/records/22662930/preview/CHANGELOG.md?include_deleted=0) + +## Build and read the forecast windows + +From the TimeNet checkout, run `make sync`, then build into a local registry: + +```bash +uv run timenet-build build res/hill-of-towie-scada@2.0.0 --no-isolation +``` + +The source download is approximately 15 GB. The connector reads two CSV tables per month directly +from the pinned archives. In the resulting TimeF release, a record holds one turbine and one +month segment. Each forecast carries an input `scope` for the 36 historical readings and a +target interval for the six observed power values. Apply the input scope before passing values to +a model. Reading a whole shared record would include readings after that task's cutoff. + +```python +from timenet import TimeNet +from timenet.types import TimeInterval + +with TimeNet().open("res/hill-of-towie-scada@2.0.0") as reader: + task = next(reader.iter_tasks(split="train", batch_size=1)) + assert isinstance(task.scope, TimeInterval) + assert task.targets is not None + (target,) = task.targets + assert isinstance(target, TimeInterval) and target.time_series + (power,) = target.time_series + context = { + signal.name: signal.slice(task.scope).to_numpy() + for signal in task.inputs[0].signals + } + observed_power_kw = power.slice(target).to_numpy() +``` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py index f0a8e9856..e4be8c70d 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py @@ -1,228 +1,199 @@ -"""Convert Hill of Towie ten-minute SCADA into turbine power forecasts.""" +"""Convert the Hill of Towie SCADA release into TimeF power forecasting tasks.""" -from collections.abc import Iterator, Sequence +from collections.abc import Iterator import csv from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from fractions import Fraction -from io import TextIOWrapper -from math import isfinite +from datetime import UTC, datetime +from functools import partial from pathlib import Path from zipfile import BadZipFile, ZipFile +import numpy as np +from numpy.typing import NDArray import pyarrow as pa +import pyarrow.csv as arrow_csv from timenet.connectors import BaseConnector -from timenet.dataset import Record, Signal, Source, TimeFDataset -from timenet.dataset.axis import RegularAxis +from timenet.dataset import TimeFDataset from timenet.errors import TimeNetConnectorError -from timenet.types import ForecastingTask, Split, TimeOrigin, TimeSeriesSpec, ureg +from timenet.types import ForecastingTask, Split, unix_us +from timenet_connectors.datasets.res.hill_of_towie_scada.windows import ( + LAST_MONTH_END, + STEP, + STEP_US, + Turbine, + WindowSeries, + forecasting_tasks, + month_periods, + prepare_windows, +) from timenet_connectors.download import Artifact, download_files _FILES_URL = "https://zenodo.org/records/22662930/files" -_ARCHIVE = "2026.zip" -_ARCHIVE_SHA256 = "4bcfd706769369eacde59ea4f3a45bc3f9c0fb334c67fea75fcf3aa0c424eaea" _METADATA = "Hill_of_Towie_turbine_metadata.csv" _METADATA_SHA256 = "dd40789c2bf2268da9d77890cc48617266f3ed9d13c5af2be6d6535671f261bc" - -_SPLITS = {1: Split.TRAIN, 2: Split.TRAIN, 3: Split.VALIDATION, 4: Split.TEST} -_INTERVAL = timedelta(minutes=10) -_AXIS = RegularAxis.from_rate_hz(Fraction(1, 600)) -_CONTEXT_STEPS = 36 -_FORECAST_STEPS = 6 -_POWER = TimeSeriesSpec(spec_type="active_power", name="Active power", unit_value=ureg.kilowatt, dtype="float64") -_WIND = TimeSeriesSpec( - spec_type="wind_speed", name="Active wind speed", unit_value=ureg.meter / ureg.second, dtype="float64" -) -_RPM = TimeSeriesSpec(spec_type="generator_speed", name="Generator speed", unit_value=ureg.rpm, dtype="float64") - -_ScadaKey = tuple[int, datetime] +_ARCHIVE_SHA256 = { + 2016: "800bb3d399e6b79278758bbd95e234e364b062a9ff7643a7f9c045b0db9e9738", + 2017: "a49c11def85623b7f0537988debfa499cca15fc6ec971256051597d91224ab1f", + 2018: "4d59191bd12ad0e85eb4de44f2f8f7d8e2556452341e89cd194a816debc48761", + 2019: "24a9b9bfdf32558d80c518a11954d7230254f5ba384671d3df6ba3e53deb5b26", + 2020: "3e55de08be27e6a6e702a99bfcccd57e20abfa69e7c65d5901caf581f128f020", + 2021: "2284741792b4bfc7ca08234f49440dad4820bac5e73098011745a3f69e295c19", + 2022: "1b0f8657273ddbb8c4f84042a0bcb1c5eb1fb7833849b4c491f2b9f10604700d", + 2023: "e65f7acd65664584e5b72e4ad7ccd4c0f0af4ad68f618b8cf0b4e5c1f4a69521", + 2024: "b6bb540e3d2fcc2cff8756d96ef4155037b1cc41b284e6249b41054f4e50fbc3", + 2025: "cd439de7bf8e297dd8422d5c93eda04785c56e3861be8d2a79f403ac39b873ee", + 2026: "4bcfd706769369eacde59ea4f3a45bc3f9c0fb334c67fea75fcf3aa0c424eaea", +} +_POWER = ("wtc_ActPower_mean",) +_TELEMETRY = ("wtc_AcWindSp_mean", "wtc_GenRpm_mean") @dataclass(frozen=True) class HillOfTowieFiles: - """Local source archive and turbine metadata.""" + """Local annual SCADA archives and their turbine lookup.""" - archive: Path + archives: tuple[Path, ...] metadata: Path -@dataclass(frozen=True) -class _Turbine: - name: str - rated_power_kw: int - - -def _read_turbines(path: Path) -> dict[int, _Turbine]: - """Read station IDs and static turbine metadata from the published lookup.""" - turbines: dict[int, _Turbine] = {} +def _read_turbines(path: Path) -> tuple[Turbine, ...]: + """Read unique station identities and rated capacities.""" + turbines: dict[int, Turbine] = {} with path.open(encoding="utf-8-sig", newline="") as stream: reader = csv.DictReader(stream) for row in reader: try: - station = int(row["Station ID"]) - turbine = _Turbine(row["Turbine Name"], int(row["Rated power (kW)"])) + turbine = Turbine(int(row["Station ID"]), row["Turbine Name"], int(row["Rated power (kW)"])) except (KeyError, TypeError, ValueError) as exc: raise TimeNetConnectorError(f"{path.name}:{reader.line_num}: invalid turbine metadata") from exc - if station in turbines or not turbine.name or turbine.rated_power_kw <= 0: - raise TimeNetConnectorError(f"{path.name}:{reader.line_num}: invalid or duplicate station {station}") - turbines[station] = turbine + if turbine.station in turbines or not turbine.name or turbine.rated_power_kw <= 0: + raise TimeNetConnectorError( + f"{path.name}:{reader.line_num}: invalid or duplicate station {turbine.station}" + ) + turbines[turbine.station] = turbine if not turbines: raise TimeNetConnectorError(f"{path.name}: no turbine metadata") - return turbines - - -def _read_measurements( - archive: ZipFile, month: int, table: str, columns: tuple[str, ...] -) -> dict[_ScadaKey, tuple[float, ...]]: - """Read complete measurements for the month owning each UTC sampling interval.""" - member = f"{table}_2026_{month:02}.csv" - values: dict[_ScadaKey, tuple[float, ...]] = {} - seen: set[_ScadaKey] = set() - with archive.open(member) as raw, TextIOWrapper(raw, encoding="utf-8-sig", newline="") as stream: - reader = csv.DictReader(stream) - required = {"StationId", "TimeStamp", *columns} - if missing := required.difference(reader.fieldnames or ()): - raise TimeNetConnectorError(f"{member}: missing columns {sorted(missing)}") - for row in reader: - try: - station = int(row["StationId"]) - timestamp = datetime.fromisoformat(row["TimeStamp"]) - if timestamp.tzinfo is None: - timestamp = timestamp.replace(tzinfo=UTC) - timestamp = timestamp.astimezone(UTC) - # The publisher assigns a boundary interval to the month in which it starts. - interval_start = timestamp - _INTERVAL - if (interval_start.year, interval_start.month) != (2026, month): - continue - if timestamp.minute % 10 or timestamp.second or timestamp.microsecond: - raise ValueError("timestamp is outside the ten-minute grid") - fields = tuple(row[column].strip() for column in columns) - if any(field.casefold() in {"", "nan"} for field in fields): - measurement = None - else: - measurement = tuple(float(field) for field in fields) - if not all(isfinite(value) for value in measurement): - raise ValueError("non-finite measurement") - except (AttributeError, TypeError, ValueError) as exc: - raise TimeNetConnectorError(f"{member}:{reader.line_num}: invalid SCADA row") from exc - key = (station, timestamp) - if key in seen: - raise TimeNetConnectorError(f"{member}:{reader.line_num}: duplicate measurement for {key}") - seen.add(key) - if measurement is not None: - values[key] = measurement - return values + return tuple(turbines[key] for key in sorted(turbines)) @dataclass(frozen=True) -class _ForecastWindow: - station: int - cutoff: datetime - history: tuple[tuple[float, ...], ...] - future_power: tuple[float, ...] - - -def _windows( - power: dict[_ScadaKey, tuple[float, ...]], telemetry: dict[_ScadaKey, tuple[float, ...]] -) -> Iterator[_ForecastWindow]: - """Yield hourly cutoffs with uninterrupted, complete inputs and future power.""" - for station, cutoff in sorted(power): - if cutoff.minute: - continue - history_keys = [(station, cutoff - step * _INTERVAL) for step in range(_CONTEXT_STEPS - 1, -1, -1)] - future_keys = [(station, cutoff + step * _INTERVAL) for step in range(1, _FORECAST_STEPS + 1)] - if all(key in power and key in telemetry for key in history_keys) and all(key in power for key in future_keys): - history = tuple(power[key] + telemetry[key] for key in history_keys) - future = tuple(power[key][0] for key in future_keys) - yield _ForecastWindow(station, cutoff, history, future) - - -def _signal(spec: TimeSeriesSpec, values: Sequence[float]) -> Signal: - """Attach the imported values as a lazily materialized regular series.""" - return Signal.from_loader( - spec=spec, - name=spec.spec_type, - time_axis=_AXIS, - loader=lambda: pa.array(values, type=pa.float64()), - n_values=len(values), - null_count=0, +class _Month: + start: datetime + end: datetime + + @property + def steps(self) -> int: + return (self.end - self.start) // STEP + + +def _months(year: int) -> Iterator[_Month]: + """Yield UTC months in an archive, respecting the release end in April 2026.""" + for month in range(1, 13): + start = datetime(year, month, 1, tzinfo=UTC) + if start >= LAST_MONTH_END: + return + end = datetime(year + month // 12, month % 12 + 1, 1, tzinfo=UTC) + yield _Month(start, end) + + +def _read_table( + archive: ZipFile, + month: _Month, + table: str, + fields: tuple[str, ...], + stations: NDArray[np.int64], +) -> NDArray[np.float64]: + """Load selected columns into a station-by-time array with source blanks preserved.""" + member = f"{table}_{month.start:%Y_%m}.csv" + options = arrow_csv.ConvertOptions( + include_columns=["TimeStamp", "StationId", *fields], + column_types={"TimeStamp": pa.timestamp("us"), "StationId": pa.int64(), **dict.fromkeys(fields, pa.float64())}, ) + try: + with archive.open(member) as raw: + data = arrow_csv.read_csv(raw, convert_options=options) + except (KeyError, pa.ArrowInvalid) as exc: + raise TimeNetConnectorError(f"{archive.filename}: unable to read {member}: {exc}") from exc + + timestamps = data["TimeStamp"] + station_ids = data["StationId"] + if timestamps.null_count or station_ids.null_count: + raise TimeNetConnectorError(f"{member}: missing timestamp or station ID") + relative_us = timestamps.to_numpy().astype(np.int64) - unix_us(month.start) + # Source labels interval ends. A row at midnight belongs to the previous interval's month. + owned = (relative_us > 0) & (relative_us <= month.steps * STEP_US) + owned_us = relative_us[owned] + if np.any(owned_us % STEP_US): + raise TimeNetConnectorError(f"{member}: timestamp is outside the ten-minute grid") + ids = station_ids.to_numpy()[owned] + if unknown := np.setdiff1d(ids, stations).tolist(): + raise TimeNetConnectorError(f"{member}: unknown station IDs {unknown}") + rows = np.searchsorted(stations, ids) + steps = owned_us // STEP_US - 1 + if np.unique(rows * month.steps + steps).size != len(steps): + raise TimeNetConnectorError(f"{member}: duplicate measurement") + values = np.full((len(fields), stations.size, month.steps), np.nan) + for field, channel in zip(fields, values, strict=True): + measurements = data[field].to_numpy()[owned] + if np.any(np.isinf(measurements)): + raise TimeNetConnectorError(f"{member}: infinite measurement in {field}") + channel[rows, steps] = measurements + return values -def _add_forecast(dataset: TimeFDataset, window: _ForecastWindow, turbine: _Turbine, split: Split) -> None: - """Register one history record and its separate power-only forecast target.""" - first_context_end = window.cutoff - (_CONTEXT_STEPS - 1) * _INTERVAL - first_target_end = window.cutoff + _INTERVAL - signals = tuple( - _signal(spec, values) - for spec, values in zip((_POWER, _WIND, _RPM), zip(*window.history, strict=True), strict=True) - ) - context = dataset.add_record( - record=Record( - start_time=TimeOrigin(first_context_end), - metadata={ - "turbine_id": turbine.name, - "station_id": window.station, - "rated_power_kw": turbine.rated_power_kw, - "window_start_utc": (first_context_end - _INTERVAL).isoformat(), - "forecast_cutoff_utc": window.cutoff.isoformat(), - }, - sources=(Source(name="SCADA", signals=signals),), - ), - ) - target = dataset.add_record( - record=Record( - start_time=TimeOrigin(first_target_end), - metadata={"turbine_id": turbine.name, "station_id": window.station}, - sources=(Source(name="SCADA", signals=(_signal(_POWER, window.future_power),)),), - ), - ) - dataset.add_task( - task=ForecastingTask( - inputs=(context,), - targets=(target,), - prompt="Forecast mean active power in kW for the next six ten-minute intervals.", - split=split, - ), - ) +def _month_series( + archive: ZipFile, month: _Month, turbines: tuple[Turbine, ...], stations: NDArray[np.int64] +) -> Iterator[WindowSeries]: + """Prepare the turbine records and eligible cutoffs for each part of a month.""" + power = _read_table(archive, month, "tblSCTurGrid", _POWER, stations) + telemetry = _read_table(archive, month, "tblSCTurbine", _TELEMETRY, stations) + values = np.concatenate((power, telemetry)) + for period in month_periods(month.start, month.end): + start = (period.start - month.start) // STEP + stop = (period.end - month.start) // STEP + for index, turbine in enumerate(turbines): + yield prepare_windows(values[:, index, start:stop], turbine, period) class HillOfTowieSCADAConnector(BaseConnector[HillOfTowieFiles]): - """Build hourly turbine power forecasting tasks from the pinned 2026 SCADA archive.""" + """Build power forecasting tasks from the pinned 2016 to 2026 annual SCADA archives.""" - # Override: the source release and artifact paths are fixed. + # Override: the pinned source release has a fixed set of files. # ruff: ignore[no-self-use] async def download_async(self, cache_dir: Path) -> HillOfTowieFiles: - """Cache the two version-pinned Zenodo artifacts.""" - archive, metadata = await download_files( + """Cache the Zenodo archives and turbine lookup.""" + pins = [(f"{year}.zip", digest) for year, digest in _ARCHIVE_SHA256.items()] + files = await download_files( [ - Artifact(f"{_FILES_URL}/{_ARCHIVE}?download=1", cache_dir / _ARCHIVE, sha256=_ARCHIVE_SHA256), - Artifact(f"{_FILES_URL}/{_METADATA}?download=1", cache_dir / _METADATA, sha256=_METADATA_SHA256), + Artifact(f"{_FILES_URL}/{name}?download=1", cache_dir / name, sha256=digest) + for name, digest in [*pins, (_METADATA, _METADATA_SHA256)] ], + max_concurrency=5, ) - return HillOfTowieFiles(archive, metadata) + return HillOfTowieFiles(tuple(files[:-1]), files[-1]) def convert(self, raw: HillOfTowieFiles, /) -> TimeFDataset: - """Join monthly SCADA tables by station and time and build chronological splits.""" + """Register the SCADA records and stream tasks over qualifying windows.""" + if len(raw.archives) != len(_ARCHIVE_SHA256): + raise TimeNetConnectorError("Hill of Towie requires all 11 annual archives and turbine metadata") dataset = TimeFDataset(metadata=self.metadata()) + series: list[WindowSeries] = [] try: turbines = _read_turbines(raw.metadata) - with ZipFile(raw.archive) as archive: - for month, split in _SPLITS.items(): - power = _read_measurements(archive, month, "tblSCTurGrid", ("wtc_ActPower_mean",)) - telemetry = _read_measurements( - archive, month, "tblSCTurbine", ("wtc_AcWindSp_mean", "wtc_GenRpm_mean") - ) - if unknown := {station for station, _ in power.keys() | telemetry.keys()}.difference(turbines): - raise TimeNetConnectorError(f"SCADA 2026-{month:02}: unknown station IDs {sorted(unknown)}") - for window in _windows(power, telemetry): - _add_forecast(dataset, window, turbines[window.station], split) - except (BadZipFile, OSError, KeyError, UnicodeError, csv.Error) as exc: + stations = np.array([turbine.station for turbine in turbines], dtype=np.int64) + for year, path in zip(_ARCHIVE_SHA256, raw.archives, strict=True): + with ZipFile(path) as archive: + for month in _months(year): + for entry in _month_series(archive, month, turbines, stations): + dataset.add_record(record=entry.record) + series.append(entry) + except (BadZipFile, OSError, UnicodeError, csv.Error) as exc: raise TimeNetConnectorError(f"Unable to read Hill of Towie source files: {exc}") from exc - if not all((dataset.get_train(), dataset.get_validation(), dataset.get_test())): + if {entry.period.split for entry in series if entry.cutoffs} != set(Split): raise TimeNetConnectorError("Hill of Towie SCADA must contain complete windows in all three splits") + dataset.set_task_stream([ForecastingTask], partial(forecasting_tasks, series)) return dataset diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml index 3d143b0a7..d65908e30 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml @@ -1,16 +1,16 @@ # yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json yaml_schema_version: 1 dataset_id: res/hill-of-towie-scada -dataset_version: 1.0.0 +dataset_version: 2.0.0 name: Hill of Towie SCADA description: >- - Ten-minute SCADA means for the Hill of Towie wind farm from January through April 2026. - Forecast six active-power readings from the preceding 36 readings of active power, active - wind speed, and generator speed. Forecasts start at hourly UTC cutoffs. Each example uses one - turbine and one source month and excludes windows with missing required readings. Timestamps - label the end of each ten-minute measurement interval. January and February are for training, - March for validation, and April for testing. This subset uses the 2026 SCADA archive and static - turbine metadata from source release 2.1.0. + Ten-minute SCADA for 21 Hill of Towie wind turbines from January 2016 through April 2026. + Forecast six observed active-power readings from the preceding 36 readings of active power, + wind speed, and generator speed. Task scopes select history on shared turbine records, and + target spans select only future power. Hourly UTC cutoffs stay within a calendar month and + an evaluation cohort, and every required reading must be available. Training ends September + 2024 and validation covers October through December 2024. The 2025 to April 2026 test split + distinguishes pre-commissioning, commissioning, and the two Dynamic Yaw trial periods. license: CC-BY-4.0 access: open source_url: https://zenodo.org/records/22662930 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py index fd2c7b935..b94169acf 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py @@ -1,5 +1,6 @@ -"""Test the Hill of Towie connector against a small SCADA archive.""" +"""Round-trip tests for Hill of Towie's SCADA forecasts.""" +from collections import Counter from datetime import UTC, datetime, timedelta from pathlib import Path from zipfile import ZipFile @@ -8,15 +9,36 @@ import pytest from timenet import TimeNet -from timenet.dataset import Record from timenet.dataset.axis import RegularAxis from timenet.engine import store_dataset from timenet.errors import TimeNetConnectorError -from timenet.types import ForecastingTask, unix_us, ureg -from timenet_connectors.datasets.res.hill_of_towie_scada.connector import ( - HillOfTowieFiles, - HillOfTowieSCADAConnector, -) +from timenet.types import ForecastingTask, TimeInterval, unix_us, ureg +from timenet_connectors.datasets.res.hill_of_towie_scada.connector import HillOfTowieFiles, HillOfTowieSCADAConnector + + +_CAMPAIGN_BOUNDARIES = { + (2025, 10): datetime(2025, 10, 28, 13), + (2026, 1): datetime(2026, 1, 7, 13), + (2026, 2): datetime(2026, 2, 3, 18), +} + + +def _month_contents(first: datetime, january_gap: bool) -> tuple[list[str], list[str]]: + power = ["TimeStamp,StationId,wtc_ActPower_mean"] + telemetry = ["TimeStamp,StationId,wtc_AcWindSp_mean,wtc_GenRpm_mean"] + starts = [first] + if boundary := _CAMPAIGN_BOUNDARIES.get((first.year, first.month)): + starts.extend((boundary - timedelta(hours=7), boundary)) + for start in starts: + for step in range(0 if start == first else 1, 43): + stamp = (start + timedelta(minutes=step * 10)).isoformat(sep=" ") + for station, offset in ((2304510, 0), (2304511, 100)): + power.append(f"{stamp},{station},{first.month * 100 + step + offset}") + if step <= 36 or start != first: + gap = january_gap and station == 2304510 and step == 24 + wind = "" if gap else 8 + offset + telemetry.append(f"{stamp},{station},{wind},{900 + offset}") + return power, telemetry def _source(tmp_path: Path, *, january_gap: bool = False, duplicate: bool = False) -> HillOfTowieFiles: @@ -25,73 +47,90 @@ def _source(tmp_path: Path, *, january_gap: bool = False, duplicate: bool = Fals "\ufeffStation ID,Turbine Name,Rated power (kW)\n2304510,T01,2300\n2304511,T02,2300\n", encoding="utf-8", ) - path = tmp_path / "2026.zip" - with ZipFile(path, "w") as archive: - for month in range(1, 5): - origin = datetime(2026, month, 1) - power = ["TimeStamp,StationId,wtc_ActPower_mean"] - telemetry = ["TimeStamp,StationId,wtc_AcWindSp_mean,wtc_GenRpm_mean"] - for step in range(43): - stamp = (origin + timedelta(minutes=step * 10)).isoformat(sep=" ") - for station, offset in ((2304510, 0), (2304511, 100)): - value = month * 100 + step + offset - power.append(f"{stamp},{station},{value}") - if step <= 36: # Future telemetry is intentionally absent. - wind = "" if january_gap and month == 1 and station == 2304510 and step == 24 else 8 + offset - telemetry.append(f"{stamp},{station},{wind},{900 + offset}") - if duplicate and month == 1: - power.append(power[-1]) - archive.writestr(f"tblSCTurGrid_2026_{month:02}.csv", "\n".join(power)) - archive.writestr(f"tblSCTurbine_2026_{month:02}.csv", "\n".join([telemetry[0], *reversed(telemetry[1:])])) - return HillOfTowieFiles(path, metadata) - - -def test_forecasts_roundtrip_preserves_station_time_units_and_splits(tmp_path): + paths = [] + for year in range(2016, 2027): + path = tmp_path / f"{year}.zip" + paths.append(path) + with ZipFile(path, "w") as archive: + for month in range(1, 5 if year == 2026 else 13): + power, telemetry = _month_contents(datetime(year, month, 1), january_gap and (year, month) == (2016, 1)) + if duplicate and (year, month) == (2016, 1): + power.append(power[-1]) + archive.writestr(f"tblSCTurGrid_{year}_{month:02}.csv", "\n".join(power)) + archive.writestr( + f"tblSCTurbine_{year}_{month:02}.csv", "\n".join([telemetry[0], *reversed(telemetry[1:])]) + ) + return HillOfTowieFiles(tuple(paths), metadata) + + +def test_forecasts_roundtrip_preserves_observations_and_units(tmp_path): converted = HillOfTowieSCADAConnector().convert(_source(tmp_path)) + assert converted.has_task_stream registry = tmp_path / "registry" store_dataset(converted, registry) - dataset = TimeNet(sources=[registry]).load("res/hill-of-towie-scada@1.0.0") - assert [len(dataset.get_train()), len(dataset.get_validation()), len(dataset.get_test())] == [4, 2, 2] - for tasks, months in ( - (dataset.get_train(), [1, 1, 2, 2]), - (dataset.get_validation(), [3, 3]), - (dataset.get_test(), [4, 4]), - ): - for index, (task, month) in enumerate(zip(tasks, months, strict=True)): - assert isinstance(task, ForecastingTask) - context = task.inputs[0] - assert task.targets is not None - target = task.targets[0] - assert isinstance(target, Record) - turbine = index % 2 + 1 - offset = 100 * (turbine - 1) - assert context.metadata["turbine_id"] == target.metadata["turbine_id"] == f"T{turbine:02}" - assert context.metadata["window_start_utc"] == f"2026-{month:02}-01T00:00:00+00:00" - assert context.start_time.timestamp == unix_us(datetime(2026, month, 1, 0, 10, tzinfo=UTC)) - assert target.start_time.timestamp == unix_us(datetime(2026, month, 1, 6, 10, tzinfo=UTC)) - - channels = {signal.name: signal for signal in context.signals} - assert channels.keys() == {"active_power", "wind_speed", "generator_speed"} - power = channels["active_power"] - wind = channels["wind_speed"] - rpm = channels["generator_speed"] - (future_power,) = target.signals - assert power.spec.unit_value == future_power.spec.unit_value == ureg.kilowatt - assert wind.spec.unit_value == ureg.meter / ureg.second - assert rpm.spec.unit_value == ureg.rpm - for signal in (power, wind, rpm, future_power): - assert isinstance(signal.time_axis, RegularAxis) - assert signal.time_axis.period_us == 600_000_000 - np.testing.assert_array_equal(power.to_numpy(), month * 100 + offset + np.arange(1, 37)) - np.testing.assert_array_equal(wind.to_numpy(), np.full(36, 8 + offset)) - np.testing.assert_array_equal(rpm.to_numpy(), np.full(36, 900 + offset)) - np.testing.assert_array_equal(future_power.to_numpy(), month * 100 + offset + np.arange(37, 43)) + with TimeNet(sources=[registry]).open("res/hill-of-towie-scada@2.0.0") as reader: + task = next(reader.iter_tasks(split="train", batch_size=1)) + assert isinstance(task, ForecastingTask) + assert isinstance(task.scope, TimeInterval) + (record,) = task.inputs + assert task.targets is not None + (target,) = task.targets + assert isinstance(target, TimeInterval) + assert target.time_series is not None + (target_signal,) = target.time_series + assert record.metadata["turbine_id"] == "T01" + assert task.metadata["window_start_utc"] == "2016-01-01T00:00:00+00:00" + assert task.metadata["forecast_cutoff_utc"] == "2016-01-01T06:00:00+00:00" + assert record.start_time.timestamp == unix_us(datetime(2016, 1, 1, 0, 10, tzinfo=UTC)) + channels = {signal.name: signal for signal in record.signals} + assert channels.keys() == {"active_power", "wind_speed", "generator_speed"} + power, wind, rpm = (channels[name] for name in ("active_power", "wind_speed", "generator_speed")) + assert target_signal is power + assert power.spec.unit_value == ureg.kilowatt + assert wind.spec.unit_value == ureg.meter / ureg.second + assert rpm.spec.unit_value == ureg.rpm + for signal in (power, wind, rpm): + assert isinstance(signal.time_axis, RegularAxis) + assert signal.time_axis.period_us == 600_000_000 + assert signal.step_range(task.scope) == (0, 36) + assert power.step_range(target) == (36, 42) + np.testing.assert_array_equal(power.slice(task.scope).to_numpy(), 100 + np.arange(1, 37)) + np.testing.assert_array_equal(wind.slice(task.scope).to_numpy(), np.full(36, 8)) + np.testing.assert_array_equal(rpm.slice(task.scope).to_numpy(), np.full(36, 900)) + np.testing.assert_array_equal(power.slice(target).to_numpy(), 100 + np.arange(37, 43)) + assert wind.read_steps(36, 42).null_count == 6 + + +def test_chronological_split_and_campaign_boundaries(tmp_path): + dataset = HillOfTowieSCADAConnector().convert(_source(tmp_path)) + tasks = list(dataset.iter_tasks()) + assert Counter(task.split for task in tasks) == {"train": 210, "validation": 6, "test": 44} + by_cohort = Counter(task.metadata["evaluation_cohort"] for task in tasks) + assert by_cohort == { + "historical_train": 210, + "post_tuneup_validation": 6, + "pre_commissioning": 22, + "dynamic_yaw_commissioning": 10, + "dynamic_yaw_west_trial": 6, + "dynamic_yaw_west_and_east_trial": 6, + } + all_cutoffs = set() + for task in tasks: + cutoff = task.metadata["forecast_cutoff_utc"] + assert isinstance(cutoff, str) + all_cutoffs.add(datetime.fromisoformat(cutoff)) + for boundary in _CAMPAIGN_BOUNDARIES.values(): + limit = boundary.replace(tzinfo=UTC) + assert limit - timedelta(hours=1) in all_cutoffs + assert limit + timedelta(hours=6) in all_cutoffs + assert not any(limit + timedelta(hours=hours) in all_cutoffs for hours in range(6)) def test_missing_history_excludes_only_affected_windows(tmp_path): dataset = HillOfTowieSCADAConnector().convert(_source(tmp_path, january_gap=True)) - assert len(dataset.get_train()) == 3 - assert dataset.get_train()[0].inputs[0].metadata["turbine_id"] == "T02" + tasks = list(dataset.iter_tasks(split="train")) + assert len(tasks) == 209 + assert tasks[0].inputs[0].metadata["turbine_id"] == "T02" def test_duplicate_measurement_fails(tmp_path): diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py new file mode 100644 index 000000000..5029aaaba --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py @@ -0,0 +1,147 @@ +"""Chronological partitions and scoped forecast windows for Hill of Towie SCADA.""" + +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from fractions import Fraction + +import numpy as np +from numpy.typing import NDArray +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source +from timenet.dataset.axis import RegularAxis +from timenet.types import ( + ForecastingTask, + MissingReason, + MissingValues, + Split, + TimeInterval, + TimeOrigin, + TimeSeriesSpec, + ureg, +) + + +STEP = timedelta(minutes=10) +STEP_US = 600_000_000 +LAST_MONTH_END = datetime(2026, 5, 1, tzinfo=UTC) +_CONTEXT = 36 +_HORIZON = 6 +_AXIS = RegularAxis.from_rate_hz(Fraction(1, 600)) +_MISSING = MissingValues(reason=MissingReason.NOT_RECORDED, description="No SCADA value for this turbine and interval.") +_POWER = TimeSeriesSpec( + spec_type="active_power", name="Active power", unit_value=ureg.kilowatt, dtype="float64", missing=_MISSING +) +_WIND = TimeSeriesSpec( + spec_type="wind_speed", + name="Active wind speed", + unit_value=ureg.meter / ureg.second, + dtype="float64", + missing=_MISSING, +) +_RPM = TimeSeriesSpec( + spec_type="generator_speed", name="Generator speed", unit_value=ureg.rpm, dtype="float64", missing=_MISSING +) +_SPECS = (_POWER, _WIND, _RPM) + + +@dataclass(frozen=True) +class Period: + """One contiguous split or evaluation cohort, with an exclusive end.""" + + start: datetime + end: datetime + split: Split + cohort: str + + +_PERIOD_STARTS = ( + (datetime(2016, 1, 1, tzinfo=UTC), Split.TRAIN, "historical_train"), + (datetime(2024, 10, 1, tzinfo=UTC), Split.VALIDATION, "post_tuneup_validation"), + (datetime(2025, 1, 1, tzinfo=UTC), Split.TEST, "pre_commissioning"), + (datetime(2025, 10, 28, 13, tzinfo=UTC), Split.TEST, "dynamic_yaw_commissioning"), + (datetime(2026, 1, 7, 13, tzinfo=UTC), Split.TEST, "dynamic_yaw_west_trial"), + (datetime(2026, 2, 3, 18, tzinfo=UTC), Split.TEST, "dynamic_yaw_west_and_east_trial"), +) +_PERIODS = tuple( + Period(start, end, split, cohort) + for (start, split, cohort), end in zip( + _PERIOD_STARTS, (*(row[0] for row in _PERIOD_STARTS[1:]), LAST_MONTH_END), strict=True + ) +) + + +def month_periods(start: datetime, end: datetime) -> Iterator[Period]: + """Intersect each release period with the UTC month being read.""" + for period in _PERIODS: + lower, upper = max(start, period.start), min(end, period.end) + if lower < upper: + yield Period(lower, upper, period.split, period.cohort) + + +@dataclass(frozen=True) +class Turbine: + """Published identifiers and rated capacity of a SCADA station.""" + + station: int + name: str + rated_power_kw: int + + +@dataclass(frozen=True) +class WindowSeries: + """One turbine record and the valid hourly cutoffs for its month and cohort.""" + + record: Record + power: Signal + period: Period + cutoffs: tuple[int, ...] + + +def prepare_windows(values: NDArray[np.float64], turbine: Turbine, period: Period) -> WindowSeries: + """Store source nulls and select only windows with complete inputs and power targets.""" + history_valid = np.isfinite(values).all(axis=0) + power_values = values[0] + total = power_values.size + cutoffs = np.arange(_CONTEXT, total - _HORIZON + 1, _HORIZON, dtype=np.int64) + history_counts = np.concatenate(([0], np.cumsum(history_valid))) + power_counts = np.concatenate(([0], np.cumsum(np.isfinite(power_values)))) + valid = (history_counts[cutoffs] - history_counts[cutoffs - _CONTEXT] == _CONTEXT) & ( + power_counts[cutoffs + _HORIZON] - power_counts[cutoffs] == _HORIZON + ) + signals = tuple( + Signal.from_values(pa.array(channel, mask=np.isnan(channel)), spec=spec, name=spec.spec_type, time_axis=_AXIS) + for spec, channel in zip(_SPECS, values, strict=True) + ) + record = Record( + start_time=TimeOrigin(period.start + STEP), + metadata={ + "station_id": turbine.station, + "turbine_id": turbine.name, + "rated_power_kw": turbine.rated_power_kw, + "evaluation_cohort": period.cohort, + }, + sources=(Source(name="SCADA", signals=signals),), + ) + return WindowSeries(record, signals[0], period, tuple(int(cutoff) for cutoff in cutoffs[valid])) + + +def forecasting_tasks(series: list[WindowSeries]) -> Iterator[ForecastingTask]: + """Stream tasks that point to disjoint history and target intervals on their source record.""" + for entry in series: + for cutoff in entry.cutoffs: + start = cutoff - _CONTEXT + stop = cutoff + _HORIZON + yield ForecastingTask( + inputs=(entry.record,), + scope=TimeInterval(start_us=start * STEP_US, end_us=cutoff * STEP_US), + targets=(TimeInterval(start_us=cutoff * STEP_US, end_us=stop * STEP_US, time_series=(entry.power,)),), + prompt="Forecast mean active power in kW for the next six ten-minute intervals.", + split=entry.period.split, + metadata={ + "evaluation_cohort": entry.period.cohort, + "window_start_utc": (entry.period.start + start * STEP).isoformat(), + "forecast_cutoff_utc": (entry.period.start + cutoff * STEP).isoformat(), + }, + )