diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py new file mode 100644 index 000000000..6af311826 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/__init__.py @@ -0,0 +1 @@ +"""Renewable Energy Systems datasets.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md new file mode 100644 index 000000000..6053d4eda --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/README.md @@ -0,0 +1,94 @@ +# Hill of Towie SCADA Forecasting + +`res/hill-of-towie-scada@2.0.0` covers all 21 turbines from January 2016 through April 2026. +It uses annual ten-minute SCADA archives from Zenodo release 2.1.0. Each task predicts six measured +active-power values in kW from the previous 36 ten-minute readings of active power, active wind +speed, and generator RPM. [Source release and CC BY 4.0 attribution](https://zenodo.org/records/22662930) + +## Dataset split + +All boundaries are UTC, with an inclusive start and exclusive end for physical measurement +intervals. Source timestamps label the **end** of each ten-minute interval, so a reading stamped +at a split boundary belongs to the preceding interval. An hourly task is eligible when its entire +six-hour input and one-hour target fit in one calendar month and one row of the table below. The +same boundaries apply to every turbine. An input needs all three channels, and a target needs +only power. Source gaps and blanks remain nulls in the shared records. + +| TimeF split | `evaluation_cohort` | Start UTC | End UTC | Windows | Rationale | +| --- | --- | --- | --- | ---: | --- | +| Train | `historical_train` | 2016-01-01 00:00 | 2024-10-01 00:00 | 1,564,202 | Retains several seasons, the AeroUp rollout, and TuneUp. Ends after the documented late-summer 2024 curtailment period. | +| Validation | `post_tuneup_validation` | 2024-10-01 00:00 | 2025-01-01 00:00 | 45,511 | Tunes on later data following the TuneUp campaign, ahead of the 2025 holdout. | +| Test | `pre_commissioning` | 2025-01-01 00:00 | 2025-10-28 13:00 | 141,619 | Main score for forecasting before documented Dynamic Yaw commissioning. | +| Test | `dynamic_yaw_commissioning` | 2025-10-28 13:00 | 2026-01-07 13:00 | 34,857 | Covers commissioning, tests, and changes to turbine roles. | +| Test | `dynamic_yaw_west_trial` | 2026-01-07 13:00 | 2026-02-03 18:00 | 13,434 | Starts at the formal west trial boundary in the RES configuration. | +| Test | `dynamic_yaw_west_and_east_trial` | 2026-02-03 18:00 | 2026-05-01 00:00 | 42,614 | Starts at the formal east cohort boundary and ends with the archive. | + +Use the main test cohort for the headline score and report the three later cohorts separately. +September 2025 lost 44.34% of expected turbine intervals, including a 12.73-day gap across every +turbine. Report a main-test sensitivity score with September omitted to show how that gap affects +coverage. The Dynamic Yaw schedule alternates 50 minutes on and 50 minutes off. The cohort labels +describe dates. Per-turbine exposure, controller state and wake effects require the RES role +definitions and controller telemetry. [Trial configuration](https://github.com/resgroup/hill-of-towie-open-source-analysis/blob/main/scripts/wfc_analysis_2026/wind_up_config/HOT_dynamic_yaw.yaml) +· [Commissioning and curtailment dates](https://github.com/resgroup/hill-of-towie-open-source-analysis/blob/main/scripts/wfc_analysis_2026/wind_up_config/exclusion_periods/HOT_DY.yaml) + +## Measurement and operating changes + +| When | What changed | +| --- | --- | +| August 2016 | The turbine SCADA schema removed the low-speed power reduction fields. | +| July 2017 | The schema added boost availability. | +| August 1, 2017 and April 10–11, 2018 | The secondary anemometer correction started on T14 and T17 first. Most other turbines followed on April 10, 2018, with T01 and T06 on April 11. The historical audit found a median +11.91 percentage-point step in raw secondary wind speed relative to primary. The active-to-primary ratio was unchanged to two decimal places. | +| September 23–29, 2021 | T13 underwent the first AeroUp retrofit. | +| April 2022 | The schema added actual wind direction, boost activation, and turbine operating state. The three forecast channels span the earlier and later schemas. | +| July 14, 2022–May 26, 2023 | RES rolled out AeroUp to the rest of the fleet. Use the per-turbine first and last work dates to distinguish pre-upgrade, installation, and post-upgrade windows. | +| March 14–August 31, 2024 | TuneUp began on T02, T03, T06, T08, T09, T13, T16, and T20 on March 14. T21 joined on May 2. RES also flags heavy curtailment from August 9 through September 28. | +| October 28, 2025 onward | Dynamic Yaw commissioning began. RES lists turbine role changes and controller configuration tests in October, November, and December. | +| January 7 and February 3, 2026 | The formal west and east trial boundaries were at 13:00 and 18:00 UTC, respectively. Dynamic Yaw changes the relationship between inflow, yaw, and measured power. | + +RES records farm-wide pre-campaign tests from October 28 at 13:00 to November 17 at 13:45 UTC. +It records a deliberate north-offset test on T15 and T18 on November 19–20, a change in T06's +role on December 2, and changes in T19's and T21's roles on December 11. These dates are why the +main test stops in October 2025. [RES commissioning log](https://github.com/resgroup/hill-of-towie-open-source-analysis/blob/main/scripts/wfc_analysis_2026/wind_up_config/exclusion_periods/HOT_DY.yaml) + +The dataset targets actual delivered power, including valid readings during curtailment and +changes in operation. The connector uses the active wind-speed channel throughout. Use the same +validation period to compare full-history training with a June 2023 onward training run, after +the AeroUp rollout. [SCADA data dictionary](https://zenodo.org/records/22662930/preview/Hill_of_Towie_data_dictionary.md?include_deleted=0) +· [AeroUp dates by turbine](https://zenodo.org/records/22662930/files/Hill_of_Towie_AeroUp_install_dates.csv?download=1) +· [RES campaign summary](https://zenodo.org/records/22662930) + +Zenodo release 2.0.0 extended SCADA from August 2024 to April 2026 and added separate fastlog, +LiDAR, and controller telemetry. Release 2.1.0 updated the documentation. The forecasting +records here use the ten-minute SCADA tables. [Source changelog](https://zenodo.org/records/22662930/preview/CHANGELOG.md?include_deleted=0) + +## Build and read the forecast windows + +From the TimeNet checkout, run `make sync`, then build into a local registry: + +```bash +uv run timenet-build build res/hill-of-towie-scada@2.0.0 --no-isolation +``` + +The source download is approximately 15 GB. The connector reads two CSV tables per month directly +from the pinned archives. In the resulting TimeF release, a record holds one turbine and one +month segment. Each forecast carries an input `scope` for the 36 historical readings and a +target interval for the six observed power values. Apply the input scope before passing values to +a model. Reading a whole shared record would include readings after that task's cutoff. + +```python +from timenet import TimeNet +from timenet.types import TimeInterval + +with TimeNet().open("res/hill-of-towie-scada@2.0.0") as reader: + task = next(reader.iter_tasks(split="train", batch_size=1)) + assert isinstance(task.scope, TimeInterval) + assert task.targets is not None + (target,) = task.targets + assert isinstance(target, TimeInterval) and target.time_series + (power,) = target.time_series + context = { + signal.name: signal.slice(task.scope).to_numpy() + for signal in task.inputs[0].signals + } + observed_power_kw = power.slice(target).to_numpy() +``` diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py new file mode 100644 index 000000000..2f14b3731 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/__init__.py @@ -0,0 +1,6 @@ +"""Hill of Towie SCADA forecasting connector.""" + +from timenet_connectors.datasets.res.hill_of_towie_scada.connector import ( + CONNECTOR as CONNECTOR, + HillOfTowieSCADAConnector as HillOfTowieSCADAConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py new file mode 100644 index 000000000..e4be8c70d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/connector.py @@ -0,0 +1,200 @@ +"""Convert the Hill of Towie SCADA release into TimeF power forecasting tasks.""" + +from collections.abc import Iterator +import csv +from dataclasses import dataclass +from datetime import UTC, datetime +from functools import partial +from pathlib import Path +from zipfile import BadZipFile, ZipFile + +import numpy as np +from numpy.typing import NDArray +import pyarrow as pa +import pyarrow.csv as arrow_csv + +from timenet.connectors import BaseConnector +from timenet.dataset import TimeFDataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ForecastingTask, Split, unix_us +from timenet_connectors.datasets.res.hill_of_towie_scada.windows import ( + LAST_MONTH_END, + STEP, + STEP_US, + Turbine, + WindowSeries, + forecasting_tasks, + month_periods, + prepare_windows, +) +from timenet_connectors.download import Artifact, download_files + + +_FILES_URL = "https://zenodo.org/records/22662930/files" +_METADATA = "Hill_of_Towie_turbine_metadata.csv" +_METADATA_SHA256 = "dd40789c2bf2268da9d77890cc48617266f3ed9d13c5af2be6d6535671f261bc" +_ARCHIVE_SHA256 = { + 2016: "800bb3d399e6b79278758bbd95e234e364b062a9ff7643a7f9c045b0db9e9738", + 2017: "a49c11def85623b7f0537988debfa499cca15fc6ec971256051597d91224ab1f", + 2018: "4d59191bd12ad0e85eb4de44f2f8f7d8e2556452341e89cd194a816debc48761", + 2019: "24a9b9bfdf32558d80c518a11954d7230254f5ba384671d3df6ba3e53deb5b26", + 2020: "3e55de08be27e6a6e702a99bfcccd57e20abfa69e7c65d5901caf581f128f020", + 2021: "2284741792b4bfc7ca08234f49440dad4820bac5e73098011745a3f69e295c19", + 2022: "1b0f8657273ddbb8c4f84042a0bcb1c5eb1fb7833849b4c491f2b9f10604700d", + 2023: "e65f7acd65664584e5b72e4ad7ccd4c0f0af4ad68f618b8cf0b4e5c1f4a69521", + 2024: "b6bb540e3d2fcc2cff8756d96ef4155037b1cc41b284e6249b41054f4e50fbc3", + 2025: "cd439de7bf8e297dd8422d5c93eda04785c56e3861be8d2a79f403ac39b873ee", + 2026: "4bcfd706769369eacde59ea4f3a45bc3f9c0fb334c67fea75fcf3aa0c424eaea", +} +_POWER = ("wtc_ActPower_mean",) +_TELEMETRY = ("wtc_AcWindSp_mean", "wtc_GenRpm_mean") + + +@dataclass(frozen=True) +class HillOfTowieFiles: + """Local annual SCADA archives and their turbine lookup.""" + + archives: tuple[Path, ...] + metadata: Path + + +def _read_turbines(path: Path) -> tuple[Turbine, ...]: + """Read unique station identities and rated capacities.""" + turbines: dict[int, Turbine] = {} + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + for row in reader: + try: + turbine = Turbine(int(row["Station ID"]), row["Turbine Name"], int(row["Rated power (kW)"])) + except (KeyError, TypeError, ValueError) as exc: + raise TimeNetConnectorError(f"{path.name}:{reader.line_num}: invalid turbine metadata") from exc + if turbine.station in turbines or not turbine.name or turbine.rated_power_kw <= 0: + raise TimeNetConnectorError( + f"{path.name}:{reader.line_num}: invalid or duplicate station {turbine.station}" + ) + turbines[turbine.station] = turbine + if not turbines: + raise TimeNetConnectorError(f"{path.name}: no turbine metadata") + return tuple(turbines[key] for key in sorted(turbines)) + + +@dataclass(frozen=True) +class _Month: + start: datetime + end: datetime + + @property + def steps(self) -> int: + return (self.end - self.start) // STEP + + +def _months(year: int) -> Iterator[_Month]: + """Yield UTC months in an archive, respecting the release end in April 2026.""" + for month in range(1, 13): + start = datetime(year, month, 1, tzinfo=UTC) + if start >= LAST_MONTH_END: + return + end = datetime(year + month // 12, month % 12 + 1, 1, tzinfo=UTC) + yield _Month(start, end) + + +def _read_table( + archive: ZipFile, + month: _Month, + table: str, + fields: tuple[str, ...], + stations: NDArray[np.int64], +) -> NDArray[np.float64]: + """Load selected columns into a station-by-time array with source blanks preserved.""" + member = f"{table}_{month.start:%Y_%m}.csv" + options = arrow_csv.ConvertOptions( + include_columns=["TimeStamp", "StationId", *fields], + column_types={"TimeStamp": pa.timestamp("us"), "StationId": pa.int64(), **dict.fromkeys(fields, pa.float64())}, + ) + try: + with archive.open(member) as raw: + data = arrow_csv.read_csv(raw, convert_options=options) + except (KeyError, pa.ArrowInvalid) as exc: + raise TimeNetConnectorError(f"{archive.filename}: unable to read {member}: {exc}") from exc + + timestamps = data["TimeStamp"] + station_ids = data["StationId"] + if timestamps.null_count or station_ids.null_count: + raise TimeNetConnectorError(f"{member}: missing timestamp or station ID") + relative_us = timestamps.to_numpy().astype(np.int64) - unix_us(month.start) + # Source labels interval ends. A row at midnight belongs to the previous interval's month. + owned = (relative_us > 0) & (relative_us <= month.steps * STEP_US) + owned_us = relative_us[owned] + if np.any(owned_us % STEP_US): + raise TimeNetConnectorError(f"{member}: timestamp is outside the ten-minute grid") + ids = station_ids.to_numpy()[owned] + if unknown := np.setdiff1d(ids, stations).tolist(): + raise TimeNetConnectorError(f"{member}: unknown station IDs {unknown}") + rows = np.searchsorted(stations, ids) + steps = owned_us // STEP_US - 1 + if np.unique(rows * month.steps + steps).size != len(steps): + raise TimeNetConnectorError(f"{member}: duplicate measurement") + values = np.full((len(fields), stations.size, month.steps), np.nan) + for field, channel in zip(fields, values, strict=True): + measurements = data[field].to_numpy()[owned] + if np.any(np.isinf(measurements)): + raise TimeNetConnectorError(f"{member}: infinite measurement in {field}") + channel[rows, steps] = measurements + return values + + +def _month_series( + archive: ZipFile, month: _Month, turbines: tuple[Turbine, ...], stations: NDArray[np.int64] +) -> Iterator[WindowSeries]: + """Prepare the turbine records and eligible cutoffs for each part of a month.""" + power = _read_table(archive, month, "tblSCTurGrid", _POWER, stations) + telemetry = _read_table(archive, month, "tblSCTurbine", _TELEMETRY, stations) + values = np.concatenate((power, telemetry)) + for period in month_periods(month.start, month.end): + start = (period.start - month.start) // STEP + stop = (period.end - month.start) // STEP + for index, turbine in enumerate(turbines): + yield prepare_windows(values[:, index, start:stop], turbine, period) + + +class HillOfTowieSCADAConnector(BaseConnector[HillOfTowieFiles]): + """Build power forecasting tasks from the pinned 2016 to 2026 annual SCADA archives.""" + + # Override: the pinned source release has a fixed set of files. + # ruff: ignore[no-self-use] + async def download_async(self, cache_dir: Path) -> HillOfTowieFiles: + """Cache the Zenodo archives and turbine lookup.""" + pins = [(f"{year}.zip", digest) for year, digest in _ARCHIVE_SHA256.items()] + files = await download_files( + [ + Artifact(f"{_FILES_URL}/{name}?download=1", cache_dir / name, sha256=digest) + for name, digest in [*pins, (_METADATA, _METADATA_SHA256)] + ], + max_concurrency=5, + ) + return HillOfTowieFiles(tuple(files[:-1]), files[-1]) + + def convert(self, raw: HillOfTowieFiles, /) -> TimeFDataset: + """Register the SCADA records and stream tasks over qualifying windows.""" + if len(raw.archives) != len(_ARCHIVE_SHA256): + raise TimeNetConnectorError("Hill of Towie requires all 11 annual archives and turbine metadata") + dataset = TimeFDataset(metadata=self.metadata()) + series: list[WindowSeries] = [] + try: + turbines = _read_turbines(raw.metadata) + stations = np.array([turbine.station for turbine in turbines], dtype=np.int64) + for year, path in zip(_ARCHIVE_SHA256, raw.archives, strict=True): + with ZipFile(path) as archive: + for month in _months(year): + for entry in _month_series(archive, month, turbines, stations): + dataset.add_record(record=entry.record) + series.append(entry) + except (BadZipFile, OSError, UnicodeError, csv.Error) as exc: + raise TimeNetConnectorError(f"Unable to read Hill of Towie source files: {exc}") from exc + if {entry.period.split for entry in series if entry.cutoffs} != set(Split): + raise TimeNetConnectorError("Hill of Towie SCADA must contain complete windows in all three splits") + dataset.set_task_stream([ForecastingTask], partial(forecasting_tasks, series)) + return dataset + + +CONNECTOR = HillOfTowieSCADAConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml new file mode 100644 index 000000000..d65908e30 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/dataset.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: res/hill-of-towie-scada +dataset_version: 2.0.0 +name: Hill of Towie SCADA +description: >- + Ten-minute SCADA for 21 Hill of Towie wind turbines from January 2016 through April 2026. + Forecast six observed active-power readings from the preceding 36 readings of active power, + wind speed, and generator speed. Task scopes select history on shared turbine records, and + target spans select only future power. Hourly UTC cutoffs stay within a calendar month and + an evaluation cohort, and every required reading must be available. Training ends September + 2024 and validation covers October through December 2024. The 2025 to April 2026 test split + distinguishes pre-commissioning, commissioning, and the two Dynamic Yaw trial periods. +license: CC-BY-4.0 +access: open +source_url: https://zenodo.org/records/22662930 +citation: >- + Clerc, A., and Lingkan, E. (2026). Hill of Towie wind farm open dataset (Version 2.1.0). + Zenodo. https://doi.org/10.5281/zenodo.22662930. Released by RES on behalf of TRIG. +domains: + - energy +tags: + - wind-energy + - scada + - forecasting diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py new file mode 100644 index 000000000..af5d44209 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the Hill of Towie SCADA connector.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py new file mode 100644 index 000000000..b94169acf --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/tests/test_connector.py @@ -0,0 +1,138 @@ +"""Round-trip tests for Hill of Towie's SCADA forecasts.""" + +from collections import Counter +from datetime import UTC, datetime, timedelta +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pytest + +from timenet import TimeNet +from timenet.dataset.axis import RegularAxis +from timenet.engine import store_dataset +from timenet.errors import TimeNetConnectorError +from timenet.types import ForecastingTask, TimeInterval, unix_us, ureg +from timenet_connectors.datasets.res.hill_of_towie_scada.connector import HillOfTowieFiles, HillOfTowieSCADAConnector + + +_CAMPAIGN_BOUNDARIES = { + (2025, 10): datetime(2025, 10, 28, 13), + (2026, 1): datetime(2026, 1, 7, 13), + (2026, 2): datetime(2026, 2, 3, 18), +} + + +def _month_contents(first: datetime, january_gap: bool) -> tuple[list[str], list[str]]: + power = ["TimeStamp,StationId,wtc_ActPower_mean"] + telemetry = ["TimeStamp,StationId,wtc_AcWindSp_mean,wtc_GenRpm_mean"] + starts = [first] + if boundary := _CAMPAIGN_BOUNDARIES.get((first.year, first.month)): + starts.extend((boundary - timedelta(hours=7), boundary)) + for start in starts: + for step in range(0 if start == first else 1, 43): + stamp = (start + timedelta(minutes=step * 10)).isoformat(sep=" ") + for station, offset in ((2304510, 0), (2304511, 100)): + power.append(f"{stamp},{station},{first.month * 100 + step + offset}") + if step <= 36 or start != first: + gap = january_gap and station == 2304510 and step == 24 + wind = "" if gap else 8 + offset + telemetry.append(f"{stamp},{station},{wind},{900 + offset}") + return power, telemetry + + +def _source(tmp_path: Path, *, january_gap: bool = False, duplicate: bool = False) -> HillOfTowieFiles: + metadata = tmp_path / "metadata.csv" + metadata.write_text( + "\ufeffStation ID,Turbine Name,Rated power (kW)\n2304510,T01,2300\n2304511,T02,2300\n", + encoding="utf-8", + ) + paths = [] + for year in range(2016, 2027): + path = tmp_path / f"{year}.zip" + paths.append(path) + with ZipFile(path, "w") as archive: + for month in range(1, 5 if year == 2026 else 13): + power, telemetry = _month_contents(datetime(year, month, 1), january_gap and (year, month) == (2016, 1)) + if duplicate and (year, month) == (2016, 1): + power.append(power[-1]) + archive.writestr(f"tblSCTurGrid_{year}_{month:02}.csv", "\n".join(power)) + archive.writestr( + f"tblSCTurbine_{year}_{month:02}.csv", "\n".join([telemetry[0], *reversed(telemetry[1:])]) + ) + return HillOfTowieFiles(tuple(paths), metadata) + + +def test_forecasts_roundtrip_preserves_observations_and_units(tmp_path): + converted = HillOfTowieSCADAConnector().convert(_source(tmp_path)) + assert converted.has_task_stream + registry = tmp_path / "registry" + store_dataset(converted, registry) + with TimeNet(sources=[registry]).open("res/hill-of-towie-scada@2.0.0") as reader: + task = next(reader.iter_tasks(split="train", batch_size=1)) + assert isinstance(task, ForecastingTask) + assert isinstance(task.scope, TimeInterval) + (record,) = task.inputs + assert task.targets is not None + (target,) = task.targets + assert isinstance(target, TimeInterval) + assert target.time_series is not None + (target_signal,) = target.time_series + assert record.metadata["turbine_id"] == "T01" + assert task.metadata["window_start_utc"] == "2016-01-01T00:00:00+00:00" + assert task.metadata["forecast_cutoff_utc"] == "2016-01-01T06:00:00+00:00" + assert record.start_time.timestamp == unix_us(datetime(2016, 1, 1, 0, 10, tzinfo=UTC)) + channels = {signal.name: signal for signal in record.signals} + assert channels.keys() == {"active_power", "wind_speed", "generator_speed"} + power, wind, rpm = (channels[name] for name in ("active_power", "wind_speed", "generator_speed")) + assert target_signal is power + assert power.spec.unit_value == ureg.kilowatt + assert wind.spec.unit_value == ureg.meter / ureg.second + assert rpm.spec.unit_value == ureg.rpm + for signal in (power, wind, rpm): + assert isinstance(signal.time_axis, RegularAxis) + assert signal.time_axis.period_us == 600_000_000 + assert signal.step_range(task.scope) == (0, 36) + assert power.step_range(target) == (36, 42) + np.testing.assert_array_equal(power.slice(task.scope).to_numpy(), 100 + np.arange(1, 37)) + np.testing.assert_array_equal(wind.slice(task.scope).to_numpy(), np.full(36, 8)) + np.testing.assert_array_equal(rpm.slice(task.scope).to_numpy(), np.full(36, 900)) + np.testing.assert_array_equal(power.slice(target).to_numpy(), 100 + np.arange(37, 43)) + assert wind.read_steps(36, 42).null_count == 6 + + +def test_chronological_split_and_campaign_boundaries(tmp_path): + dataset = HillOfTowieSCADAConnector().convert(_source(tmp_path)) + tasks = list(dataset.iter_tasks()) + assert Counter(task.split for task in tasks) == {"train": 210, "validation": 6, "test": 44} + by_cohort = Counter(task.metadata["evaluation_cohort"] for task in tasks) + assert by_cohort == { + "historical_train": 210, + "post_tuneup_validation": 6, + "pre_commissioning": 22, + "dynamic_yaw_commissioning": 10, + "dynamic_yaw_west_trial": 6, + "dynamic_yaw_west_and_east_trial": 6, + } + all_cutoffs = set() + for task in tasks: + cutoff = task.metadata["forecast_cutoff_utc"] + assert isinstance(cutoff, str) + all_cutoffs.add(datetime.fromisoformat(cutoff)) + for boundary in _CAMPAIGN_BOUNDARIES.values(): + limit = boundary.replace(tzinfo=UTC) + assert limit - timedelta(hours=1) in all_cutoffs + assert limit + timedelta(hours=6) in all_cutoffs + assert not any(limit + timedelta(hours=hours) in all_cutoffs for hours in range(6)) + + +def test_missing_history_excludes_only_affected_windows(tmp_path): + dataset = HillOfTowieSCADAConnector().convert(_source(tmp_path, january_gap=True)) + tasks = list(dataset.iter_tasks(split="train")) + assert len(tasks) == 209 + assert tasks[0].inputs[0].metadata["turbine_id"] == "T02" + + +def test_duplicate_measurement_fails(tmp_path): + with pytest.raises(TimeNetConnectorError, match="duplicate measurement"): + HillOfTowieSCADAConnector().convert(_source(tmp_path, duplicate=True)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py new file mode 100644 index 000000000..5029aaaba --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/res/hill_of_towie_scada/windows.py @@ -0,0 +1,147 @@ +"""Chronological partitions and scoped forecast windows for Hill of Towie SCADA.""" + +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from fractions import Fraction + +import numpy as np +from numpy.typing import NDArray +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source +from timenet.dataset.axis import RegularAxis +from timenet.types import ( + ForecastingTask, + MissingReason, + MissingValues, + Split, + TimeInterval, + TimeOrigin, + TimeSeriesSpec, + ureg, +) + + +STEP = timedelta(minutes=10) +STEP_US = 600_000_000 +LAST_MONTH_END = datetime(2026, 5, 1, tzinfo=UTC) +_CONTEXT = 36 +_HORIZON = 6 +_AXIS = RegularAxis.from_rate_hz(Fraction(1, 600)) +_MISSING = MissingValues(reason=MissingReason.NOT_RECORDED, description="No SCADA value for this turbine and interval.") +_POWER = TimeSeriesSpec( + spec_type="active_power", name="Active power", unit_value=ureg.kilowatt, dtype="float64", missing=_MISSING +) +_WIND = TimeSeriesSpec( + spec_type="wind_speed", + name="Active wind speed", + unit_value=ureg.meter / ureg.second, + dtype="float64", + missing=_MISSING, +) +_RPM = TimeSeriesSpec( + spec_type="generator_speed", name="Generator speed", unit_value=ureg.rpm, dtype="float64", missing=_MISSING +) +_SPECS = (_POWER, _WIND, _RPM) + + +@dataclass(frozen=True) +class Period: + """One contiguous split or evaluation cohort, with an exclusive end.""" + + start: datetime + end: datetime + split: Split + cohort: str + + +_PERIOD_STARTS = ( + (datetime(2016, 1, 1, tzinfo=UTC), Split.TRAIN, "historical_train"), + (datetime(2024, 10, 1, tzinfo=UTC), Split.VALIDATION, "post_tuneup_validation"), + (datetime(2025, 1, 1, tzinfo=UTC), Split.TEST, "pre_commissioning"), + (datetime(2025, 10, 28, 13, tzinfo=UTC), Split.TEST, "dynamic_yaw_commissioning"), + (datetime(2026, 1, 7, 13, tzinfo=UTC), Split.TEST, "dynamic_yaw_west_trial"), + (datetime(2026, 2, 3, 18, tzinfo=UTC), Split.TEST, "dynamic_yaw_west_and_east_trial"), +) +_PERIODS = tuple( + Period(start, end, split, cohort) + for (start, split, cohort), end in zip( + _PERIOD_STARTS, (*(row[0] for row in _PERIOD_STARTS[1:]), LAST_MONTH_END), strict=True + ) +) + + +def month_periods(start: datetime, end: datetime) -> Iterator[Period]: + """Intersect each release period with the UTC month being read.""" + for period in _PERIODS: + lower, upper = max(start, period.start), min(end, period.end) + if lower < upper: + yield Period(lower, upper, period.split, period.cohort) + + +@dataclass(frozen=True) +class Turbine: + """Published identifiers and rated capacity of a SCADA station.""" + + station: int + name: str + rated_power_kw: int + + +@dataclass(frozen=True) +class WindowSeries: + """One turbine record and the valid hourly cutoffs for its month and cohort.""" + + record: Record + power: Signal + period: Period + cutoffs: tuple[int, ...] + + +def prepare_windows(values: NDArray[np.float64], turbine: Turbine, period: Period) -> WindowSeries: + """Store source nulls and select only windows with complete inputs and power targets.""" + history_valid = np.isfinite(values).all(axis=0) + power_values = values[0] + total = power_values.size + cutoffs = np.arange(_CONTEXT, total - _HORIZON + 1, _HORIZON, dtype=np.int64) + history_counts = np.concatenate(([0], np.cumsum(history_valid))) + power_counts = np.concatenate(([0], np.cumsum(np.isfinite(power_values)))) + valid = (history_counts[cutoffs] - history_counts[cutoffs - _CONTEXT] == _CONTEXT) & ( + power_counts[cutoffs + _HORIZON] - power_counts[cutoffs] == _HORIZON + ) + signals = tuple( + Signal.from_values(pa.array(channel, mask=np.isnan(channel)), spec=spec, name=spec.spec_type, time_axis=_AXIS) + for spec, channel in zip(_SPECS, values, strict=True) + ) + record = Record( + start_time=TimeOrigin(period.start + STEP), + metadata={ + "station_id": turbine.station, + "turbine_id": turbine.name, + "rated_power_kw": turbine.rated_power_kw, + "evaluation_cohort": period.cohort, + }, + sources=(Source(name="SCADA", signals=signals),), + ) + return WindowSeries(record, signals[0], period, tuple(int(cutoff) for cutoff in cutoffs[valid])) + + +def forecasting_tasks(series: list[WindowSeries]) -> Iterator[ForecastingTask]: + """Stream tasks that point to disjoint history and target intervals on their source record.""" + for entry in series: + for cutoff in entry.cutoffs: + start = cutoff - _CONTEXT + stop = cutoff + _HORIZON + yield ForecastingTask( + inputs=(entry.record,), + scope=TimeInterval(start_us=start * STEP_US, end_us=cutoff * STEP_US), + targets=(TimeInterval(start_us=cutoff * STEP_US, end_us=stop * STEP_US, time_series=(entry.power,)),), + prompt="Forecast mean active power in kW for the next six ten-minute intervals.", + split=entry.period.split, + metadata={ + "evaluation_cohort": entry.period.cohort, + "window_start_utc": (entry.period.start + start * STEP).isoformat(), + "forecast_cutoff_utc": (entry.period.start + cutoff * STEP).isoformat(), + }, + )