diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py new file mode 100644 index 000000000..e931dad7d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py @@ -0,0 +1 @@ +"""Datasets published by the KIOS Research and Innovation Center of Excellence, University of Cyprus.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md new file mode 100644 index 000000000..3bf0c807a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md @@ -0,0 +1,206 @@ +# BattLeDIM: the release, and the decisions of this connector + +BattLeDIM 2020 (Battle of the Leakage Detection and Isolation Methods) released two years of +SCADA data from L-Town, a water distribution network, together with the 33 leaks the data +contains. +- The data is **simulated**. The release's own configuration names `L-TOWN_Real.inp` as the + network that generated it. +- The connector pins every file it downloads by sha256, so a changed file fails the build instead + of silently building something else. + +This document covers: +- what the release holds; +- how the connector maps it onto TimeF; +- the decisions the release leaves open; +- what a user should know before training on it. + +*(measured)* marks a number taken from the files, not from the dataset page. + +## Source, licence and citation + +**Source:** Zenodo record [4017659](https://zenodo.org/records/4017659), doi:10.5281/zenodo.4017659, published 2020-09-07. It has one version. + +**Licence:** CC BY 4.0, as stated on the Zenodo record. The organisers' GitHub repository +[KIOS-Research/BattLeDIM](https://github.com/KIOS-Research/BattLeDIM) states the same. A user must credit the authors. + +**Citation:** +> Vrachimis, S. G., Eliades, D. G., Taormina, R., Kapelan, Z., Ostfeld, A., Liu, S., Kyriakou, M., +> Pavlou, P., Qiu, M., and Polycarpou, M. M. (2022). Battle of the Leakage Detection and Isolation +> Methods. *Journal of Water Resources Planning and Management*, 148(12). +> doi:10.1061/(ASCE)WR.1943-5452.0001601 + +**Files the connector downloads** (13 of the 18 on the record): + +| File | What it holds | +| --- | --- | +| `2018_SCADA_*.csv`, `2019_SCADA_*.csv` | Pressures, Flows, Levels and Demands, one CSV per year and type | +| `2018_Leakages.csv`, `2019_Leakages.csv` | The outflow of every leak active that year, in m³/h | +| `dataset_configuration.yaml` | The configuration that generated the data: time range, the leak list, the sensor ids | +| `2018_Fixed_Leakages_Report.txt` | The repair times of the 2018 leaks that were fixed | +| `L-TOWN.inp` | The EPANET model given to participants (nominal parameters) | + +**Files it skips:** +- The two `.xlsx` files, which repeat the CSVs (README.txt of the release). +- `L-TOWN_Real.inp` (177 MB), the simulation model. Nothing in the tasks needs it. + +## The record + +The two years join into **one record** on one 5-minute grid: +- 2018-01-01 00:00 → 2019-12-31 23:55, 210 240 steps *(measured)*. +- The 2019 files continue the 2018 files with no gap. +- **Time zone:** the release writes naive timestamps and states no zone. The connector reads them as UTC. + +Each sensor type is one source. The signal names are the EPANET ids of `L-TOWN.inp`. + +| Source | Signals | Unit (README.txt of the release) | Range *(measured)* | +| --- | --- | --- | --- | +| Pressure sensors | 33: n1, n4, n31, n54, n105, n114, n163, n188, n215, n229, n288, n296, n332, n342, n410, n415, n429, n458, n469, n495, n506, n516, n519, n549, n613, n636, n644, n679, n722, n726, n740, n752, n769 | m (pressure head) | 25.7–57.0 | +| Flow sensors | PUMP_1 (pump into tank T1), p227, p235 (the two reservoir inlets) | m³/h | 0–205 | +| Tank level sensor | T1 | m | 1.95–3.90 | +| Automated meter readings | 82 nodes, the `amrs` list of the configuration | L/h | 0–37 796 | + +**Reading and checking:** +- **Values:** float64, as parsed from the decimal-comma text. Nothing is filled, trimmed or resampled. +- **Grid:** a step that is not 5 minutes fails the build. +- **Values:** a missing or non-finite value fails the build. +- **Sensors:** each year's columns must equal the sensor list of the configuration. +- **Network:** every sensor must exist in `L-TOWN.inp`, as a node or, for flow, as a link. + +## The leaks + +The configuration lists **33 leaks** *(measured)*: +- 15 abrupt and 18 incipient; +- hole diameters of 8.8–22.9 mm; +- peak outflows of 5.1–35.4 m³/h. + +| Year | Leaks starting | Active during the year | Repaired that year | Open at the end of 2019 | +| --- | --- | --- | --- | --- | +| 2018 | 14 | 14 | 10 | 4 carry over into 2019 | +| 2019 | 19 | 23 | 7 | 16 in total | + +An **abrupt** leak appears at full size at its start. An **incipient** leak grows from its start until its `peakTime`. + +Each leak becomes one **dataset annotation** with the key `leak`. Its value is the pipe's EPANET id, and its metadata holds: +- `leak_type`: `abrupt` or `incipient`; +- `start`, `end`, `peak`: the configured times, as text; +- `start_us`, `end_exclusive_us`, `peak_us`: the same times in microseconds from the record start. The interval runs from the configured start to the step after the configured end; +- `hole_diameter_mm`; +- `peak_outflow_m3h`: the largest outflow in the leak tables; +- `repaired`: whether the leak was repaired before the end of 2019. + +**The leaks are answers, so they are not on the record.** Every task reads the record, and anything that serves the record to a model also serves the record's annotations. TimeNet's `TimeFTorchDataset`, for example, returns them in each item. The record therefore holds the sensor signals and nothing else. Each window's answers are in its task targets. + +**Consistency checks.** The build fails if any of these disagree: +- the configuration, the two leak tables and the repair report, which must list every leak that ends in 2018, exactly once; +- a leak's outflow and its configured start and end (for p800 the leak tables keep one step of outflow after its end, an exception the connector names); +- a leak pipe and the pipes of `L-TOWN.inp`. + +It also fails on: +- an abrupt leak whose peak is not its start; +- a sensor listed twice in the configuration; +- a configuration without its `times` mapping or `leakages` list; +- a CSV cell that is not a timestamp or a number. + +The release passes all of these checks. + +## Windows and tasks + +**Windows:** a window is 14 days, starting every 7 days. +- The first week is context; the second week is the **target week**. +- The recording holds **103 windows** *(measured)*. +- Every leak start and every repair falls in exactly one target week: 33 starts, 17 repairs. The build checks this and fails otherwise. +- 33 windows have a new leak, 16 have a repair, and 59 have neither *(measured)*. +- **Embargo:** 5 windows get no tasks, because their context week is a held-out target week (see The split). They hold no new leak, but two of them describe a repair (p514, p142), so those two repairs appear in no task. + +**Tasks:** each of the other 98 windows gives four tasks, all scoped to the window. That is 392 tasks. + +| Task | Answer | Source of the answer | +| --- | --- | --- | +| `ClassificationTask` | the types of the leaks that start in the target week: a multi-label set {`abrupt`, `incipient`}, empty when none starts | configuration | +| `TemporalLocalizationTask` (exhaustive) | a time point at each leak start in the target week | configuration | +| `ScalarPredictionTask` `mean_leak_outflow` | the mean total leak outflow over the target week, m³/h | leak tables | +| `AnswerTask` | a short description of every leak that starts or is repaired in the target week | configuration, leak tables, `L-TOWN.inp` | + +**How the text is made.** Fixed templates fill in the answer; no model writes it. + +*New leak:* +> A sudden leak started on pipe {pipe} on {start}, with a hole of {diameter} mm and an outflow of up +> to {max outflow in the window} m3/h in this window. The nearest pressure sensor is {sensor}, {n} +> links away. + +**Incipient leaks.** The sentence opens "A gradually growing leak started on pipe {pipe} on {start}". It continues with: +- "and peaked on {peakTime}" if the peak lies inside the window; +- "and was still growing at the end of this window" if it does not. A peak after the window is not stated, because the scoped input cannot show it. + +**Outflow figures.** +- An outflow under 0.1 m³/h keeps the leak tables' second decimal, e.g. "0.02 m3/h". +- When the leak tables hold no outflow at all in the window (an incipient leak's first hours), the text says "no outflow in the leak tables yet". This applies to 3 windows *(measured)*. + +*Repair:* +> The leak on pipe {pipe} was repaired on {end}; its outflow before the repair was {outflow} m3/h. + +*Neither:* +> No leak started or was repaired in the last 7 days. + +*Always last:* +> At the end of the window {k} leaks are active, with a total outflow of {total} m3/h. + +A leak counts as active when the leak list's interval holds the window's last step. The total is the leak tables' outflow at that step. + +**Nearest sensor.** "Nearest" counts links in the `L-TOWN.inp` graph (pipes, pumps and valves) from the nearer end of the leaking pipe. Ties go to the sensor listed first in the configuration. + +**Example input and output.** The window that starts on 2018-01-29 (target week 2018-02-05) holds all 119 signals over 14 days. Its text target is: + +> The leak on pipe p232 was repaired on 2018-02-10 09:20; its outflow before the repair was 26.0 +> m3/h. At the end of the window 2 leaks are active, with a total outflow of 9.8 m3/h. + +**Task metadata:** `window_index`, `window_start`, `target_week_start`, `calendar_block`, `new_leaks` and `repaired_leaks`. + +## The split + +The release's own partition was 2018 for training and 2019 for evaluation. **This connector does not use it.** +- **Why:** 74 of the 82 meter series repeat 2018 exactly in 2019 *(measured)*. As a result, the 2019 inflow minus the 2018 inflow at the same calendar time equals the 2019 leak outflow minus the 2018 leak outflow: slope 1.00, daily correlation 0.989 *(measured)*. +- **The risk:** a year split would test on a demand background the model saw in training. + +**What it does instead:** +- Windows are grouped by the **calendar block** of their target week: the ISO week, in blocks of four. +- A block holds the same weeks of both years. +- `StratifiedSplitter` (8/1/1 folds, stratified on "a new leak starts", seed 42) assigns whole blocks. + +**Embargo.** Windows overlap by one week: a window's context week is the target week of the window before it. +- A window gets no tasks when that week is a validation or test target week of a different partition. +- Otherwise a training window would show the model the signals that a test task asks about. +- This affects 5 training windows *(measured)*. + +**Result** *(measured)*: + +| Partition | Windows | With a new leak | Tasks | Calendar blocks | +| --- | --- | --- | --- | --- | +| train | 74 | 27 | 296 | 0, 1, 2, 3, 5, 7, 8, 9, 10, 11 | +| validation | 8 | 3 | 32 | 4 | +| test | 16 | 3 | 64 | 6, 12 | + +**Remaining overlap:** a held-out window's context week can be a training target week. That shows held-out tasks training data, not the other way round. Target weeks never repeat across partitions. + +**One leaked day per block boundary:** the 2019 demand repeats 2018 by calendar date, but 2019 weekdays fall one day later. A 2019 target week (Monday to Sunday) therefore shares one date with the 2018 week before it. Where a block boundary separates partitions, one day of a target week's demand background also appears in the neighbouring partition. Week-aligned windows cannot avoid this. + +## Release details a user should know + +- **Simulated data.** The SCADA values come from a hydraulic simulation of `L-TOWN_Real.inp`. They are not field measurements. +- **Leaks almost never stop.** At least one leak is active in 97.8 % of 2018 and in 100 % of 2019, with up to 16 at once *(measured)*. There is almost no leak-free baseline. +- **Incipient onsets are not visible at once.** For incipient leaks, the leak tables show the first non-zero outflow 3 hours to 2.5 days after the configured start, because the early ramp rounds to zero *(measured)*. The localization target is the configured start. +- **One end time differs.** For p800, the last non-zero outflow is 5 minutes after the configured end *(measured)*. The connector keeps the configured end. + +## Limitations and intended use + +**Limitations:** +- 33 leak events on one network. +- Small validation and test partitions (8 and 16 windows). +- Text targets are templates, not expert explanations. +- Some window statements (the active leaks, the outflow) come from labels that a model can only infer indirectly from the sensors. + +**Intended use:** +- Training and evaluating time-series language models that detect, time, size and describe leaks from SCADA data. +- Studying explanations that can be checked against a known ground truth. + +*Contributed by Team RAMadan during the TimeNet hackathon, Round 1 (November 2026).* diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py new file mode 100644 index 000000000..79edbfc84 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py @@ -0,0 +1,6 @@ +"""The BattLeDIM connector package: its connector implementation and dataset card.""" + +from timenet_connectors.datasets.kios.battledim.connector import ( + CONNECTOR as CONNECTOR, + BattLeDIMConnector as BattLeDIMConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py new file mode 100644 index 000000000..e0f53922c --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py @@ -0,0 +1,179 @@ +"""The BattLeDIM connector: two years of SCADA data from the L-Town water network, with its leaks. + +BattLeDIM 2020 (Battle of the Leakage Detection and Isolation Methods) released simulated SCADA +data of the L-Town network for 2018 and 2019: 33 pressure sensors, 3 flow sensors, 1 tank level +sensor and 82 automated meter readings, every 5 minutes, and the 33 leaks that the simulation +contained. The two years form one Record. The leaks become dataset annotations, and two-week windows +of the record become the tasks of :mod:`~timenet_connectors.datasets.kios.battledim.tasks`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction +import functools +from pathlib import Path + +import pyarrow as pa + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import Annotation, TimeOrigin, TimeSeriesSpec, ureg +from timenet_connectors.datasets.kios.battledim.source import SCADA_FILES, STEP, Leak, Release +from timenet_connectors.datasets.kios.battledim.tasks import embargoed, split_windows, window_tasks, windows +from timenet_connectors.download import Artifact, download_files + + +ZENODO_FILE = "https://zenodo.org/api/records/4017659/files/{name}/content" +# Zenodo record 4017659 (published 2020-09-07) is the only version. The two .xlsx files repeat the +# CSVs and L-TOWN_Real.inp (177 MB) is the simulation model, so the connector downloads neither. +SOURCE_FILES = { + "2018_SCADA_Pressures.csv": "7782d890a7195699be94ddc01cba392da7ca285908afff619065a9616e04dcae", + "2018_SCADA_Flows.csv": "7f7e8625c9d92b083ee079a54cffb9ef803876de1e35017c1da0455968daf3ff", + "2018_SCADA_Levels.csv": "a1fc5f0229448a620b6220f4d079b3162a255b3070acdce448608fbfc760903c", + "2018_SCADA_Demands.csv": "713d9f03ff94128f44c7d8036a032cc1d7eebcbbc94fda9def2bbc3efc047dd5", + "2019_SCADA_Pressures.csv": "a4af6616059cc9b366bbab76d880b2c772df544970c6ff4d4a29a58704ce05b1", + "2019_SCADA_Flows.csv": "c709ea5ecc03197e1fe70cf4c2de7225f71015acb50a7445ed7765b5029f7f15", + "2019_SCADA_Levels.csv": "00624e97b72e63233beeccb48db528d43e59a87b116d0295d5c2a206ee0e75a8", + "2019_SCADA_Demands.csv": "004266aa039833b5ae0f37740487db4d90216d2ee947f14871c5d5d3f89b4151", + "2018_Leakages.csv": "e1abd9d549a18880d8ae704dc90c06c8a3d5aecbd2541fa091548ef392084ba6", + "2019_Leakages.csv": "b2ffe56b6c2f3b6cdb6aae9aeb30b9c425ba911fff8e3716ae543a56ff7576cd", + "dataset_configuration.yaml": "23471d8989d82c66a4909816ea248f7588d8ccc910e142265a7ea48db940b6a0", + "2018_Fixed_Leakages_Report.txt": "5dba254c623b867a0a91150e2b5539842ca05a989fde6cf2754065d5b415a8f3", + "L-TOWN.inp": "a7551b86745f4cc3433c78e60023077fa1386947d4d35372ffd3a0405622b436", +} + +# README.txt of the release states each unit. Pressure is head in metres of water. The values stay +# float64, as parsed from the release's two-decimal text. +SPECS = { + "pressure": TimeSeriesSpec(spec_type="pressure_head", name="Pressure head", unit_value=ureg.meter, dtype="float64"), + "flow": TimeSeriesSpec( + spec_type="volume_flow", name="Volume flow", unit_value=ureg.meter**3 / ureg.hour, dtype="float64" + ), + "tank_level": TimeSeriesSpec( + spec_type="water_level", name="Tank water level", unit_value=ureg.meter, dtype="float64" + ), + "amr_demand": TimeSeriesSpec( + spec_type="water_demand", name="Metered water demand", unit_value=ureg.liter / ureg.hour, dtype="float64" + ), +} +SOURCE_NAMES = { + "pressure": "Pressure sensors", + "flow": "Flow sensors", + "tank_level": "Tank level sensor", + "amr_demand": "Automated meter readings", +} + + +@dataclass(frozen=True) +class BattLeDIMSource: + """Handle to the downloaded release: the directory that holds the files of ``SOURCE_FILES``.""" + + root: Path + + +class BattLeDIMConnector(BaseConnector[BattLeDIMSource]): + """Connector for the BattLeDIM 2020 dataset (Zenodo record 4017659).""" + + async def download_async(self, cache_dir: Path) -> list[BattLeDIMSource]: # ruff: ignore[no-self-use] + """Download the release files once, checking each one's sha256. + + Returns: + A list of one :class:`BattLeDIMSource`. + """ + await download_files( + Artifact(ZENODO_FILE.format(name=name), cache_dir / name, sha256=digest) + for name, digest in SOURCE_FILES.items() + ) + return [BattLeDIMSource(cache_dir)] + + def convert(self, raw_refs: list[BattLeDIMSource]) -> TimeFDataset: + """Build the record, its leak annotations and the window tasks. + + The release is 190 MB of text, so every file is read once here, which also checks it before + anything is written. The signal loaders then serve the parsed columns. + + Returns: + The populated :class:`~timenet.dataset.TimeFDataset`. + + Raises: + TimeNetConnectorError: If ``raw_refs`` does not hold exactly one release. + """ + if len(raw_refs) != 1: + raise TimeNetConnectorError(f"expected one release, got {len(raw_refs)}") + release = Release(raw_refs[0].root) + dataset = TimeFDataset(metadata=self.metadata()) + record = dataset.add_record(record=self._make_record(release)) + # The leaks are the answers. They go on the dataset, not on the record that every task reads, + # so nothing that serves the record to a model serves them too. + for leak in release.leaks: + dataset.annotate(self._leak_annotation(release, leak)) + found = windows(release) + split_of = split_windows(found) + skipped = embargoed(found, split_of) + for window in found: + if window.index not in skipped: + dataset.add_tasks(tasks=window_tasks(release, record, window, split_of[window.index])) + return dataset + + @staticmethod + def _leak_annotation(release: Release, leak: Leak) -> Annotation: + """Describe one leak as a dataset annotation: its pipe, with the leak list's facts as metadata. + + A dataset annotation has no time span, so the metadata gives each time both as text and as + microseconds from the record start. The interval runs from the configured start to the step + after the configured end. + + Returns: + The annotation, keyed ``leak``. + """ + return Annotation( + key="leak", + value=leak.pipe, + description="One leak of the release: the leaking pipe's EPANET id, with its facts as metadata.", + metadata={ + "leak_type": leak.leak_type, + "start": f"{leak.start:%Y-%m-%d %H:%M}", + "end": f"{leak.end:%Y-%m-%d %H:%M}", + "peak": f"{leak.peak:%Y-%m-%d %H:%M}", + "start_us": release.micros(leak.start), + "end_exclusive_us": release.micros(leak.end + STEP), + "peak_us": release.micros(leak.peak), + "hole_diameter_mm": round(leak.diameter_m * 1000, 4), + "peak_outflow_m3h": float(release.leak_outflow[leak.pipe].max()), + "repaired": leak.repaired, + }, + ) + + @staticmethod + def _make_record(release: Release) -> Record: + """Build the one record: a source per sensor type, and no labels. + + Returns: + The record, starting at the first step of 2018. + """ + axis = RegularAxis(period_us=Fraction(int(STEP.total_seconds() * 1_000_000))) + sources = [] + for kind in SCADA_FILES: + columns = release.scada[kind].columns + signals = tuple( + Signal.from_loader( + spec=SPECS[kind], + time_axis=axis, + name=sensor, + loader=functools.partial(pa.array, columns[sensor]), + n_values=release.n_steps, + ) + for sensor in release.sensors[kind] + ) + sources.append(Source(name=SOURCE_NAMES[kind], signals=signals)) + return Record( + sources=tuple(sources), + start_time=TimeOrigin(release.start), + metadata={"network": "L-Town", "source_doi": "10.5281/zenodo.4017659"}, + ) + + +CONNECTOR = BattLeDIMConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml new file mode 100644 index 000000000..d99fa783a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml @@ -0,0 +1,42 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# +# BattLeDIM 2020 released two years (2018 and 2019) of simulated SCADA data of the L-Town water +# network, every 5 minutes: 33 pressure sensors, 3 flow sensors, 1 tank level sensor and 82 +# automated meter readings, plus the 33 leaks the simulation contained. The two years form one +# record, which holds the sensor signals only. Each leak is one dataset annotation. Two-week windows, +# one every 7 days, become four tasks each about the last week of the window. The answers come from +# the leak list, the leak tables and the network model by fixed rules, not from a model. +# +# The split groups windows by the calendar block of their target week (four ISO weeks), in both +# years at once: 74 of the 82 meter series of 2019 repeat 2018 exactly, so the same calendar week +# of both years always lands in the same partition. README.md gives the reasons and the release +# details. +yaml_schema_version: 1 +dataset_id: kios/battledim +dataset_version: 1.0.0 +name: BattLeDIM +description: >- + Two years of simulated SCADA data (pressure, flow, tank level and metered demand, every 5 minutes) + from the L-Town water distribution network, with its 33 leaks. Two-week windows ask which leaks + started, when, how much water was lost, and for a short explanation that names the pipe and the + nearest pressure sensor. +license: CC-BY-4.0 +source_url: https://zenodo.org/records/4017659 +citation: >- + Vrachimis, S. G., Eliades, D. G., Taormina, R., Kapelan, Z., Ostfeld, A., Liu, S., Kyriakou, M., + Pavlou, P., Qiu, M., and Polycarpou, M. M. (2022). Battle of the Leakage Detection and Isolation + Methods. Journal of Water Resources Planning and Management, 148(12). + https://doi.org/10.1061/(ASCE)WR.1943-5452.0001601. Dataset: https://doi.org/10.5281/zenodo.4017659 +access: open +domains: + - industrial + - environment +tags: + - water-distribution + - leak-detection + - anomaly-detection + - fault-diagnosis + - explanation + - simulated + - scada + - zenodo diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py new file mode 100644 index 000000000..c055be89f --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py @@ -0,0 +1,436 @@ +"""Read the BattLeDIM release: the SCADA and leak tables, the leak list and the network topology. + +The release holds one semicolon-separated CSV per year and sensor type, with a decimal comma and a +naive ``Timestamp`` column on a 5 minute grid. ``dataset_configuration.yaml`` is the configuration +that generated the data. It lists every leak and the sensor of each type. ``L-TOWN.inp`` is the +EPANET model of the network. :class:`Release` reads them all once and checks that they agree. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.csv as pacsv +import yaml + +from timenet.errors import TimeNetConnectorError + + +STEP = timedelta(minutes=5) +YEARS = (2018, 2019) +# The SCADA file of each sensor type, and the key that lists its sensors in the configuration. +SCADA_FILES = { + "pressure": ("SCADA_Pressures", "pressure_sensors"), + "flow": ("SCADA_Flows", "flow_sensors"), + "tank_level": ("SCADA_Levels", "level_sensors"), + "amr_demand": ("SCADA_Demands", "amrs"), +} +LEAK_TYPES = ("abrupt", "incipient") +_TIME_FORMAT = "%Y-%m-%d %H:%M" +_LEAK_FIELDS = 6 # linkID, startTime, endTime, leakDiameter (m), leakType, peakTime +_REPAIR_FIELDS = 2 # linkID, endTime +_LINK_SECTIONS = frozenset({"[PIPES]", "[PUMPS]", "[VALVES]"}) +# The leak tables keep this leak's outflow for one step after its configured end. The connector keeps +# the configured end, and README.md lists the difference. +_END_ONE_STEP_LATE = frozenset({"p800"}) + + +def _parse_time(text: str, where: str) -> datetime: + """Read a naive release time as UTC; the release states no time zone. + + Returns: + The time, in UTC. + + Raises: + TimeNetConnectorError: If the text is not a ``YYYY-MM-DD HH:MM`` time on the 5 minute grid. + """ + try: + moment = datetime.strptime(text.strip(), _TIME_FORMAT).replace(tzinfo=UTC) + except ValueError as error: + raise TimeNetConnectorError(f"{where}: {text!r} is not a {_TIME_FORMAT} time") from error + if moment.minute % 5: + raise TimeNetConnectorError(f"{where}: {text!r} is not on the 5 minute grid") + return moment + + +@dataclass(frozen=True) +class Table: + """One CSV of the release: its first timestamp and one float64 column per sensor, in file order.""" + + start: datetime + columns: dict[str, np.ndarray] + + @property + def n_rows(self) -> int: + """The number of 5 minute steps.""" + return len(next(iter(self.columns.values()))) + + +def read_table(path: Path) -> Table: + """Parse one semicolon-separated, decimal-comma CSV and check its time grid and values. + + Returns: + The table, with every value column as float64. + + Raises: + TimeNetConnectorError: If the file has no value column, a cell that is not a timestamp or a + number, a step that is not 5 minutes, a missing value, or a value that is not finite. + """ + with path.open(encoding="utf-8-sig") as handle: + header = handle.readline().strip().split(";") + if header[0] != "Timestamp" or len(header) == 1: + raise TimeNetConnectorError(f"{path.name}: expected a Timestamp column and sensor columns, got {header}") + names = header[1:] + if len(set(names)) != len(names): + raise TimeNetConnectorError(f"{path.name}: repeats a column name") + try: + table = pacsv.read_csv( + path, + parse_options=pacsv.ParseOptions(delimiter=";"), + convert_options=pacsv.ConvertOptions( + decimal_point=",", + column_types={"Timestamp": pa.timestamp("s"), **dict.fromkeys(names, pa.float64())}, + timestamp_parsers=["%Y-%m-%d %H:%M:%S"], + ), + ) + except pa.ArrowInvalid as error: + raise TimeNetConnectorError(f"{path.name}: cannot be read as timestamps and numbers: {error}") from error + times = table.column("Timestamp").to_numpy().astype("datetime64[s]").astype(np.int64) + if len(times) == 0: + raise TimeNetConnectorError(f"{path.name}: holds no rows") + steps = np.diff(times) + if (steps != STEP.total_seconds()).any(): + bad = int(np.flatnonzero(steps != STEP.total_seconds())[0]) + raise TimeNetConnectorError(f"{path.name}: row {bad + 2} is not 5 minutes after the row before it") + columns = {} + for name in names: + column = table.column(name) + if column.null_count: + raise TimeNetConnectorError(f"{path.name}: column {name} has {column.null_count} missing values") + values = column.to_numpy() + if not np.isfinite(values).all(): + raise TimeNetConnectorError(f"{path.name}: column {name} holds a value that is not finite") + columns[name] = values + return Table(start=datetime.fromtimestamp(int(times[0]), tz=UTC), columns=columns) + + +@dataclass(frozen=True) +class Leak: + """One leak of the configuration, with its outflow from the leak tables. + + ``end`` is the last 5 minute step with the leak, as the configuration and the leak tables write it. + A leak that is never repaired ends at the last step of the recording. + """ + + pipe: str + start: datetime + end: datetime + diameter_m: float + leak_type: str + peak: datetime + repaired: bool + + +def _parse_leak(line: object, where: str) -> Leak: + """Read one ``linkID, startTime, endTime, leakDiameter (m), leakType, peakTime`` line. + + Returns: + The leak, not yet marked as repaired. + + Raises: + TimeNetConnectorError: If the line does not hold six fields of the stated kinds. + """ + fields = [field.strip() for field in str(line).split(",")] + if len(fields) != _LEAK_FIELDS: + raise TimeNetConnectorError(f"{where}: leak line {line!r} does not hold six fields") + pipe, start, end, diameter, leak_type, peak = fields + if leak_type not in LEAK_TYPES: + raise TimeNetConnectorError(f"{where}: leak {pipe} has type {leak_type!r}, not one of {LEAK_TYPES}") + try: + diameter_m = float(diameter) + except ValueError as error: + raise TimeNetConnectorError(f"{where}: leak {pipe} has diameter {diameter!r}") from error + leak = Leak( + pipe=pipe, + start=_parse_time(start, where), + end=_parse_time(end, where), + diameter_m=diameter_m, + leak_type=leak_type, + peak=_parse_time(peak, where), + repaired=False, + ) + if not leak.start <= leak.peak <= leak.end: + raise TimeNetConnectorError(f"{where}: leak {pipe} peaks outside its start and end") + if leak.leak_type == "abrupt" and leak.peak != leak.start: + raise TimeNetConnectorError(f"{where}: abrupt leak {pipe} peaks after its start") + return leak + + +class Network: + """The links of ``L-TOWN.inp`` as an undirected graph, to find the pressure sensor nearest a pipe.""" + + def __init__(self, path: Path) -> None: + """Read the ``[PIPES]``, ``[PUMPS]`` and ``[VALVES]`` sections: a link id and its two nodes. + + Raises: + TimeNetConnectorError: If a link line names fewer than two nodes or a link id repeats. + """ + self.links: dict[str, tuple[str, str]] = {} + self.pipes: set[str] = set() + section = "" + for line in path.read_text(encoding="utf-8", errors="strict").splitlines(): + text = line.split(";", 1)[0].strip() + if not text: + continue + if text.startswith("["): + section = text.upper() + continue + if section not in _LINK_SECTIONS: + continue + link, *nodes = text.split() + if len(nodes) < len(("node1", "node2")): + raise TimeNetConnectorError(f"{path.name}: link line {line!r} names fewer than two nodes") + if link in self.links: + raise TimeNetConnectorError(f"{path.name}: link {link} appears twice") + self.links[link] = (nodes[0], nodes[1]) + if section == "[PIPES]": + self.pipes.add(link) + self._neighbours: dict[str, set[str]] = {} + for a, b in self.links.values(): + self._neighbours.setdefault(a, set()).add(b) + self._neighbours.setdefault(b, set()).add(a) + + @property + def nodes(self) -> set[str]: + """Every node that a link touches.""" + return set(self._neighbours) + + def nearest(self, pipe: str, sensors: list[str]) -> tuple[str, int]: + """Find the sensor node with the fewest links to either end of ``pipe``. + + Ties go to the sensor listed first, so the answer does not depend on set order. + + Returns: + The sensor and its number of links from the nearer end of the pipe. + + Raises: + TimeNetConnectorError: If no sensor is connected to the pipe. + """ + hops = dict.fromkeys(self.links[pipe], 0) + queue = deque(hops) + while queue: + node = queue.popleft() + for neighbour in self._neighbours[node]: + if neighbour not in hops: + hops[neighbour] = hops[node] + 1 + queue.append(neighbour) + reachable = [sensor for sensor in sensors if sensor in hops] + if not reachable: + raise TimeNetConnectorError(f"no pressure sensor is connected to pipe {pipe}") + best = min(reachable, key=lambda sensor: (hops[sensor], sensors.index(sensor))) + return best, hops[best] + + +class Release: + """The parsed release: the SCADA tables of both years, the leaks and the network. + + Reading checks that the two years join on one 5 minute grid with the same sensors, that the + configuration, the leak tables and the repair report agree, and that every sensor and leak pipe + is in the network model. + """ + + def __init__(self, root: Path) -> None: + """Read and check the release in ``root``. + + Raises: + TimeNetConnectorError: If a file is missing or the files disagree. + """ + self.root = root + config = self._yaml("dataset_configuration.yaml") + if not isinstance(config, dict): + raise TimeNetConnectorError("dataset_configuration.yaml is not a mapping") + self.sensors: dict[str, list[str]] = {} + for kind, (_, key) in SCADA_FILES.items(): + listed = config.get(key) + if not isinstance(listed, list) or not listed: + raise TimeNetConnectorError(f"dataset_configuration.yaml lists no {key}") + self.sensors[kind] = [str(sensor) for sensor in listed] + if len(set(self.sensors[kind])) != len(self.sensors[kind]): + raise TimeNetConnectorError(f"dataset_configuration.yaml lists a {key} entry twice") + self.scada = {kind: self._join(name, self.sensors[kind]) for kind, (name, _) in SCADA_FILES.items()} + self.start = self.scada["pressure"].start + self.n_steps = self.scada["pressure"].n_rows + for kind, table in self.scada.items(): + if table.start != self.start or table.n_rows != self.n_steps: + raise TimeNetConnectorError(f"the {kind} tables do not cover the pressure tables' time range") + times = config.get("times") + if not isinstance(times, dict): + raise TimeNetConnectorError("dataset_configuration.yaml has no times mapping") + if _parse_time(str(times.get("StartTime")), "times") != self.start or ( + _parse_time(str(times.get("EndTime")), "times") != self.end + ): + raise TimeNetConnectorError("dataset_configuration.yaml times differ from the SCADA tables") + self.leak_outflow = self._join_leaks() + # The total outflow of every leak at each step, in m3/h. + self.total_outflow = np.sum(list(self.leak_outflow.values()), axis=0) + self.leaks = self._read_leaks(config) + self.network = Network(self._path("L-TOWN.inp")) + self._check_network() + + @property + def end(self) -> datetime: + """The last 5 minute step of the recording.""" + return self.start + STEP * (self.n_steps - 1) + + def step_of(self, moment: datetime) -> int: + """The index of ``moment`` on the recording's 5 minute grid.""" + return int((moment - self.start) / STEP) + + def micros(self, moment: datetime) -> int: + """The offset of ``moment`` from the start of the recording, in whole microseconds.""" + return (moment - self.start) // timedelta(microseconds=1) + + def _path(self, name: str) -> Path: + path = self.root / name + if not path.is_file(): + raise TimeNetConnectorError(f"the release in {self.root} is missing {name}") + return path + + def _yaml(self, name: str) -> object: + try: + return yaml.safe_load(self._path(name).read_text(encoding="utf-8")) + except yaml.YAMLError as error: + raise TimeNetConnectorError(f"{name} is not valid YAML") from error + + def _join(self, name: str, sensors: list[str]) -> Table: + """Join the 2018 and 2019 tables of one sensor type into one recording. + + Returns: + One table over both years, with the sensors in configuration order. + + Raises: + TimeNetConnectorError: If the years name other sensors than the configuration lists, or + the second year does not start 5 minutes after the first ends. + """ + tables = [read_table(self._path(f"{year}_{name}.csv")) for year in YEARS] + for year, table in zip(YEARS, tables, strict=True): + if set(table.columns) != set(sensors): + raise TimeNetConnectorError( + f"{year}_{name}.csv columns differ from the sensors that dataset_configuration.yaml lists" + ) + first, second = tables + if second.start != first.start + STEP * first.n_rows: + raise TimeNetConnectorError(f"{YEARS[1]}_{name}.csv does not continue {YEARS[0]}_{name}.csv") + return Table( + start=first.start, + columns={sensor: np.concatenate([first.columns[sensor], second.columns[sensor]]) for sensor in sensors}, + ) + + def _join_leaks(self) -> dict[str, np.ndarray]: + """Join the two leak tables into one outflow series per pipe, zero where a year does not list it. + + Returns: + Each leaking pipe's outflow over both years, in m3/h. + + Raises: + TimeNetConnectorError: If a leak table does not cover its year of the SCADA tables. + """ + tables = [read_table(self._path(f"{year}_Leakages.csv")) for year in YEARS] + offset = 0 + outflow: dict[str, np.ndarray] = {} + for year, table in zip(YEARS, tables, strict=True): + if table.start != self.start + STEP * offset: + raise TimeNetConnectorError(f"{year}_Leakages.csv does not line up with the SCADA tables") + for pipe, values in table.columns.items(): + series = outflow.setdefault(pipe, np.zeros(self.n_steps)) + series[offset : offset + table.n_rows] = values + offset += table.n_rows + if offset != self.n_steps: + raise TimeNetConnectorError("the leak tables do not cover the SCADA tables") + return outflow + + def _read_repairs(self) -> dict[str, datetime]: + """Read ``2018_Fixed_Leakages_Report.txt``: one ``linkID, endTime`` line per repaired leak. + + Returns: + The repair time of each listed pipe. + + Raises: + TimeNetConnectorError: If the report is not a list, a line is malformed, or a pipe repeats. + """ + # A report of comment lines only, with no repair, reads as an empty YAML document. + report = self._yaml("2018_Fixed_Leakages_Report.txt") or [] + if not isinstance(report, list): + raise TimeNetConnectorError("2018_Fixed_Leakages_Report.txt is not a list of repairs") + repairs = {} + for line in report: + fields = [field.strip() for field in str(line).split(",")] + if len(fields) != _REPAIR_FIELDS: + raise TimeNetConnectorError(f"2018_Fixed_Leakages_Report.txt: line {line!r} is not 'linkID, endTime'") + if fields[0] in repairs: + raise TimeNetConnectorError(f"2018_Fixed_Leakages_Report.txt repairs {fields[0]} twice") + repairs[fields[0]] = _parse_time(fields[1], "2018_Fixed_Leakages_Report.txt") + return repairs + + def _read_leaks(self, config: dict) -> list[Leak]: + """Read the configured leaks and check them against the leak tables and the repair report. + + Returns: + The leaks in order of their start, each marked as repaired or not. + + Raises: + TimeNetConnectorError: If a configured leak is outside the recording or its leak table, + a leak table has a pipe the configuration lacks, or a repair time disagrees. + """ + # The list opens with a comment line that names the fields, which YAML reads as a null item. + leakages = config.get("leakages") + if not isinstance(leakages, list): + raise TimeNetConnectorError("dataset_configuration.yaml has no leakages list") + lines = [line for line in leakages if line is not None] + leaks = [_parse_leak(line, "dataset_configuration.yaml") for line in lines] + if len({leak.pipe for leak in leaks}) != len(leaks): + raise TimeNetConnectorError("dataset_configuration.yaml lists a pipe in two leaks") + if {leak.pipe for leak in leaks} != set(self.leak_outflow): + raise TimeNetConnectorError("the leak tables and dataset_configuration.yaml name different pipes") + repairs = self._read_repairs() + checked = [] + for leak in leaks: + if not self.start <= leak.start <= leak.end <= self.end: + raise TimeNetConnectorError(f"leak {leak.pipe} is outside the recording") + if leak.pipe in repairs and repairs[leak.pipe] != leak.end: + raise TimeNetConnectorError(f"leak {leak.pipe}: the repair report and the configuration disagree") + # The report covers the historical year: every leak that ends within 2018 must be in it. + if leak.end.year == YEARS[0] and leak.pipe not in repairs: + raise TimeNetConnectorError(f"leak {leak.pipe} ends in {YEARS[0]} but the repair report lacks it") + outflow = self.leak_outflow[leak.pipe] + active = np.flatnonzero(outflow > 0) + last_allowed = self.step_of(leak.end) + (1 if leak.pipe in _END_ONE_STEP_LATE else 0) + if len(active) == 0 or active[0] < self.step_of(leak.start) or active[-1] > last_allowed: + raise TimeNetConnectorError(f"leak {leak.pipe}: its outflow lies outside its configured start and end") + checked.append(replace(leak, repaired=leak.end < self.end)) + if set(repairs) - {leak.pipe for leak in checked if leak.repaired}: + raise TimeNetConnectorError("2018_Fixed_Leakages_Report.txt repairs a leak the configuration keeps open") + return sorted(checked, key=lambda leak: (leak.start, leak.pipe)) + + def _check_network(self) -> None: + """Check that every sensor and leak pipe is in the network model. + + Raises: + TimeNetConnectorError: If one is missing. + """ + nodes = self.network.nodes + for kind in ("pressure", "tank_level", "amr_demand"): + missing = [sensor for sensor in self.sensors[kind] if sensor not in nodes] + if missing: + raise TimeNetConnectorError(f"L-TOWN.inp has no node for the {kind} sensors {missing}") + missing = [sensor for sensor in self.sensors["flow"] if sensor not in self.network.links] + if missing: + raise TimeNetConnectorError(f"L-TOWN.inp has no link for the flow sensors {missing}") + missing = [leak.pipe for leak in self.leaks if leak.pipe not in self.network.pipes] + if missing: + raise TimeNetConnectorError(f"L-TOWN.inp has no pipe for the leaks {missing}") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py new file mode 100644 index 000000000..f079d9bce --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py @@ -0,0 +1,269 @@ +"""Cut the recording into two-week windows and state what the leak list says about each one. + +A window is 14 days: a context week, then a target week. It starts every 7 days, so consecutive +windows share a week. Each window gives four tasks about the target week, all scoped to the window: +the types of the leaks that start in it, when they start, the mean leak outflow, and a short text +that describes every leak that starts or is repaired in it. The answers come from the leak list, +the leak tables and the network model by fixed rules. No model writes them. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import datetime, timedelta +from itertools import pairwise + +from timenet.dataset import Record +from timenet.errors import TimeNetConnectorError +from timenet.json import JsonMapping, JsonValue +from timenet.types import ( + AnswerTask, + ClassificationTask, + Label, + LabelSet, + LocalizationMode, + ScalarPredictionTask, + Task, + TemporalLocalizationTask, + TimeInterval, + TimePoint, +) +from timenet.types.splits import Split +from timenet_connectors.datasets.kios.battledim.source import STEP, Leak, Release +from timenet_connectors.splitting import StratifiedSplitter + + +WINDOW = timedelta(days=14) +TARGET = timedelta(days=7) +STRIDE = timedelta(days=7) +# Four ISO weeks make one calendar block. Windows whose target weeks share a block share a partition +# in both years, because 2019 repeats most of the 2018 demand series (see README.md). +WEEKS_PER_BLOCK = 4 +# Below this outflow, in m3/h, the text keeps the leak tables' second decimal. +_SMALL_OUTFLOW = 0.1 + +NEW_LEAK_TYPES = LabelSet( + name="new_leak_types", + labels=( + Label(name="abrupt", description="A leak that appears at full size at its start."), + Label(name="incipient", description="A leak that grows from its start until its peak time."), + ), + multi_label=True, +) +EXPLAIN_PROMPT = ( + "Describe every leak that started or was repaired in the last 7 days of this two-week window: " + "the pipe, when, how it grew, its size, and the nearest pressure sensor." +) +LOCATE_PROMPT = "Locate the start of every leak that began in the last 7 days of this two-week window." +CLASSIFY_PROMPT = "Which types of leak began in the last 7 days of this two-week window?" +OUTFLOW_PROMPT = "What was the mean total leak outflow in the last 7 days of this two-week window?" + + +@dataclass(frozen=True) +class Window: + """One two-week window and what the leak list says about its target week.""" + + index: int + start: datetime + new: tuple[Leak, ...] + repaired: tuple[Leak, ...] + + @property + def target_start(self) -> datetime: + """The first step of the target week.""" + return self.start + WINDOW - TARGET + + @property + def end(self) -> datetime: + """The exclusive end of the window.""" + return self.start + WINDOW + + @property + def block(self) -> int: + """The calendar block of the target week: its ISO week, counted in blocks of four.""" + return (self.target_start.isocalendar().week - 1) // WEEKS_PER_BLOCK + + +def windows(release: Release) -> list[Window]: + """List every window that fits in the recording, in time order. + + Returns: + The windows, each with the leaks that start or are repaired in its target week. + + Raises: + TimeNetConnectorError: If a leak starts or is repaired outside every target week, so that no + task would mention it. + """ + found = [] + start = release.start + while start + WINDOW <= release.end + STEP: + frame = Window(index=len(found), start=start, new=(), repaired=()) + found.append( + replace( + frame, + new=tuple(leak for leak in release.leaks if frame.target_start <= leak.start < frame.end), + # The release writes a repair time as the leak's configured end. + repaired=tuple( + leak for leak in release.leaks if leak.repaired and frame.target_start <= leak.end < frame.end + ), + ) + ) + start += STRIDE + for leak in release.leaks: + if sum(leak in window.new for window in found) != 1: + raise TimeNetConnectorError(f"leak {leak.pipe} does not start in exactly one target week") + if leak.repaired and sum(leak in window.repaired for window in found) != 1: + raise TimeNetConnectorError(f"the repair of leak {leak.pipe} is not in exactly one target week") + return found + + +def split_windows(found: list[Window]) -> dict[int, Split]: + """Assign each window to a partition by the calendar block of its target week. + + Returns: + The partition of each window, keyed by window index. + """ + splitter = StratifiedSplitter( + train_folds=8, + validation_folds=1, + test_folds=1, + stratify_by=lambda window: bool(window.new), + group_by=lambda window: window.block, + seed=42, + ) + return {window.index: split for split, members in splitter.split(found).items() for window in members} + + +def embargoed(found: list[Window], split_of: dict[int, Split]) -> set[int]: + """Find the windows whose context week is a target week of another held-out partition. + + Windows overlap by one week: a window's context week is the target week of the window before + it. A training window after a test target week would show training the signals that the test + tasks ask about, so it gets no tasks. The same holds between validation and test. + + Returns: + The indices of the windows that get no tasks. + """ + held_out = {Split.VALIDATION, Split.TEST} + return { + window.index + for previous, window in pairwise(found) + if split_of[previous.index] in held_out and split_of[previous.index] != split_of[window.index] + } + + +def _rate(outflow: float) -> str: + """Format an outflow in m3/h so that the leak tables' smallest values stay visible.""" + return f"{outflow:.2f}" if outflow < _SMALL_OUTFLOW else f"{outflow:.1f}" + + +def _outflow(release: Release, leak: Leak, start: datetime, end: datetime) -> float: + """The largest outflow of ``leak`` in ``[start, end)``, in m3/h.""" + return float(release.leak_outflow[leak.pipe][release.step_of(start) : release.step_of(end)].max()) + + +def _hops(count: int) -> str: + return f"{count} link{'s' if count != 1 else ''}" + + +def explain(release: Release, window: Window) -> str: + """Write the text answer for one window from the leak list, the leak tables and the network. + + Returns: + One sentence per new or repaired leak, then the leaks still active at the end of the window. + """ + sentences = [] + for leak in window.new: + sensor, hops = release.network.nearest(leak.pipe, release.sensors["pressure"]) + if leak.leak_type == "abrupt": + kind, growth = "A sudden leak", "" + elif leak.peak < window.end: + kind, growth = "A gradually growing leak", f" and peaked on {leak.peak:%Y-%m-%d %H:%M}" + else: + # A later peak is not in the window, so the text does not state it. + kind, growth = "A gradually growing leak", " and was still growing at the end of this window" + outflow = _outflow(release, leak, leak.start, window.end) + # The leak tables round an incipient leak's first hours to zero outflow. + flow = f"an outflow of up to {_rate(outflow)} m3/h" if outflow > 0 else "no outflow in the leak tables yet" + sentences.append( + f"{kind} started on pipe {leak.pipe} on {leak.start:%Y-%m-%d %H:%M}{growth}, " + f"with a hole of {leak.diameter_m * 1000:.1f} mm and {flow} in this window. " + f"The nearest pressure sensor is {sensor}, {_hops(hops)} away." + ) + for leak in window.repaired: + last_hour = leak.end + STEP - timedelta(hours=1) + sentences.append( + f"The leak on pipe {leak.pipe} was repaired on {leak.end:%Y-%m-%d %H:%M}; " + f"its outflow before the repair was {_outflow(release, leak, last_hour, leak.end + STEP):.1f} m3/h." + ) + if not sentences: + sentences.append("No leak started or was repaired in the last 7 days.") + # Active means the leak list's interval holds the last step, whatever the rounded outflow says. + last = window.end - STEP + active = [leak.pipe for leak in release.leaks if leak.start <= last <= leak.end] + total = sum(float(release.leak_outflow[pipe][release.step_of(last)]) for pipe in active) + sentences.append( + f"At the end of the window {len(active)} leak{'s are' if len(active) != 1 else ' is'} active, " + f"with a total outflow of {total:.1f} m3/h." + ) + return " ".join(sentences) + + +def window_tasks(release: Release, record: Record, window: Window, split: Split) -> list[Task]: + """Build the four tasks of one window, all scoped to the window. + + Returns: + The classification, localization, outflow and text tasks, in that order. + """ + scope = TimeInterval.micros(release.micros(window.start), release.micros(window.end)) + target = slice(release.step_of(window.target_start), release.step_of(window.end)) + outflow = release.total_outflow[target] + new_leaks: list[JsonValue] = [leak.pipe for leak in window.new] + repaired_leaks: list[JsonValue] = [leak.pipe for leak in window.repaired] + metadata: JsonMapping = { + "window_index": window.index, + "window_start": f"{window.start:%Y-%m-%d %H:%M}", + "target_week_start": f"{window.target_start:%Y-%m-%d %H:%M}", + "calendar_block": window.block, + "new_leaks": new_leaks, + "repaired_leaks": repaired_leaks, + } + inputs = (record,) + return [ + ClassificationTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=CLASSIFY_PROMPT, + targets=tuple(sorted({leak.leak_type for leak in window.new})), + label_set=NEW_LEAK_TYPES, + ), + TemporalLocalizationTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=LOCATE_PROMPT, + mode=LocalizationMode.EXHAUSTIVE, + targets=tuple(TimePoint.micros(release.micros(leak.start)) for leak in window.new), + ), + ScalarPredictionTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=OUTFLOW_PROMPT, + targets=(round(float(outflow.mean()), 3),), + unit="m^3/h", + target_name="mean_leak_outflow", + ), + AnswerTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=EXPLAIN_PROMPT, + targets=(explain(release, window),), + ), + ] diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py new file mode 100644 index 000000000..9b7fb303a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py @@ -0,0 +1,328 @@ +from collections import Counter +from itertools import pairwise +from pathlib import Path + +import numpy as np +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, open_local_version +from timenet.types import AnswerTask, ClassificationTask, ScalarPredictionTask, TemporalLocalizationTask +from timenet.types.splits import Split +from timenet.types.text import read_value +from timenet_connectors.datasets.kios.battledim import BattLeDIMConnector +from timenet_connectors.datasets.kios.battledim.connector import BattLeDIMSource +from timenet_connectors.datasets.kios.battledim.source import Release +from timenet_connectors.datasets.kios.battledim.tasks import embargoed, split_windows, windows + + +# A synthetic release shaped exactly like the Zenodo files: semicolon-separated CSVs with a decimal +# comma on a 5 minute grid, the configuration YAML, the repair report and an EPANET network. The +# values and the network are hand-made for the tests, so the repo ships no dataset bytes. Two full +# years are needed because the split groups windows into calendar blocks and asks for 10 folds. +STEPS_PER_YEAR = 365 * 288 +_TIMES = (np.datetime64("2018-01-01T00:00") + np.arange(2 * STEPS_PER_YEAR) * np.timedelta64(5, "m")).astype(str) +TIMES = [time.replace("T", " ") + ":00" for time in _TIMES] + +CONFIG = """\ +Comments: + - Synthetic configuration for tests +Network: + filename: L-TOWN_Real.inp +times: + StartTime: 2018-01-01 00:00 + EndTime: 2019-12-31 23:55 +leakages: +- # linkID, startTime, endTime, leakDiameter (m), leakType, peakTime +- p3, 2018-03-05 15:45, 2018-03-23 10:25, 0.022916, abrupt, 2018-03-05 15:45 +- p5, 2018-11-01 00:00, 2019-12-31 23:55, 0.010000, incipient, 2018-11-15 00:00 +- p6, 2019-05-01 12:00, 2019-12-31 23:55, 0.015000, abrupt, 2019-05-01 12:00 +pressure_sensors: +- n1 +- n4 +flow_sensors: +- PUMP_1 +- p227 +level_sensors: +- T1 +amrs: +- n2 +- n3 +""" +REPORT = "# FIXED LEAKAGES\n# linkID, endTime\n- p3, 2018-03-23 10:25\n" +NETWORK = """\ +[JUNCTIONS] + n1 10 0 ; +[PIPES] +;ID Node1 Node2 Length Diameter Roughness MinorLoss Status + p227 R1 n1 10 200 140 0 Open ; + p2 n1 n2 10 150 140 0 Open ; + p3 n2 n3 10 150 140 0 Open ; + p4 n3 n4 10 150 140 0 Open ; + p5 n4 n5 10 150 140 0 Open ; + p6 n5 n6 10 150 140 0 Open ; +[PUMPS] + PUMP_1 n2 T1 HEAD 1 ;FLOW SENSOR +[VALVES] +[END] +""" + + +def _step(text: str) -> int: + return TIMES.index(text + ":00") + + +def _outflows() -> dict[str, np.ndarray]: + outflow = {pipe: np.zeros(2 * STEPS_PER_YEAR) for pipe in ("p3", "p5", "p6")} + outflow["p3"][_step("2018-03-05 15:45") : _step("2018-03-23 10:25") + 1] = 28.4 + start, peak = _step("2018-11-01 00:00"), _step("2018-11-15 00:00") + outflow["p5"][start:peak] = np.linspace(0.0, 5.0, peak - start) + outflow["p5"][peak:] = 5.0 + outflow["p6"][_step("2019-05-01 12:00") :] = 10.5 + return outflow + + +def _write_csv(path: Path, columns: dict[str, np.ndarray], rows: slice) -> None: + lines = ["Timestamp;" + ";".join(columns)] + formatted = [[f"{value:.2f}".replace(".", ",") for value in values[rows]] for values in columns.values()] + for index, time in enumerate(TIMES[rows]): + lines.append(";".join([time, *(column[index] for column in formatted)])) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def write_release(root: Path) -> Path: + root.mkdir(parents=True, exist_ok=True) + clock = np.arange(2 * STEPS_PER_YEAR) + daily = np.sin(2 * np.pi * clock / 288) + scada = { + "SCADA_Pressures": {"n1": 50.0 + daily, "n4": 40.0 + 0.5 * daily}, + "SCADA_Flows": {"p227": 80.0 + 10 * daily, "PUMP_1": np.full(clock.size, 44.04)}, + "SCADA_Levels": {"T1": 3.0 + 0.5 * daily}, + "SCADA_Demands": {"n2": 100.0 + 20 * daily, "n3": np.full(clock.size, 12.5)}, + } + outflow = _outflows() + years = {2018: slice(0, STEPS_PER_YEAR), 2019: slice(STEPS_PER_YEAR, 2 * STEPS_PER_YEAR)} + for year, rows in years.items(): + for name, columns in scada.items(): + _write_csv(root / f"{year}_{name}.csv", columns, rows) + active = {pipe: values for pipe, values in outflow.items() if values[rows].any()} + _write_csv(root / f"{year}_Leakages.csv", active, rows) + (root / "dataset_configuration.yaml").write_text(CONFIG, encoding="utf-8") + (root / "2018_Fixed_Leakages_Report.txt").write_text(REPORT, encoding="utf-8") + (root / "L-TOWN.inp").write_text(NETWORK, encoding="utf-8") + return root + + +@pytest.fixture(scope="module") +def release_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + return write_release(tmp_path_factory.mktemp("battledim")) + + +@pytest.fixture(scope="module") +def dataset(release_dir: Path): + return BattLeDIMConnector().convert([BattLeDIMSource(release_dir)]) + + +def test_record_holds_every_sensor_on_one_grid(dataset) -> None: + (record,) = dataset.records + sources = {source.name: [signal.name for signal in source.signals] for source in record.sources} + assert sources == { + "Pressure sensors": ["n1", "n4"], + "Flow sensors": ["PUMP_1", "p227"], + "Tank level sensor": ["T1"], + "Automated meter readings": ["n2", "n3"], + } + pressure = record.sources[0].signals[0] + assert pressure.n_values == 2 * STEPS_PER_YEAR + assert str(pressure.spec.unit_value) == "meter" + assert pressure.time_axis.period_us == 300_000_000 + values = pressure.to_numpy() + # The decimal comma parses, and the second year continues the first. + assert values[0] == pytest.approx(50.0) + assert values[STEPS_PER_YEAR + 72] == pytest.approx(51.0, abs=0.01) + flow = record.sources[1].signals[0] + assert str(flow.spec.unit_value) == "meter ** 3 / hour" + assert flow.to_numpy()[5] == pytest.approx(44.04) + + +def test_leaks_are_dataset_annotations_not_record_labels(dataset) -> None: + (record,) = dataset.records + # The record every task reads carries no answers. + assert record.annotations == () + leaks = [annotation for annotation in dataset.annotations if annotation.key == "leak"] + assert [annotation.value for annotation in leaks] == ["p3", "p5", "p6"] + p3 = leaks[0].metadata + assert p3["leak_type"] == "abrupt" + assert p3["start_us"] == _step("2018-03-05 15:45") * 300_000_000 + # The interval ends after the repair step, the configured end. + assert p3["end_exclusive_us"] == (_step("2018-03-23 10:25") + 1) * 300_000_000 + assert p3["repaired"] is True + assert p3["peak_outflow_m3h"] == pytest.approx(28.4) + assert leaks[1].metadata["peak"] == "2018-11-15 00:00" + assert leaks[1].metadata["repaired"] is False + + +def test_windows_give_four_tasks_each(dataset) -> None: + tasks = list(dataset.get_all()) + # 730 days hold 103 two-week windows that start every 7 days. Windows after a held-out target + # week of another partition get no tasks (see the embargo test). + windows = {task.metadata["window_index"] for task in tasks} + assert 90 <= len(windows) < 103 + assert Counter(type(task) for task in tasks) == { + ClassificationTask: len(windows), + TemporalLocalizationTask: len(windows), + ScalarPredictionTask: len(windows), + AnswerTask: len(windows), + } + first = tasks[0] + assert first.scope.start_us == 0 + assert first.scope.end_us == 14 * 24 * 3600 * 1_000_000 + + +def _tasks_of_week(dataset, target_week_start: str) -> list: + return [task for task in dataset.get_all() if task.metadata["target_week_start"] == target_week_start] + + +def test_a_new_abrupt_leak_is_classified_located_and_explained(dataset) -> None: + classify, locate, outflow, answer = _tasks_of_week(dataset, "2018-03-05 00:00") + assert classify.targets == ("abrupt",) + assert [point.start_us for point in locate.targets] == [_step("2018-03-05 15:45") * 300_000_000] + assert outflow.target_name == "mean_leak_outflow" + assert outflow.targets[0] == pytest.approx(28.4 * (7 * 288 - _step("2018-03-05 15:45") % 2016) / 2016, rel=1e-3) + text = read_value(answer.targets[0]) + assert text.startswith("A sudden leak started on pipe p3 on 2018-03-05 15:45, with a hole of 22.9 mm") + # p3 joins n2 and n3. n1 and n4 are both one link away, so the first listed sensor wins. + assert "The nearest pressure sensor is n1, 1 link away." in text + assert text.endswith("At the end of the window 1 leak is active, with a total outflow of 28.4 m3/h.") + + +def test_a_repair_and_a_quiet_week_are_explained(dataset) -> None: + (*_, answer) = _tasks_of_week(dataset, "2018-03-19 00:00") + assert "The leak on pipe p3 was repaired on 2018-03-23 10:25; its outflow before the repair was 28.4 m3/h." in ( + read_value(answer.targets[0]) + ) + classify, locate, _, answer = _tasks_of_week(dataset, "2018-07-02 00:00") + assert classify.targets == () + assert locate.targets == () + assert read_value(answer.targets[0]).startswith("No leak started or was repaired in the last 7 days.") + + +def test_a_growing_leak_states_no_peak_after_the_window(dataset) -> None: + # p5 starts on 2018-11-01 and peaks on 2018-11-15, after this window ends on 2018-11-05. + classify, _, _, answer = _tasks_of_week(dataset, "2018-10-29 00:00") + assert classify.targets == ("incipient",) + text = read_value(answer.targets[0]) + assert text.startswith( + "A gradually growing leak started on pipe p5 on 2018-11-01 00:00 and was still growing at the end of " + "this window, with a hole of 10.0 mm and an outflow of up to" + ) + assert "2018-11-15" not in text + assert "At the end of the window 1 leak is active" in text + + +def test_split_keeps_each_calendar_block_in_one_partition(dataset) -> None: + tasks = list(dataset.get_all()) + partitions: dict[int, set] = {} + for task in tasks: + partitions.setdefault(task.metadata["calendar_block"], set()).add(task.split) + assert all(len(splits) == 1 for splits in partitions.values()) + counts = Counter(task.split for task in tasks) + assert sum(counts.values()) == len(tasks) + assert len(counts) == 3 + + +def _answers(dataset) -> list[str]: + answers = [] + for task in dataset.get_all(): + if isinstance(task, AnswerTask): + assert task.targets is not None + answers.append(read_value(task.targets[0])) + return answers + + +def test_dataset_survives_storage_round_trip(dataset, tmp_path: Path) -> None: + registry = LocalRegistry(tmp_path / "registry") + registry.put_dataset(dataset) + with TimeFReader(open_local_version(registry.root / "kios/battledim/1.0.0")) as reader: + restored = reader.read() + (record,) = restored.records + assert record.sources[3].signals[1].to_numpy()[0] == pytest.approx(12.5) + assert record.annotations == () + assert [a.metadata["start"] for a in restored.annotations if a.key == "leak"] == [ + "2018-03-05 15:45", + "2018-11-01 00:00", + "2019-05-01 12:00", + ] + # The reader returns the tasks in the order the connector added them. + assert [task.split for task in restored.get_all()] == [task.split for task in dataset.get_all()] + assert _answers(restored) == _answers(dataset) + + +def test_no_window_sees_a_held_out_target_week_of_another_partition(release_dir: Path, dataset) -> None: + release = Release(release_dir) + found = windows(release) + split_of = split_windows(found) + skipped = embargoed(found, split_of) + assert skipped + kept = {task.metadata["window_index"] for task in dataset.get_all()} + assert kept == {window.index for window in found} - skipped + for previous, window in pairwise(found): + # A window's context week is the target week of the window before it. + assert previous.target_start == window.start + if window.index in kept and split_of[previous.index] in {Split.VALIDATION, Split.TEST}: + assert split_of[previous.index] == split_of[window.index] + + +def test_nearest_sensor_prefers_the_nearer_pipe_end(release_dir: Path) -> None: + network = Release(release_dir).network + assert network.nearest("p5", ["n1", "n4"]) == ("n4", 0) + assert network.nearest("p6", ["n1", "n4"]) == ("n4", 1) + + +def _corrupt(release_dir: Path, tmp_path: Path, name: str, edit) -> Path: + copy = tmp_path / "release" + copy.mkdir() + for path in release_dir.iterdir(): + (copy / path.name).write_bytes(path.read_bytes()) + target = copy / name + target.write_text(edit(target.read_text(encoding="utf-8")), encoding="utf-8") + return copy + + +@pytest.mark.parametrize( + ("name", "edit", "message"), + [ + ( + "2018_SCADA_Pressures.csv", + lambda text: text.replace("2018-01-01 00:05:00;", "2018-01-01 00:06:00;"), + "5 minutes", + ), + ("2019_SCADA_Levels.csv", lambda text: text.replace(";3,00\n", ";\n", 1), "missing values"), + ("dataset_configuration.yaml", lambda text: text.replace("- p6,", "- p4,"), "different pipes"), + ("2018_Fixed_Leakages_Report.txt", lambda text: text.replace("10:25", "10:30"), "disagree"), + ("L-TOWN.inp", lambda text: text.replace(" p5 n4 n5", " p9 n4 n5"), "no pipe for the leaks"), + ("2018_SCADA_Flows.csv", lambda text: text.replace(";44,04", ";abc", 1), "cannot be read"), + ("2018_Fixed_Leakages_Report.txt", lambda text: text.replace("- p3, 2018-03-23 10:25\n", ""), "lacks it"), + ("2018_Fixed_Leakages_Report.txt", lambda text: text + "- p3, 2018-03-23 10:25\n", "twice"), + ("dataset_configuration.yaml", lambda text: text.replace("- n4\nflow", "- n1\nflow"), "twice"), + ], +) +def test_inconsistent_release_raises(release_dir: Path, tmp_path: Path, name: str, edit, message: str) -> None: + with pytest.raises(TimeNetConnectorError, match=message): + Release(_corrupt(release_dir, tmp_path, name, edit)) + + +def test_a_leak_outside_every_target_week_raises(release_dir: Path, tmp_path: Path) -> None: + # A leak in the first week sits only in the context week of the first window. + release = _corrupt( + release_dir, + tmp_path, + "dataset_configuration.yaml", + lambda text: text.replace( + "- p3, 2018-03-05 15:45, 2018-03-23 10:25, 0.022916, abrupt, 2018-03-05 15:45", + "- p3, 2018-01-03 15:45, 2018-03-23 10:25, 0.022916, abrupt, 2018-01-03 15:45", + ), + ) + with pytest.raises(TimeNetConnectorError, match="exactly one target week"): + BattLeDIMConnector().convert([BattLeDIMSource(release)])