From abbc63c755fd35ccb04f6745394d6b5bb7b515e7 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 20:59:10 +0100 Subject: [PATCH 01/10] feat(connectors): read the BattLeDIM release Parse the BattLeDIM 2020 SCADA and leak tables (semicolon-separated, decimal comma, 5-minute grid), the leak list of the generating configuration, the 2018 repair report and the L-Town EPANET links. Joining the two years checks the grid, the sensor lists and the agreement between the configuration, the leak tables, the repair report and the network model, and raises TimeNetConnectorError on any mismatch. --- .../datasets/kios/__init__.py | 1 + .../datasets/kios/battledim/source.py | 405 ++++++++++++++++++ 2 files changed, 406 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py new file mode 100644 index 000000000..e931dad7d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/__init__.py @@ -0,0 +1 @@ +"""Datasets published by the KIOS Research and Innovation Center of Excellence, University of Cyprus.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py new file mode 100644 index 000000000..f0c7e9e9d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py @@ -0,0 +1,405 @@ +"""Read the BattLeDIM release: the SCADA and leak tables, the leak list and the network topology. + +The release holds one semicolon-separated CSV per year and sensor type, with a decimal comma and a +naive ``Timestamp`` column on a 5 minute grid. ``dataset_configuration.yaml`` is the configuration +that generated the data. It lists every leak and the sensor of each type. ``L-TOWN.inp`` is the +EPANET model of the network. :class:`Release` reads them all once and checks that they agree. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.csv as pacsv +import yaml + +from timenet.errors import TimeNetConnectorError + + +STEP = timedelta(minutes=5) +YEARS = (2018, 2019) +# The SCADA file of each sensor type, and the key that lists its sensors in the configuration. +SCADA_FILES = { + "pressure": ("SCADA_Pressures", "pressure_sensors"), + "flow": ("SCADA_Flows", "flow_sensors"), + "tank_level": ("SCADA_Levels", "level_sensors"), + "amr_demand": ("SCADA_Demands", "amrs"), +} +LEAK_TYPES = ("abrupt", "incipient") +_TIME_FORMAT = "%Y-%m-%d %H:%M" +_LEAK_FIELDS = 6 # linkID, startTime, endTime, leakDiameter (m), leakType, peakTime +_REPAIR_FIELDS = 2 # linkID, endTime +_LINK_SECTIONS = frozenset({"[PIPES]", "[PUMPS]", "[VALVES]"}) + + +def _parse_time(text: str, where: str) -> datetime: + """Read a naive release time as UTC; the release states no time zone. + + Returns: + The time, in UTC. + + Raises: + TimeNetConnectorError: If the text is not a ``YYYY-MM-DD HH:MM`` time on the 5 minute grid. + """ + try: + moment = datetime.strptime(text.strip(), _TIME_FORMAT).replace(tzinfo=UTC) + except ValueError as error: + raise TimeNetConnectorError(f"{where}: {text!r} is not a {_TIME_FORMAT} time") from error + if moment.minute % 5 or moment.second: + raise TimeNetConnectorError(f"{where}: {text!r} is not on the 5 minute grid") + return moment + + +@dataclass(frozen=True) +class Table: + """One CSV of the release: its first timestamp and one float64 column per sensor, in file order.""" + + start: datetime + columns: dict[str, np.ndarray] + + @property + def n_rows(self) -> int: + """The number of 5 minute steps.""" + return len(next(iter(self.columns.values()))) + + +def read_table(path: Path) -> Table: + """Parse one semicolon-separated, decimal-comma CSV and check its time grid and values. + + Returns: + The table, with every value column as float64. + + Raises: + TimeNetConnectorError: If the file has no value column, a step that is not 5 minutes, a + missing value, or a value that is not finite. + """ + with path.open(encoding="utf-8") as handle: + header = handle.readline().strip().split(";") + if header[0] != "Timestamp" or len(header) == 1: + raise TimeNetConnectorError(f"{path.name}: expected a Timestamp column and sensor columns, got {header}") + names = header[1:] + if len(set(names)) != len(names): + raise TimeNetConnectorError(f"{path.name}: repeats a column name") + table = pacsv.read_csv( + path, + parse_options=pacsv.ParseOptions(delimiter=";"), + convert_options=pacsv.ConvertOptions( + decimal_point=",", + column_types={"Timestamp": pa.timestamp("s"), **dict.fromkeys(names, pa.float64())}, + timestamp_parsers=["%Y-%m-%d %H:%M:%S"], + ), + ) + times = table.column("Timestamp").to_numpy().astype("datetime64[s]").astype(np.int64) + if len(times) == 0: + raise TimeNetConnectorError(f"{path.name}: holds no rows") + steps = np.diff(times) + if (steps != STEP.total_seconds()).any(): + bad = int(np.flatnonzero(steps != STEP.total_seconds())[0]) + raise TimeNetConnectorError(f"{path.name}: row {bad + 2} is not 5 minutes after the row before it") + columns = {} + for name in names: + column = table.column(name) + if column.null_count: + raise TimeNetConnectorError(f"{path.name}: column {name} has {column.null_count} missing values") + values = column.to_numpy() + if not np.isfinite(values).all(): + raise TimeNetConnectorError(f"{path.name}: column {name} holds a value that is not finite") + columns[name] = values + return Table(start=datetime.fromtimestamp(int(times[0]), tz=UTC), columns=columns) + + +@dataclass(frozen=True) +class Leak: + """One leak of the configuration, with its outflow from the leak tables. + + ``end`` is the last 5 minute step with the leak, as the configuration and the leak tables write it. + A leak that is never repaired ends at the last step of the recording. + """ + + pipe: str + start: datetime + end: datetime + diameter_m: float + leak_type: str + peak: datetime + repaired: bool + + +def _parse_leak(line: object, where: str) -> Leak: + """Read one ``linkID, startTime, endTime, leakDiameter (m), leakType, peakTime`` line. + + Returns: + The leak, not yet marked as repaired. + + Raises: + TimeNetConnectorError: If the line does not hold six fields of the stated kinds. + """ + fields = [field.strip() for field in str(line).split(",")] + if len(fields) != _LEAK_FIELDS: + raise TimeNetConnectorError(f"{where}: leak line {line!r} does not hold six fields") + pipe, start, end, diameter, leak_type, peak = fields + if leak_type not in LEAK_TYPES: + raise TimeNetConnectorError(f"{where}: leak {pipe} has type {leak_type!r}, not one of {LEAK_TYPES}") + try: + diameter_m = float(diameter) + except ValueError as error: + raise TimeNetConnectorError(f"{where}: leak {pipe} has diameter {diameter!r}") from error + leak = Leak( + pipe=pipe, + start=_parse_time(start, where), + end=_parse_time(end, where), + diameter_m=diameter_m, + leak_type=leak_type, + peak=_parse_time(peak, where), + repaired=False, + ) + if not leak.start <= leak.peak <= leak.end: + raise TimeNetConnectorError(f"{where}: leak {pipe} peaks outside its start and end") + return leak + + +class Network: + """The links of ``L-TOWN.inp`` as an undirected graph, to find the pressure sensor nearest a pipe.""" + + def __init__(self, path: Path) -> None: + """Read the ``[PIPES]``, ``[PUMPS]`` and ``[VALVES]`` sections: a link id and its two nodes. + + Raises: + TimeNetConnectorError: If a link line names fewer than two nodes or a link id repeats. + """ + self.links: dict[str, tuple[str, str]] = {} + self.pipes: set[str] = set() + section = "" + for line in path.read_text(encoding="utf-8", errors="strict").splitlines(): + text = line.split(";", 1)[0].strip() + if not text: + continue + if text.startswith("["): + section = text.upper() + continue + if section not in _LINK_SECTIONS: + continue + link, *nodes = text.split() + if len(nodes) < len(("node1", "node2")): + raise TimeNetConnectorError(f"{path.name}: link line {line!r} names fewer than two nodes") + if link in self.links: + raise TimeNetConnectorError(f"{path.name}: link {link} appears twice") + self.links[link] = (nodes[0], nodes[1]) + if section == "[PIPES]": + self.pipes.add(link) + self._neighbours: dict[str, set[str]] = {} + for a, b in self.links.values(): + self._neighbours.setdefault(a, set()).add(b) + self._neighbours.setdefault(b, set()).add(a) + + @property + def nodes(self) -> set[str]: + """Every node that a link touches.""" + return set(self._neighbours) + + def nearest(self, pipe: str, sensors: list[str]) -> tuple[str, int]: + """Find the sensor node with the fewest links to either end of ``pipe``. + + Ties go to the sensor listed first, so the answer does not depend on set order. + + Returns: + The sensor and its number of links from the nearer end of the pipe. + + Raises: + TimeNetConnectorError: If no sensor is connected to the pipe. + """ + hops = dict.fromkeys(self.links[pipe], 0) + queue = deque(hops) + while queue: + node = queue.popleft() + for neighbour in sorted(self._neighbours[node]): + if neighbour not in hops: + hops[neighbour] = hops[node] + 1 + queue.append(neighbour) + reachable = [sensor for sensor in sensors if sensor in hops] + if not reachable: + raise TimeNetConnectorError(f"no pressure sensor is connected to pipe {pipe}") + best = min(reachable, key=lambda sensor: (hops[sensor], sensors.index(sensor))) + return best, hops[best] + + +class Release: + """The parsed release: the SCADA tables of both years, the leaks and the network. + + Reading checks that the two years join on one 5 minute grid with the same sensors, that the + configuration, the leak tables and the repair report agree, and that every sensor and leak pipe + is in the network model. + """ + + def __init__(self, root: Path) -> None: + """Read and check the release in ``root``. + + Raises: + TimeNetConnectorError: If a file is missing or the files disagree. + """ + self.root = root + config = self._yaml("dataset_configuration.yaml") + if not isinstance(config, dict): + raise TimeNetConnectorError("dataset_configuration.yaml is not a mapping") + self.sensors: dict[str, list[str]] = {} + for kind, (_, key) in SCADA_FILES.items(): + listed = config.get(key) + if not isinstance(listed, list) or not listed: + raise TimeNetConnectorError(f"dataset_configuration.yaml lists no {key}") + self.sensors[kind] = [str(sensor) for sensor in listed] + self.scada = {kind: self._join(name, self.sensors[kind]) for kind, (name, _) in SCADA_FILES.items()} + self.start = self.scada["pressure"][0] + self.n_steps = len(next(iter(self.scada["pressure"][1].values()))) + for kind, (start, columns) in self.scada.items(): + if start != self.start or len(next(iter(columns.values()))) != self.n_steps: + raise TimeNetConnectorError(f"the {kind} tables do not cover the pressure tables' time range") + times = config.get("times") or {} + if _parse_time(str(times.get("StartTime")), "times") != self.start or ( + _parse_time(str(times.get("EndTime")), "times") != self.end + ): + raise TimeNetConnectorError("dataset_configuration.yaml times differ from the SCADA tables") + self.leak_outflow = self._join_leaks() + self.leaks = self._read_leaks(config) + self.network = Network(self._path("L-TOWN.inp")) + self._check_network() + + @property + def end(self) -> datetime: + """The last 5 minute step of the recording.""" + return self.start + STEP * (self.n_steps - 1) + + def step_of(self, moment: datetime) -> int: + """The index of ``moment`` on the recording's 5 minute grid.""" + return int((moment - self.start) / STEP) + + def _path(self, name: str) -> Path: + path = self.root / name + if not path.is_file(): + raise TimeNetConnectorError(f"the release in {self.root} is missing {name}") + return path + + def _yaml(self, name: str) -> object: + try: + return yaml.safe_load(self._path(name).read_text(encoding="utf-8")) + except yaml.YAMLError as error: + raise TimeNetConnectorError(f"{name} is not valid YAML") from error + + def _join(self, name: str, sensors: list[str]) -> tuple[datetime, dict[str, np.ndarray]]: + """Join the 2018 and 2019 tables of one sensor type into one recording. + + Returns: + The first timestamp, and each sensor's values over both years. + + Raises: + TimeNetConnectorError: If the years name other sensors than the configuration lists, or + the second year does not start 5 minutes after the first ends. + """ + tables = [read_table(self._path(f"{year}_{name}.csv")) for year in YEARS] + for year, table in zip(YEARS, tables, strict=True): + if set(table.columns) != set(sensors): + raise TimeNetConnectorError( + f"{year}_{name}.csv columns differ from the sensors that dataset_configuration.yaml lists" + ) + first, second = tables + if second.start != first.start + STEP * first.n_rows: + raise TimeNetConnectorError(f"{YEARS[1]}_{name}.csv does not continue {YEARS[0]}_{name}.csv") + return first.start, { + sensor: np.concatenate([first.columns[sensor], second.columns[sensor]]) for sensor in sensors + } + + def _join_leaks(self) -> dict[str, np.ndarray]: + """Join the two leak tables into one outflow series per pipe, zero where a year does not list it. + + Returns: + Each leaking pipe's outflow over both years, in m3/h. + + Raises: + TimeNetConnectorError: If a leak table does not cover its year of the SCADA tables. + """ + tables = [read_table(self._path(f"{year}_Leakages.csv")) for year in YEARS] + offset = 0 + outflow: dict[str, np.ndarray] = {} + for year, table in zip(YEARS, tables, strict=True): + if table.start != self.start + STEP * offset: + raise TimeNetConnectorError(f"{year}_Leakages.csv does not line up with the SCADA tables") + for pipe, values in table.columns.items(): + series = outflow.setdefault(pipe, np.zeros(self.n_steps)) + series[offset : offset + table.n_rows] = values + offset += table.n_rows + if offset != self.n_steps: + raise TimeNetConnectorError("the leak tables do not cover the SCADA tables") + return outflow + + def _read_leaks(self, config: dict) -> list[Leak]: + """Read the configured leaks and check them against the leak tables and the repair report. + + Returns: + The leaks in order of their start, each marked as repaired or not. + + Raises: + TimeNetConnectorError: If a configured leak is outside the recording or its leak table, + a leak table has a pipe the configuration lacks, or a repair time disagrees. + """ + # The list opens with a comment line that names the fields, which YAML reads as a null item. + lines = [line for line in (config.get("leakages") or []) if line is not None] + leaks = [_parse_leak(line, "dataset_configuration.yaml") for line in lines] + if len({leak.pipe for leak in leaks}) != len(leaks): + raise TimeNetConnectorError("dataset_configuration.yaml lists a pipe in two leaks") + if {leak.pipe for leak in leaks} != set(self.leak_outflow): + raise TimeNetConnectorError("the leak tables and dataset_configuration.yaml name different pipes") + report = self._yaml("2018_Fixed_Leakages_Report.txt") + if not isinstance(report, list): + raise TimeNetConnectorError("2018_Fixed_Leakages_Report.txt is not a list of repairs") + repairs = {} + for line in report: + fields = [field.strip() for field in str(line).split(",")] + if len(fields) != _REPAIR_FIELDS: + raise TimeNetConnectorError(f"2018_Fixed_Leakages_Report.txt: line {line!r} is not 'linkID, endTime'") + repairs[fields[0]] = _parse_time(fields[1], "2018_Fixed_Leakages_Report.txt") + checked = [] + for leak in leaks: + if not self.start <= leak.start <= leak.end <= self.end: + raise TimeNetConnectorError(f"leak {leak.pipe} is outside the recording") + if leak.pipe in repairs and repairs[leak.pipe] != leak.end: + raise TimeNetConnectorError(f"leak {leak.pipe}: the repair report and the configuration disagree") + outflow = self.leak_outflow[leak.pipe] + active = np.flatnonzero(outflow > 0) + if len(active) == 0 or active[0] < self.step_of(leak.start) or active[-1] > self.step_of(leak.end) + 1: + raise TimeNetConnectorError(f"leak {leak.pipe}: its outflow lies outside its configured start and end") + checked.append( + Leak( + pipe=leak.pipe, + start=leak.start, + end=leak.end, + diameter_m=leak.diameter_m, + leak_type=leak.leak_type, + peak=leak.peak, + repaired=leak.end < self.end, + ) + ) + if set(repairs) - {leak.pipe for leak in checked if leak.repaired}: + raise TimeNetConnectorError("2018_Fixed_Leakages_Report.txt repairs a leak the configuration keeps open") + return sorted(checked, key=lambda leak: (leak.start, leak.pipe)) + + def _check_network(self) -> None: + """Check that every sensor and leak pipe is in the network model. + + Raises: + TimeNetConnectorError: If one is missing. + """ + nodes = self.network.nodes + for kind in ("pressure", "tank_level", "amr_demand"): + missing = [sensor for sensor in self.sensors[kind] if sensor not in nodes] + if missing: + raise TimeNetConnectorError(f"L-TOWN.inp has no node for the {kind} sensors {missing}") + missing = [sensor for sensor in self.sensors["flow"] if sensor not in self.network.links] + if missing: + raise TimeNetConnectorError(f"L-TOWN.inp has no link for the flow sensors {missing}") + missing = [leak.pipe for leak in self.leaks if leak.pipe not in self.network.pipes] + if missing: + raise TimeNetConnectorError(f"L-TOWN.inp has no pipe for the leaks {missing}") From 4a15be1713aa154a88a0fdb20d1e766d3881a010 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 20:59:36 +0100 Subject: [PATCH 02/10] feat(connectors): add the kios/battledim connector and leak-window tasks Download 13 files of Zenodo record 4017659, each pinned by sha256, and build one record of the 2018-2019 recording: 119 float64 signals (33 pressure heads, 3 flows, 1 tank level, 82 metered demands) on a 5-minute grid, with every leak as record annotations over its active interval. Two-week windows, one every 7 days, give four tasks about their last week: the types of new leaks, their start times, the mean leak outflow, and a rule-generated text that names each new or repaired leak, its size and the nearest pressure sensor by network links. Windows split 8/1/1 by calendar block across both years, because 2019 repeats most of the 2018 metered demands. --- .../datasets/kios/battledim/__init__.py | 6 + .../datasets/kios/battledim/connector.py | 182 ++++++++++++++ .../datasets/kios/battledim/dataset.yaml | 42 ++++ .../datasets/kios/battledim/tasks.py | 234 ++++++++++++++++++ 4 files changed, 464 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py new file mode 100644 index 000000000..79edbfc84 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/__init__.py @@ -0,0 +1,6 @@ +"""The BattLeDIM connector package: its connector implementation and dataset card.""" + +from timenet_connectors.datasets.kios.battledim.connector import ( + CONNECTOR as CONNECTOR, + BattLeDIMConnector as BattLeDIMConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py new file mode 100644 index 000000000..3f7a8e2e7 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py @@ -0,0 +1,182 @@ +"""The BattLeDIM connector: two years of SCADA data from the L-Town water network, with its leaks. + +BattLeDIM 2020 (Battle of the Leakage Detection and Isolation Methods) released simulated SCADA +data of the L-Town network for 2018 and 2019: 33 pressure sensors, 3 flow sensors, 1 tank level +sensor and 82 automated meter readings, every 5 minutes, and the 33 leaks that the simulation +contained. The two years form one Record. The leaks become record annotations, and two-week windows +of the record become the tasks of :mod:`~timenet_connectors.datasets.kios.battledim.tasks`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction +import functools +from pathlib import Path + +import pyarrow as pa + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import Annotation, TimeInterval, TimeOrigin, TimePoint, TimeSeriesSpec, ureg +from timenet_connectors.datasets.kios.battledim.source import SCADA_FILES, STEP, Release +from timenet_connectors.datasets.kios.battledim.tasks import split_windows, window_tasks, windows +from timenet_connectors.download import Artifact, download_files + + +ZENODO_FILE = "https://zenodo.org/api/records/4017659/files/{name}/content" +# Zenodo record 4017659 (published 2020-09-07) is the only version. The two .xlsx files repeat the +# CSVs and L-TOWN_Real.inp (177 MB) is the simulation model, so the connector downloads neither. +SOURCE_FILES = { + "2018_SCADA_Pressures.csv": "7782d890a7195699be94ddc01cba392da7ca285908afff619065a9616e04dcae", + "2018_SCADA_Flows.csv": "7f7e8625c9d92b083ee079a54cffb9ef803876de1e35017c1da0455968daf3ff", + "2018_SCADA_Levels.csv": "a1fc5f0229448a620b6220f4d079b3162a255b3070acdce448608fbfc760903c", + "2018_SCADA_Demands.csv": "713d9f03ff94128f44c7d8036a032cc1d7eebcbbc94fda9def2bbc3efc047dd5", + "2019_SCADA_Pressures.csv": "a4af6616059cc9b366bbab76d880b2c772df544970c6ff4d4a29a58704ce05b1", + "2019_SCADA_Flows.csv": "c709ea5ecc03197e1fe70cf4c2de7225f71015acb50a7445ed7765b5029f7f15", + "2019_SCADA_Levels.csv": "00624e97b72e63233beeccb48db528d43e59a87b116d0295d5c2a206ee0e75a8", + "2019_SCADA_Demands.csv": "004266aa039833b5ae0f37740487db4d90216d2ee947f14871c5d5d3f89b4151", + "2018_Leakages.csv": "e1abd9d549a18880d8ae704dc90c06c8a3d5aecbd2541fa091548ef392084ba6", + "2019_Leakages.csv": "b2ffe56b6c2f3b6cdb6aae9aeb30b9c425ba911fff8e3716ae543a56ff7576cd", + "dataset_configuration.yaml": "23471d8989d82c66a4909816ea248f7588d8ccc910e142265a7ea48db940b6a0", + "2018_Fixed_Leakages_Report.txt": "5dba254c623b867a0a91150e2b5539842ca05a989fde6cf2754065d5b415a8f3", + "L-TOWN.inp": "a7551b86745f4cc3433c78e60023077fa1386947d4d35372ffd3a0405622b436", +} + +# README.txt of the release states each unit. Pressure is head in metres of water. The values stay +# float64, as parsed from the release's two-decimal text. +SPECS = { + "pressure": TimeSeriesSpec(spec_type="pressure_head", name="Pressure head", unit_value=ureg.meter, dtype="float64"), + "flow": TimeSeriesSpec( + spec_type="volume_flow", name="Volume flow", unit_value=ureg.meter**3 / ureg.hour, dtype="float64" + ), + "tank_level": TimeSeriesSpec( + spec_type="water_level", name="Tank water level", unit_value=ureg.meter, dtype="float64" + ), + "amr_demand": TimeSeriesSpec( + spec_type="water_demand", name="Metered water demand", unit_value=ureg.liter / ureg.hour, dtype="float64" + ), +} +SOURCE_NAMES = { + "pressure": "Pressure sensors", + "flow": "Flow sensors", + "tank_level": "Tank level sensor", + "amr_demand": "Automated meter readings", +} + + +@dataclass(frozen=True) +class BattLeDIMSource: + """Handle to the downloaded release: the directory that holds the files of ``SOURCE_FILES``.""" + + root: Path + + +class BattLeDIMConnector(BaseConnector[BattLeDIMSource]): + """Connector for the BattLeDIM 2020 dataset (Zenodo record 4017659).""" + + async def download_async(self, cache_dir: Path) -> list[BattLeDIMSource]: # ruff: ignore[no-self-use] + """Download the release files once, checking each one's sha256. + + Returns: + A list of one :class:`BattLeDIMSource`. + """ + await download_files( + Artifact(ZENODO_FILE.format(name=name), cache_dir / name, sha256=digest) + for name, digest in SOURCE_FILES.items() + ) + return [BattLeDIMSource(cache_dir)] + + def convert(self, raw_refs: list[BattLeDIMSource]) -> TimeFDataset: + """Build the record, its leak annotations and the window tasks. + + The release is 190 MB of text, so every file is read once here, which also checks it before + anything is written. The signal loaders then serve the parsed columns. + + Returns: + The populated :class:`~timenet.dataset.TimeFDataset`. + + Raises: + TimeNetConnectorError: If ``raw_refs`` does not hold exactly one release. + """ + if len(raw_refs) != 1: + raise TimeNetConnectorError(f"expected one release, got {len(raw_refs)}") + release = Release(raw_refs[0].root) + dataset = TimeFDataset(metadata=self.metadata()) + record = dataset.add_record(record=self._make_record(release)) + found = windows(release) + split_of = split_windows(found) + for window in found: + dataset.add_tasks(tasks=window_tasks(release, record, window, split_of[window.index])) + return dataset + + @staticmethod + def _make_record(release: Release) -> Record: + """Build the one record: a source per sensor type and an annotation set per leak. + + Returns: + The record, starting at the first step of 2018. + """ + axis = RegularAxis(period_us=Fraction(int(STEP.total_seconds() * 1_000_000))) + sources = [] + for kind in SCADA_FILES: + columns = release.scada[kind][1] + signals = tuple( + Signal.from_loader( + spec=SPECS[kind], + time_axis=axis, + name=sensor, + loader=functools.partial(pa.array, columns[sensor]), + n_values=release.n_steps, + ) + for sensor in release.sensors[kind] + ) + sources.append(Source(name=SOURCE_NAMES[kind], signals=signals)) + record = Record( + sources=tuple(sources), + start_time=TimeOrigin(release.start), + metadata={"network": "L-Town", "source_doi": "10.5281/zenodo.4017659"}, + ) + annotations = [] + for leak in release.leaks: + start_us = int((leak.start - release.start).total_seconds() * 1_000_000) + end_us = int((leak.end + STEP - release.start).total_seconds() * 1_000_000) + span = TimeInterval.micros(start_us, end_us) + outflow = release.leak_outflow[leak.pipe] + annotations += [ + Annotation(key="leak_pipe", value=leak.pipe, span=span, description="The leaking pipe's EPANET id."), + Annotation(key="leak_type", value=leak.leak_type, span=span, description="abrupt or incipient."), + Annotation( + key="leak_hole_diameter", + value=round(leak.diameter_m * 1000, 4), + unit="mm", + span=span, + description="The diameter of the simulated hole.", + ), + Annotation( + key="leak_peak_outflow", + value=float(outflow.max()), + unit="m^3/h", + span=span, + description="The largest outflow in the leak tables.", + ), + Annotation( + key="leak_repaired", + value=leak.repaired, + span=span, + description="Whether the leak was repaired before the end of 2019.", + ), + Annotation( + key="leak_peak", + value=leak.pipe, + span=TimePoint.micros(int((leak.peak - release.start).total_seconds() * 1_000_000)), + description="When the leak reaches its full size; the start, for an abrupt leak.", + ), + ] + record.add_annotations(annotations) + return record + + +CONNECTOR = BattLeDIMConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml new file mode 100644 index 000000000..93a564a9f --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml @@ -0,0 +1,42 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# +# BattLeDIM 2020 released two years (2018 and 2019) of simulated SCADA data of the L-Town water +# network, every 5 minutes: 33 pressure sensors, 3 flow sensors, 1 tank level sensor and 82 +# automated meter readings, plus the 33 leaks the simulation contained. The two years form one +# record. Each leak is a set of record annotations over its active interval. Two-week windows, +# one every 7 days, become four tasks each about the last week of the window. The answers come from +# the leak list, the leak tables and the network model by fixed rules, not from a model. +# +# The split groups windows by the calendar block of their target week (four ISO weeks), in both +# years at once: 74 of the 82 meter series of 2019 repeat 2018 exactly, so the same calendar week +# of both years always lands in the same partition. README.md gives the reasons and the release +# details. +yaml_schema_version: 1 +dataset_id: kios/battledim +dataset_version: 1.0.0 +name: BattLeDIM +description: >- + Two years of simulated SCADA data (pressure, flow, tank level and metered demand, every 5 minutes) + from the L-Town water distribution network, with its 33 leaks. Two-week windows ask which leaks + started, when, how much water was lost, and for a short explanation that names the pipe and the + nearest pressure sensor. +license: CC-BY-4.0 +source_url: https://zenodo.org/records/4017659 +citation: >- + Vrachimis, S. G., Eliades, D. G., Taormina, R., Kapelan, Z., Ostfeld, A., Liu, S., Kyriakou, M., + Pavlou, P., Qiu, M., and Polycarpou, M. M. (2022). Battle of the Leakage Detection and Isolation + Methods. Journal of Water Resources Planning and Management, 148(12). + https://doi.org/10.1061/(ASCE)WR.1943-5452.0001601. Dataset: https://doi.org/10.5281/zenodo.4017659 +access: open +domains: + - industrial + - environment +tags: + - water-distribution + - leak-detection + - anomaly-detection + - fault-diagnosis + - explanation + - simulated + - scada + - zenodo diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py new file mode 100644 index 000000000..6e5f86ead --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py @@ -0,0 +1,234 @@ +"""Cut the recording into two-week windows and state what the leak list says about each one. + +A window is 14 days: a context week, then a target week. It starts every 7 days, so consecutive +windows share a week. Each window gives four tasks about the target week, all scoped to the window: +the types of the leaks that start in it, when they start, the mean leak outflow, and a short text +that describes every leak that starts or is repaired in it. The answers come from the leak list, +the leak tables and the network model by fixed rules. No model writes them. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta + +import numpy as np + +from timenet.dataset import Record +from timenet.json import JsonMapping, JsonValue +from timenet.types import ( + AnswerTask, + ClassificationTask, + Label, + LabelSet, + LocalizationMode, + ScalarPredictionTask, + Task, + TemporalLocalizationTask, + TimeInterval, + TimePoint, +) +from timenet.types.splits import Split +from timenet_connectors.datasets.kios.battledim.source import STEP, Leak, Release +from timenet_connectors.splitting import StratifiedSplitter + + +WINDOW = timedelta(days=14) +TARGET = timedelta(days=7) +STRIDE = timedelta(days=7) +# Four ISO weeks make one calendar block. Windows whose target weeks share a block share a partition +# in both years, because 2019 repeats most of the 2018 demand series (see README.md). +WEEKS_PER_BLOCK = 4 + +NEW_LEAK_TYPES = LabelSet( + name="new_leak_types", + labels=( + Label(name="abrupt", description="A leak that appears at full size at its start."), + Label(name="incipient", description="A leak that grows from its start until its peak time."), + ), + multi_label=True, +) +EXPLAIN_PROMPT = ( + "Describe every leak that started or was repaired in the last 7 days of this two-week window: " + "the pipe, when, how it grew, its size, and the nearest pressure sensor." +) +LOCATE_PROMPT = "Locate the start of every leak that began in the last 7 days of this two-week window." +CLASSIFY_PROMPT = "Which types of leak began in the last 7 days of this two-week window?" +OUTFLOW_PROMPT = "What was the mean total leak outflow in the last 7 days of this two-week window?" + + +@dataclass(frozen=True) +class Window: + """One two-week window and what the leak list says about its target week.""" + + index: int + start: datetime + new: tuple[Leak, ...] + repaired: tuple[Leak, ...] + + @property + def target_start(self) -> datetime: + """The first step of the target week.""" + return self.start + WINDOW - TARGET + + @property + def end(self) -> datetime: + """The exclusive end of the window.""" + return self.start + WINDOW + + @property + def block(self) -> int: + """The calendar block of the target week: its ISO week, counted in blocks of four.""" + return (self.target_start.isocalendar().week - 1) // WEEKS_PER_BLOCK + + +def _micros(release: Release, moment: datetime) -> int: + return int((moment - release.start) / timedelta(microseconds=1)) + + +def windows(release: Release) -> list[Window]: + """List every window that fits in the recording, in time order. + + Returns: + The windows, each with the leaks that start or are repaired in its target week. + """ + found = [] + start = release.start + recording_end = release.end + STEP + while start + WINDOW <= recording_end: + target_start, end = start + WINDOW - TARGET, start + WINDOW + found.append( + Window( + index=len(found), + start=start, + new=tuple(leak for leak in release.leaks if target_start <= leak.start < end), + # The release writes a repair time as the leak's end: its last step with outflow. + repaired=tuple(leak for leak in release.leaks if leak.repaired and target_start <= leak.end < end), + ) + ) + start += STRIDE + return found + + +def split_windows(found: list[Window]) -> dict[int, Split]: + """Assign each window to a partition by the calendar block of its target week. + + Returns: + The partition of each window, keyed by window index. + """ + splitter = StratifiedSplitter( + train_folds=8, + validation_folds=1, + test_folds=1, + stratify_by=lambda window: bool(window.new), + group_by=lambda window: window.block, + seed=42, + ) + return {window.index: split for split, members in splitter.split(found).items() for window in members} + + +def _outflow(release: Release, leak: Leak, start: datetime, end: datetime) -> float: + """The largest outflow of ``leak`` in ``[start, end)``, in m3/h.""" + return float(release.leak_outflow[leak.pipe][release.step_of(start) : release.step_of(end)].max()) + + +def _hops(count: int) -> str: + return f"{count} link{'s' if count != 1 else ''}" + + +def explain(release: Release, window: Window) -> str: + """Write the text answer for one window from the leak list, the leak tables and the network. + + Returns: + One sentence per new or repaired leak, then the leaks still active at the end of the window. + """ + sentences = [] + for leak in window.new: + sensor, hops = release.network.nearest(leak.pipe, release.sensors["pressure"]) + growth = ( + "A sudden leak" + if leak.leak_type == "abrupt" + else f"A gradually growing leak, peaking on {leak.peak:%Y-%m-%d %H:%M}," + ) + sentences.append( + f"{growth} started on pipe {leak.pipe} on {leak.start:%Y-%m-%d %H:%M}, " + f"with a hole of {leak.diameter_m * 1000:.1f} mm and an outflow of up to " + f"{_outflow(release, leak, leak.start, window.end):.1f} m3/h in this window. " + f"The nearest pressure sensor is {sensor}, {_hops(hops)} away." + ) + for leak in window.repaired: + last_hour = leak.end + STEP - timedelta(hours=1) + sentences.append( + f"The leak on pipe {leak.pipe} was repaired on {leak.end:%Y-%m-%d %H:%M}; " + f"its outflow before the repair was {_outflow(release, leak, last_hour, leak.end + STEP):.1f} m3/h." + ) + if not sentences: + sentences.append("No leak started or was repaired in the last 7 days.") + last = release.step_of(window.end) - 1 + active = [pipe for pipe, outflow in release.leak_outflow.items() if outflow[last] > 0] + total = sum(float(release.leak_outflow[pipe][last]) for pipe in active) + sentences.append( + f"At the end of the window {len(active)} leak{'s are' if len(active) != 1 else ' is'} active, " + f"with a total outflow of {total:.1f} m3/h." + ) + return " ".join(sentences) + + +def window_tasks(release: Release, record: Record, window: Window, split: Split) -> list[Task]: + """Build the four tasks of one window, all scoped to the window. + + Returns: + The classification, localization, outflow and text tasks, in that order. + """ + scope = TimeInterval.micros(_micros(release, window.start), _micros(release, window.end)) + target = slice(release.step_of(window.target_start), release.step_of(window.end)) + outflow = np.sum([series[target] for series in release.leak_outflow.values()], axis=0) + new_leaks: list[JsonValue] = [leak.pipe for leak in window.new] + repaired_leaks: list[JsonValue] = [leak.pipe for leak in window.repaired] + metadata: JsonMapping = { + "window_index": window.index, + "window_start": f"{window.start:%Y-%m-%d %H:%M}", + "target_week_start": f"{window.target_start:%Y-%m-%d %H:%M}", + "calendar_block": window.block, + "new_leaks": new_leaks, + "repaired_leaks": repaired_leaks, + } + inputs = (record,) + return [ + ClassificationTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=CLASSIFY_PROMPT, + targets=tuple(sorted({leak.leak_type for leak in window.new})), + label_set=NEW_LEAK_TYPES, + ), + TemporalLocalizationTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=LOCATE_PROMPT, + mode=LocalizationMode.EXHAUSTIVE, + targets=tuple(TimePoint.micros(_micros(release, leak.start)) for leak in window.new), + ), + ScalarPredictionTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=OUTFLOW_PROMPT, + targets=(round(float(outflow.mean()), 3),), + unit="m^3/h", + target_name="mean_leak_outflow", + ), + AnswerTask( + inputs=inputs, + scope=scope, + split=split, + metadata=metadata, + prompt=EXPLAIN_PROMPT, + targets=(explain(release, window),), + ), + ] From e4c5c8ce6d9a03a2011ff0d322e44b2d1f7b7304 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 20:59:43 +0100 Subject: [PATCH 03/10] test(connectors): cover kios/battledim on a synthetic release Write a two-year synthetic release in the source layout and check the record, the leak annotations, the window tasks and their text, the calendar-block split, a storage round trip, and that a gap, a missing value or a disagreement between the source files raises. --- .../datasets/kios/battledim/tests/__init__.py | 0 .../kios/battledim/tests/test_connector.py | 266 ++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py new file mode 100644 index 000000000..69f6a1549 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py @@ -0,0 +1,266 @@ +from collections import Counter +from pathlib import Path + +import numpy as np +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, open_local_version +from timenet.types import AnswerTask, ClassificationTask, ScalarPredictionTask, TemporalLocalizationTask +from timenet.types.text import read_value +from timenet_connectors.datasets.kios.battledim import BattLeDIMConnector +from timenet_connectors.datasets.kios.battledim.connector import BattLeDIMSource +from timenet_connectors.datasets.kios.battledim.source import Release + + +# A synthetic release shaped exactly like the Zenodo files: semicolon-separated CSVs with a decimal +# comma on a 5 minute grid, the configuration YAML, the repair report and an EPANET network. The +# values and the network are hand-made for the tests, so the repo ships no dataset bytes. Two full +# years are needed because the split groups windows into calendar blocks and asks for 10 folds. +STEPS_PER_YEAR = 365 * 288 +_TIMES = (np.datetime64("2018-01-01T00:00") + np.arange(2 * STEPS_PER_YEAR) * np.timedelta64(5, "m")).astype(str) +TIMES = [time.replace("T", " ") + ":00" for time in _TIMES] + +CONFIG = """\ +Comments: + - Synthetic configuration for tests +Network: + filename: L-TOWN_Real.inp +times: + StartTime: 2018-01-01 00:00 + EndTime: 2019-12-31 23:55 +leakages: +- # linkID, startTime, endTime, leakDiameter (m), leakType, peakTime +- p3, 2018-03-05 15:45, 2018-03-23 10:25, 0.022916, abrupt, 2018-03-05 15:45 +- p5, 2018-11-01 00:00, 2019-12-31 23:55, 0.010000, incipient, 2018-11-15 00:00 +- p6, 2019-05-01 12:00, 2019-12-31 23:55, 0.015000, abrupt, 2019-05-01 12:00 +pressure_sensors: +- n1 +- n4 +flow_sensors: +- PUMP_1 +- p227 +level_sensors: +- T1 +amrs: +- n2 +- n3 +""" +REPORT = "# FIXED LEAKAGES\n# linkID, endTime\n- p3, 2018-03-23 10:25\n" +NETWORK = """\ +[JUNCTIONS] + n1 10 0 ; +[PIPES] +;ID Node1 Node2 Length Diameter Roughness MinorLoss Status + p227 R1 n1 10 200 140 0 Open ; + p2 n1 n2 10 150 140 0 Open ; + p3 n2 n3 10 150 140 0 Open ; + p4 n3 n4 10 150 140 0 Open ; + p5 n4 n5 10 150 140 0 Open ; + p6 n5 n6 10 150 140 0 Open ; +[PUMPS] + PUMP_1 n2 T1 HEAD 1 ;FLOW SENSOR +[VALVES] +[END] +""" + + +def _step(text: str) -> int: + return TIMES.index(text + ":00") + + +def _outflows() -> dict[str, np.ndarray]: + outflow = {pipe: np.zeros(2 * STEPS_PER_YEAR) for pipe in ("p3", "p5", "p6")} + outflow["p3"][_step("2018-03-05 15:45") : _step("2018-03-23 10:25") + 1] = 28.4 + start, peak = _step("2018-11-01 00:00"), _step("2018-11-15 00:00") + outflow["p5"][start:peak] = np.linspace(0.0, 5.0, peak - start) + outflow["p5"][peak:] = 5.0 + outflow["p6"][_step("2019-05-01 12:00") :] = 10.5 + return outflow + + +def _write_csv(path: Path, columns: dict[str, np.ndarray], rows: slice) -> None: + lines = ["Timestamp;" + ";".join(columns)] + formatted = [[f"{value:.2f}".replace(".", ",") for value in values[rows]] for values in columns.values()] + for index, time in enumerate(TIMES[rows]): + lines.append(";".join([time, *(column[index] for column in formatted)])) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def write_release(root: Path) -> Path: + root.mkdir(parents=True, exist_ok=True) + clock = np.arange(2 * STEPS_PER_YEAR) + daily = np.sin(2 * np.pi * clock / 288) + scada = { + "SCADA_Pressures": {"n1": 50.0 + daily, "n4": 40.0 + 0.5 * daily}, + "SCADA_Flows": {"p227": 80.0 + 10 * daily, "PUMP_1": np.full(clock.size, 44.04)}, + "SCADA_Levels": {"T1": 3.0 + 0.5 * daily}, + "SCADA_Demands": {"n2": 100.0 + 20 * daily, "n3": np.full(clock.size, 12.5)}, + } + outflow = _outflows() + years = {2018: slice(0, STEPS_PER_YEAR), 2019: slice(STEPS_PER_YEAR, 2 * STEPS_PER_YEAR)} + for year, rows in years.items(): + for name, columns in scada.items(): + _write_csv(root / f"{year}_{name}.csv", columns, rows) + active = {pipe: values for pipe, values in outflow.items() if values[rows].any()} + _write_csv(root / f"{year}_Leakages.csv", active, rows) + (root / "dataset_configuration.yaml").write_text(CONFIG, encoding="utf-8") + (root / "2018_Fixed_Leakages_Report.txt").write_text(REPORT, encoding="utf-8") + (root / "L-TOWN.inp").write_text(NETWORK, encoding="utf-8") + return root + + +@pytest.fixture(scope="module") +def release_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + return write_release(tmp_path_factory.mktemp("battledim")) + + +@pytest.fixture(scope="module") +def dataset(release_dir: Path): + return BattLeDIMConnector().convert([BattLeDIMSource(release_dir)]) + + +def test_record_holds_every_sensor_on_one_grid(dataset) -> None: + (record,) = dataset.records + sources = {source.name: [signal.name for signal in source.signals] for source in record.sources} + assert sources == { + "Pressure sensors": ["n1", "n4"], + "Flow sensors": ["PUMP_1", "p227"], + "Tank level sensor": ["T1"], + "Automated meter readings": ["n2", "n3"], + } + pressure = record.sources[0].signals[0] + assert pressure.n_values == 2 * STEPS_PER_YEAR + assert str(pressure.spec.unit_value) == "meter" + assert pressure.time_axis.period_us == 300_000_000 + values = pressure.to_numpy() + # The decimal comma parses, and the second year continues the first. + assert values[0] == pytest.approx(50.0) + assert values[STEPS_PER_YEAR + 72] == pytest.approx(51.0, abs=0.01) + flow = record.sources[1].signals[0] + assert str(flow.spec.unit_value) == "meter ** 3 / hour" + assert flow.to_numpy()[5] == pytest.approx(44.04) + + +def test_leaks_become_annotations_over_their_intervals(dataset) -> None: + (record,) = dataset.records + pipes = [annotation for annotation in record.annotations if annotation.key == "leak_pipe"] + assert [annotation.value for annotation in pipes] == ["p3", "p5", "p6"] + p3 = pipes[0].span + assert p3.start_us == _step("2018-03-05 15:45") * 300_000_000 + # The interval ends after the repair step, the last step with outflow. + assert p3.end_us == (_step("2018-03-23 10:25") + 1) * 300_000_000 + repaired = {a.span.start_us: a.value for a in record.annotations if a.key == "leak_repaired"} + assert sorted(repaired.values()) == [False, False, True] + peak = [a for a in record.annotations if a.key == "leak_peak" and a.value == "p5"] + assert peak[0].span.start_us == _step("2018-11-15 00:00") * 300_000_000 + + +def test_windows_give_four_tasks_each(dataset) -> None: + tasks = list(dataset.get_all()) + # 730 days hold 103 two-week windows that start every 7 days. + assert Counter(type(task) for task in tasks) == { + ClassificationTask: 103, + TemporalLocalizationTask: 103, + ScalarPredictionTask: 103, + AnswerTask: 103, + } + first = tasks[0] + assert first.scope.start_us == 0 + assert first.scope.end_us == 14 * 24 * 3600 * 1_000_000 + + +def _tasks_of_week(dataset, target_week_start: str) -> list: + return [task for task in dataset.get_all() if task.metadata["target_week_start"] == target_week_start] + + +def test_a_new_abrupt_leak_is_classified_located_and_explained(dataset) -> None: + classify, locate, outflow, answer = _tasks_of_week(dataset, "2018-03-05 00:00") + assert classify.targets == ("abrupt",) + assert [point.start_us for point in locate.targets] == [_step("2018-03-05 15:45") * 300_000_000] + assert outflow.target_name == "mean_leak_outflow" + assert outflow.targets[0] == pytest.approx(28.4 * (7 * 288 - _step("2018-03-05 15:45") % 2016) / 2016, rel=1e-3) + text = read_value(answer.targets[0]) + assert text.startswith("A sudden leak started on pipe p3 on 2018-03-05 15:45, with a hole of 22.9 mm") + # p3 joins n2 and n3. n1 and n4 are both one link away, so the first listed sensor wins. + assert "The nearest pressure sensor is n1, 1 link away." in text + assert text.endswith("At the end of the window 1 leak is active, with a total outflow of 28.4 m3/h.") + + +def test_a_repair_and_a_quiet_week_are_explained(dataset) -> None: + (*_, answer) = _tasks_of_week(dataset, "2018-03-19 00:00") + assert "The leak on pipe p3 was repaired on 2018-03-23 10:25; its outflow before the repair was 28.4 m3/h." in ( + read_value(answer.targets[0]) + ) + classify, locate, _, answer = _tasks_of_week(dataset, "2018-07-02 00:00") + assert classify.targets == () + assert locate.targets == () + assert read_value(answer.targets[0]).startswith("No leak started or was repaired in the last 7 days.") + + +def test_split_keeps_each_calendar_block_in_one_partition(dataset) -> None: + tasks = list(dataset.get_all()) + partitions: dict[int, set] = {} + for task in tasks: + partitions.setdefault(task.metadata["calendar_block"], set()).add(task.split) + assert all(len(splits) == 1 for splits in partitions.values()) + counts = Counter(task.split for task in tasks) + assert sum(counts.values()) == len(tasks) + assert len(counts) == 3 + + +def _answers(dataset) -> list[str]: + answers = [] + for task in dataset.get_all(): + if isinstance(task, AnswerTask): + assert task.targets is not None + answers.append(read_value(task.targets[0])) + return answers + + +def test_dataset_survives_storage_round_trip(dataset, tmp_path: Path) -> None: + registry = LocalRegistry(tmp_path / "registry") + registry.put_dataset(dataset) + with TimeFReader(open_local_version(registry.root / "kios/battledim/1.0.0")) as reader: + restored = reader.read() + (record,) = restored.records + assert record.sources[3].signals[1].to_numpy()[0] == pytest.approx(12.5) + # The reader returns the tasks in the order the connector added them. + assert [task.split for task in restored.get_all()] == [task.split for task in dataset.get_all()] + assert _answers(restored) == _answers(dataset) + + +def test_nearest_sensor_prefers_the_nearer_pipe_end(release_dir: Path) -> None: + network = Release(release_dir).network + assert network.nearest("p5", ["n1", "n4"]) == ("n4", 0) + assert network.nearest("p6", ["n1", "n4"]) == ("n4", 1) + + +def _corrupt(release_dir: Path, tmp_path: Path, name: str, edit) -> Path: + copy = tmp_path / "release" + copy.mkdir() + for path in release_dir.iterdir(): + (copy / path.name).write_bytes(path.read_bytes()) + target = copy / name + target.write_text(edit(target.read_text(encoding="utf-8")), encoding="utf-8") + return copy + + +@pytest.mark.parametrize( + ("name", "edit", "message"), + [ + ( + "2018_SCADA_Pressures.csv", + lambda text: text.replace("2018-01-01 00:05:00;", "2018-01-01 00:06:00;"), + "5 minutes", + ), + ("2019_SCADA_Levels.csv", lambda text: text.replace(";3,00\n", ";\n", 1), "missing values"), + ("dataset_configuration.yaml", lambda text: text.replace("- p6,", "- p4,"), "different pipes"), + ("2018_Fixed_Leakages_Report.txt", lambda text: text.replace("10:25", "10:30"), "disagree"), + ("L-TOWN.inp", lambda text: text.replace(" p5 n4 n5", " p9 n4 n5"), "no pipe for the leaks"), + ], +) +def test_inconsistent_release_raises(release_dir: Path, tmp_path: Path, name: str, edit, message: str) -> None: + with pytest.raises(TimeNetConnectorError, match=message): + Release(_corrupt(release_dir, tmp_path, name, edit)) From 6b2e2ad60a602ebdc8f0c88872bd891c65ab3540 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 20:59:44 +0100 Subject: [PATCH 04/10] docs(connectors): document the BattLeDIM release and connector decisions --- .../datasets/kios/battledim/README.md | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md new file mode 100644 index 000000000..bbb94ea76 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md @@ -0,0 +1,185 @@ +# BattLeDIM: the release, and the decisions of this connector + +BattLeDIM 2020 (Battle of the Leakage Detection and Isolation Methods) released two years of +SCADA data from L-Town, a water distribution network, together with the 33 leaks the data +contains. +- The data is **simulated**. The release's own configuration names `L-TOWN_Real.inp` as the + network that generated it. +- The connector pins every file it downloads by sha256, so a changed file fails the build instead + of silently building something else. + +This document covers: +- what the release holds; +- how the connector maps it onto TimeF; +- the decisions the release leaves open; +- what a user should know before training on it. + +*(measured)* marks a number taken from the files, not from the dataset page. + +## Source, licence and citation + +**Source:** Zenodo record [4017659](https://zenodo.org/records/4017659), doi:10.5281/zenodo.4017659, published 2020-09-07. It has one version. + +**Licence:** CC BY 4.0, as stated on the Zenodo record. The organisers' GitHub repository +[KIOS-Research/BattLeDIM](https://github.com/KIOS-Research/BattLeDIM) states the same. A user must credit the authors. + +**Citation:** +> Vrachimis, S. G., Eliades, D. G., Taormina, R., Kapelan, Z., Ostfeld, A., Liu, S., Kyriakou, M., +> Pavlou, P., Qiu, M., and Polycarpou, M. M. (2022). Battle of the Leakage Detection and Isolation +> Methods. *Journal of Water Resources Planning and Management*, 148(12). +> doi:10.1061/(ASCE)WR.1943-5452.0001601 + +**Files the connector downloads** (13 of the 18 on the record): + +| File | What it holds | +| --- | --- | +| `2018_SCADA_*.csv`, `2019_SCADA_*.csv` | Pressures, Flows, Levels and Demands, one CSV per year and type | +| `2018_Leakages.csv`, `2019_Leakages.csv` | The outflow of every leak active that year, in m³/h | +| `dataset_configuration.yaml` | The configuration that generated the data: time range, the leak list, the sensor ids | +| `2018_Fixed_Leakages_Report.txt` | The repair times of the 2018 leaks that were fixed | +| `L-TOWN.inp` | The EPANET model given to participants (nominal parameters) | + +**Files it skips:** +- The two `.xlsx` files, which repeat the CSVs (README.txt of the release). +- `L-TOWN_Real.inp` (177 MB), the simulation model. Nothing in the tasks needs it. + +## The record + +The two years join into **one record** on one 5-minute grid: +- 2018-01-01 00:00 → 2019-12-31 23:55, 210 240 steps *(measured)*. +- The 2019 files continue the 2018 files with no gap. +- **Time zone:** the release writes naive timestamps and states no zone. The connector reads them as UTC. + +Each sensor type is one source. The signal names are the EPANET ids of `L-TOWN.inp`. + +| Source | Signals | Unit (README.txt of the release) | Range *(measured)* | +| --- | --- | --- | --- | +| Pressure sensors | 33: n1, n4, n31, n54, n105, n114, n163, n188, n215, n229, n288, n296, n332, n342, n410, n415, n429, n458, n469, n495, n506, n516, n519, n549, n613, n636, n644, n679, n722, n726, n740, n752, n769 | m (pressure head) | 25.7–57.0 | +| Flow sensors | PUMP_1 (pump into tank T1), p227, p235 (the two reservoir inlets) | m³/h | 0–205 | +| Tank level sensor | T1 | m | 1.95–3.90 | +| Automated meter readings | 82 nodes, the `amrs` list of the configuration | L/h | 0–37 796 | + +**Reading and checking:** +- **Values:** float64, as parsed from the decimal-comma text. Nothing is filled, trimmed or resampled. +- **Grid:** a step that is not 5 minutes fails the build. +- **Values:** a missing or non-finite value fails the build. +- **Sensors:** each year's columns must equal the sensor list of the configuration. +- **Network:** every sensor must exist in `L-TOWN.inp`, as a node or, for flow, as a link. + +## The leaks + +The configuration lists **33 leaks** *(measured)*: +- 15 abrupt and 18 incipient; +- hole diameters of 8.8–22.9 mm; +- peak outflows of 5.1–35.4 m³/h. + +| Year | Leaks starting | Active during the year | Repaired that year | Open at the end of 2019 | +| --- | --- | --- | --- | --- | +| 2018 | 14 | 14 | 10 | 4 carry over into 2019 | +| 2019 | 19 | 23 | 7 | 16 in total | + +An **abrupt** leak appears at full size at its start. An **incipient** leak grows from its start until its `peakTime`. + +Each leak becomes a set of **record annotations** over its active interval. The interval runs from the start to the step after the last step with outflow. +- `leak_pipe`: the pipe's EPANET id. +- `leak_type`: `abrupt` or `incipient`. +- `leak_hole_diameter`, in mm. +- `leak_peak_outflow`, in m³/h: the largest outflow in the leak tables. +- `leak_repaired`: whether the leak was repaired before the end of 2019. +- `leak_peak`: a time point at `peakTime`. + +**These are answers, never task inputs.** + +**Consistency checks.** The build fails if any of these disagree: +- the configuration, the two leak tables and the repair report; +- a leak's outflow and its configured start and end; +- a leak pipe and the pipes of `L-TOWN.inp`. + +The release passes all of them. + +## Windows and tasks + +**Windows:** a window is 14 days, starting every 7 days. +- The first week is context; the second week is the **target week**. +- The recording holds **103 windows** *(measured)*. +- Every leak start and every repair falls in exactly one target week *(measured)*: 33 starts, 17 repairs. +- 33 windows have a new leak, 16 have a repair, and 59 have neither *(measured)*. + +**Tasks:** each window gives four tasks, all scoped to the window. That is 412 tasks. + +| Task | Answer | Source of the answer | +| --- | --- | --- | +| `ClassificationTask` | the types of the leaks that start in the target week: a multi-label set {`abrupt`, `incipient`}, empty when none starts | configuration | +| `TemporalLocalizationTask` (exhaustive) | a time point at each leak start in the target week | configuration | +| `ScalarPredictionTask` `mean_leak_outflow` | the mean total leak outflow over the target week, m³/h | leak tables | +| `AnswerTask` | a short description of every leak that starts or is repaired in the target week | configuration, leak tables, `L-TOWN.inp` | + +**How the text is made.** Fixed templates fill in the answer; no model writes it. + +*New leak:* +> A sudden leak started on pipe {pipe} on {start}, with a hole of {diameter} mm and an outflow of up +> to {max outflow in the window} m3/h in this window. The nearest pressure sensor is {sensor}, {n} +> links away. + +For an incipient leak, the sentence opens "A gradually growing leak, peaking on {peakTime},". + +*Repair:* +> The leak on pipe {pipe} was repaired on {end}; its outflow before the repair was {outflow} m3/h. + +*Neither:* +> No leak started or was repaired in the last 7 days. + +*Always last:* +> At the end of the window {k} leaks are active, with a total outflow of {total} m3/h. + +**Nearest sensor.** "Nearest" counts links in the `L-TOWN.inp` graph (pipes, pumps and valves) from the nearer end of the leaking pipe. Ties go to the sensor listed first in the configuration. + +**Example input and output.** The window that starts on 2018-01-29 (target week 2018-02-05) holds all 119 signals over 14 days. Its text target is: + +> The leak on pipe p232 was repaired on 2018-02-10 09:20; its outflow before the repair was 26.0 +> m3/h. At the end of the window 2 leaks are active, with a total outflow of 9.8 m3/h. + +**Task metadata:** `window_index`, `window_start`, `target_week_start`, `calendar_block`, `new_leaks` and `repaired_leaks`. + +## The split + +The release's own partition was 2018 for training and 2019 for evaluation. **This connector does not use it.** +- **Why:** 74 of the 82 meter series repeat 2018 exactly in 2019 *(measured)*. As a result, the 2019 inflow minus the 2018 inflow at the same calendar time equals the 2019 leak outflow minus the 2018 leak outflow: slope 1.00, daily correlation 0.989 *(measured)*. +- **The risk:** a year split would test on a demand background the model saw in training. + +**What it does instead:** +- Windows are grouped by the **calendar block** of their target week: the ISO week, in blocks of four. +- A block holds the same weeks of both years. +- `StratifiedSplitter` (8/1/1 folds, stratified on "a new leak starts", seed 42) assigns whole blocks. + +**Result** *(measured)*: + +| Partition | Windows | With a new leak | Tasks | Calendar blocks | +| --- | --- | --- | --- | --- | +| train | 79 | 27 | 316 | 0, 1, 2, 3, 5, 7, 8, 9, 10, 11 | +| validation | 8 | 3 | 32 | 4 | +| test | 16 | 3 | 64 | 6, 12 | + +**Overlap:** the context week of a window can lie in a neighbouring block, so context weeks can overlap across partitions. Target weeks never do. + +## Release details a user should know + +- **Simulated data.** The SCADA values come from a hydraulic simulation of `L-TOWN_Real.inp`. They are not field measurements. +- **Leaks almost never stop.** At least one leak is active in 97.8 % of 2018 and in 100 % of 2019, with up to 16 at once *(measured)*. There is almost no leak-free baseline. +- **Incipient onsets are not visible at once.** For incipient leaks, the leak tables show the first non-zero outflow 3 hours to 2.5 days after the configured start, because the early ramp rounds to zero *(measured)*. The localization target is the configured start. +- **One end time differs.** For p800, the last non-zero outflow is 5 minutes after the configured end *(measured)*. The connector keeps the configured end. +- **Small signatures.** Our own preliminary check, outside this connector, found that a 30 m³/h leak lowers night pressure by at most about 0.5 m at the nearest sensors. Leaks under about 5 m³/h are hard to see. + +## Limitations and intended use + +**Limitations:** +- 33 leak events on one network. +- Small validation and test partitions (8 and 16 windows). +- Text targets are templates, not expert explanations. +- Some window statements (the active leaks, the outflow) come from labels that a model can only infer indirectly from the sensors. + +**Intended use:** +- Training and evaluating time-series language models that detect, time, size and describe leaks from SCADA data. +- Studying explanations that can be checked against a known ground truth. + +*Added during the TimeNet hackathon, Round 1 (November 2026).* From 566dfaa2911e32a19ff91d021178c95accd80929 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 21:49:03 +0100 Subject: [PATCH 05/10] docs(connectors): drop the external signal-check note from the BattLeDIM README --- .../src/timenet_connectors/datasets/kios/battledim/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md index bbb94ea76..bb58e3dd7 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md @@ -168,7 +168,6 @@ The release's own partition was 2018 for training and 2019 for evaluation. **Thi - **Leaks almost never stop.** At least one leak is active in 97.8 % of 2018 and in 100 % of 2019, with up to 16 at once *(measured)*. There is almost no leak-free baseline. - **Incipient onsets are not visible at once.** For incipient leaks, the leak tables show the first non-zero outflow 3 hours to 2.5 days after the configured start, because the early ramp rounds to zero *(measured)*. The localization target is the configured start. - **One end time differs.** For p800, the last non-zero outflow is 5 minutes after the configured end *(measured)*. The connector keeps the configured end. -- **Small signatures.** Our own preliminary check, outside this connector, found that a 30 m³/h leak lowers night pressure by at most about 0.5 m at the nearest sensors. Leaks under about 5 m³/h are hard to see. ## Limitations and intended use From cba02281f64745b67709fbb6f9da0d80d71da0cd Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 22:19:13 +0100 Subject: [PATCH 06/10] fix(connectors): keep BattLeDIM answers within their window and check more of the release - The text no longer states an incipient leak's peak when it falls after the window; it says the leak was still growing. Zero rounded outflow reads as "no measurable outflow yet". - Active leaks at the end of a window come from the leak list, not from rounded outflow, so the text cannot contradict its own new leaks. - The build fails if a leak start or repair falls outside every target week, if the repair report misses or repeats a 2018 repair, if a sensor is listed twice, or if a CSV cell is not a timestamp or a number. - Share one microsecond conversion and drop dead code. --- .../datasets/kios/battledim/connector.py | 7 +-- .../datasets/kios/battledim/source.py | 57 ++++++++++--------- .../datasets/kios/battledim/tasks.py | 49 ++++++++++------ .../kios/battledim/tests/test_connector.py | 32 +++++++++++ 4 files changed, 97 insertions(+), 48 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py index 3f7a8e2e7..d1a3e9298 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py @@ -141,9 +141,8 @@ def _make_record(release: Release) -> Record: ) annotations = [] for leak in release.leaks: - start_us = int((leak.start - release.start).total_seconds() * 1_000_000) - end_us = int((leak.end + STEP - release.start).total_seconds() * 1_000_000) - span = TimeInterval.micros(start_us, end_us) + # From the configured start to the step after the configured end. + span = TimeInterval.micros(release.micros(leak.start), release.micros(leak.end + STEP)) outflow = release.leak_outflow[leak.pipe] annotations += [ Annotation(key="leak_pipe", value=leak.pipe, span=span, description="The leaking pipe's EPANET id."), @@ -171,7 +170,7 @@ def _make_record(release: Release) -> Record: Annotation( key="leak_peak", value=leak.pipe, - span=TimePoint.micros(int((leak.peak - release.start).total_seconds() * 1_000_000)), + span=TimePoint.micros(release.micros(leak.peak)), description="When the leak reaches its full size; the start, for an abrupt leak.", ), ] diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py index f0c7e9e9d..6e7ef5ceb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py @@ -9,7 +9,7 @@ from __future__ import annotations from collections import deque -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path @@ -50,7 +50,7 @@ def _parse_time(text: str, where: str) -> datetime: moment = datetime.strptime(text.strip(), _TIME_FORMAT).replace(tzinfo=UTC) except ValueError as error: raise TimeNetConnectorError(f"{where}: {text!r} is not a {_TIME_FORMAT} time") from error - if moment.minute % 5 or moment.second: + if moment.minute % 5: raise TimeNetConnectorError(f"{where}: {text!r} is not on the 5 minute grid") return moment @@ -75,8 +75,8 @@ def read_table(path: Path) -> Table: The table, with every value column as float64. Raises: - TimeNetConnectorError: If the file has no value column, a step that is not 5 minutes, a - missing value, or a value that is not finite. + TimeNetConnectorError: If the file has no value column, a cell that is not a timestamp or a + number, a step that is not 5 minutes, a missing value, or a value that is not finite. """ with path.open(encoding="utf-8") as handle: header = handle.readline().strip().split(";") @@ -85,15 +85,18 @@ def read_table(path: Path) -> Table: names = header[1:] if len(set(names)) != len(names): raise TimeNetConnectorError(f"{path.name}: repeats a column name") - table = pacsv.read_csv( - path, - parse_options=pacsv.ParseOptions(delimiter=";"), - convert_options=pacsv.ConvertOptions( - decimal_point=",", - column_types={"Timestamp": pa.timestamp("s"), **dict.fromkeys(names, pa.float64())}, - timestamp_parsers=["%Y-%m-%d %H:%M:%S"], - ), - ) + try: + table = pacsv.read_csv( + path, + parse_options=pacsv.ParseOptions(delimiter=";"), + convert_options=pacsv.ConvertOptions( + decimal_point=",", + column_types={"Timestamp": pa.timestamp("s"), **dict.fromkeys(names, pa.float64())}, + timestamp_parsers=["%Y-%m-%d %H:%M:%S"], + ), + ) + except pa.ArrowInvalid as error: + raise TimeNetConnectorError(f"{path.name}: cannot be read as timestamps and numbers: {error}") from error times = table.column("Timestamp").to_numpy().astype("datetime64[s]").astype(np.int64) if len(times) == 0: raise TimeNetConnectorError(f"{path.name}: holds no rows") @@ -217,7 +220,7 @@ def nearest(self, pipe: str, sensors: list[str]) -> tuple[str, int]: queue = deque(hops) while queue: node = queue.popleft() - for neighbour in sorted(self._neighbours[node]): + for neighbour in self._neighbours[node]: if neighbour not in hops: hops[neighbour] = hops[node] + 1 queue.append(neighbour) @@ -252,6 +255,8 @@ def __init__(self, root: Path) -> None: if not isinstance(listed, list) or not listed: raise TimeNetConnectorError(f"dataset_configuration.yaml lists no {key}") self.sensors[kind] = [str(sensor) for sensor in listed] + if len(set(self.sensors[kind])) != len(self.sensors[kind]): + raise TimeNetConnectorError(f"dataset_configuration.yaml lists a {key} entry twice") self.scada = {kind: self._join(name, self.sensors[kind]) for kind, (name, _) in SCADA_FILES.items()} self.start = self.scada["pressure"][0] self.n_steps = len(next(iter(self.scada["pressure"][1].values()))) @@ -277,6 +282,10 @@ def step_of(self, moment: datetime) -> int: """The index of ``moment`` on the recording's 5 minute grid.""" return int((moment - self.start) / STEP) + def micros(self, moment: datetime) -> int: + """The offset of ``moment`` from the start of the recording, in whole microseconds.""" + return (moment - self.start) // timedelta(microseconds=1) + def _path(self, name: str) -> Path: path = self.root / name if not path.is_file(): @@ -352,7 +361,8 @@ def _read_leaks(self, config: dict) -> list[Leak]: raise TimeNetConnectorError("dataset_configuration.yaml lists a pipe in two leaks") if {leak.pipe for leak in leaks} != set(self.leak_outflow): raise TimeNetConnectorError("the leak tables and dataset_configuration.yaml name different pipes") - report = self._yaml("2018_Fixed_Leakages_Report.txt") + # A report of comment lines only, with no repair, reads as an empty YAML document. + report = self._yaml("2018_Fixed_Leakages_Report.txt") or [] if not isinstance(report, list): raise TimeNetConnectorError("2018_Fixed_Leakages_Report.txt is not a list of repairs") repairs = {} @@ -360,6 +370,8 @@ def _read_leaks(self, config: dict) -> list[Leak]: fields = [field.strip() for field in str(line).split(",")] if len(fields) != _REPAIR_FIELDS: raise TimeNetConnectorError(f"2018_Fixed_Leakages_Report.txt: line {line!r} is not 'linkID, endTime'") + if fields[0] in repairs: + raise TimeNetConnectorError(f"2018_Fixed_Leakages_Report.txt repairs {fields[0]} twice") repairs[fields[0]] = _parse_time(fields[1], "2018_Fixed_Leakages_Report.txt") checked = [] for leak in leaks: @@ -367,21 +379,14 @@ def _read_leaks(self, config: dict) -> list[Leak]: raise TimeNetConnectorError(f"leak {leak.pipe} is outside the recording") if leak.pipe in repairs and repairs[leak.pipe] != leak.end: raise TimeNetConnectorError(f"leak {leak.pipe}: the repair report and the configuration disagree") + # The report covers the historical year: every leak that ends within 2018 must be in it. + if leak.end.year == YEARS[0] and leak.pipe not in repairs: + raise TimeNetConnectorError(f"leak {leak.pipe} ends in {YEARS[0]} but the repair report lacks it") outflow = self.leak_outflow[leak.pipe] active = np.flatnonzero(outflow > 0) if len(active) == 0 or active[0] < self.step_of(leak.start) or active[-1] > self.step_of(leak.end) + 1: raise TimeNetConnectorError(f"leak {leak.pipe}: its outflow lies outside its configured start and end") - checked.append( - Leak( - pipe=leak.pipe, - start=leak.start, - end=leak.end, - diameter_m=leak.diameter_m, - leak_type=leak.leak_type, - peak=leak.peak, - repaired=leak.end < self.end, - ) - ) + checked.append(replace(leak, repaired=leak.end < self.end)) if set(repairs) - {leak.pipe for leak in checked if leak.repaired}: raise TimeNetConnectorError("2018_Fixed_Leakages_Report.txt repairs a leak the configuration keeps open") return sorted(checked, key=lambda leak: (leak.start, leak.pipe)) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py index 6e5f86ead..b52ff1060 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py @@ -15,6 +15,7 @@ import numpy as np from timenet.dataset import Record +from timenet.errors import TimeNetConnectorError from timenet.json import JsonMapping, JsonValue from timenet.types import ( AnswerTask, @@ -39,6 +40,8 @@ # Four ISO weeks make one calendar block. Windows whose target weeks share a block share a partition # in both years, because 2019 repeats most of the 2018 demand series (see README.md). WEEKS_PER_BLOCK = 4 +# The leak tables write two decimals, so a smaller outflow reads as none. +_MEASURABLE = 0.05 NEW_LEAK_TYPES = LabelSet( name="new_leak_types", @@ -82,15 +85,15 @@ def block(self) -> int: return (self.target_start.isocalendar().week - 1) // WEEKS_PER_BLOCK -def _micros(release: Release, moment: datetime) -> int: - return int((moment - release.start) / timedelta(microseconds=1)) - - def windows(release: Release) -> list[Window]: """List every window that fits in the recording, in time order. Returns: The windows, each with the leaks that start or are repaired in its target week. + + Raises: + TimeNetConnectorError: If a leak starts or is repaired outside every target week, so that no + task would mention it. """ found = [] start = release.start @@ -102,11 +105,16 @@ def windows(release: Release) -> list[Window]: index=len(found), start=start, new=tuple(leak for leak in release.leaks if target_start <= leak.start < end), - # The release writes a repair time as the leak's end: its last step with outflow. + # The release writes a repair time as the leak's configured end. repaired=tuple(leak for leak in release.leaks if leak.repaired and target_start <= leak.end < end), ) ) start += STRIDE + for leak in release.leaks: + if sum(leak in window.new for window in found) != 1: + raise TimeNetConnectorError(f"leak {leak.pipe} does not start in exactly one target week") + if leak.repaired and sum(leak in window.repaired for window in found) != 1: + raise TimeNetConnectorError(f"the repair of leak {leak.pipe} is not in exactly one target week") return found @@ -145,15 +153,19 @@ def explain(release: Release, window: Window) -> str: sentences = [] for leak in window.new: sensor, hops = release.network.nearest(leak.pipe, release.sensors["pressure"]) - growth = ( - "A sudden leak" - if leak.leak_type == "abrupt" - else f"A gradually growing leak, peaking on {leak.peak:%Y-%m-%d %H:%M}," - ) + if leak.leak_type == "abrupt": + kind, growth = "A sudden leak", "" + elif leak.peak < window.end: + kind, growth = "A gradually growing leak", f" and peaked on {leak.peak:%Y-%m-%d %H:%M}" + else: + # A later peak is not in the window, so the text does not state it. + kind, growth = "A gradually growing leak", " and was still growing at the end of this window" + outflow = _outflow(release, leak, leak.start, window.end) + # The leak tables round an incipient leak's first hours to zero outflow. + flow = f"an outflow of up to {outflow:.1f} m3/h" if outflow >= _MEASURABLE else "no measurable outflow yet" sentences.append( - f"{growth} started on pipe {leak.pipe} on {leak.start:%Y-%m-%d %H:%M}, " - f"with a hole of {leak.diameter_m * 1000:.1f} mm and an outflow of up to " - f"{_outflow(release, leak, leak.start, window.end):.1f} m3/h in this window. " + f"{kind} started on pipe {leak.pipe} on {leak.start:%Y-%m-%d %H:%M}{growth}, " + f"with a hole of {leak.diameter_m * 1000:.1f} mm and {flow} in this window. " f"The nearest pressure sensor is {sensor}, {_hops(hops)} away." ) for leak in window.repaired: @@ -164,9 +176,10 @@ def explain(release: Release, window: Window) -> str: ) if not sentences: sentences.append("No leak started or was repaired in the last 7 days.") - last = release.step_of(window.end) - 1 - active = [pipe for pipe, outflow in release.leak_outflow.items() if outflow[last] > 0] - total = sum(float(release.leak_outflow[pipe][last]) for pipe in active) + # Active means the leak list's interval holds the last step, whatever the rounded outflow says. + last = window.end - STEP + active = [leak.pipe for leak in release.leaks if leak.start <= last <= leak.end] + total = sum(float(release.leak_outflow[pipe][release.step_of(last)]) for pipe in active) sentences.append( f"At the end of the window {len(active)} leak{'s are' if len(active) != 1 else ' is'} active, " f"with a total outflow of {total:.1f} m3/h." @@ -180,7 +193,7 @@ def window_tasks(release: Release, record: Record, window: Window, split: Split) Returns: The classification, localization, outflow and text tasks, in that order. """ - scope = TimeInterval.micros(_micros(release, window.start), _micros(release, window.end)) + scope = TimeInterval.micros(release.micros(window.start), release.micros(window.end)) target = slice(release.step_of(window.target_start), release.step_of(window.end)) outflow = np.sum([series[target] for series in release.leak_outflow.values()], axis=0) new_leaks: list[JsonValue] = [leak.pipe for leak in window.new] @@ -211,7 +224,7 @@ def window_tasks(release: Release, record: Record, window: Window, split: Split) metadata=metadata, prompt=LOCATE_PROMPT, mode=LocalizationMode.EXHAUSTIVE, - targets=tuple(TimePoint.micros(_micros(release, leak.start)) for leak in window.new), + targets=tuple(TimePoint.micros(release.micros(leak.start)) for leak in window.new), ), ScalarPredictionTask( inputs=inputs, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py index 69f6a1549..fb22440b6 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py @@ -199,6 +199,19 @@ def test_a_repair_and_a_quiet_week_are_explained(dataset) -> None: assert read_value(answer.targets[0]).startswith("No leak started or was repaired in the last 7 days.") +def test_a_growing_leak_states_no_peak_after_the_window(dataset) -> None: + # p5 starts on 2018-11-01 and peaks on 2018-11-15, after this window ends on 2018-11-05. + classify, _, _, answer = _tasks_of_week(dataset, "2018-10-29 00:00") + assert classify.targets == ("incipient",) + text = read_value(answer.targets[0]) + assert text.startswith( + "A gradually growing leak started on pipe p5 on 2018-11-01 00:00 and was still growing at the end of " + "this window, with a hole of 10.0 mm and an outflow of up to" + ) + assert "2018-11-15" not in text + assert "At the end of the window 1 leak is active" in text + + def test_split_keeps_each_calendar_block_in_one_partition(dataset) -> None: tasks = list(dataset.get_all()) partitions: dict[int, set] = {} @@ -259,8 +272,27 @@ def _corrupt(release_dir: Path, tmp_path: Path, name: str, edit) -> Path: ("dataset_configuration.yaml", lambda text: text.replace("- p6,", "- p4,"), "different pipes"), ("2018_Fixed_Leakages_Report.txt", lambda text: text.replace("10:25", "10:30"), "disagree"), ("L-TOWN.inp", lambda text: text.replace(" p5 n4 n5", " p9 n4 n5"), "no pipe for the leaks"), + ("2018_SCADA_Flows.csv", lambda text: text.replace(";44,04", ";abc", 1), "cannot be read"), + ("2018_Fixed_Leakages_Report.txt", lambda text: text.replace("- p3, 2018-03-23 10:25\n", ""), "lacks it"), + ("2018_Fixed_Leakages_Report.txt", lambda text: text + "- p3, 2018-03-23 10:25\n", "twice"), + ("dataset_configuration.yaml", lambda text: text.replace("- n4\nflow", "- n1\nflow"), "twice"), ], ) def test_inconsistent_release_raises(release_dir: Path, tmp_path: Path, name: str, edit, message: str) -> None: with pytest.raises(TimeNetConnectorError, match=message): Release(_corrupt(release_dir, tmp_path, name, edit)) + + +def test_a_leak_outside_every_target_week_raises(release_dir: Path, tmp_path: Path) -> None: + # A leak in the first week sits only in the context week of the first window. + release = _corrupt( + release_dir, + tmp_path, + "dataset_configuration.yaml", + lambda text: text.replace( + "- p3, 2018-03-05 15:45, 2018-03-23 10:25, 0.022916, abrupt, 2018-03-05 15:45", + "- p3, 2018-01-03 15:45, 2018-03-23 10:25, 0.022916, abrupt, 2018-01-03 15:45", + ), + ) + with pytest.raises(TimeNetConnectorError, match="exactly one target week"): + BattLeDIMConnector().convert([BattLeDIMSource(release)]) From 5f5755b4e4d93783139dcd7e8b11306420481b56 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 22:19:14 +0100 Subject: [PATCH 07/10] docs(connectors): describe the BattLeDIM answer rules, checks and split overlap --- .../datasets/kios/battledim/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md index bb58e3dd7..22157f512 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md @@ -80,7 +80,7 @@ The configuration lists **33 leaks** *(measured)*: An **abrupt** leak appears at full size at its start. An **incipient** leak grows from its start until its `peakTime`. -Each leak becomes a set of **record annotations** over its active interval. The interval runs from the start to the step after the last step with outflow. +Each leak becomes a set of **record annotations** over its active interval. The interval runs from the configured start to the step after the configured end. For one leak, p800, the outflow runs one step past that end (see below). - `leak_pipe`: the pipe's EPANET id. - `leak_type`: `abrupt` or `incipient`. - `leak_hole_diameter`, in mm. @@ -91,18 +91,18 @@ Each leak becomes a set of **record annotations** over its active interval. The **These are answers, never task inputs.** **Consistency checks.** The build fails if any of these disagree: -- the configuration, the two leak tables and the repair report; +- the configuration, the two leak tables and the repair report, which must list every leak that ends in 2018, exactly once; - a leak's outflow and its configured start and end; - a leak pipe and the pipes of `L-TOWN.inp`. -The release passes all of them. +It also fails on a sensor listed twice in the configuration, or a CSV cell that is not a timestamp or a number. The release passes all of these checks. ## Windows and tasks **Windows:** a window is 14 days, starting every 7 days. - The first week is context; the second week is the **target week**. - The recording holds **103 windows** *(measured)*. -- Every leak start and every repair falls in exactly one target week *(measured)*: 33 starts, 17 repairs. +- Every leak start and every repair falls in exactly one target week: 33 starts, 17 repairs. The build checks this and fails otherwise, so no leak drops out of the tasks. - 33 windows have a new leak, 16 have a repair, and 59 have neither *(measured)*. **Tasks:** each window gives four tasks, all scoped to the window. That is 412 tasks. @@ -121,7 +121,7 @@ The release passes all of them. > to {max outflow in the window} m3/h in this window. The nearest pressure sensor is {sensor}, {n} > links away. -For an incipient leak, the sentence opens "A gradually growing leak, peaking on {peakTime},". +For an incipient leak, the sentence opens "A gradually growing leak started on pipe {pipe} on {start}", followed by "and peaked on {peakTime}" if the peak lies inside the window, or "and was still growing at the end of this window" if it does not. A peak after the window is not stated, because the scoped input cannot show it. When the leak tables round the outflow in the window to zero (an incipient leak's first hours), the text says "no measurable outflow yet" instead of a number. This applies to 9 windows *(measured)*. *Repair:* > The leak on pipe {pipe} was repaired on {end}; its outflow before the repair was {outflow} m3/h. @@ -132,6 +132,8 @@ For an incipient leak, the sentence opens "A gradually growing leak, peaking on *Always last:* > At the end of the window {k} leaks are active, with a total outflow of {total} m3/h. +A leak counts as active when the leak list's interval holds the window's last step. The total is the leak tables' outflow at that step. + **Nearest sensor.** "Nearest" counts links in the `L-TOWN.inp` graph (pipes, pumps and valves) from the nearer end of the leaking pipe. Ties go to the sensor listed first in the configuration. **Example input and output.** The window that starts on 2018-01-29 (target week 2018-02-05) holds all 119 signals over 14 days. Its text target is: @@ -162,6 +164,8 @@ The release's own partition was 2018 for training and 2019 for evaluation. **Thi **Overlap:** the context week of a window can lie in a neighbouring block, so context weeks can overlap across partitions. Target weeks never do. +**One leaked day per block boundary:** the 2019 demand repeats 2018 by calendar date, but 2019 weekdays fall one day later. A 2019 target week (Monday to Sunday) therefore shares one date with the 2018 week before it. Where a block boundary separates partitions, one day of a target week's demand background also appears in the neighbouring partition. Week-aligned windows cannot avoid this. + ## Release details a user should know - **Simulated data.** The SCADA values come from a hydraulic simulation of `L-TOWN_Real.inp`. They are not field measurements. From e390f5bcd8d61921db5cca7aa7e98b5fef797f73 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 22:44:28 +0100 Subject: [PATCH 08/10] fix(connectors): keep BattLeDIM leak labels off the input record and embargo overlapping windows - The leaks become dataset annotations instead of record annotations. Every task reads the record, and loaders return a record's annotations with each item, so the record now holds sensor signals only. - A window whose context week is a validation or test target week of another partition gets no tasks, so training never sees the signals of a held-out target week. - Small outflows keep their second decimal; only zero reads as no outflow. - Only p800 may keep outflow one step past its configured end, abrupt leaks must peak at their start, and malformed configuration sections raise TimeNetConnectorError. - Compute the total outflow once, keep the window geometry in one place, return a Table from the year join and read headers as utf-8-sig. --- .../datasets/kios/battledim/connector.py | 88 +++++++++---------- .../datasets/kios/battledim/source.py | 74 +++++++++++----- .../datasets/kios/battledim/tasks.py | 52 +++++++---- .../kios/battledim/tests/test_connector.py | 62 +++++++++---- 4 files changed, 176 insertions(+), 100 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py index d1a3e9298..e0f53922c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/connector.py @@ -3,7 +3,7 @@ BattLeDIM 2020 (Battle of the Leakage Detection and Isolation Methods) released simulated SCADA data of the L-Town network for 2018 and 2019: 33 pressure sensors, 3 flow sensors, 1 tank level sensor and 82 automated meter readings, every 5 minutes, and the 33 leaks that the simulation -contained. The two years form one Record. The leaks become record annotations, and two-week windows +contained. The two years form one Record. The leaks become dataset annotations, and two-week windows of the record become the tasks of :mod:`~timenet_connectors.datasets.kios.battledim.tasks`. """ @@ -20,9 +20,9 @@ from timenet.dataset import Record, Signal, Source, TimeFDataset from timenet.dataset.axis import RegularAxis from timenet.errors import TimeNetConnectorError -from timenet.types import Annotation, TimeInterval, TimeOrigin, TimePoint, TimeSeriesSpec, ureg -from timenet_connectors.datasets.kios.battledim.source import SCADA_FILES, STEP, Release -from timenet_connectors.datasets.kios.battledim.tasks import split_windows, window_tasks, windows +from timenet.types import Annotation, TimeOrigin, TimeSeriesSpec, ureg +from timenet_connectors.datasets.kios.battledim.source import SCADA_FILES, STEP, Leak, Release +from timenet_connectors.datasets.kios.battledim.tasks import embargoed, split_windows, window_tasks, windows from timenet_connectors.download import Artifact, download_files @@ -106,15 +106,50 @@ def convert(self, raw_refs: list[BattLeDIMSource]) -> TimeFDataset: release = Release(raw_refs[0].root) dataset = TimeFDataset(metadata=self.metadata()) record = dataset.add_record(record=self._make_record(release)) + # The leaks are the answers. They go on the dataset, not on the record that every task reads, + # so nothing that serves the record to a model serves them too. + for leak in release.leaks: + dataset.annotate(self._leak_annotation(release, leak)) found = windows(release) split_of = split_windows(found) + skipped = embargoed(found, split_of) for window in found: - dataset.add_tasks(tasks=window_tasks(release, record, window, split_of[window.index])) + if window.index not in skipped: + dataset.add_tasks(tasks=window_tasks(release, record, window, split_of[window.index])) return dataset + @staticmethod + def _leak_annotation(release: Release, leak: Leak) -> Annotation: + """Describe one leak as a dataset annotation: its pipe, with the leak list's facts as metadata. + + A dataset annotation has no time span, so the metadata gives each time both as text and as + microseconds from the record start. The interval runs from the configured start to the step + after the configured end. + + Returns: + The annotation, keyed ``leak``. + """ + return Annotation( + key="leak", + value=leak.pipe, + description="One leak of the release: the leaking pipe's EPANET id, with its facts as metadata.", + metadata={ + "leak_type": leak.leak_type, + "start": f"{leak.start:%Y-%m-%d %H:%M}", + "end": f"{leak.end:%Y-%m-%d %H:%M}", + "peak": f"{leak.peak:%Y-%m-%d %H:%M}", + "start_us": release.micros(leak.start), + "end_exclusive_us": release.micros(leak.end + STEP), + "peak_us": release.micros(leak.peak), + "hole_diameter_mm": round(leak.diameter_m * 1000, 4), + "peak_outflow_m3h": float(release.leak_outflow[leak.pipe].max()), + "repaired": leak.repaired, + }, + ) + @staticmethod def _make_record(release: Release) -> Record: - """Build the one record: a source per sensor type and an annotation set per leak. + """Build the one record: a source per sensor type, and no labels. Returns: The record, starting at the first step of 2018. @@ -122,7 +157,7 @@ def _make_record(release: Release) -> Record: axis = RegularAxis(period_us=Fraction(int(STEP.total_seconds() * 1_000_000))) sources = [] for kind in SCADA_FILES: - columns = release.scada[kind][1] + columns = release.scada[kind].columns signals = tuple( Signal.from_loader( spec=SPECS[kind], @@ -134,48 +169,11 @@ def _make_record(release: Release) -> Record: for sensor in release.sensors[kind] ) sources.append(Source(name=SOURCE_NAMES[kind], signals=signals)) - record = Record( + return Record( sources=tuple(sources), start_time=TimeOrigin(release.start), metadata={"network": "L-Town", "source_doi": "10.5281/zenodo.4017659"}, ) - annotations = [] - for leak in release.leaks: - # From the configured start to the step after the configured end. - span = TimeInterval.micros(release.micros(leak.start), release.micros(leak.end + STEP)) - outflow = release.leak_outflow[leak.pipe] - annotations += [ - Annotation(key="leak_pipe", value=leak.pipe, span=span, description="The leaking pipe's EPANET id."), - Annotation(key="leak_type", value=leak.leak_type, span=span, description="abrupt or incipient."), - Annotation( - key="leak_hole_diameter", - value=round(leak.diameter_m * 1000, 4), - unit="mm", - span=span, - description="The diameter of the simulated hole.", - ), - Annotation( - key="leak_peak_outflow", - value=float(outflow.max()), - unit="m^3/h", - span=span, - description="The largest outflow in the leak tables.", - ), - Annotation( - key="leak_repaired", - value=leak.repaired, - span=span, - description="Whether the leak was repaired before the end of 2019.", - ), - Annotation( - key="leak_peak", - value=leak.pipe, - span=TimePoint.micros(release.micros(leak.peak)), - description="When the leak reaches its full size; the start, for an abrupt leak.", - ), - ] - record.add_annotations(annotations) - return record CONNECTOR = BattLeDIMConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py index 6e7ef5ceb..c055be89f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/source.py @@ -35,6 +35,9 @@ _LEAK_FIELDS = 6 # linkID, startTime, endTime, leakDiameter (m), leakType, peakTime _REPAIR_FIELDS = 2 # linkID, endTime _LINK_SECTIONS = frozenset({"[PIPES]", "[PUMPS]", "[VALVES]"}) +# The leak tables keep this leak's outflow for one step after its configured end. The connector keeps +# the configured end, and README.md lists the difference. +_END_ONE_STEP_LATE = frozenset({"p800"}) def _parse_time(text: str, where: str) -> datetime: @@ -78,7 +81,7 @@ def read_table(path: Path) -> Table: TimeNetConnectorError: If the file has no value column, a cell that is not a timestamp or a number, a step that is not 5 minutes, a missing value, or a value that is not finite. """ - with path.open(encoding="utf-8") as handle: + with path.open(encoding="utf-8-sig") as handle: header = handle.readline().strip().split(";") if header[0] != "Timestamp" or len(header) == 1: raise TimeNetConnectorError(f"{path.name}: expected a Timestamp column and sensor columns, got {header}") @@ -163,6 +166,8 @@ def _parse_leak(line: object, where: str) -> Leak: ) if not leak.start <= leak.peak <= leak.end: raise TimeNetConnectorError(f"{where}: leak {pipe} peaks outside its start and end") + if leak.leak_type == "abrupt" and leak.peak != leak.start: + raise TimeNetConnectorError(f"{where}: abrupt leak {pipe} peaks after its start") return leak @@ -258,17 +263,21 @@ def __init__(self, root: Path) -> None: if len(set(self.sensors[kind])) != len(self.sensors[kind]): raise TimeNetConnectorError(f"dataset_configuration.yaml lists a {key} entry twice") self.scada = {kind: self._join(name, self.sensors[kind]) for kind, (name, _) in SCADA_FILES.items()} - self.start = self.scada["pressure"][0] - self.n_steps = len(next(iter(self.scada["pressure"][1].values()))) - for kind, (start, columns) in self.scada.items(): - if start != self.start or len(next(iter(columns.values()))) != self.n_steps: + self.start = self.scada["pressure"].start + self.n_steps = self.scada["pressure"].n_rows + for kind, table in self.scada.items(): + if table.start != self.start or table.n_rows != self.n_steps: raise TimeNetConnectorError(f"the {kind} tables do not cover the pressure tables' time range") - times = config.get("times") or {} + times = config.get("times") + if not isinstance(times, dict): + raise TimeNetConnectorError("dataset_configuration.yaml has no times mapping") if _parse_time(str(times.get("StartTime")), "times") != self.start or ( _parse_time(str(times.get("EndTime")), "times") != self.end ): raise TimeNetConnectorError("dataset_configuration.yaml times differ from the SCADA tables") self.leak_outflow = self._join_leaks() + # The total outflow of every leak at each step, in m3/h. + self.total_outflow = np.sum(list(self.leak_outflow.values()), axis=0) self.leaks = self._read_leaks(config) self.network = Network(self._path("L-TOWN.inp")) self._check_network() @@ -298,11 +307,11 @@ def _yaml(self, name: str) -> object: except yaml.YAMLError as error: raise TimeNetConnectorError(f"{name} is not valid YAML") from error - def _join(self, name: str, sensors: list[str]) -> tuple[datetime, dict[str, np.ndarray]]: + def _join(self, name: str, sensors: list[str]) -> Table: """Join the 2018 and 2019 tables of one sensor type into one recording. Returns: - The first timestamp, and each sensor's values over both years. + One table over both years, with the sensors in configuration order. Raises: TimeNetConnectorError: If the years name other sensors than the configuration lists, or @@ -317,9 +326,10 @@ def _join(self, name: str, sensors: list[str]) -> tuple[datetime, dict[str, np.n first, second = tables if second.start != first.start + STEP * first.n_rows: raise TimeNetConnectorError(f"{YEARS[1]}_{name}.csv does not continue {YEARS[0]}_{name}.csv") - return first.start, { - sensor: np.concatenate([first.columns[sensor], second.columns[sensor]]) for sensor in sensors - } + return Table( + start=first.start, + columns={sensor: np.concatenate([first.columns[sensor], second.columns[sensor]]) for sensor in sensors}, + ) def _join_leaks(self) -> dict[str, np.ndarray]: """Join the two leak tables into one outflow series per pipe, zero where a year does not list it. @@ -344,23 +354,15 @@ def _join_leaks(self) -> dict[str, np.ndarray]: raise TimeNetConnectorError("the leak tables do not cover the SCADA tables") return outflow - def _read_leaks(self, config: dict) -> list[Leak]: - """Read the configured leaks and check them against the leak tables and the repair report. + def _read_repairs(self) -> dict[str, datetime]: + """Read ``2018_Fixed_Leakages_Report.txt``: one ``linkID, endTime`` line per repaired leak. Returns: - The leaks in order of their start, each marked as repaired or not. + The repair time of each listed pipe. Raises: - TimeNetConnectorError: If a configured leak is outside the recording or its leak table, - a leak table has a pipe the configuration lacks, or a repair time disagrees. + TimeNetConnectorError: If the report is not a list, a line is malformed, or a pipe repeats. """ - # The list opens with a comment line that names the fields, which YAML reads as a null item. - lines = [line for line in (config.get("leakages") or []) if line is not None] - leaks = [_parse_leak(line, "dataset_configuration.yaml") for line in lines] - if len({leak.pipe for leak in leaks}) != len(leaks): - raise TimeNetConnectorError("dataset_configuration.yaml lists a pipe in two leaks") - if {leak.pipe for leak in leaks} != set(self.leak_outflow): - raise TimeNetConnectorError("the leak tables and dataset_configuration.yaml name different pipes") # A report of comment lines only, with no repair, reads as an empty YAML document. report = self._yaml("2018_Fixed_Leakages_Report.txt") or [] if not isinstance(report, list): @@ -373,6 +375,29 @@ def _read_leaks(self, config: dict) -> list[Leak]: if fields[0] in repairs: raise TimeNetConnectorError(f"2018_Fixed_Leakages_Report.txt repairs {fields[0]} twice") repairs[fields[0]] = _parse_time(fields[1], "2018_Fixed_Leakages_Report.txt") + return repairs + + def _read_leaks(self, config: dict) -> list[Leak]: + """Read the configured leaks and check them against the leak tables and the repair report. + + Returns: + The leaks in order of their start, each marked as repaired or not. + + Raises: + TimeNetConnectorError: If a configured leak is outside the recording or its leak table, + a leak table has a pipe the configuration lacks, or a repair time disagrees. + """ + # The list opens with a comment line that names the fields, which YAML reads as a null item. + leakages = config.get("leakages") + if not isinstance(leakages, list): + raise TimeNetConnectorError("dataset_configuration.yaml has no leakages list") + lines = [line for line in leakages if line is not None] + leaks = [_parse_leak(line, "dataset_configuration.yaml") for line in lines] + if len({leak.pipe for leak in leaks}) != len(leaks): + raise TimeNetConnectorError("dataset_configuration.yaml lists a pipe in two leaks") + if {leak.pipe for leak in leaks} != set(self.leak_outflow): + raise TimeNetConnectorError("the leak tables and dataset_configuration.yaml name different pipes") + repairs = self._read_repairs() checked = [] for leak in leaks: if not self.start <= leak.start <= leak.end <= self.end: @@ -384,7 +409,8 @@ def _read_leaks(self, config: dict) -> list[Leak]: raise TimeNetConnectorError(f"leak {leak.pipe} ends in {YEARS[0]} but the repair report lacks it") outflow = self.leak_outflow[leak.pipe] active = np.flatnonzero(outflow > 0) - if len(active) == 0 or active[0] < self.step_of(leak.start) or active[-1] > self.step_of(leak.end) + 1: + last_allowed = self.step_of(leak.end) + (1 if leak.pipe in _END_ONE_STEP_LATE else 0) + if len(active) == 0 or active[0] < self.step_of(leak.start) or active[-1] > last_allowed: raise TimeNetConnectorError(f"leak {leak.pipe}: its outflow lies outside its configured start and end") checked.append(replace(leak, repaired=leak.end < self.end)) if set(repairs) - {leak.pipe for leak in checked if leak.repaired}: diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py index b52ff1060..f079d9bce 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tasks.py @@ -9,10 +9,9 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta - -import numpy as np +from itertools import pairwise from timenet.dataset import Record from timenet.errors import TimeNetConnectorError @@ -40,8 +39,8 @@ # Four ISO weeks make one calendar block. Windows whose target weeks share a block share a partition # in both years, because 2019 repeats most of the 2018 demand series (see README.md). WEEKS_PER_BLOCK = 4 -# The leak tables write two decimals, so a smaller outflow reads as none. -_MEASURABLE = 0.05 +# Below this outflow, in m3/h, the text keeps the leak tables' second decimal. +_SMALL_OUTFLOW = 0.1 NEW_LEAK_TYPES = LabelSet( name="new_leak_types", @@ -97,16 +96,16 @@ def windows(release: Release) -> list[Window]: """ found = [] start = release.start - recording_end = release.end + STEP - while start + WINDOW <= recording_end: - target_start, end = start + WINDOW - TARGET, start + WINDOW + while start + WINDOW <= release.end + STEP: + frame = Window(index=len(found), start=start, new=(), repaired=()) found.append( - Window( - index=len(found), - start=start, - new=tuple(leak for leak in release.leaks if target_start <= leak.start < end), + replace( + frame, + new=tuple(leak for leak in release.leaks if frame.target_start <= leak.start < frame.end), # The release writes a repair time as the leak's configured end. - repaired=tuple(leak for leak in release.leaks if leak.repaired and target_start <= leak.end < end), + repaired=tuple( + leak for leak in release.leaks if leak.repaired and frame.target_start <= leak.end < frame.end + ), ) ) start += STRIDE @@ -135,6 +134,29 @@ def split_windows(found: list[Window]) -> dict[int, Split]: return {window.index: split for split, members in splitter.split(found).items() for window in members} +def embargoed(found: list[Window], split_of: dict[int, Split]) -> set[int]: + """Find the windows whose context week is a target week of another held-out partition. + + Windows overlap by one week: a window's context week is the target week of the window before + it. A training window after a test target week would show training the signals that the test + tasks ask about, so it gets no tasks. The same holds between validation and test. + + Returns: + The indices of the windows that get no tasks. + """ + held_out = {Split.VALIDATION, Split.TEST} + return { + window.index + for previous, window in pairwise(found) + if split_of[previous.index] in held_out and split_of[previous.index] != split_of[window.index] + } + + +def _rate(outflow: float) -> str: + """Format an outflow in m3/h so that the leak tables' smallest values stay visible.""" + return f"{outflow:.2f}" if outflow < _SMALL_OUTFLOW else f"{outflow:.1f}" + + def _outflow(release: Release, leak: Leak, start: datetime, end: datetime) -> float: """The largest outflow of ``leak`` in ``[start, end)``, in m3/h.""" return float(release.leak_outflow[leak.pipe][release.step_of(start) : release.step_of(end)].max()) @@ -162,7 +184,7 @@ def explain(release: Release, window: Window) -> str: kind, growth = "A gradually growing leak", " and was still growing at the end of this window" outflow = _outflow(release, leak, leak.start, window.end) # The leak tables round an incipient leak's first hours to zero outflow. - flow = f"an outflow of up to {outflow:.1f} m3/h" if outflow >= _MEASURABLE else "no measurable outflow yet" + flow = f"an outflow of up to {_rate(outflow)} m3/h" if outflow > 0 else "no outflow in the leak tables yet" sentences.append( f"{kind} started on pipe {leak.pipe} on {leak.start:%Y-%m-%d %H:%M}{growth}, " f"with a hole of {leak.diameter_m * 1000:.1f} mm and {flow} in this window. " @@ -195,7 +217,7 @@ def window_tasks(release: Release, record: Record, window: Window, split: Split) """ scope = TimeInterval.micros(release.micros(window.start), release.micros(window.end)) target = slice(release.step_of(window.target_start), release.step_of(window.end)) - outflow = np.sum([series[target] for series in release.leak_outflow.values()], axis=0) + outflow = release.total_outflow[target] new_leaks: list[JsonValue] = [leak.pipe for leak in window.new] repaired_leaks: list[JsonValue] = [leak.pipe for leak in window.repaired] metadata: JsonMapping = { diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py index fb22440b6..9b7fb303a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/tests/test_connector.py @@ -1,4 +1,5 @@ from collections import Counter +from itertools import pairwise from pathlib import Path import numpy as np @@ -8,10 +9,12 @@ from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version from timenet.types import AnswerTask, ClassificationTask, ScalarPredictionTask, TemporalLocalizationTask +from timenet.types.splits import Split from timenet.types.text import read_value from timenet_connectors.datasets.kios.battledim import BattLeDIMConnector from timenet_connectors.datasets.kios.battledim.connector import BattLeDIMSource from timenet_connectors.datasets.kios.battledim.source import Release +from timenet_connectors.datasets.kios.battledim.tasks import embargoed, split_windows, windows # A synthetic release shaped exactly like the Zenodo files: semicolon-separated CSVs with a decimal @@ -143,28 +146,34 @@ def test_record_holds_every_sensor_on_one_grid(dataset) -> None: assert flow.to_numpy()[5] == pytest.approx(44.04) -def test_leaks_become_annotations_over_their_intervals(dataset) -> None: +def test_leaks_are_dataset_annotations_not_record_labels(dataset) -> None: (record,) = dataset.records - pipes = [annotation for annotation in record.annotations if annotation.key == "leak_pipe"] - assert [annotation.value for annotation in pipes] == ["p3", "p5", "p6"] - p3 = pipes[0].span - assert p3.start_us == _step("2018-03-05 15:45") * 300_000_000 - # The interval ends after the repair step, the last step with outflow. - assert p3.end_us == (_step("2018-03-23 10:25") + 1) * 300_000_000 - repaired = {a.span.start_us: a.value for a in record.annotations if a.key == "leak_repaired"} - assert sorted(repaired.values()) == [False, False, True] - peak = [a for a in record.annotations if a.key == "leak_peak" and a.value == "p5"] - assert peak[0].span.start_us == _step("2018-11-15 00:00") * 300_000_000 + # The record every task reads carries no answers. + assert record.annotations == () + leaks = [annotation for annotation in dataset.annotations if annotation.key == "leak"] + assert [annotation.value for annotation in leaks] == ["p3", "p5", "p6"] + p3 = leaks[0].metadata + assert p3["leak_type"] == "abrupt" + assert p3["start_us"] == _step("2018-03-05 15:45") * 300_000_000 + # The interval ends after the repair step, the configured end. + assert p3["end_exclusive_us"] == (_step("2018-03-23 10:25") + 1) * 300_000_000 + assert p3["repaired"] is True + assert p3["peak_outflow_m3h"] == pytest.approx(28.4) + assert leaks[1].metadata["peak"] == "2018-11-15 00:00" + assert leaks[1].metadata["repaired"] is False def test_windows_give_four_tasks_each(dataset) -> None: tasks = list(dataset.get_all()) - # 730 days hold 103 two-week windows that start every 7 days. + # 730 days hold 103 two-week windows that start every 7 days. Windows after a held-out target + # week of another partition get no tasks (see the embargo test). + windows = {task.metadata["window_index"] for task in tasks} + assert 90 <= len(windows) < 103 assert Counter(type(task) for task in tasks) == { - ClassificationTask: 103, - TemporalLocalizationTask: 103, - ScalarPredictionTask: 103, - AnswerTask: 103, + ClassificationTask: len(windows), + TemporalLocalizationTask: len(windows), + ScalarPredictionTask: len(windows), + AnswerTask: len(windows), } first = tasks[0] assert first.scope.start_us == 0 @@ -239,11 +248,32 @@ def test_dataset_survives_storage_round_trip(dataset, tmp_path: Path) -> None: restored = reader.read() (record,) = restored.records assert record.sources[3].signals[1].to_numpy()[0] == pytest.approx(12.5) + assert record.annotations == () + assert [a.metadata["start"] for a in restored.annotations if a.key == "leak"] == [ + "2018-03-05 15:45", + "2018-11-01 00:00", + "2019-05-01 12:00", + ] # The reader returns the tasks in the order the connector added them. assert [task.split for task in restored.get_all()] == [task.split for task in dataset.get_all()] assert _answers(restored) == _answers(dataset) +def test_no_window_sees_a_held_out_target_week_of_another_partition(release_dir: Path, dataset) -> None: + release = Release(release_dir) + found = windows(release) + split_of = split_windows(found) + skipped = embargoed(found, split_of) + assert skipped + kept = {task.metadata["window_index"] for task in dataset.get_all()} + assert kept == {window.index for window in found} - skipped + for previous, window in pairwise(found): + # A window's context week is the target week of the window before it. + assert previous.target_start == window.start + if window.index in kept and split_of[previous.index] in {Split.VALIDATION, Split.TEST}: + assert split_of[previous.index] == split_of[window.index] + + def test_nearest_sensor_prefers_the_nearer_pipe_end(release_dir: Path) -> None: network = Release(release_dir).network assert network.nearest("p5", ["n1", "n4"]) == ("n4", 0) From 4b6a5f0141dfe472e03d06137cef0291dd710ef2 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Thu, 19 Nov 2026 22:44:30 +0100 Subject: [PATCH 09/10] docs(connectors): document BattLeDIM dataset-level leak labels and the window embargo --- .../datasets/kios/battledim/README.md | 48 +++++++++++++------ .../datasets/kios/battledim/dataset.yaml | 2 +- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md index 22157f512..e7fb593fb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md @@ -80,32 +80,39 @@ The configuration lists **33 leaks** *(measured)*: An **abrupt** leak appears at full size at its start. An **incipient** leak grows from its start until its `peakTime`. -Each leak becomes a set of **record annotations** over its active interval. The interval runs from the configured start to the step after the configured end. For one leak, p800, the outflow runs one step past that end (see below). -- `leak_pipe`: the pipe's EPANET id. -- `leak_type`: `abrupt` or `incipient`. -- `leak_hole_diameter`, in mm. -- `leak_peak_outflow`, in m³/h: the largest outflow in the leak tables. -- `leak_repaired`: whether the leak was repaired before the end of 2019. -- `leak_peak`: a time point at `peakTime`. +Each leak becomes one **dataset annotation** with the key `leak`. Its value is the pipe's EPANET id, and its metadata holds: +- `leak_type`: `abrupt` or `incipient`; +- `start`, `end`, `peak`: the configured times, as text; +- `start_us`, `end_exclusive_us`, `peak_us`: the same times in microseconds from the record start. The interval runs from the configured start to the step after the configured end; +- `hole_diameter_mm`; +- `peak_outflow_m3h`: the largest outflow in the leak tables; +- `repaired`: whether the leak was repaired before the end of 2019. -**These are answers, never task inputs.** +**The leaks are answers, so they are not on the record.** Every task reads the record, and anything that serves the record to a model also serves the record's annotations. TimeNet's `TimeFTorchDataset`, for example, returns them in each item. The record therefore holds the sensor signals and nothing else. Each window's answers are in its task targets. **Consistency checks.** The build fails if any of these disagree: - the configuration, the two leak tables and the repair report, which must list every leak that ends in 2018, exactly once; -- a leak's outflow and its configured start and end; +- a leak's outflow and its configured start and end (for p800 the leak tables keep one step of outflow after its end, an exception the connector names); - a leak pipe and the pipes of `L-TOWN.inp`. -It also fails on a sensor listed twice in the configuration, or a CSV cell that is not a timestamp or a number. The release passes all of these checks. +It also fails on: +- an abrupt leak whose peak is not its start; +- a sensor listed twice in the configuration; +- a configuration without its `times` mapping or `leakages` list; +- a CSV cell that is not a timestamp or a number. + +The release passes all of these checks. ## Windows and tasks **Windows:** a window is 14 days, starting every 7 days. - The first week is context; the second week is the **target week**. - The recording holds **103 windows** *(measured)*. -- Every leak start and every repair falls in exactly one target week: 33 starts, 17 repairs. The build checks this and fails otherwise, so no leak drops out of the tasks. +- Every leak start and every repair falls in exactly one target week: 33 starts, 17 repairs. The build checks this and fails otherwise. - 33 windows have a new leak, 16 have a repair, and 59 have neither *(measured)*. +- **Embargo:** 5 windows get no tasks, because their context week is a held-out target week (see The split). They hold no new leak, but two of them describe a repair (p514, p142), so those two repairs appear in no task. -**Tasks:** each window gives four tasks, all scoped to the window. That is 412 tasks. +**Tasks:** each of the other 98 windows gives four tasks, all scoped to the window. That is 392 tasks. | Task | Answer | Source of the answer | | --- | --- | --- | @@ -121,7 +128,13 @@ It also fails on a sensor listed twice in the configuration, or a CSV cell that > to {max outflow in the window} m3/h in this window. The nearest pressure sensor is {sensor}, {n} > links away. -For an incipient leak, the sentence opens "A gradually growing leak started on pipe {pipe} on {start}", followed by "and peaked on {peakTime}" if the peak lies inside the window, or "and was still growing at the end of this window" if it does not. A peak after the window is not stated, because the scoped input cannot show it. When the leak tables round the outflow in the window to zero (an incipient leak's first hours), the text says "no measurable outflow yet" instead of a number. This applies to 9 windows *(measured)*. +**Incipient leaks.** The sentence opens "A gradually growing leak started on pipe {pipe} on {start}". It continues with: +- "and peaked on {peakTime}" if the peak lies inside the window; +- "and was still growing at the end of this window" if it does not. A peak after the window is not stated, because the scoped input cannot show it. + +**Outflow figures.** +- An outflow under 0.1 m³/h keeps the leak tables' second decimal, e.g. "0.02 m3/h". +- When the leak tables hold no outflow at all in the window (an incipient leak's first hours), the text says "no outflow in the leak tables yet". This applies to 3 windows *(measured)*. *Repair:* > The leak on pipe {pipe} was repaired on {end}; its outflow before the repair was {outflow} m3/h. @@ -154,15 +167,20 @@ The release's own partition was 2018 for training and 2019 for evaluation. **Thi - A block holds the same weeks of both years. - `StratifiedSplitter` (8/1/1 folds, stratified on "a new leak starts", seed 42) assigns whole blocks. +**Embargo.** Windows overlap by one week: a window's context week is the target week of the window before it. +- A window gets no tasks when that week is a validation or test target week of a different partition. +- Otherwise a training window would show the model the signals that a test task asks about. +- This affects 5 training windows *(measured)*. + **Result** *(measured)*: | Partition | Windows | With a new leak | Tasks | Calendar blocks | | --- | --- | --- | --- | --- | -| train | 79 | 27 | 316 | 0, 1, 2, 3, 5, 7, 8, 9, 10, 11 | +| train | 74 | 27 | 296 | 0, 1, 2, 3, 5, 7, 8, 9, 10, 11 | | validation | 8 | 3 | 32 | 4 | | test | 16 | 3 | 64 | 6, 12 | -**Overlap:** the context week of a window can lie in a neighbouring block, so context weeks can overlap across partitions. Target weeks never do. +**Remaining overlap:** a held-out window's context week can be a training target week. That shows held-out tasks training data, not the other way round. Target weeks never repeat across partitions. **One leaked day per block boundary:** the 2019 demand repeats 2018 by calendar date, but 2019 weekdays fall one day later. A 2019 target week (Monday to Sunday) therefore shares one date with the 2018 week before it. Where a block boundary separates partitions, one day of a target week's demand background also appears in the neighbouring partition. Week-aligned windows cannot avoid this. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml index 93a564a9f..d99fa783a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/dataset.yaml @@ -3,7 +3,7 @@ # BattLeDIM 2020 released two years (2018 and 2019) of simulated SCADA data of the L-Town water # network, every 5 minutes: 33 pressure sensors, 3 flow sensors, 1 tank level sensor and 82 # automated meter readings, plus the 33 leaks the simulation contained. The two years form one -# record. Each leak is a set of record annotations over its active interval. Two-week windows, +# record, which holds the sensor signals only. Each leak is one dataset annotation. Two-week windows, # one every 7 days, become four tasks each about the last week of the window. The answers come from # the leak list, the leak tables and the network model by fixed rules, not from a model. # From 4ffbcf350e730c044584fc714b547cb2151f1816 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 09:45:32 +0100 Subject: [PATCH 10/10] docs(connectors): credit Team RAMadan in the BattLeDIM README --- .../src/timenet_connectors/datasets/kios/battledim/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md index e7fb593fb..3bf0c807a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/kios/battledim/README.md @@ -203,4 +203,4 @@ The release's own partition was 2018 for training and 2019 for evaluation. **Thi - Training and evaluating time-series language models that detect, time, size and describe leaks from SCADA data. - Studying explanations that can be checked against a known ground truth. -*Added during the TimeNet hackathon, Round 1 (November 2026).* +*Contributed by Team RAMadan during the TimeNet hackathon, Round 1 (November 2026).*