From 0d5c17edf88ba814ddabf6fab6dcc5e5b38c3a25 Mon Sep 17 00:00:00 2001 From: "k.arias" Date: Wed, 18 Nov 2026 01:15:28 +0000 Subject: [PATCH 1/2] feat: add the Lima hourly PM2.5 forecasting benchmark --- .../datasets/airaware/__init__.py | 1 + .../airaware/lima_pm25_forecast/README.md | 44 +++++ .../airaware/lima_pm25_forecast/__init__.py | 3 + .../airaware/lima_pm25_forecast/connector.py | 184 ++++++++++++++++++ .../airaware/lima_pm25_forecast/dataset.yaml | 26 +++ .../tests/test_connector.py | 57 ++++++ 6 files changed, 315 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/airaware/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/README.md create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/__init__.py new file mode 100644 index 000000000..ea80b8047 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/__init__.py @@ -0,0 +1 @@ +"""Datasets contributed by AirAware Perú.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/README.md new file mode 100644 index 000000000..23350fa0d --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/README.md @@ -0,0 +1,44 @@ +# Lima hourly PM2.5 forecasting + +This connector builds `airaware/lima-pm25-forecast@0.1.0` from SENAMHI's +public export, *Monitoreo de los contaminantes del aire en Lima Metropolitana*. +SENAMHI licenses the source database under +[ODC-By-1.0](https://opendatacommons.org/licenses/by/1-0/). AirAware provides +the hourly deduplication, chronological benchmark, and forecasting tasks. +The connector code uses the repository's MIT license. + +## Tasks and splits + +The model receives 24 consecutive hourly PM2.5 measurements from one of seven +provisional station identities and predicts the next six values. TimeF records +contain only the history. Respective forecast values are ordered numeric targets. +The source CSV uses a pinned SHA-256, as specified on the dataset card. Exact +repeat readings share a single retained reading. Conflicting finite reports for +the same hour are excluded along with windows that require missing values. + +Validation starts on March 16, 2022 at 08:00 UTC and test starts on February +25, 2023 at 18:00 UTC. Windows crossing either boundary are purged. The published +benchmark contains 237,397 training, 50,706 validation, and 50,789 test tasks. +Use validation for model selection and reserve test for final evaluation. + +The timestamps use published hourly labels interpreted as UTC under AirAware's +operating policy. The concentration unit in µg/m³ is inferred. The position of +the hour within the reporting interval, per-reading quality flags, and publication +delays are unknown. The benchmark covers the monitored sites in the pinned export +and supports retrospective evaluation. + +## Build and verify + +With the TimeNet development environment installed: + +```sh +timenet-build build airaware/lima-pm25-forecast@0.1.0 \ + --local-registry /path/to/local-registry +``` + +For a build without network access, place the verified original at +`${TIMENET_CACHE}/downloads/airaware/lima-pm25-forecast/senamhi-lima.csv`, +use the same local registry as the remote registry in the build command, and +pass `--no-isolation` to use the already installed environment. The converter +checks the exact CSV bytes independently of the cache check. Open the result +with `TimeFReader` and run `verify()` before using the release. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/__init__.py new file mode 100644 index 000000000..6923d27cc --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/__init__.py @@ -0,0 +1,3 @@ +"""Retrospective hourly PM2.5 forecasting in Lima.""" + +from timenet_connectors.datasets.airaware.lima_pm25_forecast.connector import CONNECTOR as CONNECTOR diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py new file mode 100644 index 000000000..e9556c46f --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py @@ -0,0 +1,184 @@ +"""Build the frozen AirAware forecast from the original SENAMHI CSV.""" + +# SPDX-License-Identifier: MIT + +from collections import defaultdict, deque +from collections.abc import Iterator +import csv +from datetime import UTC, datetime, timedelta +from fractions import Fraction +from hashlib import sha256 +import io +import json +import math +from pathlib import Path + +from timenet.connectors import BaseConnector +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.types import ForecastingTask, Split, TimeOrigin, TimeSeriesSpec, ureg +from timenet_connectors.download.http import download_http + + +SOURCE_URL = ( + "https://www.datosabiertos.gob.pe/sites/default/files/Monitoreo%20de%20los%20contaminantes%20" + "del%20aire%20en%20Lima%20Metropolitana%20-%20%5BServicio%20Nacional%20de%20Meteorolog%C3%ADa" + "%20e%20Hidrolog%C3%ADa%20del%20Per%C3%BA%20-%20SENAMHI%5D_1.csv" +) +STATION_FIELDS = ("ESTACION", "LONGITUD", "LATITUD", "ALTITUD", "DEPARTAMENTO", "PROVINCIA", "DISTRITO") +VALIDATION_START = datetime(2022, 3, 16, 8, tzinfo=UTC) +TEST_START = datetime(2023, 2, 25, 18, tzinfo=UTC) +HOUR = timedelta(hours=1) +HISTORY = 24 +FUTURE = 6 +AXIS = RegularAxis(period_us=Fraction(3_600_000_000)) +PM25 = TimeSeriesSpec( + spec_type="pm25", + name="PM2.5", + unit_value=ureg.microgram / ureg.meter**3, + dtype="float64", + missing=None, +) +type_reading = tuple[float, int] + + +def _key(fields: list[str | None], prefix: str = "") -> str: + return sha256((prefix + json.dumps(fields, ensure_ascii=False, separators=(",", ":"))).encode("utf-8")).hexdigest() + + +def _parse(data: bytes) -> dict[str, dict[datetime, type_reading | None]]: + """Preserve the first equal report and leave disagreeing reports ambiguous.""" + stations: dict[str, dict[datetime, type_reading | None]] = defaultdict(dict) + required = {*STATION_FIELDS, "FECHA", "HORA", "PM2_5"} + with io.TextIOWrapper(io.BytesIO(data), encoding="utf-8-sig", newline="") as stream: + rows = csv.DictReader(stream, strict=True) + if rows.fieldnames is None or not required.issubset(rows.fieldnames): + msg = "The CSV is missing one or more required SENAMHI fields" + raise ValueError(msg) + keys = {} + for row_number, row in enumerate(rows, start=2): + if row["DEPARTAMENTO"] != "LIMA" or row["PROVINCIA"] != "LIMA" or not row["ESTACION"].strip(): + continue + try: + value = float(row["PM2_5"]) + except ValueError: + continue + if not math.isfinite(value): + continue + clock = datetime.strptime(row["FECHA"] + row["HORA"].zfill(6), "%Y%m%d%H%M%S").replace(tzinfo=UTC) + if clock.minute or clock.second: + continue + context = tuple(row[field] for field in STATION_FIELDS) + if context not in keys: + keys[context] = _key(list(context), "senamhi-lima-csv") + series = stations[keys[context]] + if clock not in series: + series[clock] = (value, row_number) + elif (previous := series[clock]) is not None and previous[0] != value: + series[clock] = None + return dict(stations) + + +def _split(start: datetime, end: datetime) -> Split | None: + if end < VALIDATION_START: + return Split.TRAIN + if start >= VALIDATION_START and end < TEST_START: + return Split.VALIDATION + if start >= TEST_START: + return Split.TEST + return None + + +def _windows( + series: dict[datetime, type_reading | None], +) -> Iterator[tuple[datetime, Split, tuple[type_reading, ...]]]: + recent: deque[type_reading] = deque(maxlen=HISTORY + FUTURE) + previous = None + for tick, reading in sorted(series.items()): + if previous is None or tick - previous != HOUR or reading is None: + recent.clear() + previous = tick + if reading is None: + continue + recent.append(reading) + if len(recent) == recent.maxlen: + start = tick - (HISTORY + FUTURE - 1) * HOUR + split = _split(start, tick) + if split is not None: + yield start, split, tuple(recent) + + +def _timestamp_key(instant: datetime) -> str: + return instant.strftime("%Y-%m-%d %H:%M:%S+00") + + +class LimaPM25Connector(BaseConnector[Path]): + """Build the Lima retrospective PM2.5 benchmark from its pinned public export.""" + + source_sha256 = "896ae8a106fb128460e81b3f58096f6b9c4d582ef28fffa46e5394c5875c40d4" + + async def download_async(self, cache_dir: Path) -> list[Path]: + """Resolve the pinned original, reusing a valid cached copy.""" + path = await download_http(SOURCE_URL, cache_dir / "senamhi-lima.csv", sha256=self.source_sha256) + return [path] + + def convert(self, raw_refs: list[Path]) -> TimeFDataset: + """Read one verified CSV and build chronological forecasting tasks.""" + (path,) = raw_refs + content = path.read_bytes() + if sha256(content).hexdigest() != self.source_sha256: + msg = "SENAMHI source SHA-256 does not match the dataset card" + raise ValueError(msg) + dataset = TimeFDataset(metadata=self.metadata()) + for station, reports in sorted(_parse(content).items(), key=lambda item: self._series_key(item[0])): + series_id = self._series_key(station) + for start, split, observations in _windows(reports): + history = observations[:HISTORY] + origin = start + (HISTORY - 1) * HOUR + end = origin + FUTURE * HOUR + window_id = _key( + [ + series_id, + _timestamp_key(origin), + _timestamp_key(start), + _timestamp_key(end), + ] + ) + signal = Signal.from_values( + [value for value, _ in history], + name="pm25", + spec=PM25, + time_axis=AXIS, + ) + record = dataset.add_record( + record=Record( + sources=(Source(name=station, signals=(signal,)),), + start_time=TimeOrigin(start), + metadata={ + "station_key": station, + "source_rows": [row for _, row in history], + "unit_basis": "inferred", + }, + ) + ) + dataset.add_task( + task=ForecastingTask( + inputs=(record,), + targets=tuple(value for value, _ in observations[HISTORY:]), + split=split, + metadata={ + "window_id": window_id, + "series_id": series_id, + "origin_at": origin.isoformat(), + "target_start": (origin + HOUR).isoformat(), + }, + ) + ) + return dataset + + @staticmethod + def _series_key(station: str) -> str: + return _key([station, None, "PM2_5", "PM2.5", "ug/m3"]) + + +CONNECTOR = LimaPM25Connector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/dataset.yaml new file mode 100644 index 000000000..02c742503 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/dataset.yaml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: airaware/lima-pm25-forecast +dataset_version: 0.1.0 +name: Lima hourly PM2.5 forecasting +description: >- + Retrospective forecasting from seven provisional SENAMHI station identities. + Predict the next six hourly PM2.5 values from the preceding 24 measured hours. + AirAware excludes ambiguous readings and purges windows crossing the fixed + chronological splits. Hourly labels are interpreted as UTC. The PM2.5 unit is + inferred, and the interval alignment, validation flags, and publication delays + of individual readings are unknown. +license: ODC-By-1.0 +license_url: https://opendatacommons.org/licenses/by/1-0/ +source_url: https://www.datosabiertos.gob.pe/dataset/monitoreo-de-los-contaminantes-del-aire-en-lima-metropolitana-servicio-nacional-de +source_revision: 896ae8a106fb128460e81b3f58096f6b9c4d582ef28fffa46e5394c5875c40d4 +citation: >- + SENAMHI (Servicio Nacional de Meteorología e Hidrología del Perú), + Monitoreo de los contaminantes del aire en Lima Metropolitana. + Adapted by AirAware Perú as deduplicated hourly forecasting windows. +domains: + - environment +tags: + - forecasting + - air-quality + - peru diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py new file mode 100644 index 000000000..58bab6ee5 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py @@ -0,0 +1,57 @@ +from datetime import UTC, datetime, timedelta +from hashlib import sha256 + +import pytest + +from timenet.reader import TimeFReader +from timenet_connectors.datasets.airaware.lima_pm25_forecast.connector import ( + LimaPM25Connector, + _parse, + _split, +) + + +def test_chronological_assignment_purges_crossing_history_or_target(): + cut = datetime(2022, 3, 16, 8, tzinfo=UTC) + assert _split(cut - timedelta(hours=30), cut - timedelta(hours=1)) == "train" + assert _split(cut - timedelta(hours=29), cut) is None + assert _split(cut, cut + timedelta(hours=29)) == "validation" + + +def test_duplicate_zero_conflict_and_missing_are_distinct(): + header = "ESTACION,LONGITUD,LATITUD,ALTITUD,DEPARTAMENTO,PROVINCIA,DISTRITO,FECHA,HORA,PM2_5" + prefix = "FICTICIA,-77,-12,10,LIMA,LIMA,TEST,20200101" + lines = [ + header, + f"{prefix},0,0", + f"{prefix},0,0.0", + f"{prefix},0,", + f"{prefix},10000,3", + f"{prefix},10000,4", + f"{prefix},20000,", + ] + (readings,) = _parse("\n".join(lines).encode()).values() + assert list(readings.values()) == [(0.0, 2), None] + + +def test_build_from_verified_fixture_and_read_timef(tmp_path, monkeypatch): + csv_path = tmp_path / "input.csv" + rows = ["ESTACION,LONGITUD,LATITUD,ALTITUD,DEPARTAMENTO,PROVINCIA,DISTRITO,FECHA,HORA,PM2_5"] + for start in ("20200101", "20220501", "20230501"): + day = datetime.strptime(start, "%Y%m%d").replace(tzinfo=UTC) + for hour in range(30): + tick = day + timedelta(hours=hour) + rows.append(f"FICTICIA,-77,-12,10,LIMA,LIMA,TEST,{tick:%Y%m%d},{tick:%H%M%S},{hour}") + body = ("\n".join(rows) + "\n").encode() + csv_path.write_bytes(body) + monkeypatch.setattr(LimaPM25Connector, "source_sha256", sha256(body).hexdigest()) + dataset = LimaPM25Connector().convert([csv_path]) + assert [str(task.split) for task in dataset.tasks] == ["train", "validation", "test"] + with TimeFReader.open(dataset.write(path=tmp_path / "registry")) as reader: + reader.verify() + task = next(reader.iter_tasks(batch_size=1)) + assert task.inputs[0].signals[0].to_arrow().to_pylist() == list(range(24)) + assert task.targets == tuple(range(24, 30)) + csv_path.write_bytes(body + b"changed") + with pytest.raises(ValueError, match="SHA-256"): + LimaPM25Connector().convert([csv_path]) From ea19b218f4383901aa0b68b47fbf62b35c2da3f6 Mon Sep 17 00:00:00 2001 From: masquare <11543761+masquare@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:45:17 -0800 Subject: [PATCH 2/2] refactor(connectors): move airaware/lima_pm25_forecast to the new connector API Main changed the connector API. A label set carries its name as an annotation, a Hugging Face connector passes its Hub settings as class keywords and reads a mapping from part to path, BaseConnector names the whole download result, and Annotation takes its value by keyword. --- .../airaware/lima_pm25_forecast/connector.py | 18 ++++++++---------- .../lima_pm25_forecast/tests/test_connector.py | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py index e9556c46f..c1fe266f0 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/connector.py @@ -117,14 +117,12 @@ class LimaPM25Connector(BaseConnector[Path]): source_sha256 = "896ae8a106fb128460e81b3f58096f6b9c4d582ef28fffa46e5394c5875c40d4" - async def download_async(self, cache_dir: Path) -> list[Path]: + async def download_async(self, cache_dir: Path) -> Path: """Resolve the pinned original, reusing a valid cached copy.""" - path = await download_http(SOURCE_URL, cache_dir / "senamhi-lima.csv", sha256=self.source_sha256) - return [path] + return await download_http(SOURCE_URL, cache_dir / "senamhi-lima.csv", sha256=self.source_sha256) - def convert(self, raw_refs: list[Path]) -> TimeFDataset: + def convert(self, path: Path) -> TimeFDataset: """Read one verified CSV and build chronological forecasting tasks.""" - (path,) = raw_refs content = path.read_bytes() if sha256(content).hexdigest() != self.source_sha256: msg = "SENAMHI source SHA-256 does not match the dataset card" @@ -151,8 +149,8 @@ def convert(self, raw_refs: list[Path]) -> TimeFDataset: time_axis=AXIS, ) record = dataset.add_record( - record=Record( - sources=(Source(name=station, signals=(signal,)),), + Record( + sources=[Source(name=station, signals=[signal])], start_time=TimeOrigin(start), metadata={ "station_key": station, @@ -162,9 +160,9 @@ def convert(self, raw_refs: list[Path]) -> TimeFDataset: ) ) dataset.add_task( - task=ForecastingTask( - inputs=(record,), - targets=tuple(value for value, _ in observations[HISTORY:]), + ForecastingTask( + inputs=[record], + targets=[value for value, _ in observations[HISTORY:]], split=split, metadata={ "window_id": window_id, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py index 58bab6ee5..3d802ca01 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/airaware/lima_pm25_forecast/tests/test_connector.py @@ -45,7 +45,7 @@ def test_build_from_verified_fixture_and_read_timef(tmp_path, monkeypatch): body = ("\n".join(rows) + "\n").encode() csv_path.write_bytes(body) monkeypatch.setattr(LimaPM25Connector, "source_sha256", sha256(body).hexdigest()) - dataset = LimaPM25Connector().convert([csv_path]) + dataset = LimaPM25Connector().convert(csv_path) assert [str(task.split) for task in dataset.tasks] == ["train", "validation", "test"] with TimeFReader.open(dataset.write(path=tmp_path / "registry")) as reader: reader.verify() @@ -54,4 +54,4 @@ def test_build_from_verified_fixture_and_read_timef(tmp_path, monkeypatch): assert task.targets == tuple(range(24, 30)) csv_path.write_bytes(body + b"changed") with pytest.raises(ValueError, match="SHA-256"): - LimaPM25Connector().convert([csv_path]) + LimaPM25Connector().convert(csv_path)