Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Datasets contributed by AirAware Perú."""
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Lima hourly PM2.5 forecasting

This connector builds `airaware/lima-pm25-forecast@0.1.0` from SENAMHI's
public export, *Monitoreo de los contaminantes del aire en Lima Metropolitana*.
SENAMHI licenses the source database under
[ODC-By-1.0](https://opendatacommons.org/licenses/by/1-0/). AirAware provides
the hourly deduplication, chronological benchmark, and forecasting tasks.
The connector code uses the repository's MIT license.

## Tasks and splits

The model receives 24 consecutive hourly PM2.5 measurements from one of seven
provisional station identities and predicts the next six values. TimeF records
contain only the history. Respective forecast values are ordered numeric targets.
The source CSV uses a pinned SHA-256, as specified on the dataset card. Exact
repeat readings share a single retained reading. Conflicting finite reports for
the same hour are excluded along with windows that require missing values.

Validation starts on March 16, 2022 at 08:00 UTC and test starts on February
25, 2023 at 18:00 UTC. Windows crossing either boundary are purged. The published
benchmark contains 237,397 training, 50,706 validation, and 50,789 test tasks.
Use validation for model selection and reserve test for final evaluation.

The timestamps use published hourly labels interpreted as UTC under AirAware's
operating policy. The concentration unit in µg/m³ is inferred. The position of
the hour within the reporting interval, per-reading quality flags, and publication
delays are unknown. The benchmark covers the monitored sites in the pinned export
and supports retrospective evaluation.

## Build and verify

With the TimeNet development environment installed:

```sh
timenet-build build airaware/lima-pm25-forecast@0.1.0 \
--local-registry /path/to/local-registry
```

For a build without network access, place the verified original at
`${TIMENET_CACHE}/downloads/airaware/lima-pm25-forecast/senamhi-lima.csv`,
use the same local registry as the remote registry in the build command, and
pass `--no-isolation` to use the already installed environment. The converter
checks the exact CSV bytes independently of the cache check. Open the result
with `TimeFReader` and run `verify()` before using the release.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
"""Retrospective hourly PM2.5 forecasting in Lima."""

from timenet_connectors.datasets.airaware.lima_pm25_forecast.connector import CONNECTOR as CONNECTOR
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
"""Build the frozen AirAware forecast from the original SENAMHI CSV."""

# SPDX-License-Identifier: MIT

from collections import defaultdict, deque
from collections.abc import Iterator
import csv
from datetime import UTC, datetime, timedelta
from fractions import Fraction
from hashlib import sha256
import io
import json
import math
from pathlib import Path

from timenet.connectors import BaseConnector
from timenet.dataset import Record, Signal, Source, TimeFDataset
from timenet.dataset.axis import RegularAxis
from timenet.types import ForecastingTask, Split, TimeOrigin, TimeSeriesSpec, ureg
from timenet_connectors.download.http import download_http


SOURCE_URL = (
"https://www.datosabiertos.gob.pe/sites/default/files/Monitoreo%20de%20los%20contaminantes%20"
"del%20aire%20en%20Lima%20Metropolitana%20-%20%5BServicio%20Nacional%20de%20Meteorolog%C3%ADa"
"%20e%20Hidrolog%C3%ADa%20del%20Per%C3%BA%20-%20SENAMHI%5D_1.csv"
)
STATION_FIELDS = ("ESTACION", "LONGITUD", "LATITUD", "ALTITUD", "DEPARTAMENTO", "PROVINCIA", "DISTRITO")
VALIDATION_START = datetime(2022, 3, 16, 8, tzinfo=UTC)
TEST_START = datetime(2023, 2, 25, 18, tzinfo=UTC)
HOUR = timedelta(hours=1)
HISTORY = 24
FUTURE = 6
AXIS = RegularAxis(period_us=Fraction(3_600_000_000))
PM25 = TimeSeriesSpec(
spec_type="pm25",
name="PM2.5",
unit_value=ureg.microgram / ureg.meter**3,
dtype="float64",
missing=None,
)
type_reading = tuple[float, int]


def _key(fields: list[str | None], prefix: str = "") -> str:
return sha256((prefix + json.dumps(fields, ensure_ascii=False, separators=(",", ":"))).encode("utf-8")).hexdigest()


def _parse(data: bytes) -> dict[str, dict[datetime, type_reading | None]]:
"""Preserve the first equal report and leave disagreeing reports ambiguous."""
stations: dict[str, dict[datetime, type_reading | None]] = defaultdict(dict)
required = {*STATION_FIELDS, "FECHA", "HORA", "PM2_5"}
with io.TextIOWrapper(io.BytesIO(data), encoding="utf-8-sig", newline="") as stream:
rows = csv.DictReader(stream, strict=True)
if rows.fieldnames is None or not required.issubset(rows.fieldnames):
msg = "The CSV is missing one or more required SENAMHI fields"
raise ValueError(msg)
keys = {}
for row_number, row in enumerate(rows, start=2):
if row["DEPARTAMENTO"] != "LIMA" or row["PROVINCIA"] != "LIMA" or not row["ESTACION"].strip():
continue
try:
value = float(row["PM2_5"])
except ValueError:
continue
if not math.isfinite(value):
continue
clock = datetime.strptime(row["FECHA"] + row["HORA"].zfill(6), "%Y%m%d%H%M%S").replace(tzinfo=UTC)
if clock.minute or clock.second:
continue
context = tuple(row[field] for field in STATION_FIELDS)
if context not in keys:
keys[context] = _key(list(context), "senamhi-lima-csv")
series = stations[keys[context]]
if clock not in series:
series[clock] = (value, row_number)
elif (previous := series[clock]) is not None and previous[0] != value:
series[clock] = None
return dict(stations)


def _split(start: datetime, end: datetime) -> Split | None:
if end < VALIDATION_START:
return Split.TRAIN
if start >= VALIDATION_START and end < TEST_START:
return Split.VALIDATION
if start >= TEST_START:
return Split.TEST
return None


def _windows(
series: dict[datetime, type_reading | None],
) -> Iterator[tuple[datetime, Split, tuple[type_reading, ...]]]:
recent: deque[type_reading] = deque(maxlen=HISTORY + FUTURE)
previous = None
for tick, reading in sorted(series.items()):
if previous is None or tick - previous != HOUR or reading is None:
recent.clear()
previous = tick
if reading is None:
continue
recent.append(reading)
if len(recent) == recent.maxlen:
start = tick - (HISTORY + FUTURE - 1) * HOUR
split = _split(start, tick)
if split is not None:
yield start, split, tuple(recent)


def _timestamp_key(instant: datetime) -> str:
return instant.strftime("%Y-%m-%d %H:%M:%S+00")


class LimaPM25Connector(BaseConnector[Path]):
"""Build the Lima retrospective PM2.5 benchmark from its pinned public export."""

source_sha256 = "896ae8a106fb128460e81b3f58096f6b9c4d582ef28fffa46e5394c5875c40d4"

async def download_async(self, cache_dir: Path) -> Path:
"""Resolve the pinned original, reusing a valid cached copy."""
return await download_http(SOURCE_URL, cache_dir / "senamhi-lima.csv", sha256=self.source_sha256)

def convert(self, path: Path) -> TimeFDataset:
"""Read one verified CSV and build chronological forecasting tasks."""
content = path.read_bytes()
if sha256(content).hexdigest() != self.source_sha256:
msg = "SENAMHI source SHA-256 does not match the dataset card"
raise ValueError(msg)
dataset = TimeFDataset(metadata=self.metadata())
for station, reports in sorted(_parse(content).items(), key=lambda item: self._series_key(item[0])):
series_id = self._series_key(station)
for start, split, observations in _windows(reports):
history = observations[:HISTORY]
origin = start + (HISTORY - 1) * HOUR
end = origin + FUTURE * HOUR
window_id = _key(
[
series_id,
_timestamp_key(origin),
_timestamp_key(start),
_timestamp_key(end),
]
)
signal = Signal.from_values(
[value for value, _ in history],
name="pm25",
spec=PM25,
time_axis=AXIS,
)
record = dataset.add_record(
Record(
sources=[Source(name=station, signals=[signal])],
start_time=TimeOrigin(start),
metadata={
"station_key": station,
"source_rows": [row for _, row in history],
"unit_basis": "inferred",
},
)
)
dataset.add_task(
ForecastingTask(
inputs=[record],
targets=[value for value, _ in observations[HISTORY:]],
split=split,
metadata={
"window_id": window_id,
"series_id": series_id,
"origin_at": origin.isoformat(),
"target_start": (origin + HOUR).isoformat(),
},
)
)
return dataset

@staticmethod
def _series_key(station: str) -> str:
return _key([station, None, "PM2_5", "PM2.5", "ug/m3"])


CONNECTOR = LimaPM25Connector
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json
yaml_schema_version: 1
dataset_id: airaware/lima-pm25-forecast
dataset_version: 0.1.0
name: Lima hourly PM2.5 forecasting
description: >-
Retrospective forecasting from seven provisional SENAMHI station identities.
Predict the next six hourly PM2.5 values from the preceding 24 measured hours.
AirAware excludes ambiguous readings and purges windows crossing the fixed
chronological splits. Hourly labels are interpreted as UTC. The PM2.5 unit is
inferred, and the interval alignment, validation flags, and publication delays
of individual readings are unknown.
license: ODC-By-1.0
license_url: https://opendatacommons.org/licenses/by/1-0/
source_url: https://www.datosabiertos.gob.pe/dataset/monitoreo-de-los-contaminantes-del-aire-en-lima-metropolitana-servicio-nacional-de
source_revision: 896ae8a106fb128460e81b3f58096f6b9c4d582ef28fffa46e5394c5875c40d4
citation: >-
SENAMHI (Servicio Nacional de Meteorología e Hidrología del Perú),
Monitoreo de los contaminantes del aire en Lima Metropolitana.
Adapted by AirAware Perú as deduplicated hourly forecasting windows.
domains:
- environment
tags:
- forecasting
- air-quality
- peru
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
from datetime import UTC, datetime, timedelta
from hashlib import sha256

import pytest

from timenet.reader import TimeFReader
from timenet_connectors.datasets.airaware.lima_pm25_forecast.connector import (
LimaPM25Connector,
_parse,
_split,
)


def test_chronological_assignment_purges_crossing_history_or_target():
cut = datetime(2022, 3, 16, 8, tzinfo=UTC)
assert _split(cut - timedelta(hours=30), cut - timedelta(hours=1)) == "train"
assert _split(cut - timedelta(hours=29), cut) is None
assert _split(cut, cut + timedelta(hours=29)) == "validation"


def test_duplicate_zero_conflict_and_missing_are_distinct():
header = "ESTACION,LONGITUD,LATITUD,ALTITUD,DEPARTAMENTO,PROVINCIA,DISTRITO,FECHA,HORA,PM2_5"
prefix = "FICTICIA,-77,-12,10,LIMA,LIMA,TEST,20200101"
lines = [
header,
f"{prefix},0,0",
f"{prefix},0,0.0",
f"{prefix},0,",
f"{prefix},10000,3",
f"{prefix},10000,4",
f"{prefix},20000,",
]
(readings,) = _parse("\n".join(lines).encode()).values()
assert list(readings.values()) == [(0.0, 2), None]


def test_build_from_verified_fixture_and_read_timef(tmp_path, monkeypatch):
csv_path = tmp_path / "input.csv"
rows = ["ESTACION,LONGITUD,LATITUD,ALTITUD,DEPARTAMENTO,PROVINCIA,DISTRITO,FECHA,HORA,PM2_5"]
for start in ("20200101", "20220501", "20230501"):
day = datetime.strptime(start, "%Y%m%d").replace(tzinfo=UTC)
for hour in range(30):
tick = day + timedelta(hours=hour)
rows.append(f"FICTICIA,-77,-12,10,LIMA,LIMA,TEST,{tick:%Y%m%d},{tick:%H%M%S},{hour}")
body = ("\n".join(rows) + "\n").encode()
csv_path.write_bytes(body)
monkeypatch.setattr(LimaPM25Connector, "source_sha256", sha256(body).hexdigest())
dataset = LimaPM25Connector().convert(csv_path)
assert [str(task.split) for task in dataset.tasks] == ["train", "validation", "test"]
with TimeFReader.open(dataset.write(path=tmp_path / "registry")) as reader:
reader.verify()
task = next(reader.iter_tasks(batch_size=1))
assert task.inputs[0].signals[0].to_arrow().to_pylist() == list(range(24))
assert task.targets == tuple(range(24, 30))
csv_path.write_bytes(body + b"changed")
with pytest.raises(ValueError, match="SHA-256"):
LimaPM25Connector().convert(csv_path)