From 966d51a4f8ce28016940da441884a4fae5948103 Mon Sep 17 00:00:00 2001 From: mahmood Date: Fri, 20 Nov 2026 15:40:43 +0400 Subject: [PATCH] feat(connectors): add williamhtan/kernelsight KernelSight: 1,444 H100 GPU counter snapshots (24 channels x 512 bins) with per-segment L1/L2 workload labels, from the pinned Hub release. - One Record per snapshot, 24 Signals in five channel-group Sources, a per-snapshot RegularAxis from the bin edges. - Segment L1/L2 interval annotations, plus multi-hot-only overlap runs (track=multihot) for the 472 cutlass_ws_overlap snapshots. - One ClassificationTask per snapshot on the authors' dominant_l1. - Split grouped by problem (KernelBench dtype/seed variants stay together), stratified by class, 3/1/1, seed 42; the released split memberships are kept in record metadata. - Reads only numeric npz arrays (allow_pickle=False). Co-Authored-By: Claude Opus 5.5 --- .../datasets/williamhtan/__init__.py | 4 + .../williamhtan/kernelsight/__init__.py | 6 + .../williamhtan/kernelsight/connector.py | 483 ++++++++++++++++++ .../williamhtan/kernelsight/dataset.yaml | 40 ++ .../williamhtan/kernelsight/requirements.txt | 2 + .../kernelsight/tests/test_connector.py | 224 ++++++++ 6 files changed, 759 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py new file mode 100644 index 000000000..3367da279 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py @@ -0,0 +1,4 @@ +"""Connectors for datasets published by the ``williamhtan`` Hub user.""" + +# Keep this a regular package, not a namespace package. pkgutil.walk_packages finds the +# connector packages below it only through this __init__.py. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py new file mode 100644 index 000000000..c60729eaf --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py @@ -0,0 +1,6 @@ +"""KernelSight: H100 GPU counter traces with per-segment workload labels.""" + +from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( + CONNECTOR as CONNECTOR, + KernelSightConnector as KernelSightConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py new file mode 100644 index 000000000..b4bada137 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py @@ -0,0 +1,483 @@ +"""The KernelSight connector: H100 GPU counter traces with per-segment workload labels. + +Each snapshot of the release is a ``[24, 512]`` counter image: 24 hardware-counter channels over 512 +equal-width bins. Each snapshot becomes one Record with one Signal per channel, grouped into the +five channel groups of the dataset card. Each labeled segment becomes a time-interval annotation +with its L1 and L2 workload class. Each snapshot gets one ClassificationTask whose target is the +authors' dominant L1 class. + +The release's own train/val/test split puts dtype and seed variants of one KernelBench problem in +different partitions. The connector therefore assigns a split grouped by problem and keeps the +authors' split memberships in the record metadata. + +The release stores its vocabularies and counter names as pickled object arrays. Unpickling a +downloaded file can run code, so the connector keeps those names as constants and reads only the +numeric arrays, with ``allow_pickle=False``. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from fractions import Fraction +import json +from operator import attrgetter +from pathlib import Path +import re +from typing import ClassVar + +import numpy as np +from numpy.lib.npyio import NpzFile +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import Annotation, ClassificationTask, Label, LabelSet, Split, TimeInterval, TimeSeriesSpec, ureg +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.splitting import DatasetSplits, StratifiedSplitter + + +N_BINS = 512 + +# The five channel groups of the dataset card, in row order of the ``data`` array. +CHANNEL_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ( + "Pipe signature", + ( + "pipe: tensor_op_hmma", + "pipe: xu", + "pipe: fma", + "pipe: alu", + "pipe: lsu", + "pipe: cbu", + "pipe: tma", + ), + ), + ("Memory access", ("hit: l2", "atom: lts_atomic_input_pct")), + ( + "Discriminators", + ("stall: short_scoreboard", "stall: barrier", "pred_on_per_inst_ratio", "gmem_coalesce_ratio"), + ), + ( + "System bandwidth", + ( + "SMs Active [Throughput %]", + "DRAM Read Bandwidth [Throughput %]", + "DRAM Write Bandwidth [Throughput %]", + "Tensor Active [Throughput %]", + ), + ), + ( + "SASS modality", + ( + "sass_compute_fma", + "sass_compute_tensor", + "sass_memory_global", + "sass_memory_shared", + "sass_memory_tma", + "sass_control", + "sass_misc", + ), + ), +) +COUNTER_NAMES: tuple[str, ...] = tuple(name for _, names in CHANNEL_GROUPS for name in names) + +# The ``vocab_l1`` and ``vocab_l2`` arrays of every labels file, in index order. +VOCAB_L1: tuple[str, ...] = ( + "matmul", + "conv", + "activation", + "normalization", + "softmax", + "pooling", + "reduction", + "attention", + "loss", + "elementwise", + "memory_movement", + "other", +) +VOCAB_L2: tuple[str, ...] = ( + "matmul_bmm", + "matmul_gemm", + "matmul_matvec", + "conv_conv1d_standard", + "conv_conv2d_depthwise", + "conv_conv2d_pointwise", + "conv_conv2d_standard", + "conv_conv3d_standard", + "conv_convtranspose1d", + "conv_convtranspose2d", + "conv_convtranspose3d", + "activation_elu", + "activation_gelu", + "activation_hardsigmoid", + "activation_hardswish", + "activation_hardtanh", + "activation_leaky_relu", + "activation_mish", + "activation_other", + "activation_relu", + "activation_selu", + "activation_sigmoid", + "activation_softplus", + "activation_softsign", + "activation_swish", + "activation_tanh", + "normalization_batchnorm", + "normalization_frobeniusnorm", + "normalization_groupnorm", + "normalization_instancenorm", + "normalization_l1norm", + "normalization_l2norm", + "normalization_layernorm", + "normalization_rmsnorm", + "softmax_log_softmax", + "softmax_logsumexp", + "softmax_softmax", + "pooling_avg_pool", + "pooling_global_avg_pool", + "pooling_max_pool", + "reduction_argmax", + "reduction_argmin", + "reduction_cumprod", + "reduction_cumsum", + "reduction_max", + "reduction_mean", + "reduction_min", + "reduction_prod", + "reduction_sum", + "attention_scaled_dot_product", + "loss_cross_entropy", + "loss_hinge", + "loss_huber", + "loss_kldiv", + "loss_mse", + "loss_triplet_margin", + "elementwise_add", + "elementwise_bias_add", + "elementwise_cast", + "elementwise_clamp", + "elementwise_div", + "elementwise_mul", + "elementwise_residual_add", + "elementwise_scalar_multiplication", + "elementwise_scaling", + "elementwise_sub", + "memory_movement_copy", + "memory_movement_embedding", + "memory_movement_gather", + "memory_movement_scatter", + "memory_movement_transpose", + "other_dropout", + "other_misc", +) + +WORKLOAD_L1 = LabelSet(name="workload_l1", labels=tuple(Label(name=name) for name in VOCAB_L1)) + +OFFICIAL_SPLITS: Mapping[str, Split] = {"train": Split.TRAIN, "val": Split.VALIDATION, "test": Split.TEST} +"""The released partition files. Each snapshot is in exactly one of them.""" +MEMBERSHIP_SPLITS = ("train", "val", "test", "iid", "param_ood", "composed", "length_ood") +"""Every released split file. A snapshot can be in several; the record metadata keeps them all.""" + +MULTIHOT_TRACK = "multihot" +"""The ``track`` of an interval that only the multi-hot overlap track of the release holds.""" + +_INPUT_SUFFIX = "/input/tensor_input.npz" +_LABELS_SUFFIX = "/labels/labels.npz" + + +def _slug(name: str) -> str: + """Turn a counter name such as ``"SMs Active [Throughput %]"`` into ``"sms_active_throughput"``. + + Returns: + The lowercase name, with each run of other characters replaced by one underscore. + """ + return re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") + + +# One spec per channel, because a spec type has one contract and each counter has its own name. The +# values are dimensionless because the authors divided each channel by a fixed physical-scale divisor. +_SPECS: Mapping[str, TimeSeriesSpec] = { + name: TimeSeriesSpec(spec_type=f"gpu_counter_{_slug(name)}", name=name, unit_value=ureg.dimensionless) + for name in COUNTER_NAMES +} + + +@dataclass(frozen=True) +class Snapshot: + """One snapshot of the release, as its split files describe it.""" + + path: str + """The input tensor, relative to the dataset root, such as + ``kernels/kernelbench/_out/L1_P100/dt-bf16_s42/input/tensor_input.npz``.""" + motif: str + dominant_l1: str + official_split: str + memberships: tuple[str, ...] + + def __post_init__(self) -> None: + """Check that the path has the layout of the release and the class is in the vocabulary. + + Raises: + TimeNetConnectorError: If the path or the dominant class is not as expected. + """ + if not (self.path.startswith(f"kernels/{self.motif}/_out/") and self.path.endswith(_INPUT_SUFFIX)): + raise TimeNetConnectorError(f"snapshot path {self.path!r} is not under kernels/{self.motif}/_out/") + if self.dominant_l1 not in VOCAB_L1: + raise TimeNetConnectorError(f"{self.path}: dominant_l1 {self.dominant_l1!r} is not an L1 class") + + @property + def variant(self) -> str: + """The variant directory below ``_out/``, or ``"baseline"`` for a snapshot directly in it.""" + start = len(f"kernels/{self.motif}/_out/") + return self.path[start : -len(_INPUT_SUFFIX)] or "baseline" + + @property + def problem(self) -> str: + """The leakage group: the KernelBench problem for its dtype and seed variants, else the variant.""" + variant = self.variant + if self.motif == "kernelbench": + variant = variant.split("/")[0] + return f"{self.motif}/{variant}" + + @property + def labels_path(self) -> str: + """The labels file of the snapshot, relative to the dataset root.""" + return self.path.removesuffix(_INPUT_SUFFIX) + _LABELS_SUFFIX + + +def read_snapshots(splits_dir: Path) -> list[Snapshot]: + """Read every snapshot from the released split files, sorted by path. + + The sort makes the split assignment independent of the order of the files. + + Args: + splits_dir: The ``splits`` directory of the release. + + Returns: + One snapshot per trace of the train, val and test files. + + Raises: + TimeNetConnectorError: If a trace is in more than one official partition, or a membership + file names a trace that no official partition has. + """ + memberships: dict[str, list[str]] = defaultdict(list) + traces: dict[str, tuple[str, dict]] = {} + for name in MEMBERSHIP_SPLITS: + index = json.loads((splits_dir / f"{name}.json").read_text(encoding="utf-8")) + for trace in index["traces"]: + memberships[trace["path"]].append(name) + if name in OFFICIAL_SPLITS: + if trace["path"] in traces: + raise TimeNetConnectorError(f"{trace['path']} is in more than one official partition") + traces[trace["path"]] = (name, trace) + unknown = sorted(memberships.keys() - traces.keys()) + if unknown: + raise TimeNetConnectorError(f"{len(unknown)} traces are in no official partition, such as {unknown[0]}") + return [ + Snapshot( + path=path, + motif=trace["motif"], + dominant_l1=trace["dominant_l1"], + official_split=official, + memberships=tuple(memberships[path]), + ) + for path, (official, trace) in sorted(traces.items()) + ] + + +def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: + """Partition the snapshots 3/1/1, grouped by problem and stratified by the dominant L1 class. + + Returns: + The train, validation and test snapshots. + """ + splitter = StratifiedSplitter( + train_folds=3, + validation_folds=1, + test_folds=1, + stratify_by=attrgetter("dominant_l1"), + group_by=attrgetter("problem"), + seed=42, + ) + return splitter.split(snapshots) + + +def _load_numeric(path: Path) -> NpzFile: + """Open an ``.npz`` file without unpickling. + + Returns: + The lazy archive. Reading an object array from it raises ``ValueError``. + + Raises: + TimeNetConnectorError: If the file is missing. + """ + if not path.is_file(): + raise TimeNetConnectorError(f"{path} is missing") + return np.load(path, allow_pickle=False) + + +def _channel_loader(path: Path, row: int) -> Callable[[], pa.Array]: + """Build a lazy loader for one channel of a snapshot. + + Returns: + A loader that reads row ``row`` of the ``data`` array as a float32 Arrow array. + """ + + def load() -> pa.Array: + with np.load(path, allow_pickle=False) as archive: + return pa.array(archive["data"][row]) + + return load + + +def _runs(active: np.ndarray) -> Iterator[tuple[int, int]]: + """Yield the ``[start, end)`` bin ranges of each run of consecutive true values.""" + padded = np.concatenate(([False], active.astype(bool), [False])) + edges = np.flatnonzero(np.diff(padded.astype(np.int8))) + yield from zip(edges[::2].tolist(), edges[1::2].tolist(), strict=True) + + +@dataclass(frozen=True) +class SnapshotFiles: + """The downloaded files of one snapshot.""" + + snapshot: Snapshot + root: Path + + def create_record(self) -> Record: + """Create the Record: one Signal per channel, grouped by channel group, with segment annotations. + + Returns: + The Record, not yet added to a dataset. + + Raises: + TimeNetConnectorError: If a file does not have the shape of the release. + """ + input_path = self.root / self.snapshot.path + with _load_numeric(input_path) as tensor: + shape = tensor["data"].shape + edges = tensor["time_edges_ns"] + if shape != (len(COUNTER_NAMES), N_BINS) or edges.shape != (N_BINS + 1,): + raise TimeNetConnectorError(f"{self.snapshot.path}: data has shape {shape}, edges {edges.shape}") + # The bins are equal-width up to one nanosecond of rounding, so the exact mean width is the period. + axis = RegularAxis(period_us=Fraction(int(edges[-1] - edges[0]), N_BINS * 1000)) + signals = iter( + Signal.from_loader( + spec=_SPECS[name], + time_axis=axis, + name=_slug(name), + loader=_channel_loader(input_path, row), + n_values=N_BINS, + metadata={"counter": name}, + ) + for row, name in enumerate(COUNTER_NAMES) + ) + sources = tuple( + Source(name=group, signals=tuple(next(signals) for _ in names)) for group, names in CHANNEL_GROUPS + ) + snapshot = self.snapshot + record = Record( + sources=sources, + metadata={ + "path": snapshot.path, + "motif": snapshot.motif, + "problem": snapshot.problem, + "variant": snapshot.variant, + "official_split": snapshot.official_split, + "kernelsight_splits": list(snapshot.memberships), + }, + ) + self._annotate_segments(record, axis) + return record + + def _annotate_segments(self, record: Record, axis: RegularAxis) -> None: + """Annotate each labeled segment, and each multi-hot run of a class that no segment has. + + Raises: + TimeNetConnectorError: If a label index or a segment bound is out of range, or an L2 class + does not refine the L1 class of its segment. + """ + path = self.snapshot.labels_path + + def interval(start: int, end: int) -> TimeInterval: + if not 0 <= start < end <= N_BINS: + raise TimeNetConnectorError(f"{path}: segment [{start}, {end}) is outside the {N_BINS} bins") + return TimeInterval.micros(int(start * axis.period_us), int(end * axis.period_us)) + + with _load_numeric(self.root / path) as labels: + starts, ends = labels["segment_starts"], labels["segment_ends"] + l1, l2 = labels["segment_label_l1"], labels["segment_label_l2"] + multihot = labels["workload_l1_multihot"] + if multihot.shape != (N_BINS, len(VOCAB_L1)): + raise TimeNetConnectorError(f"{path}: workload_l1_multihot has shape {multihot.shape}") + annotations = [] + for start, end, l1_index, l2_index in zip(starts, ends, l1, l2, strict=True): + if not (0 <= l1_index < len(VOCAB_L1) and 0 <= l2_index < len(VOCAB_L2)): + raise TimeNetConnectorError(f"{path}: label index ({l1_index}, {l2_index}) is out of range") + l1_name, l2_name = VOCAB_L1[l1_index], VOCAB_L2[l2_index] + if not l2_name.startswith(f"{l1_name}_"): + raise TimeNetConnectorError(f"{path}: L2 class {l2_name} does not refine L1 class {l1_name}") + span = interval(int(start), int(end)) + annotations += [ + Annotation(key="workload_l1", value=l1_name, span=span), + Annotation(key="workload_l2", value=l2_name, span=span), + ] + # The overlap track of the release can hold a class that no segment has, such as the TMA + # loads that run during a warp-specialized GEMM. Keep each of its runs as an interval. + segment_classes = {int(index) for index in l1} + for class_index in sorted(set(np.flatnonzero(multihot.any(axis=0)).tolist()) - segment_classes): + annotations += [ + Annotation( + key="workload_l1", + value=VOCAB_L1[class_index], + span=interval(start, end), + occurrence_metadata={"track": MULTIHOT_TRACK}, + ) + for start, end in _runs(multihot[:, class_index]) + ] + record.add_annotations(annotations) + + +class KernelSightConnector(BaseHuggingFaceConnector): + """Connector for the KernelSight release on the Hub (``williamhtan/kernelsight``).""" + + HF_REPO = "williamhtan/kernelsight" + SOURCE_REVISION = "d8cd8e804e5b619aadbb082a1fb3a8791b60a680" + # A glob's part path is its directory, so both kernel parts resolve to ``kernels/``. The ``*`` + # matches across directories, which selects the input and labels files and skips the fingerprints. + SOURCE_FILES: ClassVar[Mapping[str, str]] = { + "inputs": "kernels/*tensor_input.npz", + "labels": "kernels/*labels.npz", + "splits": "splits/*.json", + } + + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Create one Record and one ClassificationTask for each snapshot. + + Args: + raw_refs: The kernel and split directories from :meth:`download`. + + Returns: + The populated dataset. + """ + paths = self.ref_paths(raw_refs, "inputs", "labels", "splits") + root = paths["inputs"].parent + dataset = TimeFDataset(metadata=self.metadata()) + splits = assign_splits(read_snapshots(paths["splits"])) + for split, snapshots in splits.items(): + for snapshot in snapshots: + record = dataset.add_record(record=SnapshotFiles(snapshot, root).create_record()) + dataset.add_task( + task=ClassificationTask( + inputs=(record,), + targets=(snapshot.dominant_l1,), + label_set=WORKLOAD_L1, + split=split, + ) + ) + return dataset + + +CONNECTOR = KernelSightConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml new file mode 100644 index 000000000..162fee3cc --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml @@ -0,0 +1,40 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: williamhtan/kernelsight +dataset_version: 1.0.0 +name: KernelSight +description: >- + 1,444 GPU workload snapshots profiled on one NVIDIA H100 80GB HBM3 (sm_90a). Each snapshot is + 24 hardware-counter channels over 512 equal-width bins that cover the kernel-active window, so + the bin width differs per snapshot (about 0.2 ms to 68 ms). The channels form five groups: pipe + signature, memory access, discriminators, system bandwidth (Nsight Systems) and SASS modality + (NVBit). The values are kept as released. The authors divided each channel by a fixed + physical-scale divisor (for example 100 for a percent, 64 for a warp-stall ratio, 8 for + sectors per warp load), so most values are in [0, 1], but pred_on_per_inst_ratio reaches about + 32. Each labeled segment is a time-interval annotation with its L1 class (12 classes, of which + `other` never occurs) and its L2 class (73 classes). In the 472 cutlass_ws_overlap snapshots, + an overlapping memory_movement interval exists only in the source's multi-hot track; it is + added as a workload_l1 annotation with occurrence metadata track=multihot. Each snapshot has + one classification task whose target is the authors' dominant_l1 field, which can differ from + the L1 class with the most labeled bins (34 snapshots). The released train/val/test split + puts dtype and seed variants of one KernelBench problem into different partitions, so this + dataset assigns its own split instead. It is grouped by problem (a KernelBench problem keeps + all its variants together; any other variant directory is its own group), stratified by + dominant_l1, with 3/1/1 folds and seed 42. CUTLASS shape sweeps of one binary can still cross + partitions; each record's metadata keeps the authors' split memberships (train, val, test, + iid, param_ood, composed) so the released benchmark stays reproducible. softmax has only two + problems and cannot appear in all three partitions. +license: Apache-2.0 +source_url: https://huggingface.co/datasets/williamhtan/kernelsight +domains: + - observability +tags: + - gpu + - cuda + - profiling + - h100 + - kernelbench + - cutlass + - classification + - segmentation + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt new file mode 100644 index 000000000..47df73a21 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt @@ -0,0 +1,2 @@ +# huggingface_hub downloads the release from the HuggingFace Hub. Version 1.1 adds tqdm_class. +huggingface_hub>=1.1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py new file mode 100644 index 000000000..228e1fc9b --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py @@ -0,0 +1,224 @@ +import json +from pathlib import Path + +import numpy as np +import pytest + +from timenet.dataset import TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing import assert_datasets_equal +from timenet.types import ClassificationTask, Split, TimeInterval +from timenet.types.text import read_value +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( + CHANNEL_GROUPS, + COUNTER_NAMES, + MULTIHOT_TRACK, + N_BINS, + VOCAB_L1, + VOCAB_L2, + WORKLOAD_L1, + KernelSightConnector, + Snapshot, + assign_splits, + read_snapshots, +) + + +_BIN_NS = 1_000_000 +"""One millisecond per bin, so bin ``i`` starts at ``i * 1000`` microseconds.""" +_MATMUL, _MEMORY = VOCAB_L1.index("matmul"), VOCAB_L1.index("memory_movement") +_GEMM = VOCAB_L2.index("matmul_gemm") + + +def _trace(path: str, motif: str, dominant: str) -> dict: + return {"path": path, "motif": motif, "dominant_l1": dominant} + + +def _write_snapshot( + root: Path, + path: str, + *, + seed: int, + segments: list[tuple[int, int, int, int]], + multihot: np.ndarray | None = None, +) -> np.ndarray: + # Synthetic files in the shape of the release, including its pickled object arrays, which the + # connector must never read. + data = np.random.default_rng(seed).random((len(COUNTER_NAMES), N_BINS), dtype=np.float32) + edges = 5_000_000 + np.arange(N_BINS + 1, dtype=np.int64) * _BIN_NS + input_path = root / path + input_path.parent.mkdir(parents=True) + np.savez(input_path, data=data, time_edges_ns=edges, counter_names=np.array(COUNTER_NAMES, dtype=object)) + starts, ends, l1, l2 = ( + np.array(column, dtype=dtype) + for column, dtype in zip(zip(*segments, strict=True), (np.int64, np.int64, np.int32, np.int32), strict=True) + ) + if multihot is None: + multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) + labels_path = root / path.replace("/input/tensor_input.npz", "/labels/labels.npz") + labels_path.parent.mkdir(parents=True) + np.savez( + labels_path, + segment_starts=starts, + segment_ends=ends, + segment_label_l1=l1, + segment_label_l2=l2, + workload_l1_multihot=multihot, + vocab_l1=np.array(VOCAB_L1, dtype=object), + vocab_l2=np.array(VOCAB_L2, dtype=object), + ) + return data + + +def _write_splits(root: Path, official: dict[str, list[dict]], extra: dict[str, list[dict]] | None = None) -> None: + splits = root / "splits" + splits.mkdir() + files = {"train": [], "val": [], "test": [], "iid": [], "param_ood": [], "composed": [], "length_ood": []} + files |= official | (extra or {}) + for name, traces in files.items(): + (splits / f"{name}.json").write_text(json.dumps({"split": name, "n": len(traces), "traces": traces})) + + +def _refs(root: Path) -> list[HubFileRef]: + return [ + HubFileRef(root / "kernels", "inputs"), + HubFileRef(root / "kernels", "labels"), + HubFileRef(root / "splits", "splits"), + ] + + +def _gemm_path(index: int) -> str: + return f"kernels/cutlass_gemm/_out/m{index}/input/tensor_input.npz" + + +def _overlap_multihot() -> np.ndarray: + # A GEMM in bins [100, 200) with TMA loads in [120, 130) and [150, 160) that no segment holds. + multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) + multihot[100:200, _MATMUL] = 1 + multihot[120:130, _MEMORY] = 1 + multihot[150:160, _MEMORY] = 1 + return multihot + + +def _convert(root: Path) -> tuple[TimeFDataset, dict[str, np.ndarray]]: + # Five problems, the fewest that the 3/1/1 split accepts. + values = {} + traces = [] + for index in range(5): + path = _gemm_path(index) + multihot = _overlap_multihot() if index == 0 else None + values[path] = _write_snapshot(root, path, seed=index, segments=[(100, 200, _MATMUL, _GEMM)], multihot=multihot) + traces.append(_trace(path, "cutlass_gemm", "matmul")) + _write_splits(root, {"train": traces[:3], "val": traces[3:4], "test": traces[4:]}, {"param_ood": traces[:1]}) + return KernelSightConnector().convert(_refs(root)), values + + +def test_each_snapshot_becomes_a_record_and_a_classification_task(tmp_path): + dataset, values = _convert(tmp_path) + tasks = dataset.tasks_of(ClassificationTask) + assert len(tasks) == len(values) + assert {task.split for task in tasks} == {Split.TRAIN, Split.VALIDATION, Split.TEST} + for task in tasks: + (record,) = task.inputs + assert task.label_set == WORKLOAD_L1 + assert task.targets is not None + assert read_value(task.targets[0]) == "matmul" + assert [source.name for source in record.sources] == [group for group, _ in CHANNEL_GROUPS] + signals = [signal for source in record.sources for signal in source.signals] + assert [signal.metadata["counter"] for signal in signals] == list(COUNTER_NAMES) + for row, signal in enumerate(signals): + np.testing.assert_array_equal(signal.to_numpy(), values[str(record.metadata["path"])][row]) + axis = signals[0].time_axis + assert isinstance(axis, RegularAxis) + assert axis.period_us == 1000 + + +def test_record_metadata_keeps_the_released_split_memberships(tmp_path): + dataset, _ = _convert(tmp_path) + by_path = { + task.inputs[0].metadata["path"]: task.inputs[0].metadata for task in dataset.tasks_of(ClassificationTask) + } + assert by_path[_gemm_path(0)] == { + "path": _gemm_path(0), + "motif": "cutlass_gemm", + "problem": "cutlass_gemm/m0", + "variant": "m0", + "official_split": "train", + "kernelsight_splits": ["train", "param_ood"], + } + + +def test_segments_and_multihot_only_runs_become_interval_annotations(tmp_path): + dataset, _ = _convert(tmp_path) + by_path = {task.inputs[0].metadata["path"]: task.inputs[0] for task in dataset.tasks_of(ClassificationTask)} + annotations = by_path[_gemm_path(0)].annotations + found = {(a.key, read_value(a.value), a.span, a.occurrence_metadata.get("track")) for a in annotations} + segment = TimeInterval.micros(100_000, 200_000) + assert found == { + ("workload_l1", "matmul", segment, None), + ("workload_l2", "matmul_gemm", segment, None), + ("workload_l1", "memory_movement", TimeInterval.micros(120_000, 130_000), MULTIHOT_TRACK), + ("workload_l1", "memory_movement", TimeInterval.micros(150_000, 160_000), MULTIHOT_TRACK), + } + # A snapshot without an overlap track has only its segment. + assert len(by_path[_gemm_path(1)].annotations) == 2 + + +def test_variants_of_one_kernelbench_problem_share_a_partition(): + snapshots = [ + Snapshot( + path=f"kernels/kernelbench/_out/L1_P{problem}/dt-{dtype}_s42/input/tensor_input.npz", + motif="kernelbench", + dominant_l1="conv" if problem % 2 else "matmul", + official_split="train", + memberships=("train",), + ) + for problem in range(20) + for dtype in ("bf16", "fp16", "fp32") + ] + assert snapshots[0].problem == "kernelbench/L1_P0" + partition_of = {snapshot.path: split for split, members in assign_splits(snapshots).items() for snapshot in members} + for problem in range(20): + assert len({split for path, split in partition_of.items() if f"/L1_P{problem}/" in path}) == 1 + + +def test_a_snapshot_directly_in_the_output_directory_is_the_baseline_variant(): + snapshot = Snapshot( + path="kernels/cutlass_ws_overlap/_out/input/tensor_input.npz", + motif="cutlass_ws_overlap", + dominant_l1="matmul", + official_split="train", + memberships=("train",), + ) + assert snapshot.problem == "cutlass_ws_overlap/baseline" + + +def test_a_trace_in_two_official_partitions_raises(tmp_path): + trace = _trace(_gemm_path(0), "cutlass_gemm", "matmul") + _write_splits(tmp_path, {"train": [trace], "test": [trace]}) + with pytest.raises(TimeNetConnectorError, match="more than one official partition"): + read_snapshots(tmp_path / "splits") + + +def test_an_l2_class_outside_its_l1_class_raises(tmp_path): + for index in range(5): + l2 = VOCAB_L2.index("conv_conv1d_standard") if index == 0 else _GEMM + _write_snapshot(tmp_path, _gemm_path(index), seed=index, segments=[(0, 10, _MATMUL, l2)]) + traces = [_trace(_gemm_path(index), "cutlass_gemm", "matmul") for index in range(5)] + _write_splits(tmp_path, {"train": traces}) + with pytest.raises(TimeNetConnectorError, match="conv_conv1d_standard does not refine L1 class matmul"): + KernelSightConnector().convert(_refs(tmp_path)) + + +def test_the_dataset_round_trips_through_storage(tmp_path): + source = tmp_path / "source" + source.mkdir() + dataset, _ = _convert(source) + version_dir = dataset.write(path=tmp_path / "registry") + with TimeFReader(open_local_version(version_dir)) as reader: + reader.verify() + assert_datasets_equal(dataset, reader.read())