From 2de8b4c3b5cb13a706aebfabe8101f55ade415b5 Mon Sep 17 00:00:00 2001 From: mahmood Date: Fri, 20 Nov 2026 15:40:43 +0400 Subject: [PATCH 1/4] feat(connectors): add williamhtan/kernelsight KernelSight: 1,444 H100 GPU counter snapshots (24 channels x 512 bins) with per-segment L1/L2 workload labels, from the pinned Hub release. - One Record per snapshot, 24 Signals in five channel-group Sources, a per-snapshot RegularAxis from the bin edges. - Segment L1/L2 interval annotations, plus multi-hot-only overlap runs (track=multihot) for the 472 cutlass_ws_overlap snapshots. - One ClassificationTask per snapshot on the authors' dominant_l1. - Split grouped by problem (KernelBench dtype/seed variants stay together), stratified by class, 3/1/1, seed 42; the released split memberships are kept in record metadata. - Reads only numeric npz arrays (allow_pickle=False). Co-Authored-By: Claude Opus 5.5 --- .../datasets/williamhtan/__init__.py | 4 + .../williamhtan/kernelsight/__init__.py | 6 + .../williamhtan/kernelsight/connector.py | 482 ++++++++++++++++++ .../williamhtan/kernelsight/dataset.yaml | 44 ++ .../williamhtan/kernelsight/requirements.txt | 2 + .../kernelsight/tests/test_connector.py | 219 ++++++++ 6 files changed, 757 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py new file mode 100644 index 000000000..3367da279 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py @@ -0,0 +1,4 @@ +"""Connectors for datasets published by the ``williamhtan`` Hub user.""" + +# Keep this a regular package, not a namespace package. pkgutil.walk_packages finds the +# connector packages below it only through this __init__.py. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py new file mode 100644 index 000000000..c60729eaf --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py @@ -0,0 +1,6 @@ +"""KernelSight: H100 GPU counter traces with per-segment workload labels.""" + +from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( + CONNECTOR as CONNECTOR, + KernelSightConnector as KernelSightConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py new file mode 100644 index 000000000..edcc5d074 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py @@ -0,0 +1,482 @@ +"""The KernelSight connector: H100 GPU counter traces with per-segment workload labels. + +Each snapshot of the release is a ``[24, 512]`` counter image: 24 hardware-counter channels over 512 +equal-width bins. Each snapshot becomes one Record with one Signal per channel, grouped into the +five channel groups of the dataset card. Each labeled segment becomes a time-interval annotation +with its L1 and L2 workload class. Each snapshot gets one ClassificationTask whose target is the +authors' dominant L1 class. + +The release's own train/val/test split puts dtype and seed variants of one KernelBench problem in +different partitions. The connector therefore assigns a split grouped by problem and keeps the +authors' split memberships in the record metadata. + +The release stores its vocabularies and counter names as pickled object arrays. Unpickling a +downloaded file can run code, so the connector keeps those names as constants and reads only the +numeric arrays, with ``allow_pickle=False``. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from fractions import Fraction +import json +from operator import attrgetter +from pathlib import Path +import re + +import numpy as np +from numpy.lib.npyio import NpzFile +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import Annotation, ClassificationTask, LabelSet, Split, TimeInterval, TimeSeriesSpec, ureg +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector +from timenet_connectors.splitting import DatasetSplits, StratifiedSplitter + + +N_BINS = 512 + +# The five channel groups of the dataset card, in row order of the ``data`` array. +CHANNEL_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ( + "Pipe signature", + ( + "pipe: tensor_op_hmma", + "pipe: xu", + "pipe: fma", + "pipe: alu", + "pipe: lsu", + "pipe: cbu", + "pipe: tma", + ), + ), + ("Memory access", ("hit: l2", "atom: lts_atomic_input_pct")), + ( + "Discriminators", + ("stall: short_scoreboard", "stall: barrier", "pred_on_per_inst_ratio", "gmem_coalesce_ratio"), + ), + ( + "System bandwidth", + ( + "SMs Active [Throughput %]", + "DRAM Read Bandwidth [Throughput %]", + "DRAM Write Bandwidth [Throughput %]", + "Tensor Active [Throughput %]", + ), + ), + ( + "SASS modality", + ( + "sass_compute_fma", + "sass_compute_tensor", + "sass_memory_global", + "sass_memory_shared", + "sass_memory_tma", + "sass_control", + "sass_misc", + ), + ), +) +COUNTER_NAMES: tuple[str, ...] = tuple(name for _, names in CHANNEL_GROUPS for name in names) + +# The ``vocab_l1`` and ``vocab_l2`` arrays of every labels file, in index order. +VOCAB_L1: tuple[str, ...] = ( + "matmul", + "conv", + "activation", + "normalization", + "softmax", + "pooling", + "reduction", + "attention", + "loss", + "elementwise", + "memory_movement", + "other", +) +VOCAB_L2: tuple[str, ...] = ( + "matmul_bmm", + "matmul_gemm", + "matmul_matvec", + "conv_conv1d_standard", + "conv_conv2d_depthwise", + "conv_conv2d_pointwise", + "conv_conv2d_standard", + "conv_conv3d_standard", + "conv_convtranspose1d", + "conv_convtranspose2d", + "conv_convtranspose3d", + "activation_elu", + "activation_gelu", + "activation_hardsigmoid", + "activation_hardswish", + "activation_hardtanh", + "activation_leaky_relu", + "activation_mish", + "activation_other", + "activation_relu", + "activation_selu", + "activation_sigmoid", + "activation_softplus", + "activation_softsign", + "activation_swish", + "activation_tanh", + "normalization_batchnorm", + "normalization_frobeniusnorm", + "normalization_groupnorm", + "normalization_instancenorm", + "normalization_l1norm", + "normalization_l2norm", + "normalization_layernorm", + "normalization_rmsnorm", + "softmax_log_softmax", + "softmax_logsumexp", + "softmax_softmax", + "pooling_avg_pool", + "pooling_global_avg_pool", + "pooling_max_pool", + "reduction_argmax", + "reduction_argmin", + "reduction_cumprod", + "reduction_cumsum", + "reduction_max", + "reduction_mean", + "reduction_min", + "reduction_prod", + "reduction_sum", + "attention_scaled_dot_product", + "loss_cross_entropy", + "loss_hinge", + "loss_huber", + "loss_kldiv", + "loss_mse", + "loss_triplet_margin", + "elementwise_add", + "elementwise_bias_add", + "elementwise_cast", + "elementwise_clamp", + "elementwise_div", + "elementwise_mul", + "elementwise_residual_add", + "elementwise_scalar_multiplication", + "elementwise_scaling", + "elementwise_sub", + "memory_movement_copy", + "memory_movement_embedding", + "memory_movement_gather", + "memory_movement_scatter", + "memory_movement_transpose", + "other_dropout", + "other_misc", +) + +WORKLOAD_L1 = LabelSet(VOCAB_L1, annotations=[Annotation("name", value="workload_l1")]) + +OFFICIAL_SPLITS: Mapping[str, Split] = {"train": Split.TRAIN, "val": Split.VALIDATION, "test": Split.TEST} +"""The released partition files. Each snapshot is in exactly one of them.""" +MEMBERSHIP_SPLITS = ("train", "val", "test", "iid", "param_ood", "composed", "length_ood") +"""Every released split file. A snapshot can be in several; the record metadata keeps them all.""" + +MULTIHOT_TRACK = "multihot" +"""The ``track`` of an interval that only the multi-hot overlap track of the release holds.""" + +_INPUT_SUFFIX = "/input/tensor_input.npz" +_LABELS_SUFFIX = "/labels/labels.npz" + + +def _slug(name: str) -> str: + """Turn a counter name such as ``"SMs Active [Throughput %]"`` into ``"sms_active_throughput"``. + + Returns: + The lowercase name, with each run of other characters replaced by one underscore. + """ + return re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") + + +# One spec per channel, because a spec type has one contract and each counter has its own name. The +# values are dimensionless because the authors divided each channel by a fixed physical-scale divisor. +_SPECS: Mapping[str, TimeSeriesSpec] = { + name: TimeSeriesSpec(spec_type=f"gpu_counter_{_slug(name)}", name=name, unit_value=ureg.dimensionless) + for name in COUNTER_NAMES +} + + +@dataclass(frozen=True) +class Snapshot: + """One snapshot of the release, as its split files describe it.""" + + path: str + """The input tensor, relative to the dataset root, such as + ``kernels/kernelbench/_out/L1_P100/dt-bf16_s42/input/tensor_input.npz``.""" + motif: str + dominant_l1: str + official_split: str + memberships: tuple[str, ...] + + def __post_init__(self) -> None: + """Check that the path has the layout of the release and the class is in the vocabulary. + + Raises: + TimeNetConnectorError: If the path or the dominant class is not as expected. + """ + if not (self.path.startswith(f"kernels/{self.motif}/_out/") and self.path.endswith(_INPUT_SUFFIX)): + raise TimeNetConnectorError(f"snapshot path {self.path!r} is not under kernels/{self.motif}/_out/") + if self.dominant_l1 not in VOCAB_L1: + raise TimeNetConnectorError(f"{self.path}: dominant_l1 {self.dominant_l1!r} is not an L1 class") + + @property + def variant(self) -> str: + """The variant directory below ``_out/``, or ``"baseline"`` for a snapshot directly in it.""" + start = len(f"kernels/{self.motif}/_out/") + return self.path[start : -len(_INPUT_SUFFIX)] or "baseline" + + @property + def problem(self) -> str: + """The leakage group: the KernelBench problem for its dtype and seed variants, else the variant.""" + variant = self.variant + if self.motif == "kernelbench": + variant = variant.split("/")[0] + return f"{self.motif}/{variant}" + + @property + def labels_path(self) -> str: + """The labels file of the snapshot, relative to the dataset root.""" + return self.path.removesuffix(_INPUT_SUFFIX) + _LABELS_SUFFIX + + +def read_snapshots(splits_dir: Path) -> list[Snapshot]: + """Read every snapshot from the released split files, sorted by path. + + The sort makes the split assignment independent of the order of the files. + + Args: + splits_dir: The ``splits`` directory of the release. + + Returns: + One snapshot per trace of the train, val and test files. + + Raises: + TimeNetConnectorError: If a trace is in more than one official partition, or a membership + file names a trace that no official partition has. + """ + memberships: dict[str, list[str]] = defaultdict(list) + traces: dict[str, tuple[str, dict]] = {} + for name in MEMBERSHIP_SPLITS: + index = json.loads((splits_dir / f"{name}.json").read_text(encoding="utf-8")) + for trace in index["traces"]: + memberships[trace["path"]].append(name) + if name in OFFICIAL_SPLITS: + if trace["path"] in traces: + raise TimeNetConnectorError(f"{trace['path']} is in more than one official partition") + traces[trace["path"]] = (name, trace) + unknown = sorted(memberships.keys() - traces.keys()) + if unknown: + raise TimeNetConnectorError(f"{len(unknown)} traces are in no official partition, such as {unknown[0]}") + return [ + Snapshot( + path=path, + motif=trace["motif"], + dominant_l1=trace["dominant_l1"], + official_split=official, + memberships=tuple(memberships[path]), + ) + for path, (official, trace) in sorted(traces.items()) + ] + + +def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: + """Partition the snapshots 3/1/1, grouped by problem and stratified by the dominant L1 class. + + Returns: + The train, validation and test snapshots. + """ + splitter = StratifiedSplitter( + train_folds=3, + validation_folds=1, + test_folds=1, + stratify_by=attrgetter("dominant_l1"), + group_by=attrgetter("problem"), + seed=42, + ) + return splitter.split(snapshots) + + +def _load_numeric(path: Path) -> NpzFile: + """Open an ``.npz`` file without unpickling. + + Returns: + The lazy archive. Reading an object array from it raises ``ValueError``. + + Raises: + TimeNetConnectorError: If the file is missing. + """ + if not path.is_file(): + raise TimeNetConnectorError(f"{path} is missing") + return np.load(path, allow_pickle=False) + + +def _channel_loader(path: Path, row: int) -> Callable[[], pa.Array]: + """Build a lazy loader for one channel of a snapshot. + + Returns: + A loader that reads row ``row`` of the ``data`` array as a float32 Arrow array. + """ + + def load() -> pa.Array: + with np.load(path, allow_pickle=False) as archive: + return pa.array(archive["data"][row]) + + return load + + +def _runs(active: np.ndarray) -> Iterator[tuple[int, int]]: + """Yield the ``[start, end)`` bin ranges of each run of consecutive true values.""" + padded = np.concatenate(([False], active.astype(bool), [False])) + edges = np.flatnonzero(np.diff(padded.astype(np.int8))) + yield from zip(edges[::2].tolist(), edges[1::2].tolist(), strict=True) + + +@dataclass(frozen=True) +class SnapshotFiles: + """The downloaded files of one snapshot.""" + + snapshot: Snapshot + root: Path + + def create_record(self) -> Record: + """Create the Record: one Signal per channel, grouped by channel group, with segment annotations. + + Returns: + The Record, not yet added to a dataset. + + Raises: + TimeNetConnectorError: If a file does not have the shape of the release. + """ + input_path = self.root / self.snapshot.path + with _load_numeric(input_path) as tensor: + shape = tensor["data"].shape + edges = tensor["time_edges_ns"] + if shape != (len(COUNTER_NAMES), N_BINS) or edges.shape != (N_BINS + 1,): + raise TimeNetConnectorError(f"{self.snapshot.path}: data has shape {shape}, edges {edges.shape}") + # The bins are equal-width up to one nanosecond of rounding, so the exact mean width is the period. + axis = RegularAxis(period_us=Fraction(int(edges[-1] - edges[0]), N_BINS * 1000)) + signals = iter( + Signal.from_loader( + spec=_SPECS[name], + time_axis=axis, + name=_slug(name), + loader=_channel_loader(input_path, row), + n_values=N_BINS, + metadata={"counter": name}, + ) + for row, name in enumerate(COUNTER_NAMES) + ) + sources = tuple(Source(group, signals=tuple(next(signals) for _ in names)) for group, names in CHANNEL_GROUPS) + snapshot = self.snapshot + return Record( + sources=sources, + metadata={ + "path": snapshot.path, + "motif": snapshot.motif, + "problem": snapshot.problem, + "variant": snapshot.variant, + "official_split": snapshot.official_split, + "kernelsight_splits": list(snapshot.memberships), + }, + annotations=self._segment_annotations(axis), + ) + + def _segment_annotations(self, axis: RegularAxis) -> list[Annotation]: + """Annotate each labeled segment, and each multi-hot run of a class that no segment has. + + Returns: + The interval annotations of the Record. + + Raises: + TimeNetConnectorError: If a label index or a segment bound is out of range, or an L2 class + does not refine the L1 class of its segment. + """ + path = self.snapshot.labels_path + + def interval(start: int, end: int) -> TimeInterval: + if not 0 <= start < end <= N_BINS: + raise TimeNetConnectorError(f"{path}: segment [{start}, {end}) is outside the {N_BINS} bins") + return TimeInterval.micros(int(start * axis.period_us), int(end * axis.period_us)) + + with _load_numeric(self.root / path) as labels: + starts, ends = labels["segment_starts"], labels["segment_ends"] + l1, l2 = labels["segment_label_l1"], labels["segment_label_l2"] + multihot = labels["workload_l1_multihot"] + if multihot.shape != (N_BINS, len(VOCAB_L1)): + raise TimeNetConnectorError(f"{path}: workload_l1_multihot has shape {multihot.shape}") + annotations = [] + for start, end, l1_index, l2_index in zip(starts, ends, l1, l2, strict=True): + if not (0 <= l1_index < len(VOCAB_L1) and 0 <= l2_index < len(VOCAB_L2)): + raise TimeNetConnectorError(f"{path}: label index ({l1_index}, {l2_index}) is out of range") + l1_name, l2_name = VOCAB_L1[l1_index], VOCAB_L2[l2_index] + if not l2_name.startswith(f"{l1_name}_"): + raise TimeNetConnectorError(f"{path}: L2 class {l2_name} does not refine L1 class {l1_name}") + span = interval(int(start), int(end)) + annotations += [ + Annotation("workload_l1", value=l1_name, span=span), + Annotation("workload_l2", value=l2_name, span=span), + ] + # The overlap track of the release can hold a class that no segment has, such as the TMA + # loads that run during a warp-specialized GEMM. Keep each of its runs as an interval. + segment_classes = {int(index) for index in l1} + for class_index in sorted(set(np.flatnonzero(multihot.any(axis=0)).tolist()) - segment_classes): + annotations += [ + Annotation( + "workload_l1", + value=VOCAB_L1[class_index], + span=interval(start, end), + occurrence_metadata={"track": MULTIHOT_TRACK}, + ) + for start, end in _runs(multihot[:, class_index]) + ] + return annotations + + +# A glob's part path is its directory, so both kernel parts resolve to ``kernels/``. The ``*`` matches +# across directories, which selects the input and labels files and skips the fingerprints. +class KernelSightConnector( + BaseHuggingFaceConnector, + hf_repo="williamhtan/kernelsight", + source_revision="d8cd8e804e5b619aadbb082a1fb3a8791b60a680", + source_files={ + "inputs": "kernels/*tensor_input.npz", + "labels": "kernels/*labels.npz", + "splits": "splits/*.json", + }, +): + """Connector for the KernelSight release on the Hub (``williamhtan/kernelsight``).""" + + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: + """Create one Record and one ClassificationTask for each snapshot. + + Args: + files: The kernel and split directories from :meth:`download`. + + Returns: + The populated dataset. + """ + root = files["inputs"].parent + dataset = TimeFDataset(metadata=self.metadata()) + splits = assign_splits(read_snapshots(files["splits"])) + for split, snapshots in splits.items(): + for snapshot in snapshots: + record = dataset.add_record(SnapshotFiles(snapshot, root).create_record()) + dataset.add_task( + ClassificationTask( + inputs=(record,), + targets=snapshot.dominant_l1, + label_set=WORKLOAD_L1, + split=split, + ) + ) + return dataset + + +CONNECTOR = KernelSightConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml new file mode 100644 index 000000000..f70e78245 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: williamhtan/kernelsight +dataset_version: 1.0.0 +name: KernelSight +description: >- + 1,444 GPU workload snapshots profiled on one NVIDIA H100 80GB HBM3 (sm_90a). Each snapshot is + 24 hardware-counter channels over 512 equal-width bins that cover the kernel-active window, so + the bin width differs per snapshot (about 0.2 ms to 68 ms). The channels form five groups: pipe + signature, memory access, discriminators, system bandwidth (Nsight Systems) and SASS modality + (NVBit). The values are kept as released. The authors divided each channel by a fixed + physical-scale divisor (for example 100 for a percent, 64 for a warp-stall ratio, 8 for + sectors per warp load), so most values are in [0, 1], but pred_on_per_inst_ratio reaches about + 32. Each labeled segment is a time-interval annotation with its L1 class (12 classes, of which + `other` never occurs) and its L2 class (73 classes). In the 472 cutlass_ws_overlap snapshots, + an overlapping memory_movement interval exists only in the source's multi-hot track; it is + added as a workload_l1 annotation with occurrence metadata track=multihot. Each snapshot has + one classification task whose target is the authors' dominant_l1 field, which can differ from + the L1 class with the most labeled bins (34 snapshots). The released train/val/test split + puts dtype and seed variants of one KernelBench problem into different partitions, so this + dataset assigns its own split instead. It is grouped by problem (a KernelBench problem keeps + all its variants together; any other variant directory is its own group), stratified by + dominant_l1, with 3/1/1 folds and seed 42. CUTLASS shape sweeps of one binary can still cross + partitions; each record's metadata keeps the authors' split memberships (train, val, test, + iid, param_ood, composed, length_ood) so the released benchmark stays reproducible. softmax has only two + problems and cannot appear in all three partitions. +license: Apache-2.0 +source_url: https://huggingface.co/datasets/williamhtan/kernelsight +citation: >- + Tan, William. KernelSight: Per-Timestep Workload Labeling of GPU Execution + Traces. CS231N project, Stanford University, 2026. + https://huggingface.co/datasets/williamhtan/kernelsight +domains: + - observability +tags: + - gpu + - cuda + - profiling + - h100 + - kernelbench + - cutlass + - classification + - segmentation + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt new file mode 100644 index 000000000..47df73a21 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt @@ -0,0 +1,2 @@ +# huggingface_hub downloads the release from the HuggingFace Hub. Version 1.1 adds tqdm_class. +huggingface_hub>=1.1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py new file mode 100644 index 000000000..3a9c2d885 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py @@ -0,0 +1,219 @@ +import json +from pathlib import Path + +import numpy as np +import pytest + +from timenet.dataset import TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing import assert_datasets_equal +from timenet.types import ClassificationTask, Split, TimeInterval +from timenet.types.text import read_value +from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( + CHANNEL_GROUPS, + COUNTER_NAMES, + MULTIHOT_TRACK, + N_BINS, + VOCAB_L1, + VOCAB_L2, + WORKLOAD_L1, + KernelSightConnector, + Snapshot, + assign_splits, + read_snapshots, +) + + +_BIN_NS = 1_000_000 +"""One millisecond per bin, so bin ``i`` starts at ``i * 1000`` microseconds.""" +_MATMUL, _MEMORY = VOCAB_L1.index("matmul"), VOCAB_L1.index("memory_movement") +_GEMM = VOCAB_L2.index("matmul_gemm") + + +def _trace(path: str, motif: str, dominant: str) -> dict: + return {"path": path, "motif": motif, "dominant_l1": dominant} + + +def _write_snapshot( + root: Path, + path: str, + *, + seed: int, + segments: list[tuple[int, int, int, int]], + multihot: np.ndarray | None = None, +) -> np.ndarray: + # Synthetic files in the shape of the release, including its pickled object arrays, which the + # connector must never read. + data = np.random.default_rng(seed).random((len(COUNTER_NAMES), N_BINS), dtype=np.float32) + edges = 5_000_000 + np.arange(N_BINS + 1, dtype=np.int64) * _BIN_NS + input_path = root / path + input_path.parent.mkdir(parents=True) + np.savez(input_path, data=data, time_edges_ns=edges, counter_names=np.array(COUNTER_NAMES, dtype=object)) + starts, ends, l1, l2 = ( + np.array(column, dtype=dtype) + for column, dtype in zip(zip(*segments, strict=True), (np.int64, np.int64, np.int32, np.int32), strict=True) + ) + if multihot is None: + multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) + labels_path = root / path.replace("/input/tensor_input.npz", "/labels/labels.npz") + labels_path.parent.mkdir(parents=True) + np.savez( + labels_path, + segment_starts=starts, + segment_ends=ends, + segment_label_l1=l1, + segment_label_l2=l2, + workload_l1_multihot=multihot, + vocab_l1=np.array(VOCAB_L1, dtype=object), + vocab_l2=np.array(VOCAB_L2, dtype=object), + ) + return data + + +def _write_splits(root: Path, official: dict[str, list[dict]], extra: dict[str, list[dict]] | None = None) -> None: + splits = root / "splits" + splits.mkdir() + files = {"train": [], "val": [], "test": [], "iid": [], "param_ood": [], "composed": [], "length_ood": []} + files |= official | (extra or {}) + for name, traces in files.items(): + (splits / f"{name}.json").write_text(json.dumps({"split": name, "n": len(traces), "traces": traces})) + + +def _files(root: Path) -> dict[str, Path]: + return {"inputs": root / "kernels", "labels": root / "kernels", "splits": root / "splits"} + + +def _gemm_path(index: int) -> str: + return f"kernels/cutlass_gemm/_out/m{index}/input/tensor_input.npz" + + +def _overlap_multihot() -> np.ndarray: + # A GEMM in bins [100, 200) with TMA loads in [120, 130) and [150, 160) that no segment holds. + multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) + multihot[100:200, _MATMUL] = 1 + multihot[120:130, _MEMORY] = 1 + multihot[150:160, _MEMORY] = 1 + return multihot + + +def _convert(root: Path) -> tuple[TimeFDataset, dict[str, np.ndarray]]: + # Five problems, the fewest that the 3/1/1 split accepts. + values = {} + traces = [] + for index in range(5): + path = _gemm_path(index) + multihot = _overlap_multihot() if index == 0 else None + values[path] = _write_snapshot(root, path, seed=index, segments=[(100, 200, _MATMUL, _GEMM)], multihot=multihot) + traces.append(_trace(path, "cutlass_gemm", "matmul")) + _write_splits(root, {"train": traces[:3], "val": traces[3:4], "test": traces[4:]}, {"param_ood": traces[:1]}) + return KernelSightConnector().convert(_files(root)), values + + +def test_each_snapshot_becomes_a_record_and_a_classification_task(tmp_path): + dataset, values = _convert(tmp_path) + tasks = dataset.tasks_of(ClassificationTask) + assert len(tasks) == len(values) + assert {task.split for task in tasks} == {Split.TRAIN, Split.VALIDATION, Split.TEST} + for task in tasks: + (record,) = task.inputs + assert task.label_set == WORKLOAD_L1 + assert task.targets is not None + assert read_value(task.targets[0]) == "matmul" + assert [source.name for source in record.sources] == [group for group, _ in CHANNEL_GROUPS] + signals = [signal for source in record.sources for signal in source.signals] + assert [signal.metadata["counter"] for signal in signals] == list(COUNTER_NAMES) + for row, signal in enumerate(signals): + np.testing.assert_array_equal(signal.to_numpy(), values[str(record.metadata["path"])][row]) + axis = signals[0].time_axis + assert isinstance(axis, RegularAxis) + assert axis.period_us == 1000 + + +def test_record_metadata_keeps_the_released_split_memberships(tmp_path): + dataset, _ = _convert(tmp_path) + by_path = { + task.inputs[0].metadata["path"]: task.inputs[0].metadata for task in dataset.tasks_of(ClassificationTask) + } + assert by_path[_gemm_path(0)] == { + "path": _gemm_path(0), + "motif": "cutlass_gemm", + "problem": "cutlass_gemm/m0", + "variant": "m0", + "official_split": "train", + "kernelsight_splits": ["train", "param_ood"], + } + + +def test_segments_and_multihot_only_runs_become_interval_annotations(tmp_path): + dataset, _ = _convert(tmp_path) + by_path = {task.inputs[0].metadata["path"]: task.inputs[0] for task in dataset.tasks_of(ClassificationTask)} + annotations = by_path[_gemm_path(0)].annotations + found = {(a.key, read_value(a.value), a.span, a.occurrence_metadata.get("track")) for a in annotations} + segment = TimeInterval.micros(100_000, 200_000) + assert found == { + ("workload_l1", "matmul", segment, None), + ("workload_l2", "matmul_gemm", segment, None), + ("workload_l1", "memory_movement", TimeInterval.micros(120_000, 130_000), MULTIHOT_TRACK), + ("workload_l1", "memory_movement", TimeInterval.micros(150_000, 160_000), MULTIHOT_TRACK), + } + # A snapshot without an overlap track has only its segment. + assert len(by_path[_gemm_path(1)].annotations) == 2 + + +def test_variants_of_one_kernelbench_problem_share_a_partition(): + snapshots = [ + Snapshot( + path=f"kernels/kernelbench/_out/L1_P{problem}/dt-{dtype}_s42/input/tensor_input.npz", + motif="kernelbench", + dominant_l1="conv" if problem % 2 else "matmul", + official_split="train", + memberships=("train",), + ) + for problem in range(20) + for dtype in ("bf16", "fp16", "fp32") + ] + assert snapshots[0].problem == "kernelbench/L1_P0" + partition_of = {snapshot.path: split for split, members in assign_splits(snapshots).items() for snapshot in members} + for problem in range(20): + assert len({split for path, split in partition_of.items() if f"/L1_P{problem}/" in path}) == 1 + + +def test_a_snapshot_directly_in_the_output_directory_is_the_baseline_variant(): + snapshot = Snapshot( + path="kernels/cutlass_ws_overlap/_out/input/tensor_input.npz", + motif="cutlass_ws_overlap", + dominant_l1="matmul", + official_split="train", + memberships=("train",), + ) + assert snapshot.problem == "cutlass_ws_overlap/baseline" + + +def test_a_trace_in_two_official_partitions_raises(tmp_path): + trace = _trace(_gemm_path(0), "cutlass_gemm", "matmul") + _write_splits(tmp_path, {"train": [trace], "test": [trace]}) + with pytest.raises(TimeNetConnectorError, match="more than one official partition"): + read_snapshots(tmp_path / "splits") + + +def test_an_l2_class_outside_its_l1_class_raises(tmp_path): + for index in range(5): + l2 = VOCAB_L2.index("conv_conv1d_standard") if index == 0 else _GEMM + _write_snapshot(tmp_path, _gemm_path(index), seed=index, segments=[(0, 10, _MATMUL, l2)]) + traces = [_trace(_gemm_path(index), "cutlass_gemm", "matmul") for index in range(5)] + _write_splits(tmp_path, {"train": traces}) + with pytest.raises(TimeNetConnectorError, match="conv_conv1d_standard does not refine L1 class matmul"): + KernelSightConnector().convert(_files(tmp_path)) + + +def test_the_dataset_round_trips_through_storage(tmp_path): + source = tmp_path / "source" + source.mkdir() + dataset, _ = _convert(source) + version_dir = dataset.write(path=tmp_path / "registry") + with TimeFReader(open_local_version(version_dir)) as reader: + reader.verify() + assert_datasets_equal(dataset, reader.read()) From e3afd40b56325fe17ea59d66e83a2c9d4e2e009f Mon Sep 17 00:00:00 2001 From: mahmood Date: Fri, 20 Nov 2026 23:40:28 +0400 Subject: [PATCH 2/4] fix(connectors): group launch variants, label multi-hot L2, wrap read errors in kernelsight - Group a Variant by its Problem without launch options: the CUTLASS schedule suffix (_rH_s8_b1) and the microbenchmark thread-block size (_T256). No Problem crosses partitions now (0 of 427, was 32 CUTLASS shape groups with 725 snapshots). - Add the workload_l2 annotation (memory_movement_copy) to the 590 multi-hot-only intervals. - Raise TimeNetConnectorError for a missing or corrupt npz or split file, a missing array and segment arrays of different lengths. - Check the data shape in the lazy loader, so convert reads no values. - Test one real snapshot of the pinned release from tests/fixtures. - Card: bin width is 0.2 to 43 ms (not 68), the metadata key is release_splits, and the comment says why the split is 3/1/1. Co-Authored-By: Claude Opus 5.5 --- .../williamhtan/kernelsight/connector.py | 348 ++++++++++-------- .../williamhtan/kernelsight/dataset.yaml | 25 +- .../williamhtan/kernelsight/tests/__init__.py | 0 .../input/tensor_input.npz | Bin 0 -> 7696 bytes .../labels/labels.npz | Bin 0 -> 8572 bytes .../kernelsight/tests/test_connector.py | 135 ++++--- 6 files changed, 295 insertions(+), 213 deletions(-) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/input/tensor_input.npz create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/labels/labels.npz diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py index edcc5d074..33218a301 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py @@ -23,15 +23,15 @@ from operator import attrgetter from pathlib import Path import re +import zipfile import numpy as np -from numpy.lib.npyio import NpzFile import pyarrow as pa from timenet.dataset import Record, Signal, Source, TimeFDataset from timenet.dataset.axis import RegularAxis from timenet.errors import TimeNetConnectorError -from timenet.types import Annotation, ClassificationTask, LabelSet, Split, TimeInterval, TimeSeriesSpec, ureg +from timenet.types import Annotation, ClassificationTask, LabelSet, TimeInterval, TimeSeriesSpec, ureg from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector from timenet_connectors.splitting import DatasetSplits, StratifiedSplitter @@ -174,16 +174,20 @@ WORKLOAD_L1 = LabelSet(VOCAB_L1, annotations=[Annotation("name", value="workload_l1")]) -OFFICIAL_SPLITS: Mapping[str, Split] = {"train": Split.TRAIN, "val": Split.VALIDATION, "test": Split.TEST} -"""The released partition files. Each snapshot is in exactly one of them.""" -MEMBERSHIP_SPLITS = ("train", "val", "test", "iid", "param_ood", "composed", "length_ood") -"""Every released split file. A snapshot can be in several; the record metadata keeps them all.""" +OFFICIAL_SPLITS = ("train", "val", "test") +"""The released partition files. Each Snapshot is in exactly one of them.""" +RELEASE_SPLITS = (*OFFICIAL_SPLITS, "iid", "param_ood", "composed", "length_ood") +"""Every released split file. A Snapshot can be in several; the record metadata keeps them all.""" MULTIHOT_TRACK = "multihot" """The ``track`` of an interval that only the multi-hot overlap track of the release holds.""" -_INPUT_SUFFIX = "/input/tensor_input.npz" -_LABELS_SUFFIX = "/labels/labels.npz" +INPUT_SUFFIX = "/input/tensor_input.npz" +LABELS_SUFFIX = "/labels/labels.npz" + +# The launch options of a Variant, which leave its computation unchanged: the CUTLASS schedule suffix +# (``m1024_n1024_k8192_rH_s8_b1``) and the thread-block size of a microbenchmark (``N1M_T256``). +_LAUNCH_OPTIONS = re.compile(r"_r[A-Z]_s\d+_b\d+$|_T\d+") def _slug(name: str) -> str: @@ -203,9 +207,50 @@ def _slug(name: str) -> str: } +def _read_arrays(path: Path, *names: str) -> list[np.ndarray]: + """Read numeric arrays from an ``.npz`` file without unpickling. + + Returns: + The arrays, in the order of ``names``. + + Raises: + TimeNetConnectorError: If the file is missing or corrupt, lacks one of the arrays, or one of the + arrays holds Python objects. + """ + try: + with np.load(path, allow_pickle=False) as archive: + return [archive[name] for name in names] + except (OSError, EOFError, ValueError, KeyError, zipfile.BadZipFile) as error: + raise TimeNetConnectorError(f"{path}: cannot read {', '.join(names)}: {error}") from error + + +def _channel_loader(path: Path, row: int) -> Callable[[], pa.Array]: + """Build a lazy loader for one channel of a Snapshot. + + Returns: + A loader that reads row ``row`` of the ``data`` array as a float32 Arrow array. + """ + + def load() -> pa.Array: + # NOTE: The shape is checked here, not in ``convert``, so that ``convert`` never reads the values. + (data,) = _read_arrays(path, "data") + if data.shape != (len(COUNTER_NAMES), N_BINS): + raise TimeNetConnectorError(f"{path}: data has shape {data.shape}") + return pa.array(data[row]) + + return load + + +def _true_runs(active: np.ndarray) -> Iterator[tuple[int, int]]: + """Yield the ``[start, end)`` bin ranges of each run of consecutive true values.""" + padded = np.concatenate(([False], active.astype(bool), [False])) + edges = np.flatnonzero(np.diff(padded.astype(np.int8))) + yield from zip(edges[::2].tolist(), edges[1::2].tolist(), strict=True) + + @dataclass(frozen=True) class Snapshot: - """One snapshot of the release, as its split files describe it.""" + """One Snapshot of the release, as its split files describe it.""" path: str """The input tensor, relative to the dataset root, such as @@ -213,7 +258,7 @@ class Snapshot: motif: str dominant_l1: str official_split: str - memberships: tuple[str, ...] + release_splits: tuple[str, ...] def __post_init__(self) -> None: """Check that the path has the layout of the release and the class is in the vocabulary. @@ -221,145 +266,49 @@ def __post_init__(self) -> None: Raises: TimeNetConnectorError: If the path or the dominant class is not as expected. """ - if not (self.path.startswith(f"kernels/{self.motif}/_out/") and self.path.endswith(_INPUT_SUFFIX)): - raise TimeNetConnectorError(f"snapshot path {self.path!r} is not under kernels/{self.motif}/_out/") + if not (self.path.startswith(self._variants_dir) and self.path.endswith(INPUT_SUFFIX)): + raise TimeNetConnectorError(f"snapshot path {self.path!r} is not under {self._variants_dir}") if self.dominant_l1 not in VOCAB_L1: raise TimeNetConnectorError(f"{self.path}: dominant_l1 {self.dominant_l1!r} is not an L1 class") + @property + def _variants_dir(self) -> str: + return f"kernels/{self.motif}/_out/" + @property def variant(self) -> str: - """The variant directory below ``_out/``, or ``"baseline"`` for a snapshot directly in it.""" - start = len(f"kernels/{self.motif}/_out/") - return self.path[start : -len(_INPUT_SUFFIX)] or "baseline" + """The Variant directory below ``_out/``, or ``"baseline"`` for a Snapshot directly in it.""" + return self.path[len(self._variants_dir) : -len(INPUT_SUFFIX)] or "baseline" @property def problem(self) -> str: - """The leakage group: the KernelBench problem for its dtype and seed variants, else the variant.""" - variant = self.variant + """The leakage group: the Variant without its dtype, seed and launch options.""" if self.motif == "kernelbench": - variant = variant.split("/")[0] - return f"{self.motif}/{variant}" + return f"{self.motif}/{self.variant.split('/')[0]}" + return f"{self.motif}/{_LAUNCH_OPTIONS.sub('', self.variant)}" @property def labels_path(self) -> str: - """The labels file of the snapshot, relative to the dataset root.""" - return self.path.removesuffix(_INPUT_SUFFIX) + _LABELS_SUFFIX - - -def read_snapshots(splits_dir: Path) -> list[Snapshot]: - """Read every snapshot from the released split files, sorted by path. - - The sort makes the split assignment independent of the order of the files. - - Args: - splits_dir: The ``splits`` directory of the release. - - Returns: - One snapshot per trace of the train, val and test files. - - Raises: - TimeNetConnectorError: If a trace is in more than one official partition, or a membership - file names a trace that no official partition has. - """ - memberships: dict[str, list[str]] = defaultdict(list) - traces: dict[str, tuple[str, dict]] = {} - for name in MEMBERSHIP_SPLITS: - index = json.loads((splits_dir / f"{name}.json").read_text(encoding="utf-8")) - for trace in index["traces"]: - memberships[trace["path"]].append(name) - if name in OFFICIAL_SPLITS: - if trace["path"] in traces: - raise TimeNetConnectorError(f"{trace['path']} is in more than one official partition") - traces[trace["path"]] = (name, trace) - unknown = sorted(memberships.keys() - traces.keys()) - if unknown: - raise TimeNetConnectorError(f"{len(unknown)} traces are in no official partition, such as {unknown[0]}") - return [ - Snapshot( - path=path, - motif=trace["motif"], - dominant_l1=trace["dominant_l1"], - official_split=official, - memberships=tuple(memberships[path]), - ) - for path, (official, trace) in sorted(traces.items()) - ] - - -def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: - """Partition the snapshots 3/1/1, grouped by problem and stratified by the dominant L1 class. + """The labels file of the Snapshot, relative to the dataset root.""" + return self.path.removesuffix(INPUT_SUFFIX) + LABELS_SUFFIX - Returns: - The train, validation and test snapshots. - """ - splitter = StratifiedSplitter( - train_folds=3, - validation_folds=1, - test_folds=1, - stratify_by=attrgetter("dominant_l1"), - group_by=attrgetter("problem"), - seed=42, - ) - return splitter.split(snapshots) - - -def _load_numeric(path: Path) -> NpzFile: - """Open an ``.npz`` file without unpickling. + def create_record(self, root: Path) -> Record: + """Create the Record: one Signal per channel, grouped by channel group, with Segment annotations. - Returns: - The lazy archive. Reading an object array from it raises ``ValueError``. - - Raises: - TimeNetConnectorError: If the file is missing. - """ - if not path.is_file(): - raise TimeNetConnectorError(f"{path} is missing") - return np.load(path, allow_pickle=False) - - -def _channel_loader(path: Path, row: int) -> Callable[[], pa.Array]: - """Build a lazy loader for one channel of a snapshot. - - Returns: - A loader that reads row ``row`` of the ``data`` array as a float32 Arrow array. - """ - - def load() -> pa.Array: - with np.load(path, allow_pickle=False) as archive: - return pa.array(archive["data"][row]) - - return load - - -def _runs(active: np.ndarray) -> Iterator[tuple[int, int]]: - """Yield the ``[start, end)`` bin ranges of each run of consecutive true values.""" - padded = np.concatenate(([False], active.astype(bool), [False])) - edges = np.flatnonzero(np.diff(padded.astype(np.int8))) - yield from zip(edges[::2].tolist(), edges[1::2].tolist(), strict=True) - - -@dataclass(frozen=True) -class SnapshotFiles: - """The downloaded files of one snapshot.""" - - snapshot: Snapshot - root: Path - - def create_record(self) -> Record: - """Create the Record: one Signal per channel, grouped by channel group, with segment annotations. + Args: + root: The dataset root that the paths of the Snapshot are relative to. Returns: The Record, not yet added to a dataset. Raises: - TimeNetConnectorError: If a file does not have the shape of the release. + TimeNetConnectorError: If a file of the Snapshot is missing or corrupt, or does not have the + layout of the release. """ - input_path = self.root / self.snapshot.path - with _load_numeric(input_path) as tensor: - shape = tensor["data"].shape - edges = tensor["time_edges_ns"] - if shape != (len(COUNTER_NAMES), N_BINS) or edges.shape != (N_BINS + 1,): - raise TimeNetConnectorError(f"{self.snapshot.path}: data has shape {shape}, edges {edges.shape}") + input_path = root / self.path + (edges,) = _read_arrays(input_path, "time_edges_ns") + if edges.shape != (N_BINS + 1,): + raise TimeNetConnectorError(f"{self.path}: time_edges_ns has shape {edges.shape}") # The bins are equal-width up to one nanosecond of rounding, so the exact mean width is the period. axis = RegularAxis(period_us=Fraction(int(edges[-1] - edges[0]), N_BINS * 1000)) signals = iter( @@ -374,43 +323,52 @@ def create_record(self) -> Record: for row, name in enumerate(COUNTER_NAMES) ) sources = tuple(Source(group, signals=tuple(next(signals) for _ in names)) for group, names in CHANNEL_GROUPS) - snapshot = self.snapshot return Record( sources=sources, metadata={ - "path": snapshot.path, - "motif": snapshot.motif, - "problem": snapshot.problem, - "variant": snapshot.variant, - "official_split": snapshot.official_split, - "kernelsight_splits": list(snapshot.memberships), + "path": self.path, + "motif": self.motif, + "problem": self.problem, + "variant": self.variant, + "official_split": self.official_split, + "release_splits": list(self.release_splits), }, - annotations=self._segment_annotations(axis), + annotations=self._segment_annotations(root, axis), ) - def _segment_annotations(self, axis: RegularAxis) -> list[Annotation]: - """Annotate each labeled segment, and each multi-hot run of a class that no segment has. + def _segment_annotations(self, root: Path, axis: RegularAxis) -> list[Annotation]: + """Annotate each labeled Segment, and each multi-hot run of a class that no Segment has. Returns: The interval annotations of the Record. Raises: - TimeNetConnectorError: If a label index or a segment bound is out of range, or an L2 class - does not refine the L1 class of its segment. + TimeNetConnectorError: If the labels file is missing or corrupt, an array has the wrong shape, + a label index or a Segment bound is out of range, or an L2 class does not refine the L1 + class of its Segment. """ - path = self.snapshot.labels_path + path = self.labels_path def interval(start: int, end: int) -> TimeInterval: if not 0 <= start < end <= N_BINS: raise TimeNetConnectorError(f"{path}: segment [{start}, {end}) is outside the {N_BINS} bins") return TimeInterval.micros(int(start * axis.period_us), int(end * axis.period_us)) - with _load_numeric(self.root / path) as labels: - starts, ends = labels["segment_starts"], labels["segment_ends"] - l1, l2 = labels["segment_label_l1"], labels["segment_label_l2"] - multihot = labels["workload_l1_multihot"] - if multihot.shape != (N_BINS, len(VOCAB_L1)): - raise TimeNetConnectorError(f"{path}: workload_l1_multihot has shape {multihot.shape}") + starts, ends, l1, l2, l1_multihot, l2_multihot = _read_arrays( + root / path, + "segment_starts", + "segment_ends", + "segment_label_l1", + "segment_label_l2", + "workload_l1_multihot", + "workload_l2_multihot", + ) + if not len(starts) == len(ends) == len(l1) == len(l2): + raise TimeNetConnectorError(f"{path}: the segment arrays have different lengths") + tracks = (("workload_l1", l1, l1_multihot, VOCAB_L1), ("workload_l2", l2, l2_multihot, VOCAB_L2)) + for key, _, multihot, vocab in tracks: + if multihot.shape != (N_BINS, len(vocab)): + raise TimeNetConnectorError(f"{path}: {key}_multihot has shape {multihot.shape}") annotations = [] for start, end, l1_index, l2_index in zip(starts, ends, l1, l2, strict=True): if not (0 <= l1_index < len(VOCAB_L1) and 0 <= l2_index < len(VOCAB_L2)): @@ -423,22 +381,98 @@ def interval(start: int, end: int) -> TimeInterval: Annotation("workload_l1", value=l1_name, span=span), Annotation("workload_l2", value=l2_name, span=span), ] - # The overlap track of the release can hold a class that no segment has, such as the TMA - # loads that run during a warp-specialized GEMM. Keep each of its runs as an interval. - segment_classes = {int(index) for index in l1} - for class_index in sorted(set(np.flatnonzero(multihot.any(axis=0)).tolist()) - segment_classes): + # The overlap track of the release can hold a class that no Segment has, such as the TMA loads + # (memory_movement, memory_movement_copy) that run during a warp-specialized GEMM. Keep each of + # its runs as an interval. + for key, segment_classes, multihot, vocab in tracks: + only_multihot = set(np.flatnonzero(multihot.any(axis=0)).tolist()) - set(segment_classes.tolist()) annotations += [ Annotation( - "workload_l1", - value=VOCAB_L1[class_index], + key, + value=vocab[class_index], span=interval(start, end), occurrence_metadata={"track": MULTIHOT_TRACK}, ) - for start, end in _runs(multihot[:, class_index]) + for class_index in sorted(only_multihot) + for start, end in _true_runs(multihot[:, class_index]) ] return annotations +def _read_split_file(path: Path) -> list[tuple[str, str, str]]: + """Read the ``(path, motif, dominant_l1)`` of each Snapshot that a released split file lists. + + Returns: + One tuple per Snapshot, in the order of the file. + + Raises: + TimeNetConnectorError: If the file is missing or does not have the layout of the release. + """ + try: + index = json.loads(path.read_text(encoding="utf-8")) + return [(entry["path"], entry["motif"], entry["dominant_l1"]) for entry in index["traces"]] + except (OSError, ValueError, KeyError, TypeError) as error: + raise TimeNetConnectorError(f"{path} is not a KernelSight split file: {error!r}") from error + + +def read_snapshots(splits_dir: Path) -> list[Snapshot]: + """Read every Snapshot from the released split files, sorted by path. + + The sort makes the split assignment independent of the order of the files. + + Args: + splits_dir: The ``splits`` directory of the release. + + Returns: + One Snapshot per entry of the train, val and test files. + + Raises: + TimeNetConnectorError: If a split file is missing or malformed, a Snapshot is in more than one + official partition or in none, or its path or dominant class is not as in the release. + """ + release_splits: dict[str, list[str]] = defaultdict(list) + official: dict[str, tuple[str, str, str]] = {} + for name in RELEASE_SPLITS: + for path, motif, dominant_l1 in _read_split_file(splits_dir / f"{name}.json"): + release_splits[path].append(name) + if name in OFFICIAL_SPLITS: + if path in official: + raise TimeNetConnectorError(f"{path} is in more than one official partition") + official[path] = (name, motif, dominant_l1) + unknown = sorted(release_splits.keys() - official.keys()) + if unknown: + raise TimeNetConnectorError(f"{len(unknown)} snapshots are in no official partition, such as {unknown[0]}") + return [ + Snapshot( + path=path, + motif=motif, + dominant_l1=dominant_l1, + official_split=split, + release_splits=tuple(release_splits[path]), + ) + for path, (split, motif, dominant_l1) in sorted(official.items()) + ] + + +def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: + """Partition the Snapshots 3/1/1, grouped by Problem and stratified by the dominant L1 class. + + Returns: + The train, validation and test Snapshots. + """ + # NOTE: 3/1/1, not the 8/1/1 of docs/connectors.md. Several classes have only 6 to 8 Problems, and with + # 8/1/1 loss, normalization and pooling each get no validation or test Snapshot. + splitter = StratifiedSplitter( + train_folds=3, + validation_folds=1, + test_folds=1, + stratify_by=attrgetter("dominant_l1"), + group_by=attrgetter("problem"), + seed=42, + ) + return splitter.split(snapshots) + + # A glob's part path is its directory, so both kernel parts resolve to ``kernels/``. The ``*`` matches # across directories, which selects the input and labels files and skips the fingerprints. class KernelSightConnector( @@ -454,7 +488,7 @@ class KernelSightConnector( """Connector for the KernelSight release on the Hub (``williamhtan/kernelsight``).""" def convert(self, files: Mapping[str, Path]) -> TimeFDataset: - """Create one Record and one ClassificationTask for each snapshot. + """Create one Record and one ClassificationTask for each Snapshot. Args: files: The kernel and split directories from :meth:`download`. @@ -467,7 +501,7 @@ def convert(self, files: Mapping[str, Path]) -> TimeFDataset: splits = assign_splits(read_snapshots(files["splits"])) for split, snapshots in splits.items(): for snapshot in snapshots: - record = dataset.add_record(SnapshotFiles(snapshot, root).create_record()) + record = dataset.add_record(snapshot.create_record(root)) dataset.add_task( ClassificationTask( inputs=(record,), diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml index f70e78245..1dfe947ba 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml @@ -6,24 +6,25 @@ name: KernelSight description: >- 1,444 GPU workload snapshots profiled on one NVIDIA H100 80GB HBM3 (sm_90a). Each snapshot is 24 hardware-counter channels over 512 equal-width bins that cover the kernel-active window, so - the bin width differs per snapshot (about 0.2 ms to 68 ms). The channels form five groups: pipe + the bin width differs per snapshot (0.2 ms to 43 ms). The channels form five groups: pipe signature, memory access, discriminators, system bandwidth (Nsight Systems) and SASS modality (NVBit). The values are kept as released. The authors divided each channel by a fixed physical-scale divisor (for example 100 for a percent, 64 for a warp-stall ratio, 8 for sectors per warp load), so most values are in [0, 1], but pred_on_per_inst_ratio reaches about 32. Each labeled segment is a time-interval annotation with its L1 class (12 classes, of which `other` never occurs) and its L2 class (73 classes). In the 472 cutlass_ws_overlap snapshots, - an overlapping memory_movement interval exists only in the source's multi-hot track; it is - added as a workload_l1 annotation with occurrence metadata track=multihot. Each snapshot has - one classification task whose target is the authors' dominant_l1 field, which can differ from - the L1 class with the most labeled bins (34 snapshots). The released train/val/test split - puts dtype and seed variants of one KernelBench problem into different partitions, so this - dataset assigns its own split instead. It is grouped by problem (a KernelBench problem keeps - all its variants together; any other variant directory is its own group), stratified by - dominant_l1, with 3/1/1 folds and seed 42. CUTLASS shape sweeps of one binary can still cross - partitions; each record's metadata keeps the authors' split memberships (train, val, test, - iid, param_ood, composed, length_ood) so the released benchmark stays reproducible. softmax has only two - problems and cannot appear in all three partitions. + an overlapping memory_movement (L2 memory_movement_copy) interval exists only in the source's + multi-hot track; it is added as a workload_l1 and a workload_l2 annotation with occurrence + metadata track=multihot. Each snapshot has one classification task whose target is the + authors' dominant_l1 field, which can differ from the L1 class with the most labeled bins (34 + snapshots). The released train/val/test split puts dtype and seed variants of one KernelBench + problem into different partitions, so this dataset assigns its own split instead. It is + grouped by problem, which is the variant without its dtype, seed and launch options (the + CUTLASS schedule suffix such as _rH_s8_b1, and a microbenchmark's thread-block size such as + _T256), so no problem crosses partitions. It is stratified by dominant_l1, with 3/1/1 folds and + seed 42. softmax has only two problems and has no validation snapshot. Each record's metadata + keeps the authors' split memberships under release_splits (train, val, test, iid, param_ood, + composed, length_ood), so the released benchmark stays reproducible. license: Apache-2.0 source_url: https://huggingface.co/datasets/williamhtan/kernelsight citation: >- diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/input/tensor_input.npz b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/input/tensor_input.npz new file mode 100644 index 0000000000000000000000000000000000000000..f2eb175e6bcf5552c71f1b773753553f8cca1106 GIT binary patch literal 7696 zcmeHM_d8tO_9udbgd`$HPm$=o_k0-)-WMBO?q9%=P@5b zikCnwd4XYUF)&F4`3^6(J2tBpMM&ixwtCpvi_Lcp@E=H53lzdGCt5iurnx{k{J#$R z4t}DQ!^Qc@+(q!n#1tX!f_OQ2BGqT@;)hHQe4cjhN|y9M(lDB50qP)jvC)^CLouoe z&`$3Y!lWS#bR&{^6__SnkWf2wMKZ-Jj9tb~*K!%}V&Vzy7jWU(5PC2wL>&l!8m4pF z*74jH>CNe)d2&vtZi$+G0Zth4qb87?!QI^KVMWOT`jG+Z zcrjEtsr^H?r`_Lkqv;_JV?Z@eY5WoIkb=;6v1K1+rPu9=rPkWFdHXFHzyHBn&mk&4pm*xd@|3_FS%pK6V}q}Vh1i#>f9FF%Opaaj;Z zVkxYaxF2**ii6X{cAz-{ zU8}4rH&VD*?w!-qzZ9<1u-=%I0Ev})*75i11~A_`^hV?BCPe%{X@2vDi63$6fju2Jt?!U5)fY9G!1{a!%=(=T%Hncr-3R8DzNR_6^(T z)bX`S=QcqwzrK7nKEy?Njc6%h3?Q~^4}H|11dOxS9~ z!hLG>loI-2GiM_xiC%E1hR>m^5E73lJvbbg-Z`DLo&fnC`K$5CHgEvHjxNJDD7pO! zsZgx;Nn4FWK{wJrsuhgj(L{^(2ZFX$G!t)82XN!>joZpUO8nmoAsbZi5wV#M(Tn)$KF;s zWeA48#!AGf0FSJK4%nPIc1m2s&w*(L5B@hRfoy3V;lax`@?;!Qx@k4 z-%&l-@=;@s-kHQF5xLDJZrJj6&^qu`g)>+<++WajI>IQ|`;2s9wR3>MBDJSV|grzvgUO35Hcy^NPpi%3l8!iq`@Z>1s89T)`PxKWrx))9;4SboEflBIU8Dbk%#~r>uEG9;M9J9P8kLq$MniK zwT)6n_KcYoFp4k|T-SbZ^NHH54U5OEYQ@B`?I|R3U3?=7JSVxfTcBv#tx&|#sF_fc zc3|ar3*-Cgolc~EdywM8)_m7cDH2!*`Gr@c~OkLvhs(9(792-NIX4;9@r z11VnOp$~~IasfMe`o;$+UOF%w(HQoHz9n#OW$q&k8$NW8;B}@eL0%b8w?|u0PV<)H zVM?sn=WVGX#|~cTpj*YD*71m@a`U$?Y}~_mLU|P(1-J%96@#601UCOsq7~k3KNsHW zm!}VNxXeISWp~0F`0t<4#=L!@`GOb_=gOe|PGAFCAiVJ4VrOOFN&QY1e?*0R+-Hu2 z$}uOqCx<(!dJ6r3owTysQz&wjt*amrsw zlff1gW+SMTrMhj%r&`>??MuH6PmyKA9^;dR2g$g^>ZH10Xcvt|N}UicY$8LkjLwJZ zD?pE$YgEufIpHYWBjD?+Q@m7fxJmj~HU(*xTYt=M|LhyxR|@u3p64>q!XIJ&GUpf) zB{Zl)Mh#VNHhd@I!{9_v!agilA3nzPSd2pQ2lf0-^z*=?simI z=i1kZ=jB}LI|LZ2g*hphs8CM<4oG$i{=S0Hr?wgZ6%cF2|87;7xqW8>$hwm?L@lY5RVcPL5n)20D)LX)L0&usUyv9Y+a`%rVBfo`-ce8|%GgVput_P3J0sUlbRDu-vu{UDY_&3LaB-W%bHM&9^PXbMvJrz z0c*Gt9Pg9gikb*X8?Jj7_+)9%>Sz`1$z*M&9Z7jnnDml72gL+dw~AlYmLi$S zXuZbPE&94cyRW)_UWntp#H;A!ol!eK0(uz3?5C!^YIX_oZg-BH%!hc5P?Hw9J0sjC z{Dk#yT;|2#0W>HOsi8%5Dc9cZ_C$$x81 z*oMH%PY78tip40=0qD%!9eOSM$ybe`c-x=9(}@*9DFTZ_`X*U#z-2?{!3iyTMu4#E zK)9=-H(O7VXxOeuL0R|1+w|2LEo2XvtA$tZP%RhC|5z(Js(eJeB|qH?=r&w2`w-z_ zv^BS#DdiXqiaZv63^3lgJ5rfA>)7F*pyMAF+pg`?wZl%g1CdS|B`r)Eiqx<&-In$s zp1SMM=9;&)VmbP100o-AKkZx`yT)5JQtWyA2J0%gh`P<~OXSt)>jW0D>3k-Yba5y3 z3%wlXeD--n+ZowzN3|-H1GS7V-%dU^D#?i=V=s7VN=jRN?b9oY_TPx=NV?kWe6&K0gmADQ|wOmPZFm&I~dt1mEM zSzk2W#@LjNiBJ;Pamht+7l}YjAHy349_Xw2vF4 z#L}L6%IH&Adw0aMp^1_wktyqdV7=R(YohWV>D79J^t154)UL);!^sITVAcSLE({OifZwnG6ek0$}{X`_; zq~7*?M}8%zIBdxye{M8!V7maqaZe7>8hQh7a@U80fX$q+8;Iuy=PepDvV$#B=oaf%7an?ILWWe~wq+0%O_+C42Z!rnWeKUu|kW@sjYM|-0b-Pw}i z*jAb;Ln{unz3sT!5d)@drG4Ebw?Pk9zFW7+yjRLmO9!wq->%wKs-XWo4VEyII40O% z1qkT$$nxIG33IIPS-1x6j(><4H_pvPBWF9&aS zGYBiAX#tzhy!8?1UyWQIAOxRI5GqR^lyj$ocb@saK=Z^LAs2i3TM8d1KUj$T4K6i&Wuz1wQ7uHOhf1oDwpOhh){+h1P5x1%UmY&#P66z}U%%P0uSz~BL%a5VP= z1QV2dmzYRfZEuT2G^_K$hjcylD4l|_Zi{`Lke5c8Mv=2%z}f5kC5_F>K8cWg(OUUh z)fU$RFB7!n(0PvW8G4nzX$JqziJ1^#4*r^?^m*Q)RgxPg_?ZXaS1y`puh3nh8iM() zMAaNfZ7O>3=`^mfO~w46>f02q@#r0wHO@70pw3&(K5HhlR{ARrRo{%wlSDGz-Sqwy zs%)2R2}0#fg)@3C&v9Dy^H?u(-5pU@vuyyTsxzlI_L<>*;!4)$l{ZuI70Kvf*=hV1 z=QTz%KJs~#JJc&F5yCWLD6P_RU-3}poTJtkK=<6^MZLZvh__ab%l?%$S9{AK=RgoS zh76E8h>g2~S+kwUSq+_a;OP&1)Si-vume4VC6JbKVCVpr=6tl>N~l8?P&QIq-<~#5 z1S1QuETRuqZ@{$emWqX!eD_Olys7Nq>BI$KhTB4Q&((I?U}8rg0x$0%p6@!49=fk5(h zMo-DdMfY~j__@}COL#GEoi3T|I*r=L-bl{u&*ksXx!D~})r{kmoxbC!mYHBot8yKL zul;5pUmF);h!K(^%i9**IS6#XIZ;oAl{{)!&7uV_Det$9JGIQzZ6@uf`&aFq*4t+2Zie-V0)9%N%P^{a3FFOJ z*&9?@zJMGofJCDaaHdgV+U3%Rtc~|4zcG>N6tQV_V8N{iG0Htb>=t7TU`JDTZ@@a~ zR`N5`O6rK`>&K1z#LWcJd~f=>N&0MZJ{6>bn<)TvI-mi%;^2II*^AX>(MHWGcsc+j znH4=Do5$%4d}-UJf5tD1NK)S4@$1&V;@6*vZr%8=On3C}BnLHwaG}q(^T(3U4sR^ zs5-x!;V0Q2__}X=66DDVMP^f;8J3G;;Iq9L#}Dv5|F5X}6UjOd8tr|KK150eoS;+%1O$`4&O$oO#bAE7 z2nJX{TqX&?ciP%sm33H?!rf+19!ODJrb&ao38$_q-H!Xy)JsV+r^kGdC%&I5BsdB}R8{k}8L5ptxG`RyDBKf;!9@ttm$ujX5+%w={}eV-)?0 z?KXWe(7Odh_^rm5W{`aQ2$N@AP0f=ehC=#D{B6dDJk`BNp5?+Hgx{8gHpAyre4r^S zAUfytI_1&BJ$W|#y>=VMGlo_KYV>Q`lb^j{+;nkaKR{1L#sLg2X9oiE+tI7S%U8wd zr^0*VUIppY1<^{!9O`1!@%`zYr#OPRdD!N zaZi!;a$2Y*+DxL;yk>3k#vm(<=c7fVUX*xEZITmQ-G5@5;IgA0uGh`dR zVcaUojW8P&OJG>mHztSlFEz?~*Z*|zd|>xA`n0%?Xv}q#Hdc0hvwq4vRkv-%MI$M3B+WUhnkvuJxV~=8%&jo>9k9w#~h1 z4s&fP{=^a=zj4ylAV86ueJT~Yd00qfH8)=6f4Bel8T3Hb0!~IQx)u3m(aibuESmp2 zyZ`su{pn|W=L5%wC!h&EekJa!MqPQ|WznKB$ zznOtG(AmP%&c)f(&e_t+=XWi9o8VWTWBhBtN!g%SqrTPgL#_HZuY67w|4>^|)8q@&4>YUUnKUyN;K=$=?$ES7H8lU-F-*%NqY> z>He~efBQG&VhqNeym;4tRs8=6zbp@5R(&rkz<VKjC+r(;$_3bX)dw<2VDWn4FCWD literal 0 HcmV?d00001 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/labels/labels.npz b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/labels/labels.npz new file mode 100644 index 0000000000000000000000000000000000000000..7fd3100ee75640675e56c96c044c9336a9ffee21 GIT binary patch literal 8572 zcmc&(2UJs8w@yHcq4!=C5D=tG7Z9W<9Vt=-0z_H}CG-*LMY<>;N=KwOrAhCIQU?{0 zB1I6CDvAgQFIS0@=*;|Yy?O85tZ=w%W$*jl@0@+k{`S^U$31ci1OgGFzQ;gL3wFW# z2NxNL8sz2d<_LF&+E~Da1rRPiU=TSd0Qe>9GW|Y)?k>^2V6@C+$6)FO2uYRHW}nL#iM)xTX6! zmekQfP&|6OBlxc@fE%25I3S(B#72t3hJ@KV!4U2i_BQ|T&|(8lbD?RYJOmBt4s~-! zqTxjxFsAko%w_!(b79|kqXw0h5tDyO33SbDI}3$wMpUTfJXBlK6f$>1v9pQNHkl!=?X()A3vs22!M7s9(yJR`{JXgu%R zdC_#!{k%z_jNwxOEt`i*ww}dr;R9z2hx4;?$nN5(eG!80VV7QW3INQdbnb+|dc4`WO{HZ(fs^rd_X##yb3`WQaH9!cq!y zcJ7W5jPRyO$m1nu`mcN`Dq!>8Gs{*vVRfAsKF62DS6*jQj>FHEk&53P_<7|kZhPFy z=dAZ)LaJ6hQ4W;cl+d>1NB#82iylW?g`9)rx~A9__I$j-rly(o@V>g2N`|}Y z>>!D%qlvqQ;By1;*p0MB2B#*I8tZWyWnK5L)6T<$=HF2E#tgY>Rj>F&mP*?OpRnM}!*v1;M{=jveYtuzo9qOLs<#^8 z6kJj>D;lJ+c%h%>J3RKlM%xcWnC!~h@`BLpQlZ#&f+8I)pa>s}@2d~JLj>S_UvRny z>+ZLEtNBHT{D0aZ4(+W1VK)frD}n+&p9f%v#9>sxR`Cthx8Q{R+*AK=A@M|sUPKc` zfr8|2?*y}e+1SF67Kk6=8G+jLfNPfE`G==0oC%2GuAqO(vZzq(8N<;J|NSmY>GF#w78aKmrz#wB-ZH+OL@brJKK?m}6ggUf3yIWUDm{gfqYnpFt*xSeS_{w$J@E7*H`Q}%T z?ugx+ecCi4q%6mT>+U?GS7a(VDN0sPI@$SvDAappPib7&G$^V*Jv(6uxqG^bIfMUN zsI7#4miC&dn09fyun*f>_gxkKy)SrFVb_$MV+>QBw^`;cOP4zhIP*(CkKsma>MnR7 z9}zOvY^q9K2XA=<9hnqmj20&Sb74h-`EAGJ>9yXWmmf6(d^Bf;O!V=D2`Fj2CkiY;_Rbe%>Id*x=H(&m~{ZbeEFnguz zXdD%;Qib#G+2UFs!i!HiT#erAnw_ACDlFu#7~=tF$fR~|TqgXE^*`EjUSDEZis3h>NtEI0Imu7mxro6Uw%;D}YFTn*hC9|}KeVs)@}+nl&V^5Q=|ZJt+XN)i z9Xcw$B>X3gz7GFM_O)Fe=G2@3wx``zT1Av9Y8ox43*Sk$U&7^L=C)4D z5Wmu`(T$tpyPzXYRCSj`*Q}TD^~wHrNBsNr@EY=EKYkatXL?H+h#2QcBTL`lGb=Qn zEi@2EmxOd8h-q2-1#Pbe>ZYS!dEQwD(}wy*8_HQLXO`n@UPUdmk<{I;MRmu@z z^V0;4(ay87-B+2H{mD$NBg=(0D7q5mx-|O+2Gy4Eo}4tWjB0gka5ReP_M4mz%l8%2 z*pFzxAUB;HcA|k}mn4V!)u84oLH~o5j$`Kj<;NNbkQcXO7eBc`<=soElqXieJiOf3 zSF6IOJct-%7vykQlvHLaIenPpmW(}lDK8cfQl^L+;y4ZnyDE|HhqWjz~&I#wSqQQ*`gXczvAGbaX#X1m5|xETkV(3 z=tGZsbi1f~@%Jaognp3DtLQsU!B(5mPIZj8R%(%w@onDh<99_{RUAaC=U;cw_n9t0 zlfsc^t21kvittGPh%ZuH^q1GEN2(I1;XQ7*4F3p?=`!$sx_*jvy-X?jC0Ty*^9Vhd z#hMb5Xp-8o(U3LB>){m=Z{L^P{WL9^B0E z41s}6Ou4GBoZN_5`^J^1!GY(Lr-GBR+=58<`YSqZY50T-`oMl%X|7p!_+mwk<5*%@ zFU>SFXm|V0z9@FGGpS~wG1)D@44078hqPS`+Zh^`qt?7iHj}@gv}NJTIVmU=SrH># ztNVs?k58?IV%xT2RGNXPL13yPBC1yIGzm#$X{LtXiEArOu{CpU!g_B>>r7~;G~#w7 zB|n0%X`Jjz-R1VqucIDuZ)#x>N#x1;g#W(JL1?Usmh4EQYIq3#v!1WfGl_j$)yqug zH{spnRT*~9ABebRJfQoaEZ0WEPKdSd3;Eg_4Pv88;&UXfywrxQk%?jLYYC0~E9)$w zhC`72Bw^Z6R=(+{UF+Uw&#`>PW2`;rFA`fb)Fp2|Tp^}NcJrd`bnI~@GyQ$Rlbg3K z^HO=`hKwU@`A@mr?bZ_`s_GK%;amHo>5N>VdCYp6*21aTppD8G({@NNxW`0@@>z5G zE`x$SJ-IjvicWG%Ek3JpcoWN~*xGV6!IV#EG8bwjcFM3(OSBuRIwxNhSo zwk`{)obYn_xJOL|-jdrQrG1YTCb~y-K3o)UqU&j83L5j)k!NFZTA3DGJ-4QAc%hYOBIXtJi?W~;-CJFzoW z7sMi2-^G^ezucuQ^2d6Tp%AMunG#)%%d^*i(C2EYPKS?Mdp{JR<#?r?ee;=pYH4~z z@O)akbRKmtq&5WVx>zf{bJ9V6K(V4sqFiN{*4&`KeM-t}-5)cZqP8k3KxYY=J&A+Dz|FTSFpr^MBB={?(`j9)X91RbU8i!Q0g_8%|-QLdG{Rd9^?*P3Ha4ruV zBW+Wl4w){QqaHY;_5w~tOpqV^Oke*7$>kvSQ=?A8m3dQb)Zwk{Cm-(@q_}R#>hC=0 z@12mXJi18j!7WXyRkH9X%J1WHMU zVO!ka_}mOUHfbFPlV5c*ZN}fffXJ^eknkP(-R}cvxwnF}RRy4t zc!!nHd8JNvwJ1F8>Cop6_o`RN`Wg1#!iCcWhu_>8pSfQ=V!#~k-TX$F$n}rnegoG_ zEJd1M_P9Hn1nxFZ(&RlDoq$dbwAl?lZR5C_ck|#;KZi8}dsFZ+)Rom7Dixxt|>Xmo)GsFePv=)TorsKEj%SOtYFdP;3HnBS=Q z9}$OE@&8ob_;09q6?}YrEHvB}G`wguj3{c*RUj8Pm<`MthD18M9a61mO#fbGp>+S3 z+4_$l{X*0KDx|-vytp_c?cMF25$MJ9@ALDU%1c9F@;UUB0pb_x?(Sx9<>3yqxB`dT z{#11b3;1t~QvZqg4U>k@9uyV@@OiLft0Hr)0gdKRq2tH=Kn_IZn}ev&z4g@jnGkt+ z{jKCAqt~)zbCt@4C#%i`)Ve4r21Vi>bDav{=T5m&^2*glMm;%vDhSk5bElcEYt!?6 zNaGD9@r|!tqSgbxT4Bx>P1i@McQfDGhdpAgJ?K+e&!ec^F09o5&m}888 zPf-G1E(zZmBdeBZhVOL`?|bfj9*H8Fu3$3Yc-KSPlN-b`_(m2=Si;hJTi(KFU!!qT-i8) zkYsM!3fA^!krfu$`84S?+LVcpys1HGTc#01+2G;iFg-x;W;;@TUwFN2BFIk3PdN19 zd(j4?cpL}O?G`b2?^DLG;}+^)jBkrJ$g}OtFPJ#IOpE(GKkL^qVvl%voACr6PJ`>F z3eFP+aY~W4-aCnkB5V)%rUX>_OFe~XqutMBF)s`qks0mdZrjccQ4_^^?z!!iLTbas zmB)3z%Ff5mVRFHrx6Wv}VYuX;@r*daQ19rI(QZOfhYvSY`zM|F!Y^FWsZdd^dRVIe zpdiOBucZ8mmt;6kX_a!54BKj-X4~Qsef<|>k(9#^48LyTojGBaWV5}zQUhgN4gevnNP}?Pkc2Pv&$f&@89AwJ`B(u6@esE_Eq5-IM5_b2s4{ zfz`Sjs+u>TCoZd}(RUlF)z7s69`&c;2$y-C%nL0U`jq@{^lv)h8uH>$0rt zMIs9;%I9cj^q-}9XNAzV>dr5f+~OH5w!2E;m$XyxhAj9b<$bq4W7a+0+8d>0tcu(* z2$As!(6Q$sgi@rCSWbVpntJ+k_{tPPaQCW&t_qije3s_$J9nrM;vdV0p!xAQrYD$W z%db_sZoMIO*;?Jj!#o&2ajX!aBLp~oFyj@oQtEtg&f*`8xiM#fvS28cb z6pHa(n8P}cO}-g$C$Dye1GW8?AlH@la2O! zL-ETdE}`*HsztbzY*EB)QR(jeJ z8?wmw@)*GegHv;uv*p**b3WI;zOQ5(4bAG@ykRBtf<%eL2A3nnamt{GgnhKaW*hIw z=y}|Hb=wzVOFfWU-AheDJe8)*k}#E?1iI)AL+Mh=fme1oBk_A+Og_gc3QEy@(g8=7 z3)I~n3b#OY2T*(9w?Zp=KejxeasOa1%+;y5(1EJ5%C-kW#I4cAl&$PmWxsj#q zb`K*GgbMG8L_&_!1m9@NiNWuFf5&rbdrRbzTHEI}dC6_b-M#e_FOfqv6VhC{`tj9% z){+L1r9k_B>hhc0neScsF{ znJ%6n(l!<(u79h=?7j)(#QESU{$Y7c9>?;KNuf6o7fE^jQMo082S`!CIHzzi z+h{<44Cs}S9UuXQfN$sy$pK*A|IU~c!#EXnht7AtcfGJ}fkqUNe?_3alo+-Ws7Ir1 zf3~Ht?SbYIaQ*`J2SwIzUIM`FhwRaj2UkAgiGMX7~M109Xrkn4+J4xch_c z0cg?yO&6d6b0Cbqxd;G{9rA$PvcdKO^fG`x1<}sP*IGEXIUtJxz5DC3 z%R?=spUduBu7hn1h*m&^2B3Jyq+SKJgnt?fV<5T0JIE;Wp1~_mq13e8j&DckijyfJHfC13- Npkt{2l$n6L{{pPs>V*IR literal 0 HcmV?d00001 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py index 3a9c2d885..af3db720f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py @@ -15,8 +15,11 @@ from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( CHANNEL_GROUPS, COUNTER_NAMES, + INPUT_SUFFIX, + LABELS_SUFFIX, MULTIHOT_TRACK, N_BINS, + RELEASE_SPLITS, VOCAB_L1, VOCAB_L2, WORKLOAD_L1, @@ -27,23 +30,29 @@ ) +FIXTURES = Path(__file__).parent / "fixtures" +"""One real Snapshot of the pinned release: a warp-specialized GEMM with a multi-hot-only TMA track.""" _BIN_NS = 1_000_000 """One millisecond per bin, so bin ``i`` starts at ``i * 1000`` microseconds.""" _MATMUL, _MEMORY = VOCAB_L1.index("matmul"), VOCAB_L1.index("memory_movement") -_GEMM = VOCAB_L2.index("matmul_gemm") +_GEMM, _COPY = VOCAB_L2.index("matmul_gemm"), VOCAB_L2.index("memory_movement_copy") def _trace(path: str, motif: str, dominant: str) -> dict: return {"path": path, "motif": motif, "dominant_l1": dominant} +def _snapshot(path: str, motif: str, dominant: str = "matmul") -> Snapshot: + return Snapshot(path=path, motif=motif, dominant_l1=dominant, official_split="train", release_splits=("train",)) + + def _write_snapshot( root: Path, path: str, *, seed: int, segments: list[tuple[int, int, int, int]], - multihot: np.ndarray | None = None, + overlap: bool = False, ) -> np.ndarray: # Synthetic files in the shape of the release, including its pickled object arrays, which the # connector must never read. @@ -56,9 +65,14 @@ def _write_snapshot( np.array(column, dtype=dtype) for column, dtype in zip(zip(*segments, strict=True), (np.int64, np.int64, np.int32, np.int32), strict=True) ) - if multihot is None: - multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) - labels_path = root / path.replace("/input/tensor_input.npz", "/labels/labels.npz") + l1_multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) + l2_multihot = np.zeros((N_BINS, len(VOCAB_L2)), dtype=np.uint8) + if overlap: + # A GEMM in bins [100, 200) with TMA loads in [120, 130) and [150, 160) that no Segment holds. + l1_multihot[100:200, _MATMUL] = l2_multihot[100:200, _GEMM] = 1 + for start, end in ((120, 130), (150, 160)): + l1_multihot[start:end, _MEMORY] = l2_multihot[start:end, _COPY] = 1 + labels_path = root / (path.removesuffix(INPUT_SUFFIX) + LABELS_SUFFIX) labels_path.parent.mkdir(parents=True) np.savez( labels_path, @@ -66,7 +80,8 @@ def _write_snapshot( segment_ends=ends, segment_label_l1=l1, segment_label_l2=l2, - workload_l1_multihot=multihot, + workload_l1_multihot=l1_multihot, + workload_l2_multihot=l2_multihot, vocab_l1=np.array(VOCAB_L1, dtype=object), vocab_l2=np.array(VOCAB_L2, dtype=object), ) @@ -76,8 +91,7 @@ def _write_snapshot( def _write_splits(root: Path, official: dict[str, list[dict]], extra: dict[str, list[dict]] | None = None) -> None: splits = root / "splits" splits.mkdir() - files = {"train": [], "val": [], "test": [], "iid": [], "param_ood": [], "composed": [], "length_ood": []} - files |= official | (extra or {}) + files = {name: [] for name in RELEASE_SPLITS} | official | (extra or {}) for name, traces in files.items(): (splits / f"{name}.json").write_text(json.dumps({"split": name, "n": len(traces), "traces": traces})) @@ -87,31 +101,27 @@ def _files(root: Path) -> dict[str, Path]: def _gemm_path(index: int) -> str: - return f"kernels/cutlass_gemm/_out/m{index}/input/tensor_input.npz" - - -def _overlap_multihot() -> np.ndarray: - # A GEMM in bins [100, 200) with TMA loads in [120, 130) and [150, 160) that no segment holds. - multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) - multihot[100:200, _MATMUL] = 1 - multihot[120:130, _MEMORY] = 1 - multihot[150:160, _MEMORY] = 1 - return multihot + return f"kernels/cutlass_gemm/_out/m{index}{INPUT_SUFFIX}" def _convert(root: Path) -> tuple[TimeFDataset, dict[str, np.ndarray]]: - # Five problems, the fewest that the 3/1/1 split accepts. + # Five Problems, the fewest that the 3/1/1 split accepts. values = {} traces = [] for index in range(5): path = _gemm_path(index) - multihot = _overlap_multihot() if index == 0 else None - values[path] = _write_snapshot(root, path, seed=index, segments=[(100, 200, _MATMUL, _GEMM)], multihot=multihot) + values[path] = _write_snapshot( + root, path, seed=index, segments=[(100, 200, _MATMUL, _GEMM)], overlap=index == 0 + ) traces.append(_trace(path, "cutlass_gemm", "matmul")) _write_splits(root, {"train": traces[:3], "val": traces[3:4], "test": traces[4:]}, {"param_ood": traces[:1]}) return KernelSightConnector().convert(_files(root)), values +def _found(annotations) -> set: + return {(a.key, read_value(a.value), a.span, a.occurrence_metadata.get("track")) for a in annotations} + + def test_each_snapshot_becomes_a_record_and_a_classification_task(tmp_path): dataset, values = _convert(tmp_path) tasks = dataset.tasks_of(ClassificationTask) @@ -143,34 +153,51 @@ def test_record_metadata_keeps_the_released_split_memberships(tmp_path): "problem": "cutlass_gemm/m0", "variant": "m0", "official_split": "train", - "kernelsight_splits": ["train", "param_ood"], + "release_splits": ["train", "param_ood"], } def test_segments_and_multihot_only_runs_become_interval_annotations(tmp_path): dataset, _ = _convert(tmp_path) by_path = {task.inputs[0].metadata["path"]: task.inputs[0] for task in dataset.tasks_of(ClassificationTask)} - annotations = by_path[_gemm_path(0)].annotations - found = {(a.key, read_value(a.value), a.span, a.occurrence_metadata.get("track")) for a in annotations} segment = TimeInterval.micros(100_000, 200_000) - assert found == { + loads = (TimeInterval.micros(120_000, 130_000), TimeInterval.micros(150_000, 160_000)) + assert _found(by_path[_gemm_path(0)].annotations) == { ("workload_l1", "matmul", segment, None), ("workload_l2", "matmul_gemm", segment, None), - ("workload_l1", "memory_movement", TimeInterval.micros(120_000, 130_000), MULTIHOT_TRACK), - ("workload_l1", "memory_movement", TimeInterval.micros(150_000, 160_000), MULTIHOT_TRACK), + *(("workload_l1", "memory_movement", span, MULTIHOT_TRACK) for span in loads), + *(("workload_l2", "memory_movement_copy", span, MULTIHOT_TRACK) for span in loads), } - # A snapshot without an overlap track has only its segment. + # A Snapshot without an overlap track has only its Segment. assert len(by_path[_gemm_path(1)].annotations) == 2 +def test_a_real_snapshot_of_the_release_becomes_a_record(): + path = f"kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0{INPUT_SUFFIX}" + record = _snapshot(path, "cutlass_ws_overlap").create_record(FIXTURES) + signals = [signal for source in record.sources for signal in source.signals] + assert len(signals) == len(COUNTER_NAMES) + with np.load(FIXTURES / path, allow_pickle=False) as archive: + np.testing.assert_array_equal(signals[5].to_numpy(), archive["data"][5]) + axis = signals[0].time_axis + assert isinstance(axis, RegularAxis) + assert axis.period_us == pytest.approx(203.73275) + segment = TimeInterval.micros(49_914, 54_396) + loads = (TimeInterval.micros(49_914, 53_785), TimeInterval.micros(53_989, 54_396)) + assert _found(record.annotations) == { + ("workload_l1", "matmul", segment, None), + ("workload_l2", "matmul_gemm", segment, None), + *(("workload_l1", "memory_movement", span, MULTIHOT_TRACK) for span in loads), + *(("workload_l2", "memory_movement_copy", span, MULTIHOT_TRACK) for span in loads), + } + + def test_variants_of_one_kernelbench_problem_share_a_partition(): snapshots = [ - Snapshot( - path=f"kernels/kernelbench/_out/L1_P{problem}/dt-{dtype}_s42/input/tensor_input.npz", - motif="kernelbench", - dominant_l1="conv" if problem % 2 else "matmul", - official_split="train", - memberships=("train",), + _snapshot( + f"kernels/kernelbench/_out/L1_P{problem}/dt-{dtype}_s42{INPUT_SUFFIX}", + "kernelbench", + "conv" if problem % 2 else "matmul", ) for problem in range(20) for dtype in ("bf16", "fp16", "fp32") @@ -181,24 +208,44 @@ def test_variants_of_one_kernelbench_problem_share_a_partition(): assert len({split for path, split in partition_of.items() if f"/L1_P{problem}/" in path}) == 1 -def test_a_snapshot_directly_in_the_output_directory_is_the_baseline_variant(): - snapshot = Snapshot( - path="kernels/cutlass_ws_overlap/_out/input/tensor_input.npz", - motif="cutlass_ws_overlap", - dominant_l1="matmul", - official_split="train", - memberships=("train",), - ) - assert snapshot.problem == "cutlass_ws_overlap/baseline" +@pytest.mark.parametrize( + ("motif", "variant", "problem"), + [ + ("cutlass_gemm", "m1024_n1024_k8192_rH_s8_b1", "cutlass_gemm/m1024_n1024_k8192"), + ("cutlass_ws_overlap", "m512_n512_k8192_rM_s1_b0", "cutlass_ws_overlap/m512_n512_k8192"), + ("cutlass_ws_overlap", "m1024_k2048", "cutlass_ws_overlap/m1024_k2048"), + ("vector_add", "N16M_T1024", "vector_add/N16M"), + ("reduction", "N16M_T256_I4", "reduction/N16M_I4"), + ("cutlass_fmha", "b2h16_q1024_d64_causal", "cutlass_fmha/b2h16_q1024_d64_causal"), + ("cutlass_ws_overlap", "", "cutlass_ws_overlap/baseline"), + ], +) +def test_the_problem_of_a_variant_drops_its_launch_options(motif, variant, problem): + directory = f"{variant}/" if variant else "" + assert _snapshot(f"kernels/{motif}/_out/{directory}{INPUT_SUFFIX.lstrip('/')}", motif).problem == problem -def test_a_trace_in_two_official_partitions_raises(tmp_path): +def test_a_snapshot_in_two_official_partitions_raises(tmp_path): trace = _trace(_gemm_path(0), "cutlass_gemm", "matmul") _write_splits(tmp_path, {"train": [trace], "test": [trace]}) with pytest.raises(TimeNetConnectorError, match="more than one official partition"): read_snapshots(tmp_path / "splits") +def test_a_malformed_split_file_raises(tmp_path): + _write_splits(tmp_path, {"train": [{"path": _gemm_path(0)}]}) + with pytest.raises(TimeNetConnectorError, match=r"train\.json is not a KernelSight split file"): + read_snapshots(tmp_path / "splits") + + +def test_a_corrupt_labels_file_raises(tmp_path): + _write_snapshot(tmp_path, _gemm_path(0), seed=0, segments=[(0, 10, _MATMUL, _GEMM)]) + labels = tmp_path / _snapshot(_gemm_path(0), "cutlass_gemm").labels_path + labels.write_bytes(b"not a zip archive") + with pytest.raises(TimeNetConnectorError, match="cannot read segment_starts"): + _snapshot(_gemm_path(0), "cutlass_gemm").create_record(tmp_path) + + def test_an_l2_class_outside_its_l1_class_raises(tmp_path): for index in range(5): l2 = VOCAB_L2.index("conv_conv1d_standard") if index == 0 else _GEMM From 9957d9e7fa5ad0f1218f057d0aec9d30f01dc582 Mon Sep 17 00:00:00 2001 From: mahmood Date: Sat, 21 Nov 2026 00:18:32 +0400 Subject: [PATCH 3/4] fix(connectors): close kernelsight split leaks and catch zlib errors --- .../williamhtan/kernelsight/connector.py | 20 +++++++---- .../williamhtan/kernelsight/dataset.yaml | 8 +++-- .../kernelsight/tests/test_connector.py | 35 +++++++++++++++---- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py index 33218a301..443b14770 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py @@ -24,6 +24,7 @@ from pathlib import Path import re import zipfile +import zlib import numpy as np import pyarrow as pa @@ -185,9 +186,14 @@ INPUT_SUFFIX = "/input/tensor_input.npz" LABELS_SUFFIX = "/labels/labels.npz" -# The launch options of a Variant, which leave its computation unchanged: the CUTLASS schedule suffix -# (``m1024_n1024_k8192_rH_s8_b1``) and the thread-block size of a microbenchmark (``N1M_T256``). -_LAUNCH_OPTIONS = re.compile(r"_r[A-Z]_s\d+_b\d+$|_T\d+") +# The launch options of a Variant: the CUTLASS raster, swizzle and ``b`` options, alone or together +# (``m1024_n1024_k8192_rH_s8_b1``, ``m4096_rM``, ``m2048_sw4``, ``m2048_rN_b1``), and the thread-block size +# of a microbenchmark (``N1M_T256``). +_LAUNCH_OPTIONS = re.compile(r"_(?:r[A-Z]|s\d+|sw\d+|b\d+)(?=_|$)|_T\d+") + +# NOTE: cutlass_ws_overlap runs the CUTLASS ex48 GEMM of cutlass_gemm with device timer markers added. One +# shape of the two motifs gives near-identical traces, so it is one Problem. +_SAME_KERNEL = {"cutlass_ws_overlap": "cutlass_gemm"} def _slug(name: str) -> str: @@ -220,7 +226,7 @@ def _read_arrays(path: Path, *names: str) -> list[np.ndarray]: try: with np.load(path, allow_pickle=False) as archive: return [archive[name] for name in names] - except (OSError, EOFError, ValueError, KeyError, zipfile.BadZipFile) as error: + except (OSError, EOFError, ValueError, KeyError, zipfile.BadZipFile, zlib.error) as error: raise TimeNetConnectorError(f"{path}: cannot read {', '.join(names)}: {error}") from error @@ -282,10 +288,10 @@ def variant(self) -> str: @property def problem(self) -> str: - """The leakage group: the Variant without its dtype, seed and launch options.""" + """The leakage group: the motif of its kernel, and the Variant without dtype, seed and launch options.""" if self.motif == "kernelbench": return f"{self.motif}/{self.variant.split('/')[0]}" - return f"{self.motif}/{_LAUNCH_OPTIONS.sub('', self.variant)}" + return f"{_SAME_KERNEL.get(self.motif, self.motif)}/{_LAUNCH_OPTIONS.sub('', self.variant)}" @property def labels_path(self) -> str: @@ -461,7 +467,7 @@ def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: The train, validation and test Snapshots. """ # NOTE: 3/1/1, not the 8/1/1 of docs/connectors.md. Several classes have only 6 to 8 Problems, and with - # 8/1/1 loss, normalization and pooling each get no validation or test Snapshot. + # 8/1/1 loss gets no validation or test Snapshot. splitter = StratifiedSplitter( train_folds=3, validation_folds=1, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml index 1dfe947ba..d0cf19363 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml @@ -20,9 +20,11 @@ description: >- snapshots). The released train/val/test split puts dtype and seed variants of one KernelBench problem into different partitions, so this dataset assigns its own split instead. It is grouped by problem, which is the variant without its dtype, seed and launch options (the - CUTLASS schedule suffix such as _rH_s8_b1, and a microbenchmark's thread-block size such as - _T256), so no problem crosses partitions. It is stratified by dominant_l1, with 3/1/1 folds and - seed 42. softmax has only two problems and has no validation snapshot. Each record's metadata + CUTLASS raster, swizzle and b options such as _rH_s8_b1 or _sw4, and a microbenchmark's + thread-block size such as _T256). cutlass_ws_overlap runs the CUTLASS GEMM of cutlass_gemm with + device timer markers added, so one shape of the two motifs is one problem. No problem crosses + partitions. The split is stratified by dominant_l1, with 3/1/1 folds and seed 42. softmax has + only two problems, and both are in the training split. Each record's metadata keeps the authors' split memberships under release_splits (train, val, test, iid, param_ood, composed, length_ood), so the released benchmark stays reproducible. license: Apache-2.0 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py index af3db720f..aba9a4fd3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py @@ -1,5 +1,8 @@ import json from pathlib import Path +import shutil +import struct +import zipfile import numpy as np import pytest @@ -32,6 +35,7 @@ FIXTURES = Path(__file__).parent / "fixtures" """One real Snapshot of the pinned release: a warp-specialized GEMM with a multi-hot-only TMA track.""" +_REAL_PATH = f"kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0{INPUT_SUFFIX}" _BIN_NS = 1_000_000 """One millisecond per bin, so bin ``i`` starts at ``i * 1000`` microseconds.""" _MATMUL, _MEMORY = VOCAB_L1.index("matmul"), VOCAB_L1.index("memory_movement") @@ -173,11 +177,10 @@ def test_segments_and_multihot_only_runs_become_interval_annotations(tmp_path): def test_a_real_snapshot_of_the_release_becomes_a_record(): - path = f"kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0{INPUT_SUFFIX}" - record = _snapshot(path, "cutlass_ws_overlap").create_record(FIXTURES) + record = _snapshot(_REAL_PATH, "cutlass_ws_overlap").create_record(FIXTURES) signals = [signal for source in record.sources for signal in source.signals] assert len(signals) == len(COUNTER_NAMES) - with np.load(FIXTURES / path, allow_pickle=False) as archive: + with np.load(FIXTURES / _REAL_PATH, allow_pickle=False) as archive: np.testing.assert_array_equal(signals[5].to_numpy(), archive["data"][5]) axis = signals[0].time_axis assert isinstance(axis, RegularAxis) @@ -212,12 +215,17 @@ def test_variants_of_one_kernelbench_problem_share_a_partition(): ("motif", "variant", "problem"), [ ("cutlass_gemm", "m1024_n1024_k8192_rH_s8_b1", "cutlass_gemm/m1024_n1024_k8192"), - ("cutlass_ws_overlap", "m512_n512_k8192_rM_s1_b0", "cutlass_ws_overlap/m512_n512_k8192"), - ("cutlass_ws_overlap", "m1024_k2048", "cutlass_ws_overlap/m1024_k2048"), + ("cutlass_gemm", "m4096_rM", "cutlass_gemm/m4096"), + ("cutlass_gemm", "m2048_rN_b1", "cutlass_gemm/m2048"), + # cutlass_ws_overlap runs the kernel of cutlass_gemm. + ("cutlass_ws_overlap", "m512_n512_k8192_rM_s1_b0", "cutlass_gemm/m512_n512_k8192"), + ("cutlass_ws_overlap", "m2048_sw4", "cutlass_gemm/m2048"), + ("cutlass_ws_overlap", "m1024_k2048", "cutlass_gemm/m1024_k2048"), ("vector_add", "N16M_T1024", "vector_add/N16M"), ("reduction", "N16M_T256_I4", "reduction/N16M_I4"), + ("scatter", "N1M_B16_T512", "scatter/N1M_B16"), ("cutlass_fmha", "b2h16_q1024_d64_causal", "cutlass_fmha/b2h16_q1024_d64_causal"), - ("cutlass_ws_overlap", "", "cutlass_ws_overlap/baseline"), + ("cutlass_ws_overlap", "", "cutlass_gemm/baseline"), ], ) def test_the_problem_of_a_variant_drops_its_launch_options(motif, variant, problem): @@ -246,6 +254,21 @@ def test_a_corrupt_labels_file_raises(tmp_path): _snapshot(_gemm_path(0), "cutlass_gemm").create_record(tmp_path) +def test_a_damaged_compressed_member_raises(tmp_path): + shutil.copytree(FIXTURES, tmp_path, dirs_exist_ok=True) + snapshot = _snapshot(_REAL_PATH, "cutlass_ws_overlap") + labels = tmp_path / snapshot.labels_path + with zipfile.ZipFile(labels) as archive: + offset = archive.getinfo("segment_starts.npy").header_offset + raw = bytearray(labels.read_bytes()) + name_length, extra_length = struct.unpack_from(" Date: Sat, 21 Nov 2026 00:46:49 +0400 Subject: [PATCH 4/4] fix(connectors): group short kernelsight CUTLASS names with their shapes --- .../williamhtan/kernelsight/connector.py | 28 ++++++- .../williamhtan/kernelsight/dataset.yaml | 12 +-- .../kernelsight/tests/test_connector.py | 80 ++++++++++++++----- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py index 443b14770..e6cd351b1 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py @@ -195,6 +195,26 @@ # shape of the two motifs gives near-identical traces, so it is one Problem. _SAME_KERNEL = {"cutlass_ws_overlap": "cutlass_gemm"} +# NOTE: A short ex48 Variant name leaves the other sizes at a default, and the two motifs have different +# defaults. Each alias below runs the same shape as its target: the launch times of the CUTLASS GEMM kernel +# and of the example's reference GEMM kernel agree within about 1%. The sw4 run of cutlass_ws_overlap/m1024 +# agrees on the reference GEMM only, because its swizzle changes the CUTLASS kernel. +# No full shape has the times of cutlass_gemm/m8192_k1024, so its n is not its m. +_ALIASES = { + "cutlass_gemm/baseline": "m4096", + "cutlass_gemm/kheavy": "m1024_n1024_k8192", + "cutlass_gemm/m1024": "m1024_n1024_k1024", + "cutlass_gemm/m2048": "m2048_n2048_k2048", + "cutlass_ws_overlap/baseline": "k2048", + "cutlass_ws_overlap/k4096": "m512_n512_k4096", + "cutlass_ws_overlap/m1024": "m1024_n1024_k2048", + "cutlass_ws_overlap/m1024_k2048": "m1024_n1024_k2048", + "cutlass_ws_overlap/m2048": "m2048_n2048_k2048", + "cutlass_ws_overlap/m2048_k2048": "m2048_n2048_k2048", + "cutlass_ws_overlap/m4096": "m4096_n4096_k2048", + "cutlass_ws_overlap/m4096_k2048": "m4096_n4096_k2048", +} + def _slug(name: str) -> str: """Turn a counter name such as ``"SMs Active [Throughput %]"`` into ``"sms_active_throughput"``. @@ -288,10 +308,12 @@ def variant(self) -> str: @property def problem(self) -> str: - """The leakage group: the motif of its kernel, and the Variant without dtype, seed and launch options.""" + """The leakage group: the kernel's motif and the Variant's shape, without dtype, seed and launch options.""" if self.motif == "kernelbench": return f"{self.motif}/{self.variant.split('/')[0]}" - return f"{_SAME_KERNEL.get(self.motif, self.motif)}/{_LAUNCH_OPTIONS.sub('', self.variant)}" + variant = _LAUNCH_OPTIONS.sub("", self.variant) + shape = _ALIASES.get(f"{self.motif}/{variant}", variant) + return f"{_SAME_KERNEL.get(self.motif, self.motif)}/{shape}" @property def labels_path(self) -> str: @@ -467,7 +489,7 @@ def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: The train, validation and test Snapshots. """ # NOTE: 3/1/1, not the 8/1/1 of docs/connectors.md. Several classes have only 6 to 8 Problems, and with - # 8/1/1 loss gets no validation or test Snapshot. + # 8/1/1 loss, pooling and elementwise each lose the validation or the test partition. splitter = StratifiedSplitter( train_folds=3, validation_folds=1, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml index d0cf19363..bf176c99b 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml @@ -22,11 +22,13 @@ description: >- grouped by problem, which is the variant without its dtype, seed and launch options (the CUTLASS raster, swizzle and b options such as _rH_s8_b1 or _sw4, and a microbenchmark's thread-block size such as _T256). cutlass_ws_overlap runs the CUTLASS GEMM of cutlass_gemm with - device timer markers added, so one shape of the two motifs is one problem. No problem crosses - partitions. The split is stratified by dominant_l1, with 3/1/1 folds and seed 42. softmax has - only two problems, and both are in the training split. Each record's metadata - keeps the authors' split memberships under release_splits (train, val, test, iid, param_ood, - composed, length_ood), so the released benchmark stays reproducible. + device timer markers added, so one shape of the two motifs is one problem. A short CUTLASS name, + such as m4096_k2048 or kheavy, leaves the other sizes at a default. Where the launch times of + the example's kernels match a full shape, such as m4096_n4096_k2048, the variant is in the + problem of that shape. No problem crosses partitions. The split is stratified by dominant_l1, + with 3/1/1 folds and seed 42. softmax has only two problems and has no validation snapshot. + Each record's metadata keeps the authors' split memberships under release_splits (train, val, + test, iid, param_ood, composed, length_ood), so the released benchmark stays reproducible. license: Apache-2.0 source_url: https://huggingface.co/datasets/williamhtan/kernelsight citation: >- diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py index aba9a4fd3..a41c62371 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py @@ -2,6 +2,7 @@ from pathlib import Path import shutil import struct +from typing import Any import zipfile import numpy as np @@ -108,20 +109,30 @@ def _gemm_path(index: int) -> str: return f"kernels/cutlass_gemm/_out/m{index}{INPUT_SUFFIX}" +_DOMINANT = {_gemm_path(index): "matmul" if index < 5 else "conv" for index in range(10)} +"""Five Problems of each class, the fewest that the 3/1/1 split accepts.""" +_TRACE = _trace(_gemm_path(0), "cutlass_gemm", "matmul") + + def _convert(root: Path) -> tuple[TimeFDataset, dict[str, np.ndarray]]: - # Five Problems, the fewest that the 3/1/1 split accepts. values = {} - traces = [] - for index in range(5): - path = _gemm_path(index) + for index, path in enumerate(_DOMINANT): values[path] = _write_snapshot( root, path, seed=index, segments=[(100, 200, _MATMUL, _GEMM)], overlap=index == 0 ) - traces.append(_trace(path, "cutlass_gemm", "matmul")) - _write_splits(root, {"train": traces[:3], "val": traces[3:4], "test": traces[4:]}, {"param_ood": traces[:1]}) + traces = [_trace(path, "cutlass_gemm", dominant) for path, dominant in _DOMINANT.items()] + _write_splits(root, {"train": traces[:6], "val": traces[6:8], "test": traces[8:]}, {"param_ood": traces[:1]}) return KernelSightConnector().convert(_files(root)), values +def _replace_arrays(path: Path, **arrays: np.ndarray) -> None: + # The file is one that the test wrote, so its object arrays are safe to unpickle. + with np.load(path, allow_pickle=True) as archive: + kept: dict[str, Any] = {name: archive[name] for name in archive.files} + kept.update(arrays) + np.savez(path, **kept) + + def _found(annotations) -> set: return {(a.key, read_value(a.value), a.span, a.occurrence_metadata.get("track")) for a in annotations} @@ -135,7 +146,7 @@ def test_each_snapshot_becomes_a_record_and_a_classification_task(tmp_path): (record,) = task.inputs assert task.label_set == WORKLOAD_L1 assert task.targets is not None - assert read_value(task.targets[0]) == "matmul" + assert read_value(task.targets[0]) == _DOMINANT[str(record.metadata["path"])] assert [source.name for source in record.sources] == [group for group, _ in CHANNEL_GROUPS] signals = [signal for source in record.sources for signal in source.signals] assert [signal.metadata["counter"] for signal in signals] == list(COUNTER_NAMES) @@ -216,34 +227,61 @@ def test_variants_of_one_kernelbench_problem_share_a_partition(): [ ("cutlass_gemm", "m1024_n1024_k8192_rH_s8_b1", "cutlass_gemm/m1024_n1024_k8192"), ("cutlass_gemm", "m4096_rM", "cutlass_gemm/m4096"), - ("cutlass_gemm", "m2048_rN_b1", "cutlass_gemm/m2048"), - # cutlass_ws_overlap runs the kernel of cutlass_gemm. + ("cutlass_gemm", "", "cutlass_gemm/m4096"), + ("cutlass_gemm", "m2048_rN_b1", "cutlass_gemm/m2048_n2048_k2048"), + ("cutlass_gemm", "kheavy", "cutlass_gemm/m1024_n1024_k8192"), + ("cutlass_gemm", "m8192_k1024", "cutlass_gemm/m8192_k1024"), + # cutlass_ws_overlap runs the kernel of cutlass_gemm, but a short name has the defaults of its own motif. ("cutlass_ws_overlap", "m512_n512_k8192_rM_s1_b0", "cutlass_gemm/m512_n512_k8192"), - ("cutlass_ws_overlap", "m2048_sw4", "cutlass_gemm/m2048"), - ("cutlass_ws_overlap", "m1024_k2048", "cutlass_gemm/m1024_k2048"), + ("cutlass_ws_overlap", "m4096_rN", "cutlass_gemm/m4096_n4096_k2048"), + ("cutlass_ws_overlap", "m2048_sw4", "cutlass_gemm/m2048_n2048_k2048"), + ("cutlass_ws_overlap", "m1024_k2048", "cutlass_gemm/m1024_n1024_k2048"), + ("cutlass_ws_overlap", "", "cutlass_gemm/k2048"), ("vector_add", "N16M_T1024", "vector_add/N16M"), ("reduction", "N16M_T256_I4", "reduction/N16M_I4"), ("scatter", "N1M_B16_T512", "scatter/N1M_B16"), ("cutlass_fmha", "b2h16_q1024_d64_causal", "cutlass_fmha/b2h16_q1024_d64_causal"), - ("cutlass_ws_overlap", "", "cutlass_gemm/baseline"), ], ) -def test_the_problem_of_a_variant_drops_its_launch_options(motif, variant, problem): +def test_the_problem_of_a_variant_is_its_shape_without_launch_options(motif, variant, problem): directory = f"{variant}/" if variant else "" assert _snapshot(f"kernels/{motif}/_out/{directory}{INPUT_SUFFIX.lstrip('/')}", motif).problem == problem -def test_a_snapshot_in_two_official_partitions_raises(tmp_path): - trace = _trace(_gemm_path(0), "cutlass_gemm", "matmul") - _write_splits(tmp_path, {"train": [trace], "test": [trace]}) - with pytest.raises(TimeNetConnectorError, match="more than one official partition"): +@pytest.mark.parametrize( + ("official", "extra", "message"), + [ + ({"train": [_TRACE], "test": [_TRACE]}, {}, "more than one official partition"), + ({}, {"iid": [_TRACE]}, "in no official partition"), + ({"train": [{"path": _gemm_path(0)}]}, {}, r"train\.json is not a KernelSight split file"), + ({"train": [_trace(_gemm_path(0), "cutlass_fmha", "matmul")]}, {}, "is not under kernels/cutlass_fmha/_out/"), + ({"train": [_trace(_gemm_path(0), "cutlass_gemm", "gemm")]}, {}, "'gemm' is not an L1 class"), + ], +) +def test_a_malformed_split_index_raises(tmp_path, official, extra, message): + _write_splits(tmp_path, official, extra) + with pytest.raises(TimeNetConnectorError, match=message): read_snapshots(tmp_path / "splits") -def test_a_malformed_split_file_raises(tmp_path): - _write_splits(tmp_path, {"train": [{"path": _gemm_path(0)}]}) - with pytest.raises(TimeNetConnectorError, match=r"train\.json is not a KernelSight split file"): - read_snapshots(tmp_path / "splits") +@pytest.mark.parametrize( + ("suffix", "arrays", "message"), + [ + (LABELS_SUFFIX, {"segment_ends": np.array([10, 20])}, "different lengths"), + (LABELS_SUFFIX, {"segment_ends": np.array([N_BINS + 1])}, "outside the 512 bins"), + (LABELS_SUFFIX, {"segment_label_l1": np.array([-1]), "segment_label_l2": np.array([-1])}, "out of range"), + (LABELS_SUFFIX, {"workload_l2_multihot": np.zeros((N_BINS, 3), np.uint8)}, "workload_l2_multihot has shape"), + (INPUT_SUFFIX, {"time_edges_ns": np.arange(N_BINS)}, "time_edges_ns has shape"), + (INPUT_SUFFIX, {"data": np.zeros((len(COUNTER_NAMES), N_BINS - 1), np.float32)}, "data has shape"), + ], +) +def test_a_malformed_array_raises(tmp_path, suffix, arrays, message): + _write_snapshot(tmp_path, _gemm_path(0), seed=0, segments=[(0, 10, _MATMUL, _GEMM)]) + _replace_arrays(tmp_path / (_gemm_path(0).removesuffix(INPUT_SUFFIX) + suffix), **arrays) + with pytest.raises(TimeNetConnectorError, match=message): + record = _snapshot(_gemm_path(0), "cutlass_gemm").create_record(tmp_path) + # The data shape is checked only when a value is read. + record.sources[0].signals[0].to_numpy() def test_a_corrupt_labels_file_raises(tmp_path):