diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py new file mode 100644 index 000000000..3367da279 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/__init__.py @@ -0,0 +1,4 @@ +"""Connectors for datasets published by the ``williamhtan`` Hub user.""" + +# Keep this a regular package, not a namespace package. pkgutil.walk_packages finds the +# connector packages below it only through this __init__.py. diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py new file mode 100644 index 000000000..c60729eaf --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/__init__.py @@ -0,0 +1,6 @@ +"""KernelSight: H100 GPU counter traces with per-segment workload labels.""" + +from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( + CONNECTOR as CONNECTOR, + KernelSightConnector as KernelSightConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py new file mode 100644 index 000000000..e6cd351b1 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/connector.py @@ -0,0 +1,544 @@ +"""The KernelSight connector: H100 GPU counter traces with per-segment workload labels. + +Each snapshot of the release is a ``[24, 512]`` counter image: 24 hardware-counter channels over 512 +equal-width bins. Each snapshot becomes one Record with one Signal per channel, grouped into the +five channel groups of the dataset card. Each labeled segment becomes a time-interval annotation +with its L1 and L2 workload class. Each snapshot gets one ClassificationTask whose target is the +authors' dominant L1 class. + +The release's own train/val/test split puts dtype and seed variants of one KernelBench problem in +different partitions. The connector therefore assigns a split grouped by problem and keeps the +authors' split memberships in the record metadata. + +The release stores its vocabularies and counter names as pickled object arrays. Unpickling a +downloaded file can run code, so the connector keeps those names as constants and reads only the +numeric arrays, with ``allow_pickle=False``. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from fractions import Fraction +import json +from operator import attrgetter +from pathlib import Path +import re +import zipfile +import zlib + +import numpy as np +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.types import Annotation, ClassificationTask, LabelSet, TimeInterval, TimeSeriesSpec, ureg +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector +from timenet_connectors.splitting import DatasetSplits, StratifiedSplitter + + +N_BINS = 512 + +# The five channel groups of the dataset card, in row order of the ``data`` array. +CHANNEL_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ( + "Pipe signature", + ( + "pipe: tensor_op_hmma", + "pipe: xu", + "pipe: fma", + "pipe: alu", + "pipe: lsu", + "pipe: cbu", + "pipe: tma", + ), + ), + ("Memory access", ("hit: l2", "atom: lts_atomic_input_pct")), + ( + "Discriminators", + ("stall: short_scoreboard", "stall: barrier", "pred_on_per_inst_ratio", "gmem_coalesce_ratio"), + ), + ( + "System bandwidth", + ( + "SMs Active [Throughput %]", + "DRAM Read Bandwidth [Throughput %]", + "DRAM Write Bandwidth [Throughput %]", + "Tensor Active [Throughput %]", + ), + ), + ( + "SASS modality", + ( + "sass_compute_fma", + "sass_compute_tensor", + "sass_memory_global", + "sass_memory_shared", + "sass_memory_tma", + "sass_control", + "sass_misc", + ), + ), +) +COUNTER_NAMES: tuple[str, ...] = tuple(name for _, names in CHANNEL_GROUPS for name in names) + +# The ``vocab_l1`` and ``vocab_l2`` arrays of every labels file, in index order. +VOCAB_L1: tuple[str, ...] = ( + "matmul", + "conv", + "activation", + "normalization", + "softmax", + "pooling", + "reduction", + "attention", + "loss", + "elementwise", + "memory_movement", + "other", +) +VOCAB_L2: tuple[str, ...] = ( + "matmul_bmm", + "matmul_gemm", + "matmul_matvec", + "conv_conv1d_standard", + "conv_conv2d_depthwise", + "conv_conv2d_pointwise", + "conv_conv2d_standard", + "conv_conv3d_standard", + "conv_convtranspose1d", + "conv_convtranspose2d", + "conv_convtranspose3d", + "activation_elu", + "activation_gelu", + "activation_hardsigmoid", + "activation_hardswish", + "activation_hardtanh", + "activation_leaky_relu", + "activation_mish", + "activation_other", + "activation_relu", + "activation_selu", + "activation_sigmoid", + "activation_softplus", + "activation_softsign", + "activation_swish", + "activation_tanh", + "normalization_batchnorm", + "normalization_frobeniusnorm", + "normalization_groupnorm", + "normalization_instancenorm", + "normalization_l1norm", + "normalization_l2norm", + "normalization_layernorm", + "normalization_rmsnorm", + "softmax_log_softmax", + "softmax_logsumexp", + "softmax_softmax", + "pooling_avg_pool", + "pooling_global_avg_pool", + "pooling_max_pool", + "reduction_argmax", + "reduction_argmin", + "reduction_cumprod", + "reduction_cumsum", + "reduction_max", + "reduction_mean", + "reduction_min", + "reduction_prod", + "reduction_sum", + "attention_scaled_dot_product", + "loss_cross_entropy", + "loss_hinge", + "loss_huber", + "loss_kldiv", + "loss_mse", + "loss_triplet_margin", + "elementwise_add", + "elementwise_bias_add", + "elementwise_cast", + "elementwise_clamp", + "elementwise_div", + "elementwise_mul", + "elementwise_residual_add", + "elementwise_scalar_multiplication", + "elementwise_scaling", + "elementwise_sub", + "memory_movement_copy", + "memory_movement_embedding", + "memory_movement_gather", + "memory_movement_scatter", + "memory_movement_transpose", + "other_dropout", + "other_misc", +) + +WORKLOAD_L1 = LabelSet(VOCAB_L1, annotations=[Annotation("name", value="workload_l1")]) + +OFFICIAL_SPLITS = ("train", "val", "test") +"""The released partition files. Each Snapshot is in exactly one of them.""" +RELEASE_SPLITS = (*OFFICIAL_SPLITS, "iid", "param_ood", "composed", "length_ood") +"""Every released split file. A Snapshot can be in several; the record metadata keeps them all.""" + +MULTIHOT_TRACK = "multihot" +"""The ``track`` of an interval that only the multi-hot overlap track of the release holds.""" + +INPUT_SUFFIX = "/input/tensor_input.npz" +LABELS_SUFFIX = "/labels/labels.npz" + +# The launch options of a Variant: the CUTLASS raster, swizzle and ``b`` options, alone or together +# (``m1024_n1024_k8192_rH_s8_b1``, ``m4096_rM``, ``m2048_sw4``, ``m2048_rN_b1``), and the thread-block size +# of a microbenchmark (``N1M_T256``). +_LAUNCH_OPTIONS = re.compile(r"_(?:r[A-Z]|s\d+|sw\d+|b\d+)(?=_|$)|_T\d+") + +# NOTE: cutlass_ws_overlap runs the CUTLASS ex48 GEMM of cutlass_gemm with device timer markers added. One +# shape of the two motifs gives near-identical traces, so it is one Problem. +_SAME_KERNEL = {"cutlass_ws_overlap": "cutlass_gemm"} + +# NOTE: A short ex48 Variant name leaves the other sizes at a default, and the two motifs have different +# defaults. Each alias below runs the same shape as its target: the launch times of the CUTLASS GEMM kernel +# and of the example's reference GEMM kernel agree within about 1%. The sw4 run of cutlass_ws_overlap/m1024 +# agrees on the reference GEMM only, because its swizzle changes the CUTLASS kernel. +# No full shape has the times of cutlass_gemm/m8192_k1024, so its n is not its m. +_ALIASES = { + "cutlass_gemm/baseline": "m4096", + "cutlass_gemm/kheavy": "m1024_n1024_k8192", + "cutlass_gemm/m1024": "m1024_n1024_k1024", + "cutlass_gemm/m2048": "m2048_n2048_k2048", + "cutlass_ws_overlap/baseline": "k2048", + "cutlass_ws_overlap/k4096": "m512_n512_k4096", + "cutlass_ws_overlap/m1024": "m1024_n1024_k2048", + "cutlass_ws_overlap/m1024_k2048": "m1024_n1024_k2048", + "cutlass_ws_overlap/m2048": "m2048_n2048_k2048", + "cutlass_ws_overlap/m2048_k2048": "m2048_n2048_k2048", + "cutlass_ws_overlap/m4096": "m4096_n4096_k2048", + "cutlass_ws_overlap/m4096_k2048": "m4096_n4096_k2048", +} + + +def _slug(name: str) -> str: + """Turn a counter name such as ``"SMs Active [Throughput %]"`` into ``"sms_active_throughput"``. + + Returns: + The lowercase name, with each run of other characters replaced by one underscore. + """ + return re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") + + +# One spec per channel, because a spec type has one contract and each counter has its own name. The +# values are dimensionless because the authors divided each channel by a fixed physical-scale divisor. +_SPECS: Mapping[str, TimeSeriesSpec] = { + name: TimeSeriesSpec(spec_type=f"gpu_counter_{_slug(name)}", name=name, unit_value=ureg.dimensionless) + for name in COUNTER_NAMES +} + + +def _read_arrays(path: Path, *names: str) -> list[np.ndarray]: + """Read numeric arrays from an ``.npz`` file without unpickling. + + Returns: + The arrays, in the order of ``names``. + + Raises: + TimeNetConnectorError: If the file is missing or corrupt, lacks one of the arrays, or one of the + arrays holds Python objects. + """ + try: + with np.load(path, allow_pickle=False) as archive: + return [archive[name] for name in names] + except (OSError, EOFError, ValueError, KeyError, zipfile.BadZipFile, zlib.error) as error: + raise TimeNetConnectorError(f"{path}: cannot read {', '.join(names)}: {error}") from error + + +def _channel_loader(path: Path, row: int) -> Callable[[], pa.Array]: + """Build a lazy loader for one channel of a Snapshot. + + Returns: + A loader that reads row ``row`` of the ``data`` array as a float32 Arrow array. + """ + + def load() -> pa.Array: + # NOTE: The shape is checked here, not in ``convert``, so that ``convert`` never reads the values. + (data,) = _read_arrays(path, "data") + if data.shape != (len(COUNTER_NAMES), N_BINS): + raise TimeNetConnectorError(f"{path}: data has shape {data.shape}") + return pa.array(data[row]) + + return load + + +def _true_runs(active: np.ndarray) -> Iterator[tuple[int, int]]: + """Yield the ``[start, end)`` bin ranges of each run of consecutive true values.""" + padded = np.concatenate(([False], active.astype(bool), [False])) + edges = np.flatnonzero(np.diff(padded.astype(np.int8))) + yield from zip(edges[::2].tolist(), edges[1::2].tolist(), strict=True) + + +@dataclass(frozen=True) +class Snapshot: + """One Snapshot of the release, as its split files describe it.""" + + path: str + """The input tensor, relative to the dataset root, such as + ``kernels/kernelbench/_out/L1_P100/dt-bf16_s42/input/tensor_input.npz``.""" + motif: str + dominant_l1: str + official_split: str + release_splits: tuple[str, ...] + + def __post_init__(self) -> None: + """Check that the path has the layout of the release and the class is in the vocabulary. + + Raises: + TimeNetConnectorError: If the path or the dominant class is not as expected. + """ + if not (self.path.startswith(self._variants_dir) and self.path.endswith(INPUT_SUFFIX)): + raise TimeNetConnectorError(f"snapshot path {self.path!r} is not under {self._variants_dir}") + if self.dominant_l1 not in VOCAB_L1: + raise TimeNetConnectorError(f"{self.path}: dominant_l1 {self.dominant_l1!r} is not an L1 class") + + @property + def _variants_dir(self) -> str: + return f"kernels/{self.motif}/_out/" + + @property + def variant(self) -> str: + """The Variant directory below ``_out/``, or ``"baseline"`` for a Snapshot directly in it.""" + return self.path[len(self._variants_dir) : -len(INPUT_SUFFIX)] or "baseline" + + @property + def problem(self) -> str: + """The leakage group: the kernel's motif and the Variant's shape, without dtype, seed and launch options.""" + if self.motif == "kernelbench": + return f"{self.motif}/{self.variant.split('/')[0]}" + variant = _LAUNCH_OPTIONS.sub("", self.variant) + shape = _ALIASES.get(f"{self.motif}/{variant}", variant) + return f"{_SAME_KERNEL.get(self.motif, self.motif)}/{shape}" + + @property + def labels_path(self) -> str: + """The labels file of the Snapshot, relative to the dataset root.""" + return self.path.removesuffix(INPUT_SUFFIX) + LABELS_SUFFIX + + def create_record(self, root: Path) -> Record: + """Create the Record: one Signal per channel, grouped by channel group, with Segment annotations. + + Args: + root: The dataset root that the paths of the Snapshot are relative to. + + Returns: + The Record, not yet added to a dataset. + + Raises: + TimeNetConnectorError: If a file of the Snapshot is missing or corrupt, or does not have the + layout of the release. + """ + input_path = root / self.path + (edges,) = _read_arrays(input_path, "time_edges_ns") + if edges.shape != (N_BINS + 1,): + raise TimeNetConnectorError(f"{self.path}: time_edges_ns has shape {edges.shape}") + # The bins are equal-width up to one nanosecond of rounding, so the exact mean width is the period. + axis = RegularAxis(period_us=Fraction(int(edges[-1] - edges[0]), N_BINS * 1000)) + signals = iter( + Signal.from_loader( + spec=_SPECS[name], + time_axis=axis, + name=_slug(name), + loader=_channel_loader(input_path, row), + n_values=N_BINS, + metadata={"counter": name}, + ) + for row, name in enumerate(COUNTER_NAMES) + ) + sources = tuple(Source(group, signals=tuple(next(signals) for _ in names)) for group, names in CHANNEL_GROUPS) + return Record( + sources=sources, + metadata={ + "path": self.path, + "motif": self.motif, + "problem": self.problem, + "variant": self.variant, + "official_split": self.official_split, + "release_splits": list(self.release_splits), + }, + annotations=self._segment_annotations(root, axis), + ) + + def _segment_annotations(self, root: Path, axis: RegularAxis) -> list[Annotation]: + """Annotate each labeled Segment, and each multi-hot run of a class that no Segment has. + + Returns: + The interval annotations of the Record. + + Raises: + TimeNetConnectorError: If the labels file is missing or corrupt, an array has the wrong shape, + a label index or a Segment bound is out of range, or an L2 class does not refine the L1 + class of its Segment. + """ + path = self.labels_path + + def interval(start: int, end: int) -> TimeInterval: + if not 0 <= start < end <= N_BINS: + raise TimeNetConnectorError(f"{path}: segment [{start}, {end}) is outside the {N_BINS} bins") + return TimeInterval.micros(int(start * axis.period_us), int(end * axis.period_us)) + + starts, ends, l1, l2, l1_multihot, l2_multihot = _read_arrays( + root / path, + "segment_starts", + "segment_ends", + "segment_label_l1", + "segment_label_l2", + "workload_l1_multihot", + "workload_l2_multihot", + ) + if not len(starts) == len(ends) == len(l1) == len(l2): + raise TimeNetConnectorError(f"{path}: the segment arrays have different lengths") + tracks = (("workload_l1", l1, l1_multihot, VOCAB_L1), ("workload_l2", l2, l2_multihot, VOCAB_L2)) + for key, _, multihot, vocab in tracks: + if multihot.shape != (N_BINS, len(vocab)): + raise TimeNetConnectorError(f"{path}: {key}_multihot has shape {multihot.shape}") + annotations = [] + for start, end, l1_index, l2_index in zip(starts, ends, l1, l2, strict=True): + if not (0 <= l1_index < len(VOCAB_L1) and 0 <= l2_index < len(VOCAB_L2)): + raise TimeNetConnectorError(f"{path}: label index ({l1_index}, {l2_index}) is out of range") + l1_name, l2_name = VOCAB_L1[l1_index], VOCAB_L2[l2_index] + if not l2_name.startswith(f"{l1_name}_"): + raise TimeNetConnectorError(f"{path}: L2 class {l2_name} does not refine L1 class {l1_name}") + span = interval(int(start), int(end)) + annotations += [ + Annotation("workload_l1", value=l1_name, span=span), + Annotation("workload_l2", value=l2_name, span=span), + ] + # The overlap track of the release can hold a class that no Segment has, such as the TMA loads + # (memory_movement, memory_movement_copy) that run during a warp-specialized GEMM. Keep each of + # its runs as an interval. + for key, segment_classes, multihot, vocab in tracks: + only_multihot = set(np.flatnonzero(multihot.any(axis=0)).tolist()) - set(segment_classes.tolist()) + annotations += [ + Annotation( + key, + value=vocab[class_index], + span=interval(start, end), + occurrence_metadata={"track": MULTIHOT_TRACK}, + ) + for class_index in sorted(only_multihot) + for start, end in _true_runs(multihot[:, class_index]) + ] + return annotations + + +def _read_split_file(path: Path) -> list[tuple[str, str, str]]: + """Read the ``(path, motif, dominant_l1)`` of each Snapshot that a released split file lists. + + Returns: + One tuple per Snapshot, in the order of the file. + + Raises: + TimeNetConnectorError: If the file is missing or does not have the layout of the release. + """ + try: + index = json.loads(path.read_text(encoding="utf-8")) + return [(entry["path"], entry["motif"], entry["dominant_l1"]) for entry in index["traces"]] + except (OSError, ValueError, KeyError, TypeError) as error: + raise TimeNetConnectorError(f"{path} is not a KernelSight split file: {error!r}") from error + + +def read_snapshots(splits_dir: Path) -> list[Snapshot]: + """Read every Snapshot from the released split files, sorted by path. + + The sort makes the split assignment independent of the order of the files. + + Args: + splits_dir: The ``splits`` directory of the release. + + Returns: + One Snapshot per entry of the train, val and test files. + + Raises: + TimeNetConnectorError: If a split file is missing or malformed, a Snapshot is in more than one + official partition or in none, or its path or dominant class is not as in the release. + """ + release_splits: dict[str, list[str]] = defaultdict(list) + official: dict[str, tuple[str, str, str]] = {} + for name in RELEASE_SPLITS: + for path, motif, dominant_l1 in _read_split_file(splits_dir / f"{name}.json"): + release_splits[path].append(name) + if name in OFFICIAL_SPLITS: + if path in official: + raise TimeNetConnectorError(f"{path} is in more than one official partition") + official[path] = (name, motif, dominant_l1) + unknown = sorted(release_splits.keys() - official.keys()) + if unknown: + raise TimeNetConnectorError(f"{len(unknown)} snapshots are in no official partition, such as {unknown[0]}") + return [ + Snapshot( + path=path, + motif=motif, + dominant_l1=dominant_l1, + official_split=split, + release_splits=tuple(release_splits[path]), + ) + for path, (split, motif, dominant_l1) in sorted(official.items()) + ] + + +def assign_splits(snapshots: list[Snapshot]) -> DatasetSplits[Snapshot]: + """Partition the Snapshots 3/1/1, grouped by Problem and stratified by the dominant L1 class. + + Returns: + The train, validation and test Snapshots. + """ + # NOTE: 3/1/1, not the 8/1/1 of docs/connectors.md. Several classes have only 6 to 8 Problems, and with + # 8/1/1 loss, pooling and elementwise each lose the validation or the test partition. + splitter = StratifiedSplitter( + train_folds=3, + validation_folds=1, + test_folds=1, + stratify_by=attrgetter("dominant_l1"), + group_by=attrgetter("problem"), + seed=42, + ) + return splitter.split(snapshots) + + +# A glob's part path is its directory, so both kernel parts resolve to ``kernels/``. The ``*`` matches +# across directories, which selects the input and labels files and skips the fingerprints. +class KernelSightConnector( + BaseHuggingFaceConnector, + hf_repo="williamhtan/kernelsight", + source_revision="d8cd8e804e5b619aadbb082a1fb3a8791b60a680", + source_files={ + "inputs": "kernels/*tensor_input.npz", + "labels": "kernels/*labels.npz", + "splits": "splits/*.json", + }, +): + """Connector for the KernelSight release on the Hub (``williamhtan/kernelsight``).""" + + def convert(self, files: Mapping[str, Path]) -> TimeFDataset: + """Create one Record and one ClassificationTask for each Snapshot. + + Args: + files: The kernel and split directories from :meth:`download`. + + Returns: + The populated dataset. + """ + root = files["inputs"].parent + dataset = TimeFDataset(metadata=self.metadata()) + splits = assign_splits(read_snapshots(files["splits"])) + for split, snapshots in splits.items(): + for snapshot in snapshots: + record = dataset.add_record(snapshot.create_record(root)) + dataset.add_task( + ClassificationTask( + inputs=(record,), + targets=snapshot.dominant_l1, + label_set=WORKLOAD_L1, + split=split, + ) + ) + return dataset + + +CONNECTOR = KernelSightConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml new file mode 100644 index 000000000..bf176c99b --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/dataset.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +yaml_schema_version: 1 +dataset_id: williamhtan/kernelsight +dataset_version: 1.0.0 +name: KernelSight +description: >- + 1,444 GPU workload snapshots profiled on one NVIDIA H100 80GB HBM3 (sm_90a). Each snapshot is + 24 hardware-counter channels over 512 equal-width bins that cover the kernel-active window, so + the bin width differs per snapshot (0.2 ms to 43 ms). The channels form five groups: pipe + signature, memory access, discriminators, system bandwidth (Nsight Systems) and SASS modality + (NVBit). The values are kept as released. The authors divided each channel by a fixed + physical-scale divisor (for example 100 for a percent, 64 for a warp-stall ratio, 8 for + sectors per warp load), so most values are in [0, 1], but pred_on_per_inst_ratio reaches about + 32. Each labeled segment is a time-interval annotation with its L1 class (12 classes, of which + `other` never occurs) and its L2 class (73 classes). In the 472 cutlass_ws_overlap snapshots, + an overlapping memory_movement (L2 memory_movement_copy) interval exists only in the source's + multi-hot track; it is added as a workload_l1 and a workload_l2 annotation with occurrence + metadata track=multihot. Each snapshot has one classification task whose target is the + authors' dominant_l1 field, which can differ from the L1 class with the most labeled bins (34 + snapshots). The released train/val/test split puts dtype and seed variants of one KernelBench + problem into different partitions, so this dataset assigns its own split instead. It is + grouped by problem, which is the variant without its dtype, seed and launch options (the + CUTLASS raster, swizzle and b options such as _rH_s8_b1 or _sw4, and a microbenchmark's + thread-block size such as _T256). cutlass_ws_overlap runs the CUTLASS GEMM of cutlass_gemm with + device timer markers added, so one shape of the two motifs is one problem. A short CUTLASS name, + such as m4096_k2048 or kheavy, leaves the other sizes at a default. Where the launch times of + the example's kernels match a full shape, such as m4096_n4096_k2048, the variant is in the + problem of that shape. No problem crosses partitions. The split is stratified by dominant_l1, + with 3/1/1 folds and seed 42. softmax has only two problems and has no validation snapshot. + Each record's metadata keeps the authors' split memberships under release_splits (train, val, + test, iid, param_ood, composed, length_ood), so the released benchmark stays reproducible. +license: Apache-2.0 +source_url: https://huggingface.co/datasets/williamhtan/kernelsight +citation: >- + Tan, William. KernelSight: Per-Timestep Workload Labeling of GPU Execution + Traces. CS231N project, Stanford University, 2026. + https://huggingface.co/datasets/williamhtan/kernelsight +domains: + - observability +tags: + - gpu + - cuda + - profiling + - h100 + - kernelbench + - cutlass + - classification + - segmentation + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt new file mode 100644 index 000000000..47df73a21 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/requirements.txt @@ -0,0 +1,2 @@ +# huggingface_hub downloads the release from the HuggingFace Hub. Version 1.1 adds tqdm_class. +huggingface_hub>=1.1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/input/tensor_input.npz b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/input/tensor_input.npz new file mode 100644 index 000000000..f2eb175e6 Binary files /dev/null and b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/input/tensor_input.npz differ diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/labels/labels.npz b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/labels/labels.npz new file mode 100644 index 000000000..7fd3100ee Binary files /dev/null and b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/fixtures/kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0/labels/labels.npz differ diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py new file mode 100644 index 000000000..a41c62371 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/williamhtan/kernelsight/tests/test_connector.py @@ -0,0 +1,327 @@ +import json +from pathlib import Path +import shutil +import struct +from typing import Any +import zipfile + +import numpy as np +import pytest + +from timenet.dataset import TimeFDataset +from timenet.dataset.axis import RegularAxis +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import open_local_version +from timenet.testing import assert_datasets_equal +from timenet.types import ClassificationTask, Split, TimeInterval +from timenet.types.text import read_value +from timenet_connectors.datasets.williamhtan.kernelsight.connector import ( + CHANNEL_GROUPS, + COUNTER_NAMES, + INPUT_SUFFIX, + LABELS_SUFFIX, + MULTIHOT_TRACK, + N_BINS, + RELEASE_SPLITS, + VOCAB_L1, + VOCAB_L2, + WORKLOAD_L1, + KernelSightConnector, + Snapshot, + assign_splits, + read_snapshots, +) + + +FIXTURES = Path(__file__).parent / "fixtures" +"""One real Snapshot of the pinned release: a warp-specialized GEMM with a multi-hot-only TMA track.""" +_REAL_PATH = f"kernels/cutlass_ws_overlap/_out/m1024_n1024_k1024_rH_s1_b0{INPUT_SUFFIX}" +_BIN_NS = 1_000_000 +"""One millisecond per bin, so bin ``i`` starts at ``i * 1000`` microseconds.""" +_MATMUL, _MEMORY = VOCAB_L1.index("matmul"), VOCAB_L1.index("memory_movement") +_GEMM, _COPY = VOCAB_L2.index("matmul_gemm"), VOCAB_L2.index("memory_movement_copy") + + +def _trace(path: str, motif: str, dominant: str) -> dict: + return {"path": path, "motif": motif, "dominant_l1": dominant} + + +def _snapshot(path: str, motif: str, dominant: str = "matmul") -> Snapshot: + return Snapshot(path=path, motif=motif, dominant_l1=dominant, official_split="train", release_splits=("train",)) + + +def _write_snapshot( + root: Path, + path: str, + *, + seed: int, + segments: list[tuple[int, int, int, int]], + overlap: bool = False, +) -> np.ndarray: + # Synthetic files in the shape of the release, including its pickled object arrays, which the + # connector must never read. + data = np.random.default_rng(seed).random((len(COUNTER_NAMES), N_BINS), dtype=np.float32) + edges = 5_000_000 + np.arange(N_BINS + 1, dtype=np.int64) * _BIN_NS + input_path = root / path + input_path.parent.mkdir(parents=True) + np.savez(input_path, data=data, time_edges_ns=edges, counter_names=np.array(COUNTER_NAMES, dtype=object)) + starts, ends, l1, l2 = ( + np.array(column, dtype=dtype) + for column, dtype in zip(zip(*segments, strict=True), (np.int64, np.int64, np.int32, np.int32), strict=True) + ) + l1_multihot = np.zeros((N_BINS, len(VOCAB_L1)), dtype=np.uint8) + l2_multihot = np.zeros((N_BINS, len(VOCAB_L2)), dtype=np.uint8) + if overlap: + # A GEMM in bins [100, 200) with TMA loads in [120, 130) and [150, 160) that no Segment holds. + l1_multihot[100:200, _MATMUL] = l2_multihot[100:200, _GEMM] = 1 + for start, end in ((120, 130), (150, 160)): + l1_multihot[start:end, _MEMORY] = l2_multihot[start:end, _COPY] = 1 + labels_path = root / (path.removesuffix(INPUT_SUFFIX) + LABELS_SUFFIX) + labels_path.parent.mkdir(parents=True) + np.savez( + labels_path, + segment_starts=starts, + segment_ends=ends, + segment_label_l1=l1, + segment_label_l2=l2, + workload_l1_multihot=l1_multihot, + workload_l2_multihot=l2_multihot, + vocab_l1=np.array(VOCAB_L1, dtype=object), + vocab_l2=np.array(VOCAB_L2, dtype=object), + ) + return data + + +def _write_splits(root: Path, official: dict[str, list[dict]], extra: dict[str, list[dict]] | None = None) -> None: + splits = root / "splits" + splits.mkdir() + files = {name: [] for name in RELEASE_SPLITS} | official | (extra or {}) + for name, traces in files.items(): + (splits / f"{name}.json").write_text(json.dumps({"split": name, "n": len(traces), "traces": traces})) + + +def _files(root: Path) -> dict[str, Path]: + return {"inputs": root / "kernels", "labels": root / "kernels", "splits": root / "splits"} + + +def _gemm_path(index: int) -> str: + return f"kernels/cutlass_gemm/_out/m{index}{INPUT_SUFFIX}" + + +_DOMINANT = {_gemm_path(index): "matmul" if index < 5 else "conv" for index in range(10)} +"""Five Problems of each class, the fewest that the 3/1/1 split accepts.""" +_TRACE = _trace(_gemm_path(0), "cutlass_gemm", "matmul") + + +def _convert(root: Path) -> tuple[TimeFDataset, dict[str, np.ndarray]]: + values = {} + for index, path in enumerate(_DOMINANT): + values[path] = _write_snapshot( + root, path, seed=index, segments=[(100, 200, _MATMUL, _GEMM)], overlap=index == 0 + ) + traces = [_trace(path, "cutlass_gemm", dominant) for path, dominant in _DOMINANT.items()] + _write_splits(root, {"train": traces[:6], "val": traces[6:8], "test": traces[8:]}, {"param_ood": traces[:1]}) + return KernelSightConnector().convert(_files(root)), values + + +def _replace_arrays(path: Path, **arrays: np.ndarray) -> None: + # The file is one that the test wrote, so its object arrays are safe to unpickle. + with np.load(path, allow_pickle=True) as archive: + kept: dict[str, Any] = {name: archive[name] for name in archive.files} + kept.update(arrays) + np.savez(path, **kept) + + +def _found(annotations) -> set: + return {(a.key, read_value(a.value), a.span, a.occurrence_metadata.get("track")) for a in annotations} + + +def test_each_snapshot_becomes_a_record_and_a_classification_task(tmp_path): + dataset, values = _convert(tmp_path) + tasks = dataset.tasks_of(ClassificationTask) + assert len(tasks) == len(values) + assert {task.split for task in tasks} == {Split.TRAIN, Split.VALIDATION, Split.TEST} + for task in tasks: + (record,) = task.inputs + assert task.label_set == WORKLOAD_L1 + assert task.targets is not None + assert read_value(task.targets[0]) == _DOMINANT[str(record.metadata["path"])] + assert [source.name for source in record.sources] == [group for group, _ in CHANNEL_GROUPS] + signals = [signal for source in record.sources for signal in source.signals] + assert [signal.metadata["counter"] for signal in signals] == list(COUNTER_NAMES) + for row, signal in enumerate(signals): + np.testing.assert_array_equal(signal.to_numpy(), values[str(record.metadata["path"])][row]) + axis = signals[0].time_axis + assert isinstance(axis, RegularAxis) + assert axis.period_us == 1000 + + +def test_record_metadata_keeps_the_released_split_memberships(tmp_path): + dataset, _ = _convert(tmp_path) + by_path = { + task.inputs[0].metadata["path"]: task.inputs[0].metadata for task in dataset.tasks_of(ClassificationTask) + } + assert by_path[_gemm_path(0)] == { + "path": _gemm_path(0), + "motif": "cutlass_gemm", + "problem": "cutlass_gemm/m0", + "variant": "m0", + "official_split": "train", + "release_splits": ["train", "param_ood"], + } + + +def test_segments_and_multihot_only_runs_become_interval_annotations(tmp_path): + dataset, _ = _convert(tmp_path) + by_path = {task.inputs[0].metadata["path"]: task.inputs[0] for task in dataset.tasks_of(ClassificationTask)} + segment = TimeInterval.micros(100_000, 200_000) + loads = (TimeInterval.micros(120_000, 130_000), TimeInterval.micros(150_000, 160_000)) + assert _found(by_path[_gemm_path(0)].annotations) == { + ("workload_l1", "matmul", segment, None), + ("workload_l2", "matmul_gemm", segment, None), + *(("workload_l1", "memory_movement", span, MULTIHOT_TRACK) for span in loads), + *(("workload_l2", "memory_movement_copy", span, MULTIHOT_TRACK) for span in loads), + } + # A Snapshot without an overlap track has only its Segment. + assert len(by_path[_gemm_path(1)].annotations) == 2 + + +def test_a_real_snapshot_of_the_release_becomes_a_record(): + record = _snapshot(_REAL_PATH, "cutlass_ws_overlap").create_record(FIXTURES) + signals = [signal for source in record.sources for signal in source.signals] + assert len(signals) == len(COUNTER_NAMES) + with np.load(FIXTURES / _REAL_PATH, allow_pickle=False) as archive: + np.testing.assert_array_equal(signals[5].to_numpy(), archive["data"][5]) + axis = signals[0].time_axis + assert isinstance(axis, RegularAxis) + assert axis.period_us == pytest.approx(203.73275) + segment = TimeInterval.micros(49_914, 54_396) + loads = (TimeInterval.micros(49_914, 53_785), TimeInterval.micros(53_989, 54_396)) + assert _found(record.annotations) == { + ("workload_l1", "matmul", segment, None), + ("workload_l2", "matmul_gemm", segment, None), + *(("workload_l1", "memory_movement", span, MULTIHOT_TRACK) for span in loads), + *(("workload_l2", "memory_movement_copy", span, MULTIHOT_TRACK) for span in loads), + } + + +def test_variants_of_one_kernelbench_problem_share_a_partition(): + snapshots = [ + _snapshot( + f"kernels/kernelbench/_out/L1_P{problem}/dt-{dtype}_s42{INPUT_SUFFIX}", + "kernelbench", + "conv" if problem % 2 else "matmul", + ) + for problem in range(20) + for dtype in ("bf16", "fp16", "fp32") + ] + assert snapshots[0].problem == "kernelbench/L1_P0" + partition_of = {snapshot.path: split for split, members in assign_splits(snapshots).items() for snapshot in members} + for problem in range(20): + assert len({split for path, split in partition_of.items() if f"/L1_P{problem}/" in path}) == 1 + + +@pytest.mark.parametrize( + ("motif", "variant", "problem"), + [ + ("cutlass_gemm", "m1024_n1024_k8192_rH_s8_b1", "cutlass_gemm/m1024_n1024_k8192"), + ("cutlass_gemm", "m4096_rM", "cutlass_gemm/m4096"), + ("cutlass_gemm", "", "cutlass_gemm/m4096"), + ("cutlass_gemm", "m2048_rN_b1", "cutlass_gemm/m2048_n2048_k2048"), + ("cutlass_gemm", "kheavy", "cutlass_gemm/m1024_n1024_k8192"), + ("cutlass_gemm", "m8192_k1024", "cutlass_gemm/m8192_k1024"), + # cutlass_ws_overlap runs the kernel of cutlass_gemm, but a short name has the defaults of its own motif. + ("cutlass_ws_overlap", "m512_n512_k8192_rM_s1_b0", "cutlass_gemm/m512_n512_k8192"), + ("cutlass_ws_overlap", "m4096_rN", "cutlass_gemm/m4096_n4096_k2048"), + ("cutlass_ws_overlap", "m2048_sw4", "cutlass_gemm/m2048_n2048_k2048"), + ("cutlass_ws_overlap", "m1024_k2048", "cutlass_gemm/m1024_n1024_k2048"), + ("cutlass_ws_overlap", "", "cutlass_gemm/k2048"), + ("vector_add", "N16M_T1024", "vector_add/N16M"), + ("reduction", "N16M_T256_I4", "reduction/N16M_I4"), + ("scatter", "N1M_B16_T512", "scatter/N1M_B16"), + ("cutlass_fmha", "b2h16_q1024_d64_causal", "cutlass_fmha/b2h16_q1024_d64_causal"), + ], +) +def test_the_problem_of_a_variant_is_its_shape_without_launch_options(motif, variant, problem): + directory = f"{variant}/" if variant else "" + assert _snapshot(f"kernels/{motif}/_out/{directory}{INPUT_SUFFIX.lstrip('/')}", motif).problem == problem + + +@pytest.mark.parametrize( + ("official", "extra", "message"), + [ + ({"train": [_TRACE], "test": [_TRACE]}, {}, "more than one official partition"), + ({}, {"iid": [_TRACE]}, "in no official partition"), + ({"train": [{"path": _gemm_path(0)}]}, {}, r"train\.json is not a KernelSight split file"), + ({"train": [_trace(_gemm_path(0), "cutlass_fmha", "matmul")]}, {}, "is not under kernels/cutlass_fmha/_out/"), + ({"train": [_trace(_gemm_path(0), "cutlass_gemm", "gemm")]}, {}, "'gemm' is not an L1 class"), + ], +) +def test_a_malformed_split_index_raises(tmp_path, official, extra, message): + _write_splits(tmp_path, official, extra) + with pytest.raises(TimeNetConnectorError, match=message): + read_snapshots(tmp_path / "splits") + + +@pytest.mark.parametrize( + ("suffix", "arrays", "message"), + [ + (LABELS_SUFFIX, {"segment_ends": np.array([10, 20])}, "different lengths"), + (LABELS_SUFFIX, {"segment_ends": np.array([N_BINS + 1])}, "outside the 512 bins"), + (LABELS_SUFFIX, {"segment_label_l1": np.array([-1]), "segment_label_l2": np.array([-1])}, "out of range"), + (LABELS_SUFFIX, {"workload_l2_multihot": np.zeros((N_BINS, 3), np.uint8)}, "workload_l2_multihot has shape"), + (INPUT_SUFFIX, {"time_edges_ns": np.arange(N_BINS)}, "time_edges_ns has shape"), + (INPUT_SUFFIX, {"data": np.zeros((len(COUNTER_NAMES), N_BINS - 1), np.float32)}, "data has shape"), + ], +) +def test_a_malformed_array_raises(tmp_path, suffix, arrays, message): + _write_snapshot(tmp_path, _gemm_path(0), seed=0, segments=[(0, 10, _MATMUL, _GEMM)]) + _replace_arrays(tmp_path / (_gemm_path(0).removesuffix(INPUT_SUFFIX) + suffix), **arrays) + with pytest.raises(TimeNetConnectorError, match=message): + record = _snapshot(_gemm_path(0), "cutlass_gemm").create_record(tmp_path) + # The data shape is checked only when a value is read. + record.sources[0].signals[0].to_numpy() + + +def test_a_corrupt_labels_file_raises(tmp_path): + _write_snapshot(tmp_path, _gemm_path(0), seed=0, segments=[(0, 10, _MATMUL, _GEMM)]) + labels = tmp_path / _snapshot(_gemm_path(0), "cutlass_gemm").labels_path + labels.write_bytes(b"not a zip archive") + with pytest.raises(TimeNetConnectorError, match="cannot read segment_starts"): + _snapshot(_gemm_path(0), "cutlass_gemm").create_record(tmp_path) + + +def test_a_damaged_compressed_member_raises(tmp_path): + shutil.copytree(FIXTURES, tmp_path, dirs_exist_ok=True) + snapshot = _snapshot(_REAL_PATH, "cutlass_ws_overlap") + labels = tmp_path / snapshot.labels_path + with zipfile.ZipFile(labels) as archive: + offset = archive.getinfo("segment_starts.npy").header_offset + raw = bytearray(labels.read_bytes()) + name_length, extra_length = struct.unpack_from("