From c8b80255a6696f002a2b8dc59203e912a8c09a0c Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 13:20:33 +0100 Subject: [PATCH 1/8] feat(connectors): read the SWE-agent trajectories release Parse each run of nebius/SWE-agent-trajectories into its outcome and the per-step signals of its first 30 steps: thought, command and reply lengths, conversation length, repeated commands, command kind and reply kind. The build fails on an unknown exit status, a malformed instance id, a trajectory out of turn, or a shard with another row count. --- .../datasets/nebius/__init__.py | 1 + .../nebius/swe_agent_trajectories/source.py | 364 ++++++++++++++++++ 2 files changed, 365 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py new file mode 100644 index 000000000..db7d1a712 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py @@ -0,0 +1 @@ +"""Connectors for datasets published by the ``nebius`` org on the Hugging Face Hub.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py new file mode 100644 index 000000000..12689291a --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py @@ -0,0 +1,364 @@ +"""Read the SWE-agent trajectories release and turn each run's first steps into per-step signals. + +A run is one attempt of SWE-agent to resolve one GitHub issue. Its trajectory is a system prompt, the +issue message, then alternating agent and environment messages. One agent message and the +environment's reply to it make one step. The connector keeps the first :data:`MAX_STEPS` steps of +every run, because no checkpoint looks further, and the run's step count and outcome. The outcome +is never part of a step. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from enum import StrEnum, unique +from pathlib import Path +import re +import zlib + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from timenet.errors import TimeNetConnectorError + + +# The twelve parquet shards of the pinned revision and their row counts. +SHARD_ROWS = { + "train-00000-of-00012.parquet": 6670, + "train-00001-of-00012.parquet": 6670, + "train-00002-of-00012.parquet": 6670, + "train-00003-of-00012.parquet": 6670, + "train-00004-of-00012.parquet": 6670, + "train-00005-of-00012.parquet": 6670, + "train-00006-of-00012.parquet": 6670, + "train-00007-of-00012.parquet": 6670, + "train-00008-of-00012.parquet": 6669, + "train-00009-of-00012.parquet": 6669, + "train-00010-of-00012.parquet": 6669, + "train-00011-of-00012.parquet": 6669, +} +# The latest checkpoint is at 30 steps, so no step after the 30th is kept. +MAX_STEPS = 30 +# A trajectory opens with the system prompt and the issue message; the steps follow. +_OPENING = 2 +_COLUMNS = ["instance_id", "model_name", "target", "trajectory", "exit_status"] +_BATCH_ROWS = 256 +_ISSUE = re.compile(r"ISSUE:\n(.*?)\n\nINSTRUCTIONS:", re.DOTALL) +# An instance id is "__-". +_INSTANCE = re.compile(r"(?P.+__.+)-\d+") +# SWE-agent's ThoughtActionParser: an opening fence may name a language, a closing fence is bare. +_FENCE = re.compile(r"^```(\S*)\s*\n|^```\s*$", re.MULTILINE) + + +@unique +class Ending(StrEnum): + """How a run ended, grouped from the release's ``exit_status``.""" + + SUBMITTED = "submitted" + CONTEXT_LIMIT = "context_limit" + EARLY_EXIT = "early_exit" + SUBMITTED_NO_PATCH = "submitted_no_patch" + FORMAT_ERRORS = "format_errors" + COST_LIMIT = "cost_limit" + + +# Every exit_status of the release. "submitted (exit_x)" means SWE-agent hit limit x and submitted the +# changes so far by itself. +ENDINGS = { + "submitted": Ending.SUBMITTED, + "submitted (exit_context)": Ending.CONTEXT_LIMIT, + "exit_context": Ending.CONTEXT_LIMIT, + "early_exit": Ending.EARLY_EXIT, + "submitted_no_patch": Ending.SUBMITTED_NO_PATCH, + "submitted (exit_format)": Ending.FORMAT_ERRORS, + "exit_format": Ending.FORMAT_ERRORS, + "submitted (exit_cost)": Ending.COST_LIMIT, + "exit_cost": Ending.COST_LIMIT, +} + + +@unique +class ActionKind(StrEnum): + """What a step's command does, by its first word.""" + + EDIT = "edit" + CREATE = "create" + RUN = "run" + READ = "read" + SEARCH = "search" + SUBMIT = "submit" + OTHER = "other" + + +_ACTION_WORDS = { + "edit": ActionKind.EDIT, + "create": ActionKind.CREATE, + "python": ActionKind.RUN, + "python3": ActionKind.RUN, + "pytest": ActionKind.RUN, + "open": ActionKind.READ, + "goto": ActionKind.READ, + "scroll_down": ActionKind.READ, + "scroll_up": ActionKind.READ, + "cat": ActionKind.READ, + "search_dir": ActionKind.SEARCH, + "search_file": ActionKind.SEARCH, + "find_file": ActionKind.SEARCH, + "find": ActionKind.SEARCH, + "grep": ActionKind.SEARCH, + "ls": ActionKind.SEARCH, + "submit": ActionKind.SUBMIT, +} + + +@unique +class ObservationKind(StrEnum): + """What the environment's reply to a step shows, by the first rule of :func:`observation_kind` it meets.""" + + EDIT_REJECTED = "edit_rejected" + TIMEOUT = "timeout" + TRACEBACK = "traceback" + COMMAND_ERROR = "command_error" + SILENT_SUCCESS = "silent_success" + OUTPUT = "output" + + +_EDIT_REJECTED = "Your proposed edit has introduced new syntax error" +_TIMEOUT = "EXECUTION TIMED OUT" +_TRACEBACK = "Traceback (most recent call last)" +_SILENT_SUCCESS = "Your command ran successfully and did not produce any output" +_COMMAND_ERRORS = ("command not found", "No such file or directory", "syntax error near unexpected token") + + +def split_message(text: str) -> tuple[str, str]: + """Split an agent message into its thought and its command, as SWE-agent's ThoughtActionParser does. + + The command is the content of the last complete top-level fenced block. The thought is the rest of + the message. A message without a complete block is all thought, with an empty command. + + Returns: + The thought and the command, both stripped. + """ + stack: list[re.Match[str]] = [] + last: tuple[re.Match[str], re.Match[str]] | None = None + for match in _FENCE.finditer(text): + if stack and not match.group(1): + start = stack.pop() + if not stack: + last = (start, match) + elif match.group(1) is not None: + stack.append(match) + if last is None: + return text.strip(), "" + start, end = last + return (text[: start.start()] + text[end.end() :]).strip(), text[start.end() : end.start()].strip() + + +def action_kind(command: str) -> ActionKind: + """Classify a command by its first word. + + Returns: + The kind; ``other`` for an empty command or a word not in the table. + """ + words = command.split(maxsplit=1) + return _ACTION_WORDS.get(words[0], ActionKind.OTHER) if words else ActionKind.OTHER + + +def observation_kind(text: str) -> ObservationKind: + """Classify the environment's reply, without its leading whitespace, by the first of these rules that holds. + + 1. It starts with SWE-agent's message for an edit that its linter rejected. + 2. It starts with ``EXECUTION TIMED OUT``. + 3. It contains a Python traceback. + 4. Its first line starts with ``Usage:`` or ``Error:``, or names a shell error. + 5. It is SWE-agent's message for a command that printed nothing. + 6. Otherwise it is plain output. + + Returns: + The kind. + """ + # Some replies open with a blank line, e.g. the timeout message. + text = text.lstrip() + if text.startswith(_EDIT_REJECTED): + return ObservationKind.EDIT_REJECTED + if text.startswith(_TIMEOUT): + return ObservationKind.TIMEOUT + if _TRACEBACK in text: + return ObservationKind.TRACEBACK + first = text.split("\n", 1)[0] + if first.startswith(("Usage:", "Error:")) or any(error in first for error in _COMMAND_ERRORS): + return ObservationKind.COMMAND_ERROR + if text.startswith(_SILENT_SUCCESS): + return ObservationKind.SILENT_SUCCESS + return ObservationKind.OUTPUT + + +@dataclass(frozen=True) +class Steps: + """The per-step signals of a run's first steps, one entry per step.""" + + thought_chars: np.ndarray + action_chars: np.ndarray + observation_chars: np.ndarray + context_chars: np.ndarray + """The characters of the whole conversation after the step's reply, system prompt included.""" + repeated_action: np.ndarray + """Whether the step's command equals an earlier command of the run, character for character.""" + action_kind: tuple[ActionKind, ...] + observation_kind: tuple[ObservationKind, ...] + thoughts: bytes + """The thoughts, NUL-joined and compressed, because the release holds about 0.7 GB of them.""" + + def thought_texts(self, count: int) -> list[str]: + """The thoughts of the first ``count`` steps. + + Returns: + One string per step. + """ + return zlib.decompress(self.thoughts).decode().split("\0")[:count] + + +@dataclass(frozen=True) +class Run: + """One run of the release: its identity, its outcome, and the signals of its first steps.""" + + shard: str + row: int + instance_id: str + repository: str + model_name: str + resolved: bool + exit_status: str + issue: str + n_steps: int + """The run's step count. Its last step is the run's final action.""" + steps: Steps + + @property + def run_id(self) -> str: + """The run's place in the release: shard stem and row.""" + return f"{self.shard.removesuffix('.parquet')}:{self.row}" + + @property + def ending(self) -> Ending: + """How the run ended.""" + return ENDINGS[self.exit_status] + + +def _steps(messages: list[dict], opening: int, context: str) -> tuple[int, Steps]: + """Parse the agent and environment messages that follow the issue message. + + Args: + messages: The messages after the issue message. + opening: The characters the agent reads before its first step: system prompt and issue message. + context: The row, for error messages. + + Returns: + The run's step count and the signals of its first :data:`MAX_STEPS` steps. + + Raises: + TimeNetConnectorError: If the messages do not alternate agent, environment, ending on the agent. + """ + for index, message in enumerate(messages): + if message["role"] != ("ai" if index % 2 == 0 else "user"): + raise TimeNetConnectorError(f"{context}: message {index + 2} is {message['role']!r} out of turn") + if messages and messages[-1]["role"] != "ai": + raise TimeNetConnectorError(f"{context}: the trajectory does not end on an agent message") + n_steps = (len(messages) + 1) // 2 + kept = min(n_steps, MAX_STEPS) + numbers = np.zeros((4, kept), dtype=np.int32) + repeated = np.zeros(kept, dtype=bool) + actions, observations, thoughts, seen = [], [], [], set() + for step in range(kept): + thought, command = split_message(messages[2 * step]["text"] or "") + # The final step has no reply; every kept step of a checkpoint does. + reply = messages[2 * step + 1]["text"] or "" if 2 * step + 1 < len(messages) else "" + numbers[:3, step] = len(thought), len(command), len(reply) + numbers[3, step] = len(messages[2 * step]["text"] or "") + len(reply) + repeated[step] = command in seen + seen.add(command) + actions.append(action_kind(command)) + observations.append(observation_kind(reply)) + thoughts.append(thought.replace("\0", "")) + return n_steps, Steps( + thought_chars=numbers[0], + action_chars=numbers[1], + observation_chars=numbers[2], + context_chars=(opening + np.cumsum(numbers[3])).astype(np.int32), + repeated_action=repeated, + action_kind=tuple(actions), + observation_kind=tuple(observations), + thoughts=zlib.compress("\0".join(thoughts).encode()), + ) + + +def parse_run(row: dict, shard: str, index: int) -> Run: + """Check one release row and parse it into a :class:`Run`. + + Returns: + The run. + + Raises: + TimeNetConnectorError: If the row breaks the release's shape: an unknown exit status, a + malformed instance id, a trajectory that does not open with the system prompt and the + issue, or messages out of turn. + """ + context = f"{shard} row {index}" + instance = _INSTANCE.fullmatch(row["instance_id"] or "") + if instance is None: + raise TimeNetConnectorError(f"{context}: malformed instance_id {row['instance_id']!r}") + if row["exit_status"] not in ENDINGS: + raise TimeNetConnectorError(f"{context}: unknown exit_status {row['exit_status']!r}") + if not isinstance(row["target"], bool) or not row["model_name"]: + raise TimeNetConnectorError(f"{context}: target must be true or false and model_name set") + messages = row["trajectory"] or [] + if len(messages) < _OPENING or messages[0]["role"] != "system" or messages[1]["role"] != "user": + raise TimeNetConnectorError(f"{context}: the trajectory must open with the system prompt and the issue") + issue = _ISSUE.search(messages[1]["text"] or "") + if issue is None: + raise TimeNetConnectorError(f"{context}: the first message holds no ISSUE section") + opening = len(messages[0]["system_prompt"] or "") + len(messages[0]["text"] or "") + len(messages[1]["text"]) + n_steps, steps = _steps(messages[_OPENING:], opening, context) + return Run( + shard=shard, + row=index, + instance_id=row["instance_id"], + repository=instance["repository"], + model_name=row["model_name"], + resolved=row["target"], + exit_status=row["exit_status"], + issue=issue.group(1), + n_steps=n_steps, + steps=steps, + ) + + +def read_runs(root: Path) -> Iterator[Run]: + """Read every run of the release, shard by shard, in row order. + + Args: + root: The directory that holds the parquet shards. + + Yields: + Each run. + + Raises: + TimeNetConnectorError: If a shard is missing, unreadable, or holds a different number of rows + than the pinned revision. + """ + for shard, expected in SHARD_ROWS.items(): + path = root / shard + if not path.is_file(): + raise TimeNetConnectorError(f"{path} is missing") + count = 0 + try: + batches = pq.ParquetFile(path).iter_batches(batch_size=_BATCH_ROWS, columns=_COLUMNS) + for batch in batches: + for row in batch.to_pylist(): + yield parse_run(row, shard, count) + count += 1 + except (OSError, pa.ArrowException) as exc: + raise TimeNetConnectorError(f"{path}: {exc}") from exc + if count != expected: + raise TimeNetConnectorError(f"{path}: expected {expected} rows, found {count}") From d581b1ca0ff657e1b26b12594b90c7b0cab96a78 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 13:20:35 +0100 Subject: [PATCH 2/8] feat(connectors): add the nebius/swe_agent_trajectories connector and checkpoint tasks Cut each run after 5, 10, 20 and 30 steps while it is still going, so no input shows how the run ends. Each checkpoint is one Record of ordinal step signals with the agent's thoughts as text, the issue is a dataset annotation, and three streamed tasks ask whether the run will resolve its issue, how it will end, and for a rule-made status report. Runs are split by repository. --- .../nebius/swe_agent_trajectories/__init__.py | 6 + .../swe_agent_trajectories/connector.py | 219 ++++++++++++++++++ .../swe_agent_trajectories/dataset.yaml | 33 +++ .../swe_agent_trajectories/requirements.txt | 2 + .../nebius/swe_agent_trajectories/tasks.py | 146 ++++++++++++ 5 files changed, 406 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py new file mode 100644 index 000000000..b321c89bc --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py @@ -0,0 +1,6 @@ +"""The SWE-agent trajectories connector package: its connector implementation and dataset card.""" + +from timenet_connectors.datasets.nebius.swe_agent_trajectories.connector import ( + CONNECTOR as CONNECTOR, + SWEAgentTrajectoriesConnector as SWEAgentTrajectoriesConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py new file mode 100644 index 000000000..97c83f7fe --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py @@ -0,0 +1,219 @@ +"""The SWE-agent trajectories connector: 80,036 coding-agent runs, cut into checkpoints for early warning. + +Nebius released the trajectories of SWE-agent runs that try to resolve GitHub issues, each with +whether its final patch passed the tests of the issue's pull request. Each checkpoint of a run (its +first 5, 10, 20 or 30 steps, if the run is still going) becomes one Record of per-step signals, with +the agent's thoughts for those steps as text. The issue text is a dataset annotation, stored once per +distinct text. :mod:`~timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks` describes the +three tasks of each checkpoint. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping +import functools +from typing import ClassVar + +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import OrdinalAxis +from timenet.json import JsonMapping +from timenet.types import Annotation, AnswerTask, ClassificationTask, Task, TimeSeriesSpec, ureg +from timenet.types.splits import Split +from timenet.types.text import LazyTextList +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + ActionKind, + ObservationKind, + Run, + read_runs, +) +from timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks import ( + ENDING_PROMPT, + ENDINGS, + OUTCOMES, + RESOLVE_PROMPT, + RESOLVED, + STATUS_PROMPT, + UNRESOLVED, + Checkpoint, + checkpoints, + split_runs, + status, +) + + +AXIS = OrdinalAxis() +SOURCE_NAME = "agent_steps" +THOUGHTS = "agent_thoughts" +ISSUE = "issue" + + +def _count_spec(spec_type: str, name: str) -> TimeSeriesSpec: + return TimeSeriesSpec(spec_type=spec_type, name=name, unit_value=ureg.dimensionless, dtype="int32") + + +# One signal per step property, in Record order. Lengths count characters of the release's text. +SPECS = { + "thought_chars": _count_spec("agent_thought_length", "Agent thought length (characters)"), + "action_chars": _count_spec("agent_action_length", "Agent command length (characters)"), + "observation_chars": _count_spec("agent_observation_length", "Environment reply length (characters)"), + "context_chars": _count_spec("agent_context_length", "Conversation length so far (characters)"), + "repeated_action": TimeSeriesSpec( + spec_type="agent_repeated_action", name="Command repeats an earlier one", unit_value=None, dtype="bool" + ), + "action_kind": TimeSeriesSpec( + spec_type="agent_action_kind", + name="Agent command kind", + unit_value=ureg.dimensionless, + dtype="enum", + categories=tuple(ActionKind), + ), + "observation_kind": TimeSeriesSpec( + spec_type="agent_observation_kind", + name="Environment reply kind", + unit_value=ureg.dimensionless, + dtype="enum", + categories=tuple(ObservationKind), + ), +} + + +def _values(run: Run, name: str, steps: int) -> pa.Array: + """The first ``steps`` values of one step signal of a run.""" + values = getattr(run.steps, name)[:steps] + if name in {"action_kind", "observation_kind"}: + # An enum series loads as dictionary-encoded labels. + return pa.array([str(value) for value in values], type=pa.string()).dictionary_encode() + return pa.array(values) + + +class SWEAgentTrajectoriesConnector(BaseHuggingFaceConnector): + """Connector for nebius/SWE-agent-trajectories on the Hugging Face Hub.""" + + HF_REPO = "nebius/SWE-agent-trajectories" + SOURCE_REVISION = "68195a1450865274106246d0d0296a1d6807b88e" + SOURCE_FILES: ClassVar[Mapping[str, str]] = {"train": "data/train-*.parquet"} + + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Read every run, then build one Record per checkpoint and stream three tasks per Record. + + Every shard is read once, which checks every row before anything is written. Each run keeps + only the signals of its first 30 steps and its thoughts, compressed, so the loaders and the + task stream need no second read. + + Returns: + The populated :class:`~timenet.dataset.TimeFDataset`. + """ + runs = list(read_runs(self.ref_paths(raw_refs, "train")["train"])) + dataset = TimeFDataset(metadata=self.metadata()) + for key, value in (("source_repository", self.HF_REPO), ("source_revision", self.SOURCE_REVISION)): + dataset.annotate(Annotation(key=key, value=value)) + issues: dict[str, Annotation] = {} + for run in runs: + if run.issue not in issues: + issues[run.issue] = dataset.annotate( + Annotation( + key=ISSUE, + value=run.issue, + description="The GitHub issue a run tries to resolve, as the agent read it.", + metadata={"instance_id": run.instance_id}, + ) + ) + split_of = split_runs(runs) + records: list[tuple[Checkpoint, Record]] = [] + for run in runs: + for checkpoint in checkpoints(run): + records.append((checkpoint, dataset.add_record(record=self._make_record(checkpoint)))) + dataset.set_task_stream( + [ClassificationTask, AnswerTask], functools.partial(_tasks, records, issues, split_of.__getitem__) + ) + return dataset + + @staticmethod + def _make_record(checkpoint: Checkpoint) -> Record: + """Build one checkpoint's Record: a signal per step property, and the thoughts as text. + + Returns: + The Record. It holds nothing about the run after the checkpoint. + """ + run, steps = checkpoint.run, checkpoint.steps + signals = tuple( + Signal.from_loader( + spec=spec, + time_axis=AXIS, + name=name, + loader=functools.partial(_values, run, name, steps), + n_values=steps, + ) + for name, spec in SPECS.items() + ) + record = Record( + sources=(Source(name=SOURCE_NAME, signals=signals),), + metadata={ + "run_id": run.run_id, + "instance_id": run.instance_id, + "repository": run.repository, + "model_name": run.model_name, + "checkpoint_steps": steps, + }, + ) + record.annotate( + Annotation( + key=THOUGHTS, + value=LazyTextList(functools.partial(run.steps.thought_texts, steps), length=steps), + description="The agent's thought at each step, in step order: its message without the command.", + ) + ) + return record + + +def _tasks( + records: list[tuple[Checkpoint, Record]], issues: dict[str, Annotation], split_of: Callable[[str], Split] +) -> Iterator[Task]: + """Yield the three tasks of every checkpoint, in Record order. + + Yields: + The outcome and ending classifications, then the status report, of each checkpoint. + """ + for checkpoint, record in records: + run = checkpoint.run + context = (issues[run.issue], *(a for a in record.annotations if a.key == THOUGHTS)) + metadata: JsonMapping = { + "run_id": run.run_id, + "instance_id": run.instance_id, + "repository": run.repository, + "model_name": run.model_name, + "checkpoint_steps": checkpoint.steps, + } + inputs, split = (record,), split_of(run.run_id) + yield ClassificationTask( + inputs=inputs, + split=split, + input_annotations=context, + metadata=metadata, + prompt=RESOLVE_PROMPT, + targets=(RESOLVED if run.resolved else UNRESOLVED,), + label_set=OUTCOMES, + ) + yield ClassificationTask( + inputs=inputs, + split=split, + input_annotations=context, + metadata=metadata, + prompt=ENDING_PROMPT, + targets=(str(run.ending),), + label_set=ENDINGS, + ) + yield AnswerTask( + inputs=inputs, + split=split, + input_annotations=context, + metadata=metadata, + prompt=STATUS_PROMPT, + targets=(status(checkpoint),), + ) + + +CONNECTOR = SWEAgentTrajectoriesConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml new file mode 100644 index 000000000..36d151ca7 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml @@ -0,0 +1,33 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# Each Record is one checkpoint of one run: the run's first 5, 10, 20 or 30 steps, kept only if the run +# goes on after them. It holds seven per-step signals and the agent's thoughts for those steps. No input +# holds the run's outcome, its ending or its length. The split groups runs by repository. +yaml_schema_version: 1 +dataset_id: nebius/swe_agent_trajectories +dataset_version: 1.0.0 +name: SWE-agent trajectories (early warning) +description: >- + 80,036 SWE-agent runs that try to resolve real GitHub issues, released by Nebius, each labelled + with whether its final patch passed the tests of the issue's pull request. Each run is cut into + checkpoints after 5, 10, 20 and 30 steps while it is still going: 192,428 Records of per-step + signals (thought, command and reply lengths, conversation length, repeated commands, command + kind, reply kind) with the agent's thoughts as text and the issue as context. Each checkpoint asks + whether the run will resolve its issue, how it will end, and for a rule-made status report. +license: CC-BY-4.0 +source_url: https://huggingface.co/datasets/nebius/SWE-agent-trajectories +citation: >- + Nebius. SWE-agent trajectories (nebius/SWE-agent-trajectories), Hugging Face dataset, 2024, + revision 68195a1450865274106246d0d0296a1d6807b88e. + https://huggingface.co/datasets/nebius/SWE-agent-trajectories +access: open +domains: + - observability +tags: + - software-engineering + - coding-agents + - agent-monitoring + - early-warning + - llm-trajectories + - failure-prediction + - explanation + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt new file mode 100644 index 000000000..fe57efebd --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt @@ -0,0 +1,2 @@ +# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class. +huggingface_hub>=1.1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py new file mode 100644 index 000000000..d01ff8c44 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py @@ -0,0 +1,146 @@ +"""Cut each run at fixed checkpoints and ask, from the steps so far, how the run will turn out. + +A checkpoint is the first ``k`` steps of a run that is still going after them: the run has more than +``k`` steps, so its final action is never among them. How a run ends (a submission, the context limit, +its length) would give its outcome away, and a monitor never sees it before the end. Each checkpoint +gives three tasks: will the run resolve its issue, how will it end, and a short status report. The +answers come from the release's ``target`` and ``exit_status`` and the steps' signals by fixed rules. +No model writes them. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass + +from timenet.types import Label, LabelSet +from timenet.types.splits import Split +from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + ActionKind, + Ending, + ObservationKind, + Run, +) +from timenet_connectors.splitting import StratifiedSplitter + + +CHECKPOINTS = (5, 10, 20, 30) + +RESOLVED = "resolved" +UNRESOLVED = "unresolved" +OUTCOMES = LabelSet( + name="run_outcome", + labels=( + Label(name=RESOLVED, description="The run's final patch passed the tests of the issue's pull request."), + Label(name=UNRESOLVED, description="The run did not resolve the issue."), + ), +) +ENDINGS = LabelSet( + name="run_ending", + labels=( + Label(name=Ending.SUBMITTED, description="The agent submitted its changes."), + Label( + name=Ending.CONTEXT_LIMIT, + description="The run reached the model's context limit; SWE-agent may have submitted the changes so far.", + ), + Label(name=Ending.EARLY_EXIT, description="The release's early_exit status."), + Label(name=Ending.SUBMITTED_NO_PATCH, description="The agent submitted without a patch."), + Label( + name=Ending.FORMAT_ERRORS, + description="SWE-agent stopped the run after repeated malformed agent messages.", + ), + Label(name=Ending.COST_LIMIT, description="The run reached its cost limit."), + ), +) + +_CONTEXT = ( + "These are the first steps of a software engineering agent (SWE-agent) that is trying to resolve " + "the GitHub issue above. The run is still going." +) +RESOLVE_PROMPT = f"{_CONTEXT} Will the agent's final patch resolve the issue?" +ENDING_PROMPT = f"{_CONTEXT} How will the run end?" +STATUS_PROMPT = ( + f"{_CONTEXT} Report how the agent is doing so far, then say how the run will end and whether it " + "will resolve the issue." +) + +_ENDING_TEXT = { + "submitted": "the agent submitted its changes", + "submitted (exit_context)": "it reached the context limit and SWE-agent submitted its changes", + "exit_context": "it reached the context limit", + "early_exit": "it ended with the status early_exit", + "submitted_no_patch": "the agent submitted without a patch", + "submitted (exit_format)": "SWE-agent stopped it after repeated malformed messages and submitted its changes", + "exit_format": "SWE-agent stopped it after repeated malformed messages", + "submitted (exit_cost)": "it reached its cost limit and SWE-agent submitted its changes", + "exit_cost": "it reached its cost limit", +} + + +@dataclass(frozen=True) +class Checkpoint: + """The first ``steps`` steps of one run.""" + + run: Run + steps: int + + +def checkpoints(run: Run) -> list[Checkpoint]: + """List the checkpoints of a run: every ``k`` of :data:`CHECKPOINTS` that the run outlasts. + + Returns: + The checkpoints, earliest first. + """ + return [Checkpoint(run, k) for k in CHECKPOINTS if run.n_steps > k] + + +def split_runs(runs: list[Run]) -> dict[str, Split]: + """Assign each run to a partition by its repository, so no repository is in two partitions. + + A repository's runs share its code, and an issue's runs share the issue, so a model could + otherwise recognise a test run's repository or issue from training. + + Returns: + The partition of each run, keyed by run id. + """ + splitter = StratifiedSplitter( + train_folds=8, + validation_folds=1, + test_folds=1, + stratify_by=lambda run: run.resolved, + group_by=lambda run: run.repository, + seed=42, + ) + return {run.run_id: split for split, members in splitter.split(runs).items() for run in members} + + +def _count(number: int, singular: str, plural: str | None = None) -> str: + return f"{number} {singular if number == 1 else plural or singular + 's'}" + + +def status(checkpoint: Checkpoint) -> str: + """Write the status report of a checkpoint from its steps' signals and the run's outcome. + + Returns: + Three sentences on the steps so far, then one on how the run ended and whether it resolved + the issue. + """ + k, run = checkpoint.steps, checkpoint.run + actions = Counter(run.steps.action_kind[:k]) + replies = Counter(run.steps.observation_kind[:k]) + repeats = int(run.steps.repeated_action[:k].sum()) + looked = actions[ActionKind.READ] + actions[ActionKind.SEARCH] + rest = run.n_steps - k + return ( + f"In its first {k} steps the agent edited files {_count(actions[ActionKind.EDIT], 'time')}, " + f"created {_count(actions[ActionKind.CREATE], 'file')}, ran code {_count(actions[ActionKind.RUN], 'time')}, " + f"read or searched the repository {_count(looked, 'time')} " + f"and repeated an earlier command {_count(repeats, 'time')}. " + f"The linter rejected {_count(replies[ObservationKind.EDIT_REJECTED], 'edit')}, " + f"{_count(replies[ObservationKind.TRACEBACK], 'reply', 'replies')} showed a Python traceback, " + f"{_count(replies[ObservationKind.COMMAND_ERROR], 'reply', 'replies')} a command error " + f"and {_count(replies[ObservationKind.TIMEOUT], 'command')} timed out. " + f"The conversation holds {int(run.steps.context_chars[k - 1]):,} characters. " + f"The run went on for {_count(rest, 'more step')}, then {_ENDING_TEXT[run.exit_status]}, " + f"and {'resolved' if run.resolved else 'did not resolve'} the issue." + ) From f0f4959104c96853ce66d8f4285a3829fb03af57 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 13:20:36 +0100 Subject: [PATCH 3/8] test(connectors): cover nebius/swe_agent_trajectories on a synthetic shard --- .../swe_agent_trajectories/tests/__init__.py | 0 .../tests/test_connector.py | 270 ++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/__init__.py create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py new file mode 100644 index 000000000..bfe2c09ca --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py @@ -0,0 +1,270 @@ +from collections import Counter +from itertools import pairwise +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, open_local_version +from timenet.types import AnswerTask, ClassificationTask +from timenet.types.text import read_value +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.datasets.nebius.swe_agent_trajectories import SWEAgentTrajectoriesConnector, source +from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + ActionKind, + ObservationKind, + action_kind, + observation_kind, + parse_run, + read_runs, + split_message, +) +from timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks import checkpoints, status + + +# A synthetic release shaped like the Hub parquet: one shard of runs over twelve repositories, so the +# split has enough groups for its ten folds. The texts are hand-made, so the repo ships no dataset bytes. +SHARD = "train-00000-of-00012.parquet" +SYSTEM = "SETTING: You are an autonomous programmer." +ISSUE = "Parser crashes on empty input\nCalling parse('') raises IndexError." + + +def _message(role: str, text: str, system_prompt: str | None = None) -> dict: + return {"cutoff_date": None, "mask": role == "ai", "role": role, "system_prompt": system_prompt, "text": text} + + +def _agent(thought: str, command: str) -> str: + return f"{thought}\n\n```\n{command}\n```" + + +# One cycle of steps; a run of n steps repeats it and ends on a submit. +CYCLE = ( + (_agent("Let's reproduce the bug first.", "create reproduce.py"), "[File: /repo/reproduce.py (1 lines total)]"), + ( + _agent("Write the script.", "edit 1:1\nprint(parse(''))\nend_of_edit"), + "Your proposed edit has introduced new syntax error(s).", + ), + (_agent("Run it.", "python reproduce.py"), "Traceback (most recent call last):\n File ...\nIndexError"), + (_agent("Look at the parser.", "open src/parser.py"), "[File: /repo/src/parser.py (120 lines total)]"), + (_agent("Search for the call.", "search_dir parse"), 'Found 3 matches for "parse"'), +) + + +def _trajectory(n_steps: int) -> list[dict]: + messages = [_message("system", "", SYSTEM), _message("user", f"ISSUE:\n{ISSUE}\n\nINSTRUCTIONS:\nFix it.")] + for step in range(n_steps - 1): + agent, reply = CYCLE[step % len(CYCLE)] + messages += [_message("ai", agent), _message("user", reply)] + return [*messages, _message("ai", _agent("Done.", "submit"))] + + +def _row(repository: int, number: int, n_steps: int, *, resolved: bool, exit_status: str = "submitted") -> dict: + return { + "instance_id": f"owner{repository}__repo{repository}-{number}", + "model_name": "swe-agent-llama-70b", + "target": resolved, + "trajectory": _trajectory(n_steps), + "exit_status": exit_status, + "generated_patch": "", + "eval_logs": "", + } + + +def _rows() -> list[dict]: + rows = [] + for repository in range(12): + rows.append(_row(repository, 1, 4, resolved=True)) + rows.append(_row(repository, 1, 12, resolved=repository % 3 == 0)) + rows.append(_row(repository, 2, 33, resolved=False, exit_status="submitted (exit_context)")) + return rows + + +def _write(root: Path, rows: list[dict]) -> Path: + root.mkdir(parents=True, exist_ok=True) + pq.write_table(pa.Table.from_pylist(rows), root / SHARD) + return root + + +@pytest.fixture +def one_shard(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + def release(rows: list[dict]) -> Path: + monkeypatch.setattr(source, "SHARD_ROWS", {SHARD: len(rows)}) + return _write(tmp_path / "release", rows) + + return release + + +@pytest.fixture +def dataset(one_shard): + return SWEAgentTrajectoriesConnector().convert([HubFileRef(one_shard(_rows()), "train")]) + + +def _target(task) -> str: + assert task.targets is not None + return read_value(task.targets[0]) + + +def _thoughts(record) -> list[str]: + (annotation,) = record.annotations + assert annotation.key == "agent_thoughts" + value = read_value(annotation.value) + assert isinstance(value, list) + return value + + +def test_split_message_takes_the_last_fenced_block_as_the_command() -> None: + text = "Here is code:\n```python\nx = 1\n```\nNow run it.\n```\npython x.py\n```\n" + assert split_message(text) == ("Here is code:\n```python\nx = 1\n```\nNow run it.", "python x.py") + assert split_message("Only thinking.") == ("Only thinking.", "") + + +@pytest.mark.parametrize( + ("command", "kind"), + [ + ("edit 1:2\nx\nend_of_edit", ActionKind.EDIT), + ("create a.py", ActionKind.CREATE), + ("python a.py", ActionKind.RUN), + ("goto 40", ActionKind.READ), + ("find_file a.py", ActionKind.SEARCH), + ("submit", ActionKind.SUBMIT), + ("git diff", ActionKind.OTHER), + ("", ActionKind.OTHER), + ], +) +def test_action_kind_reads_the_first_word(command: str, kind: ActionKind) -> None: + assert action_kind(command) == kind + + +@pytest.mark.parametrize( + ("reply", "kind"), + [ + ("Your proposed edit has introduced new syntax error(s). Traceback (most recent call last)", "edit_rejected"), + ("\nEXECUTION TIMED OUT", "timeout"), + ("Running...\nTraceback (most recent call last):\n", "traceback"), + ("Usage: open ", "command_error"), + ("/bin/bash: line 1: foo: command not found", "command_error"), + ("Your command ran successfully and did not produce any output.", "silent_success"), + ("[File: /repo/a.py (10 lines total)]", "output"), + ], +) +def test_observation_kind_applies_its_rules_in_order(reply: str, kind: str) -> None: + assert observation_kind(reply) == ObservationKind(kind) + + +def test_a_run_keeps_its_first_steps_and_its_outcome_apart() -> None: + run = parse_run(_row(0, 1, 7, resolved=False), SHARD, 0) + assert run.n_steps == 7 + assert run.issue == ISSUE + assert run.repository == "owner0__repo0" + assert list(run.steps.action_kind) == ["create", "edit", "run", "read", "search", "create", "submit"] + assert list(run.steps.observation_kind[:3]) == ["output", "edit_rejected", "traceback"] + # The final step has no reply. + assert run.steps.observation_chars[-1] == 0 + assert list(run.steps.repeated_action) == [False, False, False, False, False, True, False] + opening = len(SYSTEM) + len(f"ISSUE:\n{ISSUE}\n\nINSTRUCTIONS:\nFix it.") + first = len(CYCLE[0][0]) + len(CYCLE[0][1]) + assert run.steps.context_chars[0] == opening + first + assert all(later > earlier for earlier, later in pairwise(run.steps.context_chars)) + assert run.steps.thought_texts(2) == ["Let's reproduce the bug first.", "Write the script."] + + +def test_only_a_run_that_goes_on_gets_a_checkpoint() -> None: + assert [c.steps for c in checkpoints(parse_run(_row(0, 1, 5, resolved=False), SHARD, 0))] == [] + assert [c.steps for c in checkpoints(parse_run(_row(0, 1, 6, resolved=False), SHARD, 0))] == [5] + assert [c.steps for c in checkpoints(parse_run(_row(0, 1, 33, resolved=False), SHARD, 0))] == [5, 10, 20, 30] + + +def test_records_hold_the_steps_so_far_and_nothing_about_the_outcome(dataset) -> None: + records = list(dataset.records) + # Per repository: the 4-step run gives none, the 12-step run 2, the 33-step run 4. + assert len(records) == 12 * 6 + for record in records: + k = record.metadata["checkpoint_steps"] + assert set(record.metadata) == {"run_id", "instance_id", "repository", "model_name", "checkpoint_steps"} + assert {signal.name for signal in record.sources[0].signals} == set(source.Steps.__annotations__) - {"thoughts"} + assert all(signal.n_values == k for signal in record.sources[0].signals) + assert len(_thoughts(record)) == k + # The last visible step is never the run's submit. + kinds = next(s for s in record.sources[0].signals if s.name == "action_kind") + assert "submit" not in kinds.to_arrow().to_pylist() + + +def test_each_checkpoint_gives_three_tasks_with_the_issue_and_thoughts_as_input(dataset) -> None: + tasks = list(dataset.iter_tasks()) + assert len(tasks) == 3 * 12 * 6 + assert Counter(type(task) for task in tasks) == {ClassificationTask: 2 * 72, AnswerTask: 72} + issues = [a for a in dataset.annotations if a.key == "issue"] + assert len(issues) == 1 + for task in tasks: + assert [a.key for a in task.input_annotations] == ["issue", "agent_thoughts"] + assert task.input_annotations[0] is issues[0] + endings = { + read_value(t.targets[0]) + for t in tasks + if isinstance(t, ClassificationTask) and t.label_set.name == "run_ending" + } + assert endings == {"submitted", "context_limit"} + + +def test_the_status_report_states_the_steps_so_far_and_the_outcome() -> None: + run = parse_run(_row(0, 1, 12, resolved=True), SHARD, 0) + (first, *_) = checkpoints(run) + assert status(first) == ( + "In its first 5 steps the agent edited files 1 time, created 1 file, ran code 1 time, " + "read or searched the repository 2 times and repeated an earlier command 0 times. " + "The linter rejected 1 edit, 1 reply showed a Python traceback, 0 replies a command error " + f"and 0 commands timed out. The conversation holds {int(run.steps.context_chars[4]):,} characters. " + "The run went on for 7 more steps, then the agent submitted its changes, and resolved the issue." + ) + + +def test_no_repository_is_in_two_partitions(dataset) -> None: + splits: dict[str, set] = {} + for task in dataset.iter_tasks(): + splits.setdefault(task.metadata["repository"], set()).add(task.split) + assert all(len(found) == 1 for found in splits.values()) + assert len({split for found in splits.values() for split in found}) == 3 + + +def test_dataset_survives_storage_round_trip(dataset, tmp_path: Path) -> None: + registry = LocalRegistry(tmp_path / "registry") + registry.put_dataset(dataset) + with TimeFReader(open_local_version(registry.root / "nebius/swe_agent_trajectories/1.0.0")) as reader: + restored = reader.read() + record = next(iter(restored.records)) + signals = {signal.name: signal for signal in record.sources[0].signals} + assert signals["action_kind"].to_arrow().to_pylist() == ["create", "edit", "run", "read", "search"] + assert signals["observation_kind"].to_arrow().to_pylist()[1] == "edit_rejected" + assert _thoughts(record)[0] == "Let's reproduce the bug first." + restored_tasks = list(restored.iter_tasks()) + original = list(dataset.iter_tasks()) + assert [t.split for t in restored_tasks] == [t.split for t in original] + assert [_target(t) for t in restored_tasks] == [_target(t) for t in original] + + +@pytest.mark.parametrize( + ("change", "message"), + [ + (lambda row: row.update(exit_status="exit_unknown"), "unknown exit_status"), + (lambda row: row.update(instance_id="no-separator"), "malformed instance_id"), + (lambda row: row["trajectory"][1].update(text="no issue here"), "no ISSUE section"), + (lambda row: row["trajectory"][3].update(role="ai"), "out of turn"), + (lambda row: row["trajectory"].pop(), "does not end on an agent message"), + (lambda row: row["trajectory"].pop(0), "must open with the system prompt"), + ], +) +def test_a_malformed_row_fails_the_build(change, message: str) -> None: + row = _row(0, 1, 6, resolved=False) + change(row) + with pytest.raises(TimeNetConnectorError, match=message): + parse_run(row, SHARD, 0) + + +def test_a_shard_with_another_row_count_fails_the_build(one_shard, monkeypatch: pytest.MonkeyPatch) -> None: + root = one_shard(_rows()) + monkeypatch.setattr(source, "SHARD_ROWS", {SHARD: 1}) + with pytest.raises(TimeNetConnectorError, match="expected 1 rows"): + list(read_runs(root)) From 2d26e7320dbdbb1f7765855132038a32f4e2c259 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 13:20:38 +0100 Subject: [PATCH 4/8] docs(connectors): document the SWE-agent trajectories release and connector decisions --- .../nebius/swe_agent_trajectories/README.md | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md new file mode 100644 index 000000000..cfea9debc --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md @@ -0,0 +1,220 @@ +# SWE-agent trajectories: the release, and the decisions of this connector + +Nebius released 80,036 runs of SWE-agent, a coding agent, each trying to resolve one real GitHub +issue. Each run is labelled with whether its final patch passed the tests of the issue's pull request. + +This connector turns the runs into an **early-warning** dataset: a model sees the first steps of a run +that is still going and says how the run will turn out, like a monitor watching an agent at work. + +This document covers: +- what the release holds; +- how the connector maps it onto TimeF; +- why no input shows how a run ends; +- what a user should know before training or evaluating on it. + +*(measured)* marks a number taken from the files, not from the dataset page. + +## Source, licence and citation + +**Source:** the Hugging Face dataset [nebius/SWE-agent-trajectories](https://huggingface.co/datasets/nebius/SWE-agent-trajectories), pinned at revision `68195a1450865274106246d0d0296a1d6807b88e` (last modified 2024-12-23). +- 12 parquet shards, 1.1 GB to download, one `train` split of 80,036 rows *(measured)*. +- The connector pins the revision and each shard's row count, so a changed release fails the build. + +**Licence:** CC BY 4.0, as stated on the dataset page. The page adds two conditions that this card's +single licence field cannot hold: +- the repositories behind the issues keep their own licences, listed in nebius/SWE-bench-extra; +- using the model outputs requires complying with the Llama 3.1 License, because Llama models wrote them. + +**Citation:** the release has no citation entry. Cite the dataset page and the pinned revision: +> Nebius. SWE-agent trajectories (nebius/SWE-agent-trajectories). Hugging Face, 2024. Revision +> 68195a1450865274106246d0d0296a1d6807b88e. + +## What a run is + +Each row is one run of one model on one issue: +- `instance_id`: `__-`; +- `model_name`: `swe-agent-llama-70b` (74,792 runs), `-8b` (4,053) or `-405b` (1,191) *(measured)*; +- `target`: whether the run resolved the issue: 13,389 true, 66,647 false *(measured)*; +- `exit_status`: how the run ended; +- `trajectory`: the messages; +- `generated_patch` and `eval_logs`: the final patch and its test log. **The connector reads neither.** + +**The trajectory** is a system prompt, the issue message, then alternating agent and environment +messages, ending on an agent message. Every run follows this shape *(measured)*. +- **A step** is one agent message and the environment's reply to it. +- **An agent message** is a thought followed by one command in a fenced block. +- **The connector splits them** as SWE-agent's `ThoughtActionParser` does: the command is the last complete fenced block, and the thought is the rest. + +| | Value *(measured)* | +| --- | --- | +| Steps per run | median 17, 90th percentile 50, maximum 408; one run has none | +| Repositories, issues | 1,276 repositories, 3,591 issues; 807 issues have both resolved and unresolved runs | +| Largest repositories | pydantic/pydantic 6,279 runs, iterative/dvc 5,646 | + +**How runs end** (`exit_status` *(measured)*), and the ending label the connector gives it: + +| `exit_status` | Runs | Ending label | +| --- | --- | --- | +| `submitted` | 51,087 | `submitted` | +| `submitted (exit_context)` | 21,026 | `context_limit` | +| `exit_context` | 3,568 | `context_limit` | +| `early_exit` | 3,176 | `early_exit` | +| `submitted_no_patch` | 1,066 | `submitted_no_patch` | +| `submitted (exit_format)`, `exit_format` | 80, 20 | `format_errors` | +| `submitted (exit_cost)`, `exit_cost` | 10, 3 | `cost_limit` | + +`submitted (exit_x)` means SWE-agent reached limit x and submitted the changes so far itself. The +release does not explain `early_exit`; the connector keeps the name. + +## Why the inputs never show the end of a run + +The end of a run gives its outcome away: +- **The final step:** every `submitted` run ends with the command `submit` *(measured)*. +- **Running out of context:** 36 % of unresolved runs stop at the context limit, against 5 % of resolved ones *(measured)*. +- **Length:** unresolved runs are longer. + +A model that sees whole runs can learn "long, or cut off, means failure". That is not monitoring, and +a live monitor never sees the end before it happens. + +**So the connector only cuts checkpoints.** A checkpoint is the first *k* steps of a run, for +*k* = 5, 10, 20 and 30, and only if the run has more than *k* steps. Its last visible step is +therefore never the run's final action. +- **What no input holds:** the outcome, the exit status, the run's length, the patch and the test log. +- **What the inputs do hold:** a growing conversation, which is real information a monitor has. The longest checkpoint ends at 30 steps. + +## Records + +Each checkpoint is **one Record**: 192,428 in all *(measured)*. + +| Checkpoint | Runs still going | Resolved among them | +| --- | --- | --- | +| 5 steps | 76,182 | 16.6 % | +| 10 steps | 59,708 | 13.8 % | +| 20 steps | 34,059 | 6.4 % | +| 30 steps | 22,479 | 4.2 % | + +**Signals.** The Record has one source, `agent_steps`, with seven signals of *k* values each, one per +step, on an ordinal axis. The steps have an order but no timestamps: the release records none. + +| Signal | dtype | Value at each step | +| --- | --- | --- | +| `thought_chars` | int32 | characters of the agent's thought | +| `action_chars` | int32 | characters of the command | +| `observation_chars` | int32 | characters of the environment's reply | +| `context_chars` | int32 | characters of the whole conversation after the reply, from the system prompt on | +| `repeated_action` | bool | the command equals an earlier command of the run, character for character | +| `action_kind` | enum | `edit`, `create`, `run`, `read`, `search`, `submit`, `other`, by the command's first word | +| `observation_kind` | enum | `edit_rejected`, `timeout`, `traceback`, `command_error`, `silent_success`, `output` | + +**`action_kind` rules**, by the command's first word: +- `edit`: `edit`; +- `create`: `create`; +- `run`: `python`, `python3`, `pytest`; +- `read`: `open`, `goto`, `scroll_down`, `scroll_up`, `cat`; +- `search`: `search_dir`, `search_file`, `find_file`, `find`, `grep`, `ls`; +- `submit`: `submit`; +- `other`: every other command, or none. + +**`observation_kind` rules.** The first of these that holds wins: +1. `edit_rejected`: the reply starts with SWE-agent's message for an edit its linter rejected ("Your proposed edit has introduced new syntax error"). +2. `timeout`: it starts with `EXECUTION TIMED OUT`. +3. `traceback`: it contains `Traceback (most recent call last)`. +4. `command_error`: its first line starts with `Usage:` or `Error:`, or contains `command not found`, `No such file or directory` or `syntax error near unexpected token`. +5. `silent_success`: it starts with SWE-agent's message for a command that printed nothing. +6. `output`: anything else. + +Over the first 30 steps of every run *(measured)*: +- **Replies:** plain output 1,030,976; linter rejections 179,147; tracebacks 114,145; silent successes 55,182; command errors 49,989; timeouts 6,235. +- **Repeats:** 28 % of steps repeat an earlier command. Rerunning a script counts as a repeat. + +**Text.** The agent's thoughts for the *k* steps are the record annotation `agent_thoughts`, a list +of *k* strings in step order. They are text, not a signal, because a text series has no tensor form +in `TimeFTorchDataset`. + +**Metadata** (provenance, never model input): `run_id` (shard and row), `instance_id`, `repository`, +`model_name`, `checkpoint_steps`. + +**The issue** is a dataset annotation with the key `issue`, stored once per distinct text, because +each issue has many runs. Its text is the issue message's `ISSUE:` section. 13 issues appear with +two slightly different texts *(measured)*; each text is its own annotation. + +## Tasks + +Each checkpoint gives three tasks. All take the Record as input and name the issue and the thoughts +in `input_annotations`. That makes 577,284 tasks *(measured)*. + +| Task | Target | Source of the answer | +| --- | --- | --- | +| `ClassificationTask` `run_outcome` | `resolved` or `unresolved` | `target` | +| `ClassificationTask` `run_ending` | the ending label (table above) | `exit_status` | +| `AnswerTask` | a status report: the steps so far, then how the run ended | the signals, `target`, `exit_status`, the step count | + +**How the text is made.** A fixed template fills in the numbers; no model writes it: + +> In its first {k} steps the agent edited files {n} times, created {n} files, ran code {n} times, +> read or searched the repository {n} times and repeated an earlier command {n} times. The linter +> rejected {n} edits, {n} replies showed a Python traceback, {n} replies a command error and {n} +> commands timed out. The conversation holds {context} characters. The run went on for {m} more +> steps, then {ending}, and {resolved | did not resolve} the issue. + +"Read or searched" adds the `read` and `search` steps. {ending} is one fixed phrase per `exit_status`, +e.g. "it reached the context limit and SWE-agent submitted its changes". + +**Example.** The first checkpoint of the release (run `train-00000-of-00012:0`, issue +`AnalogJ__lexicon-336`, 5 steps) has the target: + +> In its first 5 steps the agent edited files 0 times, created 0 files, ran code 0 times, read or +> searched the repository 4 times and repeated an earlier command 0 times. The linter rejected 0 +> edits, 1 reply showed a Python traceback, 1 reply a command error and 0 commands timed out. The +> conversation holds 12,857 characters. The run went on for 41 more steps, then it reached the +> context limit and SWE-agent submitted its changes, and did not resolve the issue. + +**Task metadata:** the Record's metadata fields. + +## The split + +**Runs are grouped by repository.** `StratifiedSplitter` (8/1/1 folds, stratified on `target`, +seed 42) assigns whole repositories, and each checkpoint takes its run's partition. +- **Why:** a repository's runs share its code, and an issue's runs share the issue. With 807 issues holding both resolved and unresolved runs, a random split would test on issues seen in training. +- **The release** has a single `train` split, so there is no official split to follow. + +**Result** *(measured)*: + +| Partition | Repositories | Runs with a checkpoint | Tasks | Resolved at 5 / 10 / 20 / 30 steps | +| --- | --- | --- | --- | --- | +| train | 993 | 60,985 | 464,748 | 16.6 / 13.7 / 6.2 / 3.9 % | +| validation | 139 | 7,557 | 55,029 | 16.8 / 14.6 / 6.8 / 4.4 % | +| test | 143 | 7,640 | 57,507 | 16.4 / 14.1 / 7.1 / 5.9 % | + +No repository and no issue is in two partitions. The checks count only the 1,275 repositories that +have a checkpoint. + +## Evaluating on it + +**Most runs fail.** A model that always answers `unresolved` is right 83 % of the time at 5 steps and +96 % at 30 steps. Accuracy therefore says little. We suggest: +- **Report each checkpoint separately.** The base rate changes with *k*, so pooled results mix four problems. +- **Use measures that do not reward the majority class:** AUROC, the precision-recall curve of `resolved` (its baseline is the resolved rate), balanced accuracy, and calibration (Brier score) for a health score. +- **Count each run once.** A long run gives up to four checkpoints, and long runs fail more often. When training, weight a checkpoint by 1 / (checkpoints of its run), or sample one checkpoint per run. +- **Compare against simple baselines:** always `unresolved`; the numeric signals alone; the thoughts alone. + +## Release details a user should know + +- **The labels depend on the model as much as the issue.** 93 % of runs come from one model (Llama 70B), and each issue is attempted 22 times on average *(measured)*. +- **The thoughts are long:** 5.8 kB per run over the first 30 steps, on average *(measured)*. +- **Reply lengths count the logged text.** That includes SWE-agent's status lines (open file, current directory). +- **`mask` and `cutoff_date`** are not explained by the release. The connector does not use them. + +## Limitations and intended use + +**Limitations:** +- One agent framework (SWE-agent) and one model family (Llama 3.1). +- The signals and the kinds are rule-made from the logged text. The rules are fixed and listed above, but they are not exhaustive: a shell command the table does not list is `other`. +- Status reports are templates, not expert assessments. +- The outcome is the release's test result, which may miss correct fixes or accept incomplete ones. + +**Intended use:** +- Training and evaluating time-series language models that monitor a coding agent while it works, and warn before a run fails. +- Studying which step patterns precede success or failure. + +*Contributed by Team RAMadan during the TimeNet hackathon, Round 1 (November 2026).* From fa2d068ae65f662cdf0f348dda00ffe9cbcd8e10 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 15:09:37 +0100 Subject: [PATCH 5/8] fix(connectors): validate SWE-agent rows and key issues by instance - Read the shards with the shared ParquetDataReader into a SourceRow model, and raise on a missing message text or a system message with text, instead of filling in empty strings. - Keep thoughts exactly as written; thought_chars now marks where each one ends. - Store one issue annotation per instance and text, only for instances with a checkpoint, so no task names another instance's issue. - Keep the exit statuses in one table, rename the label sets, and reuse the Record metadata for the tasks. - Ask the status report for the remaining steps, the ending and the outcome, and write its target in the same future tense. - Give repeated_action a dimensionless unit like the other signals. --- .../swe_agent_trajectories/connector.py | 53 +++---- .../nebius/swe_agent_trajectories/source.py | 148 ++++++++++++------ .../nebius/swe_agent_trajectories/tasks.py | 29 ++-- .../tests/test_connector.py | 52 ++++-- 4 files changed, 173 insertions(+), 109 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py index 97c83f7fe..352d574e0 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py @@ -4,7 +4,7 @@ whether its final patch passed the tests of the issue's pull request. Each checkpoint of a run (its first 5, 10, 20 or 30 steps, if the run is still going) becomes one Record of per-step signals, with the agent's thoughts for those steps as text. The issue text is a dataset annotation, stored once per -distinct text. :mod:`~timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks` describes the +instance. :mod:`~timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks` describes the three tasks of each checkpoint. """ @@ -18,7 +18,6 @@ from timenet.dataset import Record, Signal, Source, TimeFDataset from timenet.dataset.axis import OrdinalAxis -from timenet.json import JsonMapping from timenet.types import Annotation, AnswerTask, ClassificationTask, Task, TimeSeriesSpec, ureg from timenet.types.splits import Split from timenet.types.text import LazyTextList @@ -30,9 +29,9 @@ read_runs, ) from timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks import ( + ENDING_LABELS, ENDING_PROMPT, - ENDINGS, - OUTCOMES, + OUTCOME_LABELS, RESOLVE_PROMPT, RESOLVED, STATUS_PROMPT, @@ -61,7 +60,10 @@ def _count_spec(spec_type: str, name: str) -> TimeSeriesSpec: "observation_chars": _count_spec("agent_observation_length", "Environment reply length (characters)"), "context_chars": _count_spec("agent_context_length", "Conversation length so far (characters)"), "repeated_action": TimeSeriesSpec( - spec_type="agent_repeated_action", name="Command repeats an earlier one", unit_value=None, dtype="bool" + spec_type="agent_repeated_action", + name="Command repeats an earlier one", + unit_value=ureg.dimensionless, + dtype="bool", ), "action_kind": TimeSeriesSpec( spec_type="agent_action_kind", @@ -110,21 +112,21 @@ def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: dataset = TimeFDataset(metadata=self.metadata()) for key, value in (("source_repository", self.HF_REPO), ("source_revision", self.SOURCE_REVISION)): dataset.annotate(Annotation(key=key, value=value)) - issues: dict[str, Annotation] = {} - for run in runs: - if run.issue not in issues: - issues[run.issue] = dataset.annotate( - Annotation( - key=ISSUE, - value=run.issue, - description="The GitHub issue a run tries to resolve, as the agent read it.", - metadata={"instance_id": run.instance_id}, - ) - ) split_of = split_runs(runs) + # One issue annotation per instance and text, made when a task first needs it. + issues: dict[tuple[str, str], Annotation] = {} records: list[tuple[Checkpoint, Record]] = [] for run in runs: for checkpoint in checkpoints(run): + if (run.instance_id, run.issue) not in issues: + issues[run.instance_id, run.issue] = dataset.annotate( + Annotation( + key=ISSUE, + value=run.issue, + description="The GitHub issue a run tries to resolve, as the agent read it.", + metadata={"instance_id": run.instance_id}, + ) + ) records.append((checkpoint, dataset.add_record(record=self._make_record(checkpoint)))) dataset.set_task_stream( [ClassificationTask, AnswerTask], functools.partial(_tasks, records, issues, split_of.__getitem__) @@ -170,7 +172,9 @@ def _make_record(checkpoint: Checkpoint) -> Record: def _tasks( - records: list[tuple[Checkpoint, Record]], issues: dict[str, Annotation], split_of: Callable[[str], Split] + records: list[tuple[Checkpoint, Record]], + issues: dict[tuple[str, str], Annotation], + split_of: Callable[[str], Split], ) -> Iterator[Task]: """Yield the three tasks of every checkpoint, in Record order. @@ -179,15 +183,8 @@ def _tasks( """ for checkpoint, record in records: run = checkpoint.run - context = (issues[run.issue], *(a for a in record.annotations if a.key == THOUGHTS)) - metadata: JsonMapping = { - "run_id": run.run_id, - "instance_id": run.instance_id, - "repository": run.repository, - "model_name": run.model_name, - "checkpoint_steps": checkpoint.steps, - } - inputs, split = (record,), split_of(run.run_id) + context = (issues[run.instance_id, run.issue], *(a for a in record.annotations if a.key == THOUGHTS)) + inputs, split, metadata = (record,), split_of(run.run_id), record.metadata yield ClassificationTask( inputs=inputs, split=split, @@ -195,7 +192,7 @@ def _tasks( metadata=metadata, prompt=RESOLVE_PROMPT, targets=(RESOLVED if run.resolved else UNRESOLVED,), - label_set=OUTCOMES, + label_set=OUTCOME_LABELS, ) yield ClassificationTask( inputs=inputs, @@ -204,7 +201,7 @@ def _tasks( metadata=metadata, prompt=ENDING_PROMPT, targets=(str(run.ending),), - label_set=ENDINGS, + label_set=ENDING_LABELS, ) yield AnswerTask( inputs=inputs, diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py index 12689291a..58a445bcf 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py @@ -10,6 +10,7 @@ from __future__ import annotations from collections.abc import Iterator +import dataclasses from dataclasses import dataclass from enum import StrEnum, unique from pathlib import Path @@ -17,10 +18,10 @@ import zlib import numpy as np -import pyarrow as pa -import pyarrow.parquet as pq +from pydantic import ConfigDict, StrictBool from timenet.errors import TimeNetConnectorError +from timenet_connectors.sources.readers import DataRow, ParquetDataReader # The twelve parquet shards of the pinned revision and their row counts. @@ -43,7 +44,6 @@ # A trajectory opens with the system prompt and the issue message; the steps follow. _OPENING = 2 _COLUMNS = ["instance_id", "model_name", "target", "trajectory", "exit_status"] -_BATCH_ROWS = 256 _ISSUE = re.compile(r"ISSUE:\n(.*?)\n\nINSTRUCTIONS:", re.DOTALL) # An instance id is "__-". _INSTANCE = re.compile(r"(?P.+__.+)-\d+") @@ -63,18 +63,33 @@ class Ending(StrEnum): COST_LIMIT = "cost_limit" +@dataclass(frozen=True) +class ExitStatus: + """How the release's ``exit_status`` maps onto an ending, and how a status report says it.""" + + ending: Ending + phrase: str + """The end of the sentence "The run will go on for n more steps, then ...".""" + + # Every exit_status of the release. "submitted (exit_x)" means SWE-agent hit limit x and submitted the # changes so far by itself. -ENDINGS = { - "submitted": Ending.SUBMITTED, - "submitted (exit_context)": Ending.CONTEXT_LIMIT, - "exit_context": Ending.CONTEXT_LIMIT, - "early_exit": Ending.EARLY_EXIT, - "submitted_no_patch": Ending.SUBMITTED_NO_PATCH, - "submitted (exit_format)": Ending.FORMAT_ERRORS, - "exit_format": Ending.FORMAT_ERRORS, - "submitted (exit_cost)": Ending.COST_LIMIT, - "exit_cost": Ending.COST_LIMIT, +EXIT_STATUSES = { + "submitted": ExitStatus(Ending.SUBMITTED, "the agent will submit its changes"), + "submitted (exit_context)": ExitStatus( + Ending.CONTEXT_LIMIT, "it will reach the context limit and SWE-agent will submit its changes" + ), + "exit_context": ExitStatus(Ending.CONTEXT_LIMIT, "it will reach the context limit"), + "early_exit": ExitStatus(Ending.EARLY_EXIT, "it will end with the status early_exit"), + "submitted_no_patch": ExitStatus(Ending.SUBMITTED_NO_PATCH, "the agent will submit without a patch"), + "submitted (exit_format)": ExitStatus( + Ending.FORMAT_ERRORS, "SWE-agent will stop it after repeated malformed messages and submit its changes" + ), + "exit_format": ExitStatus(Ending.FORMAT_ERRORS, "SWE-agent will stop it after repeated malformed messages"), + "submitted (exit_cost)": ExitStatus( + Ending.COST_LIMIT, "it will reach its cost limit and SWE-agent will submit its changes" + ), + "exit_cost": ExitStatus(Ending.COST_LIMIT, "it will reach its cost limit"), } @@ -170,7 +185,7 @@ def observation_kind(text: str) -> ObservationKind: 1. It starts with SWE-agent's message for an edit that its linter rejected. 2. It starts with ``EXECUTION TIMED OUT``. - 3. It contains a Python traceback. + 3. It contains a Python traceback, also one shown inside a file the agent opened or edited. 4. Its first line starts with ``Usage:`` or ``Error:``, or names a shell error. 5. It is SWE-agent's message for a command that printed nothing. 6. Otherwise it is plain output. @@ -208,7 +223,8 @@ class Steps: action_kind: tuple[ActionKind, ...] observation_kind: tuple[ObservationKind, ...] thoughts: bytes - """The thoughts, NUL-joined and compressed, because the release holds about 0.7 GB of them.""" + """The thoughts, concatenated and compressed, because the release holds about 0.7 GB of them. + ``thought_chars`` holds where each one ends.""" def thought_texts(self, count: int) -> list[str]: """The thoughts of the first ``count`` steps. @@ -216,7 +232,9 @@ def thought_texts(self, count: int) -> list[str]: Returns: One string per step. """ - return zlib.decompress(self.thoughts).decode().split("\0")[:count] + text = zlib.decompress(self.thoughts).decode() + ends = np.cumsum(self.thought_chars[:count]) + return [text[end - length : end] for end, length in zip(ends, self.thought_chars[:count], strict=True)] @dataclass(frozen=True) @@ -243,10 +261,36 @@ def run_id(self) -> str: @property def ending(self) -> Ending: """How the run ended.""" - return ENDINGS[self.exit_status] + return EXIT_STATUSES[self.exit_status].ending + + +class Message(DataRow): + """One message of a trajectory, as the release stores it.""" + + model_config = ConfigDict(extra="ignore") + + role: str + text: str | None + system_prompt: str | None + + +class SourceRow(DataRow): + """The columns of a release row that the connector reads.""" + + instance_id: str + model_name: str + target: StrictBool + trajectory: list[Message] + exit_status: str + + +def _text(message: Message, index: int, context: str) -> str: + if message.text is None: + raise TimeNetConnectorError(f"{context}: message {index} has no text") + return message.text -def _steps(messages: list[dict], opening: int, context: str) -> tuple[int, Steps]: +def _steps(messages: list[Message], opening: int, context: str) -> tuple[int, Steps]: """Parse the agent and environment messages that follow the issue message. Args: @@ -261,26 +305,27 @@ def _steps(messages: list[dict], opening: int, context: str) -> tuple[int, Steps TimeNetConnectorError: If the messages do not alternate agent, environment, ending on the agent. """ for index, message in enumerate(messages): - if message["role"] != ("ai" if index % 2 == 0 else "user"): - raise TimeNetConnectorError(f"{context}: message {index + 2} is {message['role']!r} out of turn") - if messages and messages[-1]["role"] != "ai": + if message.role != ("ai" if index % 2 == 0 else "user"): + raise TimeNetConnectorError(f"{context}: message {index + _OPENING} is {message.role!r} out of turn") + if messages and messages[-1].role != "ai": raise TimeNetConnectorError(f"{context}: the trajectory does not end on an agent message") + texts = [_text(message, index + _OPENING, context) for index, message in enumerate(messages[: 2 * MAX_STEPS])] n_steps = (len(messages) + 1) // 2 kept = min(n_steps, MAX_STEPS) numbers = np.zeros((4, kept), dtype=np.int32) repeated = np.zeros(kept, dtype=bool) actions, observations, thoughts, seen = [], [], [], set() for step in range(kept): - thought, command = split_message(messages[2 * step]["text"] or "") + thought, command = split_message(texts[2 * step]) # The final step has no reply; every kept step of a checkpoint does. - reply = messages[2 * step + 1]["text"] or "" if 2 * step + 1 < len(messages) else "" + reply = texts[2 * step + 1] if 2 * step + 1 < len(texts) else "" numbers[:3, step] = len(thought), len(command), len(reply) - numbers[3, step] = len(messages[2 * step]["text"] or "") + len(reply) + numbers[3, step] = len(texts[2 * step]) + len(reply) repeated[step] = command in seen seen.add(command) actions.append(action_kind(command)) observations.append(observation_kind(reply)) - thoughts.append(thought.replace("\0", "")) + thoughts.append(thought) return n_steps, Steps( thought_chars=numbers[0], action_chars=numbers[1], @@ -289,11 +334,11 @@ def _steps(messages: list[dict], opening: int, context: str) -> tuple[int, Steps repeated_action=repeated, action_kind=tuple(actions), observation_kind=tuple(observations), - thoughts=zlib.compress("\0".join(thoughts).encode()), + thoughts=zlib.compress("".join(thoughts).encode()), ) -def parse_run(row: dict, shard: str, index: int) -> Run: +def parse_run(row: SourceRow, shard: str, index: int) -> Run: """Check one release row and parse it into a :class:`Run`. Returns: @@ -302,32 +347,37 @@ def parse_run(row: dict, shard: str, index: int) -> Run: Raises: TimeNetConnectorError: If the row breaks the release's shape: an unknown exit status, a malformed instance id, a trajectory that does not open with the system prompt and the - issue, or messages out of turn. + issue, a message without text, or messages out of turn. """ context = f"{shard} row {index}" - instance = _INSTANCE.fullmatch(row["instance_id"] or "") + instance = _INSTANCE.fullmatch(row.instance_id) if instance is None: - raise TimeNetConnectorError(f"{context}: malformed instance_id {row['instance_id']!r}") - if row["exit_status"] not in ENDINGS: - raise TimeNetConnectorError(f"{context}: unknown exit_status {row['exit_status']!r}") - if not isinstance(row["target"], bool) or not row["model_name"]: - raise TimeNetConnectorError(f"{context}: target must be true or false and model_name set") - messages = row["trajectory"] or [] - if len(messages) < _OPENING or messages[0]["role"] != "system" or messages[1]["role"] != "user": + raise TimeNetConnectorError(f"{context}: malformed instance_id {row.instance_id!r}") + if row.exit_status not in EXIT_STATUSES: + raise TimeNetConnectorError(f"{context}: unknown exit_status {row.exit_status!r}") + if not row.model_name: + raise TimeNetConnectorError(f"{context}: model_name is empty") + messages = row.trajectory + if len(messages) < _OPENING or messages[0].role != "system" or messages[1].role != "user": raise TimeNetConnectorError(f"{context}: the trajectory must open with the system prompt and the issue") - issue = _ISSUE.search(messages[1]["text"] or "") + system = messages[0] + # The system message holds the prompt in system_prompt; its text is empty in every row of the release. + if system.system_prompt is None or system.text: + raise TimeNetConnectorError(f"{context}: the system message must hold its prompt in system_prompt only") + issue_message = _text(messages[1], 1, context) + issue = _ISSUE.search(issue_message) if issue is None: raise TimeNetConnectorError(f"{context}: the first message holds no ISSUE section") - opening = len(messages[0]["system_prompt"] or "") + len(messages[0]["text"] or "") + len(messages[1]["text"]) + opening = len(system.system_prompt) + len(issue_message) n_steps, steps = _steps(messages[_OPENING:], opening, context) return Run( shard=shard, row=index, - instance_id=row["instance_id"], + instance_id=row.instance_id, repository=instance["repository"], - model_name=row["model_name"], - resolved=row["target"], - exit_status=row["exit_status"], + model_name=row.model_name, + resolved=row.target, + exit_status=row.exit_status, issue=issue.group(1), n_steps=n_steps, steps=steps, @@ -347,18 +397,16 @@ def read_runs(root: Path) -> Iterator[Run]: TimeNetConnectorError: If a shard is missing, unreadable, or holds a different number of rows than the pinned revision. """ + # Each issue has many runs; they share one copy of its text. + issues: dict[str, str] = {} for shard, expected in SHARD_ROWS.items(): path = root / shard if not path.is_file(): raise TimeNetConnectorError(f"{path} is missing") count = 0 - try: - batches = pq.ParquetFile(path).iter_batches(batch_size=_BATCH_ROWS, columns=_COLUMNS) - for batch in batches: - for row in batch.to_pylist(): - yield parse_run(row, shard, count) - count += 1 - except (OSError, pa.ArrowException) as exc: - raise TimeNetConnectorError(f"{path}: {exc}") from exc + for row in ParquetDataReader(path, SourceRow, selected_columns=_COLUMNS).iter_rows(): + run = parse_run(row, shard, count) + yield dataclasses.replace(run, issue=issues.setdefault(run.issue, run.issue)) + count += 1 if count != expected: raise TimeNetConnectorError(f"{path}: expected {expected} rows, found {count}") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py index d01ff8c44..1af70aaa0 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py @@ -16,6 +16,7 @@ from timenet.types import Label, LabelSet from timenet.types.splits import Split from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + EXIT_STATUSES, ActionKind, Ending, ObservationKind, @@ -28,14 +29,14 @@ RESOLVED = "resolved" UNRESOLVED = "unresolved" -OUTCOMES = LabelSet( +OUTCOME_LABELS = LabelSet( name="run_outcome", labels=( Label(name=RESOLVED, description="The run's final patch passed the tests of the issue's pull request."), Label(name=UNRESOLVED, description="The run did not resolve the issue."), ), ) -ENDINGS = LabelSet( +ENDING_LABELS = LabelSet( name="run_ending", labels=( Label(name=Ending.SUBMITTED, description="The agent submitted its changes."), @@ -60,22 +61,10 @@ RESOLVE_PROMPT = f"{_CONTEXT} Will the agent's final patch resolve the issue?" ENDING_PROMPT = f"{_CONTEXT} How will the run end?" STATUS_PROMPT = ( - f"{_CONTEXT} Report how the agent is doing so far, then say how the run will end and whether it " - "will resolve the issue." + f"{_CONTEXT} Report how the agent is doing so far, then say how many more steps the run will take, " + "how it will end and whether the agent will resolve the issue." ) -_ENDING_TEXT = { - "submitted": "the agent submitted its changes", - "submitted (exit_context)": "it reached the context limit and SWE-agent submitted its changes", - "exit_context": "it reached the context limit", - "early_exit": "it ended with the status early_exit", - "submitted_no_patch": "the agent submitted without a patch", - "submitted (exit_format)": "SWE-agent stopped it after repeated malformed messages and submitted its changes", - "exit_format": "SWE-agent stopped it after repeated malformed messages", - "submitted (exit_cost)": "it reached its cost limit and SWE-agent submitted its changes", - "exit_cost": "it reached its cost limit", -} - @dataclass(frozen=True) class Checkpoint: @@ -122,8 +111,8 @@ def status(checkpoint: Checkpoint) -> str: """Write the status report of a checkpoint from its steps' signals and the run's outcome. Returns: - Three sentences on the steps so far, then one on how the run ended and whether it resolved - the issue. + Three sentences on the steps so far, then one on how many more steps the run will take, how it + will end and whether the agent will resolve the issue. """ k, run = checkpoint.steps, checkpoint.run actions = Counter(run.steps.action_kind[:k]) @@ -141,6 +130,6 @@ def status(checkpoint: Checkpoint) -> str: f"{_count(replies[ObservationKind.COMMAND_ERROR], 'reply', 'replies')} a command error " f"and {_count(replies[ObservationKind.TIMEOUT], 'command')} timed out. " f"The conversation holds {int(run.steps.context_chars[k - 1]):,} characters. " - f"The run went on for {_count(rest, 'more step')}, then {_ENDING_TEXT[run.exit_status]}, " - f"and {'resolved' if run.resolved else 'did not resolve'} the issue." + f"The run will go on for {_count(rest, 'more step')}, then {EXIT_STATUSES[run.exit_status].phrase}, " + f"and the agent will {'resolve' if run.resolved else 'not resolve'} the issue." ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py index bfe2c09ca..6a33604e7 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py @@ -16,6 +16,8 @@ from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( ActionKind, ObservationKind, + Run, + SourceRow, action_kind, observation_kind, parse_run, @@ -73,10 +75,16 @@ def _row(repository: int, number: int, n_steps: int, *, resolved: bool, exit_sta } +def _parse(row: dict) -> Run: + # The reader selects the columns the connector reads; the patch and the test log are not among them. + return parse_run(SourceRow.model_validate({name: row[name] for name in SourceRow.model_fields}), SHARD, 0) + + def _rows() -> list[dict]: rows = [] for repository in range(12): - rows.append(_row(repository, 1, 4, resolved=True)) + # Instance 3 has only a 4-step run, so no checkpoint and no issue annotation. + rows.append(_row(repository, 3, 4, resolved=True)) rows.append(_row(repository, 1, 12, resolved=repository % 3 == 0)) rows.append(_row(repository, 2, 33, resolved=False, exit_status="submitted (exit_context)")) return rows @@ -155,7 +163,7 @@ def test_observation_kind_applies_its_rules_in_order(reply: str, kind: str) -> N def test_a_run_keeps_its_first_steps_and_its_outcome_apart() -> None: - run = parse_run(_row(0, 1, 7, resolved=False), SHARD, 0) + run = _parse(_row(0, 1, 7, resolved=False)) assert run.n_steps == 7 assert run.issue == ISSUE assert run.repository == "owner0__repo0" @@ -171,10 +179,18 @@ def test_a_run_keeps_its_first_steps_and_its_outcome_apart() -> None: assert run.steps.thought_texts(2) == ["Let's reproduce the bug first.", "Write the script."] +def test_a_thought_is_kept_as_written() -> None: + row = _row(0, 1, 7, resolved=False) + row["trajectory"][2]["text"] = _agent("A thought with a NUL \0 and ```inline``` fences.", "ls") + run = _parse(row) + assert run.steps.thought_texts(2) == ["A thought with a NUL \0 and ```inline``` fences.", "Write the script."] + assert run.steps.thought_chars[0] == len(run.steps.thought_texts(1)[0]) + + def test_only_a_run_that_goes_on_gets_a_checkpoint() -> None: - assert [c.steps for c in checkpoints(parse_run(_row(0, 1, 5, resolved=False), SHARD, 0))] == [] - assert [c.steps for c in checkpoints(parse_run(_row(0, 1, 6, resolved=False), SHARD, 0))] == [5] - assert [c.steps for c in checkpoints(parse_run(_row(0, 1, 33, resolved=False), SHARD, 0))] == [5, 10, 20, 30] + assert [c.steps for c in checkpoints(_parse(_row(0, 1, 5, resolved=False)))] == [] + assert [c.steps for c in checkpoints(_parse(_row(0, 1, 6, resolved=False)))] == [5] + assert [c.steps for c in checkpoints(_parse(_row(0, 1, 33, resolved=False)))] == [5, 10, 20, 30] def test_records_hold_the_steps_so_far_and_nothing_about_the_outcome(dataset) -> None: @@ -196,11 +212,14 @@ def test_each_checkpoint_gives_three_tasks_with_the_issue_and_thoughts_as_input( tasks = list(dataset.iter_tasks()) assert len(tasks) == 3 * 12 * 6 assert Counter(type(task) for task in tasks) == {ClassificationTask: 2 * 72, AnswerTask: 72} + # One issue annotation per instance with a checkpoint: instances 1 and 2 of each repository. issues = [a for a in dataset.annotations if a.key == "issue"] - assert len(issues) == 1 + assert len(issues) == 24 + assert not any(a.metadata["instance_id"].endswith("-3") for a in issues) for task in tasks: assert [a.key for a in task.input_annotations] == ["issue", "agent_thoughts"] - assert task.input_annotations[0] is issues[0] + assert task.input_annotations[0].metadata["instance_id"] == task.metadata["instance_id"] + assert task.metadata == task.inputs[0].metadata endings = { read_value(t.targets[0]) for t in tasks @@ -209,15 +228,16 @@ def test_each_checkpoint_gives_three_tasks_with_the_issue_and_thoughts_as_input( assert endings == {"submitted", "context_limit"} -def test_the_status_report_states_the_steps_so_far_and_the_outcome() -> None: - run = parse_run(_row(0, 1, 12, resolved=True), SHARD, 0) +def test_the_status_report_states_the_steps_so_far_and_what_comes() -> None: + run = _parse(_row(0, 1, 12, resolved=True)) (first, *_) = checkpoints(run) assert status(first) == ( "In its first 5 steps the agent edited files 1 time, created 1 file, ran code 1 time, " "read or searched the repository 2 times and repeated an earlier command 0 times. " "The linter rejected 1 edit, 1 reply showed a Python traceback, 0 replies a command error " f"and 0 commands timed out. The conversation holds {int(run.steps.context_chars[4]):,} characters. " - "The run went on for 7 more steps, then the agent submitted its changes, and resolved the issue." + "The run will go on for 7 more steps, then the agent will submit its changes, and the agent will " + "resolve the issue." ) @@ -254,13 +274,23 @@ def test_dataset_survives_storage_round_trip(dataset, tmp_path: Path) -> None: (lambda row: row["trajectory"][3].update(role="ai"), "out of turn"), (lambda row: row["trajectory"].pop(), "does not end on an agent message"), (lambda row: row["trajectory"].pop(0), "must open with the system prompt"), + (lambda row: row["trajectory"][2].update(text=None), "message 2 has no text"), + (lambda row: row["trajectory"][0].update(text="more prompt"), "system_prompt only"), + (lambda row: row.update(model_name=""), "model_name is empty"), ], ) def test_a_malformed_row_fails_the_build(change, message: str) -> None: row = _row(0, 1, 6, resolved=False) change(row) with pytest.raises(TimeNetConnectorError, match=message): - parse_run(row, SHARD, 0) + _parse(row) + + +def test_a_row_without_a_target_fails_the_build(one_shard) -> None: + rows = _rows() + rows[1]["target"] = None + with pytest.raises(TimeNetConnectorError, match="target"): + list(read_runs(one_shard(rows))) def test_a_shard_with_another_row_count_fails_the_build(one_shard, monkeypatch: pytest.MonkeyPatch) -> None: From 6c2cca8c1601e38229f4c159795b2d27befd33a9 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 15:09:40 +0100 Subject: [PATCH 6/8] docs(connectors): update the SWE-agent card for per-instance issues and the status report --- .../nebius/swe_agent_trajectories/README.md | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md index cfea9debc..5b1e5e618 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md @@ -118,7 +118,9 @@ step, on an ordinal axis. The steps have an order but no timestamps: the release **`observation_kind` rules.** The first of these that holds wins: 1. `edit_rejected`: the reply starts with SWE-agent's message for an edit its linter rejected ("Your proposed edit has introduced new syntax error"). 2. `timeout`: it starts with `EXECUTION TIMED OUT`. -3. `traceback`: it contains `Traceback (most recent call last)`. +3. `traceback`: it contains `Traceback (most recent call last)`. This includes a file view that + merely shows such text, e.g. after `open` or `edit`. In shard 0 that is 69 of 9,375 tracebacks + *(measured)*. 4. `command_error`: its first line starts with `Usage:` or `Error:`, or contains `command not found`, `No such file or directory` or `syntax error near unexpected token`. 5. `silent_success`: it starts with SWE-agent's message for a command that printed nothing. 6. `output`: anything else. @@ -128,15 +130,18 @@ Over the first 30 steps of every run *(measured)*: - **Repeats:** 28 % of steps repeat an earlier command. Rerunning a script counts as a repeat. **Text.** The agent's thoughts for the *k* steps are the record annotation `agent_thoughts`, a list -of *k* strings in step order. They are text, not a signal, because a text series has no tensor form -in `TimeFTorchDataset`. +of *k* strings in step order, kept exactly as written. They are text, not a signal, because a text +series has no tensor form in `TimeFTorchDataset`. **Metadata** (provenance, never model input): `run_id` (shard and row), `instance_id`, `repository`, `model_name`, `checkpoint_steps`. -**The issue** is a dataset annotation with the key `issue`, stored once per distinct text, because -each issue has many runs. Its text is the issue message's `ISSUE:` section. 13 issues appear with -two slightly different texts *(measured)*; each text is its own annotation. +**The issue** is a dataset annotation with the key `issue`, stored once per instance, because each +issue has many runs. Its text is the issue message's `ISSUE:` section, and its metadata names the +`instance_id`. Only instances with a checkpoint get one: 3,599 annotations *(measured)*. +- 13 instances appear with two slightly different texts *(measured)*; each text is its own annotation. +- 16 texts are shared by two instances of the same repository, e.g. `iterative__dvc-5335` and + `iterative__dvc-5336` *(measured)*; each instance gets its own annotation. ## Tasks @@ -147,18 +152,19 @@ in `input_annotations`. That makes 577,284 tasks *(measured)*. | --- | --- | --- | | `ClassificationTask` `run_outcome` | `resolved` or `unresolved` | `target` | | `ClassificationTask` `run_ending` | the ending label (table above) | `exit_status` | -| `AnswerTask` | a status report: the steps so far, then how the run ended | the signals, `target`, `exit_status`, the step count | +| `AnswerTask` | a status report: the steps so far, then how many more steps the run will take, how it will end and whether it will resolve the issue | the signals, `target`, `exit_status`, the step count | **How the text is made.** A fixed template fills in the numbers; no model writes it: > In its first {k} steps the agent edited files {n} times, created {n} files, ran code {n} times, > read or searched the repository {n} times and repeated an earlier command {n} times. The linter > rejected {n} edits, {n} replies showed a Python traceback, {n} replies a command error and {n} -> commands timed out. The conversation holds {context} characters. The run went on for {m} more -> steps, then {ending}, and {resolved | did not resolve} the issue. +> commands timed out. The conversation holds {context} characters. The run will go on for {m} more +> steps, then {ending}, and the agent will {resolve | not resolve} the issue. "Read or searched" adds the `read` and `search` steps. {ending} is one fixed phrase per `exit_status`, -e.g. "it reached the context limit and SWE-agent submitted its changes". +e.g. "it will reach the context limit and SWE-agent will submit its changes". The prompt asks for +the same three things in the same tense. **Example.** The first checkpoint of the release (run `train-00000-of-00012:0`, issue `AnalogJ__lexicon-336`, 5 steps) has the target: @@ -166,10 +172,10 @@ e.g. "it reached the context limit and SWE-agent submitted its changes". > In its first 5 steps the agent edited files 0 times, created 0 files, ran code 0 times, read or > searched the repository 4 times and repeated an earlier command 0 times. The linter rejected 0 > edits, 1 reply showed a Python traceback, 1 reply a command error and 0 commands timed out. The -> conversation holds 12,857 characters. The run went on for 41 more steps, then it reached the -> context limit and SWE-agent submitted its changes, and did not resolve the issue. +> conversation holds 12,857 characters. The run will go on for 41 more steps, then it will reach +> the context limit and SWE-agent will submit its changes, and the agent will not resolve the issue. -**Task metadata:** the Record's metadata fields. +**Task metadata:** the Record's metadata. ## The split From 56da8bd2e5a46cf29ba41a702a64592bfd39b8f5 Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 15:21:39 +0100 Subject: [PATCH 7/8] docs(connectors): say how to score the SWE-agent status report --- .../datasets/nebius/swe_agent_trajectories/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md index 5b1e5e618..74336bff5 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md @@ -203,6 +203,9 @@ have a checkpoint. - **Use measures that do not reward the majority class:** AUROC, the precision-recall curve of `resolved` (its baseline is the resolved rate), balanced accuracy, and calibration (Brier score) for a health score. - **Count each run once.** A long run gives up to four checkpoints, and long runs fail more often. When training, weight a checkpoint by 1 / (checkpoints of its run), or sample one checkpoint per run. - **Compare against simple baselines:** always `unresolved`; the numeric signals alone; the thoughts alone. +- **Score the status report by its last sentence.** The first three restate counts the inputs show, + so text overlap (ROUGE, BLEU) on the whole answer stays high even when the prediction is wrong. The + last sentence starts with "The run will go on for"; score its step count, ending and outcome apart. ## Release details a user should know From de1582366cab547a1fc273fee18beca56b1bb4ea Mon Sep 17 00:00:00 2001 From: etienneRV Date: Fri, 20 Nov 2026 15:25:06 +0100 Subject: [PATCH 8/8] docs(connectors): explain the status-report scoring note more plainly --- .../datasets/nebius/swe_agent_trajectories/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md index 74336bff5..d0d9871f3 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md @@ -203,9 +203,11 @@ have a checkpoint. - **Use measures that do not reward the majority class:** AUROC, the precision-recall curve of `resolved` (its baseline is the resolved rate), balanced accuracy, and calibration (Brier score) for a health score. - **Count each run once.** A long run gives up to four checkpoints, and long runs fail more often. When training, weight a checkpoint by 1 / (checkpoints of its run), or sample one checkpoint per run. - **Compare against simple baselines:** always `unresolved`; the numeric signals alone; the thoughts alone. -- **Score the status report by its last sentence.** The first three restate counts the inputs show, - so text overlap (ROUGE, BLEU) on the whole answer stays high even when the prediction is wrong. The - last sentence starts with "The run will go on for"; score its step count, ending and outcome apart. +- **Score only the prediction in the status report.** Its first three sentences count what happened + so far, which the model can read off its input. Only the last sentence predicts: how many more + steps, how the run will end, and whether it will resolve the issue. A word-overlap score (ROUGE, + BLEU) on the whole answer therefore stays high even when that prediction is wrong. Score the last + sentence, which starts with "The run will go on for", and check its three facts one by one. ## Release details a user should know