diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py new file mode 100644 index 000000000..db7d1a712 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/__init__.py @@ -0,0 +1 @@ +"""Connectors for datasets published by the ``nebius`` org on the Hugging Face Hub.""" diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md new file mode 100644 index 000000000..d0d9871f3 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/README.md @@ -0,0 +1,231 @@ +# SWE-agent trajectories: the release, and the decisions of this connector + +Nebius released 80,036 runs of SWE-agent, a coding agent, each trying to resolve one real GitHub +issue. Each run is labelled with whether its final patch passed the tests of the issue's pull request. + +This connector turns the runs into an **early-warning** dataset: a model sees the first steps of a run +that is still going and says how the run will turn out, like a monitor watching an agent at work. + +This document covers: +- what the release holds; +- how the connector maps it onto TimeF; +- why no input shows how a run ends; +- what a user should know before training or evaluating on it. + +*(measured)* marks a number taken from the files, not from the dataset page. + +## Source, licence and citation + +**Source:** the Hugging Face dataset [nebius/SWE-agent-trajectories](https://huggingface.co/datasets/nebius/SWE-agent-trajectories), pinned at revision `68195a1450865274106246d0d0296a1d6807b88e` (last modified 2024-12-23). +- 12 parquet shards, 1.1 GB to download, one `train` split of 80,036 rows *(measured)*. +- The connector pins the revision and each shard's row count, so a changed release fails the build. + +**Licence:** CC BY 4.0, as stated on the dataset page. The page adds two conditions that this card's +single licence field cannot hold: +- the repositories behind the issues keep their own licences, listed in nebius/SWE-bench-extra; +- using the model outputs requires complying with the Llama 3.1 License, because Llama models wrote them. + +**Citation:** the release has no citation entry. Cite the dataset page and the pinned revision: +> Nebius. SWE-agent trajectories (nebius/SWE-agent-trajectories). Hugging Face, 2024. Revision +> 68195a1450865274106246d0d0296a1d6807b88e. + +## What a run is + +Each row is one run of one model on one issue: +- `instance_id`: `__-`; +- `model_name`: `swe-agent-llama-70b` (74,792 runs), `-8b` (4,053) or `-405b` (1,191) *(measured)*; +- `target`: whether the run resolved the issue: 13,389 true, 66,647 false *(measured)*; +- `exit_status`: how the run ended; +- `trajectory`: the messages; +- `generated_patch` and `eval_logs`: the final patch and its test log. **The connector reads neither.** + +**The trajectory** is a system prompt, the issue message, then alternating agent and environment +messages, ending on an agent message. Every run follows this shape *(measured)*. +- **A step** is one agent message and the environment's reply to it. +- **An agent message** is a thought followed by one command in a fenced block. +- **The connector splits them** as SWE-agent's `ThoughtActionParser` does: the command is the last complete fenced block, and the thought is the rest. + +| | Value *(measured)* | +| --- | --- | +| Steps per run | median 17, 90th percentile 50, maximum 408; one run has none | +| Repositories, issues | 1,276 repositories, 3,591 issues; 807 issues have both resolved and unresolved runs | +| Largest repositories | pydantic/pydantic 6,279 runs, iterative/dvc 5,646 | + +**How runs end** (`exit_status` *(measured)*), and the ending label the connector gives it: + +| `exit_status` | Runs | Ending label | +| --- | --- | --- | +| `submitted` | 51,087 | `submitted` | +| `submitted (exit_context)` | 21,026 | `context_limit` | +| `exit_context` | 3,568 | `context_limit` | +| `early_exit` | 3,176 | `early_exit` | +| `submitted_no_patch` | 1,066 | `submitted_no_patch` | +| `submitted (exit_format)`, `exit_format` | 80, 20 | `format_errors` | +| `submitted (exit_cost)`, `exit_cost` | 10, 3 | `cost_limit` | + +`submitted (exit_x)` means SWE-agent reached limit x and submitted the changes so far itself. The +release does not explain `early_exit`; the connector keeps the name. + +## Why the inputs never show the end of a run + +The end of a run gives its outcome away: +- **The final step:** every `submitted` run ends with the command `submit` *(measured)*. +- **Running out of context:** 36 % of unresolved runs stop at the context limit, against 5 % of resolved ones *(measured)*. +- **Length:** unresolved runs are longer. + +A model that sees whole runs can learn "long, or cut off, means failure". That is not monitoring, and +a live monitor never sees the end before it happens. + +**So the connector only cuts checkpoints.** A checkpoint is the first *k* steps of a run, for +*k* = 5, 10, 20 and 30, and only if the run has more than *k* steps. Its last visible step is +therefore never the run's final action. +- **What no input holds:** the outcome, the exit status, the run's length, the patch and the test log. +- **What the inputs do hold:** a growing conversation, which is real information a monitor has. The longest checkpoint ends at 30 steps. + +## Records + +Each checkpoint is **one Record**: 192,428 in all *(measured)*. + +| Checkpoint | Runs still going | Resolved among them | +| --- | --- | --- | +| 5 steps | 76,182 | 16.6 % | +| 10 steps | 59,708 | 13.8 % | +| 20 steps | 34,059 | 6.4 % | +| 30 steps | 22,479 | 4.2 % | + +**Signals.** The Record has one source, `agent_steps`, with seven signals of *k* values each, one per +step, on an ordinal axis. The steps have an order but no timestamps: the release records none. + +| Signal | dtype | Value at each step | +| --- | --- | --- | +| `thought_chars` | int32 | characters of the agent's thought | +| `action_chars` | int32 | characters of the command | +| `observation_chars` | int32 | characters of the environment's reply | +| `context_chars` | int32 | characters of the whole conversation after the reply, from the system prompt on | +| `repeated_action` | bool | the command equals an earlier command of the run, character for character | +| `action_kind` | enum | `edit`, `create`, `run`, `read`, `search`, `submit`, `other`, by the command's first word | +| `observation_kind` | enum | `edit_rejected`, `timeout`, `traceback`, `command_error`, `silent_success`, `output` | + +**`action_kind` rules**, by the command's first word: +- `edit`: `edit`; +- `create`: `create`; +- `run`: `python`, `python3`, `pytest`; +- `read`: `open`, `goto`, `scroll_down`, `scroll_up`, `cat`; +- `search`: `search_dir`, `search_file`, `find_file`, `find`, `grep`, `ls`; +- `submit`: `submit`; +- `other`: every other command, or none. + +**`observation_kind` rules.** The first of these that holds wins: +1. `edit_rejected`: the reply starts with SWE-agent's message for an edit its linter rejected ("Your proposed edit has introduced new syntax error"). +2. `timeout`: it starts with `EXECUTION TIMED OUT`. +3. `traceback`: it contains `Traceback (most recent call last)`. This includes a file view that + merely shows such text, e.g. after `open` or `edit`. In shard 0 that is 69 of 9,375 tracebacks + *(measured)*. +4. `command_error`: its first line starts with `Usage:` or `Error:`, or contains `command not found`, `No such file or directory` or `syntax error near unexpected token`. +5. `silent_success`: it starts with SWE-agent's message for a command that printed nothing. +6. `output`: anything else. + +Over the first 30 steps of every run *(measured)*: +- **Replies:** plain output 1,030,976; linter rejections 179,147; tracebacks 114,145; silent successes 55,182; command errors 49,989; timeouts 6,235. +- **Repeats:** 28 % of steps repeat an earlier command. Rerunning a script counts as a repeat. + +**Text.** The agent's thoughts for the *k* steps are the record annotation `agent_thoughts`, a list +of *k* strings in step order, kept exactly as written. They are text, not a signal, because a text +series has no tensor form in `TimeFTorchDataset`. + +**Metadata** (provenance, never model input): `run_id` (shard and row), `instance_id`, `repository`, +`model_name`, `checkpoint_steps`. + +**The issue** is a dataset annotation with the key `issue`, stored once per instance, because each +issue has many runs. Its text is the issue message's `ISSUE:` section, and its metadata names the +`instance_id`. Only instances with a checkpoint get one: 3,599 annotations *(measured)*. +- 13 instances appear with two slightly different texts *(measured)*; each text is its own annotation. +- 16 texts are shared by two instances of the same repository, e.g. `iterative__dvc-5335` and + `iterative__dvc-5336` *(measured)*; each instance gets its own annotation. + +## Tasks + +Each checkpoint gives three tasks. All take the Record as input and name the issue and the thoughts +in `input_annotations`. That makes 577,284 tasks *(measured)*. + +| Task | Target | Source of the answer | +| --- | --- | --- | +| `ClassificationTask` `run_outcome` | `resolved` or `unresolved` | `target` | +| `ClassificationTask` `run_ending` | the ending label (table above) | `exit_status` | +| `AnswerTask` | a status report: the steps so far, then how many more steps the run will take, how it will end and whether it will resolve the issue | the signals, `target`, `exit_status`, the step count | + +**How the text is made.** A fixed template fills in the numbers; no model writes it: + +> In its first {k} steps the agent edited files {n} times, created {n} files, ran code {n} times, +> read or searched the repository {n} times and repeated an earlier command {n} times. The linter +> rejected {n} edits, {n} replies showed a Python traceback, {n} replies a command error and {n} +> commands timed out. The conversation holds {context} characters. The run will go on for {m} more +> steps, then {ending}, and the agent will {resolve | not resolve} the issue. + +"Read or searched" adds the `read` and `search` steps. {ending} is one fixed phrase per `exit_status`, +e.g. "it will reach the context limit and SWE-agent will submit its changes". The prompt asks for +the same three things in the same tense. + +**Example.** The first checkpoint of the release (run `train-00000-of-00012:0`, issue +`AnalogJ__lexicon-336`, 5 steps) has the target: + +> In its first 5 steps the agent edited files 0 times, created 0 files, ran code 0 times, read or +> searched the repository 4 times and repeated an earlier command 0 times. The linter rejected 0 +> edits, 1 reply showed a Python traceback, 1 reply a command error and 0 commands timed out. The +> conversation holds 12,857 characters. The run will go on for 41 more steps, then it will reach +> the context limit and SWE-agent will submit its changes, and the agent will not resolve the issue. + +**Task metadata:** the Record's metadata. + +## The split + +**Runs are grouped by repository.** `StratifiedSplitter` (8/1/1 folds, stratified on `target`, +seed 42) assigns whole repositories, and each checkpoint takes its run's partition. +- **Why:** a repository's runs share its code, and an issue's runs share the issue. With 807 issues holding both resolved and unresolved runs, a random split would test on issues seen in training. +- **The release** has a single `train` split, so there is no official split to follow. + +**Result** *(measured)*: + +| Partition | Repositories | Runs with a checkpoint | Tasks | Resolved at 5 / 10 / 20 / 30 steps | +| --- | --- | --- | --- | --- | +| train | 993 | 60,985 | 464,748 | 16.6 / 13.7 / 6.2 / 3.9 % | +| validation | 139 | 7,557 | 55,029 | 16.8 / 14.6 / 6.8 / 4.4 % | +| test | 143 | 7,640 | 57,507 | 16.4 / 14.1 / 7.1 / 5.9 % | + +No repository and no issue is in two partitions. The checks count only the 1,275 repositories that +have a checkpoint. + +## Evaluating on it + +**Most runs fail.** A model that always answers `unresolved` is right 83 % of the time at 5 steps and +96 % at 30 steps. Accuracy therefore says little. We suggest: +- **Report each checkpoint separately.** The base rate changes with *k*, so pooled results mix four problems. +- **Use measures that do not reward the majority class:** AUROC, the precision-recall curve of `resolved` (its baseline is the resolved rate), balanced accuracy, and calibration (Brier score) for a health score. +- **Count each run once.** A long run gives up to four checkpoints, and long runs fail more often. When training, weight a checkpoint by 1 / (checkpoints of its run), or sample one checkpoint per run. +- **Compare against simple baselines:** always `unresolved`; the numeric signals alone; the thoughts alone. +- **Score only the prediction in the status report.** Its first three sentences count what happened + so far, which the model can read off its input. Only the last sentence predicts: how many more + steps, how the run will end, and whether it will resolve the issue. A word-overlap score (ROUGE, + BLEU) on the whole answer therefore stays high even when that prediction is wrong. Score the last + sentence, which starts with "The run will go on for", and check its three facts one by one. + +## Release details a user should know + +- **The labels depend on the model as much as the issue.** 93 % of runs come from one model (Llama 70B), and each issue is attempted 22 times on average *(measured)*. +- **The thoughts are long:** 5.8 kB per run over the first 30 steps, on average *(measured)*. +- **Reply lengths count the logged text.** That includes SWE-agent's status lines (open file, current directory). +- **`mask` and `cutoff_date`** are not explained by the release. The connector does not use them. + +## Limitations and intended use + +**Limitations:** +- One agent framework (SWE-agent) and one model family (Llama 3.1). +- The signals and the kinds are rule-made from the logged text. The rules are fixed and listed above, but they are not exhaustive: a shell command the table does not list is `other`. +- Status reports are templates, not expert assessments. +- The outcome is the release's test result, which may miss correct fixes or accept incomplete ones. + +**Intended use:** +- Training and evaluating time-series language models that monitor a coding agent while it works, and warn before a run fails. +- Studying which step patterns precede success or failure. + +*Contributed by Team RAMadan during the TimeNet hackathon, Round 1 (November 2026).* diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py new file mode 100644 index 000000000..b321c89bc --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/__init__.py @@ -0,0 +1,6 @@ +"""The SWE-agent trajectories connector package: its connector implementation and dataset card.""" + +from timenet_connectors.datasets.nebius.swe_agent_trajectories.connector import ( + CONNECTOR as CONNECTOR, + SWEAgentTrajectoriesConnector as SWEAgentTrajectoriesConnector, +) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py new file mode 100644 index 000000000..352d574e0 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/connector.py @@ -0,0 +1,216 @@ +"""The SWE-agent trajectories connector: 80,036 coding-agent runs, cut into checkpoints for early warning. + +Nebius released the trajectories of SWE-agent runs that try to resolve GitHub issues, each with +whether its final patch passed the tests of the issue's pull request. Each checkpoint of a run (its +first 5, 10, 20 or 30 steps, if the run is still going) becomes one Record of per-step signals, with +the agent's thoughts for those steps as text. The issue text is a dataset annotation, stored once per +instance. :mod:`~timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks` describes the +three tasks of each checkpoint. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping +import functools +from typing import ClassVar + +import pyarrow as pa + +from timenet.dataset import Record, Signal, Source, TimeFDataset +from timenet.dataset.axis import OrdinalAxis +from timenet.types import Annotation, AnswerTask, ClassificationTask, Task, TimeSeriesSpec, ureg +from timenet.types.splits import Split +from timenet.types.text import LazyTextList +from timenet_connectors.bases.huggingface import BaseHuggingFaceConnector, HubFileRef +from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + ActionKind, + ObservationKind, + Run, + read_runs, +) +from timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks import ( + ENDING_LABELS, + ENDING_PROMPT, + OUTCOME_LABELS, + RESOLVE_PROMPT, + RESOLVED, + STATUS_PROMPT, + UNRESOLVED, + Checkpoint, + checkpoints, + split_runs, + status, +) + + +AXIS = OrdinalAxis() +SOURCE_NAME = "agent_steps" +THOUGHTS = "agent_thoughts" +ISSUE = "issue" + + +def _count_spec(spec_type: str, name: str) -> TimeSeriesSpec: + return TimeSeriesSpec(spec_type=spec_type, name=name, unit_value=ureg.dimensionless, dtype="int32") + + +# One signal per step property, in Record order. Lengths count characters of the release's text. +SPECS = { + "thought_chars": _count_spec("agent_thought_length", "Agent thought length (characters)"), + "action_chars": _count_spec("agent_action_length", "Agent command length (characters)"), + "observation_chars": _count_spec("agent_observation_length", "Environment reply length (characters)"), + "context_chars": _count_spec("agent_context_length", "Conversation length so far (characters)"), + "repeated_action": TimeSeriesSpec( + spec_type="agent_repeated_action", + name="Command repeats an earlier one", + unit_value=ureg.dimensionless, + dtype="bool", + ), + "action_kind": TimeSeriesSpec( + spec_type="agent_action_kind", + name="Agent command kind", + unit_value=ureg.dimensionless, + dtype="enum", + categories=tuple(ActionKind), + ), + "observation_kind": TimeSeriesSpec( + spec_type="agent_observation_kind", + name="Environment reply kind", + unit_value=ureg.dimensionless, + dtype="enum", + categories=tuple(ObservationKind), + ), +} + + +def _values(run: Run, name: str, steps: int) -> pa.Array: + """The first ``steps`` values of one step signal of a run.""" + values = getattr(run.steps, name)[:steps] + if name in {"action_kind", "observation_kind"}: + # An enum series loads as dictionary-encoded labels. + return pa.array([str(value) for value in values], type=pa.string()).dictionary_encode() + return pa.array(values) + + +class SWEAgentTrajectoriesConnector(BaseHuggingFaceConnector): + """Connector for nebius/SWE-agent-trajectories on the Hugging Face Hub.""" + + HF_REPO = "nebius/SWE-agent-trajectories" + SOURCE_REVISION = "68195a1450865274106246d0d0296a1d6807b88e" + SOURCE_FILES: ClassVar[Mapping[str, str]] = {"train": "data/train-*.parquet"} + + def convert(self, raw_refs: list[HubFileRef]) -> TimeFDataset: + """Read every run, then build one Record per checkpoint and stream three tasks per Record. + + Every shard is read once, which checks every row before anything is written. Each run keeps + only the signals of its first 30 steps and its thoughts, compressed, so the loaders and the + task stream need no second read. + + Returns: + The populated :class:`~timenet.dataset.TimeFDataset`. + """ + runs = list(read_runs(self.ref_paths(raw_refs, "train")["train"])) + dataset = TimeFDataset(metadata=self.metadata()) + for key, value in (("source_repository", self.HF_REPO), ("source_revision", self.SOURCE_REVISION)): + dataset.annotate(Annotation(key=key, value=value)) + split_of = split_runs(runs) + # One issue annotation per instance and text, made when a task first needs it. + issues: dict[tuple[str, str], Annotation] = {} + records: list[tuple[Checkpoint, Record]] = [] + for run in runs: + for checkpoint in checkpoints(run): + if (run.instance_id, run.issue) not in issues: + issues[run.instance_id, run.issue] = dataset.annotate( + Annotation( + key=ISSUE, + value=run.issue, + description="The GitHub issue a run tries to resolve, as the agent read it.", + metadata={"instance_id": run.instance_id}, + ) + ) + records.append((checkpoint, dataset.add_record(record=self._make_record(checkpoint)))) + dataset.set_task_stream( + [ClassificationTask, AnswerTask], functools.partial(_tasks, records, issues, split_of.__getitem__) + ) + return dataset + + @staticmethod + def _make_record(checkpoint: Checkpoint) -> Record: + """Build one checkpoint's Record: a signal per step property, and the thoughts as text. + + Returns: + The Record. It holds nothing about the run after the checkpoint. + """ + run, steps = checkpoint.run, checkpoint.steps + signals = tuple( + Signal.from_loader( + spec=spec, + time_axis=AXIS, + name=name, + loader=functools.partial(_values, run, name, steps), + n_values=steps, + ) + for name, spec in SPECS.items() + ) + record = Record( + sources=(Source(name=SOURCE_NAME, signals=signals),), + metadata={ + "run_id": run.run_id, + "instance_id": run.instance_id, + "repository": run.repository, + "model_name": run.model_name, + "checkpoint_steps": steps, + }, + ) + record.annotate( + Annotation( + key=THOUGHTS, + value=LazyTextList(functools.partial(run.steps.thought_texts, steps), length=steps), + description="The agent's thought at each step, in step order: its message without the command.", + ) + ) + return record + + +def _tasks( + records: list[tuple[Checkpoint, Record]], + issues: dict[tuple[str, str], Annotation], + split_of: Callable[[str], Split], +) -> Iterator[Task]: + """Yield the three tasks of every checkpoint, in Record order. + + Yields: + The outcome and ending classifications, then the status report, of each checkpoint. + """ + for checkpoint, record in records: + run = checkpoint.run + context = (issues[run.instance_id, run.issue], *(a for a in record.annotations if a.key == THOUGHTS)) + inputs, split, metadata = (record,), split_of(run.run_id), record.metadata + yield ClassificationTask( + inputs=inputs, + split=split, + input_annotations=context, + metadata=metadata, + prompt=RESOLVE_PROMPT, + targets=(RESOLVED if run.resolved else UNRESOLVED,), + label_set=OUTCOME_LABELS, + ) + yield ClassificationTask( + inputs=inputs, + split=split, + input_annotations=context, + metadata=metadata, + prompt=ENDING_PROMPT, + targets=(str(run.ending),), + label_set=ENDING_LABELS, + ) + yield AnswerTask( + inputs=inputs, + split=split, + input_annotations=context, + metadata=metadata, + prompt=STATUS_PROMPT, + targets=(status(checkpoint),), + ) + + +CONNECTOR = SWEAgentTrajectoriesConnector diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml new file mode 100644 index 000000000..36d151ca7 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/dataset.yaml @@ -0,0 +1,33 @@ +# yaml-language-server: $schema=https://docs.timenet.ai/schemas/dataset-card-v1.schema.json +# Each Record is one checkpoint of one run: the run's first 5, 10, 20 or 30 steps, kept only if the run +# goes on after them. It holds seven per-step signals and the agent's thoughts for those steps. No input +# holds the run's outcome, its ending or its length. The split groups runs by repository. +yaml_schema_version: 1 +dataset_id: nebius/swe_agent_trajectories +dataset_version: 1.0.0 +name: SWE-agent trajectories (early warning) +description: >- + 80,036 SWE-agent runs that try to resolve real GitHub issues, released by Nebius, each labelled + with whether its final patch passed the tests of the issue's pull request. Each run is cut into + checkpoints after 5, 10, 20 and 30 steps while it is still going: 192,428 Records of per-step + signals (thought, command and reply lengths, conversation length, repeated commands, command + kind, reply kind) with the agent's thoughts as text and the issue as context. Each checkpoint asks + whether the run will resolve its issue, how it will end, and for a rule-made status report. +license: CC-BY-4.0 +source_url: https://huggingface.co/datasets/nebius/SWE-agent-trajectories +citation: >- + Nebius. SWE-agent trajectories (nebius/SWE-agent-trajectories), Hugging Face dataset, 2024, + revision 68195a1450865274106246d0d0296a1d6807b88e. + https://huggingface.co/datasets/nebius/SWE-agent-trajectories +access: open +domains: + - observability +tags: + - software-engineering + - coding-agents + - agent-monitoring + - early-warning + - llm-trajectories + - failure-prediction + - explanation + - huggingface diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt new file mode 100644 index 000000000..fe57efebd --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/requirements.txt @@ -0,0 +1,2 @@ +# huggingface_hub downloads the source dataset from the HuggingFace Hub; 1.1 adds tqdm_class. +huggingface_hub>=1.1 diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py new file mode 100644 index 000000000..58a445bcf --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/source.py @@ -0,0 +1,412 @@ +"""Read the SWE-agent trajectories release and turn each run's first steps into per-step signals. + +A run is one attempt of SWE-agent to resolve one GitHub issue. Its trajectory is a system prompt, the +issue message, then alternating agent and environment messages. One agent message and the +environment's reply to it make one step. The connector keeps the first :data:`MAX_STEPS` steps of +every run, because no checkpoint looks further, and the run's step count and outcome. The outcome +is never part of a step. +""" + +from __future__ import annotations + +from collections.abc import Iterator +import dataclasses +from dataclasses import dataclass +from enum import StrEnum, unique +from pathlib import Path +import re +import zlib + +import numpy as np +from pydantic import ConfigDict, StrictBool + +from timenet.errors import TimeNetConnectorError +from timenet_connectors.sources.readers import DataRow, ParquetDataReader + + +# The twelve parquet shards of the pinned revision and their row counts. +SHARD_ROWS = { + "train-00000-of-00012.parquet": 6670, + "train-00001-of-00012.parquet": 6670, + "train-00002-of-00012.parquet": 6670, + "train-00003-of-00012.parquet": 6670, + "train-00004-of-00012.parquet": 6670, + "train-00005-of-00012.parquet": 6670, + "train-00006-of-00012.parquet": 6670, + "train-00007-of-00012.parquet": 6670, + "train-00008-of-00012.parquet": 6669, + "train-00009-of-00012.parquet": 6669, + "train-00010-of-00012.parquet": 6669, + "train-00011-of-00012.parquet": 6669, +} +# The latest checkpoint is at 30 steps, so no step after the 30th is kept. +MAX_STEPS = 30 +# A trajectory opens with the system prompt and the issue message; the steps follow. +_OPENING = 2 +_COLUMNS = ["instance_id", "model_name", "target", "trajectory", "exit_status"] +_ISSUE = re.compile(r"ISSUE:\n(.*?)\n\nINSTRUCTIONS:", re.DOTALL) +# An instance id is "__-". +_INSTANCE = re.compile(r"(?P.+__.+)-\d+") +# SWE-agent's ThoughtActionParser: an opening fence may name a language, a closing fence is bare. +_FENCE = re.compile(r"^```(\S*)\s*\n|^```\s*$", re.MULTILINE) + + +@unique +class Ending(StrEnum): + """How a run ended, grouped from the release's ``exit_status``.""" + + SUBMITTED = "submitted" + CONTEXT_LIMIT = "context_limit" + EARLY_EXIT = "early_exit" + SUBMITTED_NO_PATCH = "submitted_no_patch" + FORMAT_ERRORS = "format_errors" + COST_LIMIT = "cost_limit" + + +@dataclass(frozen=True) +class ExitStatus: + """How the release's ``exit_status`` maps onto an ending, and how a status report says it.""" + + ending: Ending + phrase: str + """The end of the sentence "The run will go on for n more steps, then ...".""" + + +# Every exit_status of the release. "submitted (exit_x)" means SWE-agent hit limit x and submitted the +# changes so far by itself. +EXIT_STATUSES = { + "submitted": ExitStatus(Ending.SUBMITTED, "the agent will submit its changes"), + "submitted (exit_context)": ExitStatus( + Ending.CONTEXT_LIMIT, "it will reach the context limit and SWE-agent will submit its changes" + ), + "exit_context": ExitStatus(Ending.CONTEXT_LIMIT, "it will reach the context limit"), + "early_exit": ExitStatus(Ending.EARLY_EXIT, "it will end with the status early_exit"), + "submitted_no_patch": ExitStatus(Ending.SUBMITTED_NO_PATCH, "the agent will submit without a patch"), + "submitted (exit_format)": ExitStatus( + Ending.FORMAT_ERRORS, "SWE-agent will stop it after repeated malformed messages and submit its changes" + ), + "exit_format": ExitStatus(Ending.FORMAT_ERRORS, "SWE-agent will stop it after repeated malformed messages"), + "submitted (exit_cost)": ExitStatus( + Ending.COST_LIMIT, "it will reach its cost limit and SWE-agent will submit its changes" + ), + "exit_cost": ExitStatus(Ending.COST_LIMIT, "it will reach its cost limit"), +} + + +@unique +class ActionKind(StrEnum): + """What a step's command does, by its first word.""" + + EDIT = "edit" + CREATE = "create" + RUN = "run" + READ = "read" + SEARCH = "search" + SUBMIT = "submit" + OTHER = "other" + + +_ACTION_WORDS = { + "edit": ActionKind.EDIT, + "create": ActionKind.CREATE, + "python": ActionKind.RUN, + "python3": ActionKind.RUN, + "pytest": ActionKind.RUN, + "open": ActionKind.READ, + "goto": ActionKind.READ, + "scroll_down": ActionKind.READ, + "scroll_up": ActionKind.READ, + "cat": ActionKind.READ, + "search_dir": ActionKind.SEARCH, + "search_file": ActionKind.SEARCH, + "find_file": ActionKind.SEARCH, + "find": ActionKind.SEARCH, + "grep": ActionKind.SEARCH, + "ls": ActionKind.SEARCH, + "submit": ActionKind.SUBMIT, +} + + +@unique +class ObservationKind(StrEnum): + """What the environment's reply to a step shows, by the first rule of :func:`observation_kind` it meets.""" + + EDIT_REJECTED = "edit_rejected" + TIMEOUT = "timeout" + TRACEBACK = "traceback" + COMMAND_ERROR = "command_error" + SILENT_SUCCESS = "silent_success" + OUTPUT = "output" + + +_EDIT_REJECTED = "Your proposed edit has introduced new syntax error" +_TIMEOUT = "EXECUTION TIMED OUT" +_TRACEBACK = "Traceback (most recent call last)" +_SILENT_SUCCESS = "Your command ran successfully and did not produce any output" +_COMMAND_ERRORS = ("command not found", "No such file or directory", "syntax error near unexpected token") + + +def split_message(text: str) -> tuple[str, str]: + """Split an agent message into its thought and its command, as SWE-agent's ThoughtActionParser does. + + The command is the content of the last complete top-level fenced block. The thought is the rest of + the message. A message without a complete block is all thought, with an empty command. + + Returns: + The thought and the command, both stripped. + """ + stack: list[re.Match[str]] = [] + last: tuple[re.Match[str], re.Match[str]] | None = None + for match in _FENCE.finditer(text): + if stack and not match.group(1): + start = stack.pop() + if not stack: + last = (start, match) + elif match.group(1) is not None: + stack.append(match) + if last is None: + return text.strip(), "" + start, end = last + return (text[: start.start()] + text[end.end() :]).strip(), text[start.end() : end.start()].strip() + + +def action_kind(command: str) -> ActionKind: + """Classify a command by its first word. + + Returns: + The kind; ``other`` for an empty command or a word not in the table. + """ + words = command.split(maxsplit=1) + return _ACTION_WORDS.get(words[0], ActionKind.OTHER) if words else ActionKind.OTHER + + +def observation_kind(text: str) -> ObservationKind: + """Classify the environment's reply, without its leading whitespace, by the first of these rules that holds. + + 1. It starts with SWE-agent's message for an edit that its linter rejected. + 2. It starts with ``EXECUTION TIMED OUT``. + 3. It contains a Python traceback, also one shown inside a file the agent opened or edited. + 4. Its first line starts with ``Usage:`` or ``Error:``, or names a shell error. + 5. It is SWE-agent's message for a command that printed nothing. + 6. Otherwise it is plain output. + + Returns: + The kind. + """ + # Some replies open with a blank line, e.g. the timeout message. + text = text.lstrip() + if text.startswith(_EDIT_REJECTED): + return ObservationKind.EDIT_REJECTED + if text.startswith(_TIMEOUT): + return ObservationKind.TIMEOUT + if _TRACEBACK in text: + return ObservationKind.TRACEBACK + first = text.split("\n", 1)[0] + if first.startswith(("Usage:", "Error:")) or any(error in first for error in _COMMAND_ERRORS): + return ObservationKind.COMMAND_ERROR + if text.startswith(_SILENT_SUCCESS): + return ObservationKind.SILENT_SUCCESS + return ObservationKind.OUTPUT + + +@dataclass(frozen=True) +class Steps: + """The per-step signals of a run's first steps, one entry per step.""" + + thought_chars: np.ndarray + action_chars: np.ndarray + observation_chars: np.ndarray + context_chars: np.ndarray + """The characters of the whole conversation after the step's reply, system prompt included.""" + repeated_action: np.ndarray + """Whether the step's command equals an earlier command of the run, character for character.""" + action_kind: tuple[ActionKind, ...] + observation_kind: tuple[ObservationKind, ...] + thoughts: bytes + """The thoughts, concatenated and compressed, because the release holds about 0.7 GB of them. + ``thought_chars`` holds where each one ends.""" + + def thought_texts(self, count: int) -> list[str]: + """The thoughts of the first ``count`` steps. + + Returns: + One string per step. + """ + text = zlib.decompress(self.thoughts).decode() + ends = np.cumsum(self.thought_chars[:count]) + return [text[end - length : end] for end, length in zip(ends, self.thought_chars[:count], strict=True)] + + +@dataclass(frozen=True) +class Run: + """One run of the release: its identity, its outcome, and the signals of its first steps.""" + + shard: str + row: int + instance_id: str + repository: str + model_name: str + resolved: bool + exit_status: str + issue: str + n_steps: int + """The run's step count. Its last step is the run's final action.""" + steps: Steps + + @property + def run_id(self) -> str: + """The run's place in the release: shard stem and row.""" + return f"{self.shard.removesuffix('.parquet')}:{self.row}" + + @property + def ending(self) -> Ending: + """How the run ended.""" + return EXIT_STATUSES[self.exit_status].ending + + +class Message(DataRow): + """One message of a trajectory, as the release stores it.""" + + model_config = ConfigDict(extra="ignore") + + role: str + text: str | None + system_prompt: str | None + + +class SourceRow(DataRow): + """The columns of a release row that the connector reads.""" + + instance_id: str + model_name: str + target: StrictBool + trajectory: list[Message] + exit_status: str + + +def _text(message: Message, index: int, context: str) -> str: + if message.text is None: + raise TimeNetConnectorError(f"{context}: message {index} has no text") + return message.text + + +def _steps(messages: list[Message], opening: int, context: str) -> tuple[int, Steps]: + """Parse the agent and environment messages that follow the issue message. + + Args: + messages: The messages after the issue message. + opening: The characters the agent reads before its first step: system prompt and issue message. + context: The row, for error messages. + + Returns: + The run's step count and the signals of its first :data:`MAX_STEPS` steps. + + Raises: + TimeNetConnectorError: If the messages do not alternate agent, environment, ending on the agent. + """ + for index, message in enumerate(messages): + if message.role != ("ai" if index % 2 == 0 else "user"): + raise TimeNetConnectorError(f"{context}: message {index + _OPENING} is {message.role!r} out of turn") + if messages and messages[-1].role != "ai": + raise TimeNetConnectorError(f"{context}: the trajectory does not end on an agent message") + texts = [_text(message, index + _OPENING, context) for index, message in enumerate(messages[: 2 * MAX_STEPS])] + n_steps = (len(messages) + 1) // 2 + kept = min(n_steps, MAX_STEPS) + numbers = np.zeros((4, kept), dtype=np.int32) + repeated = np.zeros(kept, dtype=bool) + actions, observations, thoughts, seen = [], [], [], set() + for step in range(kept): + thought, command = split_message(texts[2 * step]) + # The final step has no reply; every kept step of a checkpoint does. + reply = texts[2 * step + 1] if 2 * step + 1 < len(texts) else "" + numbers[:3, step] = len(thought), len(command), len(reply) + numbers[3, step] = len(texts[2 * step]) + len(reply) + repeated[step] = command in seen + seen.add(command) + actions.append(action_kind(command)) + observations.append(observation_kind(reply)) + thoughts.append(thought) + return n_steps, Steps( + thought_chars=numbers[0], + action_chars=numbers[1], + observation_chars=numbers[2], + context_chars=(opening + np.cumsum(numbers[3])).astype(np.int32), + repeated_action=repeated, + action_kind=tuple(actions), + observation_kind=tuple(observations), + thoughts=zlib.compress("".join(thoughts).encode()), + ) + + +def parse_run(row: SourceRow, shard: str, index: int) -> Run: + """Check one release row and parse it into a :class:`Run`. + + Returns: + The run. + + Raises: + TimeNetConnectorError: If the row breaks the release's shape: an unknown exit status, a + malformed instance id, a trajectory that does not open with the system prompt and the + issue, a message without text, or messages out of turn. + """ + context = f"{shard} row {index}" + instance = _INSTANCE.fullmatch(row.instance_id) + if instance is None: + raise TimeNetConnectorError(f"{context}: malformed instance_id {row.instance_id!r}") + if row.exit_status not in EXIT_STATUSES: + raise TimeNetConnectorError(f"{context}: unknown exit_status {row.exit_status!r}") + if not row.model_name: + raise TimeNetConnectorError(f"{context}: model_name is empty") + messages = row.trajectory + if len(messages) < _OPENING or messages[0].role != "system" or messages[1].role != "user": + raise TimeNetConnectorError(f"{context}: the trajectory must open with the system prompt and the issue") + system = messages[0] + # The system message holds the prompt in system_prompt; its text is empty in every row of the release. + if system.system_prompt is None or system.text: + raise TimeNetConnectorError(f"{context}: the system message must hold its prompt in system_prompt only") + issue_message = _text(messages[1], 1, context) + issue = _ISSUE.search(issue_message) + if issue is None: + raise TimeNetConnectorError(f"{context}: the first message holds no ISSUE section") + opening = len(system.system_prompt) + len(issue_message) + n_steps, steps = _steps(messages[_OPENING:], opening, context) + return Run( + shard=shard, + row=index, + instance_id=row.instance_id, + repository=instance["repository"], + model_name=row.model_name, + resolved=row.target, + exit_status=row.exit_status, + issue=issue.group(1), + n_steps=n_steps, + steps=steps, + ) + + +def read_runs(root: Path) -> Iterator[Run]: + """Read every run of the release, shard by shard, in row order. + + Args: + root: The directory that holds the parquet shards. + + Yields: + Each run. + + Raises: + TimeNetConnectorError: If a shard is missing, unreadable, or holds a different number of rows + than the pinned revision. + """ + # Each issue has many runs; they share one copy of its text. + issues: dict[str, str] = {} + for shard, expected in SHARD_ROWS.items(): + path = root / shard + if not path.is_file(): + raise TimeNetConnectorError(f"{path} is missing") + count = 0 + for row in ParquetDataReader(path, SourceRow, selected_columns=_COLUMNS).iter_rows(): + run = parse_run(row, shard, count) + yield dataclasses.replace(run, issue=issues.setdefault(run.issue, run.issue)) + count += 1 + if count != expected: + raise TimeNetConnectorError(f"{path}: expected {expected} rows, found {count}") diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py new file mode 100644 index 000000000..1af70aaa0 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tasks.py @@ -0,0 +1,135 @@ +"""Cut each run at fixed checkpoints and ask, from the steps so far, how the run will turn out. + +A checkpoint is the first ``k`` steps of a run that is still going after them: the run has more than +``k`` steps, so its final action is never among them. How a run ends (a submission, the context limit, +its length) would give its outcome away, and a monitor never sees it before the end. Each checkpoint +gives three tasks: will the run resolve its issue, how will it end, and a short status report. The +answers come from the release's ``target`` and ``exit_status`` and the steps' signals by fixed rules. +No model writes them. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass + +from timenet.types import Label, LabelSet +from timenet.types.splits import Split +from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + EXIT_STATUSES, + ActionKind, + Ending, + ObservationKind, + Run, +) +from timenet_connectors.splitting import StratifiedSplitter + + +CHECKPOINTS = (5, 10, 20, 30) + +RESOLVED = "resolved" +UNRESOLVED = "unresolved" +OUTCOME_LABELS = LabelSet( + name="run_outcome", + labels=( + Label(name=RESOLVED, description="The run's final patch passed the tests of the issue's pull request."), + Label(name=UNRESOLVED, description="The run did not resolve the issue."), + ), +) +ENDING_LABELS = LabelSet( + name="run_ending", + labels=( + Label(name=Ending.SUBMITTED, description="The agent submitted its changes."), + Label( + name=Ending.CONTEXT_LIMIT, + description="The run reached the model's context limit; SWE-agent may have submitted the changes so far.", + ), + Label(name=Ending.EARLY_EXIT, description="The release's early_exit status."), + Label(name=Ending.SUBMITTED_NO_PATCH, description="The agent submitted without a patch."), + Label( + name=Ending.FORMAT_ERRORS, + description="SWE-agent stopped the run after repeated malformed agent messages.", + ), + Label(name=Ending.COST_LIMIT, description="The run reached its cost limit."), + ), +) + +_CONTEXT = ( + "These are the first steps of a software engineering agent (SWE-agent) that is trying to resolve " + "the GitHub issue above. The run is still going." +) +RESOLVE_PROMPT = f"{_CONTEXT} Will the agent's final patch resolve the issue?" +ENDING_PROMPT = f"{_CONTEXT} How will the run end?" +STATUS_PROMPT = ( + f"{_CONTEXT} Report how the agent is doing so far, then say how many more steps the run will take, " + "how it will end and whether the agent will resolve the issue." +) + + +@dataclass(frozen=True) +class Checkpoint: + """The first ``steps`` steps of one run.""" + + run: Run + steps: int + + +def checkpoints(run: Run) -> list[Checkpoint]: + """List the checkpoints of a run: every ``k`` of :data:`CHECKPOINTS` that the run outlasts. + + Returns: + The checkpoints, earliest first. + """ + return [Checkpoint(run, k) for k in CHECKPOINTS if run.n_steps > k] + + +def split_runs(runs: list[Run]) -> dict[str, Split]: + """Assign each run to a partition by its repository, so no repository is in two partitions. + + A repository's runs share its code, and an issue's runs share the issue, so a model could + otherwise recognise a test run's repository or issue from training. + + Returns: + The partition of each run, keyed by run id. + """ + splitter = StratifiedSplitter( + train_folds=8, + validation_folds=1, + test_folds=1, + stratify_by=lambda run: run.resolved, + group_by=lambda run: run.repository, + seed=42, + ) + return {run.run_id: split for split, members in splitter.split(runs).items() for run in members} + + +def _count(number: int, singular: str, plural: str | None = None) -> str: + return f"{number} {singular if number == 1 else plural or singular + 's'}" + + +def status(checkpoint: Checkpoint) -> str: + """Write the status report of a checkpoint from its steps' signals and the run's outcome. + + Returns: + Three sentences on the steps so far, then one on how many more steps the run will take, how it + will end and whether the agent will resolve the issue. + """ + k, run = checkpoint.steps, checkpoint.run + actions = Counter(run.steps.action_kind[:k]) + replies = Counter(run.steps.observation_kind[:k]) + repeats = int(run.steps.repeated_action[:k].sum()) + looked = actions[ActionKind.READ] + actions[ActionKind.SEARCH] + rest = run.n_steps - k + return ( + f"In its first {k} steps the agent edited files {_count(actions[ActionKind.EDIT], 'time')}, " + f"created {_count(actions[ActionKind.CREATE], 'file')}, ran code {_count(actions[ActionKind.RUN], 'time')}, " + f"read or searched the repository {_count(looked, 'time')} " + f"and repeated an earlier command {_count(repeats, 'time')}. " + f"The linter rejected {_count(replies[ObservationKind.EDIT_REJECTED], 'edit')}, " + f"{_count(replies[ObservationKind.TRACEBACK], 'reply', 'replies')} showed a Python traceback, " + f"{_count(replies[ObservationKind.COMMAND_ERROR], 'reply', 'replies')} a command error " + f"and {_count(replies[ObservationKind.TIMEOUT], 'command')} timed out. " + f"The conversation holds {int(run.steps.context_chars[k - 1]):,} characters. " + f"The run will go on for {_count(rest, 'more step')}, then {EXIT_STATUSES[run.exit_status].phrase}, " + f"and the agent will {'resolve' if run.resolved else 'not resolve'} the issue." + ) diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/__init__.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py new file mode 100644 index 000000000..6a33604e7 --- /dev/null +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/nebius/swe_agent_trajectories/tests/test_connector.py @@ -0,0 +1,300 @@ +from collections import Counter +from itertools import pairwise +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from timenet.errors import TimeNetConnectorError +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, open_local_version +from timenet.types import AnswerTask, ClassificationTask +from timenet.types.text import read_value +from timenet_connectors.bases.huggingface import HubFileRef +from timenet_connectors.datasets.nebius.swe_agent_trajectories import SWEAgentTrajectoriesConnector, source +from timenet_connectors.datasets.nebius.swe_agent_trajectories.source import ( + ActionKind, + ObservationKind, + Run, + SourceRow, + action_kind, + observation_kind, + parse_run, + read_runs, + split_message, +) +from timenet_connectors.datasets.nebius.swe_agent_trajectories.tasks import checkpoints, status + + +# A synthetic release shaped like the Hub parquet: one shard of runs over twelve repositories, so the +# split has enough groups for its ten folds. The texts are hand-made, so the repo ships no dataset bytes. +SHARD = "train-00000-of-00012.parquet" +SYSTEM = "SETTING: You are an autonomous programmer." +ISSUE = "Parser crashes on empty input\nCalling parse('') raises IndexError." + + +def _message(role: str, text: str, system_prompt: str | None = None) -> dict: + return {"cutoff_date": None, "mask": role == "ai", "role": role, "system_prompt": system_prompt, "text": text} + + +def _agent(thought: str, command: str) -> str: + return f"{thought}\n\n```\n{command}\n```" + + +# One cycle of steps; a run of n steps repeats it and ends on a submit. +CYCLE = ( + (_agent("Let's reproduce the bug first.", "create reproduce.py"), "[File: /repo/reproduce.py (1 lines total)]"), + ( + _agent("Write the script.", "edit 1:1\nprint(parse(''))\nend_of_edit"), + "Your proposed edit has introduced new syntax error(s).", + ), + (_agent("Run it.", "python reproduce.py"), "Traceback (most recent call last):\n File ...\nIndexError"), + (_agent("Look at the parser.", "open src/parser.py"), "[File: /repo/src/parser.py (120 lines total)]"), + (_agent("Search for the call.", "search_dir parse"), 'Found 3 matches for "parse"'), +) + + +def _trajectory(n_steps: int) -> list[dict]: + messages = [_message("system", "", SYSTEM), _message("user", f"ISSUE:\n{ISSUE}\n\nINSTRUCTIONS:\nFix it.")] + for step in range(n_steps - 1): + agent, reply = CYCLE[step % len(CYCLE)] + messages += [_message("ai", agent), _message("user", reply)] + return [*messages, _message("ai", _agent("Done.", "submit"))] + + +def _row(repository: int, number: int, n_steps: int, *, resolved: bool, exit_status: str = "submitted") -> dict: + return { + "instance_id": f"owner{repository}__repo{repository}-{number}", + "model_name": "swe-agent-llama-70b", + "target": resolved, + "trajectory": _trajectory(n_steps), + "exit_status": exit_status, + "generated_patch": "", + "eval_logs": "", + } + + +def _parse(row: dict) -> Run: + # The reader selects the columns the connector reads; the patch and the test log are not among them. + return parse_run(SourceRow.model_validate({name: row[name] for name in SourceRow.model_fields}), SHARD, 0) + + +def _rows() -> list[dict]: + rows = [] + for repository in range(12): + # Instance 3 has only a 4-step run, so no checkpoint and no issue annotation. + rows.append(_row(repository, 3, 4, resolved=True)) + rows.append(_row(repository, 1, 12, resolved=repository % 3 == 0)) + rows.append(_row(repository, 2, 33, resolved=False, exit_status="submitted (exit_context)")) + return rows + + +def _write(root: Path, rows: list[dict]) -> Path: + root.mkdir(parents=True, exist_ok=True) + pq.write_table(pa.Table.from_pylist(rows), root / SHARD) + return root + + +@pytest.fixture +def one_shard(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + def release(rows: list[dict]) -> Path: + monkeypatch.setattr(source, "SHARD_ROWS", {SHARD: len(rows)}) + return _write(tmp_path / "release", rows) + + return release + + +@pytest.fixture +def dataset(one_shard): + return SWEAgentTrajectoriesConnector().convert([HubFileRef(one_shard(_rows()), "train")]) + + +def _target(task) -> str: + assert task.targets is not None + return read_value(task.targets[0]) + + +def _thoughts(record) -> list[str]: + (annotation,) = record.annotations + assert annotation.key == "agent_thoughts" + value = read_value(annotation.value) + assert isinstance(value, list) + return value + + +def test_split_message_takes_the_last_fenced_block_as_the_command() -> None: + text = "Here is code:\n```python\nx = 1\n```\nNow run it.\n```\npython x.py\n```\n" + assert split_message(text) == ("Here is code:\n```python\nx = 1\n```\nNow run it.", "python x.py") + assert split_message("Only thinking.") == ("Only thinking.", "") + + +@pytest.mark.parametrize( + ("command", "kind"), + [ + ("edit 1:2\nx\nend_of_edit", ActionKind.EDIT), + ("create a.py", ActionKind.CREATE), + ("python a.py", ActionKind.RUN), + ("goto 40", ActionKind.READ), + ("find_file a.py", ActionKind.SEARCH), + ("submit", ActionKind.SUBMIT), + ("git diff", ActionKind.OTHER), + ("", ActionKind.OTHER), + ], +) +def test_action_kind_reads_the_first_word(command: str, kind: ActionKind) -> None: + assert action_kind(command) == kind + + +@pytest.mark.parametrize( + ("reply", "kind"), + [ + ("Your proposed edit has introduced new syntax error(s). Traceback (most recent call last)", "edit_rejected"), + ("\nEXECUTION TIMED OUT", "timeout"), + ("Running...\nTraceback (most recent call last):\n", "traceback"), + ("Usage: open ", "command_error"), + ("/bin/bash: line 1: foo: command not found", "command_error"), + ("Your command ran successfully and did not produce any output.", "silent_success"), + ("[File: /repo/a.py (10 lines total)]", "output"), + ], +) +def test_observation_kind_applies_its_rules_in_order(reply: str, kind: str) -> None: + assert observation_kind(reply) == ObservationKind(kind) + + +def test_a_run_keeps_its_first_steps_and_its_outcome_apart() -> None: + run = _parse(_row(0, 1, 7, resolved=False)) + assert run.n_steps == 7 + assert run.issue == ISSUE + assert run.repository == "owner0__repo0" + assert list(run.steps.action_kind) == ["create", "edit", "run", "read", "search", "create", "submit"] + assert list(run.steps.observation_kind[:3]) == ["output", "edit_rejected", "traceback"] + # The final step has no reply. + assert run.steps.observation_chars[-1] == 0 + assert list(run.steps.repeated_action) == [False, False, False, False, False, True, False] + opening = len(SYSTEM) + len(f"ISSUE:\n{ISSUE}\n\nINSTRUCTIONS:\nFix it.") + first = len(CYCLE[0][0]) + len(CYCLE[0][1]) + assert run.steps.context_chars[0] == opening + first + assert all(later > earlier for earlier, later in pairwise(run.steps.context_chars)) + assert run.steps.thought_texts(2) == ["Let's reproduce the bug first.", "Write the script."] + + +def test_a_thought_is_kept_as_written() -> None: + row = _row(0, 1, 7, resolved=False) + row["trajectory"][2]["text"] = _agent("A thought with a NUL \0 and ```inline``` fences.", "ls") + run = _parse(row) + assert run.steps.thought_texts(2) == ["A thought with a NUL \0 and ```inline``` fences.", "Write the script."] + assert run.steps.thought_chars[0] == len(run.steps.thought_texts(1)[0]) + + +def test_only_a_run_that_goes_on_gets_a_checkpoint() -> None: + assert [c.steps for c in checkpoints(_parse(_row(0, 1, 5, resolved=False)))] == [] + assert [c.steps for c in checkpoints(_parse(_row(0, 1, 6, resolved=False)))] == [5] + assert [c.steps for c in checkpoints(_parse(_row(0, 1, 33, resolved=False)))] == [5, 10, 20, 30] + + +def test_records_hold_the_steps_so_far_and_nothing_about_the_outcome(dataset) -> None: + records = list(dataset.records) + # Per repository: the 4-step run gives none, the 12-step run 2, the 33-step run 4. + assert len(records) == 12 * 6 + for record in records: + k = record.metadata["checkpoint_steps"] + assert set(record.metadata) == {"run_id", "instance_id", "repository", "model_name", "checkpoint_steps"} + assert {signal.name for signal in record.sources[0].signals} == set(source.Steps.__annotations__) - {"thoughts"} + assert all(signal.n_values == k for signal in record.sources[0].signals) + assert len(_thoughts(record)) == k + # The last visible step is never the run's submit. + kinds = next(s for s in record.sources[0].signals if s.name == "action_kind") + assert "submit" not in kinds.to_arrow().to_pylist() + + +def test_each_checkpoint_gives_three_tasks_with_the_issue_and_thoughts_as_input(dataset) -> None: + tasks = list(dataset.iter_tasks()) + assert len(tasks) == 3 * 12 * 6 + assert Counter(type(task) for task in tasks) == {ClassificationTask: 2 * 72, AnswerTask: 72} + # One issue annotation per instance with a checkpoint: instances 1 and 2 of each repository. + issues = [a for a in dataset.annotations if a.key == "issue"] + assert len(issues) == 24 + assert not any(a.metadata["instance_id"].endswith("-3") for a in issues) + for task in tasks: + assert [a.key for a in task.input_annotations] == ["issue", "agent_thoughts"] + assert task.input_annotations[0].metadata["instance_id"] == task.metadata["instance_id"] + assert task.metadata == task.inputs[0].metadata + endings = { + read_value(t.targets[0]) + for t in tasks + if isinstance(t, ClassificationTask) and t.label_set.name == "run_ending" + } + assert endings == {"submitted", "context_limit"} + + +def test_the_status_report_states_the_steps_so_far_and_what_comes() -> None: + run = _parse(_row(0, 1, 12, resolved=True)) + (first, *_) = checkpoints(run) + assert status(first) == ( + "In its first 5 steps the agent edited files 1 time, created 1 file, ran code 1 time, " + "read or searched the repository 2 times and repeated an earlier command 0 times. " + "The linter rejected 1 edit, 1 reply showed a Python traceback, 0 replies a command error " + f"and 0 commands timed out. The conversation holds {int(run.steps.context_chars[4]):,} characters. " + "The run will go on for 7 more steps, then the agent will submit its changes, and the agent will " + "resolve the issue." + ) + + +def test_no_repository_is_in_two_partitions(dataset) -> None: + splits: dict[str, set] = {} + for task in dataset.iter_tasks(): + splits.setdefault(task.metadata["repository"], set()).add(task.split) + assert all(len(found) == 1 for found in splits.values()) + assert len({split for found in splits.values() for split in found}) == 3 + + +def test_dataset_survives_storage_round_trip(dataset, tmp_path: Path) -> None: + registry = LocalRegistry(tmp_path / "registry") + registry.put_dataset(dataset) + with TimeFReader(open_local_version(registry.root / "nebius/swe_agent_trajectories/1.0.0")) as reader: + restored = reader.read() + record = next(iter(restored.records)) + signals = {signal.name: signal for signal in record.sources[0].signals} + assert signals["action_kind"].to_arrow().to_pylist() == ["create", "edit", "run", "read", "search"] + assert signals["observation_kind"].to_arrow().to_pylist()[1] == "edit_rejected" + assert _thoughts(record)[0] == "Let's reproduce the bug first." + restored_tasks = list(restored.iter_tasks()) + original = list(dataset.iter_tasks()) + assert [t.split for t in restored_tasks] == [t.split for t in original] + assert [_target(t) for t in restored_tasks] == [_target(t) for t in original] + + +@pytest.mark.parametrize( + ("change", "message"), + [ + (lambda row: row.update(exit_status="exit_unknown"), "unknown exit_status"), + (lambda row: row.update(instance_id="no-separator"), "malformed instance_id"), + (lambda row: row["trajectory"][1].update(text="no issue here"), "no ISSUE section"), + (lambda row: row["trajectory"][3].update(role="ai"), "out of turn"), + (lambda row: row["trajectory"].pop(), "does not end on an agent message"), + (lambda row: row["trajectory"].pop(0), "must open with the system prompt"), + (lambda row: row["trajectory"][2].update(text=None), "message 2 has no text"), + (lambda row: row["trajectory"][0].update(text="more prompt"), "system_prompt only"), + (lambda row: row.update(model_name=""), "model_name is empty"), + ], +) +def test_a_malformed_row_fails_the_build(change, message: str) -> None: + row = _row(0, 1, 6, resolved=False) + change(row) + with pytest.raises(TimeNetConnectorError, match=message): + _parse(row) + + +def test_a_row_without_a_target_fails_the_build(one_shard) -> None: + rows = _rows() + rows[1]["target"] = None + with pytest.raises(TimeNetConnectorError, match="target"): + list(read_runs(one_shard(rows))) + + +def test_a_shard_with_another_row_count_fails_the_build(one_shard, monkeypatch: pytest.MonkeyPatch) -> None: + root = one_shard(_rows()) + monkeypatch.setattr(source, "SHARD_ROWS", {SHARD: 1}) + with pytest.raises(TimeNetConnectorError, match="expected 1 rows"): + list(read_runs(root))