From 36bb1abe26f57040372cf6ec77bd074ef7bd0f32 Mon Sep 17 00:00:00 2001 From: Nico Hinderling Date: Wed, 18 Nov 2026 16:05:43 -0800 Subject: [PATCH 1/3] fix(seer): Support single-run Autofix responses Single-run Autofix finds the root cause, plans and fixes in one run. explain waited for the whole run, which can pass the 6-minute timeout, and plan sent a step continuation that 404s. Stop polling once a single run has its root cause, read relevant_repos from its root cause artifact, and have plan explain that there is no separate plan step. --- packages/cli/src/commands/issue/plan.ts | 8 ++++ packages/cli/src/commands/issue/utils.ts | 14 ++++++- packages/cli/src/types/seer.ts | 14 +++++-- .../cli/test/commands/issue/plan.func.test.ts | 12 ++++++ .../cli/test/commands/issue/utils.test.ts | 42 +++++++++++++++++++ packages/cli/test/types/seer.test.ts | 30 +++++++++++++ 6 files changed, 116 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/issue/plan.ts b/packages/cli/src/commands/issue/plan.ts index 33359a146..ea4a52aa6 100644 --- a/packages/cli/src/commands/issue/plan.ts +++ b/packages/cli/src/commands/issue/plan.ts @@ -30,6 +30,7 @@ import { extractNoSolutionReason, extractRootCauses, extractSolution, + isSingleRun, requireAutofixRunId, type SolutionArtifact, } from "../../types/seer.js"; @@ -284,6 +285,13 @@ export const planCommand = buildCommand({ json: flags.json, }); + if (isSingleRun(state)) { + throw new CliError( + "Seer plans and fixes this issue in the same run as the root cause, so there is no separate plan.\n" + + `Run ${sentry`issue explain ${issueArg.text}`} for the root cause, or open the issue in Sentry to see the fix.`, + ); + } + // Root cause analysis can pause waiting for the user to confirm the // root cause before planning. When running interactively, confirm in the // terminal and continue; otherwise point them at the Sentry UI. diff --git a/packages/cli/src/commands/issue/utils.ts b/packages/cli/src/commands/issue/utils.ts index fa5a57120..1f60e9ecf 100644 --- a/packages/cli/src/commands/issue/utils.ts +++ b/packages/cli/src/commands/issue/utils.ts @@ -61,7 +61,12 @@ import { buildIssueUrl } from "../../lib/sentry-web-urls.js"; import { setOrgProjectContext } from "../../lib/telemetry.js"; import { isAllDigits, shellQuote } from "../../lib/utils.js"; import type { SentryIssue } from "../../types/index.js"; -import { type AutofixState, isTerminalStatus } from "../../types/seer.js"; +import { + type AutofixState, + extractRootCauses, + isSingleRun, + isTerminalStatus, +} from "../../types/seer.js"; const log = logger.withTag("issue.utils"); @@ -1152,6 +1157,13 @@ function shouldStopPolling( if (isTerminalStatus(state.status)) { return true; } + if ( + stopOnWaitingForUser && + isSingleRun(state) && + extractRootCauses(state).length > 0 + ) { + return true; + } if ( stopOnWaitingForUser && (state.status === "WAITING_FOR_USER_RESPONSE" || diff --git a/packages/cli/src/types/seer.ts b/packages/cli/src/types/seer.ts index e607c21ba..f4cb6a2b6 100644 --- a/packages/cli/src/types/seer.ts +++ b/packages/cli/src/types/seer.ts @@ -52,6 +52,8 @@ export const STOPPING_POINTS = [ export type StoppingPoint = (typeof STOPPING_POINTS)[number]; +const SINGLE_RUN_FEATURE_ID = "autofix_single_run"; + // Progress Message export const ProgressMessageSchema = object({ @@ -189,6 +191,7 @@ export const AutofixStateSchema = looseObject({ run_id: optional(number()), /** Null for legacy runs predating this field. */ sentry_run_id: optional(nullable(string())), + feature_id: optional(string()), status: string(), /** * Why a failed run stopped, e.g. `out_of_credits`. Null when the run didn't @@ -264,6 +267,10 @@ export function isTerminalStatus(status: string): boolean { return TERMINAL_STATUSES.includes(status as AutofixStatus); } +export function isSingleRun(state: AutofixState): boolean { + return state.feature_id === SINGLE_RUN_FEATURE_ID; +} + // At least one of sentry_run_id/run_id is always present on a non-null // autofix state, so a missing run ID means a malformed response. export function requireAutofixRunId(state: AutofixState): string | number { @@ -309,6 +316,7 @@ type AgentRootCauseData = { five_whys?: string[]; reproduction_steps?: string[]; relevant_repo?: string | null; + relevant_repos?: string[]; }; /** @@ -337,9 +345,9 @@ function searchBlocksForAgentRootCause( id: 0, description, cause: agentData.cause, - relevant_repos: agentData.relevant_repo - ? [agentData.relevant_repo] - : undefined, + relevant_repos: + agentData.relevant_repos ?? + (agentData.relevant_repo ? [agentData.relevant_repo] : undefined), }; return [cause]; } diff --git a/packages/cli/test/commands/issue/plan.func.test.ts b/packages/cli/test/commands/issue/plan.func.test.ts index 1045829b6..162c9c421 100644 --- a/packages/cli/test/commands/issue/plan.func.test.ts +++ b/packages/cli/test/commands/issue/plan.func.test.ts @@ -181,6 +181,18 @@ describe("issue plan after solution planning", () => { expect(pollSpy).not.toHaveBeenCalled(); }); + test("points single runs at explain instead of requesting a plan", async () => { + analyzeSpy.mockResolvedValue({ + ...completedState(), + feature_id: "autofix_single_run", + }); + + await expect(runPlan(createMockContext())).rejects.toThrow( + /sentry issue explain IOS-1/, + ); + expect(triggerSolutionPlanning).not.toHaveBeenCalled(); + }); + test("throws when planning ends in ERROR", async () => { pollSpy.mockResolvedValue({ status: "ERROR", run_id: 42, steps: [] }); diff --git a/packages/cli/test/commands/issue/utils.test.ts b/packages/cli/test/commands/issue/utils.test.ts index cce70a051..f2e20cc74 100644 --- a/packages/cli/test/commands/issue/utils.test.ts +++ b/packages/cli/test/commands/issue/utils.test.ts @@ -1141,6 +1141,48 @@ describe("pollAutofixState", () => { expect(result.status).toBe("ERROR"); }); + test("stops once a single run has its root cause", async () => { + globalThis.fetch = async () => + new Response( + JSON.stringify({ + autofix: { + run_id: 12_345, + status: "PROCESSING", + feature_id: "autofix_single_run", + blocks: [ + { + artifacts: [ + { + key: "root_cause", + data: { + headline: "Regex rejects ids", + cause: "Too strict.", + }, + reason: "", + }, + ], + }, + ], + }, + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + }, + ); + + const result = await pollAutofixState({ + orgSlug: "test-org", + issueId: "123456789", + json: true, + stopOnWaitingForUser: true, + pollIntervalMs: 10, + timeoutMs: 100, + }); + + expect(result.status).toBe("PROCESSING"); + }); + test("stops at WAITING_FOR_USER_RESPONSE when stopOnWaitingForUser is true", async () => { globalThis.fetch = async () => new Response( diff --git a/packages/cli/test/types/seer.test.ts b/packages/cli/test/types/seer.test.ts index f43e8a1e4..accfc6f41 100644 --- a/packages/cli/test/types/seer.test.ts +++ b/packages/cli/test/types/seer.test.ts @@ -11,6 +11,7 @@ import { extractNoSolutionReason, extractRootCauses, extractSolution, + isSingleRun, isTerminalStatus, type RootCause, requireAutofixRunId, @@ -276,6 +277,35 @@ describe("extractRootCauses", () => { expect(causes[0]?.cause).toBe("The regex is too strict."); }); + test("reads relevant_repos from a single-run root cause artifact", () => { + const state = { + run_id: 103, + status: "COMPLETED", + feature_id: "autofix_single_run", + blocks: [ + { + artifacts: [ + { + key: "root_cause", + data: { + headline: "Regex rejects ids", + cause: "The regex is too strict.", + relevant_repos: ["org/api-server", "org/web"], + }, + reason: "", + }, + ], + }, + ], + } as unknown as AutofixState; + + expect(isSingleRun(state)).toBe(true); + expect(extractRootCauses(state)[0]?.relevant_repos).toEqual([ + "org/api-server", + "org/web", + ]); + }); + test("extracts root cause from agent artifact without relevant_repo", () => { const state = { run_id: 101, From d39fe7006cfa8310d6b89b8f02bad93b5fefe82f Mon Sep 17 00:00:00 2001 From: Nico Hinderling Date: Wed, 18 Nov 2026 16:31:34 -0800 Subject: [PATCH 2/3] feat(seer): Show the single-run fix in issue plan Single runs have no separate plan step but already write the fix, so plan now waits for the run and shows the root cause, changed files and any PR, or why Seer stopped. Failed runs still error. --- packages/cli/src/commands/issue/explain.ts | 12 +- packages/cli/src/commands/issue/plan.ts | 110 ++++++++++- packages/cli/src/lib/formatters/seer.ts | 1 + packages/cli/src/types/seer.ts | 72 ++++++++ .../test/commands/issue/explain.func.test.ts | 15 ++ .../cli/test/commands/issue/plan.func.test.ts | 171 ++++++++++++++++-- 6 files changed, 361 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/commands/issue/explain.ts b/packages/cli/src/commands/issue/explain.ts index 0c8d9034a..241758e6d 100644 --- a/packages/cli/src/commands/issue/explain.ts +++ b/packages/cli/src/commands/issue/explain.ts @@ -23,7 +23,7 @@ import { } from "../../lib/list-command.js"; import type { IssueArgument } from "../../lib/issue-arguments.js"; import { logger } from "../../lib/logger.js"; -import { extractRootCauses } from "../../types/seer.js"; +import { extractRootCauses, isSingleRun } from "../../types/seer.js"; import { type IssuesArguments, issuesPositional } from "./positional.js"; import { ensureRootCauseAnalysis, @@ -71,7 +71,13 @@ async function analyzeIssue( ); } - return { issue: issueArg.text, org, issueId, rootCauses }; + return { + issue: issueArg.text, + org, + issueId, + rootCauses, + singleRun: isSingleRun(state), + }; } catch (error) { if (error instanceof ApiError) { throw handleSeerApiError( @@ -194,7 +200,7 @@ export const explainCommand = buildCommand({ return isBatch ? undefined : { - hint: `To create a plan, run: ${sentry`issue plan ${primaryArg}`}`, + hint: `${results[0]?.singleRun ? "To see the fix" : "To create a plan"}, run: ${sentry`issue plan ${primaryArg}`}`, }; }, }); diff --git a/packages/cli/src/commands/issue/plan.ts b/packages/cli/src/commands/issue/plan.ts index ea4a52aa6..cd90c771f 100644 --- a/packages/cli/src/commands/issue/plan.ts +++ b/packages/cli/src/commands/issue/plan.ts @@ -6,12 +6,14 @@ */ import { isatty } from "node:tty"; +import { text } from "@sentry/toolkit-core/markdown"; import type { SentryContext } from "../../context.js"; import { triggerSolutionPlanning } from "../../lib/api-client.js"; import { sentry } from "../../lib/app-identity.js"; import { issueSelectorHelp } from "../../lib/arg-parsing.js"; import { buildCommand } from "../../lib/command.js"; import { ApiError, CliError, SeerError } from "../../lib/errors.js"; +import { renderMarkdown } from "../../lib/formatters/markdown.js"; import { CommandOutput } from "../../lib/formatters/output.js"; import { formatSolution, @@ -26,12 +28,19 @@ import { logger } from "../../lib/logger.js"; import { prompt } from "../../lib/prompt.js"; import { type AutofixState, + type CodeChange, + extractCodeChanges, extractExaminedFiles, extractNoSolutionReason, + extractPullRequests, extractRootCauses, + extractSingleRunStop, extractSolution, isSingleRun, + isTerminalStatus, requireAutofixRunId, + type PullRequestLink, + type SingleRunStop, type SolutionArtifact, } from "../../types/seer.js"; import { @@ -70,8 +79,50 @@ type PlanData = { solution: SolutionArtifact["data"] | null; /** Context about why no solution was produced. Only present when solution is null. */ no_solution_context?: NoSolutionContext; + code_changes?: CodeChange[]; + pull_requests?: PullRequestLink[]; + stop?: SingleRunStop; }; +function formatSingleRunFix(data: PlanData, changes: CodeChange[]): string { + const lines: string[] = []; + if (data.no_solution_context?.root_cause) { + lines.push( + `**Root cause:** ${text(data.no_solution_context.root_cause)}`, + "", + ); + } + + if (changes.length > 0) { + lines.push("## Code changes", ""); + for (const repo of new Set(changes.map((change) => change.repo))) { + lines.push(`**${text(repo)}**`, ""); + for (const change of changes.filter((c) => c.repo === repo)) { + lines.push( + `- ${text(change.path)} (+${change.added} −${change.removed})`, + ); + } + lines.push(""); + } + } else if (data.stop) { + lines.push(`**Seer stopped:** ${text(data.stop.reason)}`, ""); + lines.push(text(data.stop.suggestion), ""); + } else { + lines.push( + "Seer stopped after the root cause without changing code, for example because of the project's settings. Open the issue in Sentry for details.", + ); + } + + if (data.pull_requests?.length) { + lines.push("## Pull requests", ""); + for (const pr of data.pull_requests) { + lines.push(`- ${text(pr.url)}`); + } + } + + return renderMarkdown(lines.join("\n")); +} + /** * Format solution plan data for human-readable terminal output. * @@ -79,6 +130,9 @@ type PlanData = { * why no solution was produced. */ function formatPlanOutput(data: PlanData): string { + if (data.code_changes) { + return formatSingleRunFix(data, data.code_changes); + } if (data.solution) { return formatSolution({ key: "solution", data: data.solution }); } @@ -166,6 +220,51 @@ function buildPlanData(state: AutofixState): PlanData { return data; } +async function waitForSingleRun( + state: AutofixState, + org: string, + issueId: string, + json: boolean, + issueText: string, +): Promise { + const finalState = isTerminalStatus(state.status) + ? state + : await pollAutofixState({ + orgSlug: org, + issueId, + json, + timeoutMessage: + "Seer is still working on the fix after 6 minutes. Try again or check the issue in Sentry web UI.", + timeoutHint: + "The fix may still be in progress.\n" + + ` Or retry: ${sentry`issue plan ${issueText}`}`, + }); + + if (finalState.status === "ERROR") { + if (finalState.failure_reason === "out_of_credits") { + throw new SeerError("out_of_credits", org); + } + throw new Error("Autofix failed. Check the Sentry web UI for details."); + } + if (finalState.status === "CANCELLED") { + throw new Error("Autofix was cancelled."); + } + + const data: PlanData = { + ...buildPlanData(finalState), + code_changes: extractCodeChanges(finalState), + }; + const pullRequests = extractPullRequests(finalState); + if (pullRequests.length > 0) { + data.pull_requests = pullRequests; + } + const stop = extractSingleRunStop(finalState); + if (stop) { + data.stop = stop; + } + return data; +} + /** * Root cause analysis paused waiting for the user to confirm the identified root * cause before solution planning. When the session is interactive (a TTY and not @@ -286,9 +385,14 @@ export const planCommand = buildCommand({ }); if (isSingleRun(state)) { - throw new CliError( - "Seer plans and fixes this issue in the same run as the root cause, so there is no separate plan.\n" + - `Run ${sentry`issue explain ${issueArg.text}`} for the root cause, or open the issue in Sentry to see the fix.`, + return yield new CommandOutput( + await waitForSingleRun( + state, + org, + numericId, + flags.json, + issueArg.text, + ), ); } diff --git a/packages/cli/src/lib/formatters/seer.ts b/packages/cli/src/lib/formatters/seer.ts index 5ca99173c..d76fdc753 100644 --- a/packages/cli/src/lib/formatters/seer.ts +++ b/packages/cli/src/lib/formatters/seer.ts @@ -144,6 +144,7 @@ export type IssueExplainResult = { issueId: string; /** Root causes returned by Seer. */ rootCauses: RootCause[]; + singleRun?: boolean; }; /** Aggregate root-cause output for single- and multi-issue invocations. */ diff --git a/packages/cli/src/types/seer.ts b/packages/cli/src/types/seer.ts index f4cb6a2b6..0deb406a8 100644 --- a/packages/cli/src/types/seer.ts +++ b/packages/cli/src/types/seer.ts @@ -645,6 +645,78 @@ function findNoSolutionReasonInContainers( return searchContainersForNoSolutionReason(containers); } +export type CodeChange = { + repo: string; + path: string; + type: string; + added: number; + removed: number; +}; + +export type SingleRunStop = { reason: string; suggestion: string }; + +type MergedFilePatch = { + repo_name: string; + patch: { path: string; type: string; added: number; removed: number }; +}; + +type SingleRunBlock = WithArtifacts & { + merged_file_patches?: MergedFilePatch[]; +}; + +function singleRunBlocks(state: AutofixState): SingleRunBlock[] { + return (state as { blocks?: SingleRunBlock[] }).blocks ?? []; +} + +export function extractCodeChanges(state: AutofixState): CodeChange[] { + const byFile = new Map(); + for (const block of singleRunBlocks(state)) { + for (const { repo_name, patch } of block.merged_file_patches ?? []) { + byFile.set(`${repo_name}:${patch.path}`, { + repo: repo_name, + path: patch.path, + type: patch.type, + added: patch.added, + removed: patch.removed, + }); + } + } + return [...byFile.values()]; +} + +export type PullRequestLink = { + repo: string | undefined; + url: string; + number: number | undefined; +}; + +export function extractPullRequests(state: AutofixState): PullRequestLink[] { + const prStates = + (state as { repo_pr_states?: Record }).repo_pr_states ?? + {}; + return Object.values(prStates).flatMap((prState) => { + const result = safeParse(PullRequestInfoSchema, prState); + if (!(result.success && result.output.pr_url)) { + return []; + } + const { repo_name, pr_url, pr_number } = result.output; + return [{ repo: repo_name, url: pr_url, number: pr_number }]; + }); +} + +export function extractSingleRunStop( + state: AutofixState, +): SingleRunStop | undefined { + const stops = singleRunBlocks(state) + .flatMap((block) => block.artifacts ?? []) + .filter((artifact) => artifact.key === "stop"); + const data = stops.at(-1)?.data as Partial | undefined; + if (typeof data?.reason === "string" && typeof data.suggestion === "string") { + return { reason: data.reason, suggestion: data.suggestion }; + } + return; +} + /** * Extract file paths examined during root cause analysis. * diff --git a/packages/cli/test/commands/issue/explain.func.test.ts b/packages/cli/test/commands/issue/explain.func.test.ts index 42f42b004..f406f36d8 100644 --- a/packages/cli/test/commands/issue/explain.func.test.ts +++ b/packages/cli/test/commands/issue/explain.func.test.ts @@ -231,6 +231,21 @@ describe("issue explain multiple IDs", () => { }); }); + test("points single runs at the fix in the closing hint", async () => { + resolveSpy.mockResolvedValue({ org: "test-org", issueId: "1" }); + analyzeSpy.mockResolvedValue({ + ...sampleState(sampleCause("Single cause")), + feature_id: "autofix_single_run", + }); + + const { context, stdoutWrite } = createMockContext(); + const func = await loadCommand(explainCommand); + await func.call(context, { ...EXPLAIN_FLAGS, json: false }, "IOS-1"); + + const output = stdoutWrite.mock.calls.map((call) => call[0]).join(""); + expect(output).toContain("To see the fix, run: sentry issue plan IOS-1"); + }); + test("fails to parse when no issue ID is provided", async () => { const func = await loadCommand(explainCommand); await expect( diff --git a/packages/cli/test/commands/issue/plan.func.test.ts b/packages/cli/test/commands/issue/plan.func.test.ts index 162c9c421..ee2e0cc9c 100644 --- a/packages/cli/test/commands/issue/plan.func.test.ts +++ b/packages/cli/test/commands/issue/plan.func.test.ts @@ -63,9 +63,12 @@ function createMockContext() { }; } -async function runPlan(context: ReturnType) { +async function runPlan( + context: ReturnType, + flags: { json: boolean; force: boolean; fresh: boolean } = PLAN_FLAGS, +) { const func = await loadCommand(planCommand); - await func.call(context, PLAN_FLAGS, "IOS-1"); + await func.call(context, flags, "IOS-1"); } describe("issue plan waiting-for-user-response", () => { @@ -181,18 +184,6 @@ describe("issue plan after solution planning", () => { expect(pollSpy).not.toHaveBeenCalled(); }); - test("points single runs at explain instead of requesting a plan", async () => { - analyzeSpy.mockResolvedValue({ - ...completedState(), - feature_id: "autofix_single_run", - }); - - await expect(runPlan(createMockContext())).rejects.toThrow( - /sentry issue explain IOS-1/, - ); - expect(triggerSolutionPlanning).not.toHaveBeenCalled(); - }); - test("throws when planning ends in ERROR", async () => { pollSpy.mockResolvedValue({ status: "ERROR", run_id: 42, steps: [] }); @@ -273,3 +264,155 @@ describe("issue plan after solution planning", () => { expect(pollSpy).toHaveBeenCalled(); }); }); + +function singleRunState( + status: string, + extra: { artifacts?: unknown[]; merged_file_patches?: unknown[] } = {}, + repoPrStates: Record = {}, +): AutofixState { + return { + status, + run_id: 42, + feature_id: "autofix_single_run", + repo_pr_states: repoPrStates, + blocks: [ + { + artifacts: [ + { + key: "root_cause", + data: { headline: "Regex rejects ids", cause: "Too strict." }, + }, + ...(extra.artifacts ?? []), + ], + merged_file_patches: extra.merged_file_patches ?? [], + }, + ], + }; +} + +function filePatch(path: string, added: number, removed: number) { + return { + repo_name: "org/api", + patch: { path, type: "M", added, removed, hunks: [] }, + diff: "@@", + }; +} + +function writtenOutput(context: ReturnType): string { + return context.stdout.write.mock.calls + .map((call) => String(call[0])) + .join(""); +} + +describe("issue plan on single runs", () => { + let analyzeSpy: ReturnType; + let pollSpy: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(isatty).mockReturnValue(false); + vi.spyOn(issueUtils, "resolveOrgAndIssueId").mockResolvedValue({ + org: "test-org", + issueId: "1", + }); + analyzeSpy = vi.spyOn(issueUtils, "ensureRootCauseAnalysis"); + pollSpy = vi.spyOn(issueUtils, "pollAutofixState"); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + const fixedState = () => + singleRunState( + "COMPLETED", + { merged_file_patches: [filePatch("src/ids.ts", 3, 1)] }, + { + "org/api": { + repo_name: "org/api", + pr_number: 7, + pr_url: "https://github.com/org/api/pull/7", + }, + }, + ); + + test("shows the changed files and PR of a completed run", async () => { + analyzeSpy.mockResolvedValue(fixedState()); + + const human = createMockContext(); + await runPlan(human); + const output = writtenOutput(human); + expect(output).toContain("Regex rejects ids"); + expect(output).toContain("src/ids.ts (+3 −1)"); + expect(output).toContain("https://github.com/org/api/pull/7"); + + const json = createMockContext(); + await runPlan(json, { ...PLAN_FLAGS, json: true }); + expect(JSON.parse(writtenOutput(json))).toMatchObject({ + solution: null, + code_changes: [ + { + repo: "org/api", + path: "src/ids.ts", + type: "M", + added: 3, + removed: 1, + }, + ], + pull_requests: [ + { + repo: "org/api", + url: "https://github.com/org/api/pull/7", + number: 7, + }, + ], + }); + expect(triggerSolutionPlanning).not.toHaveBeenCalled(); + expect(pollSpy).not.toHaveBeenCalled(); + }); + + test("shows why Seer stopped when it changed no code", async () => { + analyzeSpy.mockResolvedValue( + singleRunState("COMPLETED", { + artifacts: [ + { + key: "stop", + data: { + reason: "Fix needs a migration.", + suggestion: "Add it by hand.", + }, + }, + ], + }), + ); + + const context = createMockContext(); + await runPlan(context); + + const output = writtenOutput(context); + expect(output).toContain("Fix needs a migration."); + expect(output).toContain("Add it by hand."); + }); + + test("throws when the run failed", async () => { + analyzeSpy.mockResolvedValue(singleRunState("ERROR")); + + await expect(runPlan(createMockContext())).rejects.toThrow( + /Autofix failed/, + ); + }); + + test("waits for a run still writing the fix", async () => { + analyzeSpy.mockResolvedValue(singleRunState("PROCESSING")); + pollSpy.mockResolvedValue(fixedState()); + + const context = createMockContext(); + await runPlan(context); + + expect(pollSpy).toHaveBeenCalledWith( + expect.not.objectContaining({ stopOnWaitingForUser: true }), + ); + expect(triggerSolutionPlanning).not.toHaveBeenCalled(); + expect(writtenOutput(context)).toContain("src/ids.ts (+3 −1)"); + }); +}); From 2df9f9ddf868425f3062e37c1e7a230587af128c Mon Sep 17 00:00:00 2001 From: Nico Hinderling Date: Wed, 18 Nov 2026 17:33:55 -0800 Subject: [PATCH 3/3] test(seer): Share single-run fixtures in issue tests Build the single-run states from one small helper and fold the single-run plan tests into the existing post-root-cause suite, so the PR's tests cover the same behavior with far less inline fixture data. --- .../cli/test/commands/issue/plan.func.test.ts | 146 +++++++----------- .../cli/test/commands/issue/utils.test.ts | 35 +---- packages/cli/test/types/seer.test.ts | 31 +--- 3 files changed, 62 insertions(+), 150 deletions(-) diff --git a/packages/cli/test/commands/issue/plan.func.test.ts b/packages/cli/test/commands/issue/plan.func.test.ts index ee2e0cc9c..b0c8127fd 100644 --- a/packages/cli/test/commands/issue/plan.func.test.ts +++ b/packages/cli/test/commands/issue/plan.func.test.ts @@ -65,10 +65,10 @@ function createMockContext() { async function runPlan( context: ReturnType, - flags: { json: boolean; force: boolean; fresh: boolean } = PLAN_FLAGS, + json = false, ) { const func = await loadCommand(planCommand); - await func.call(context, flags, "IOS-1"); + await func.call(context, { ...PLAN_FLAGS, json }, "IOS-1"); } describe("issue plan waiting-for-user-response", () => { @@ -140,11 +140,48 @@ describe("issue plan waiting-for-user-response", () => { }); }); +function singleRunState(status: string, ...blocks: unknown[]): AutofixState { + const rootCause = { + key: "root_cause", + data: { headline: "Regex rejects ids" }, + }; + return { + status, + run_id: 42, + feature_id: "autofix_single_run", + blocks: [{ artifacts: [rootCause] }, ...blocks], + }; +} + +const fixedState = (): AutofixState => ({ + ...singleRunState("COMPLETED", { + merged_file_patches: [ + { + repo_name: "org/api", + patch: { path: "src/ids.ts", type: "M", added: 3, removed: 1 }, + }, + ], + }), + repo_pr_states: { + "org/api": { + repo_name: "org/api", + pr_number: 7, + pr_url: "https://github.com/org/api/pull/7", + }, + }, +}); + +function writtenOutput(context: ReturnType): string { + return context.stdout.write.mock.calls + .map((call) => String(call[0])) + .join(""); +} + function completedState(): AutofixState { return { status: "COMPLETED", run_id: 42, steps: [] }; } -describe("issue plan after solution planning", () => { +describe("issue plan after root cause analysis", () => { let resolveSpy: ReturnType; let analyzeSpy: ReturnType; let pollSpy: ReturnType; @@ -263,80 +300,8 @@ describe("issue plan after solution planning", () => { expect(triggerSolutionPlanning).toHaveBeenCalledWith("test-org", "1", 42); expect(pollSpy).toHaveBeenCalled(); }); -}); - -function singleRunState( - status: string, - extra: { artifacts?: unknown[]; merged_file_patches?: unknown[] } = {}, - repoPrStates: Record = {}, -): AutofixState { - return { - status, - run_id: 42, - feature_id: "autofix_single_run", - repo_pr_states: repoPrStates, - blocks: [ - { - artifacts: [ - { - key: "root_cause", - data: { headline: "Regex rejects ids", cause: "Too strict." }, - }, - ...(extra.artifacts ?? []), - ], - merged_file_patches: extra.merged_file_patches ?? [], - }, - ], - }; -} - -function filePatch(path: string, added: number, removed: number) { - return { - repo_name: "org/api", - patch: { path, type: "M", added, removed, hunks: [] }, - diff: "@@", - }; -} - -function writtenOutput(context: ReturnType): string { - return context.stdout.write.mock.calls - .map((call) => String(call[0])) - .join(""); -} - -describe("issue plan on single runs", () => { - let analyzeSpy: ReturnType; - let pollSpy: ReturnType; - - beforeEach(() => { - vi.clearAllMocks(); - vi.mocked(isatty).mockReturnValue(false); - vi.spyOn(issueUtils, "resolveOrgAndIssueId").mockResolvedValue({ - org: "test-org", - issueId: "1", - }); - analyzeSpy = vi.spyOn(issueUtils, "ensureRootCauseAnalysis"); - pollSpy = vi.spyOn(issueUtils, "pollAutofixState"); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - const fixedState = () => - singleRunState( - "COMPLETED", - { merged_file_patches: [filePatch("src/ids.ts", 3, 1)] }, - { - "org/api": { - repo_name: "org/api", - pr_number: 7, - pr_url: "https://github.com/org/api/pull/7", - }, - }, - ); - test("shows the changed files and PR of a completed run", async () => { + test("shows the changed files and PR of a completed single run", async () => { analyzeSpy.mockResolvedValue(fixedState()); const human = createMockContext(); @@ -347,7 +312,7 @@ describe("issue plan on single runs", () => { expect(output).toContain("https://github.com/org/api/pull/7"); const json = createMockContext(); - await runPlan(json, { ...PLAN_FLAGS, json: true }); + await runPlan(json, true); expect(JSON.parse(writtenOutput(json))).toMatchObject({ solution: null, code_changes: [ @@ -371,30 +336,23 @@ describe("issue plan on single runs", () => { expect(pollSpy).not.toHaveBeenCalled(); }); - test("shows why Seer stopped when it changed no code", async () => { + test("shows why a single run stopped without changing code", async () => { + const stop = { + reason: "Fix needs a migration.", + suggestion: "Add it by hand.", + }; analyzeSpy.mockResolvedValue( - singleRunState("COMPLETED", { - artifacts: [ - { - key: "stop", - data: { - reason: "Fix needs a migration.", - suggestion: "Add it by hand.", - }, - }, - ], - }), + singleRunState("COMPLETED", { artifacts: [{ key: "stop", data: stop }] }), ); const context = createMockContext(); await runPlan(context); - const output = writtenOutput(context); - expect(output).toContain("Fix needs a migration."); - expect(output).toContain("Add it by hand."); + expect(writtenOutput(context)).toContain(stop.reason); + expect(writtenOutput(context)).toContain(stop.suggestion); }); - test("throws when the run failed", async () => { + test("throws when a single run failed", async () => { analyzeSpy.mockResolvedValue(singleRunState("ERROR")); await expect(runPlan(createMockContext())).rejects.toThrow( @@ -402,7 +360,7 @@ describe("issue plan on single runs", () => { ); }); - test("waits for a run still writing the fix", async () => { + test("waits for a single run still writing the fix", async () => { analyzeSpy.mockResolvedValue(singleRunState("PROCESSING")); pollSpy.mockResolvedValue(fixedState()); diff --git a/packages/cli/test/commands/issue/utils.test.ts b/packages/cli/test/commands/issue/utils.test.ts index f2e20cc74..785265979 100644 --- a/packages/cli/test/commands/issue/utils.test.ts +++ b/packages/cli/test/commands/issue/utils.test.ts @@ -1142,34 +1142,13 @@ describe("pollAutofixState", () => { }); test("stops once a single run has its root cause", async () => { - globalThis.fetch = async () => - new Response( - JSON.stringify({ - autofix: { - run_id: 12_345, - status: "PROCESSING", - feature_id: "autofix_single_run", - blocks: [ - { - artifacts: [ - { - key: "root_cause", - data: { - headline: "Regex rejects ids", - cause: "Too strict.", - }, - reason: "", - }, - ], - }, - ], - }, - }), - { - status: 200, - headers: { "Content-Type": "application/json" }, - }, - ); + const autofix = { + run_id: 12_345, + status: "PROCESSING", + feature_id: "autofix_single_run", + blocks: [{ artifacts: [{ key: "root_cause", data: { headline: "x" } }] }], + }; + globalThis.fetch = async () => Response.json({ autofix }); const result = await pollAutofixState({ orgSlug: "test-org", diff --git a/packages/cli/test/types/seer.test.ts b/packages/cli/test/types/seer.test.ts index accfc6f41..795912494 100644 --- a/packages/cli/test/types/seer.test.ts +++ b/packages/cli/test/types/seer.test.ts @@ -255,6 +255,7 @@ describe("extractRootCauses", () => { const state = { run_id: 102, status: "COMPLETED", + feature_id: "autofix_single_run", blocks: [ { artifacts: [ @@ -263,6 +264,7 @@ describe("extractRootCauses", () => { data: { headline: "Regex rejects ids", cause: "The regex is too strict.", + relevant_repos: ["org/api-server", "org/web"], }, reason: "", }, @@ -275,35 +277,8 @@ describe("extractRootCauses", () => { expect(causes).toHaveLength(1); expect(causes[0]?.description).toBe("Regex rejects ids"); expect(causes[0]?.cause).toBe("The regex is too strict."); - }); - - test("reads relevant_repos from a single-run root cause artifact", () => { - const state = { - run_id: 103, - status: "COMPLETED", - feature_id: "autofix_single_run", - blocks: [ - { - artifacts: [ - { - key: "root_cause", - data: { - headline: "Regex rejects ids", - cause: "The regex is too strict.", - relevant_repos: ["org/api-server", "org/web"], - }, - reason: "", - }, - ], - }, - ], - } as unknown as AutofixState; - + expect(causes[0]?.relevant_repos).toEqual(["org/api-server", "org/web"]); expect(isSingleRun(state)).toBe(true); - expect(extractRootCauses(state)[0]?.relevant_repos).toEqual([ - "org/api-server", - "org/web", - ]); }); test("extracts root cause from agent artifact without relevant_repo", () => {