From 19ebb09fc6afc1d4954237b09e0aaf45d7bdee1b Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Sun, 20 Sep 2026 22:17:06 +0900 Subject: [PATCH 01/34] fix(webview): detect dead webview renderer via heartbeat and auto-reload --- packages/types/src/vscode-extension-host.ts | 2 + src/core/webview/ClineProvider.ts | 50 +++++++++++ .../webview/__tests__/ClineProvider.spec.ts | 90 +++++++++++++++++++ src/core/webview/webviewMessageHandler.ts | 4 + webview-ui/src/App.tsx | 9 ++ webview-ui/src/__tests__/App.spec.tsx | 38 ++++++++ 6 files changed, 193 insertions(+) diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index 5f6b579779..84e03d0980 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -472,6 +472,7 @@ export interface WebviewMessage { | "getListApiConfiguration" | "customInstructions" | "webviewDidLaunch" + | "webviewHeartbeat" | "newTask" | "askResponse" | "terminalOperation" @@ -696,6 +697,7 @@ export interface WebviewMessage { ids?: string[] terminalOperation?: "continue" | "abort" messageTs?: number + timestamp?: number // For webviewHeartbeat restoreCheckpoint?: boolean historyPreviewCollapsed?: boolean filters?: { type?: string; search?: string; tags?: string[] } diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 86ce5d8e67..64c64f7dc5 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -223,6 +223,10 @@ export class ClineProvider private taskEventListeners: WeakMap void>> = new WeakMap() private currentWorkspacePath: string | undefined private _disposed = false + private lastWebviewHeartbeatAt = 0 + private webviewWatchdogInterval: ReturnType | null = null + private static readonly WEBVIEW_WATCHDOG_TICK_MS = 60_000 + private static readonly WEBVIEW_HEARTBEAT_STALE_MS = 90_000 private readonly _postStateToWebviewThrottled = debounce( async () => { try { @@ -842,6 +846,10 @@ export class ClineProvider this._disposed = true this._postStateToWebviewThrottled.cancel() + if (this.webviewWatchdogInterval) { + clearInterval(this.webviewWatchdogInterval) + this.webviewWatchdogInterval = null + } this.log("Disposing ClineProvider...") // Reject any tasks still waiting for a scheduler permit so they don't @@ -1082,6 +1090,9 @@ export class ClineProvider // and executes code based on the message that is received. this.setWebviewMessageListener(webviewView.webview) + // Detect a dead webview renderer process (gray screen) via heartbeat timeout. + this.startWebviewWatchdog() + // Initialize code index status subscription for the current workspace. this.updateCodeIndexStatusSubscription() @@ -1100,6 +1111,9 @@ export class ClineProvider // for this visibility listener panel. const viewStateDisposable = webviewView.onDidChangeViewState(() => { if (this.view?.visible) { + // Hidden webviews throttle timers, so grant a fresh grace + // window instead of counting throttled heartbeats as a crash. + this.updateWebviewHeartbeat() void this.postMessageToWebview({ type: "action", action: "didBecomeVisible" }) } else { this.logWebviewHiddenDiagnostics() @@ -1111,6 +1125,9 @@ export class ClineProvider // sidebar const visibilityDisposable = webviewView.onDidChangeVisibility(() => { if (this.view?.visible) { + // Hidden webviews throttle timers, so grant a fresh grace + // window instead of counting throttled heartbeats as a crash. + this.updateWebviewHeartbeat() void this.postMessageToWebview({ type: "action", action: "didBecomeVisible" }) } else { this.logWebviewHiddenDiagnostics() @@ -3381,6 +3398,39 @@ export class ClineProvider ) } + /** Records that the webview renderer is alive; called on every webviewHeartbeat message. */ + public updateWebviewHeartbeat(): void { + this.lastWebviewHeartbeatAt = Date.now() + } + + /** + * Starts (or restarts) the watchdog that detects a dead webview renderer + * process. Hidden webviews throttle timers, so becoming visible resets the + * grace window instead of counting throttled heartbeats as a crash. + */ + private startWebviewWatchdog(): void { + this.updateWebviewHeartbeat() + if (this.webviewWatchdogInterval) { + clearInterval(this.webviewWatchdogInterval) + } + this.webviewWatchdogInterval = setInterval(() => { + if (this.view?.visible !== true) { + return + } + if (Date.now() - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS) { + return + } + this.log("[Zoo Code] Webview heartbeat stale while visible; reloading webview (dead renderer?)") + void Promise.resolve(vscode.commands.executeCommand("workbench.action.webview.reloadWebviewAction")).catch( + (error) => { + this.log( + `[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`, + ) + }, + ) + }, ClineProvider.WEBVIEW_WATCHDOG_TICK_MS) + } + public getRecentTasks(): string[] { if (this.recentTasksCache) { return this.recentTasksCache diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 97c4dd877e..dfd165f678 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -663,6 +663,96 @@ describe("ClineProvider", () => { }) }) + describe("webview heartbeat watchdog", () => { + let visibilityCallback: () => void + + beforeEach(() => { + // Fake timers must be active before resolveWebviewView so the + // watchdog interval is registered on the fake clock. + vi.useFakeTimers() + mockWebviewView.onDidChangeVisibility = vi.fn().mockImplementation((cb: () => void) => { + visibilityCallback = cb + return { dispose: vi.fn() } + }) + }) + + afterEach(async () => { + await provider.dispose() + vi.useRealTimers() + }) + + test("does not reload the webview while heartbeats are fresh", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.executeCommand).mockClear() + + await vi.advanceTimersByTimeAsync(110_000) + await webviewMessageHandler(provider, { type: "webviewHeartbeat", timestamp: Date.now() }) + await vi.advanceTimersByTimeAsync(60_000) + + expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + }) + + test("reloads the webview when the heartbeat is stale and the view is visible", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.executeCommand).mockClear() + + await vi.advanceTimersByTimeAsync(120_000) + + expect(vscode.commands.executeCommand).toHaveBeenCalledTimes(1) + expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.webview.reloadWebviewAction") + }) + + test("does not reload while the view is hidden even when the heartbeat is stale", async () => { + await provider.resolveWebviewView(mockWebviewView) + Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) + vi.mocked(vscode.commands.executeCommand).mockClear() + + await vi.advanceTimersByTimeAsync(180_000) + + expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + }) + + test("resets the grace window when the view becomes visible", async () => { + await provider.resolveWebviewView(mockWebviewView) + Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) + vi.mocked(vscode.commands.executeCommand).mockClear() + + await vi.advanceTimersByTimeAsync(120_000) + expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + + Object.defineProperty(mockWebviewView, "visible", { value: true, configurable: true }) + visibilityCallback() + + await vi.advanceTimersByTimeAsync(60_000) + expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + + // Watchdog ticks every 60s; 120s after the flip the heartbeat is stale again. + await vi.advanceTimersByTimeAsync(60_000) + expect(vscode.commands.executeCommand).toHaveBeenCalledTimes(1) + expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.webview.reloadWebviewAction") + }) + + test("stops watching after the provider is disposed", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.executeCommand).mockClear() + + await provider.dispose() + await vi.advanceTimersByTimeAsync(180_000) + + expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + }) + + test("does not stack watchdog intervals when resolveWebviewView runs again", async () => { + await provider.resolveWebviewView(mockWebviewView) + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.executeCommand).mockClear() + + await vi.advanceTimersByTimeAsync(120_000) + + expect(vscode.commands.executeCommand).toHaveBeenCalledTimes(1) + }) + }) + test("resolveWebviewView sets up webview correctly in development mode even if local server is not running", async () => { provider = new ClineProvider( { ...mockContext, extensionMode: vscode.ExtensionMode.Development }, diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index 34a35ea3ca..d5408cca0b 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -688,6 +688,10 @@ export const webviewMessageHandler = async ( provider.isViewLaunched = true break + case "webviewHeartbeat": + // Timestamp-only update for the dead-renderer watchdog; no other side effects. + provider.updateWebviewHeartbeat() + break case "newTask": // Initializing new instance of Cline will make sure that any // agentically running promises in old instance don't affect our new diff --git a/webview-ui/src/App.tsx b/webview-ui/src/App.tsx index b1fbf82999..f4e80ac480 100644 --- a/webview-ui/src/App.tsx +++ b/webview-ui/src/App.tsx @@ -206,6 +206,15 @@ const App = () => { // Tell the extension that we are ready to receive messages. useEffect(() => vscode.postMessage({ type: "webviewDidLaunch" }), []) + // Heartbeat so the extension watchdog can detect a crashed webview renderer + // process (gray screen) and reload the view. + useEffect(() => { + const postHeartbeat = () => vscode.postMessage({ type: "webviewHeartbeat", timestamp: Date.now() }) + postHeartbeat() + const interval = setInterval(postHeartbeat, 30_000) + return () => clearInterval(interval) + }, []) + // Initialize source map support for better error reporting useEffect(() => { // Initialize source maps for better error reporting in production diff --git a/webview-ui/src/__tests__/App.spec.tsx b/webview-ui/src/__tests__/App.spec.tsx index 137bed5d70..4178b999e6 100644 --- a/webview-ui/src/__tests__/App.spec.tsx +++ b/webview-ui/src/__tests__/App.spec.tsx @@ -4,6 +4,7 @@ import React from "react" import { render, screen, act, cleanup } from "@/utils/test-utils" import AppWithProviders from "../App" +import { vscode } from "@src/utils/vscode" vi.mock("@src/utils/vscode", () => ({ vscode: { @@ -476,4 +477,41 @@ describe("App", () => { expect(chatView.getAttribute("data-hidden")).toBe("false") expect(screen.queryByTestId("marketplace-view")).not.toBeInTheDocument() }) + + describe("webview heartbeat", () => { + afterEach(() => { + vi.useRealTimers() + }) + + it("posts an immediate heartbeat on mount and then every 30 seconds", () => { + vi.useFakeTimers() + render() + + const postMessageMock = vi.mocked(vscode.postMessage) + expect(postMessageMock).toHaveBeenCalledWith( + expect.objectContaining({ type: "webviewHeartbeat", timestamp: expect.any(Number) }), + ) + const callsAfterMount = postMessageMock.mock.calls.length + + vi.advanceTimersByTime(30_000) + expect(postMessageMock.mock.calls.length).toBe(callsAfterMount + 1) + + vi.advanceTimersByTime(30_000) + expect(postMessageMock.mock.calls.length).toBe(callsAfterMount + 2) + expect(postMessageMock).toHaveBeenLastCalledWith(expect.objectContaining({ type: "webviewHeartbeat" })) + }) + + it("stops posting heartbeats after unmount", () => { + vi.useFakeTimers() + const { unmount } = render() + + const postMessageMock = vi.mocked(vscode.postMessage) + const callsAfterMount = postMessageMock.mock.calls.length + + unmount() + vi.advanceTimersByTime(90_000) + + expect(postMessageMock.mock.calls.length).toBe(callsAfterMount) + }) + }) }) From 77d02f3f85add531443765e5e2b580e868f0615a Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Sun, 20 Sep 2026 23:28:36 +0900 Subject: [PATCH 02/34] fix(webview): scope recovery reload to the provider's own webview --- src/core/webview/ClineProvider.ts | 39 +++++--- .../webview/__tests__/ClineProvider.spec.ts | 94 +++++++++++++++---- 2 files changed, 104 insertions(+), 29 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 64c64f7dc5..620a24cc8a 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1050,11 +1050,7 @@ export class ClineProvider localResourceRoots: resourceRoots, } - webviewView.webview.html = - this.contextProxy.extensionMode === vscode.ExtensionMode.Development && - process.env.ROO_CODE_THEME_FIXTURE_PROBE !== "1" - ? await this.getHMRHtmlContent(webviewView.webview) - : await this.getHtmlContent(webviewView.webview) + webviewView.webview.html = await this.getWebviewHtml(webviewView.webview) // Initialize out-of-scope variables that need to receive persistent // global state values. @@ -3421,16 +3417,35 @@ export class ClineProvider return } this.log("[Zoo Code] Webview heartbeat stale while visible; reloading webview (dead renderer?)") - void Promise.resolve(vscode.commands.executeCommand("workbench.action.webview.reloadWebviewAction")).catch( - (error) => { - this.log( - `[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`, - ) - }, - ) + void this.reloadWebviewForRecovery() }, ClineProvider.WEBVIEW_WATCHDOG_TICK_MS) } + /** + * Reloads only this provider's own webview by regenerating its HTML (fresh + * nonce) and reassigning `webview.html`, which forces VS Code to reload that + * webview. Works for both sidebar (WebviewView) and tab (WebviewPanel) shapes. + */ + private async reloadWebviewForRecovery(): Promise { + const view = this.view + if (!view?.webview) { + return + } + try { + view.webview.html = await this.getWebviewHtml(view.webview) + } catch (error) { + this.log(`[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`) + } + } + + /** Builds the webview HTML using the same path as resolveWebviewView (HMR in development). */ + private async getWebviewHtml(webview: vscode.Webview): Promise { + return this.contextProxy.extensionMode === vscode.ExtensionMode.Development && + process.env.ROO_CODE_THEME_FIXTURE_PROBE !== "1" + ? await this.getHMRHtmlContent(webview) + : await this.getHtmlContent(webview) + } + public getRecentTasks(): string[] { if (this.recentTasksCache) { return this.recentTasksCache diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index dfd165f678..b5e6277576 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -683,73 +683,133 @@ describe("ClineProvider", () => { test("does not reload the webview while heartbeats are fresh", async () => { await provider.resolveWebviewView(mockWebviewView) - vi.mocked(vscode.commands.executeCommand).mockClear() + const htmlAfterResolve = mockWebviewView.webview.html await vi.advanceTimersByTimeAsync(110_000) await webviewMessageHandler(provider, { type: "webviewHeartbeat", timestamp: Date.now() }) await vi.advanceTimersByTimeAsync(60_000) - expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) }) - test("reloads the webview when the heartbeat is stale and the view is visible", async () => { + test("reloads the webview with fresh HTML when the heartbeat is stale and the view is visible", async () => { await provider.resolveWebviewView(mockWebviewView) - vi.mocked(vscode.commands.executeCommand).mockClear() + const htmlAfterResolve = mockWebviewView.webview.html await vi.advanceTimersByTimeAsync(120_000) - expect(vscode.commands.executeCommand).toHaveBeenCalledTimes(1) - expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.webview.reloadWebviewAction") + // Reassigning webview.html with a fresh nonce is what forces the reload. + expect(mockWebviewView.webview.html).not.toBe(htmlAfterResolve) + expect(mockWebviewView.webview.html).toContain("Zoo Code") }) test("does not reload while the view is hidden even when the heartbeat is stale", async () => { await provider.resolveWebviewView(mockWebviewView) Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) - vi.mocked(vscode.commands.executeCommand).mockClear() + const htmlAfterResolve = mockWebviewView.webview.html await vi.advanceTimersByTimeAsync(180_000) - expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) }) test("resets the grace window when the view becomes visible", async () => { await provider.resolveWebviewView(mockWebviewView) Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) - vi.mocked(vscode.commands.executeCommand).mockClear() + const htmlAfterResolve = mockWebviewView.webview.html await vi.advanceTimersByTimeAsync(120_000) - expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) Object.defineProperty(mockWebviewView, "visible", { value: true, configurable: true }) visibilityCallback() + const htmlAfterBecomingVisible = mockWebviewView.webview.html await vi.advanceTimersByTimeAsync(60_000) - expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + expect(mockWebviewView.webview.html).toBe(htmlAfterBecomingVisible) // Watchdog ticks every 60s; 120s after the flip the heartbeat is stale again. await vi.advanceTimersByTimeAsync(60_000) - expect(vscode.commands.executeCommand).toHaveBeenCalledTimes(1) - expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.webview.reloadWebviewAction") + expect(mockWebviewView.webview.html).not.toBe(htmlAfterBecomingVisible) }) test("stops watching after the provider is disposed", async () => { await provider.resolveWebviewView(mockWebviewView) - vi.mocked(vscode.commands.executeCommand).mockClear() + const htmlAfterResolve = mockWebviewView.webview.html await provider.dispose() await vi.advanceTimersByTimeAsync(180_000) - expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) }) test("does not stack watchdog intervals when resolveWebviewView runs again", async () => { await provider.resolveWebviewView(mockWebviewView) await provider.resolveWebviewView(mockWebviewView) - vi.mocked(vscode.commands.executeCommand).mockClear() + + // getHtmlContent regenerates the HTML via one getState() call per reload, + // so getState invocations after this point count watchdog reloads. + const getStateSpy = vi.spyOn(provider, "getState") + + await vi.advanceTimersByTimeAsync(120_000) + + expect(getStateSpy).toHaveBeenCalledTimes(1) + }) + + test("logs when regenerating the reload HTML fails", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.spyOn(provider, "getState").mockRejectedValue(new Error("regen boom")) + ;(mockOutputChannel.appendLine as ReturnType).mockClear() await vi.advanceTimersByTimeAsync(120_000) - expect(vscode.commands.executeCommand).toHaveBeenCalledTimes(1) + expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( + expect.stringContaining("[Zoo Code] Failed to reload webview: regen boom"), + ) + }) + + test("reloads only its own webview when multiple providers are active", async () => { + // Structural stand-in for the VS Code webview API surface this scenario + // exercises, same as the mockContext cast below. + const mockWebviewViewB = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi.fn(), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + onDidDispose: vi.fn().mockImplementation((callback: () => void) => { + callback() + return { dispose: vi.fn() } + }), + onDidChangeVisibility: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), + } as unknown as vscode.WebviewView + const providerB = new ClineProvider( + mockContext, + mockOutputChannel, + "sidebar", + new ContextProxy(mockContext), + ) + try { + await provider.resolveWebviewView(mockWebviewView) + await providerB.resolveWebviewView(mockWebviewViewB) + + const htmlBeforeA = mockWebviewView.webview.html + const htmlBeforeB = mockWebviewViewB.webview.html + + await vi.advanceTimersByTimeAsync(119_000) + // Keep provider B's heartbeat fresh so only A's watchdog fires. + await webviewMessageHandler(providerB, { type: "webviewHeartbeat", timestamp: Date.now() }) + await vi.advanceTimersByTimeAsync(1_000) + + expect(mockWebviewView.webview.html).not.toBe(htmlBeforeA) + expect(mockWebviewViewB.webview.html).toBe(htmlBeforeB) + } finally { + await providerB.dispose() + } }) }) From dfbcc6ee00d656b499cdda93cdfbcaba90978a53 Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Mon, 21 Sep 2026 04:26:21 +0900 Subject: [PATCH 03/34] fix(webview): stop recovery watchdog when sidebar webview is disposed --- src/core/webview/ClineProvider.ts | 19 ++++++-- .../webview/__tests__/ClineProvider.spec.ts | 45 ++++++++++++------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 620a24cc8a..25d4bfb296 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -820,6 +820,7 @@ export class ClineProvider */ private clearWebviewResources() { this.rejectPendingThemeFixtureProbes(new Error("Webview was disposed before the theme fixture probe completed")) + this.stopWebviewWatchdog() while (this.webviewDisposables.length) { const x = this.webviewDisposables.pop() if (x) { @@ -846,10 +847,7 @@ export class ClineProvider this._disposed = true this._postStateToWebviewThrottled.cancel() - if (this.webviewWatchdogInterval) { - clearInterval(this.webviewWatchdogInterval) - this.webviewWatchdogInterval = null - } + this.stopWebviewWatchdog() this.log("Disposing ClineProvider...") // Reject any tasks still waiting for a scheduler permit so they don't @@ -1143,6 +1141,11 @@ export class ClineProvider } else { this.log("Clearing webview resources for sidebar view") this.clearWebviewResources() + if (this.view === webviewView) { + // Drop the disposed view so nothing keeps polling it + // (e.g. the recovery watchdog) for the provider's lifetime. + this.view = undefined + } // Reset current workspace manager reference when view is disposed this.codeIndexManager = undefined } @@ -3421,6 +3424,14 @@ export class ClineProvider }, ClineProvider.WEBVIEW_WATCHDOG_TICK_MS) } + /** Stops the renderer heartbeat watchdog; the webview it watches is gone. */ + private stopWebviewWatchdog(): void { + if (this.webviewWatchdogInterval) { + clearInterval(this.webviewWatchdogInterval) + this.webviewWatchdogInterval = null + } + } + /** * Reloads only this provider's own webview by regenerating its HTML (fresh * nonce) and reassigning `webview.html`, which forces VS Code to reload that diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index b5e6277576..420cfd904e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -527,10 +527,7 @@ describe("ClineProvider", () => { cspSource: "vscode-webview://test-csp-source", }, visible: true, - onDidDispose: vi.fn().mockImplementation((callback) => { - callback() - return { dispose: vi.fn() } - }), + onDidDispose: vi.fn(), onDidChangeVisibility: vi.fn().mockImplementation(() => { return { dispose: vi.fn() } }), @@ -665,6 +662,7 @@ describe("ClineProvider", () => { describe("webview heartbeat watchdog", () => { let visibilityCallback: () => void + let disposeCallback: () => void beforeEach(() => { // Fake timers must be active before resolveWebviewView so the @@ -674,6 +672,10 @@ describe("ClineProvider", () => { visibilityCallback = cb return { dispose: vi.fn() } }) + mockWebviewView.onDidDispose = vi.fn().mockImplementation((cb: () => void) => { + disposeCallback = cb + return { dispose: vi.fn() } + }) }) afterEach(async () => { @@ -743,6 +745,23 @@ describe("ClineProvider", () => { expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) }) + test("stops watching and clears the view when the sidebar webview is disposed", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + // The provider outlives a disposed sidebar view; VS Code re-resolves + // a fresh view later. Disposal must stop the watchdog and drop the + // stale view reference so no recovery reload targets the dead view. + disposeCallback() + expect(provider["webviewWatchdogInterval"]).toBeNull() + // @ts-ignore - accessing private property for testing + expect(provider.view).toBeUndefined() + + await vi.advanceTimersByTimeAsync(180_000) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + test("does not stack watchdog intervals when resolveWebviewView runs again", async () => { await provider.resolveWebviewView(mockWebviewView) await provider.resolveWebviewView(mockWebviewView) @@ -781,10 +800,7 @@ describe("ClineProvider", () => { cspSource: "vscode-webview://test-csp-source", }, visible: true, - onDidDispose: vi.fn().mockImplementation((callback: () => void) => { - callback() - return { dispose: vi.fn() } - }), + onDidDispose: vi.fn(), onDidChangeVisibility: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), } as unknown as vscode.WebviewView const providerB = new ClineProvider( @@ -801,7 +817,8 @@ describe("ClineProvider", () => { const htmlBeforeB = mockWebviewViewB.webview.html await vi.advanceTimersByTimeAsync(119_000) - // Keep provider B's heartbeat fresh so only A's watchdog fires. + // B's view is still alive; refresh its heartbeat so a stale one + // would reload it too, leaving only A's watchdog to fire. await webviewMessageHandler(providerB, { type: "webviewHeartbeat", timestamp: Date.now() }) await vi.advanceTimersByTimeAsync(1_000) @@ -3867,10 +3884,7 @@ describe("ClineProvider - Router Models", () => { asWebviewUri: vi.fn(), }, visible: true, - onDidDispose: vi.fn().mockImplementation((callback) => { - callback() - return { dispose: vi.fn() } - }), + onDidDispose: vi.fn(), onDidChangeVisibility: vi.fn().mockImplementation(() => { return { dispose: vi.fn() } }), @@ -4221,10 +4235,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => { asWebviewUri: vi.fn(), }, visible: true, - onDidDispose: vi.fn().mockImplementation((callback) => { - callback() - return { dispose: vi.fn() } - }), + onDidDispose: vi.fn(), onDidChangeVisibility: vi.fn().mockImplementation(() => { return { dispose: vi.fn() } }), From 36e6bcb757d30aa116d8456e245f38521c1ce333 Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Mon, 21 Sep 2026 05:35:12 +0900 Subject: [PATCH 04/34] test(webview): capture sidebar dispose callback in task history spec --- src/core/webview/__tests__/ClineProvider.taskHistory.spec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/webview/__tests__/ClineProvider.taskHistory.spec.ts b/src/core/webview/__tests__/ClineProvider.taskHistory.spec.ts index 2bbf0736c6..8b4cb5575f 100644 --- a/src/core/webview/__tests__/ClineProvider.taskHistory.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.taskHistory.spec.ts @@ -339,10 +339,7 @@ describe("ClineProvider Task History Synchronization", () => { cspSource: "vscode-webview://test-csp-source", }, visible: true, - onDidDispose: vi.fn().mockImplementation((callback) => { - callback() - return { dispose: vi.fn() } - }), + onDidDispose: vi.fn(), onDidChangeVisibility: vi.fn().mockImplementation(() => { return { dispose: vi.fn() } }), From ec7d3751e0e17ab13c80e4b30ab87b262b1cf188 Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Wed, 23 Sep 2026 06:35:55 +0900 Subject: [PATCH 05/34] fix(webview): invalidate in-flight recovery reload on dispose or replace The watchdog fires reloadWebviewForRecovery() without tracking it, so the recovery could reassign webview.html after the provider was disposed or the watched view was disposed/replaced while the HTML was being generated. Track the operation with an epoch token: capture it before the await and bail out before assigning webview.html when the provider is disposed, the epoch changed, or this.view no longer references the captured view. clearWebviewResources() bumps the epoch so sidebar view disposal and provider disposal invalidate any in-flight recovery. Also add fake-timer watchdog coverage for the tab-panel branch (WebviewPanel shape): a hidden tab does not reload, the onDidChangeViewState callback resets the heartbeat grace window when the tab becomes visible, and a stale heartbeat reloads a visible tab. --- src/core/webview/ClineProvider.ts | 19 ++- .../webview/__tests__/ClineProvider.spec.ts | 153 ++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 25d4bfb296..4be16135c2 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -225,6 +225,10 @@ export class ClineProvider private _disposed = false private lastWebviewHeartbeatAt = 0 private webviewWatchdogInterval: ReturnType | null = null + // Bumped to invalidate an in-flight recovery reload when the provider or + // the watched view is disposed; the reload must not reassign webview.html + // afterwards. + private webviewRecoveryEpoch = 0 private static readonly WEBVIEW_WATCHDOG_TICK_MS = 60_000 private static readonly WEBVIEW_HEARTBEAT_STALE_MS = 90_000 private readonly _postStateToWebviewThrottled = debounce( @@ -821,6 +825,9 @@ export class ClineProvider private clearWebviewResources() { this.rejectPendingThemeFixtureProbes(new Error("Webview was disposed before the theme fixture probe completed")) this.stopWebviewWatchdog() + // Invalidate any recovery reload still awaiting its HTML so it cannot + // reassign webview.html on the disposed view. + this.webviewRecoveryEpoch++ while (this.webviewDisposables.length) { const x = this.webviewDisposables.pop() if (x) { @@ -3442,8 +3449,18 @@ export class ClineProvider if (!view?.webview) { return } + // Capture the epoch so a disposal or view replacement can invalidate + // this operation while the HTML is being generated. + const epoch = this.webviewRecoveryEpoch try { - view.webview.html = await this.getWebviewHtml(view.webview) + const html = await this.getWebviewHtml(view.webview) + // The await yields; assigning html now that the provider is disposed + // or the watched view was disposed/replaced would touch a dead or + // unrelated webview. + if (this._disposed || this.webviewRecoveryEpoch !== epoch || this.view !== view) { + return + } + view.webview.html = html } catch (error) { this.log(`[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`) } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 420cfd904e..88cf2977e2 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -787,6 +787,89 @@ describe("ClineProvider", () => { ) }) + test("does not reassign html when the provider is disposed mid-recovery", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + // Hold the recovery reload's HTML regeneration in flight until the + // disposal below lands. + let finishReload: (state: ExtensionState) => void = () => {} + vi.spyOn(provider, "getState").mockImplementation( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + // The stale heartbeat started a recovery reload, but its HTML is still pending. + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + await provider.dispose() + finishReload({ apiConfiguration: {} } as unknown as ExtensionState) + await vi.advanceTimersByTimeAsync(0) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + + test("does not reassign html when the sidebar view is disposed mid-recovery", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + let finishReload: (state: ExtensionState) => void = () => {} + vi.spyOn(provider, "getState").mockImplementation( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + disposeCallback() + finishReload({ apiConfiguration: {} } as unknown as ExtensionState) + await vi.advanceTimersByTimeAsync(0) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + + test("does not reassign html when the watched view is replaced mid-recovery", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + let finishReload: (state: ExtensionState) => void = () => {} + vi.spyOn(provider, "getState").mockImplementation( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + // VS Code re-resolves a fresh view (e.g. sidebar re-opened) while the + // recovery reload for the old view is still awaiting its HTML. + // @ts-ignore - accessing private property for testing + provider.view = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi.fn(), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + } + + finishReload({ apiConfiguration: {} } as unknown as ExtensionState) + await vi.advanceTimersByTimeAsync(0) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + test("reloads only its own webview when multiple providers are active", async () => { // Structural stand-in for the VS Code webview API surface this scenario // exercises, same as the mockContext cast below. @@ -828,6 +911,76 @@ describe("ClineProvider", () => { await providerB.dispose() } }) + + describe("tab panel (WebviewPanel shape)", () => { + let viewStateCallback: () => void + // Structural stand-in for the VS Code webview API surface this + // scenario exercises, same as the mockWebviewViewB cast below. + let mockWebviewPanel: vscode.WebviewPanel + + beforeEach(() => { + // WebviewPanel-shaped stand-in: same webview surface, but the + // visibility listener is onDidChangeViewState instead of + // onDidChangeVisibility, matching resolveWebviewView's tab branch. + mockWebviewPanel = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi.fn(), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + onDidDispose: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), + onDidChangeViewState: vi.fn().mockImplementation((cb: () => void) => { + viewStateCallback = cb + return { dispose: vi.fn() } + }), + dispose: vi.fn(), + } as unknown as vscode.WebviewPanel + }) + + test("reloads the tab webview when the heartbeat is stale and the tab is visible", async () => { + await provider.resolveWebviewView(mockWebviewPanel) + const htmlAfterResolve = mockWebviewPanel.webview.html + + await vi.advanceTimersByTimeAsync(120_000) + + expect(mockWebviewPanel.webview.html).not.toBe(htmlAfterResolve) + expect(mockWebviewPanel.webview.html).toContain("Zoo Code") + }) + + test("does not reload a hidden tab even when the heartbeat is stale", async () => { + await provider.resolveWebviewView(mockWebviewPanel) + Object.defineProperty(mockWebviewPanel, "visible", { value: false, configurable: true }) + const htmlAfterResolve = mockWebviewPanel.webview.html + + await vi.advanceTimersByTimeAsync(180_000) + + expect(mockWebviewPanel.webview.html).toBe(htmlAfterResolve) + }) + + test("resets the grace window when the tab becomes visible", async () => { + await provider.resolveWebviewView(mockWebviewPanel) + Object.defineProperty(mockWebviewPanel, "visible", { value: false, configurable: true }) + const htmlAfterResolve = mockWebviewPanel.webview.html + + await vi.advanceTimersByTimeAsync(120_000) + expect(mockWebviewPanel.webview.html).toBe(htmlAfterResolve) + + Object.defineProperty(mockWebviewPanel, "visible", { value: true, configurable: true }) + viewStateCallback() + const htmlAfterBecomingVisible = mockWebviewPanel.webview.html + + await vi.advanceTimersByTimeAsync(60_000) + expect(mockWebviewPanel.webview.html).toBe(htmlAfterBecomingVisible) + + // Watchdog ticks every 60s; 120s after the flip the heartbeat is stale again. + await vi.advanceTimersByTimeAsync(60_000) + expect(mockWebviewPanel.webview.html).not.toBe(htmlAfterBecomingVisible) + }) + }) }) test("resolveWebviewView sets up webview correctly in development mode even if local server is not running", async () => { From ba72a4e3ee3483ec8ba4fc24f680fc45d30419e1 Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Wed, 23 Sep 2026 06:37:38 +0900 Subject: [PATCH 06/34] test(webview): cover recovery rejection and watchdog precondition Address two minor review threads on the watchdog spec: - Cover the reload rejection path: stub the recovery HTML regeneration to reject and assert the failure is logged and webview.html is untouched. - Assert the watchdog interval was actually scheduled before the sidebar disposal test disposes the view, so the post-disposal null check proves the watchdog was stopped rather than never having started. --- .../webview/__tests__/ClineProvider.spec.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 88cf2977e2..d19298dda7 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -749,6 +749,11 @@ describe("ClineProvider", () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html + // Precondition: the watchdog interval was actually scheduled, so the + // null check below proves disposal stopped it rather than it never + // having started. + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + // The provider outlives a disposed sidebar view; VS Code re-resolves // a fresh view later. Disposal must stop the watchdog and drop the // stale view reference so no recovery reload targets the dead view. @@ -787,6 +792,22 @@ describe("ClineProvider", () => { ) }) + test("logs and keeps the webview html when the recovery reload rejects", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + // The watchdog reload no longer goes through a VS Code command; stub + // the recovery HTML regeneration itself to reject. + provider["getWebviewHtml"] = vi.fn().mockRejectedValue(new Error("reload boom")) + ;(mockOutputChannel.appendLine as ReturnType).mockClear() + + await vi.advanceTimersByTimeAsync(120_000) + + expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( + expect.stringContaining("[Zoo Code] Failed to reload webview: reload boom"), + ) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + test("does not reassign html when the provider is disposed mid-recovery", async () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html From 1e6fd39684d5321af0790b2e74d064ad0621e852 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sun, 27 Sep 2026 22:25:51 +0900 Subject: [PATCH 07/34] test(webview): assert global webview reload command is not used in provider isolation What: spy on vscode.commands.executeCommand in the multi-provider watchdog isolation test and assert workbench.action.webview.reloadWebviewAction was never invoked while the stale-heartbeat watchdog fires. Why: the pre-PR recovery path fired that global command, which resets every webview. The html-only assertions still pass under a regression back to the global command because the vscode mock swallows executeCommand without touching any view, so the test could not fail for the exact bug class it guards (code-reviewer P2 F-01 on PR #1715). --- src/core/webview/__tests__/ClineProvider.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index d19298dda7..fb392f3eba 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -920,6 +920,14 @@ describe("ClineProvider", () => { const htmlBeforeA = mockWebviewView.webview.html const htmlBeforeB = mockWebviewViewB.webview.html + // The pre-PR reload path fired the global + // workbench.action.webview.reloadWebviewAction command, which + // resets every webview. The html assertions below cannot catch a + // regression back to that command because the vscode mock + // swallows executeCommand without touching any view, so also + // assert the global command was never invoked. + const executeCommandMock = vscode.commands.executeCommand as ReturnType + await vi.advanceTimersByTimeAsync(119_000) // B's view is still alive; refresh its heartbeat so a stale one // would reload it too, leaving only A's watchdog to fire. @@ -928,6 +936,7 @@ describe("ClineProvider", () => { expect(mockWebviewView.webview.html).not.toBe(htmlBeforeA) expect(mockWebviewViewB.webview.html).toBe(htmlBeforeB) + expect(executeCommandMock).not.toHaveBeenCalledWith("workbench.action.webview.reloadWebviewAction") } finally { await providerB.dispose() } From da8d752ffb35c0861403899b41929a047df7c533 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Mon, 28 Sep 2026 00:09:27 +0900 Subject: [PATCH 08/34] fix(webview): harden recovery reload and scope sidebar disposal cleanup What: The recovery reload captures lastWebviewHeartbeatAt when recovery starts and, after the awaited HTML generation, additionally requires the heartbeat timestamp to be unchanged and the captured view to still be visible before reassigning webview.html. The sidebar onDidDispose handler now runs clearWebviewResources() and the codeIndexManager reset only when the disposed view is the currently watched one. Cancellation specs defer getWebviewHtml() itself (instead of getState() with a fabricated state) so they prove disposal and replacement land while recovery is in flight, and new specs cover heartbeat-during-recovery and hide-during-recovery skips. Why: A heartbeat arriving during recovery means the renderer is alive again, and a view that hides mid-recovery no longer proves a dead renderer (hidden webviews throttle timers), so reloading in either case needlessly reloads a healthy or throttled webview. This mitigates the security audit finding LOW-1 (self-reload loop when the postMessage bridge is half-broken): a renderer that resumes heartbeating during the recovery window is no longer reloaded on every watchdog tick. Disposing an outdated (replaced) sidebar view previously wiped the replacement view's resources and code index manager; scoping the cleanup to the watched view keeps the replacement intact. --- src/core/webview/ClineProvider.ts | 27 +++++-- .../webview/__tests__/ClineProvider.spec.ts | 79 +++++++++++++++++-- 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 4be16135c2..811ab40e3f 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1146,15 +1146,15 @@ export class ClineProvider this.log("Disposing ClineProvider instance for tab view") await this.dispose() } else { - this.log("Clearing webview resources for sidebar view") - this.clearWebviewResources() if (this.view === webviewView) { + this.log("Clearing webview resources for sidebar view") + this.clearWebviewResources() // Drop the disposed view so nothing keeps polling it // (e.g. the recovery watchdog) for the provider's lifetime. this.view = undefined + // Reset current workspace manager reference when view is disposed + this.codeIndexManager = undefined } - // Reset current workspace manager reference when view is disposed - this.codeIndexManager = undefined } }, null, @@ -3452,12 +3452,23 @@ export class ClineProvider // Capture the epoch so a disposal or view replacement can invalidate // this operation while the HTML is being generated. const epoch = this.webviewRecoveryEpoch + // A heartbeat that arrives while the HTML is being generated means the + // renderer is alive again; comparing against the timestamp captured here + // lets the post-await check skip the reload in that case. + const heartbeatAtRecoveryStart = this.lastWebviewHeartbeatAt try { const html = await this.getWebviewHtml(view.webview) - // The await yields; assigning html now that the provider is disposed - // or the watched view was disposed/replaced would touch a dead or - // unrelated webview. - if (this._disposed || this.webviewRecoveryEpoch !== epoch || this.view !== view) { + // The await yields; assigning html now that the provider is disposed, + // the watched view was disposed/replaced, the renderer heartbeat + // recovered, or the view hid again would either touch a dead or + // unrelated webview or reload one that no longer needs recovery. + if ( + this._disposed || + this.webviewRecoveryEpoch !== epoch || + this.view !== view || + this.lastWebviewHeartbeatAt !== heartbeatAtRecoveryStart || + view.visible !== true + ) { return } view.webview.html = html diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index fb392f3eba..79f0168365 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -837,19 +837,24 @@ describe("ClineProvider", () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html - let finishReload: (state: ExtensionState) => void = () => {} - vi.spyOn(provider, "getState").mockImplementation( + // Defer the recovery reload's own HTML generation so the test proves + // the disposal lands while recovery is in flight, not before it starts. + let finishReload: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( () => - new Promise((resolve) => { + new Promise((resolve) => { finishReload = resolve }), ) await vi.advanceTimersByTimeAsync(120_000) + // The stale heartbeat started a recovery reload, and its HTML + // generation is still pending. + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) disposeCallback() - finishReload({ apiConfiguration: {} } as unknown as ExtensionState) + finishReload("recovered") await vi.advanceTimersByTimeAsync(0) expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) @@ -859,15 +864,21 @@ describe("ClineProvider", () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html - let finishReload: (state: ExtensionState) => void = () => {} - vi.spyOn(provider, "getState").mockImplementation( + // Defer the recovery reload's own HTML generation so the test proves + // the replacement lands while recovery is in flight, not before it + // starts. + let finishReload: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( () => - new Promise((resolve) => { + new Promise((resolve) => { finishReload = resolve }), ) await vi.advanceTimersByTimeAsync(120_000) + // The stale heartbeat started a recovery reload, and its HTML + // generation is still pending. + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) // VS Code re-resolves a fresh view (e.g. sidebar re-opened) while the @@ -885,7 +896,59 @@ describe("ClineProvider", () => { visible: true, } - finishReload({ apiConfiguration: {} } as unknown as ExtensionState) + finishReload("recovered") + await vi.advanceTimersByTimeAsync(0) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + + test("skips the recovery reload when a heartbeat arrives while regenerating HTML", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + let finishReload: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + // The renderer process reported in again while the recovery HTML was + // still being generated, so the webview is alive and must not reload. + provider.updateWebviewHeartbeat() + + finishReload("recovered") + await vi.advanceTimersByTimeAsync(0) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + + test("skips the recovery reload when the view hides while regenerating HTML", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + let finishReload: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + + // The view hid while the recovery HTML was still being generated; a + // hidden webview throttles heartbeats, so the stale heartbeat no + // longer proves a dead renderer. + Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) + + finishReload("recovered") await vi.advanceTimersByTimeAsync(0) expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) From 148b6416b609e9806f5bffe341891f4fc443899e Mon Sep 17 00:00:00 2001 From: myk1yt Date: Mon, 28 Sep 2026 01:23:35 +0900 Subject: [PATCH 09/34] test(e2e): poll restart task-history presence before asserting What: runVerify now gates the Task should be present after restart assertion on a bounded waitForTaskInHistory poll that uses the same api.isTaskInHistory read as the assertion, instead of a single unguarded read immediately after isReady. Why: the e2e-mock lane failed on da8d752ff with exactly this assertion (false !== true) while the same commit passed 6/6 local runs and the changed production paths cannot execute during the scenario (watchdog tick 60s vs ~2s phases; the sidebar dispose scoping is behavior-identical for the single watched view). The read races the persisted state becoming observable on a fresh host, the same swap-window failure class documented in #1641 for this scenario, which #1663 fixed for the conversation-history reads but left the isTaskInHistory read single-shot. Polling with the same read removes the false negative without touching production code, keeping the CodeRabbit-requested recovery and dispose guarantees intact. --- apps/vscode-e2e/src/suite/restart-persistence.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/vscode-e2e/src/suite/restart-persistence.test.ts b/apps/vscode-e2e/src/suite/restart-persistence.test.ts index 699b64e9cc..902dc25f51 100644 --- a/apps/vscode-e2e/src/suite/restart-persistence.test.ts +++ b/apps/vscode-e2e/src/suite/restart-persistence.test.ts @@ -35,6 +35,14 @@ async function waitForMarkedCompletion(api: RooCodeAPI, taskId: string): Promise ) } +async function waitForTaskInHistory(api: RooCodeAPI, taskId: string): Promise { + // Poll the same read the assertion uses. A single read can observe the + // persisted state before it is visible on a fresh host (the atomic-rename + // swap window documented in #1641), so gate the assertion on a bounded + // wait instead of failing on the first miss. + await waitFor(() => api.isTaskInHistory(taskId)) +} + async function runCreate(api: RooCodeAPI): Promise { let taskId: string | undefined let createPhasePassed = false @@ -92,6 +100,7 @@ async function runVerify(api: RooCodeAPI): Promise { api.on(RooCodeEventName.Message, messageHandler) await waitFor(() => api.isReady()) + await waitForTaskInHistory(api, taskId) assert.strictEqual(await api.isTaskInHistory(taskId), true, "Task should be present after restart") const historyItem = await api.getTaskHistoryItem(taskId) assert.ok(historyItem, "Task history item should be available after restart") From 54f6a93d57fed5e3d4337106c093edfd63a144c7 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Mon, 28 Sep 2026 01:50:12 +0900 Subject: [PATCH 10/34] fix(webview): guard recovery with heartbeat revision and in-flight flag What: The recovery reload now compares a webviewHeartbeatRevision counter (captured before the awaited HTML generation against the value after) instead of comparing heartbeat timestamps, and reloadWebviewForRecovery skips starting when webviewRecoveryInFlight is already set, clearing that flag in a finally block so a failed generation cannot block future recoveries. updateWebviewHeartbeat bumps the revision on every accepted heartbeat. New specs cover the same-millisecond double-heartbeat case and the no-second-recovery-while-in-flight case including the reject path. Why: Two heartbeats landing in the same millisecond advanced the old timestamp not at all, so a recovery that started before them could still replace a page whose renderer had reported back twice; a monotonic counter catches every accepted heartbeat. Without the in-flight flag, every 60s watchdog tick during a slow HTML generation started another recovery for the same view, piling forced reloads onto it; the flag limits the view to one recovery at a time while the finally block keeps one failed attempt from disabling recovery permanently. The existing webviewRecoveryEpoch checks stay in place: the epoch invalidates completions after dispose or view replacement, while the in-flight flag only serializes generation starts, so the two guards address different races and both remain. --- src/core/webview/ClineProvider.ts | 24 +++++-- .../webview/__tests__/ClineProvider.spec.ts | 62 +++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 811ab40e3f..69a8cf0614 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -224,11 +224,18 @@ export class ClineProvider private currentWorkspacePath: string | undefined private _disposed = false private lastWebviewHeartbeatAt = 0 + // Bumped on every accepted heartbeat, including multiple heartbeats in the + // same millisecond; the recovery reload compares revisions around its + // awaited HTML generation so no accepted heartbeat is missed. + private webviewHeartbeatRevision = 0 private webviewWatchdogInterval: ReturnType | null = null // Bumped to invalidate an in-flight recovery reload when the provider or // the watched view is disposed; the reload must not reassign webview.html // afterwards. private webviewRecoveryEpoch = 0 + // True while a recovery reload is awaiting HTML generation; the watchdog + // must not pile a second reload onto the same view until this one settles. + private webviewRecoveryInFlight = false private static readonly WEBVIEW_WATCHDOG_TICK_MS = 60_000 private static readonly WEBVIEW_HEARTBEAT_STALE_MS = 90_000 private readonly _postStateToWebviewThrottled = debounce( @@ -3407,6 +3414,7 @@ export class ClineProvider /** Records that the webview renderer is alive; called on every webviewHeartbeat message. */ public updateWebviewHeartbeat(): void { this.lastWebviewHeartbeatAt = Date.now() + this.webviewHeartbeatRevision++ } /** @@ -3449,13 +3457,19 @@ export class ClineProvider if (!view?.webview) { return } + // A recovery already awaiting its HTML generation owns this view; a + // second one would only pile another forced reload onto it. + if (this.webviewRecoveryInFlight) { + return + } + this.webviewRecoveryInFlight = true // Capture the epoch so a disposal or view replacement can invalidate // this operation while the HTML is being generated. const epoch = this.webviewRecoveryEpoch // A heartbeat that arrives while the HTML is being generated means the - // renderer is alive again; comparing against the timestamp captured here - // lets the post-await check skip the reload in that case. - const heartbeatAtRecoveryStart = this.lastWebviewHeartbeatAt + // renderer is alive again; comparing revisions (not timestamps) lets the + // post-await check catch heartbeats that land in the same millisecond. + const heartbeatRevision = this.webviewHeartbeatRevision try { const html = await this.getWebviewHtml(view.webview) // The await yields; assigning html now that the provider is disposed, @@ -3466,7 +3480,7 @@ export class ClineProvider this._disposed || this.webviewRecoveryEpoch !== epoch || this.view !== view || - this.lastWebviewHeartbeatAt !== heartbeatAtRecoveryStart || + this.webviewHeartbeatRevision !== heartbeatRevision || view.visible !== true ) { return @@ -3474,6 +3488,8 @@ export class ClineProvider view.webview.html = html } catch (error) { this.log(`[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`) + } finally { + this.webviewRecoveryInFlight = false } } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 79f0168365..c9c9d8ce31 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -928,6 +928,36 @@ describe("ClineProvider", () => { expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) }) + test("skips the recovery reload when two heartbeats land in the same millisecond during recovery", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + let finishReload: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + // Pin the clock so both heartbeats stamp the same millisecond: a + // timestamp comparison would see no change, but the heartbeat + // revision moved, so the reload must still be skipped. + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(Date.now()) + provider.updateWebviewHeartbeat() + provider.updateWebviewHeartbeat() + nowSpy.mockRestore() + + finishReload("recovered") + await vi.advanceTimersByTimeAsync(0) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + }) + test("skips the recovery reload when the view hides while regenerating HTML", async () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html @@ -954,6 +984,38 @@ describe("ClineProvider", () => { expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) }) + test("does not start a second recovery while one is in flight and recovers after a failure", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + // The first recovery blocks on a controlled pending generation. + let failReload: (error: Error) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( + () => + new Promise((_resolve, reject) => { + failReload = reject + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + + // The watchdog ticks again while the first recovery still awaits + // its HTML; no second generation may start for the same view. + await vi.advanceTimersByTimeAsync(60_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + // The pending generation rejects; the finally block must clear the + // in-flight state so a later watchdog tick can start a fresh + // recovery instead of being blocked forever. + failReload(new Error("reload boom")) + await vi.advanceTimersByTimeAsync(0) + + await vi.advanceTimersByTimeAsync(60_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(2) + }) + test("reloads only its own webview when multiple providers are active", async () => { // Structural stand-in for the VS Code webview API surface this scenario // exercises, same as the mockContext cast below. From fad059e26e0e973460f74cef359f46c3aace51cf Mon Sep 17 00:00:00 2001 From: myk1yt Date: Mon, 28 Sep 2026 02:32:21 +0900 Subject: [PATCH 11/34] fix(webview): scope recovery in-flight guard to epoch What: The in-flight guard for recovery reloads now records which webviewRecoveryEpoch owns the pending recovery (webviewRecoveryInFlightEpoch) instead of a plain boolean. A new recovery is only blocked when the pending one belongs to the same epoch, and the finally block releases ownership only if the completing recovery still holds it. Specs: the same-millisecond heartbeat spec captures the pre-spy timestamp, feeds it to the Date.now spy, and asserts the stamp is unchanged after each heartbeat; the retry spec resolves the second getWebviewHtml call with distinct HTML and asserts it lands in the webview; a new spec covers recovery A pending, view disposed/replaced (epoch bump), recovery B starting immediately, and A's finally not releasing B's ownership. Why: With a boolean flag, a recovery pending on a disposed or replaced view blocked the replacement view's recovery until the stale generation settled, delaying recovery of the live view by a whole generation window. Scoping ownership to the epoch keeps the pile-up protection for the same view while letting the replacement's recovery start immediately; the conditional finally clear stops a stale completion from releasing ownership the replacement recovery took. The per-heartbeat timestamp assertions prove both same-millisecond heartbeats leave the recovery-start stamp unchanged, which is exactly the case the revision counter exists for, and the retry HTML assertion proves a post-failure recovery actually replaces the page content. --- src/core/webview/ClineProvider.ts | 28 +++-- .../webview/__tests__/ClineProvider.spec.ts | 102 +++++++++++++++--- 2 files changed, 107 insertions(+), 23 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 69a8cf0614..eac211fe89 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -233,9 +233,11 @@ export class ClineProvider // the watched view is disposed; the reload must not reassign webview.html // afterwards. private webviewRecoveryEpoch = 0 - // True while a recovery reload is awaiting HTML generation; the watchdog - // must not pile a second reload onto the same view until this one settles. - private webviewRecoveryInFlight = false + // Epoch that owns an in-flight recovery reload, if any. Only a recovery + // from the same epoch is blocked, so disposing/replacing the view (which + // bumps webviewRecoveryEpoch) never makes the replacement wait on the + // stale recovery's completion. + private webviewRecoveryInFlightEpoch: number | undefined = undefined private static readonly WEBVIEW_WATCHDOG_TICK_MS = 60_000 private static readonly WEBVIEW_HEARTBEAT_STALE_MS = 90_000 private readonly _postStateToWebviewThrottled = debounce( @@ -3457,15 +3459,17 @@ export class ClineProvider if (!view?.webview) { return } - // A recovery already awaiting its HTML generation owns this view; a - // second one would only pile another forced reload onto it. - if (this.webviewRecoveryInFlight) { - return - } - this.webviewRecoveryInFlight = true // Capture the epoch so a disposal or view replacement can invalidate // this operation while the HTML is being generated. const epoch = this.webviewRecoveryEpoch + // A recovery already awaiting its HTML generation owns this view; a + // second one would only pile another forced reload onto it. Ownership + // is scoped to the epoch so a stale recovery (its view disposed or + // replaced, epoch bumped) never blocks the replacement view's recovery. + if (this.webviewRecoveryInFlightEpoch === epoch) { + return + } + this.webviewRecoveryInFlightEpoch = epoch // A heartbeat that arrives while the HTML is being generated means the // renderer is alive again; comparing revisions (not timestamps) lets the // post-await check catch heartbeats that land in the same millisecond. @@ -3489,7 +3493,11 @@ export class ClineProvider } catch (error) { this.log(`[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`) } finally { - this.webviewRecoveryInFlight = false + // Clear ownership only while this completion still holds it; a + // stale recovery must not release the replacement's in-flight state. + if (this.webviewRecoveryInFlightEpoch === epoch) { + this.webviewRecoveryInFlightEpoch = undefined + } } } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index c9c9d8ce31..776e12194e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -944,12 +944,16 @@ describe("ClineProvider", () => { expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) - // Pin the clock so both heartbeats stamp the same millisecond: a - // timestamp comparison would see no change, but the heartbeat - // revision moved, so the reload must still be skipped. - const nowSpy = vi.spyOn(Date, "now").mockReturnValue(Date.now()) + // Pin the clock to the timestamp captured before recovery: both + // heartbeats stamp the same millisecond, so a timestamp comparison + // would see no change, but the heartbeat revision moved, so the + // reload must still be skipped. + const heartbeatAtCapture = provider["lastWebviewHeartbeatAt"] + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(heartbeatAtCapture) provider.updateWebviewHeartbeat() + expect(provider["lastWebviewHeartbeatAt"]).toBe(heartbeatAtCapture) provider.updateWebviewHeartbeat() + expect(provider["lastWebviewHeartbeatAt"]).toBe(heartbeatAtCapture) nowSpy.mockRestore() finishReload("recovered") @@ -988,14 +992,24 @@ describe("ClineProvider", () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html - // The first recovery blocks on a controlled pending generation. - let failReload: (error: Error) => void = () => {} - provider["getWebviewHtml"] = vi.fn().mockImplementation( - () => - new Promise((_resolve, reject) => { - failReload = reject - }), - ) + // The first recovery blocks on a controlled pending generation; the + // retry (second call) gets its own deferred generation. + let failFirst: (error: Error) => void = () => {} + let finishRetry: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi + .fn() + .mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + failFirst = reject + }), + ) + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishRetry = resolve + }), + ) await vi.advanceTimersByTimeAsync(120_000) expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) @@ -1009,11 +1023,73 @@ describe("ClineProvider", () => { // The pending generation rejects; the finally block must clear the // in-flight state so a later watchdog tick can start a fresh // recovery instead of being blocked forever. - failReload(new Error("reload boom")) + failFirst(new Error("reload boom")) await vi.advanceTimersByTimeAsync(0) await vi.advanceTimersByTimeAsync(60_000) expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(2) + + // The retry's generation resolves with fresh HTML and must land in + // the webview. + finishRetry("recovered-retry") + await vi.advanceTimersByTimeAsync(0) + expect(mockWebviewView.webview.html).toContain("recovered-retry") + }) + + test("lets the replacement view's recovery start while a stale one is pending", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + // Each recovery generation gets its own deferred finish callback. + const generations: Array<{ finish: (html: string) => void }> = [] + provider["getWebviewHtml"] = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + generations.push({ finish: resolve }) + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + + // The sidebar view is disposed (bumping webviewRecoveryEpoch) and a + // replacement view takes over while recovery A's HTML generation is + // still pending, making A stale. + disposeCallback() + // @ts-ignore - accessing private property for testing + provider.view = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi.fn(), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + } + const replacementEpoch = provider["webviewRecoveryEpoch"] + + // Recovery B for the new view must start even though A is still + // pending: the in-flight guard is scoped to the epoch. + void provider["reloadWebviewForRecovery"]() + await vi.advanceTimersByTimeAsync(0) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(2) + + // A's completion is stale: it must not reassign the old view's html, + // and its finally must NOT release the ownership B took. + generations[0].finish("stale-A") + await vi.advanceTimersByTimeAsync(0) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + expect(provider["webviewRecoveryInFlightEpoch"]).toBe(replacementEpoch) + + // B completes and lands its HTML on the new view, then releases + // ownership so future recoveries can start. + generations[1].finish("recovered-B") + await vi.advanceTimersByTimeAsync(0) + // @ts-ignore - accessing private property for testing + expect(provider.view.webview.html).toContain("recovered-B") + expect(provider["webviewRecoveryInFlightEpoch"]).toBeUndefined() }) test("reloads only its own webview when multiple providers are active", async () => { From 939e85ff92d97950ca2a41310a3b7017f8369554 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Mon, 28 Sep 2026 06:12:19 +0900 Subject: [PATCH 12/34] fix(webview): bump recovery epoch when resolveWebviewView replaces the view What: resolveWebviewView now advances webviewRecoveryEpoch (the same increment clearWebviewResources uses) when the incoming view is a different object than the currently watched one, before assigning this.view. Re-resolving the same view and the first resolve leave the epoch unchanged. New specs: a replacement resolve without an intervening disposal advances the epoch, lets the replacement view's recovery start while the stale one is still pending, keeps the stale completion from reassigning the old view's html or releasing the replacement's in-flight ownership, and lands the replacement's HTML; a same-view re-resolve asserts no epoch bump and that the pending recovery's ownership still blocks a second recovery until it settles. Why: Without the bump, a recovery pending on the old view kept the epoch-scoped in-flight ownership, so a replacement view resolved without an intervening disposal (a path the dispose handler never saw) could not start its own recovery until the stale generation settled, delaying the live view's recovery by a whole generation window. The bump closes that interleaving while leaving the normal dispose-then-resolve path unchanged: disposal already bumps via clearWebviewResources, and by the time the replacement resolves this.view is undefined, so the two bump sites never double-count one replacement. Same-view re-resolves (a known VS Code re-resolve pattern) must not invalidate an in-flight recovery, so they are excluded from the bump. --- src/core/webview/ClineProvider.ts | 8 ++ .../webview/__tests__/ClineProvider.spec.ts | 112 ++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index eac211fe89..5e964998bd 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1042,6 +1042,14 @@ export class ClineProvider } async resolveWebviewView(webviewView: vscode.WebviewView | vscode.WebviewPanel) { + // Replacing the watched view invalidates a recovery reload still + // awaiting its HTML (same epoch bump as clearWebviewResources), so the + // stale recovery can neither block nor reassign the replacement view's + // recovery. Re-resolving the same view, or the first resolve, leaves + // the epoch alone. + if (this.view && this.view !== webviewView) { + this.webviewRecoveryEpoch++ + } this.view = webviewView const inTabMode = "onDidChangeViewState" in webviewView diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 776e12194e..ccc7136118 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1092,6 +1092,118 @@ describe("ClineProvider", () => { expect(provider["webviewRecoveryInFlightEpoch"]).toBeUndefined() }) + test("advances the recovery epoch when resolveWebviewView replaces the view and lets the new recovery start", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + const epochBeforeReplacement = provider["webviewRecoveryEpoch"] + + // Recovery generations get their own deferred finish callbacks; the + // replacement view's initial HTML generation resolves immediately so + // resolveWebviewView can complete. + const generations: Array<{ finish: (html: string) => void }> = [] + provider["getWebviewHtml"] = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + generations.push({ finish: resolve }) + }), + ) + .mockImplementationOnce(() => Promise.resolve("view2-initial")) + .mockImplementation( + () => + new Promise((resolve) => { + generations.push({ finish: resolve }) + }), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + + // VS Code re-resolves a DIFFERENT view without a disposal in between + // (the gap path): the epoch must advance so the stale recovery can + // neither block nor reassign the replacement view's recovery. + const mockWebviewView2 = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi.fn(), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + onDidDispose: vi.fn(), + onDidChangeVisibility: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), + } as unknown as vscode.WebviewView + await provider.resolveWebviewView(mockWebviewView2) + expect(provider["webviewRecoveryEpoch"]).toBe(epochBeforeReplacement + 1) + + // Recovery B for the replacement view starts immediately, unblocked + // by A's still-pending generation. + void provider["reloadWebviewForRecovery"]() + await vi.advanceTimersByTimeAsync(0) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(3) + + // A's completion is stale: the old view's html is untouched and A's + // finally does not release the ownership B took. + generations[0].finish("stale-A") + await vi.advanceTimersByTimeAsync(0) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + expect(provider["webviewRecoveryInFlightEpoch"]).toBe(provider["webviewRecoveryEpoch"]) + + // B completes and lands its HTML on the replacement view, then + // releases ownership. + generations[1].finish("recovered-B") + await vi.advanceTimersByTimeAsync(0) + expect(mockWebviewView2.webview.html).toContain("recovered-B") + expect(provider["webviewRecoveryInFlightEpoch"]).toBeUndefined() + }) + + test("does not advance the recovery epoch when the same view is re-resolved and keeps the pending recovery intact", async () => { + await provider.resolveWebviewView(mockWebviewView) + const epochAfterFirstResolve = provider["webviewRecoveryEpoch"] + + // Recovery A blocks on a controlled pending generation; the same + // view's re-resolve initial HTML resolves immediately. + let finishReload: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + .mockImplementation(() => + Promise.resolve("same-view-reresolve"), + ) + + await vi.advanceTimersByTimeAsync(120_000) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(1) + + // Re-resolving the SAME view must not bump the epoch nor disrupt the + // pending recovery that belongs to it. + await provider.resolveWebviewView(mockWebviewView) + expect(provider["webviewRecoveryEpoch"]).toBe(epochAfterFirstResolve) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(2) + expect(provider["webviewRecoveryInFlightEpoch"]).toBe(epochAfterFirstResolve) + + // A still owns the view across the re-resolve: another recovery + // attempt is blocked until A settles. + void provider["reloadWebviewForRecovery"]() + await vi.advanceTimersByTimeAsync(0) + expect(provider["getWebviewHtml"]).toHaveBeenCalledTimes(2) + + // A settles cleanly: the re-resolve restarted the watchdog, which + // re-stamped the heartbeat, so the revision guard (not the epoch) + // skips the stale assignment; the ownership A held is released. + finishReload("recovered-A") + await vi.advanceTimersByTimeAsync(0) + expect(mockWebviewView.webview.html).toContain("same-view-reresolve") + expect(provider["webviewRecoveryInFlightEpoch"]).toBeUndefined() + }) + test("reloads only its own webview when multiple providers are active", async () => { // Structural stand-in for the VS Code webview API surface this scenario // exercises, same as the mockContext cast below. From 1c9ac529b2d278f030040306d5eb26f60fddb7df Mon Sep 17 00:00:00 2001 From: myk1yt Date: Tue, 29 Sep 2026 00:55:36 +0900 Subject: [PATCH 13/34] chore(e2e): restore restart-persistence test to merge-base state What: Restores apps/vscode-e2e/src/suite/restart-persistence.test.ts to the merge-base (08d05eb0f), removing the waitForTaskInHistory polling helper and its call in runVerify that commit 148b6416b added. Why: The restart-persistence polling change is unrelated to webview renderer heartbeat recovery and was flagged as an out-of-scope change in the PR's pre-merge review; it will be submitted separately. Impact: The PR diff no longer touches apps/vscode-e2e; no runtime behavior changes. --- apps/vscode-e2e/src/suite/restart-persistence.test.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apps/vscode-e2e/src/suite/restart-persistence.test.ts b/apps/vscode-e2e/src/suite/restart-persistence.test.ts index 902dc25f51..699b64e9cc 100644 --- a/apps/vscode-e2e/src/suite/restart-persistence.test.ts +++ b/apps/vscode-e2e/src/suite/restart-persistence.test.ts @@ -35,14 +35,6 @@ async function waitForMarkedCompletion(api: RooCodeAPI, taskId: string): Promise ) } -async function waitForTaskInHistory(api: RooCodeAPI, taskId: string): Promise { - // Poll the same read the assertion uses. A single read can observe the - // persisted state before it is visible on a fresh host (the atomic-rename - // swap window documented in #1641), so gate the assertion on a bounded - // wait instead of failing on the first miss. - await waitFor(() => api.isTaskInHistory(taskId)) -} - async function runCreate(api: RooCodeAPI): Promise { let taskId: string | undefined let createPhasePassed = false @@ -100,7 +92,6 @@ async function runVerify(api: RooCodeAPI): Promise { api.on(RooCodeEventName.Message, messageHandler) await waitFor(() => api.isReady()) - await waitForTaskInHistory(api, taskId) assert.strictEqual(await api.isTaskInHistory(taskId), true, "Task should be present after restart") const historyItem = await api.getTaskHistoryItem(taskId) assert.ok(historyItem, "Task history item should be available after restart") From aa666fc402504e5f89337fda3238dfe655654162 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Tue, 29 Sep 2026 01:12:31 +0900 Subject: [PATCH 14/34] fix(webview): gate obsolete-view messages and dispose replaced-view subscriptions What: setWebviewMessageListener now takes the resolved view/panel and skips dispatch when this.view no longer references it, so a replaced view's live renderer cannot post into the provider. resolveWebviewView now tracks the view-specific subscriptions (message, visibility, active-editor, and configuration listeners plus the view's onDidDispose registration) in a dedicated resolvedViewDisposables array; switching to a different view disposes the previous entries before installing the replacement's, and both visibility callbacks (onDidChangeViewState and onDidChangeVisibility) also guard on this.view === webviewView before refreshing the provider-wide heartbeat. clearWebviewResources disposes the per-view subscriptions too, so provider dispose() and sidebar disposal still clean up everything. New specs: a replaced view's message is ignored while the current view's dispatches; the replaced view's subscriptions are each disposed exactly once and only the replacement's remain; a replaced sidebar view's late dispose callback neither stops the replacement's watchdog nor blocks its stale-heartbeat recovery; a replaced tab panel's view state changes and messages are ignored while the current panel's still dispatch; and a racing stale visibility event cannot refresh the heartbeat. Why: A live old renderer kept the provider-wide heartbeat timestamp fresh after its view was replaced, so the dead-renderer watchdog could skip recovery and leave the current webview gray, and every resolution accumulated duplicate listeners in the shared webviewDisposables array. Impact: Only ClineProvider and its spec change; same-view re-resolves keep today's behavior (no disposal, no epoch bump), and the existing sidebar onDidDispose guard semantics are unchanged. --- src/core/webview/ClineProvider.ts | 62 +++++-- .../webview/__tests__/ClineProvider.spec.ts | 166 ++++++++++++++++++ 2 files changed, 216 insertions(+), 12 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 5e964998bd..2422b4c724 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -198,6 +198,12 @@ export class ClineProvider private static activeInstances: Set = new Set() private disposables: vscode.Disposable[] = [] private webviewDisposables: vscode.Disposable[] = [] + // Subscriptions tied to the currently resolved view (message, visibility, + // active-editor, and configuration listeners plus the view's disposal + // registration). Replacing the view disposes the previous entries before + // the replacement's are installed, so a stale view can neither dispatch + // messages nor refresh the provider-wide heartbeat. + private resolvedViewDisposables: vscode.Disposable[] = [] private pendingThemeFixtureProbes = new Map< string, { @@ -837,6 +843,7 @@ export class ClineProvider // Invalidate any recovery reload still awaiting its HTML so it cannot // reassign webview.html on the disposed view. this.webviewRecoveryEpoch++ + this.disposeResolvedViewResources() while (this.webviewDisposables.length) { const x = this.webviewDisposables.pop() if (x) { @@ -845,6 +852,16 @@ export class ClineProvider } } + /** Disposes the subscriptions registered for the previously resolved view. */ + private disposeResolvedViewResources() { + while (this.resolvedViewDisposables.length) { + const x = this.resolvedViewDisposables.pop() + if (x) { + x.dispose() + } + } + } + /** Drain one task's memoized cleanup without preventing the remaining provider shutdown work. */ private async drainTaskDisposal(task: Task): Promise { try { @@ -1049,6 +1066,11 @@ export class ClineProvider // the epoch alone. if (this.view && this.view !== webviewView) { this.webviewRecoveryEpoch++ + // The replaced view's listeners must not outlive it: dispose them + // before the replacement's subscriptions are installed below so no + // duplicate message/visibility/editor/configuration listeners + // accumulate across A-to-B replacements. + this.disposeResolvedViewResources() } this.view = webviewView const inTabMode = "onDidChangeViewState" in webviewView @@ -1106,7 +1128,7 @@ export class ClineProvider // Sets up an event listener to listen for messages passed from the webview view context // and executes code based on the message that is received. - this.setWebviewMessageListener(webviewView.webview) + this.setWebviewMessageListener(webviewView) // Detect a dead webview renderer process (gray screen) via heartbeat timeout. this.startWebviewWatchdog() @@ -1120,7 +1142,7 @@ export class ClineProvider // Update subscription when workspace might have changed. this.updateCodeIndexStatusSubscription() }) - this.webviewDisposables.push(activeEditorSubscription) + this.resolvedViewDisposables.push(activeEditorSubscription) // Listen for when the panel becomes visible. // https://github.com/microsoft/vscode-discussions/discussions/840 @@ -1128,6 +1150,11 @@ export class ClineProvider // WebviewView and WebviewPanel have all the same properties except // for this visibility listener panel. const viewStateDisposable = webviewView.onDidChangeViewState(() => { + if (this.view !== webviewView) { + // A replaced view must not refresh the provider-wide + // heartbeat the current view's watchdog relies on. + return + } if (this.view?.visible) { // Hidden webviews throttle timers, so grant a fresh grace // window instead of counting throttled heartbeats as a crash. @@ -1138,10 +1165,15 @@ export class ClineProvider } }) - this.webviewDisposables.push(viewStateDisposable) + this.resolvedViewDisposables.push(viewStateDisposable) } else if ("onDidChangeVisibility" in webviewView) { // sidebar const visibilityDisposable = webviewView.onDidChangeVisibility(() => { + if (this.view !== webviewView) { + // A replaced view must not refresh the provider-wide + // heartbeat the current view's watchdog relies on. + return + } if (this.view?.visible) { // Hidden webviews throttle timers, so grant a fresh grace // window instead of counting throttled heartbeats as a crash. @@ -1152,7 +1184,7 @@ export class ClineProvider } }) - this.webviewDisposables.push(visibilityDisposable) + this.resolvedViewDisposables.push(visibilityDisposable) } // Listen for when the view is disposed @@ -1175,7 +1207,7 @@ export class ClineProvider } }, null, - this.disposables, + this.resolvedViewDisposables, ) // Listen for when color changes @@ -1185,7 +1217,7 @@ export class ClineProvider await this.postMessageToWebview({ type: "theme", text: JSON.stringify(await getTheme()) }) } }) - this.webviewDisposables.push(configDisposable) + this.resolvedViewDisposables.push(configDisposable) // If the extension is starting a new session, clear previous task state. // But don't clear if there's already an active task (e.g., resumed via IPC/bridge). @@ -1748,14 +1780,20 @@ export class ClineProvider * Sets up an event listener to listen for messages passed from the webview context and * executes code based on the message that is received. * - * @param webview A reference to the extension webview + * @param webviewView The resolved webview view or panel */ - private setWebviewMessageListener(webview: vscode.Webview) { - const onReceiveMessage = async (message: WebviewMessage) => - webviewMessageHandler(this, message, this.marketplaceManager) + private setWebviewMessageListener(webviewView: vscode.WebviewView | vscode.WebviewPanel) { + const onReceiveMessage = async (message: WebviewMessage) => { + // A replaced view's listener stays registered until the provider + // clears its subscriptions; never let a stale renderer dispatch. + if (this.view !== webviewView) { + return + } + await webviewMessageHandler(this, message, this.marketplaceManager) + } - const messageDisposable = webview.onDidReceiveMessage(onReceiveMessage) - this.webviewDisposables.push(messageDisposable) + const messageDisposable = webviewView.webview.onDidReceiveMessage(onReceiveMessage) + this.resolvedViewDisposables.push(messageDisposable) } /** diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index ccc7136118..44e7424e7b 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1255,6 +1255,172 @@ describe("ClineProvider", () => { } }) + describe("obsolete view gating and replacement cleanup", () => { + // Sidebar-shaped replacement view; callbacks are read through thunks + // because the mock assigns them when resolveWebviewView registers. + const createViewB = () => { + let messageCallbackB: (message: WebviewMessage) => Promise = async () => {} + const viewB = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi + .fn() + .mockImplementation((cb: (message: WebviewMessage) => Promise) => { + messageCallbackB = cb + return { dispose: vi.fn() } + }), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + onDidDispose: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), + onDidChangeVisibility: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), + } as unknown as vscode.WebviewView + return { viewB, sendMessage: (message: WebviewMessage) => messageCallbackB(message) } + } + + test("ignores messages from a replaced view but still dispatches for the current view", async () => { + let messageCallbackA: (message: WebviewMessage) => Promise = async () => {} + mockWebviewView.webview.onDidReceiveMessage = vi + .fn() + .mockImplementation((cb: (message: WebviewMessage) => Promise) => { + messageCallbackA = cb + return { dispose: vi.fn() } + }) + await provider.resolveWebviewView(mockWebviewView) + const { viewB, sendMessage } = createViewB() + await provider.resolveWebviewView(viewB) + + const revisionBefore = provider["webviewHeartbeatRevision"] + // The stale renderer reports in; its dispatch must be skipped. + await messageCallbackA({ type: "webviewHeartbeat", timestamp: Date.now() }) + expect(provider["webviewHeartbeatRevision"]).toBe(revisionBefore) + + // The current view's dispatch reaches the handler as before. + await sendMessage({ type: "webviewHeartbeat", timestamp: Date.now() }) + expect(provider["webviewHeartbeatRevision"]).toBe(revisionBefore + 1) + }) + + test("disposes the replaced view's subscriptions so listeners do not accumulate", async () => { + const aDisposables: Array<{ dispose: ReturnType }> = [] + let visibilityCallbackA: () => void = () => {} + mockWebviewView.webview.onDidReceiveMessage = vi.fn().mockImplementation(() => { + const d = { dispose: vi.fn() } + aDisposables.push(d) + return d + }) + mockWebviewView.onDidChangeVisibility = vi.fn().mockImplementation((cb: () => void) => { + visibilityCallbackA = cb + const d = { dispose: vi.fn() } + aDisposables.push(d) + return d + }) + // Emulate the real onDidDispose(listener, thisArgs, disposables) + // contract so the disposal registration is tracked per view too. + mockWebviewView.onDidDispose = vi + .fn() + .mockImplementation((cb: () => void, _thisArgs: null, disposables?: vscode.Disposable[]) => { + const d = { dispose: vi.fn() } + aDisposables.push(d) + disposables?.push(d) + return d + }) + await provider.resolveWebviewView(mockWebviewView) + expect(aDisposables.length).toBe(3) // message, visibility, disposal registration + + const { viewB } = createViewB() + await provider.resolveWebviewView(viewB) + + expect(aDisposables.map((d) => d.dispose.mock.calls.length)).toEqual([1, 1, 1]) + // Only B's subscriptions remain: message, visibility, active + // editor, configuration. + expect(provider["resolvedViewDisposables"].length).toBe(4) + + // Even if a stale visibility event races in, it must not refresh + // the provider-wide heartbeat the current view's watchdog reads. + const revisionBefore = provider["webviewHeartbeatRevision"] + visibilityCallbackA() + expect(provider["webviewHeartbeatRevision"]).toBe(revisionBefore) + }) + + test("keeps the replacement view's watchdog and recovery when the replaced view's dispose callback fires", async () => { + let disposeCallbackA: () => void = () => {} + mockWebviewView.onDidDispose = vi.fn().mockImplementation((cb: () => void) => { + disposeCallbackA = cb + return { dispose: vi.fn() } + }) + await provider.resolveWebviewView(mockWebviewView) + const { viewB } = createViewB() + await provider.resolveWebviewView(viewB) + + // The stale view is finally torn down; its disposal must not + // clear the replacement's resources. + disposeCallbackA() + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + // @ts-ignore - accessing private property for testing + expect(provider.view).toBe(viewB) + + // B remains eligible for recovery: once its heartbeat goes stale + // the watchdog reloads B's webview. + const htmlAfterBResolve = viewB.webview.html + await vi.advanceTimersByTimeAsync(120_000) + expect(viewB.webview.html).not.toBe(htmlAfterBResolve) + }) + + test("ignores a replaced tab panel's view state changes and messages", async () => { + // WebviewPanel-shaped views: visibility arrives via + // onDidChangeViewState instead of onDidChangeVisibility. + const createPanel = () => { + let viewStateCallback: () => void = () => {} + let messageCallback: (message: WebviewMessage) => Promise = async () => {} + const panel = { + webview: { + postMessage: vi.fn(), + html: "", + options: {}, + onDidReceiveMessage: vi + .fn() + .mockImplementation((cb: (message: WebviewMessage) => Promise) => { + messageCallback = cb + return { dispose: vi.fn() } + }), + asWebviewUri: vi.fn(), + cspSource: "vscode-webview://test-csp-source", + }, + visible: true, + onDidDispose: vi.fn().mockImplementation(() => ({ dispose: vi.fn() })), + onDidChangeViewState: vi.fn().mockImplementation((cb: () => void) => { + viewStateCallback = cb + return { dispose: vi.fn() } + }), + dispose: vi.fn(), + } as unknown as vscode.WebviewPanel + return { + panel, + fireViewState: () => viewStateCallback(), + sendMessage: (message: WebviewMessage) => messageCallback(message), + } + } + const panelA = createPanel() + await provider.resolveWebviewView(panelA.panel) + const panelB = createPanel() + await provider.resolveWebviewView(panelB.panel) + + // @ts-ignore - accessing private property for testing + expect(provider.view).toBe(panelB.panel) + + const revisionBefore = provider["webviewHeartbeatRevision"] + panelA.fireViewState() + await panelA.sendMessage({ type: "webviewHeartbeat", timestamp: Date.now() }) + expect(provider["webviewHeartbeatRevision"]).toBe(revisionBefore) + + panelB.fireViewState() + expect(provider["webviewHeartbeatRevision"]).toBe(revisionBefore + 1) + }) + }) + describe("tab panel (WebviewPanel shape)", () => { let viewStateCallback: () => void // Structural stand-in for the VS Code webview API surface this From 9f05e75d22d7a8a2c79673486f629fe8a3f5f450 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Tue, 29 Sep 2026 03:25:19 +0900 Subject: [PATCH 15/34] fix(webview): bail out of a resolve that went stale during its awaits What: resolveWebviewView now checks _disposed and this.view === webviewView after its getWebviewHtml/getState awaits and before installing the message listener, restarting the watchdog, or registering any other view subscription; a stale resolve returns without installing anything. New specs: a provider disposed mid-resolve ends with no watchdog interval and zero view subscriptions; a view that replaces a still-pending resolve ends with only the replacement's four subscriptions and the replacement keeping the watchdog. Why: Disposal (or a view replacement) during the awaited HTML/state generation used to stop the watchdog and clear resources, after which the pending resolve resumed and installed a fresh watchdog interval plus listeners for a disposed provider or a replaced view. That interval retained the provider and kept attempting recovery forever, and a replaced pending resolve duplicated the replacement's subscriptions on top of its own. Impact: Only the pending-resolve race is closed; normal resolves, same-view re-resolves, and dispose-then-revive via a new provider instance are unchanged. Both the sidebar (WebviewView) and tab (WebviewPanel) shapes share this code path, so both are covered by the single guard. --- src/core/webview/ClineProvider.ts | 9 ++++ .../webview/__tests__/ClineProvider.spec.ts | 49 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 2422b4c724..60311c028b 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1126,6 +1126,15 @@ export class ClineProvider }, ) + // The awaits above yield; disposal or a view replacement may have + // landed while this resolve was pending. Installing listeners or the + // watchdog now would leak an interval on a disposed provider (the + // recovery path's _disposed check cannot stop the interval itself) or + // duplicate the replacement view's subscriptions. + if (this._disposed || this.view !== webviewView) { + return + } + // Sets up an event listener to listen for messages passed from the webview view context // and executes code based on the message that is received. this.setWebviewMessageListener(webviewView) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 44e7424e7b..cf8bcb2ab1 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1419,6 +1419,55 @@ describe("ClineProvider", () => { panelB.fireViewState() expect(provider["webviewHeartbeatRevision"]).toBe(revisionBefore + 1) }) + + test("installs no listeners or watchdog when the provider is disposed mid-resolve", async () => { + // Hold the resolve's initial HTML generation so disposal lands + // inside the pending resolve. + let finishHtml: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + finishHtml = resolve + }), + ) + + const resolvePromise = provider.resolveWebviewView(mockWebviewView) + await provider.dispose() + finishHtml("initial") + await resolvePromise + + expect(provider["webviewWatchdogInterval"]).toBeNull() + expect(provider["resolvedViewDisposables"].length).toBe(0) + }) + + test("installs nothing when a different view replaces the pending resolve", async () => { + // A's initial HTML stays pending; B's resolves immediately. + let finishHtmlA: (html: string) => void = () => {} + provider["getWebviewHtml"] = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishHtmlA = resolve + }), + ) + .mockImplementation(() => Promise.resolve("viewB-initial")) + + const resolveA = provider.resolveWebviewView(mockWebviewView) + const { viewB } = createViewB() + await provider.resolveWebviewView(viewB) + + finishHtmlA("stale-A") + await resolveA + + // A's stale resolve bailed out: only B's subscriptions (message, + // visibility, active editor, configuration) are installed and B + // keeps the watchdog. + expect(provider["resolvedViewDisposables"].length).toBe(4) + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + // @ts-ignore - accessing private property for testing + expect(provider.view).toBe(viewB) + }) }) describe("tab panel (WebviewPanel shape)", () => { From 325436db84a4f12061f767ecd70cb2e8cead4ab2 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Tue, 29 Sep 2026 04:14:05 +0900 Subject: [PATCH 16/34] fix(webview): skip the initial html assignment for a stale resolve What: resolveWebviewView now checks _disposed and this.view === webviewView after the getWebviewHtml await and before assigning webview.html; a stale resolve returns without touching the view. The existing post-awaits guard before listener installation is unchanged. The two mid-resolve regression specs now also assert no html landed on the disposed/replaced view. Why: Disposal or replacement during html generation used to let the pending resolve assign html afterwards. The VS Code API throws when assigning to a destroyed webview, so the rejection surfaced from resolveWebviewView instead of returning cleanly, and a replaced resolve pushed html into the obsolete view it no longer owned. Impact: Live resolves assign exactly as before (a single combined guard after both awaits was rejected: it would leave the webview blank when the subsequent getState read fails, where today the html stays assigned). Same- view re-resolves, replacements, and revive flows are unaffected. --- src/core/webview/ClineProvider.ts | 10 +++++++++- src/core/webview/__tests__/ClineProvider.spec.ts | 9 ++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 60311c028b..fbd7901daf 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1094,7 +1094,15 @@ export class ClineProvider localResourceRoots: resourceRoots, } - webviewView.webview.html = await this.getWebviewHtml(webviewView.webview) + const html = await this.getWebviewHtml(webviewView.webview) + // The await yields; a disposal or replacement that landed mid-generation + // must not receive this HTML. The VS Code API throws when assigning to a + // destroyed webview, and a stale resolve must not touch the obsolete + // view it no longer owns. + if (this._disposed || this.view !== webviewView) { + return + } + webviewView.webview.html = html // Initialize out-of-scope variables that need to receive persistent // global state values. diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index cf8bcb2ab1..cd58da5493 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1436,6 +1436,8 @@ describe("ClineProvider", () => { finishHtml("initial") await resolvePromise + // The stale resolve assigned no HTML and installed nothing. + expect(mockWebviewView.webview.html).toBe("") expect(provider["webviewWatchdogInterval"]).toBeNull() expect(provider["resolvedViewDisposables"].length).toBe(0) }) @@ -1460,9 +1462,10 @@ describe("ClineProvider", () => { finishHtmlA("stale-A") await resolveA - // A's stale resolve bailed out: only B's subscriptions (message, - // visibility, active editor, configuration) are installed and B - // keeps the watchdog. + // A's stale resolve bailed out: no HTML landed on the obsolete + // view, only B's subscriptions (message, visibility, active + // editor, configuration) are installed and B keeps the watchdog. + expect(mockWebviewView.webview.html).toBe("") expect(provider["resolvedViewDisposables"].length).toBe(4) expect(provider["webviewWatchdogInterval"]).not.toBeNull() // @ts-ignore - accessing private property for testing From 63c82c3113ad1526e058d9aceeacbe266fa8cb4c Mon Sep 17 00:00:00 2001 From: myk1yt Date: Tue, 6 Oct 2026 04:13:51 +0900 Subject: [PATCH 17/34] chore: retrigger review-state reconciliation From 09855104bae12d41b17c62565a88dc00fba47747 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Tue, 6 Oct 2026 05:29:10 +0900 Subject: [PATCH 18/34] chore: retrigger review-state reconciliation From ff9aa8acfa0e6cd31922873547958858b4c3d1e9 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Fri, 9 Oct 2026 09:12:24 +0900 Subject: [PATCH 19/34] fix(webview): ignore suspend gap in webview heartbeat watchdog What: record the wall-clock reading of the previous watchdog tick; a tick arriving more than 1.5x the interval late restarts the heartbeat grace window instead of counting the gap against the renderer. Why: during system sleep both the extension-host interval and the webview heartbeat timer pause, but Date.now() keeps advancing. The first post-wake tick then saw a stale heartbeat on a live renderer and reloaded it, discarding unsent composer text and pending attachments. Impact: no false recovery reload after wake from sleep; the watchdog still reloads when the heartbeat genuinely goes stale (regression test advances the clock past the stale threshold between ticks and asserts no HTML reassignment, then asserts a real stale period still reloads). --- src/core/webview/ClineProvider.ts | 17 ++++++++++++++++- .../webview/__tests__/ClineProvider.spec.ts | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index dfcaab207f..fe7b43752b 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -250,6 +250,10 @@ export class ClineProvider // bumps webviewRecoveryEpoch) never makes the replacement wait on the // stale recovery's completion. private webviewRecoveryInFlightEpoch: number | undefined = undefined + // Wall-clock reading of the previous watchdog tick; a tick arriving far + // later than scheduled means the host slept, so the grace window restarts + // instead of counting the suspended interval against a live renderer. + private lastWebviewWatchdogTickAt = 0 private static readonly WEBVIEW_WATCHDOG_TICK_MS = 60_000 private static readonly WEBVIEW_HEARTBEAT_STALE_MS = 90_000 private readonly _postStateToWebviewThrottled = debounce( @@ -3398,14 +3402,25 @@ export class ClineProvider */ private startWebviewWatchdog(): void { this.updateWebviewHeartbeat() + this.lastWebviewWatchdogTickAt = Date.now() if (this.webviewWatchdogInterval) { clearInterval(this.webviewWatchdogInterval) } this.webviewWatchdogInterval = setInterval(() => { + const now = Date.now() + const tickGap = now - this.lastWebviewWatchdogTickAt + this.lastWebviewWatchdogTickAt = now + // A tick far later than scheduled means the host was suspended or + // blocked; the wall clock jumped while both timers were paused. + // Grant a fresh grace window instead of reloading a live renderer. + if (tickGap > ClineProvider.WEBVIEW_WATCHDOG_TICK_MS * 1.5) { + this.updateWebviewHeartbeat() + return + } if (this.view?.visible !== true) { return } - if (Date.now() - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS) { + if (now - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS) { return } this.log("[Zoo Code] Webview heartbeat stale while visible; reloading webview (dead renderer?)") diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 3cb442bca1..6c13008b2b 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1117,6 +1117,25 @@ describe("ClineProvider", () => { expect(mockWebviewView.webview.html).not.toBe(htmlAfterBecomingVisible) }) + test("restarts the grace window when a tick gap shows the host slept", async () => { + await provider.resolveWebviewView(mockWebviewView) + const htmlAfterResolve = mockWebviewView.webview.html + + // One ordinary tick, then a wall-clock jump past the stale + // threshold before the next tick. The gap means the host slept + // while both timers were paused, not that the renderer died. + await vi.advanceTimersByTimeAsync(60_000) + vi.setSystemTime(Date.now() + 120_000) + await vi.advanceTimersByTimeAsync(60_000) + + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + // The grace window restarted at the post-sleep tick, so the + // watchdog still reloads once the heartbeat genuinely goes stale. + await vi.advanceTimersByTimeAsync(120_000) + expect(mockWebviewView.webview.html).not.toBe(htmlAfterResolve) + }) + test("stops watching after the provider is disposed", async () => { await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html From ce63424a1723cfce65c1963ad5ae20ee51495bc7 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 26 Oct 2026 00:45:49 +0000 Subject: [PATCH 20/34] fix(webview): stop stale-heartbeat logging after retry limit --- src/core/webview/ClineProvider.ts | 10 ++++++---- src/core/webview/__tests__/ClineProvider.spec.ts | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 16187296a8..67bd8b742a 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -251,9 +251,8 @@ export class ClineProvider private currentWorkspacePath: string | undefined private _disposed = false private lastWebviewHeartbeatAt = 0 - // Bumped on every accepted heartbeat, including multiple heartbeats in the - // same millisecond; the recovery reload compares revisions around its - // awaited HTML generation so no accepted heartbeat is missed. + // Bumped on every heartbeat or grace reset, including events in the same + // millisecond; pending recovery must yield to either sign of renewed grace. private webviewHeartbeatRevision = 0 private webviewRecoveryAttempts = 0 private static readonly WEBVIEW_RECOVERY_ATTEMPT_LIMIT = 3 @@ -3598,7 +3597,10 @@ export class ClineProvider if (this.view?.visible !== true) { return } - if (now - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS) { + if ( + now - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS || + this.webviewRecoveryAttempts >= ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT + ) { return } this.log("[Zoo Code] Webview heartbeat stale while visible; reloading webview (dead renderer?)") diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index fdf88742cc..6b44218319 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1147,12 +1147,15 @@ describe("ClineProvider", () => { await vi.advanceTimersByTimeAsync(180_000) expect(generateHtml).toHaveBeenCalledTimes(3) + const log = vi.spyOn(provider, "log") + // Visibility changes and host suspension grant grace, but cannot // replenish attempts for a renderer that has never responded. visibilityCallback() vi.setSystemTime(Date.now() + 120_000) await vi.advanceTimersByTimeAsync(600_000) expect(generateHtml).toHaveBeenCalledTimes(3) + expect(log).not.toHaveBeenCalledWith(expect.stringContaining("Webview heartbeat stale")) await webviewMessageHandler(provider, { type: "webviewHeartbeat" }) await vi.advanceTimersByTimeAsync(120_000) From f98b9412d59670ca6061f92233cae72ab548258a Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 26 Oct 2026 01:17:59 +0000 Subject: [PATCH 21/34] test(webview): verify sidebar disposal cleanup and recovery invalidation --- src/core/webview/__tests__/ClineProvider.spec.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 6b44218319..358c2b2133 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1222,8 +1222,13 @@ describe("ClineProvider", () => { }) test("stops watching and clears the view when the sidebar webview is disposed", async () => { + mockWebviewView.webview.onDidReceiveMessage = vi.fn(() => ({ dispose: vi.fn() })) await provider.resolveWebviewView(mockWebviewView) const htmlAfterResolve = mockWebviewView.webview.html + const epochBefore = provider["webviewRecoveryEpoch"] + const trackedDisposables = [...provider["resolvedViewDisposables"]] + expect(trackedDisposables).not.toHaveLength(0) + const disposeSpies = trackedDisposables.map((disposable) => vi.spyOn(disposable, "dispose")) // Precondition: the watchdog interval was actually scheduled, so the // null check below proves disposal stopped it rather than it never @@ -1235,8 +1240,12 @@ describe("ClineProvider", () => { // stale view reference so no recovery reload targets the dead view. disposeCallback() expect(provider["webviewWatchdogInterval"]).toBeNull() - // @ts-ignore - accessing private property for testing - expect(provider.view).toBeUndefined() + expect(provider["view"]).toBeUndefined() + expect(provider["resolvedViewDisposables"]).toHaveLength(0) + for (const disposeSpy of disposeSpies) { + expect(disposeSpy).toHaveBeenCalledOnce() + } + expect(provider["webviewRecoveryEpoch"]).toBe(epochBefore + 1) await vi.advanceTimersByTimeAsync(180_000) From a985f03ba922e4728dcfd99971ffd3527d0a81a5 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 28 Oct 2026 23:16:32 +0000 Subject: [PATCH 22/34] fix(webview): reset recovery budget on view replacement and strengthen watchdog tests --- src/core/webview/ClineProvider.ts | 2 ++ .../webview/__tests__/ClineProvider.spec.ts | 34 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 5758d060e7..ada1d21658 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -854,6 +854,8 @@ export class ClineProvider // Invalidate any recovery reload still awaiting its HTML so it cannot // reassign webview.html on the disposed view. this.webviewRecoveryEpoch++ + // A replacement renderer needs its own recovery-attempt budget. + this.webviewRecoveryAttempts = 0 while (this.webviewDisposables.length) { const x = this.webviewDisposables.pop() if (x) { diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index d64aff60f7..754f531f60 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1168,14 +1168,41 @@ describe("ClineProvider", () => { expect(generateHtml).toHaveBeenCalledTimes(4) }) + test("gives a replacement view a fresh recovery budget after exhausting attempts", async () => { + await provider.resolveWebviewView(mockWebviewView) + const generateHtml = vi.spyOn(provider, "getState") + + await vi.advanceTimersByTimeAsync(480_000) + expect(generateHtml).toHaveBeenCalledTimes(3) + + disposeCallback() + const replacementView = { + ...mockWebviewView, + webview: { ...mockWebviewView.webview, html: "" }, + } + await provider.resolveWebviewView(replacementView) + const htmlAfterResolve = replacementView.webview.html + generateHtml.mockClear() + + await vi.advanceTimersByTimeAsync(60_000) + expect(replacementView.webview.html).toBe(htmlAfterResolve) + await vi.advanceTimersByTimeAsync(60_000) + + expect(generateHtml).toHaveBeenCalledTimes(1) + expect(replacementView.webview.html).not.toBe(htmlAfterResolve) + expect(replacementView.webview.html).toContain("Zoo Code") + }) + test("does not reload while the view is hidden even when the heartbeat is stale", async () => { await provider.resolveWebviewView(mockWebviewView) Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) const htmlAfterResolve = mockWebviewView.webview.html + const log = vi.spyOn(provider, "log") await vi.advanceTimersByTimeAsync(180_000) expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + expect(log).not.toHaveBeenCalledWith(expect.stringContaining("Webview heartbeat stale")) }) test("resets the grace window when the view becomes visible", async () => { @@ -1213,7 +1240,10 @@ describe("ClineProvider", () => { // The grace window restarted at the post-sleep tick, so the // watchdog still reloads once the heartbeat genuinely goes stale. - await vi.advanceTimersByTimeAsync(120_000) + await vi.advanceTimersByTimeAsync(60_000) + expect(mockWebviewView.webview.html).toBe(htmlAfterResolve) + + await vi.advanceTimersByTimeAsync(60_000) expect(mockWebviewView.webview.html).not.toBe(htmlAfterResolve) }) @@ -1260,7 +1290,9 @@ describe("ClineProvider", () => { test("does not stack watchdog intervals when resolveWebviewView runs again", async () => { await provider.resolveWebviewView(mockWebviewView) + const timersAfterFirstResolve = vi.getTimerCount() await provider.resolveWebviewView(mockWebviewView) + expect(vi.getTimerCount()).toBe(timersAfterFirstResolve) // getHtmlContent regenerates the HTML via one getState() call per reload, // so getState invocations after this point count watchdog reloads. From 3106b780e1c16d7c9c17dfa6206c9af8799d38c9 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 29 Oct 2026 01:11:20 +0000 Subject: [PATCH 23/34] test: cover webview heartbeat recovery and reproduce renderer crash failure --- apps/vscode-e2e/playwright-electron.config.ts | 6 +- apps/vscode-e2e/src/visual/electron.visual.ts | 189 +--------------- apps/vscode-e2e/src/visual/electronHarness.ts | 178 +++++++++++++++ .../src/visual/heartbeat.electron.ts | 214 ++++++++++++++++++ apps/vscode-e2e/src/visual/heartbeat.md | 20 ++ apps/vscode-e2e/src/visual/sceneController.ts | 10 +- .../webview/__tests__/ClineProvider.spec.ts | 89 +++++++- 7 files changed, 518 insertions(+), 188 deletions(-) create mode 100644 apps/vscode-e2e/src/visual/electronHarness.ts create mode 100644 apps/vscode-e2e/src/visual/heartbeat.electron.ts create mode 100644 apps/vscode-e2e/src/visual/heartbeat.md diff --git a/apps/vscode-e2e/playwright-electron.config.ts b/apps/vscode-e2e/playwright-electron.config.ts index 871bc3dead..250b748196 100644 --- a/apps/vscode-e2e/playwright-electron.config.ts +++ b/apps/vscode-e2e/playwright-electron.config.ts @@ -4,12 +4,14 @@ import { defineConfig } from "@playwright/test" export default defineConfig({ testDir: "./src/visual", - testMatch: "electron.visual.ts", outputDir: path.resolve(__dirname, "test-results-electron"), snapshotPathTemplate: "{testDir}/__screenshots__/{arg}{ext}", fullyParallel: false, workers: 1, - repeatEach: process.env.CI ? 3 : 1, + projects: [ + { name: "visual", testMatch: "electron.visual.ts", repeatEach: process.env.CI ? 3 : 1 }, + { name: "heartbeat", testMatch: "heartbeat.electron.ts", repeatEach: 1 }, + ], timeout: 3 * 60 * 1_000, expect: { timeout: 60_000, diff --git a/apps/vscode-e2e/src/visual/electron.visual.ts b/apps/vscode-e2e/src/visual/electron.visual.ts index 155f496980..3ea34742e7 100644 --- a/apps/vscode-e2e/src/visual/electron.visual.ts +++ b/apps/vscode-e2e/src/visual/electron.visual.ts @@ -1,33 +1,15 @@ -import fs from "fs/promises" -import { readFileSync } from "fs" -import os from "os" -import path from "path" - -import { LLMock } from "@copilotkit/aimock" import { downloadAndUnzipVSCode } from "@vscode/test-electron" -import { _electron, expect, test, type ElectronApplication, type Page, type TestInfo } from "@playwright/test" -import { glob } from "glob" - -const packageRoot = path.resolve(__dirname, "../..") -const repositoryRoot = path.resolve(packageRoot, "../..") -const extensionDevelopmentPath = path.join(repositoryRoot, "src") -const extensionTestsPath = path.join(packageRoot, "out", "visual", "sceneController") -const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as { - devDependencies: Record -} -const vscodeVersion = packageJson.devDependencies["@types/vscode"] - -const CHAT_PROMPT = "electron-visual-smoke: render a deterministic completion" -const CHAT_RESULT = "Electron visual smoke response rendered inside the real VS Code webview." - -interface VisualScenario { - name: string - scene: "welcome" | "chat" - themeId: "Default Dark Modern" | "Default High Contrast" - landmark: string - webviewSnapshot?: string - startMock?: boolean -} +import { expect, test } from "@playwright/test" + +import { + attachFailureArtifacts, + CHAT_RESULT, + extensionDevelopmentPath, + startScene, + stopScene, + vscodeVersion, + type VisualScenario, +} from "./electronHarness" const scenarios: VisualScenario[] = [ { @@ -52,155 +34,6 @@ const scenarios: VisualScenario[] = [ }, ] -interface RunningScene { - app: ElectronApplication - donePath: string - logPath: string - mock?: InstanceType - page: Page - temporaryRoot: string - userDataPath: string -} - -async function waitForJson(filePath: string): Promise<{ landmark: string; scene: string; themeId: string }> { - const deadline = Date.now() + 90_000 - while (Date.now() < deadline) { - try { - return JSON.parse(await fs.readFile(filePath, "utf8")) as { - landmark: string - scene: string - themeId: string - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT" && !(error instanceof SyntaxError)) throw error - await new Promise((resolve) => setTimeout(resolve, 100)) - } - } - throw new Error(`Timed out waiting for Electron visual readiness file: ${filePath}`) -} - -async function attachLogs(userDataPath: string, logPath: string, testInfo: TestInfo) { - const sceneLog = await fs.readFile(logPath, "utf8").catch(() => "") - if (sceneLog) await testInfo.attach("scene.log", { body: sceneLog, contentType: "text/plain" }) - - for (const pattern of ["logs/**/exthost.log", "logs/**/renderer.log", "logs/**/*Zoo-Code.log"]) { - const logs = await glob(pattern, { cwd: userDataPath }) - const latest = logs.at(-1) - if (!latest) continue - const contents = await fs.readFile(path.join(userDataPath, latest), "utf8").catch(() => "") - if (contents) await testInfo.attach(path.basename(latest), { body: contents, contentType: "text/plain" }) - } -} - -async function stopTracing(app: ElectronApplication, testInfo: TestInfo) { - await app - .context() - .tracing.stop({ path: testInfo.outputPath("trace.zip") }) - .catch(() => undefined) -} - -async function startScene( - vscodeExecutablePath: string, - scenario: VisualScenario, - testInfo: TestInfo, -): Promise { - const temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-code-electron-visual-")) - const workspacePath = path.join(temporaryRoot, "workspace") - const userDataPath = path.join(temporaryRoot, "user-data") - const extensionsPath = path.join(temporaryRoot, "extensions") - const readyPath = path.join(temporaryRoot, "ready.json") - const donePath = path.join(temporaryRoot, "done") - const logPath = path.join(temporaryRoot, "scene.log") - await Promise.all([fs.mkdir(workspacePath), fs.mkdir(userDataPath), fs.mkdir(extensionsPath)]) - - let app: ElectronApplication | undefined - let mock: InstanceType | undefined - let page: Page | undefined - try { - if (scenario.startMock) { - mock = new LLMock({ port: 0 }) - mock.addFixture({ - match: { userMessage: CHAT_PROMPT }, - response: { - toolCalls: [ - { - name: "attempt_completion", - arguments: JSON.stringify({ result: CHAT_RESULT }), - id: "call_electron_visual_completion", - }, - ], - }, - }) - await mock.start() - } - - app = await _electron.launch({ - executablePath: vscodeExecutablePath, - args: [ - workspacePath, - `--extensionDevelopmentPath=${extensionDevelopmentPath}`, - `--extensionTestsPath=${extensionTestsPath}`, - `--user-data-dir=${userDataPath}`, - `--extensions-dir=${extensionsPath}`, - "--disable-workspace-trust", - "--skip-welcome", - "--skip-release-notes", - "--disable-updates", - "--disable-gpu", - "--force-device-scale-factor=1", - ], - env: { - ...process.env, - AIMOCK_URL: mock?.url ?? "", - ROO_CODE_VISUAL_CHAT_PROMPT: CHAT_PROMPT, - ROO_CODE_VISUAL_CHAT_RESULT: CHAT_RESULT, - ROO_CODE_VISUAL_DONE_PATH: donePath, - ROO_CODE_VISUAL_LOG_PATH: logPath, - ROO_CODE_VISUAL_READY_PATH: readyPath, - ROO_CODE_VISUAL_SCENE: scenario.scene, - ROO_CODE_VISUAL_THEME: scenario.themeId, - ROO_CODE_THEME_FIXTURE_PROBE: "1", - }, - locale: "en-US", - timeout: 60_000, - timezoneId: "UTC", - }) - await app.context().tracing.start({ screenshots: true, snapshots: true, sources: true }) - page = await app.firstWindow({ timeout: 60_000 }) - await app.evaluate(({ BrowserWindow }) => { - const window = BrowserWindow.getAllWindows()[0] - window?.setBounds({ x: 0, y: 0, width: 1200, height: 800 }) - window?.focus() - }) - const ready = await waitForJson(readyPath) - expect(ready).toMatchObject({ landmark: scenario.landmark, scene: scenario.scene, themeId: scenario.themeId }) - return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath } - } catch (error) { - if (page) await page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) - await attachLogs(userDataPath, logPath, testInfo) - await fs.writeFile(donePath, "done\n", "utf8").catch(() => undefined) - if (app) await stopTracing(app, testInfo) - await app?.close().catch(() => undefined) - await mock?.stop().catch(() => undefined) - await fs.rm(temporaryRoot, { recursive: true, force: true }) - if (error instanceof Error) throw error - throw new Error(String(error)) - } -} - -async function attachFailureArtifacts(scene: RunningScene, testInfo: TestInfo) { - await scene.page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) - await attachLogs(scene.userDataPath, scene.logPath, testInfo) -} - -async function stopScene(scene: RunningScene, testInfo: TestInfo) { - await fs.writeFile(scene.donePath, "done\n", "utf8").catch(() => undefined) - await stopTracing(scene.app, testInfo) - await scene.app.close().catch(() => undefined) - await scene.mock?.stop().catch(() => undefined) - await fs.rm(scene.temporaryRoot, { recursive: true, force: true }) -} - test.describe.configure({ mode: "serial" }) let vscodeExecutablePath: string diff --git a/apps/vscode-e2e/src/visual/electronHarness.ts b/apps/vscode-e2e/src/visual/electronHarness.ts new file mode 100644 index 0000000000..861b612946 --- /dev/null +++ b/apps/vscode-e2e/src/visual/electronHarness.ts @@ -0,0 +1,178 @@ +import fs from "fs/promises" +import { readFileSync } from "fs" +import os from "os" +import path from "path" + +import { LLMock } from "@copilotkit/aimock" +import { _electron, expect, type ElectronApplication, type Page, type TestInfo } from "@playwright/test" +import { glob } from "glob" + +const packageRoot = path.resolve(__dirname, "../..") +const repositoryRoot = path.resolve(packageRoot, "../..") +export const extensionDevelopmentPath = path.join(repositoryRoot, "src") +const extensionTestsPath = path.join(packageRoot, "out", "visual", "sceneController") +const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as { + devDependencies: Record +} +export const vscodeVersion = packageJson.devDependencies["@types/vscode"] + +const CHAT_PROMPT = "electron-visual-smoke: render a deterministic completion" +export const CHAT_RESULT = "Electron visual smoke response rendered inside the real VS Code webview." + +export interface VisualScenario { + name: string + scene: "welcome" | "chat" + themeId: "Default Dark Modern" | "Default High Contrast" + landmark: string + webviewSnapshot?: string + startMock?: boolean +} + +interface RunningScene { + app: ElectronApplication + donePath: string + logPath: string + mock?: InstanceType + page: Page + temporaryRoot: string + userDataPath: string +} + +async function waitForJson(filePath: string): Promise<{ landmark: string; scene: string; themeId: string }> { + const deadline = Date.now() + 90_000 + while (Date.now() < deadline) { + try { + return JSON.parse(await fs.readFile(filePath, "utf8")) as { + landmark: string + scene: string + themeId: string + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" && !(error instanceof SyntaxError)) throw error + await new Promise((resolve) => setTimeout(resolve, 100)) + } + } + throw new Error(`Timed out waiting for Electron visual readiness file: ${filePath}`) +} + +async function attachLogs(userDataPath: string, logPath: string, testInfo: TestInfo) { + const sceneLog = await fs.readFile(logPath, "utf8").catch(() => "") + if (sceneLog) await testInfo.attach("scene.log", { body: sceneLog, contentType: "text/plain" }) + + for (const pattern of ["logs/**/exthost.log", "logs/**/renderer.log", "logs/**/*Zoo-Code.log"]) { + const logs = await glob(pattern, { cwd: userDataPath }) + const latest = logs.at(-1) + if (!latest) continue + const contents = await fs.readFile(path.join(userDataPath, latest), "utf8").catch(() => "") + if (contents) await testInfo.attach(path.basename(latest), { body: contents, contentType: "text/plain" }) + } +} + +async function stopTracing(app: ElectronApplication, testInfo: TestInfo) { + await app + .context() + .tracing.stop({ path: testInfo.outputPath("trace.zip") }) + .catch(() => undefined) +} + +export async function startScene( + vscodeExecutablePath: string, + scenario: VisualScenario, + testInfo: TestInfo, +): Promise { + const temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-code-electron-visual-")) + const workspacePath = path.join(temporaryRoot, "workspace") + const userDataPath = path.join(temporaryRoot, "user-data") + const extensionsPath = path.join(temporaryRoot, "extensions") + const readyPath = path.join(temporaryRoot, "ready.json") + const donePath = path.join(temporaryRoot, "done") + const logPath = path.join(temporaryRoot, "scene.log") + await Promise.all([fs.mkdir(workspacePath), fs.mkdir(userDataPath), fs.mkdir(extensionsPath)]) + + let app: ElectronApplication | undefined + let mock: InstanceType | undefined + let page: Page | undefined + try { + if (scenario.startMock) { + mock = new LLMock({ port: 0 }) + mock.addFixture({ + match: { userMessage: CHAT_PROMPT }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: CHAT_RESULT }), + id: "call_electron_visual_completion", + }, + ], + }, + }) + await mock.start() + } + + app = await _electron.launch({ + executablePath: vscodeExecutablePath, + args: [ + workspacePath, + `--extensionDevelopmentPath=${extensionDevelopmentPath}`, + `--extensionTestsPath=${extensionTestsPath}`, + `--user-data-dir=${userDataPath}`, + `--extensions-dir=${extensionsPath}`, + "--disable-workspace-trust", + "--skip-welcome", + "--skip-release-notes", + "--disable-updates", + "--disable-gpu", + "--force-device-scale-factor=1", + ], + env: { + ...process.env, + AIMOCK_URL: mock?.url ?? "", + ROO_CODE_VISUAL_CHAT_PROMPT: CHAT_PROMPT, + ROO_CODE_VISUAL_CHAT_RESULT: CHAT_RESULT, + ROO_CODE_VISUAL_DONE_PATH: donePath, + ROO_CODE_VISUAL_LOG_PATH: logPath, + ROO_CODE_VISUAL_READY_PATH: readyPath, + ROO_CODE_VISUAL_SCENE: scenario.scene, + ROO_CODE_VISUAL_THEME: scenario.themeId, + ROO_CODE_THEME_FIXTURE_PROBE: "1", + }, + locale: "en-US", + timeout: 60_000, + timezoneId: "UTC", + }) + await app.context().tracing.start({ screenshots: true, snapshots: true, sources: true }) + page = await app.firstWindow({ timeout: 60_000 }) + await app.evaluate(({ BrowserWindow }) => { + const window = BrowserWindow.getAllWindows()[0] + window?.setBounds({ x: 0, y: 0, width: 1200, height: 800 }) + window?.focus() + }) + const ready = await waitForJson(readyPath) + expect(ready).toMatchObject({ landmark: scenario.landmark, scene: scenario.scene, themeId: scenario.themeId }) + return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath } + } catch (error) { + if (page) await page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) + await attachLogs(userDataPath, logPath, testInfo) + await fs.writeFile(donePath, "done\n", "utf8").catch(() => undefined) + if (app) await stopTracing(app, testInfo) + await app?.close().catch(() => undefined) + await mock?.stop().catch(() => undefined) + await fs.rm(temporaryRoot, { recursive: true, force: true }) + if (error instanceof Error) throw error + throw new Error(String(error)) + } +} + +export async function attachFailureArtifacts(scene: RunningScene, testInfo: TestInfo) { + await scene.page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) + await attachLogs(scene.userDataPath, scene.logPath, testInfo) +} + +export async function stopScene(scene: RunningScene, testInfo: TestInfo) { + await fs.writeFile(scene.donePath, "done\n", "utf8").catch(() => undefined) + await stopTracing(scene.app, testInfo) + await scene.app.close().catch(() => undefined) + await scene.mock?.stop().catch(() => undefined) + await fs.rm(scene.temporaryRoot, { recursive: true, force: true }) +} diff --git a/apps/vscode-e2e/src/visual/heartbeat.electron.ts b/apps/vscode-e2e/src/visual/heartbeat.electron.ts new file mode 100644 index 0000000000..adc0cd1460 --- /dev/null +++ b/apps/vscode-e2e/src/visual/heartbeat.electron.ts @@ -0,0 +1,214 @@ +import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { expect, test, type Page } from "@playwright/test" + +import { + attachFailureArtifacts, + extensionDevelopmentPath, + startScene, + stopScene, + vscodeVersion, + type VisualScenario, +} from "./electronHarness" + +const scenario: VisualScenario = { + name: "heartbeat", + scene: "welcome", + themeId: "Default Dark Modern", + landmark: "Welcome to Zoo Code!", +} + +function contentFrame(page: Page) { + return page + .frames() + .find((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code")) + ?.childFrames() + .find((frame) => frame.url().includes("/fake.html")) +} + +async function waitForWelcome(page: Page) { + await expect.poll(() => Boolean(contentFrame(page))).toBe(true) + const frame = contentFrame(page) + if (!frame) throw new Error("Zoo Code content frame not found") + await expect(frame.getByText(scenario.landmark, { exact: false })).toBeVisible() + return frame +} + +async function observeHeartbeats(page: Page, block = false) { + const parent = page.frames().find((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code")) + if (!parent) throw new Error("Zoo Code parent frame not found") + return parent.evaluateHandle((blockHeartbeats) => { + const observed = { block: blockHeartbeats, blocked: 0, forwarded: 0, launches: 0 } + const original = MessagePort.prototype.postMessage + // VS Code 1.100 forwards webview messages through this parent's port. + // Intercept only heartbeat payloads; all other host traffic stays real. + MessagePort.prototype.postMessage = function (this: MessagePort, envelope: unknown, ...args: unknown[]) { + if ( + typeof envelope === "object" && + envelope !== null && + "channel" in envelope && + envelope.channel === "onmessage" && + "data" in envelope + ) { + const data = envelope.data + if (typeof data === "object" && data !== null && "message" in data) { + const message = data.message + if (typeof message === "object" && message !== null && "type" in message) { + if (message.type === "webviewDidLaunch") { + observed.launches++ + observed.block = false + } + if (message.type === "webviewHeartbeat") { + if (observed.block) { + observed.blocked++ + return + } + observed.forwarded++ + } + } + } + } + Reflect.apply(original, this, [envelope, ...args]) + } + return observed + }, block) +} + +let vscodeExecutablePath: string +test.beforeAll(async () => { + vscodeExecutablePath = await downloadAndUnzipVSCode({ version: vscodeVersion, extensionDevelopmentPath }) +}) + +test("recovers when heartbeat delivery stops and resumes periodic heartbeats", async ({ + browserName: _browserName, +}, testInfo) => { + test.setTimeout(4 * 60_000) + const running = await startScene(vscodeExecutablePath, scenario, testInfo) + try { + const frame = await waitForWelcome(running.page) + await frame.evaluate(() => { + document.documentElement.dataset.heartbeatDocument = "before-timeout" + }) + const heartbeat = await observeHeartbeats(running.page, true) + await expect + .poll(() => heartbeat.evaluate((state) => state.launches), { timeout: 150_000, intervals: [1_000] }) + .toBeGreaterThan(0) + expect(await heartbeat.evaluate((state) => state.blocked)).toBeGreaterThan(0) + const recovered = await waitForWelcome(running.page) + expect(await recovered.evaluate(() => document.documentElement.dataset.heartbeatDocument)).toBeUndefined() + // Mount effects can emit more than one launch/heartbeat message. Capture + // their counts after rendering, then require a later interval heartbeat. + const afterMount = await heartbeat.jsonValue() + expect(afterMount.forwarded).toBeGreaterThan(0) + await expect + .poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }) + .toBeGreaterThan(afterMount.forwarded) + expect(await heartbeat.evaluate((state) => state.launches)).toBe(afterMount.launches) + } catch (error) { + await attachFailureArtifacts(running, testInfo) + throw error + } finally { + await stopScene(running, testInfo) + } +}) + +test("recovers after the isolated Zoo Code renderer is terminated", async ({ browserName: _browserName }, testInfo) => { + test.setTimeout(4 * 60_000) + const running = await startScene(vscodeExecutablePath, scenario, testInfo) + try { + const initial = await waitForWelcome(running.page) + await initial.evaluate(() => { + document.documentElement.dataset.heartbeatDocument = "before-crash" + }) + const processes = await running.app.evaluate(({ BrowserWindow }) => { + const mainFrame = BrowserWindow.getAllWindows()[0]?.webContents.mainFrame + if (!mainFrame) throw new Error("VS Code workbench frame not found") + const renderer = mainFrame.framesInSubtree.find( + (frame: { url: string; parent?: { url: string }; osProcessId: number }) => + frame.url.includes("/fake.html") && + frame.parent?.url.includes("extensionId=ZooCodeOrganization.zoo-code"), + ) + if (!renderer) throw new Error("Zoo Code renderer process not found") + return { workbench: mainFrame.osProcessId, webview: renderer.osProcessId } + }) + await testInfo.attach("renderer-processes.json", { + body: JSON.stringify(processes), + contentType: "application/json", + }) + // Never terminate the workbench: this test requires a separate OOPIF process. + expect(processes.webview).toBeGreaterThan(0) + expect(processes.webview).not.toBe(processes.workbench) + process.kill(processes.webview, "SIGKILL") + + // Use production timers: the next stale tick is at most two minutes away. + await expect + .poll( + async () => { + try { + const frame = contentFrame(running.page) + if (!frame || !(await frame.getByText(scenario.landmark, { exact: false }).isVisible())) + return "missing" + return ( + (await frame.locator("html").getAttribute("data-heartbeat-document", { timeout: 1_000 })) ?? + "recovered" + ) + } catch { + return "unavailable" + } + }, + { timeout: 150_000, intervals: [1_000] }, + ) + .toBe("recovered") + await waitForWelcome(running.page) + const heartbeat = await observeHeartbeats(running.page) + await expect.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }).toBeGreaterThan(0) + } catch (error) { + await attachFailureArtifacts(running, testInfo) + throw error + } finally { + await stopScene(running, testInfo) + } +}) + +test("keeps the same document when the sidebar is hidden past the stale threshold", async ({ + browserName: _browserName, +}, testInfo) => { + test.setTimeout(4 * 60_000) + const running = await startScene(vscodeExecutablePath, scenario, testInfo) + try { + const frame = await waitForWelcome(running.page) + await frame.evaluate(() => { + document.documentElement.dataset.heartbeatDocument = "retained" + }) + const heartbeat = await observeHeartbeats(running.page, true) + await running.page.keyboard.press("Control+b") + await expect(running.page.locator(".part.sidebar")).toBeHidden() + const deadline = Date.now() + 125_000 + // Observe a complete production stale window and two watchdog ticks; + // this is a negative assertion window, not a delay for UI readiness. + await expect + .poll( + async () => { + expect(await frame.evaluate(() => document.documentElement.dataset.heartbeatDocument)).toBe( + "retained", + ) + return Date.now() + }, + { timeout: 135_000, intervals: [1_000] }, + ) + .toBeGreaterThanOrEqual(deadline) + expect(await heartbeat.evaluate((state) => state.blocked)).toBeGreaterThan(0) + await running.page.keyboard.press("Control+b") + await expect(running.page.locator(".part.sidebar")).toBeVisible() + const restored = await waitForWelcome(running.page) + expect(await restored.evaluate(() => document.documentElement.dataset.heartbeatDocument)).toBe("retained") + await heartbeat.evaluate((state) => { + state.block = false + }) + await expect.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }).toBeGreaterThan(0) + } catch (error) { + await attachFailureArtifacts(running, testInfo) + throw error + } finally { + await stopScene(running, testInfo) + } +}) diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md new file mode 100644 index 0000000000..ad4fddf727 --- /dev/null +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -0,0 +1,20 @@ +# Webview heartbeat checks + +Build the extension, production webview assets, and extension-host controller, then run the heartbeat project: + +```sh +pnpm --filter @roo-code/vscode-e2e visual:build +xvfb-run -a pnpm --filter @roo-code/vscode-e2e exec playwright test -c playwright-electron.config.ts --project=heartbeat +``` + +The scenarios use isolated VS Code instances and the production 60-second watchdog tick and 90-second stale threshold: + +- Block only heartbeat messages at VS Code's webview message port, verify the watchdog replaces the document, and verify mount and periodic heartbeats resume. +- Terminate the webview renderer after verifying that its OS process differs from the workbench process, then require the welcome UI and periodic heartbeats to return. A timeout records a screenshot, trace, extension logs, and process IDs. +- Hide the sidebar with heartbeats blocked for 125 seconds, verify the document survives, reopen it, and restore heartbeat delivery. + +The message-port interceptor follows the pinned VS Code implementation. Check it when upgrading VS Code. It does not modify the extension or fake extension-host timers. Heartbeat blocking and renderer termination are separate failure modes; passing the former does not establish crash recovery. + +At PR head `a985f03ba`, terminating the isolated renderer reproduces a gray sidebar: the watchdog logs recovery, but assigning fresh HTML does not restore the UI within 150 seconds. The crash test intentionally remains an active regression test. It must pass before claiming recovery from a terminated renderer. + +Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/apps/vscode-e2e/src/visual/sceneController.ts b/apps/vscode-e2e/src/visual/sceneController.ts index a4372c95ae..2511036c65 100644 --- a/apps/vscode-e2e/src/visual/sceneController.ts +++ b/apps/vscode-e2e/src/visual/sceneController.ts @@ -20,8 +20,13 @@ const themeKinds: Record = { "Default High Contrast": vscode.ColorThemeKind.HighContrast, } -async function poll(capture: () => Promise, matches: (value: T) => boolean, description: string): Promise { - const deadline = Date.now() + SCENE_TIMEOUT_MS +async function poll( + capture: () => Promise, + matches: (value: T) => boolean, + description: string, + timeoutMs = SCENE_TIMEOUT_MS, +): Promise { + const deadline = Date.now() + timeoutMs let latest: T | undefined while (Date.now() < deadline) { @@ -45,6 +50,7 @@ async function waitForFile(filePath: string) { }, Boolean, `file ${filePath}`, + 10 * 60_000, ) } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 754f531f60..5fffd00644 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1097,6 +1097,19 @@ describe("ClineProvider", () => { let visibilityCallback: () => void let disposeCallback: () => void + function trackHtmlAssignments() { + let html = mockWebviewView.webview.html + const assignHtml = vi.fn((value: string) => { + html = value + }) + Object.defineProperty(mockWebviewView.webview, "html", { + configurable: true, + get: () => html, + set: assignHtml, + }) + return assignHtml + } + beforeEach(() => { // Fake timers must be active before resolveWebviewView so the // watchdog interval is registered on the fake clock. @@ -1142,16 +1155,80 @@ describe("ClineProvider", () => { expect(mockWebviewView.webview.html).toContain("Zoo Code") }) + test.each([ + { age: 89_999, reloads: 0 }, + { age: 90_000, reloads: 0 }, + { age: 90_001, reloads: 1 }, + ])("at heartbeat age $age ms performs $reloads reloads", async ({ age, reloads }) => { + await provider.resolveWebviewView(mockWebviewView) + const assignHtml = trackHtmlAssignments() + // Keep the next tick ordinary while placing heartbeat age exactly on + // either side of the stale threshold at that tick. + provider["lastWebviewHeartbeatAt"] = Date.now() + 60_000 - age + + await vi.advanceTimersByTimeAsync(60_000) + + expect(assignHtml).toHaveBeenCalledTimes(reloads) + }) + + test.each([ + { gap: 89_999, reloads: 1 }, + { gap: 90_000, reloads: 1 }, + { gap: 90_001, reloads: 0 }, + ])("at watchdog tick gap $gap ms performs $reloads reloads", async ({ gap, reloads }) => { + await provider.resolveWebviewView(mockWebviewView) + const assignHtml = trackHtmlAssignments() + // The heartbeat is stale in every row; only the suspend-gap guard + // can suppress recovery. No wall-clock jumps or extra ticks mask it. + provider["lastWebviewHeartbeatAt"] = Date.now() - 120_000 + provider["lastWebviewWatchdogTickAt"] = Date.now() + 60_000 - gap + + await vi.advanceTimersByTimeAsync(60_000) + + expect(assignHtml).toHaveBeenCalledTimes(reloads) + }) + + test.each(["none", "disposed", "epoch"] as const)( + "checks only the %s post-await recovery guard", + async (guard) => { + await provider.resolveWebviewView(mockWebviewView) + const assignHtml = trackHtmlAssignments() + let finishReload: (html: string) => void = () => { + throw new Error("Recovery HTML generation has not started") + } + provider["getWebviewHtml"] = vi.fn( + () => + new Promise((resolve) => { + finishReload = resolve + }), + ) + const recovery = provider["reloadWebviewForRecovery"]() + // Change one field only: real dispose/re-resolve paths change + // multiple fields and would let other guards mask this one. + if (guard === "disposed") provider["_disposed"] = true + if (guard === "epoch") provider["webviewRecoveryEpoch"]++ + try { + finishReload("recovered-html") + await recovery + expect(assignHtml).toHaveBeenCalledTimes(guard === "none" ? 1 : 0) + if (guard === "none") expect(mockWebviewView.webview.html).toBe("recovered-html") + } finally { + // Allow the regular afterEach to dispose the provider's resources. + provider["_disposed"] = false + } + }, + ) + test("grants a new grace window after recovery and caps reloads until an actual heartbeat", async () => { await provider.resolveWebviewView(mockWebviewView) - const generateHtml = vi.spyOn(provider, "getState") + const assignHtml = trackHtmlAssignments() await vi.advanceTimersByTimeAsync(120_000) - expect(generateHtml).toHaveBeenCalledTimes(1) + expect(assignHtml).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(60_000) - expect(generateHtml).toHaveBeenCalledTimes(1) + expect(assignHtml).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(180_000) - expect(generateHtml).toHaveBeenCalledTimes(3) + expect(assignHtml).toHaveBeenCalledTimes(3) const log = vi.spyOn(provider, "log") @@ -1160,12 +1237,12 @@ describe("ClineProvider", () => { visibilityCallback() vi.setSystemTime(Date.now() + 120_000) await vi.advanceTimersByTimeAsync(600_000) - expect(generateHtml).toHaveBeenCalledTimes(3) + expect(assignHtml).toHaveBeenCalledTimes(3) expect(log).not.toHaveBeenCalledWith(expect.stringContaining("Webview heartbeat stale")) await webviewMessageHandler(provider, { type: "webviewHeartbeat" }) await vi.advanceTimersByTimeAsync(120_000) - expect(generateHtml).toHaveBeenCalledTimes(4) + expect(assignHtml).toHaveBeenCalledTimes(4) }) test("gives a replacement view a fresh recovery budget after exhausting attempts", async () => { From c90830a62dee4e299c56b3019c60d944bb897dad Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 29 Oct 2026 21:23:41 +0000 Subject: [PATCH 24/34] fix: recreate unresponsive webview hosts and preserve active tasks --- apps/vscode-e2e/src/visual/electronHarness.ts | 8 +- .../src/visual/heartbeat.electron.ts | 90 +++++-- apps/vscode-e2e/src/visual/heartbeat.md | 6 +- apps/vscode-e2e/src/visual/sceneController.ts | 23 +- .../__tests__/registerCommands.spec.ts | 21 +- src/activate/registerCommands.ts | 3 +- src/core/webview/ClineProvider.ts | 196 ++++++++++++-- .../webview/__tests__/ClineProvider.spec.ts | 252 +++++++++++++++++- src/core/webview/webviewMessageHandler.ts | 2 +- src/eslint-suppressions.json | 2 +- 10 files changed, 551 insertions(+), 52 deletions(-) diff --git a/apps/vscode-e2e/src/visual/electronHarness.ts b/apps/vscode-e2e/src/visual/electronHarness.ts index 861b612946..62e36ec712 100644 --- a/apps/vscode-e2e/src/visual/electronHarness.ts +++ b/apps/vscode-e2e/src/visual/electronHarness.ts @@ -36,9 +36,12 @@ interface RunningScene { page: Page temporaryRoot: string userDataPath: string + taskIds: string[] } -async function waitForJson(filePath: string): Promise<{ landmark: string; scene: string; themeId: string }> { +async function waitForJson( + filePath: string, +): Promise<{ landmark: string; scene: string; themeId: string; taskIds: string[] }> { const deadline = Date.now() + 90_000 while (Date.now() < deadline) { try { @@ -46,6 +49,7 @@ async function waitForJson(filePath: string): Promise<{ landmark: string; scene: landmark: string scene: string themeId: string + taskIds: string[] } } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT" && !(error instanceof SyntaxError)) throw error @@ -150,7 +154,7 @@ export async function startScene( }) const ready = await waitForJson(readyPath) expect(ready).toMatchObject({ landmark: scenario.landmark, scene: scenario.scene, themeId: scenario.themeId }) - return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath } + return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath, taskIds: ready.taskIds } } catch (error) { if (page) await page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) await attachLogs(userDataPath, logPath, testInfo) diff --git a/apps/vscode-e2e/src/visual/heartbeat.electron.ts b/apps/vscode-e2e/src/visual/heartbeat.electron.ts index adc0cd1460..410691916b 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.electron.ts +++ b/apps/vscode-e2e/src/visual/heartbeat.electron.ts @@ -1,8 +1,14 @@ +import fs from "fs/promises" +import path from "path" + +import { glob } from "glob" + import { downloadAndUnzipVSCode } from "@vscode/test-electron" import { expect, test, type Page } from "@playwright/test" import { attachFailureArtifacts, + CHAT_RESULT, extensionDevelopmentPath, startScene, stopScene, @@ -25,11 +31,11 @@ function contentFrame(page: Page) { .find((frame) => frame.url().includes("/fake.html")) } -async function waitForWelcome(page: Page) { +async function waitForContent(page: Page, landmark = scenario.landmark) { await expect.poll(() => Boolean(contentFrame(page))).toBe(true) const frame = contentFrame(page) if (!frame) throw new Error("Zoo Code content frame not found") - await expect(frame.getByText(scenario.landmark, { exact: false })).toBeVisible() + await expect(frame.getByText(landmark, { exact: false })).toBeVisible() return frame } @@ -38,6 +44,9 @@ async function observeHeartbeats(page: Page, block = false) { if (!parent) throw new Error("Zoo Code parent frame not found") return parent.evaluateHandle((blockHeartbeats) => { const observed = { block: blockHeartbeats, blocked: 0, forwarded: 0, launches: 0 } + // Reacquire the observation after a retained hidden frame reconnects; + // CDP handles from before suspension are not a readiness contract. + Reflect.set(window, Symbol.for("zooCode.heartbeatObservation"), observed) const original = MessagePort.prototype.postMessage // VS Code 1.100 forwards webview messages through this parent's port. // Intercept only heartbeat payloads; all other host traffic stays real. @@ -84,7 +93,7 @@ test("recovers when heartbeat delivery stops and resumes periodic heartbeats", a test.setTimeout(4 * 60_000) const running = await startScene(vscodeExecutablePath, scenario, testInfo) try { - const frame = await waitForWelcome(running.page) + const frame = await waitForContent(running.page) await frame.evaluate(() => { document.documentElement.dataset.heartbeatDocument = "before-timeout" }) @@ -93,7 +102,7 @@ test("recovers when heartbeat delivery stops and resumes periodic heartbeats", a .poll(() => heartbeat.evaluate((state) => state.launches), { timeout: 150_000, intervals: [1_000] }) .toBeGreaterThan(0) expect(await heartbeat.evaluate((state) => state.blocked)).toBeGreaterThan(0) - const recovered = await waitForWelcome(running.page) + const recovered = await waitForContent(running.page) expect(await recovered.evaluate(() => document.documentElement.dataset.heartbeatDocument)).toBeUndefined() // Mount effects can emit more than one launch/heartbeat message. Capture // their counts after rendering, then require a later interval heartbeat. @@ -113,9 +122,14 @@ test("recovers when heartbeat delivery stops and resumes periodic heartbeats", a test("recovers after the isolated Zoo Code renderer is terminated", async ({ browserName: _browserName }, testInfo) => { test.setTimeout(4 * 60_000) - const running = await startScene(vscodeExecutablePath, scenario, testInfo) + const running = await startScene( + vscodeExecutablePath, + { ...scenario, scene: "chat", startMock: true, landmark: CHAT_RESULT }, + testInfo, + ) try { - const initial = await waitForWelcome(running.page) + expect(running.taskIds).toHaveLength(1) + const initial = await waitForContent(running.page, CHAT_RESULT) await initial.evaluate(() => { document.documentElement.dataset.heartbeatDocument = "before-crash" }) @@ -145,7 +159,7 @@ test("recovers after the isolated Zoo Code renderer is terminated", async ({ bro async () => { try { const frame = contentFrame(running.page) - if (!frame || !(await frame.getByText(scenario.landmark, { exact: false }).isVisible())) + if (!frame || !(await frame.getByText(CHAT_RESULT, { exact: false }).isVisible())) return "missing" return ( (await frame.locator("html").getAttribute("data-heartbeat-document", { timeout: 1_000 })) ?? @@ -158,7 +172,14 @@ test("recovers after the isolated Zoo Code renderer is terminated", async ({ bro { timeout: 150_000, intervals: [1_000] }, ) .toBe("recovered") - await waitForWelcome(running.page) + await waitForContent(running.page, CHAT_RESULT) + await fs.writeFile(`${running.donePath}.probe`, "probe") + await expect + .poll(async () => { + const result = await fs.readFile(`${running.donePath}.probe.result`, "utf8").catch(() => "") + return result ? (JSON.parse(result) as { taskIds: string[] }) : undefined + }) + .toMatchObject({ taskIds: running.taskIds }) const heartbeat = await observeHeartbeats(running.page) await expect.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }).toBeGreaterThan(0) } catch (error) { @@ -175,36 +196,59 @@ test("keeps the same document when the sidebar is hidden past the stale threshol test.setTimeout(4 * 60_000) const running = await startScene(vscodeExecutablePath, scenario, testInfo) try { - const frame = await waitForWelcome(running.page) + const frame = await waitForContent(running.page) await frame.evaluate(() => { document.documentElement.dataset.heartbeatDocument = "retained" }) const heartbeat = await observeHeartbeats(running.page, true) + // Establish that the interceptor works before hiding. Chromium may + // suspend hidden renderer timers entirely, so no hidden tick is promised. + await expect.poll(() => heartbeat.evaluate((state) => state.blocked), { timeout: 45_000 }).toBeGreaterThan(0) await running.page.keyboard.press("Control+b") await expect(running.page.locator(".part.sidebar")).toBeHidden() - const deadline = Date.now() + 125_000 + const deadline = performance.now() + 125_000 // Observe a complete production stale window and two watchdog ticks; // this is a negative assertion window, not a delay for UI readiness. await expect - .poll( - async () => { - expect(await frame.evaluate(() => document.documentElement.dataset.heartbeatDocument)).toBe( - "retained", - ) - return Date.now() - }, - { timeout: 135_000, intervals: [1_000] }, - ) + .poll(() => performance.now(), { timeout: 135_000, intervals: [1_000] }) .toBeGreaterThanOrEqual(deadline) - expect(await heartbeat.evaluate((state) => state.blocked)).toBeGreaterThan(0) + const logs = await glob("logs/**/*Zoo-Code.log", { cwd: running.userDataPath }) + expect(logs.length).toBeGreaterThan(0) + for (const logPath of logs) { + const output = await fs.readFile(path.join(running.userDataPath, logPath), "utf8") + expect(output).not.toContain("Webview heartbeat stale while visible") + expect(output).not.toContain("recreating this webview host") + } await running.page.keyboard.press("Control+b") await expect(running.page.locator(".part.sidebar")).toBeVisible() - const restored = await waitForWelcome(running.page) + const restored = await waitForContent(running.page) expect(await restored.evaluate(() => document.documentElement.dataset.heartbeatDocument)).toBe("retained") - await heartbeat.evaluate((state) => { + const parent = restored.parentFrame() + if (!parent) throw new Error("Retained webview parent frame not found") + await parent.evaluate(() => { + const state: unknown = Reflect.get(window, Symbol.for("zooCode.heartbeatObservation")) + if (!state || typeof state !== "object" || !("block" in state)) + throw new Error("Heartbeat observation was lost") state.block = false }) - await expect.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }).toBeGreaterThan(0) + await expect + .poll( + () => + parent.evaluate(() => { + const state: unknown = Reflect.get(window, Symbol.for("zooCode.heartbeatObservation")) + if ( + !state || + typeof state !== "object" || + !("forwarded" in state) || + typeof state.forwarded !== "number" + ) { + throw new Error("Heartbeat observation was lost") + } + return state.forwarded + }), + { timeout: 45_000 }, + ) + .toBeGreaterThan(0) } catch (error) { await attachFailureArtifacts(running, testInfo) throw error diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index ad4fddf727..d8bb924e0f 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -10,11 +10,13 @@ xvfb-run -a pnpm --filter @roo-code/vscode-e2e exec playwright test -c playwrigh The scenarios use isolated VS Code instances and the production 60-second watchdog tick and 90-second stale threshold: - Block only heartbeat messages at VS Code's webview message port, verify the watchdog replaces the document, and verify mount and periodic heartbeats resume. -- Terminate the webview renderer after verifying that its OS process differs from the workbench process, then require the welcome UI and periodic heartbeats to return. A timeout records a screenshot, trace, extension logs, and process IDs. +- Terminate the webview renderer after verifying that its OS process differs from the workbench process, then require the task UI and periodic heartbeats to return. A fresh extension-host probe must report the same nonempty task stack. A timeout records a screenshot, trace, extension logs, and process IDs. - Hide the sidebar with heartbeats blocked for 125 seconds, verify the document survives, reopen it, and restore heartbeat delivery. The message-port interceptor follows the pinned VS Code implementation. Check it when upgrading VS Code. It does not modify the extension or fake extension-host timers. Heartbeat blocking and renderer termination are separate failure modes; passing the former does not establish crash recovery. -At PR head `a985f03ba`, terminating the isolated renderer reproduces a gray sidebar: the watchdog logs recovery, but assigning fresh HTML does not restore the UI within 150 seconds. The crash test intentionally remains an active regression test. It must pass before claiming recovery from a terminated renderer. +At PR head `a985f03ba`, terminating the isolated renderer reproduces a gray sidebar: the watchdog logs recovery, but assigning fresh HTML does not restore the UI within 150 seconds. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. + +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. Window reload is never automatic. A late action from an old notification is ignored after liveness returns. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/apps/vscode-e2e/src/visual/sceneController.ts b/apps/vscode-e2e/src/visual/sceneController.ts index 2511036c65..f2aba52046 100644 --- a/apps/vscode-e2e/src/visual/sceneController.ts +++ b/apps/vscode-e2e/src/visual/sceneController.ts @@ -38,9 +38,10 @@ async function poll( throw new Error(`Timed out waiting for ${description}; latest value: ${JSON.stringify(latest)}`) } -async function waitForFile(filePath: string) { +async function waitForFile(filePath: string, probe?: () => Promise) { await poll( async () => { + await probe?.() try { await fs.access(filePath) return true @@ -144,6 +145,22 @@ export async function run(): Promise { landmark = result } - await fs.writeFile(readyPath, `${JSON.stringify({ scene, themeId, landmark, themeFixture }, null, 2)}\n`, "utf8") - await waitForFile(donePath) + await fs.writeFile( + readyPath, + `${JSON.stringify({ scene, themeId, landmark, themeFixture, taskIds: api.getCurrentTaskStack() }, null, 2)}\n`, + "utf8", + ) + // A request written by Playwright after recovery proves a fresh round trip + // through the real host and compares its task stack with the initial one. + await waitForFile(donePath, async () => { + const request = `${donePath}.probe` + try { + await fs.access(request) + } catch { + return + } + await fs.unlink(request) + const fixture = await api.captureWebviewThemeFixture() + await fs.writeFile(`${request}.result`, JSON.stringify({ fixture, taskIds: api.getCurrentTaskStack() }), "utf8") + }) } diff --git a/src/activate/__tests__/registerCommands.spec.ts b/src/activate/__tests__/registerCommands.spec.ts index d041dada2e..6ef57dacb6 100644 --- a/src/activate/__tests__/registerCommands.spec.ts +++ b/src/activate/__tests__/registerCommands.spec.ts @@ -5,7 +5,7 @@ import { WebviewFocusTracker } from "../../core/webview/WebviewFocusTracker" import { ContextProxy } from "../../core/config/ContextProxy" import { makeExtensionContext } from "../../test-utils/vscode" -import { getVisibleProviderOrLog, openClineInNewTab, registerCommands, setPanel } from "../registerCommands" +import { getPanel, getVisibleProviderOrLog, openClineInNewTab, registerCommands, setPanel } from "../registerCommands" vi.mock("execa", () => ({ execa: vi.fn(), @@ -461,6 +461,25 @@ describe("openClineInNewTab", () => { setPanel(undefined, "tab") }) + it.each([false, true])( + "clears a closed tab reference only if it is still current (replacement: %s)", + async (replaced) => { + await openClineInNewTab({ + context: mockContext, + outputChannel: mockOutputChannel, + webviewFocusTracker: new WebviewFocusTracker(), + }) + const panel = vi.mocked(vscode.window.createWebviewPanel).mock.results[0].value + const onDispose = vi.mocked(panel.onDidDispose).mock.calls[0][0] + const replacement = { ...panel } + if (replaced) setPanel(replacement, "tab") + + onDispose() + + expect(getPanel()).toBe(replaced ? replacement : undefined) + }, + ) + it("creates a webview panel with title 'Zoo Code'", async () => { const webviewFocusTracker = new WebviewFocusTracker() await openClineInNewTab({ context: mockContext, outputChannel: mockOutputChannel, webviewFocusTracker }) diff --git a/src/activate/registerCommands.ts b/src/activate/registerCommands.ts index 9ee3443f9a..2851dac016 100644 --- a/src/activate/registerCommands.ts +++ b/src/activate/registerCommands.ts @@ -295,7 +295,8 @@ export const openClineInNewTab = async ({ context, outputChannel, webviewFocusTr // Handle panel closing events. newPanel.onDidDispose( () => { - setPanel(undefined, "tab") + // A recovery may already have installed a replacement panel. + if (tabPanel === newPanel) setPanel(undefined, "tab") }, null, context.subscriptions, // Also register dispose listener diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index ada1d21658..895030b91a 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -104,7 +104,7 @@ import { getWorkspaceGitInfo } from "../../utils/git" import { getWorkspacePath } from "../../utils/path" import { OrganizationAllowListViolationError } from "../../utils/errors" -import { setPanel } from "../../activate/registerCommands" +import { getPanel, setPanel } from "../../activate/registerCommands" import { t } from "../../i18n" @@ -248,6 +248,13 @@ export class ClineProvider // millisecond; pending recovery must yield to either sign of renewed grace. private webviewHeartbeatRevision = 0 private webviewRecoveryAttempts = 0 + private webviewRecoveryConfirmation: ReturnType | undefined + private webviewHostRecoveryInProgress = false + private webviewRecoveryFailureReported = false + private lastReceivedWebviewHeartbeatAt: number | undefined + private webviewHealthySince: number | undefined + private static readonly WEBVIEW_RECOVERY_CONFIRM_MS = 10_000 + private static readonly WEBVIEW_HEALTHY_WINDOW_MS = 60_000 private static readonly WEBVIEW_RECOVERY_ATTEMPT_LIMIT = 3 private webviewWatchdogInterval: ReturnType | null = null // Bumped to invalidate an in-flight recovery reload when the provider or @@ -854,8 +861,14 @@ export class ClineProvider // Invalidate any recovery reload still awaiting its HTML so it cannot // reassign webview.html on the disposed view. this.webviewRecoveryEpoch++ - // A replacement renderer needs its own recovery-attempt budget. - this.webviewRecoveryAttempts = 0 + // User-opened replacements get a fresh budget; recreating a failed host + // must preserve its incident budget so crash loops remain bounded. + if (!this.webviewHostRecoveryInProgress) { + this.webviewRecoveryAttempts = 0 + this.webviewRecoveryFailureReported = false + } + this.lastReceivedWebviewHeartbeatAt = undefined + this.webviewHealthySince = undefined while (this.webviewDisposables.length) { const x = this.webviewDisposables.pop() if (x) { @@ -1079,6 +1092,7 @@ export class ClineProvider } if (inTabMode) { + if (getPanel() === webviewView) setPanel(undefined, "tab") this.log("Disposing ClineProvider instance for tab view") await this.dispose() } else { @@ -1226,7 +1240,9 @@ export class ClineProvider // If the extension is starting a new session, clear previous task state. // But don't clear if there's already an active task (e.g., resumed via IPC/bridge). const currentTask = this.getCurrentTask() - if (!currentTask || currentTask.abandoned || currentTask.abort) { + // Host recreation is a UI repair, including when the displayed task is + // interrupted. It must not evict or resume that task as a side effect. + if (!this.webviewHostRecoveryInProgress && (!currentTask || currentTask.abandoned || currentTask.abort)) { await this.removeClineFromStack() } @@ -3544,14 +3560,34 @@ export class ClineProvider ) } - /** Records that the webview renderer is alive; called on every webviewHeartbeat message. */ + /** A heartbeat confirms liveness; only sustained health replenishes recovery attempts. */ public updateWebviewHeartbeat(): void { + const now = Date.now() + if ( + this.lastReceivedWebviewHeartbeatAt === undefined || + now - this.lastReceivedWebviewHeartbeatAt > ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS + ) { + this.webviewHealthySince = now + } + this.lastReceivedWebviewHeartbeatAt = now + if (this.webviewRecoveryConfirmation) { + this.log("[Zoo Code] Webview heartbeat returned after recovery") + } this.resetWebviewHeartbeatGrace() - this.webviewRecoveryAttempts = 0 + if ( + this.webviewHealthySince !== undefined && + now - this.webviewHealthySince >= ClineProvider.WEBVIEW_HEALTHY_WINDOW_MS + ) { + if (this.webviewRecoveryAttempts > 0) + this.log("[Zoo Code] Webview recovery confirmed by sustained heartbeats") + this.webviewRecoveryAttempts = 0 + this.webviewRecoveryFailureReported = false + } } /** Restarts the grace window and invalidates pending recovery without recording a heartbeat. */ private resetWebviewHeartbeatGrace(): void { + this.clearWebviewRecoveryConfirmation() this.lastWebviewHeartbeatAt = Date.now() this.webviewHeartbeatRevision++ } @@ -3581,19 +3617,19 @@ export class ClineProvider if (this.view?.visible !== true) { return } - if ( - now - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS || - this.webviewRecoveryAttempts >= ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT - ) { + if (now - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS) return + if (this.webviewRecoveryAttempts >= ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT) { + this.reportWebviewRecoveryFailure("Recovery attempt limit reached without sustained heartbeats") return } - this.log("[Zoo Code] Webview heartbeat stale while visible; reloading webview (dead renderer?)") + this.log("[Zoo Code] Webview heartbeat stale while visible; attempting recovery") void this.reloadWebviewForRecovery() }, ClineProvider.WEBVIEW_WATCHDOG_TICK_MS) } /** Stops the renderer heartbeat watchdog; the webview it watches is gone. */ private stopWebviewWatchdog(): void { + this.clearWebviewRecoveryConfirmation() if (this.webviewWatchdogInterval) { clearInterval(this.webviewWatchdogInterval) this.webviewWatchdogInterval = null @@ -3601,9 +3637,8 @@ export class ClineProvider } /** - * Reloads only this provider's own webview by regenerating its HTML (fresh - * nonce) and reassigning `webview.html`, which forces VS Code to reload that - * webview. Works for both sidebar (WebviewView) and tab (WebviewPanel) shapes. + * First attempt: regenerate this view's HTML. A missing heartbeat escalates + * to host recreation because HTML delivery cannot restart a dead renderer. */ private async reloadWebviewForRecovery(): Promise { const view = this.view @@ -3621,6 +3656,10 @@ export class ClineProvider return } this.webviewRecoveryInFlightEpoch = epoch + // Count the whole attempt, including HTML-generation failures. + this.webviewRecoveryAttempts++ + this.lastReceivedWebviewHeartbeatAt = undefined + this.webviewHealthySince = undefined // A heartbeat that arrives while the HTML is being generated means the // renderer is alive again; comparing revisions (not timestamps) lets the // post-await check catch heartbeats that land in the same millisecond. @@ -3641,8 +3680,31 @@ export class ClineProvider return } view.webview.html = html - this.webviewRecoveryAttempts++ this.resetWebviewHeartbeatGrace() + const revision = this.webviewHeartbeatRevision + const requestedAt = Date.now() + this.webviewRecoveryConfirmation = setTimeout(() => { + this.webviewRecoveryConfirmation = undefined + // A late confirmation timer can be a host suspend, just like a + // late watchdog tick. Let renderer timers resume before escalating. + if (Date.now() - requestedAt > ClineProvider.WEBVIEW_RECOVERY_CONFIRM_MS * 1.5) { + this.resetWebviewHeartbeatGrace() + return + } + if ( + this._disposed || + this.view !== view || + this.webviewRecoveryEpoch !== epoch || + this.webviewHeartbeatRevision !== revision || + !view.visible + ) + return + void this.recreateWebviewHostForRecovery(view).catch((error: unknown) => { + this.reportWebviewRecoveryFailure( + `Host recreation failed: ${error instanceof Error ? error.message : String(error)}`, + ) + }) + }, ClineProvider.WEBVIEW_RECOVERY_CONFIRM_MS) } catch (error) { this.log(`[Zoo Code] Failed to reload webview: ${error instanceof Error ? error.message : String(error)}`) } finally { @@ -3654,6 +3716,110 @@ export class ClineProvider } } + private clearWebviewRecoveryConfirmation(): void { + if (this.webviewRecoveryConfirmation) clearTimeout(this.webviewRecoveryConfirmation) + this.webviewRecoveryConfirmation = undefined + } + + /** Recreate the host/transport: sending more HTML cannot revive its dead renderer. */ + private async recreateWebviewHostForRecovery(view: vscode.WebviewView | vscode.WebviewPanel): Promise { + if (this.webviewHostRecoveryInProgress || this._disposed || this.view !== view || !view.visible) return + const revision = this.webviewHeartbeatRevision + let started = false + try { + let commands: string[] | undefined + if (!("onDidChangeViewState" in view)) { + // Generated VS Code view commands are not an extension API contract. + // Check availability and cover them against the pinned real host in E2E. + commands = await vscode.commands.getCommands(true) + } + if ( + this.webviewHostRecoveryInProgress || + this._disposed || + this.view !== view || + !view.visible || + this.webviewHeartbeatRevision !== revision + ) + return + if ( + commands && + (!commands.includes(`${ClineProvider.sideBarId}.removeView`) || + !commands.includes(`${ClineProvider.sideBarId}.focus`)) + ) { + throw new Error("VS Code does not provide the scoped sidebar recreation commands") + } + this.webviewHostRecoveryInProgress = true + started = true + this.log("[Zoo Code] No heartbeat after HTML reload; recreating this webview host") + if ("onDidChangeViewState" in view) { + const replacement = vscode.window.createWebviewPanel( + ClineProvider.tabPanelId, + view.title, + { viewColumn: view.viewColumn ?? vscode.ViewColumn.Active, preserveFocus: true }, + { ...view.options, ...view.webview.options }, + ) + replacement.iconPath = view.iconPath + // Resolve publishes the replacement before yielding. The old view's + // disposal must see it no longer owns this provider and its tasks. + const resolved = this.resolveWebviewView(replacement) + if (this.view === replacement) view.dispose() + else replacement.dispose() + await resolved + } else { + await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.removeView`) + if (this._disposed) return + await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.focus`, { preserveFocus: true }) + await pWaitFor( + () => + this._disposed || + (this.view !== undefined && this.view !== view && this.webviewWatchdogInterval !== null), + { + interval: 50, + timeout: 10_000, + }, + ) + } + } catch (error) { + // A late discovery failure must not stop a recovered/replaced view. + // Once recreation starts, disposal of the old view is expected. + if ( + this._disposed || + (!started && (this.view !== view || this.webviewHeartbeatRevision !== revision || !view.visible)) + ) + return + if (started && this.lastReceivedWebviewHeartbeatAt !== undefined) return + this.reportWebviewRecoveryFailure( + `Host recreation failed: ${error instanceof Error ? error.message : String(error)}`, + ) + } finally { + if (started) this.webviewHostRecoveryInProgress = false + } + } + + private reportWebviewRecoveryFailure(reason: string): void { + if (this._disposed || this.webviewRecoveryFailureReported) return + this.webviewRecoveryFailureReported = true + this.webviewRecoveryAttempts = ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT + this.log(`[Zoo Code] Webview recovery stopped: ${reason}`) + const revision = this.webviewHeartbeatRevision + void Promise.resolve( + vscode.window.showWarningMessage( + "Zoo Code's interface is not responding. Reloading the VS Code window will restart extensions and may interrupt running tasks.", + "Reload Window", + "Show Logs", + ), + ) + .then(async (action) => { + if (action === "Reload Window" && !this._disposed && this.webviewHeartbeatRevision === revision) { + await vscode.commands.executeCommand("workbench.action.reloadWindow") + } + if (action === "Show Logs") this.outputChannel.show() + }) + .then(undefined, (error: unknown) => { + this.log(`[Zoo Code] Recovery notification failed: ${String(error)}`) + }) + } + /** Builds the webview HTML using the same path as resolveWebviewView (HMR in development). */ private async getWebviewHtml(webview: vscode.Webview): Promise { return this.contextProxy.extensionMode === vscode.ExtensionMode.Development && diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 5fffd00644..8ea5fca664 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -175,12 +175,15 @@ vi.mock("vscode", () => ({ QuickFix: { value: "quickfix" }, RefactorRewrite: { value: "refactor.rewrite" }, }, + ViewColumn: { Active: -1 }, commands: { + getCommands: vi.fn().mockResolvedValue([]), executeCommand: vi.fn().mockResolvedValue(undefined), }, window: { + createWebviewPanel: vi.fn(), showInformationMessage: vi.fn(), - showWarningMessage: vi.fn(), + showWarningMessage: vi.fn().mockResolvedValue(undefined), showErrorMessage: vi.fn(), showSaveDialog: vi.fn(), showOpenDialog: vi.fn(), @@ -273,7 +276,7 @@ vi.mock("../../../integrations/workspace/WorkspaceTracker", () => { }) vi.mock("../../task/Task", () => ({ - Task: vi.fn().mockImplementation(function (options: any) { + Task: vi.fn().mockImplementation(function (options: TaskOptions) { return { api: undefined, abortTask: vi.fn(), @@ -518,6 +521,7 @@ describe("ClineProvider", () => { // Mock output channel mockOutputChannel = { + show: vi.fn(), appendLine: vi.fn(), clear: vi.fn(), dispose: vi.fn(), @@ -1111,6 +1115,12 @@ describe("ClineProvider", () => { } beforeEach(() => { + vi.mocked(vscode.commands.executeCommand).mockReset().mockResolvedValue(undefined) + vi.mocked(vscode.window.showWarningMessage).mockResolvedValue(undefined) + vi.mocked(vscode.commands.getCommands).mockResolvedValue([ + `${ClineProvider.sideBarId}.removeView`, + `${ClineProvider.sideBarId}.focus`, + ]) // Fake timers must be active before resolveWebviewView so the // watchdog interval is registered on the fake clock. vi.useFakeTimers() @@ -1219,7 +1229,7 @@ describe("ClineProvider", () => { }, ) - test("grants a new grace window after recovery and caps reloads until an actual heartbeat", async () => { + test("grants a new grace window after recovery and caps reloads until sustained heartbeats", async () => { await provider.resolveWebviewView(mockWebviewView) const assignHtml = trackHtmlAssignments() @@ -1242,9 +1252,184 @@ describe("ClineProvider", () => { await webviewMessageHandler(provider, { type: "webviewHeartbeat" }) await vi.advanceTimersByTimeAsync(120_000) + expect(assignHtml).toHaveBeenCalledTimes(3) + // One mount heartbeat is insufficient: a new healthy window must + // span a full minute of periodic heartbeats before replenishing. + provider.updateWebviewHeartbeat() + await vi.advanceTimersByTimeAsync(30_000) + provider.updateWebviewHeartbeat() + await vi.advanceTimersByTimeAsync(30_000) + provider.updateWebviewHeartbeat() + await vi.advanceTimersByTimeAsync(120_000) expect(assignHtml).toHaveBeenCalledTimes(4) }) + test.each([false, true])( + "recreates only the failed sidebar host, retaining its task and budget (interrupted: %s)", + async (interrupted) => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + task.abort = interrupted + provider["taskRegistry"].push(task) + const evict = vi.spyOn(provider, "evictCurrentTask") + const remove = vi.spyOn(provider, "removeClineFromStack") + const replacement = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) await provider.resolveWebviewView(replacement) + }) + + await vi.advanceTimersByTimeAsync(120_000) + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith(`${ClineProvider.sideBarId}.removeView`) + await vi.advanceTimersByTimeAsync(10_000) + + expect(vscode.commands.executeCommand).toHaveBeenCalledWith(`${ClineProvider.sideBarId}.removeView`) + expect(vscode.commands.executeCommand).toHaveBeenCalledWith(`${ClineProvider.sideBarId}.focus`, { + preserveFocus: true, + }) + expect(provider["view"]).toBe(replacement) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(provider.getCurrentTask()).toBe(task) + expect(evict).not.toHaveBeenCalled() + expect(remove).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + }, + ) + + test.each(["heartbeat", "hidden", "disposed", "suspended"] as const)( + "does not recreate the host after %s during the confirmation window", + async (event) => { + await provider.resolveWebviewView(mockWebviewView) + await vi.advanceTimersByTimeAsync(120_000) + if (event === "heartbeat") provider.updateWebviewHeartbeat() + if (event === "hidden") + Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) + if (event === "disposed") await provider.dispose() + if (event === "suspended") vi.setSystemTime(Date.now() + 120_000) + + await vi.advanceTimersByTimeAsync(10_000) + + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith(`${ClineProvider.sideBarId}.removeView`) + }, + ) + + test.each(["heartbeat", "hidden", "disposed", "replacement"] as const)( + "cancels host recreation after %s during command discovery", + async (event) => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockImplementation(async () => { + if (event === "heartbeat") provider.updateWebviewHeartbeat() + if (event === "hidden") + Object.defineProperty(mockWebviewView, "visible", { value: false, configurable: true }) + if (event === "disposed") await provider.dispose() + if (event === "replacement") provider["view"] = { ...mockWebviewView } + // Even an unavailable-command result must be ignored after cancellation. + return [] + }) + + await vi.advanceTimersByTimeAsync(130_000) + + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith(`${ClineProvider.sideBarId}.removeView`) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + }, + ) + + test("ignores a late command-discovery error after a heartbeat returns", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockImplementation(async () => { + provider.updateWebviewHeartbeat() + throw new Error("discovery failed") + }) + + await vi.advanceTimersByTimeAsync(130_000) + + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + }) + + test.each(["unavailable commands", "failed removal"] as const)( + "stops and notifies once when scoped recovery has %s", + async (failure) => { + await provider.resolveWebviewView(mockWebviewView) + if (failure === "unavailable commands") vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) + if (failure === "failed removal") + vi.mocked(vscode.commands.executeCommand).mockRejectedValue(new Error("removal failed")) + const assignHtml = trackHtmlAssignments() + + await vi.advanceTimersByTimeAsync(600_000) + + expect(assignHtml).toHaveBeenCalledTimes(1) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") + expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( + expect.stringContaining("Webview recovery stopped:"), + ) + }, + ) + + test.each(["Reload Window", "Show Logs"] as const)( + "runs only the selected recovery action: %s", + async (action) => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) + vi.mocked(vscode.window.showWarningMessage).mockImplementation(vi.fn().mockResolvedValue(action)) + + await vi.advanceTimersByTimeAsync(130_000) + + if (action === "Reload Window") + expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.reloadWindow") + if (action === "Show Logs") { + expect(mockOutputChannel.show).toHaveBeenCalledTimes(1) + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") + } + }, + ) + + test("ignores an old Reload Window choice after a heartbeat returns", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) + let choose: (value: string) => void = () => { + throw new Error("Notification not shown") + } + const choice = new Promise((resolve) => { + choose = resolve + }) + vi.mocked(vscode.window.showWarningMessage).mockImplementation(vi.fn().mockReturnValue(choice)) + await vi.advanceTimersByTimeAsync(130_000) + + provider.updateWebviewHeartbeat() + choose("Reload Window") + await vi.advanceTimersByTimeAsync(0) + + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") + }) + + test("bounds failed HTML generation and reports the exhausted budget once", async () => { + await provider.resolveWebviewView(mockWebviewView) + const generate = vi.spyOn(provider, "getState").mockRejectedValue(new Error("cannot generate")) + const assignHtml = trackHtmlAssignments() + + await vi.advanceTimersByTimeAsync(600_000) + + expect(generate).toHaveBeenCalledTimes(3) + expect(assignHtml).not.toHaveBeenCalled() + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + + test("does not replenish attempts in a mount-heartbeat-crash loop", async () => { + await provider.resolveWebviewView(mockWebviewView) + const assignHtml = trackHtmlAssignments() + for (let attempt = 0; attempt < 3; attempt++) { + await vi.advanceTimersByTimeAsync(120_000) + provider.updateWebviewHeartbeat() + } + await vi.advanceTimersByTimeAsync(240_000) + + expect(assignHtml).toHaveBeenCalledTimes(3) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + test("gives a replacement view a fresh recovery budget after exhausting attempts", async () => { await provider.resolveWebviewView(mockWebviewView) const generateHtml = vi.spyOn(provider, "getState") @@ -1883,6 +2068,67 @@ describe("ClineProvider", () => { } as unknown as vscode.WebviewPanel }) + test.each([false, true])( + "replaces the tab host without disposing its provider or task (interrupted: %s)", + async (interrupted) => { + const callbacks: Array<() => unknown> = [] + vi.mocked(mockWebviewPanel.onDidDispose).mockImplementation((listener) => { + callbacks.push(listener) + return { dispose: vi.fn() } + }) + vi.mocked(mockWebviewPanel.dispose).mockImplementation(() => { + callbacks.forEach((callback) => callback()) + }) + mockWebviewPanel.title = "Zoo Code task" + await provider.resolveWebviewView(mockWebviewPanel) + const task = new Task(defaultTaskOptions) + task.abort = interrupted + provider["taskRegistry"].push(task) + const evict = vi.spyOn(provider, "evictCurrentTask") + const remove = vi.spyOn(provider, "removeClineFromStack") + const replacement = { + ...mockWebviewPanel, + webview: { ...mockWebviewPanel.webview, html: "" }, + onDidDispose: vi.fn(() => ({ dispose: vi.fn() })), + dispose: vi.fn(), + } + vi.mocked(vscode.window.createWebviewPanel).mockReturnValue(replacement) + + await vi.advanceTimersByTimeAsync(130_000) + + expect(mockWebviewPanel.dispose).toHaveBeenCalledTimes(1) + expect(provider["view"]).toBe(replacement) + expect(provider["_disposed"]).toBe(false) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(provider.getCurrentTask()).toBe(task) + expect(evict).not.toHaveBeenCalled() + expect(remove).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith( + `${ClineProvider.sideBarId}.removeView`, + ) + }, + ) + + test("keeps the old tab and its task when replacement resolution fails before publication", async () => { + await provider.resolveWebviewView(mockWebviewPanel) + const task = new Task(defaultTaskOptions) + provider["taskRegistry"].push(task) + const replacement = { ...mockWebviewPanel, dispose: vi.fn() } + vi.mocked(vscode.window.createWebviewPanel).mockReturnValue(replacement) + vi.spyOn(provider, "resolveWebviewView").mockRejectedValue(new Error("resolver unavailable")) + + await provider["recreateWebviewHostForRecovery"](mockWebviewPanel) + + expect(mockWebviewPanel.dispose).not.toHaveBeenCalled() + expect(replacement.dispose).toHaveBeenCalledTimes(1) + expect(provider["view"]).toBe(mockWebviewPanel) + expect(provider.getCurrentTask()).toBe(task) + expect(task.dispose).not.toHaveBeenCalled() + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + test("reloads the tab webview when the heartbeat is stale and the tab is visible", async () => { await provider.resolveWebviewView(mockWebviewPanel) const htmlAfterResolve = mockWebviewPanel.webview.html diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index 8f3fa69b0d..231acdd448 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -701,7 +701,7 @@ export const webviewMessageHandler = async ( provider.isViewLaunched = true break case "webviewHeartbeat": - // Timestamp-only update for the dead-renderer watchdog; no other side effects. + // Confirm renderer liveness and advance the watchdog recovery health window. provider.updateWebviewHeartbeat() break case "newTask": diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 8a9e1fa4df..1960139787 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -1016,7 +1016,7 @@ }, "core/webview/__tests__/ClineProvider.spec.ts": { "@typescript-eslint/no-explicit-any": { - "count": 196 + "count": 195 } }, "core/webview/__tests__/ClineProvider.sticky-mode.spec.ts": { From 361a48be53242455da57604f36a23e84b397b3f2 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 29 Oct 2026 22:23:10 +0000 Subject: [PATCH 25/34] fix: address heartbeat recovery review feedback --- apps/vscode-e2e/src/visual/electronHarness.ts | 12 +++- .../src/visual/heartbeat.electron.ts | 9 ++- apps/vscode-e2e/src/visual/sceneController.ts | 8 ++- src/core/webview/ClineProvider.ts | 11 ++- .../webview/__tests__/ClineProvider.spec.ts | 67 +++++++++++++++++++ 5 files changed, 98 insertions(+), 9 deletions(-) diff --git a/apps/vscode-e2e/src/visual/electronHarness.ts b/apps/vscode-e2e/src/visual/electronHarness.ts index 62e36ec712..96bab3dd0f 100644 --- a/apps/vscode-e2e/src/visual/electronHarness.ts +++ b/apps/vscode-e2e/src/visual/electronHarness.ts @@ -91,12 +91,20 @@ export async function startScene( const readyPath = path.join(temporaryRoot, "ready.json") const donePath = path.join(temporaryRoot, "done") const logPath = path.join(temporaryRoot, "scene.log") - await Promise.all([fs.mkdir(workspacePath), fs.mkdir(userDataPath), fs.mkdir(extensionsPath)]) let app: ElectronApplication | undefined let mock: InstanceType | undefined let page: Page | undefined try { + // Settle all directory operations before failure cleanup removes their parent. + const directories = await Promise.allSettled([ + fs.mkdir(workspacePath), + fs.mkdir(userDataPath), + fs.mkdir(extensionsPath), + ]) + for (const result of directories) { + if (result.status === "rejected") throw result.reason + } if (scenario.startMock) { mock = new LLMock({ port: 0 }) mock.addFixture({ @@ -157,7 +165,7 @@ export async function startScene( return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath, taskIds: ready.taskIds } } catch (error) { if (page) await page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) - await attachLogs(userDataPath, logPath, testInfo) + await attachLogs(userDataPath, logPath, testInfo).catch(() => undefined) await fs.writeFile(donePath, "done\n", "utf8").catch(() => undefined) if (app) await stopTracing(app, testInfo) await app?.close().catch(() => undefined) diff --git a/apps/vscode-e2e/src/visual/heartbeat.electron.ts b/apps/vscode-e2e/src/visual/heartbeat.electron.ts index 410691916b..de503453db 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.electron.ts +++ b/apps/vscode-e2e/src/visual/heartbeat.electron.ts @@ -90,7 +90,8 @@ test.beforeAll(async () => { test("recovers when heartbeat delivery stops and resumes periodic heartbeats", async ({ browserName: _browserName, }, testInfo) => { - test.setTimeout(4 * 60_000) + // Allow startup, stale detection, recovery observations, and cleanup. + test.setTimeout(8 * 60_000) const running = await startScene(vscodeExecutablePath, scenario, testInfo) try { const frame = await waitForContent(running.page) @@ -121,7 +122,8 @@ test("recovers when heartbeat delivery stops and resumes periodic heartbeats", a }) test("recovers after the isolated Zoo Code renderer is terminated", async ({ browserName: _browserName }, testInfo) => { - test.setTimeout(4 * 60_000) + // Allow startup, stale detection, recovery observations, and cleanup. + test.setTimeout(8 * 60_000) const running = await startScene( vscodeExecutablePath, { ...scenario, scene: "chat", startMock: true, landmark: CHAT_RESULT }, @@ -193,7 +195,8 @@ test("recovers after the isolated Zoo Code renderer is terminated", async ({ bro test("keeps the same document when the sidebar is hidden past the stale threshold", async ({ browserName: _browserName, }, testInfo) => { - test.setTimeout(4 * 60_000) + // Allow 210s startup, 125s hidden, two 45s polls, and cleanup. + test.setTimeout(8 * 60_000) const running = await startScene(vscodeExecutablePath, scenario, testInfo) try { const frame = await waitForContent(running.page) diff --git a/apps/vscode-e2e/src/visual/sceneController.ts b/apps/vscode-e2e/src/visual/sceneController.ts index f2aba52046..c01c1e60f3 100644 --- a/apps/vscode-e2e/src/visual/sceneController.ts +++ b/apps/vscode-e2e/src/visual/sceneController.ts @@ -159,8 +159,14 @@ export async function run(): Promise { } catch { return } + let fixture: WebviewThemeFixture + try { + fixture = await api.captureWebviewThemeFixture() + } catch (error) { + await log(`theme probe will retry: ${String(error)}`) + return + } await fs.unlink(request) - const fixture = await api.captureWebviewThemeFixture() await fs.writeFile(`${request}.result`, JSON.stringify({ fixture, taskIds: api.getCurrentTaskStack() }), "utf8") }) } diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 895030b91a..f84d4688b9 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -252,6 +252,7 @@ export class ClineProvider private webviewHostRecoveryInProgress = false private webviewRecoveryFailureReported = false private lastReceivedWebviewHeartbeatAt: number | undefined + private webviewHeartbeatCount = 0 private webviewHealthySince: number | undefined private static readonly WEBVIEW_RECOVERY_CONFIRM_MS = 10_000 private static readonly WEBVIEW_HEALTHY_WINDOW_MS = 60_000 @@ -3570,6 +3571,7 @@ export class ClineProvider this.webviewHealthySince = now } this.lastReceivedWebviewHeartbeatAt = now + this.webviewHeartbeatCount++ if (this.webviewRecoveryConfirmation) { this.log("[Zoo Code] Webview heartbeat returned after recovery") } @@ -3619,7 +3621,10 @@ export class ClineProvider } if (now - this.lastWebviewHeartbeatAt <= ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS) return if (this.webviewRecoveryAttempts >= ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT) { - this.reportWebviewRecoveryFailure("Recovery attempt limit reached without sustained heartbeats") + // Let the final attempt finish generating HTML before declaring recovery stopped. + if (this.webviewRecoveryInFlightEpoch !== this.webviewRecoveryEpoch) { + this.reportWebviewRecoveryFailure("Recovery attempt limit reached without sustained heartbeats") + } return } this.log("[Zoo Code] Webview heartbeat stale while visible; attempting recovery") @@ -3801,7 +3806,7 @@ export class ClineProvider this.webviewRecoveryFailureReported = true this.webviewRecoveryAttempts = ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT this.log(`[Zoo Code] Webview recovery stopped: ${reason}`) - const revision = this.webviewHeartbeatRevision + const heartbeatCount = this.webviewHeartbeatCount void Promise.resolve( vscode.window.showWarningMessage( "Zoo Code's interface is not responding. Reloading the VS Code window will restart extensions and may interrupt running tasks.", @@ -3810,7 +3815,7 @@ export class ClineProvider ), ) .then(async (action) => { - if (action === "Reload Window" && !this._disposed && this.webviewHeartbeatRevision === revision) { + if (action === "Reload Window" && !this._disposed && this.webviewHeartbeatCount === heartbeatCount) { await vscode.commands.executeCommand("workbench.action.reloadWindow") } if (action === "Show Logs") this.outputChannel.show() diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 8ea5fca664..0135cab29f 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1255,11 +1255,16 @@ describe("ClineProvider", () => { expect(assignHtml).toHaveBeenCalledTimes(3) // One mount heartbeat is insufficient: a new healthy window must // span a full minute of periodic heartbeats before replenishing. + const attempts = provider["webviewRecoveryAttempts"] + expect(attempts).toBe(3) provider.updateWebviewHeartbeat() + expect(provider["webviewRecoveryAttempts"]).toBe(attempts) await vi.advanceTimersByTimeAsync(30_000) provider.updateWebviewHeartbeat() + expect(provider["webviewRecoveryAttempts"]).toBe(attempts) await vi.advanceTimersByTimeAsync(30_000) provider.updateWebviewHeartbeat() + expect(provider["webviewRecoveryAttempts"]).toBe(0) await vi.advanceTimersByTimeAsync(120_000) expect(assignHtml).toHaveBeenCalledTimes(4) }) @@ -1404,6 +1409,44 @@ describe("ClineProvider", () => { expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") }) + test("keeps the Reload Window action available after visibility grace resets", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) + let choose: (value: string) => void = () => { + throw new Error("Notification not shown") + } + const choice = new Promise((resolve) => { + choose = resolve + }) + vi.mocked(vscode.window.showWarningMessage).mockImplementation(vi.fn().mockReturnValue(choice)) + await vi.advanceTimersByTimeAsync(130_000) + + visibilityCallback() + choose("Reload Window") + await vi.advanceTimersByTimeAsync(0) + + expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.reloadWindow") + }) + + test("waits for the final HTML attempt to settle before reporting the limit", async () => { + await provider.resolveWebviewView(mockWebviewView) + provider["webviewRecoveryAttempts"] = 2 + let rejectHtml: (error: Error) => void = () => { + throw new Error("HTML not requested") + } + const html = new Promise((_resolve, reject) => { + rejectHtml = reject + }) + provider["getWebviewHtml"] = vi.fn<(webview: vscode.Webview) => Promise>().mockReturnValue(html) + await vi.advanceTimersByTimeAsync(240_000) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + + rejectHtml(new Error("cannot generate")) + await vi.advanceTimersByTimeAsync(60_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + test("bounds failed HTML generation and reports the exhausted budget once", async () => { await provider.resolveWebviewView(mockWebviewView) const generate = vi.spyOn(provider, "getState").mockRejectedValue(new Error("cannot generate")) @@ -2068,6 +2111,30 @@ describe("ClineProvider", () => { } as unknown as vscode.WebviewPanel }) + test("disposes the current tab provider and its watchdog registration", async () => { + const onDispose: Array<() => unknown> = [] + const registration = { dispose: vi.fn() } + vi.mocked(mockWebviewPanel.onDidDispose).mockImplementation((listener, _thisArg, disposables) => { + onDispose.push(listener) + const disposable = onDispose.length === 1 ? registration : { dispose: vi.fn() } + disposables?.push(disposable) + return disposable + }) + await provider.resolveWebviewView(mockWebviewPanel) + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + const dispose = vi.spyOn(provider, "dispose") + const html = mockWebviewPanel.webview.html + + await onDispose[0]() + + expect(dispose).toHaveBeenCalledTimes(1) + expect(provider["_disposed"]).toBe(true) + expect(provider["webviewWatchdogInterval"]).toBeNull() + expect(registration.dispose).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(600_000) + expect(mockWebviewPanel.webview.html).toBe(html) + }) + test.each([false, true])( "replaces the tab host without disposing its provider or task (interrupted: %s)", async (interrupted) => { From dea0067bcaf07539b3f02063611ab515b67dffd5 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 29 Oct 2026 23:30:28 +0000 Subject: [PATCH 26/34] fix(webview): keep recovery warnings and probe results retryable --- apps/vscode-e2e/src/visual/sceneController.ts | 20 +++++++++---- src/core/webview/ClineProvider.ts | 5 +++- .../webview/__tests__/ClineProvider.spec.ts | 29 ++++++++++++++++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/apps/vscode-e2e/src/visual/sceneController.ts b/apps/vscode-e2e/src/visual/sceneController.ts index c01c1e60f3..d8857ae043 100644 --- a/apps/vscode-e2e/src/visual/sceneController.ts +++ b/apps/vscode-e2e/src/visual/sceneController.ts @@ -159,14 +159,22 @@ export async function run(): Promise { } catch { return } - let fixture: WebviewThemeFixture + const resultPath = `${request}.result` + const temporaryResultPath = `${resultPath}.tmp` try { - fixture = await api.captureWebviewThemeFixture() + const fixture = await api.captureWebviewThemeFixture() + await fs.writeFile( + temporaryResultPath, + JSON.stringify({ fixture, taskIds: api.getCurrentTaskStack() }), + "utf8", + ) + await fs.rename(temporaryResultPath, resultPath) + // Acknowledge only after publishing a complete result. Repeating this + // write after a failed acknowledgement is safe for this read-only probe. + await fs.unlink(request) } catch (error) { - await log(`theme probe will retry: ${String(error)}`) - return + await fs.rm(temporaryResultPath, { force: true }).catch(() => undefined) + await log(`theme probe will retry: ${String(error)}`).catch(() => undefined) } - await fs.unlink(request) - await fs.writeFile(`${request}.result`, JSON.stringify({ fixture, taskIds: api.getCurrentTaskStack() }), "utf8") }) } diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index f84d4688b9..bcd04b03da 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1638,7 +1638,7 @@ export class ClineProvider // Check if local dev server is running. try { - await axios.get(`http://${localServerUrl}`) + await axios.get(`http://${localServerUrl}`, { timeout: 5_000 }) } catch { vscode.window.showErrorMessage(t("common:errors.hmr_not_running")) return this.getHtmlContent(webview) @@ -3572,6 +3572,9 @@ export class ClineProvider } this.lastReceivedWebviewHeartbeatAt = now this.webviewHeartbeatCount++ + // A new heartbeat invalidates the old warning action. Allow a fresh warning + // if this renderer stalls again, without replenishing its retry budget. + this.webviewRecoveryFailureReported = false if (this.webviewRecoveryConfirmation) { this.log("[Zoo Code] Webview heartbeat returned after recovery") } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 0135cab29f..69676fd757 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1409,6 +1409,33 @@ describe("ClineProvider", () => { expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") }) + test("offers a fresh actionable warning when one heartbeat is followed by another stall", async () => { + await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) + let chooseOld: (value: string) => void = () => { + throw new Error("Notification not shown") + } + const oldChoice = new Promise((resolve) => { + chooseOld = resolve + }) + vi.mocked(vscode.window.showWarningMessage).mockImplementationOnce(vi.fn().mockReturnValue(oldChoice)) + await vi.advanceTimersByTimeAsync(130_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + + provider.updateWebviewHeartbeat() + expect(provider["webviewRecoveryAttempts"]).toBe(3) + chooseOld("Reload Window") + await vi.advanceTimersByTimeAsync(0) + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") + + vi.mocked(vscode.window.showWarningMessage).mockImplementation(vi.fn().mockResolvedValue("Reload Window")) + await vi.advanceTimersByTimeAsync(170_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(2) + expect(vscode.commands.executeCommand).toHaveBeenCalledWith("workbench.action.reloadWindow") + await vi.advanceTimersByTimeAsync(240_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(2) + }) + test("keeps the Reload Window action available after visibility grace resets", async () => { await provider.resolveWebviewView(mockWebviewView) vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) @@ -2304,7 +2331,7 @@ describe("ClineProvider", () => { // Pin the health-check URL itself: the mock resolves for any URL, so only // this assertion keeps the probe from silently drifting back to // `localhost` (the IPv6 resolution failure this branch fixes). - expect(axios.get).toHaveBeenCalledWith("http://127.0.0.1:5173") + expect(axios.get).toHaveBeenCalledWith("http://127.0.0.1:5173", { timeout: 5_000 }) const html = mockWebviewView.webview.html // The dev server URL must be baked into the module script tag and CSP directives. From 437abced0b0b300cd21436bbdd3469307b2628c5 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 30 Oct 2026 02:27:44 +0000 Subject: [PATCH 27/34] fix(webview): distinguish HMR probe timeouts from unavailable dev server --- src/core/webview/ClineProvider.ts | 7 +++++-- .../webview/__tests__/ClineProvider.spec.ts | 18 +++++++++++++++++- src/i18n/locales/ca/common.json | 1 + src/i18n/locales/de/common.json | 1 + src/i18n/locales/en/common.json | 1 + src/i18n/locales/es/common.json | 1 + src/i18n/locales/fr/common.json | 1 + src/i18n/locales/hi/common.json | 1 + src/i18n/locales/id/common.json | 1 + src/i18n/locales/it/common.json | 1 + src/i18n/locales/ja/common.json | 1 + src/i18n/locales/ko/common.json | 1 + src/i18n/locales/nl/common.json | 1 + src/i18n/locales/pl/common.json | 1 + src/i18n/locales/pt-BR/common.json | 1 + src/i18n/locales/ru/common.json | 1 + src/i18n/locales/tr/common.json | 1 + src/i18n/locales/vi/common.json | 1 + src/i18n/locales/zh-CN/common.json | 1 + src/i18n/locales/zh-TW/common.json | 1 + 20 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index bcd04b03da..7df8ba678d 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1639,8 +1639,11 @@ export class ClineProvider // Check if local dev server is running. try { await axios.get(`http://${localServerUrl}`, { timeout: 5_000 }) - } catch { - vscode.window.showErrorMessage(t("common:errors.hmr_not_running")) + } catch (error) { + const timedOut = axios.isAxiosError(error) && (error.code === "ECONNABORTED" || error.code === "ETIMEDOUT") + void vscode.window.showErrorMessage( + t(timedOut ? "common:errors.hmr_probe_timeout" : "common:errors.hmr_not_running"), + ) return this.getHtmlContent(webview) } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 69676fd757..7202423a05 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -69,8 +69,9 @@ vi.mock("fs/promises", async (importOriginal) => { } }) -vi.mock("axios", () => ({ +vi.mock("axios", async (importOriginal) => ({ default: { + isAxiosError: (await importOriginal()).default.isAxiosError, get: vi.fn().mockResolvedValue({ data: { data: [] } }), post: vi.fn(), }, @@ -2300,6 +2301,21 @@ describe("ClineProvider", () => { expect(scriptSrcMatch![0]).toContain("'wasm-unsafe-eval'") }) + test.each([ + ["ECONNABORTED", "errors.hmr_probe_timeout"], + ["ETIMEDOUT", "errors.hmr_probe_timeout"], + ["ECONNREFUSED", "errors.hmr_not_running"], + ])("reports dev-server probe failure %s accurately and falls back to built HTML", async (code, message) => { + const { AxiosError } = await vi.importActual("axios") + vi.mocked(axios.get).mockRejectedValueOnce(new AxiosError("probe failed", code)) + + const html = await provider["getHMRHtmlContent"](mockWebviewView.webview) + + expect(vscode.window.showErrorMessage).toHaveBeenCalledWith(message) + expect(html).toContain("") + expect(html).not.toContain("/@react-refresh") + }) + test("resolveWebviewView builds HMR content against the local dev server when it is reachable", async () => { const originalThemeFixtureProbe = process.env.ROO_CODE_THEME_FIXTURE_PROBE delete process.env.ROO_CODE_THEME_FIXTURE_PROBE diff --git a/src/i18n/locales/ca/common.json b/src/i18n/locales/ca/common.json index 579cd6b7d7..59130e8889 100644 --- a/src/i18n/locales/ca/common.json +++ b/src/i18n/locales/ca/common.json @@ -55,6 +55,7 @@ "list_api_config": "Ha fallat l'obtenció de la llista de configuracions de l'API", "update_server_timeout": "Ha fallat l'actualització del temps d'espera del servidor", "hmr_not_running": "El servidor de desenvolupament local no està executant-se, l'HMR no funcionarà. Si us plau, executa 'npm run dev' abans de llançar l'extensió per habilitar l'HMR.", + "hmr_probe_timeout": "El servidor de desenvolupament local no ha respost en 5 segons. Es farà servir la interfície compilada de l’extensió sense HMR.", "retrieve_current_mode": "Error en recuperar el mode actual de l'estat.", "failed_delete_repo": "Ha fallat l'eliminació del repositori o branca associada: {{error}}", "failed_remove_directory": "Ha fallat l'eliminació del directori de tasques: {{error}}", diff --git a/src/i18n/locales/de/common.json b/src/i18n/locales/de/common.json index 3d8175a77e..3b74e83e2b 100644 --- a/src/i18n/locales/de/common.json +++ b/src/i18n/locales/de/common.json @@ -51,6 +51,7 @@ "list_api_config": "Fehler beim Abrufen der API-Konfigurationsliste", "update_server_timeout": "Fehler beim Aktualisieren des Server-Timeouts", "hmr_not_running": "Der lokale Entwicklungsserver läuft nicht, HMR wird nicht funktionieren. Bitte führen Sie 'npm run dev' vor dem Start der Erweiterung aus, um HMR zu aktivieren.", + "hmr_probe_timeout": "Der lokale Entwicklungsserver hat nicht innerhalb von 5 Sekunden geantwortet. Die erstellte Erweiterungsoberfläche wird ohne HMR verwendet.", "retrieve_current_mode": "Fehler beim Abrufen des aktuellen Modus aus dem Zustand.", "failed_delete_repo": "Fehler beim Löschen des zugehörigen Shadow-Repositorys oder -Zweigs: {{error}}", "failed_remove_directory": "Fehler beim Entfernen des Aufgabenverzeichnisses: {{error}}", diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index a7d616028f..3ec79a5a60 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -51,6 +51,7 @@ "list_api_config": "Failed to get list api configuration", "update_server_timeout": "Failed to update server timeout", "hmr_not_running": "Local development server is not running, HMR will not work. Please run 'npm run dev' before launching the extension to enable HMR.", + "hmr_probe_timeout": "The local development server did not respond within 5 seconds. Using the built extension UI without HMR.", "retrieve_current_mode": "Error: failed to retrieve current mode from state.", "failed_delete_repo": "Failed to delete associated shadow repository or branch: {{error}}", "failed_remove_directory": "Failed to remove task directory: {{error}}", diff --git a/src/i18n/locales/es/common.json b/src/i18n/locales/es/common.json index 2eb6848f11..fc621129c0 100644 --- a/src/i18n/locales/es/common.json +++ b/src/i18n/locales/es/common.json @@ -51,6 +51,7 @@ "list_api_config": "Error al obtener la lista de configuraciones de API", "update_server_timeout": "Error al actualizar el tiempo de espera del servidor", "hmr_not_running": "El servidor de desarrollo local no está en ejecución, HMR no funcionará. Por favor, ejecuta 'npm run dev' antes de lanzar la extensión para habilitar HMR.", + "hmr_probe_timeout": "El servidor de desarrollo local no respondió en 5 segundos. Se usará la interfaz compilada de la extensión sin HMR.", "retrieve_current_mode": "Error al recuperar el modo actual del estado.", "failed_delete_repo": "Error al eliminar el repositorio o rama asociada: {{error}}", "failed_remove_directory": "Error al eliminar el directorio de tareas: {{error}}", diff --git a/src/i18n/locales/fr/common.json b/src/i18n/locales/fr/common.json index b6b65a88de..65bbd3535f 100644 --- a/src/i18n/locales/fr/common.json +++ b/src/i18n/locales/fr/common.json @@ -51,6 +51,7 @@ "list_api_config": "Erreur lors de l'obtention de la liste des configurations API", "update_server_timeout": "Erreur lors de la mise à jour du délai d'attente du serveur", "hmr_not_running": "Le serveur de développement local n'est pas en cours d'exécution, HMR ne fonctionnera pas. Veuillez exécuter 'npm run dev' avant de lancer l'extension pour activer l'HMR.", + "hmr_probe_timeout": "Le serveur de développement local n’a pas répondu dans les 5 secondes. L’interface compilée de l’extension sera utilisée sans HMR.", "retrieve_current_mode": "Erreur lors de la récupération du mode actuel à partir du state.", "failed_delete_repo": "Échec de la suppression du repo fantôme ou de la branche associée : {{error}}", "failed_remove_directory": "Échec de la suppression du répertoire de tâches : {{error}}", diff --git a/src/i18n/locales/hi/common.json b/src/i18n/locales/hi/common.json index f69b5adccf..56318890ca 100644 --- a/src/i18n/locales/hi/common.json +++ b/src/i18n/locales/hi/common.json @@ -51,6 +51,7 @@ "list_api_config": "API कॉन्फ़िगरेशन की सूची प्राप्त करने में विफल", "update_server_timeout": "सर्वर टाइमआउट अपडेट करने में विफल", "hmr_not_running": "स्थानीय विकास सर्वर चल नहीं रहा है, HMR काम नहीं करेगा। कृपया HMR सक्षम करने के लिए एक्सटेंशन लॉन्च करने से पहले 'npm run dev' चलाएँ।", + "hmr_probe_timeout": "स्थानीय डेवलपमेंट सर्वर ने 5 सेकंड में जवाब नहीं दिया। HMR के बिना एक्सटेंशन के बिल्ड किए गए UI का उपयोग किया जा रहा है।", "retrieve_current_mode": "स्टेट से वर्तमान मोड प्राप्त करने में त्रुटि।", "failed_delete_repo": "संबंधित शैडो रिपॉजिटरी या ब्रांच हटाने में विफल: {{error}}", "failed_remove_directory": "टास्क डायरेक्टरी हटाने में विफल: {{error}}", diff --git a/src/i18n/locales/id/common.json b/src/i18n/locales/id/common.json index e401612bf3..339fdc9e6e 100644 --- a/src/i18n/locales/id/common.json +++ b/src/i18n/locales/id/common.json @@ -51,6 +51,7 @@ "list_api_config": "Gagal mendapatkan daftar konfigurasi api", "update_server_timeout": "Gagal memperbarui timeout server", "hmr_not_running": "Server pengembangan lokal tidak berjalan, HMR tidak akan bekerja. Silakan jalankan 'npm run dev' sebelum meluncurkan ekstensi untuk mengaktifkan HMR.", + "hmr_probe_timeout": "Server pengembangan lokal tidak merespons dalam 5 detik. Menggunakan UI ekstensi yang telah dibangun tanpa HMR.", "retrieve_current_mode": "Error: gagal mengambil mode saat ini dari state.", "failed_delete_repo": "Gagal menghapus shadow repository atau branch yang terkait: {{error}}", "failed_remove_directory": "Gagal menghapus direktori tugas: {{error}}", diff --git a/src/i18n/locales/it/common.json b/src/i18n/locales/it/common.json index aeb984940a..20b8d01f68 100644 --- a/src/i18n/locales/it/common.json +++ b/src/i18n/locales/it/common.json @@ -51,6 +51,7 @@ "list_api_config": "Errore durante l'ottenimento dell'elenco delle configurazioni API", "update_server_timeout": "Errore durante l'aggiornamento del timeout del server", "hmr_not_running": "Il server di sviluppo locale non è in esecuzione, l'HMR non funzionerà. Esegui 'npm run dev' prima di avviare l'estensione per abilitare l'HMR.", + "hmr_probe_timeout": "Il server di sviluppo locale non ha risposto entro 5 secondi. Verrà usata l’interfaccia compilata dell’estensione senza HMR.", "retrieve_current_mode": "Errore durante il recupero della modalità corrente dallo stato.", "failed_delete_repo": "Impossibile eliminare il repository o il ramo associato: {{error}}", "failed_remove_directory": "Impossibile rimuovere la directory delle attività: {{error}}", diff --git a/src/i18n/locales/ja/common.json b/src/i18n/locales/ja/common.json index a7ec0c4de3..9752ccc544 100644 --- a/src/i18n/locales/ja/common.json +++ b/src/i18n/locales/ja/common.json @@ -51,6 +51,7 @@ "list_api_config": "API設定リストの取得に失敗しました", "update_server_timeout": "サーバータイムアウトの更新に失敗しました", "hmr_not_running": "ローカル開発サーバーが実行されていないため、HMRは機能しません。HMRを有効にするには、拡張機能を起動する前に'npm run dev'を実行してください。", + "hmr_probe_timeout": "ローカル開発サーバーが5秒以内に応答しませんでした。HMRなしでビルド済みの拡張機能UIを使用します。", "retrieve_current_mode": "現在のモードを状態から取得する際にエラーが発生しました。", "failed_delete_repo": "関連するシャドウリポジトリまたはブランチの削除に失敗しました：{{error}}", "failed_remove_directory": "タスクディレクトリの削除に失敗しました：{{error}}", diff --git a/src/i18n/locales/ko/common.json b/src/i18n/locales/ko/common.json index 906652ac14..7865deff42 100644 --- a/src/i18n/locales/ko/common.json +++ b/src/i18n/locales/ko/common.json @@ -51,6 +51,7 @@ "list_api_config": "API 구성 목록 가져오기에 실패했습니다", "update_server_timeout": "서버 타임아웃 업데이트에 실패했습니다", "hmr_not_running": "로컬 개발 서버가 실행되고 있지 않아 HMR이 작동하지 않습니다. HMR을 활성화하려면 확장 프로그램을 실행하기 전에 'npm run dev'를 실행하세요.", + "hmr_probe_timeout": "로컬 개발 서버가 5초 이내에 응답하지 않았습니다. HMR 없이 빌드된 확장 프로그램 UI를 사용합니다.", "retrieve_current_mode": "상태에서 현재 모드를 검색하는 데 오류가 발생했습니다.", "failed_delete_repo": "관련 shadow 저장소 또는 브랜치 삭제 실패: {{error}}", "failed_remove_directory": "작업 디렉토리 제거 실패: {{error}}", diff --git a/src/i18n/locales/nl/common.json b/src/i18n/locales/nl/common.json index 34cdbbbe89..5c30be24ca 100644 --- a/src/i18n/locales/nl/common.json +++ b/src/i18n/locales/nl/common.json @@ -51,6 +51,7 @@ "list_api_config": "Ophalen van lijst met API-configuraties mislukt", "update_server_timeout": "Bijwerken van server-timeout mislukt", "hmr_not_running": "Lokale ontwikkelserver draait niet, HMR werkt niet. Voer 'npm run dev' uit voordat je de extensie start om HMR in te schakelen.", + "hmr_probe_timeout": "De lokale ontwikkelserver heeft niet binnen 5 seconden gereageerd. De gebouwde extensie-interface wordt zonder HMR gebruikt.", "retrieve_current_mode": "Fout: ophalen van huidige modus uit status mislukt.", "failed_delete_repo": "Verwijderen van gekoppelde schaduwrepository of branch mislukt: {{error}}", "failed_remove_directory": "Verwijderen van taakmap mislukt: {{error}}", diff --git a/src/i18n/locales/pl/common.json b/src/i18n/locales/pl/common.json index 2a743fc009..b0e17e243b 100644 --- a/src/i18n/locales/pl/common.json +++ b/src/i18n/locales/pl/common.json @@ -51,6 +51,7 @@ "list_api_config": "Nie udało się pobrać listy konfiguracji API", "update_server_timeout": "Nie udało się zaktualizować limitu czasu serwera", "hmr_not_running": "Lokalny serwer deweloperski nie jest uruchomiony, HMR nie będzie działać. Uruchom 'npm run dev' przed uruchomieniem rozszerzenia, aby włączyć HMR.", + "hmr_probe_timeout": "Lokalny serwer deweloperski nie odpowiedział w ciągu 5 sekund. Używany jest zbudowany interfejs rozszerzenia bez HMR.", "retrieve_current_mode": "Błąd podczas pobierania bieżącego trybu ze stanu.", "failed_delete_repo": "Nie udało się usunąć powiązanego repozytorium lub gałęzi pomocniczej: {{error}}", "failed_remove_directory": "Nie udało się usunąć katalogu zadania: {{error}}", diff --git a/src/i18n/locales/pt-BR/common.json b/src/i18n/locales/pt-BR/common.json index eaf1cae06b..4a17aa731c 100644 --- a/src/i18n/locales/pt-BR/common.json +++ b/src/i18n/locales/pt-BR/common.json @@ -55,6 +55,7 @@ "list_api_config": "Falha ao obter a lista de configurações da API", "update_server_timeout": "Falha ao atualizar o tempo limite do servidor", "hmr_not_running": "O servidor de desenvolvimento local não está em execução, o HMR não funcionará. Por favor, execute 'npm run dev' antes de iniciar a extensão para habilitar o HMR.", + "hmr_probe_timeout": "O servidor de desenvolvimento local não respondeu em 5 segundos. Usando a interface compilada da extensão sem HMR.", "retrieve_current_mode": "Erro ao recuperar o modo atual do estado.", "failed_delete_repo": "Falha ao excluir o repositório ou ramificação associada: {{error}}", "failed_remove_directory": "Falha ao remover o diretório de tarefas: {{error}}", diff --git a/src/i18n/locales/ru/common.json b/src/i18n/locales/ru/common.json index 6f48365ec1..aa9db995e2 100644 --- a/src/i18n/locales/ru/common.json +++ b/src/i18n/locales/ru/common.json @@ -51,6 +51,7 @@ "list_api_config": "Не удалось получить список конфигураций API", "update_server_timeout": "Не удалось обновить таймаут сервера", "hmr_not_running": "Локальный сервер разработки не запущен, HMR не будет работать. Пожалуйста, запустите 'npm run dev' перед запуском расширения для включения HMR.", + "hmr_probe_timeout": "Локальный сервер разработки не ответил в течение 5 секунд. Используется собранный интерфейс расширения без HMR.", "retrieve_current_mode": "Ошибка: не удалось получить текущий режим из состояния.", "failed_delete_repo": "Не удалось удалить связанный теневой репозиторий или ветку: {{error}}", "failed_remove_directory": "Не удалось удалить директорию задачи: {{error}}", diff --git a/src/i18n/locales/tr/common.json b/src/i18n/locales/tr/common.json index b5c543c405..ab812e810e 100644 --- a/src/i18n/locales/tr/common.json +++ b/src/i18n/locales/tr/common.json @@ -51,6 +51,7 @@ "list_api_config": "API yapılandırma listesi alınamadı", "update_server_timeout": "Sunucu zaman aşımı güncellenemedi", "hmr_not_running": "Yerel geliştirme sunucusu çalışmıyor, HMR çalışmayacak. HMR'yi etkinleştirmek için uzantıyı başlatmadan önce lütfen 'npm run dev' komutunu çalıştırın.", + "hmr_probe_timeout": "Yerel geliştirme sunucusu 5 saniye içinde yanıt vermedi. Derlenmiş eklenti arayüzü HMR olmadan kullanılıyor.", "retrieve_current_mode": "Mevcut mod durumdan alınırken hata oluştu.", "failed_delete_repo": "İlişkili gölge depo veya dal silinemedi: {{error}}", "failed_remove_directory": "Görev dizini kaldırılamadı: {{error}}", diff --git a/src/i18n/locales/vi/common.json b/src/i18n/locales/vi/common.json index cb2ed8e9b8..829fbb6f28 100644 --- a/src/i18n/locales/vi/common.json +++ b/src/i18n/locales/vi/common.json @@ -51,6 +51,7 @@ "list_api_config": "Không thể lấy danh sách cấu hình API", "update_server_timeout": "Không thể cập nhật thời gian chờ máy chủ", "hmr_not_running": "Máy chủ phát triển cục bộ không chạy, HMR sẽ không hoạt động. Vui lòng chạy 'npm run dev' trước khi khởi chạy tiện ích mở rộng để bật HMR.", + "hmr_probe_timeout": "Máy chủ phát triển cục bộ không phản hồi trong 5 giây. Sử dụng giao diện tiện ích mở rộng đã dựng mà không có HMR.", "retrieve_current_mode": "Lỗi không thể truy xuất chế độ hiện tại từ trạng thái.", "failed_delete_repo": "Không thể xóa kho lưu trữ hoặc nhánh liên quan: {{error}}", "failed_remove_directory": "Không thể xóa thư mục nhiệm vụ: {{error}}", diff --git a/src/i18n/locales/zh-CN/common.json b/src/i18n/locales/zh-CN/common.json index 08cfe736b5..c28726e6fa 100644 --- a/src/i18n/locales/zh-CN/common.json +++ b/src/i18n/locales/zh-CN/common.json @@ -56,6 +56,7 @@ "list_api_config": "获取API配置列表失败", "update_server_timeout": "更新服务器超时设置失败", "hmr_not_running": "本地开发服务器未运行，HMR将不起作用。请在启动扩展前运行'npm run dev'以启用HMR。", + "hmr_probe_timeout": "本地开发服务器未在 5 秒内响应。将使用已构建的扩展界面，不启用 HMR。", "retrieve_current_mode": "从状态中检索当前模式失败。", "failed_delete_repo": "删除关联的影子仓库或分支失败：{{error}}", "failed_remove_directory": "删除任务目录失败：{{error}}", diff --git a/src/i18n/locales/zh-TW/common.json b/src/i18n/locales/zh-TW/common.json index 92fdbaeb96..de535c391a 100644 --- a/src/i18n/locales/zh-TW/common.json +++ b/src/i18n/locales/zh-TW/common.json @@ -51,6 +51,7 @@ "list_api_config": "取得 API 設定列表失敗", "update_server_timeout": "更新伺服器超時設定失敗", "hmr_not_running": "本機開發伺服器沒有執行，HMR 將不起作用。請在啟動擴充套件前執行'npm run dev'以啟用 HMR。", + "hmr_probe_timeout": "本機開發伺服器未在 5 秒內回應。將使用已建置的擴充套件介面，不啟用 HMR。", "retrieve_current_mode": "從狀態中檢索目前模式失敗。", "failed_delete_repo": "刪除關聯的影子倉庫或分支失敗：{{error}}", "failed_remove_directory": "刪除工作目錄失敗：{{error}}", From 190f6c15178876170d4020108c6146127e2b97d7 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 03:28:37 +0000 Subject: [PATCH 28/34] fix(webview): preserve recovery state for delayed sidebar replacements --- .github/workflows/visual-regression.yml | 3 +- .../src/visual/heartbeat.electron.ts | 5 +- apps/vscode-e2e/src/visual/heartbeat.md | 4 +- src/core/webview/ClineProvider.ts | 17 +- .../webview/__tests__/ClineProvider.spec.ts | 148 +++++++++++++++++- 5 files changed, 167 insertions(+), 10 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 93a27931e4..52cbc3a9e3 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -85,7 +85,8 @@ jobs: extension-host-visual: runs-on: ubuntu-latest - timeout-minutes: 20 + # Includes dependency setup, builds, repeated visuals, and production watchdog waits. + timeout-minutes: 35 container: image: mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e options: --ipc=host diff --git a/apps/vscode-e2e/src/visual/heartbeat.electron.ts b/apps/vscode-e2e/src/visual/heartbeat.electron.ts index de503453db..2fa61ebfc6 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.electron.ts +++ b/apps/vscode-e2e/src/visual/heartbeat.electron.ts @@ -155,7 +155,8 @@ test("recovers after the isolated Zoo Code renderer is terminated", async ({ bro expect(processes.webview).not.toBe(processes.workbench) process.kill(processes.webview, "SIGKILL") - // Use production timers: the next stale tick is at most two minutes away. + // Allow stale detection (up to 150s), confirmation and replacement waits, + // plus webview loading. A passing fast run does not bound the slow path. await expect .poll( async () => { @@ -171,7 +172,7 @@ test("recovers after the isolated Zoo Code renderer is terminated", async ({ bro return "unavailable" } }, - { timeout: 150_000, intervals: [1_000] }, + { timeout: 210_000, intervals: [1_000] }, ) .toBe("recovered") await waitForContent(running.page, CHAT_RESULT) diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index d8bb924e0f..38af215fc3 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -15,8 +15,8 @@ The scenarios use isolated VS Code instances and the production 60-second watchd The message-port interceptor follows the pinned VS Code implementation. Check it when upgrading VS Code. It does not modify the extension or fake extension-host timers. Heartbeat blocking and renderer termination are separate failure modes; passing the former does not establish crash recovery. -At PR head `a985f03ba`, terminating the isolated renderer reproduces a gray sidebar: the watchdog logs recovery, but assigning fresh HTML does not restore the UI within 150 seconds. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. +A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. -A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. Window reload is never automatic. A late action from an old notification is ignored after liveness returns. +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. Only sustained healthy heartbeats replenish that budget. Window reload is never automatic. A late action from an old notification is ignored after liveness returns. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 7df8ba678d..f4039dd110 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -250,6 +250,9 @@ export class ClineProvider private webviewRecoveryAttempts = 0 private webviewRecoveryConfirmation: ReturnType | undefined private webviewHostRecoveryInProgress = false + // VS Code may resolve the focused sidebar after our bounded wait expires. + // Keep that repair distinct from a user-opened view until it arrives. + private pendingSidebarRecovery = false private webviewRecoveryFailureReported = false private lastReceivedWebviewHeartbeatAt: number | undefined private webviewHeartbeatCount = 0 @@ -864,7 +867,7 @@ export class ClineProvider this.webviewRecoveryEpoch++ // User-opened replacements get a fresh budget; recreating a failed host // must preserve its incident budget so crash loops remain bounded. - if (!this.webviewHostRecoveryInProgress) { + if (!this.webviewHostRecoveryInProgress && !this.pendingSidebarRecovery) { this.webviewRecoveryAttempts = 0 this.webviewRecoveryFailureReported = false } @@ -1073,12 +1076,16 @@ export class ClineProvider } async resolveWebviewView(webviewView: vscode.WebviewView | vscode.WebviewPanel) { + // Capture intent before yielding: host recovery can time out while this + // resolve is still generating HTML or hydrating state. + const recoveringHost = this.webviewHostRecoveryInProgress || this.pendingSidebarRecovery // A different view replaces the current one: dispose its subscriptions so // the replaced view can no longer dispatch messages or refresh provider state. if (this.view !== webviewView) { this.clearWebviewResources() } + this.pendingSidebarRecovery = false this.view = webviewView const inTabMode = "onDidChangeViewState" in webviewView @@ -1243,7 +1250,7 @@ export class ClineProvider const currentTask = this.getCurrentTask() // Host recreation is a UI repair, including when the displayed task is // interrupted. It must not evict or resume that task as a side effect. - if (!this.webviewHostRecoveryInProgress && (!currentTask || currentTask.abandoned || currentTask.abort)) { + if (!recoveringHost && (!currentTask || currentTask.abandoned || currentTask.abort)) { await this.removeClineFromStack() } @@ -3737,6 +3744,7 @@ export class ClineProvider if (this.webviewHostRecoveryInProgress || this._disposed || this.view !== view || !view.visible) return const revision = this.webviewHeartbeatRevision let started = false + let sidebarRemoved = false try { let commands: string[] | undefined if (!("onDidChangeViewState" in view)) { @@ -3777,7 +3785,9 @@ export class ClineProvider else replacement.dispose() await resolved } else { + this.pendingSidebarRecovery = true await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.removeView`) + sidebarRemoved = true if (this._disposed) return await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.focus`, { preserveFocus: true }) await pWaitFor( @@ -3791,6 +3801,9 @@ export class ClineProvider ) } } catch (error) { + // If removal failed, no replacement is expected. A removed sidebar + // can still resolve after the wait expires and must retain its task. + if (started && !sidebarRemoved) this.pendingSidebarRecovery = false // A late discovery failure must not stop a recovered/replaced view. // Once recreation starts, disposal of the old view is expected. if ( diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 7202423a05..4d048e3a3e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -7,6 +7,7 @@ import { TaskRegistry } from "../../task/TaskRegistry" import Anthropic from "@anthropic-ai/sdk" import * as vscode from "vscode" import axios from "axios" +import pWaitFor from "p-wait-for" import { type ProviderSettingsEntry, @@ -1294,6 +1295,7 @@ describe("ClineProvider", () => { preserveFocus: true, }) expect(provider["view"]).toBe(replacement) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) expect(provider["webviewRecoveryAttempts"]).toBe(1) expect(provider.getCurrentTask()).toBe(task) expect(evict).not.toHaveBeenCalled() @@ -1303,6 +1305,140 @@ describe("ClineProvider", () => { }, ) + describe("sidebar replacement timing", () => { + beforeEach(async () => { + const actual = await vi.importActual("p-wait-for") + vi.mocked(pWaitFor).mockImplementation(actual.default) + }) + + afterEach(() => { + vi.mocked(pWaitFor).mockReset().mockResolvedValue(undefined) + }) + + function removeSidebarOnCommand() { + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + }) + } + + test("warns once after ten seconds without a replacement and releases recovery ownership", async () => { + await provider.resolveWebviewView(mockWebviewView) + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(130_000) + expect(provider["webviewHostRecoveryInProgress"]).toBe(true) + await vi.advanceTimersByTimeAsync(9_999) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(120_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + + test("settles the replacement wait when the provider is disposed", async () => { + await provider.resolveWebviewView(mockWebviewView) + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(130_000) + expect(provider["webviewHostRecoveryInProgress"]).toBe(true) + await provider.dispose() + await vi.advanceTimersByTimeAsync(50) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + }) + + test.each([5_000, 11_000])( + "retains an interrupted task and budget when replacement starts after %s ms", + async (delay) => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(130_000 + delay) + const attempts = delay < 10_000 ? 1 : 3 + expect(provider["webviewRecoveryAttempts"]).toBe(attempts) + expect(provider["webviewHostRecoveryInProgress"]).toBe(delay < 10_000) + + const replacement = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + await provider.resolveWebviewView(replacement) + await vi.advanceTimersByTimeAsync(50) + + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider["pendingSidebarRecovery"]).toBe(false) + expect(provider["webviewRecoveryAttempts"]).toBe(attempts) + expect(provider["webviewRecoveryFailureReported"]).toBe(delay >= 10_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(delay < 10_000 ? 0 : 1) + expect(provider.getCurrentTask()).toBe(task) + expect(remove).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + }, + ) + + test("retains recovery intent while replacement HTML generation outlasts the wait", async () => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(130_000) + let finishHtml: (html: string) => void = () => { + throw new Error("HTML generation has not started") + } + provider["getWebviewHtml"] = vi.fn( + () => + new Promise((resolve) => { + finishHtml = resolve + }), + ) + const replacement = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + const resolving = provider.resolveWebviewView(replacement) + await vi.advanceTimersByTimeAsync(10_000) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + finishHtml("replacement-html") + await resolving + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(provider.getCurrentTask()).toBe(task) + expect(remove).not.toHaveBeenCalled() + }) + }) + + test("recreates the sidebar again after a second incident", async () => { + await provider.resolveWebviewView(mockWebviewView) + const replacements = [0, 1].map(() => ({ + ...mockWebviewView, + webview: { ...mockWebviewView.webview, html: "" }, + })) + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) { + const next = replacements.shift() + if (!next) throw new Error("Unexpected third replacement") + await provider.resolveWebviewView(next) + } + }) + await vi.advanceTimersByTimeAsync(130_000) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + provider.updateWebviewHeartbeat() + await vi.advanceTimersByTimeAsync(30_000) + provider.updateWebviewHeartbeat() + await vi.advanceTimersByTimeAsync(30_000) + provider.updateWebviewHeartbeat() + expect(provider["webviewRecoveryAttempts"]).toBe(0) + await vi.advanceTimersByTimeAsync(130_000) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(replacements).toHaveLength(0) + expect( + vi + .mocked(vscode.commands.executeCommand) + .mock.calls.filter(([command]) => command === `${ClineProvider.sideBarId}.removeView`), + ).toHaveLength(2) + }) + test.each(["heartbeat", "hidden", "disposed", "suspended"] as const)( "does not recreate the host after %s during the confirmation window", async (event) => { @@ -1365,6 +1501,8 @@ describe("ClineProvider", () => { await vi.advanceTimersByTimeAsync(600_000) expect(assignHtml).toHaveBeenCalledTimes(1) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider["pendingSidebarRecovery"]).toBe(false) expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( @@ -1501,12 +1639,14 @@ describe("ClineProvider", () => { expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) }) - test("gives a replacement view a fresh recovery budget after exhausting attempts", async () => { + test("gives a user-opened replacement a fresh budget after unavailable host recovery", async () => { await provider.resolveWebviewView(mockWebviewView) + vi.mocked(vscode.commands.getCommands).mockResolvedValue([]) const generateHtml = vi.spyOn(provider, "getState") - await vi.advanceTimersByTimeAsync(480_000) - expect(generateHtml).toHaveBeenCalledTimes(3) + await vi.advanceTimersByTimeAsync(130_000) + expect(generateHtml).toHaveBeenCalledTimes(1) + expect(provider["webviewRecoveryAttempts"]).toBe(3) disposeCallback() const replacementView = { @@ -2194,6 +2334,7 @@ describe("ClineProvider", () => { expect(mockWebviewPanel.dispose).toHaveBeenCalledTimes(1) expect(provider["view"]).toBe(replacement) expect(provider["_disposed"]).toBe(false) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) expect(provider["webviewRecoveryAttempts"]).toBe(1) expect(provider.getCurrentTask()).toBe(task) expect(evict).not.toHaveBeenCalled() @@ -2218,6 +2359,7 @@ describe("ClineProvider", () => { expect(mockWebviewPanel.dispose).not.toHaveBeenCalled() expect(replacement.dispose).toHaveBeenCalledTimes(1) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) expect(provider["view"]).toBe(mockWebviewPanel) expect(provider.getCurrentTask()).toBe(task) expect(task.dispose).not.toHaveBeenCalled() From cefc4ec8845e2c41abded1b4bab275a3b6f439f2 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 04:53:17 +0000 Subject: [PATCH 29/34] fix(webview): retry failed tab initialization and clear failed sidebar recovery --- apps/vscode-e2e/src/visual/heartbeat.md | 2 +- src/core/webview/ClineProvider.ts | 23 ++- .../webview/__tests__/ClineProvider.spec.ts | 144 ++++++++++++++++++ 3 files changed, 162 insertions(+), 7 deletions(-) diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index 38af215fc3..a541176efa 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -17,6 +17,6 @@ The message-port interceptor follows the pinned VS Code implementation. Check it A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. -A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. Only sustained healthy heartbeats replenish that budget. Window reload is never automatic. A late action from an old notification is ignored after liveness returns. +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A tab that fails initialization after publication keeps its task and a watchdog for bounded retries. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late action from an old notification is ignored after liveness returns. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index f4039dd110..f1d8538885 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -3744,7 +3744,7 @@ export class ClineProvider if (this.webviewHostRecoveryInProgress || this._disposed || this.view !== view || !view.visible) return const revision = this.webviewHeartbeatRevision let started = false - let sidebarRemoved = false + let sidebarFocused = false try { let commands: string[] | undefined if (!("onDidChangeViewState" in view)) { @@ -3783,13 +3783,24 @@ export class ClineProvider const resolved = this.resolveWebviewView(replacement) if (this.view === replacement) view.dispose() else replacement.dispose() - await resolved + try { + await resolved + } catch (error) { + // A published panel already owns this provider and its task. + // Keep its disposal subscription, and restore the watchdog even + // if HTML generation or state hydration failed before installation. + // The next stale tick uses the existing bounded recovery budget. + if (this.view === view) throw error + if (this._disposed || this.view !== replacement) return + this.log(`[Zoo Code] Failed to initialize replacement tab: ${String(error)}`) + this.startWebviewWatchdog() + } } else { this.pendingSidebarRecovery = true await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.removeView`) - sidebarRemoved = true if (this._disposed) return await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.focus`, { preserveFocus: true }) + sidebarFocused = true await pWaitFor( () => this._disposed || @@ -3801,9 +3812,9 @@ export class ClineProvider ) } } catch (error) { - // If removal failed, no replacement is expected. A removed sidebar - // can still resolve after the wait expires and must retain its task. - if (started && !sidebarRemoved) this.pendingSidebarRecovery = false + // Only a successful focus can leave a delayed replacement pending. + // Command failures must not mark a later manual open as recovery. + if (started && !sidebarFocused) this.pendingSidebarRecovery = false // A late discovery failure must not stop a recovered/replaced view. // Once recreation starts, disposal of the old view is expected. if ( diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 4d048e3a3e..67a91b103e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1511,6 +1511,36 @@ describe("ClineProvider", () => { }, ) + test("clears a failed focus incident before a manual sidebar open", async () => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) throw new Error("focus failed") + }) + + await vi.advanceTimersByTimeAsync(130_000) + + expect(provider["view"]).toBeUndefined() + expect(provider["pendingSidebarRecovery"]).toBe(false) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(remove).not.toHaveBeenCalled() + + const manualView = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + await provider.resolveWebviewView(manualView) + + expect(provider["view"]).toBe(manualView) + expect(provider["webviewRecoveryAttempts"]).toBe(0) + expect(provider["webviewRecoveryFailureReported"]).toBe(false) + expect(remove).toHaveBeenCalledTimes(1) + expect(provider.getCurrentTask()).toBeUndefined() + }) + test.each(["Reload Window", "Show Logs"] as const)( "runs only the selected recovery action: %s", async (action) => { @@ -2366,6 +2396,120 @@ describe("ClineProvider", () => { expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) }) + function replacementPanel() { + return { + ...mockWebviewPanel, + webview: { + ...mockWebviewPanel.webview, + html: "", + onDidReceiveMessage: vi.fn(() => ({ dispose: vi.fn() })), + }, + onDidDispose: vi.fn(() => ({ dispose: vi.fn() })), + dispose: vi.fn(), + } + } + + test.each(["HTML generation", "state hydration"] as const)( + "retries a published tab after failed %s without disposing its interrupted task", + async (stage) => { + await provider.resolveWebviewView(mockWebviewPanel) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("recovered-html") + await provider["reloadWebviewForRecovery"]() + const failed = replacementPanel() + const healthy = replacementPanel() + vi.mocked(vscode.window.createWebviewPanel).mockReturnValueOnce(failed).mockReturnValue(healthy) + if (stage === "HTML generation") { + vi.mocked(provider["getWebviewHtml"]).mockRejectedValueOnce(new Error("HTML failed")) + } else { + vi.spyOn(provider, "getState").mockRejectedValueOnce(new Error("hydration failed")) + } + await vi.advanceTimersByTimeAsync(10_000) + + expect(provider["view"]).toBe(failed) + expect(failed.webview.onDidReceiveMessage).not.toHaveBeenCalled() + expect(failed.dispose).not.toHaveBeenCalled() + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( + expect.stringContaining("Failed to initialize replacement tab"), + ) + + await vi.advanceTimersByTimeAsync(130_000) + + expect(vscode.window.createWebviewPanel).toHaveBeenCalledTimes(2) + expect(provider["view"]).toBe(healthy) + // The provider and focus tracker each install one message listener. + expect(healthy.webview.onDidReceiveMessage).toHaveBeenCalledTimes(2) + expect(failed.dispose).toHaveBeenCalledTimes(1) + expect(provider["webviewRecoveryAttempts"]).toBe(2) + expect(provider.getCurrentTask()).toBe(task) + expect(provider["_disposed"]).toBe(false) + expect(remove).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + }, + ) + + test.each(["disposed", "replaced"] as const)( + "does not restart a stale replacement watchdog after the provider is %s", + async (event) => { + await provider.resolveWebviewView(mockWebviewPanel) + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("recovered-html") + await provider["reloadWebviewForRecovery"]() + let rejectHtml: (error: Error) => void = () => { + throw new Error("Replacement HTML generation has not started") + } + vi.mocked(provider["getWebviewHtml"]).mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectHtml = reject + }), + ) + const failed = replacementPanel() + vi.mocked(vscode.window.createWebviewPanel).mockReturnValue(failed) + await vi.advanceTimersByTimeAsync(10_000) + expect(provider["view"]).toBe(failed) + if (event === "disposed") await provider.dispose() + else await provider.resolveWebviewView(replacementPanel()) + const watchdog = provider["webviewWatchdogInterval"] + rejectHtml(new Error("stale HTML failure")) + await vi.advanceTimersByTimeAsync(0) + expect(provider["webviewWatchdogInterval"]).toBe(watchdog) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + }, + ) + + test("bounds repeated published-tab initialization failures and keeps the task available", async () => { + await provider.resolveWebviewView(mockWebviewPanel) + const task = new Task(defaultTaskOptions) + provider["taskRegistry"].push(task) + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("recovered-html") + vi.mocked(vscode.window.createWebviewPanel).mockImplementation(() => { + const replacement = replacementPanel() + // Fail after publication, before any message listener is installed. + vi.spyOn(provider, "getState").mockRejectedValueOnce(new Error("hydration failed")) + return replacement + }) + + await vi.advanceTimersByTimeAsync(720_000) + + expect(vscode.window.createWebviewPanel).toHaveBeenCalledTimes(3) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(provider["_disposed"]).toBe(false) + expect(provider.getCurrentTask()).toBe(task) + expect(task.dispose).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + }) + test("reloads the tab webview when the heartbeat is stale and the tab is visible", async () => { await provider.resolveWebviewView(mockWebviewPanel) const htmlAfterResolve = mockWebviewPanel.webview.html From ec42ed12ce289511c13d2085fd7edb67f2eff590 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 14:53:01 +0000 Subject: [PATCH 30/34] fix(webview): warn on tab initialization failures without stopping retries --- apps/vscode-e2e/src/visual/heartbeat.md | 2 +- src/core/webview/ClineProvider.ts | 14 +++++++----- .../webview/__tests__/ClineProvider.spec.ts | 22 ++++++++++++++++++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index a541176efa..e59685c9f7 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -17,6 +17,6 @@ The message-port interceptor follows the pinned VS Code implementation. Check it A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. -A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A tab that fails initialization after publication keeps its task and a watchdog for bounded retries. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late action from an old notification is ignored after liveness returns. +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index f1d8538885..aab68be4b8 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -3792,7 +3792,8 @@ export class ClineProvider // The next stale tick uses the existing bounded recovery budget. if (this.view === view) throw error if (this._disposed || this.view !== replacement) return - this.log(`[Zoo Code] Failed to initialize replacement tab: ${String(error)}`) + // Surface the unusable tab without exhausting its remaining retries. + this.reportWebviewRecoveryFailure(`Failed to initialize replacement tab: ${String(error)}`, false) this.startWebviewWatchdog() } } else { @@ -3831,11 +3832,14 @@ export class ClineProvider } } - private reportWebviewRecoveryFailure(reason: string): void { - if (this._disposed || this.webviewRecoveryFailureReported) return + private reportWebviewRecoveryFailure(reason: string, stopRecovery = true): void { + if (this._disposed) return + // A terminal failure still stops retries if an earlier retryable failure + // already showed the notification for this incident. + if (stopRecovery) this.webviewRecoveryAttempts = ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT + if (this.webviewRecoveryFailureReported) return this.webviewRecoveryFailureReported = true - this.webviewRecoveryAttempts = ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT - this.log(`[Zoo Code] Webview recovery stopped: ${reason}`) + this.log(`[Zoo Code] Webview recovery ${stopRecovery ? "stopped" : "will retry"}: ${reason}`) const heartbeatCount = this.webviewHeartbeatCount void Promise.resolve( vscode.window.showWarningMessage( diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 67a91b103e..b08f5cc8ef 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1541,6 +1541,21 @@ describe("ClineProvider", () => { expect(provider.getCurrentTask()).toBeUndefined() }) + test("stops retries after a terminal failure even when a retry warning was already shown", async () => { + await provider.resolveWebviewView(mockWebviewView) + const html = mockWebviewView.webview.html + provider["reportWebviewRecoveryFailure"]("Initialization failed; retry pending", false) + expect(provider["webviewRecoveryAttempts"]).toBe(0) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + + provider["reportWebviewRecoveryFailure"]("Recovery is unavailable") + await vi.advanceTimersByTimeAsync(120_000) + + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(mockWebviewView.webview.html).toBe(html) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + test.each(["Reload Window", "Show Logs"] as const)( "runs only the selected recovery action: %s", async (action) => { @@ -2435,7 +2450,12 @@ describe("ClineProvider", () => { expect(provider["webviewWatchdogInterval"]).not.toBeNull() expect(provider["webviewHostRecoveryInProgress"]).toBe(false) expect(provider["webviewRecoveryAttempts"]).toBe(1) - expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(vscode.window.showWarningMessage).toHaveBeenCalledWith( + expect.stringContaining("interface is not responding"), + "Reload Window", + "Show Logs", + ) expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( expect.stringContaining("Failed to initialize replacement tab"), ) From 18dc5c21e9bf985fba2f6b4d76749ab41c733143 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 19:05:33 +0000 Subject: [PATCH 31/34] fix(webview): restore bounded recovery for failed sidebar initialization --- apps/vscode-e2e/src/visual/heartbeat.md | 2 +- src/core/webview/ClineProvider.ts | 37 +++-- .../webview/__tests__/ClineProvider.spec.ts | 131 +++++++++++++++++- 3 files changed, 154 insertions(+), 16 deletions(-) diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index e59685c9f7..61ccc3ef73 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -17,6 +17,6 @@ The message-port interceptor follows the pinned VS Code implementation. Check it A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. -A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index aab68be4b8..ad04884b74 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1079,6 +1079,24 @@ export class ClineProvider // Capture intent before yielding: host recovery can time out while this // resolve is still generating HTML or hydrating state. const recoveringHost = this.webviewHostRecoveryInProgress || this.pendingSidebarRecovery + try { + await this.initializeWebviewView(webviewView, recoveringHost) + } catch (error) { + if (!recoveringHost) throw error + if (this.isStaleViewResolve(webviewView)) return + // VS Code invokes sidebar resolution itself, outside the focus command's + // promise. Handle both host types here so failed initialization retains + // its task, disposal subscription, and remaining recovery attempts. + const host = "onDidChangeViewState" in webviewView ? "tab" : "sidebar" + this.reportWebviewRecoveryFailure(`Failed to initialize replacement ${host}: ${String(error)}`, false) + this.startWebviewWatchdog() + } + } + + private async initializeWebviewView( + webviewView: vscode.WebviewView | vscode.WebviewPanel, + recoveringHost: boolean, + ): Promise { // A different view replaces the current one: dispose its subscriptions so // the replaced view can no longer dispatch messages or refresh provider state. if (this.view !== webviewView) { @@ -3783,19 +3801,7 @@ export class ClineProvider const resolved = this.resolveWebviewView(replacement) if (this.view === replacement) view.dispose() else replacement.dispose() - try { - await resolved - } catch (error) { - // A published panel already owns this provider and its task. - // Keep its disposal subscription, and restore the watchdog even - // if HTML generation or state hydration failed before installation. - // The next stale tick uses the existing bounded recovery budget. - if (this.view === view) throw error - if (this._disposed || this.view !== replacement) return - // Surface the unusable tab without exhausting its remaining retries. - this.reportWebviewRecoveryFailure(`Failed to initialize replacement tab: ${String(error)}`, false) - this.startWebviewWatchdog() - } + await resolved } else { this.pendingSidebarRecovery = true await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.removeView`) @@ -3824,8 +3830,13 @@ export class ClineProvider ) return if (started && this.lastReceivedWebviewHeartbeatAt !== undefined) return + // A published replacement may still be initializing after the wait + // expires. Its resolver will restore the watchdog on failure; keep the + // remaining attempts available instead of turning this into a dead end. + const replacementPublished = this.view !== undefined && this.view !== view this.reportWebviewRecoveryFailure( `Host recreation failed: ${error instanceof Error ? error.message : String(error)}`, + !replacementPublished, ) } finally { if (started) this.webviewHostRecoveryInProgress = false diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index b08f5cc8ef..fa0db8d1f2 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1305,6 +1305,22 @@ describe("ClineProvider", () => { }, ) + test.each(["HTML generation", "state hydration"] as const)( + "preserves normal sidebar startup rejection for failed %s outside recovery", + async (stage) => { + const error = new Error("initial view failed") + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("initial-html") + if (stage === "HTML generation") vi.mocked(provider["getWebviewHtml"]).mockRejectedValueOnce(error) + else vi.spyOn(provider, "getState").mockRejectedValueOnce(error) + + await expect(provider.resolveWebviewView(mockWebviewView)).rejects.toBe(error) + + expect(provider["webviewWatchdogInterval"]).toBeNull() + expect(provider["webviewRecoveryAttempts"]).toBe(0) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + }, + ) + describe("sidebar replacement timing", () => { beforeEach(async () => { const actual = await vi.importActual("p-wait-for") @@ -1377,7 +1393,118 @@ describe("ClineProvider", () => { }, ) - test("retains recovery intent while replacement HTML generation outlasts the wait", async () => { + test.each(["HTML generation", "state hydration"] as const)( + "recovers an interrupted sidebar task after failed replacement %s", + async (stage) => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("recovered-html") + const failed = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + const healthy = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + const replacements = [failed, healthy] + let resolution: Promise | undefined + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) { + const next = replacements.shift() + if (!next) throw new Error("Unexpected third replacement") + if (next === failed) { + if (stage === "HTML generation") + vi.mocked(provider["getWebviewHtml"]).mockRejectedValueOnce( + new Error("HTML failed"), + ) + else vi.spyOn(provider, "getState").mockRejectedValueOnce(new Error("hydration failed")) + } + // VS Code resolves the view independently of its focus command. + resolution = provider.resolveWebviewView(next) + } + }) + + await vi.advanceTimersByTimeAsync(130_050) + await expect(resolution).resolves.toBeUndefined() + + expect(provider["view"]).toBe(failed) + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(provider.getCurrentTask()).toBe(task) + expect(remove).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(130_000) + await expect(resolution).resolves.toBeUndefined() + + expect(provider["view"]).toBe(healthy) + expect(healthy.webview.html).toBe("recovered-html") + expect(provider["webviewRecoveryAttempts"]).toBe(2) + expect(replacements).toHaveLength(0) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(provider.getCurrentTask()).toBe(task) + expect(remove).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + }, + ) + + test("caps repeated failed sidebar replacements without evicting the task", async () => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + provider["taskRegistry"].push(task) + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("recovered-html") + const resolutions: Promise[] = [] + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) { + vi.spyOn(provider, "getState").mockRejectedValueOnce(new Error("hydration failed")) + const replacement = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + resolutions.push(provider.resolveWebviewView(replacement)) + } + }) + + await vi.advanceTimersByTimeAsync(720_000) + await Promise.all(resolutions) + + expect(resolutions).toHaveLength(3) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect(provider.getCurrentTask()).toBe(task) + expect(task.abortTask).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + }) + + test("keeps retries available when a published sidebar rejects after the replacement wait", async () => { + await provider.resolveWebviewView(mockWebviewView) + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(130_000) + let rejectHtml: (error: Error) => void = () => { + throw new Error("HTML generation has not started") + } + provider["getWebviewHtml"] = vi.fn( + () => + new Promise((_resolve, reject) => { + rejectHtml = reject + }), + ) + const replacement = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + const resolution = provider.resolveWebviewView(replacement) + await vi.advanceTimersByTimeAsync(10_000) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + + rejectHtml(new Error("late HTML failure")) + await expect(resolution).resolves.toBeUndefined() + + expect(provider["view"]).toBe(replacement) + expect(provider["webviewWatchdogInterval"]).not.toBeNull() + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }) + + test("retains recovery intent and remaining attempts while replacement HTML generation outlasts the wait", async () => { await provider.resolveWebviewView(mockWebviewView) const task = new Task(defaultTaskOptions) task.abort = true @@ -1400,7 +1527,7 @@ describe("ClineProvider", () => { expect(provider["webviewHostRecoveryInProgress"]).toBe(false) finishHtml("replacement-html") await resolving - expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(provider["webviewRecoveryAttempts"]).toBe(1) expect(provider.getCurrentTask()).toBe(task) expect(remove).not.toHaveBeenCalled() }) From 82c2a473e0a25d479dd7c497d87c40808f881582 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 19:16:23 +0000 Subject: [PATCH 32/34] test(webview): add bounded model checking for recovery protocols --- .github/workflows/code-qa.yml | 2 + apps/vscode-e2e/src/visual/heartbeat.md | 2 + docs/architecture/webview-recovery-model.md | 71 +++ package.json | 2 + scripts/check-webview-recovery.ts | 490 ++++++++++++++++++++ scripts/tsconfig.webview-recovery.json | 15 + src/core/webview/ClineProvider.ts | 109 +++-- src/core/webview/webviewRecoveryPolicy.ts | 77 +++ 8 files changed, 728 insertions(+), 40 deletions(-) create mode 100644 docs/architecture/webview-recovery-model.md create mode 100644 scripts/check-webview-recovery.ts create mode 100644 scripts/tsconfig.webview-recovery.json create mode 100644 src/core/webview/webviewRecoveryPolicy.ts diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index f03ee6b408..f5a67ac478 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -110,6 +110,8 @@ jobs: run: pnpm agent-evals:check-types - name: Model-check task lifecycle protocols run: pnpm lifecycle:model-check + - name: Model-check webview recovery + run: pnpm webview-recovery:check-types && pnpm webview-recovery:model-check - name: Validate MCP OAuth integration run: pnpm mcp:integration-check diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index 61ccc3ef73..164e5724d2 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -20,3 +20,5 @@ A gray sidebar can persist after renderer termination even when the watchdog ass A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. + +The [bounded recovery model](../../../../docs/architecture/webview-recovery-model.md) checks production recovery policies and abstract interleavings. Run `pnpm webview-recovery:model-check` alongside the provider and Electron tests. It does not prove VS Code effects or recovery liveness. diff --git a/docs/architecture/webview-recovery-model.md b/docs/architecture/webview-recovery-model.md new file mode 100644 index 0000000000..d4f6d12836 --- /dev/null +++ b/docs/architecture/webview-recovery-model.md @@ -0,0 +1,71 @@ +# Webview recovery model check + +Run the bounded checker and its type check with: + +```sh +pnpm webview-recovery:check-types +pnpm webview-recovery:model-check +``` + +Code QA runs both commands in its compile job. This is an independent webview protocol check, separate from the persisted task lifecycle models. It adds no task-status transitions. + +## Production connection + +`src/core/webview/webviewRecoveryPolicy.ts` owns pure decisions used by both `ClineProvider` and the checker. These include attempt admission, recovery-incident preservation, resolve-failure classification, HTML commit eligibility, release of an async recovery owner, command-failure marker cleanup, failure accounting and notification deduplication, heartbeat health, and warning actions. + +The provider still performs VS Code effects. It installs subscriptions, starts timers, assigns HTML, resolves views, and retains its task stack. The policy extraction does not move those effects into the model. + +The checker combines two kinds of evidence: + +- finite-state exploration calls production policy functions while modeling Promise settlements and host effects; +- exhaustive finite input tables check policy contracts, including both acceptance and rejection paths and the healthy-window/stale-gap boundaries. + +This is production-policy-backed bounded verification, not a refinement proof of the complete provider. A missing production call, a misplaced effect, or incorrect VS Code behavior can escape the abstract model. Focused provider tests and Electron tests remain necessary. + +## State spaces and bounds + +The checker explores three independent state spaces to a fixed point. Each has a 100,000-state budget and a maximum discovery depth of 60. Discovering a new state beyond either bound fails the check; reaching the depth bound does not count as success. The printed state totals are diagnostics, not ratchets. + +**Async ownership** models one pending HTML operation and one possible replacement operation. Visibility can change, a heartbeat or grace reset can invalidate the pending work, the epoch can change, and the provider can be disposed or replaced before the old operation settles. The completion records which condition held at settlement, rather than inferring event order from a later state. Old completion must not release a replacement operation's owner. + +**Sidebar and tab hosts** model recovery attempts, HTML generation success/failure, host replacement, captured recovery intent, initialization success/failure, notification state, watchdog availability, and host-operation ownership. Sidebar actions additionally include removal/focus failures, publication before or after the replacement wait, and a wait expiring during initialization. Repeated failures consume a three-attempt budget. One manual reopen and one sustained-health refill are permitted, so later incidents are reachable without an unbounded counter. + +The graphs are checked separately. They do not establish every possible joint interleaving between ownership, host effects, and task execution. The host graph abstracts identities as original/missing/replacement; the ownership graph checks distinct old/replacement owners. Numeric health contracts check 59,999/60,000 and 90,000/90,001 millisecond boundaries. Timer scheduling itself is an environment assumption, not a model guarantee. + +## Safety contracts and required witnesses + +The checker rejects: + +1. HTML committed to a disposed, replaced, hidden, epoch-invalidated, or heartbeat-invalidated view; +2. an old async completion releasing a newer recovery's ownership; +3. duplicate in-flight recovery or attempts exceeding the cap; +4. a recovery replacement resetting its incident budget or being classified as normal startup; +5. a current recovery resolve failure losing the retry disposition; +6. retryable warnings consuming the remaining attempts, or duplicate notifications without a notification-state reset; +7. a failed sidebar focus leaving a marker that contaminates a later manual reopen; +8. a startup heartbeat replenishing attempts before sustained health; +9. a stale Reload Window action reloading after a heartbeat or disposal; and +10. a late Show Logs action being incorrectly suppressed. + +Required actions and semantic witnesses prevent a vacuous pass. They include a valid HTML commit, each invalidated completion class, a replacement owner surviving old completion, a retryable initialization failure, repeated initialization failures reaching the cap, healthy budget refill, a published sidebar timing out with attempts remaining, late publication retaining an exhausted budget, and a fresh manual open after focus failure. + +Ten representative faulty policies are injected without editing source. Each must fail with its expected contract violation. Exploration truncation or an unrelated exception does not count as rejecting a regression. + +## Effect and integration evidence + +`src/core/webview/__tests__/ClineProvider.spec.ts` checks the corresponding real provider effects with fake timers. Relevant scenarios include: + +- HTML and hydration rejection from VS Code-style independent sidebar resolution; +- successful retry while retaining an interrupted task; +- repeated failed sidebar and tab replacements stopping at the cap; +- late initialization failure after the sidebar replacement wait; +- disposal/replacement before an old resolve completes; +- recovery ownership clearing and a second incident remaining recoverable; +- failed focus followed by a normal manual open; and +- warning actions, heartbeat invalidation, and healthy-window boundaries. + +These are integration checks of the handwritten effect layer, not generated trace conformance tests. The host model assumes that a retryable resolve failure installs a watchdog and that a completed host operation releases its busy flag; the provider tests verify those assumptions. The model's incident-preservation decision is used by the provider's budget reset and task-cleanup guards; the task registry itself is not modeled. + +The Electron heartbeat project checks real sidebar recreation, isolated renderer termination, task-stack preservation, resumed periodic heartbeats, and hidden-sidebar retention. It does not deliberately make a real VS Code resolver reject. No model here proves renderer memory safety, eventual Promise settlement, OS sleep/resume ordering, arbitrary overlapping operations, or recovery liveness under an unfair or permanently failing environment. + +When changing recovery behavior, update the production policy, model actions/contracts and required witnesses, and the narrowest provider test that checks its effects. Rerun Electron checks when a VS Code host or transport boundary changes. diff --git a/package.json b/package.json index 6ec1d9067f..010b9749d8 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", + "webview-recovery:model-check": "node --import tsx scripts/check-webview-recovery.ts", + "webview-recovery:check-types": "tsc -p scripts/tsconfig.webview-recovery.json", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", "cleanup-protocol:model-check": "tsx scripts/check-task-cleanup-protocol.ts", "tool-turn:model-check": "tsx scripts/check-tool-turn-continuation.ts", diff --git a/scripts/check-webview-recovery.ts b/scripts/check-webview-recovery.ts new file mode 100644 index 0000000000..537e1d6eec --- /dev/null +++ b/scripts/check-webview-recovery.ts @@ -0,0 +1,490 @@ +import assert from "node:assert/strict" +import * as policy from "../src/core/webview/webviewRecoveryPolicy" + +// This explores finite protocol abstractions using production decisions. Effects +// (VS Code calls, listener installation, timer delivery) are modeled explicitly; +// ClineProvider.spec.ts and Electron tests verify those effects in production. +type Policy = typeof policy +interface Step { + action: string + state: S +} +interface Node { + state: S + parent: number + action: string + depth: number +} +const MAX_STATES = 100_000 +const MAX_DEPTH = 60 + +function explore( + name: string, + initial: S, + transitions: (state: S) => Step[], + inspect: (state: S) => string[], +): Set { + const queue: Node[] = [{ state: initial, parent: -1, action: "initial", depth: 0 }] + const seen = new Set([JSON.stringify(initial)]) + const reached = new Set() + for (let i = 0; i < queue.length; i++) { + const node = queue[i]! + try { + for (const landmark of inspect(node.state)) reached.add(landmark) + for (const step of transitions(node.state)) { + reached.add(`action:${step.action}`) + const key = JSON.stringify(step.state) + if (seen.has(key)) continue + assert(node.depth < MAX_DEPTH, `${name}: exploration truncated at depth ${MAX_DEPTH}`) + assert(seen.size < MAX_STATES, `${name}: state budget exceeded`) + seen.add(key) + queue.push({ ...step, parent: i, depth: node.depth + 1 }) + } + } catch (error) { + const trace: string[] = [] + let cursor = i + while (cursor >= 0) { + const entry = queue[cursor]! + trace.unshift(`${entry.action}: ${JSON.stringify(entry.state)}`) + cursor = entry.parent + } + throw new Error(`${name}: ${String(error)}\nShortest BFS trace:\n${trace.join("\n")}`) + } + } + console.log(`${name}: ${seen.size} states, fixed point reached, depth <= ${MAX_DEPTH}`) + return reached +} + +interface Ownership { + disposed: boolean + isCurrent: boolean + epochMatches: boolean + heartbeatMatches: boolean + visible: boolean + owner: number | undefined + completed: boolean + completionCase: "pending" | "valid" | "disposed" | "replaced" | "epoch" | "heartbeat" | "hidden" + fault?: string +} + +function ownershipCheck(p: Policy): void { + const reached = explore( + "async ownership", + { + disposed: false, + isCurrent: true, + epochMatches: true, + heartbeatMatches: true, + visible: true, + owner: 0, + completed: false, + completionCase: "pending", + }, + (state) => { + const steps: Step[] = [] + const add = (action: string, update: Partial) => + steps.push({ action, state: { ...state, ...update } }) + if (!state.disposed) add("dispose", { disposed: true }) + if (state.isCurrent) + add("replace-and-start-new-recovery", { isCurrent: false, epochMatches: false, owner: 1 }) + if (state.epochMatches) add("invalidate-epoch", { epochMatches: false }) + if (state.heartbeatMatches) add("heartbeat-or-grace", { heartbeatMatches: false }) + add(state.visible ? "hide" : "show", { visible: !state.visible }) + if (!state.completed) { + const commit = p.canCommitRecovery(state) + const expected = + !state.disposed && state.isCurrent && state.epochMatches && state.heartbeatMatches && state.visible + const owner = p.releaseRecoveryOwner(state.owner, 0) + add("complete-old-html", { + completed: true, + completionCase: state.disposed + ? "disposed" + : !state.isCurrent + ? "replaced" + : !state.epochMatches + ? "epoch" + : !state.heartbeatMatches + ? "heartbeat" + : !state.visible + ? "hidden" + : "valid", + owner, + fault: + commit !== expected + ? "stale, hidden, or disposed HTML commit" + : owner !== (state.owner === 0 ? undefined : state.owner) + ? "old completion released a replacement owner" + : undefined, + }) + } + return steps + }, + (s) => { + assert(!s.fault, s.fault) + const marks: string[] = [] + if (s.completionCase === "valid") marks.push("valid-html-commit") + if (s.completionCase === "epoch") marks.push("epoch-invalidates-html") + if (s.completionCase === "replaced" && s.owner === 1) + marks.push("replacement-owner-survives-old-completion") + if (s.completionCase === "disposed") marks.push("disposed-completion") + if (s.completionCase === "heartbeat") marks.push("heartbeat-invalidates-html") + if (s.completionCase === "hidden") marks.push("hidden-completion") + return marks + }, + ) + for (const mark of [ + "valid-html-commit", + "epoch-invalidates-html", + "replacement-owner-survives-old-completion", + "disposed-completion", + "heartbeat-invalidates-html", + "hidden-completion", + ]) + assert(reached.has(mark), `ownership landmark missing: ${mark}`) +} + +type Phase = "idle" | "html" | "remove" | "focus" | "wait" | "initialize" | "stopped" +interface Host { + phase: Phase + attempts: number + reported: boolean + busy: boolean + pending: boolean + view: "old" | "none" | "replacement" + watchdog: boolean + initialized: boolean + recoveringResolve: boolean + waitExpired: boolean + focusFailed: boolean + manualOpened: boolean + healthStarted: boolean + healthReset: boolean + fault?: string +} + +function hostCheck(kind: "sidebar" | "tab", p: Policy): void { + const initial: Host = { + phase: "idle", + attempts: 0, + reported: false, + busy: false, + pending: false, + view: "old", + watchdog: true, + initialized: true, + recoveringResolve: false, + waitExpired: false, + focusFailed: false, + manualOpened: false, + healthStarted: false, + healthReset: false, + } + const reached = explore( + `${kind} host`, + initial, + (s) => { + const steps: Step[] = [] + const add = (action: string, update: Partial) => steps.push({ action, state: { ...s, ...update } }) + const failure = (stop: boolean): Partial => { + const next = p.recordRecoveryFailure({ attempts: s.attempts, reported: s.reported }, stop) + return { + attempts: next.attempts, + reported: next.reported, + fault: + next.attempts !== (stop ? 3 : s.attempts) || next.notify === s.reported + ? "failure consumed retries or repeated a warning" + : undefined, + } + } + const publish = () => { + const recovering = p.preservesRecoveryIncident(s.busy, s.pending) + add("publish-replacement", { + phase: "initialize", + view: "replacement", + watchdog: false, + initialized: false, + recoveringResolve: recovering, + pending: false, + attempts: recovering ? s.attempts : 0, + fault: !recovering + ? "repair reset its incident budget or could evict the interrupted task" + : undefined, + }) + } + if (s.phase === "idle" && s.watchdog && s.attempts < 3) { + const attempt = p.nextRecoveryAttempt(s.attempts, false) + add("stale-watchdog", { + phase: "html", + attempts: attempt ?? s.attempts, + healthStarted: false, + fault: attempt !== s.attempts + 1 ? "attempt accounting" : undefined, + }) + } + if (s.phase === "html") { + add("duplicate-recovery", { + fault: + p.nextRecoveryAttempt(s.attempts, true) !== undefined + ? "duplicate in-flight recovery" + : undefined, + }) + add("html-failure", { phase: "idle" }) + if (kind === "sidebar") + add("html-success", { phase: "remove", busy: true, pending: true, waitExpired: false }) + else + add("html-success", { + phase: "initialize", + busy: true, + view: "replacement", + watchdog: false, + initialized: false, + recoveringResolve: true, + waitExpired: false, + }) + } + if (s.phase === "remove" || s.phase === "focus") { + const pending = p.pendingSidebarAfterCommandFailure({ + started: true, + focusSucceeded: false, + pending: s.pending, + }) + add(`${s.phase}-failure`, { + ...failure(true), + phase: "stopped", + busy: false, + pending, + focusFailed: s.phase === "focus", + fault: pending ? "command failure left a stale sidebar marker" : undefined, + }) + if (s.phase === "remove") + add("remove-success", { phase: "focus", view: "none", watchdog: false, initialized: false }) + else add("focus-success", { phase: "wait" }) + } + if (s.phase === "wait") { + publish() + if (!s.waitExpired) + add("timeout-before-publication", { ...failure(true), busy: false, waitExpired: true }) + } + if (s.phase === "initialize") { + add("resolve-success", { phase: "idle", busy: false, watchdog: true, initialized: true }) + const action = p.resolutionFailureAction({ + recoveringHost: s.recoveringResolve, + disposed: false, + isCurrent: true, + }) + add("resolve-failure", { + ...failure(false), + phase: "idle", + busy: false, + watchdog: action === "retry", + initialized: false, + fault: action !== "retry" ? "published resolution failure lost recovery" : undefined, + }) + if (kind === "sidebar" && !s.waitExpired) + add("timeout-after-publication", { ...failure(false), busy: false, waitExpired: true }) + } + if (s.phase === "idle" && s.attempts === 3) add("exhaust-budget", { ...failure(true), phase: "stopped" }) + if (s.phase === "stopped" && !s.pending && !s.manualOpened) { + const recovering = p.preservesRecoveryIncident(s.busy, s.pending) + add("manual-open", { + phase: "idle", + view: "old", + attempts: recovering ? s.attempts : 0, + watchdog: true, + initialized: true, + reported: false, + manualOpened: true, + fault: recovering ? "manual open inherited a failed incident" : undefined, + }) + } + if ((s.phase === "idle" || s.phase === "stopped") && s.watchdog && !s.healthReset) { + if (!s.healthStarted) { + const health = p.recoveryHealth( + { attempts: s.attempts, lastHeartbeatAt: undefined, healthySince: undefined }, + 0, + ) + add("startup-heartbeat", { + healthStarted: true, + reported: false, + attempts: health.attempts, + fault: health.attempts !== s.attempts ? "one heartbeat refilled the budget" : undefined, + }) + } else { + const health = p.recoveryHealth( + { attempts: s.attempts, lastHeartbeatAt: 0, healthySince: 0 }, + 60_000, + ) + add("sustained-health", { + phase: "idle", + healthReset: true, + attempts: health.attempts, + reported: false, + fault: health.attempts !== 0 ? "healthy window failed to replenish attempts" : undefined, + }) + } + } + return steps + }, + (s) => { + assert(!s.fault, s.fault) + assert(Number.isInteger(s.attempts) && s.attempts >= 0 && s.attempts <= 3, "attempt budget") + if (s.phase === "idle") + assert(s.watchdog && !s.busy, "settled recovery lost watchdog or retained ownership") + const marks: string[] = [] + if (s.phase === "idle" && !s.initialized && s.attempts === 1) marks.push("initialization-failure-retryable") + if (s.phase === "stopped" && s.attempts === 3 && !s.initialized && s.view === "replacement") + marks.push("failed-resolutions-bounded") + if (s.phase === "initialize" && s.waitExpired && s.attempts === 1) + marks.push("published-timeout-keeps-attempts") + if (s.phase === "initialize" && s.waitExpired && s.attempts === 3 && s.recoveringResolve) + marks.push("late-publication-preserves-cap") + if (s.manualOpened && s.focusFailed && s.attempts === 0) marks.push("manual-open-after-focus-failure") + if (s.healthReset && s.attempts === 0) marks.push("healthy-budget-refill") + return marks + }, + ) + const required = ["initialization-failure-retryable", "failed-resolutions-bounded", "healthy-budget-refill"] + if (kind === "sidebar") + required.push( + "published-timeout-keeps-attempts", + "late-publication-preserves-cap", + "manual-open-after-focus-failure", + ) + for (const mark of required) assert(reached.has(mark), `${kind} landmark missing: ${mark}`) + for (const action of [ + "stale-watchdog", + "duplicate-recovery", + "html-failure", + "resolve-success", + "resolve-failure", + "exhaust-budget", + "sustained-health", + ]) + assert(reached.has(`action:${action}`), `${kind} action unreachable: ${action}`) +} + +function policyContracts(p: Policy): void { + for (const choice of [undefined, "Reload Window", "Show Logs", "dismiss"]) + for (const disposed of [false, true]) + for (const sameHeartbeat of [false, true]) { + assert.equal( + p.recoveryWarningAction(choice, disposed, sameHeartbeat), + choice === "Show Logs" + ? "logs" + : choice === "Reload Window" && !disposed && sameHeartbeat + ? "reload" + : "none", + "notification action contract", + ) + } + for (const recoveringHost of [false, true]) + for (const disposed of [false, true]) + for (const isCurrent of [false, true]) { + assert.equal( + p.resolutionFailureAction({ recoveringHost, disposed, isCurrent }), + !recoveringHost ? "propagate" : disposed || !isCurrent ? "ignore" : "retry", + "resolve failure ownership", + ) + } + for (let attempts = 0; attempts <= 3; attempts++) { + for (const inFlight of [false, true]) + assert.equal( + p.nextRecoveryAttempt(attempts, inFlight), + inFlight || attempts === 3 ? undefined : attempts + 1, + "attempt cap and ownership", + ) + for (const lastHeartbeatAt of [undefined, 0]) + for (const healthySince of [undefined, 0]) { + for (const now of [0, 59_999, 60_000, 90_000, 90_001]) { + const actual = p.recoveryHealth({ attempts, lastHeartbeatAt, healthySince }, now) + const since = lastHeartbeatAt === undefined || now > 90_000 ? now : healthySince + assert.deepEqual( + actual, + { attempts: since !== undefined && now - since >= 60_000 ? 0 : attempts, healthySince: since }, + "health boundary", + ) + } + } + } + for (let attempts = 0; attempts <= 3; attempts++) + for (const reported of [false, true]) + for (const stop of [false, true]) { + assert.deepEqual( + p.recordRecoveryFailure({ attempts, reported }, stop), + { attempts: stop ? 3 : attempts, reported: true, notify: !reported }, + "failure accounting and notification deduplication", + ) + } + for (const started of [false, true]) + for (const focusSucceeded of [false, true]) + for (const pending of [false, true]) { + assert.equal( + p.pendingSidebarAfterCommandFailure({ started, focusSucceeded, pending }), + started && !focusSucceeded ? false : pending, + "sidebar command failure marker", + ) + } +} + +function check(p: Policy): void { + policyContracts(p) + ownershipCheck(p) + hostCheck("sidebar", p) + hostCheck("tab", p) +} + +check(policy) +// Representative policy regressions must be rejected, rather than merely +// leaving a self-consistent abstract model green. These do not mutate source. +const mutants: Array<[string, Policy, RegExp]> = [ + [ + "stale notification reloads window", + { ...policy, recoveryWarningAction: (choice) => (choice === "Reload Window" ? "reload" : "none") }, + /notification action contract/, + ], + [ + "late Show Logs is suppressed", + { + ...policy, + recoveryWarningAction: (choice, disposed, sameHeartbeat) => + sameHeartbeat ? policy.recoveryWarningAction(choice, disposed, sameHeartbeat) : "none", + }, + /notification action contract/, + ], + ["stale HTML commit", { ...policy, canCommitRecovery: () => true }, /stale, hidden, or disposed HTML commit/], + [ + "old completion clears new owner", + { ...policy, releaseRecoveryOwner: () => undefined }, + /old completion released a replacement owner/, + ], + [ + "late repair resets budget", + { ...policy, preservesRecoveryIncident: () => false }, + /repair reset its incident budget/, + ], + [ + "failed resolution loses retry", + { ...policy, resolutionFailureAction: () => "propagate" }, + /resolve failure ownership/, + ], + [ + "retry warning exhausts budget", + { ...policy, recordRecoveryFailure: (state) => policy.recordRecoveryFailure(state, true) }, + /failure accounting/, + ], + [ + "failed focus retains marker", + { ...policy, pendingSidebarAfterCommandFailure: (state) => state.pending }, + /sidebar command failure marker/, + ], + [ + "startup heartbeat refills budget", + { ...policy, recoveryHealth: (state, now) => ({ ...policy.recoveryHealth(state, now), attempts: 0 }) }, + /health boundary/, + ], + ["unbounded attempts", { ...policy, nextRecoveryAttempt: (attempts) => attempts + 1 }, /attempt cap and ownership/], +] +for (const [name, mutant, expectedViolation] of mutants) { + assert.throws(() => check(mutant), expectedViolation, `checker missed regression: ${name}`) + console.log(`Rejected regression: ${name}`) +} +console.log(`Webview recovery model check passed; ${mutants.length} representative regressions rejected.`) diff --git a/scripts/tsconfig.webview-recovery.json b/scripts/tsconfig.webview-recovery.json new file mode 100644 index 0000000000..776dceaf80 --- /dev/null +++ b/scripts/tsconfig.webview-recovery.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "isolatedModules": true, + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./check-webview-recovery.ts", "../src/core/webview/webviewRecoveryPolicy.ts"] +} diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index ad04884b74..bc6d2f6985 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -10,6 +10,18 @@ import axios from "axios" import debounce from "lodash.debounce" import pWaitFor from "p-wait-for" import * as vscode from "vscode" +import { + WEBVIEW_RECOVERY_LIMITS, + preservesRecoveryIncident, + resolutionFailureAction, + canCommitRecovery, + nextRecoveryAttempt, + releaseRecoveryOwner, + pendingSidebarAfterCommandFailure, + recordRecoveryFailure, + recoveryHealth, + recoveryWarningAction, +} from "./webviewRecoveryPolicy" import { type TaskProviderLike, @@ -258,8 +270,7 @@ export class ClineProvider private webviewHeartbeatCount = 0 private webviewHealthySince: number | undefined private static readonly WEBVIEW_RECOVERY_CONFIRM_MS = 10_000 - private static readonly WEBVIEW_HEALTHY_WINDOW_MS = 60_000 - private static readonly WEBVIEW_RECOVERY_ATTEMPT_LIMIT = 3 + private static readonly WEBVIEW_RECOVERY_ATTEMPT_LIMIT = WEBVIEW_RECOVERY_LIMITS.attempts private webviewWatchdogInterval: ReturnType | null = null // Bumped to invalidate an in-flight recovery reload when the provider or // the watched view is disposed; the reload must not reassign webview.html @@ -275,7 +286,7 @@ export class ClineProvider // instead of counting the suspended interval against a live renderer. private lastWebviewWatchdogTickAt = 0 private static readonly WEBVIEW_WATCHDOG_TICK_MS = 60_000 - private static readonly WEBVIEW_HEARTBEAT_STALE_MS = 90_000 + private static readonly WEBVIEW_HEARTBEAT_STALE_MS = WEBVIEW_RECOVERY_LIMITS.staleMs private readonly _postStateToWebviewThrottled = debounce( async () => { try { @@ -867,7 +878,7 @@ export class ClineProvider this.webviewRecoveryEpoch++ // User-opened replacements get a fresh budget; recreating a failed host // must preserve its incident budget so crash loops remain bounded. - if (!this.webviewHostRecoveryInProgress && !this.pendingSidebarRecovery) { + if (!preservesRecoveryIncident(this.webviewHostRecoveryInProgress, this.pendingSidebarRecovery)) { this.webviewRecoveryAttempts = 0 this.webviewRecoveryFailureReported = false } @@ -1078,12 +1089,20 @@ export class ClineProvider async resolveWebviewView(webviewView: vscode.WebviewView | vscode.WebviewPanel) { // Capture intent before yielding: host recovery can time out while this // resolve is still generating HTML or hydrating state. - const recoveringHost = this.webviewHostRecoveryInProgress || this.pendingSidebarRecovery + const recoveringHost = preservesRecoveryIncident( + this.webviewHostRecoveryInProgress, + this.pendingSidebarRecovery, + ) try { await this.initializeWebviewView(webviewView, recoveringHost) } catch (error) { - if (!recoveringHost) throw error - if (this.isStaleViewResolve(webviewView)) return + const action = resolutionFailureAction({ + recoveringHost, + disposed: this._disposed, + isCurrent: this.view === webviewView, + }) + if (action === "propagate") throw error + if (action === "ignore") return // VS Code invokes sidebar resolution itself, outside the focus command's // promise. Handle both host types here so failed initialization retains // its task, disposal subscription, and remaining recovery attempts. @@ -3592,12 +3611,15 @@ export class ClineProvider /** A heartbeat confirms liveness; only sustained health replenishes recovery attempts. */ public updateWebviewHeartbeat(): void { const now = Date.now() - if ( - this.lastReceivedWebviewHeartbeatAt === undefined || - now - this.lastReceivedWebviewHeartbeatAt > ClineProvider.WEBVIEW_HEARTBEAT_STALE_MS - ) { - this.webviewHealthySince = now - } + const health = recoveryHealth( + { + attempts: this.webviewRecoveryAttempts, + lastHeartbeatAt: this.lastReceivedWebviewHeartbeatAt, + healthySince: this.webviewHealthySince, + }, + now, + ) + this.webviewHealthySince = health.healthySince this.lastReceivedWebviewHeartbeatAt = now this.webviewHeartbeatCount++ // A new heartbeat invalidates the old warning action. Allow a fresh warning @@ -3607,15 +3629,10 @@ export class ClineProvider this.log("[Zoo Code] Webview heartbeat returned after recovery") } this.resetWebviewHeartbeatGrace() - if ( - this.webviewHealthySince !== undefined && - now - this.webviewHealthySince >= ClineProvider.WEBVIEW_HEALTHY_WINDOW_MS - ) { - if (this.webviewRecoveryAttempts > 0) - this.log("[Zoo Code] Webview recovery confirmed by sustained heartbeats") - this.webviewRecoveryAttempts = 0 - this.webviewRecoveryFailureReported = false + if (health.attempts === 0 && this.webviewRecoveryAttempts > 0) { + this.log("[Zoo Code] Webview recovery confirmed by sustained heartbeats") } + this.webviewRecoveryAttempts = health.attempts } /** Restarts the grace window and invalidates pending recovery without recording a heartbeat. */ @@ -3688,12 +3705,11 @@ export class ClineProvider // second one would only pile another forced reload onto it. Ownership // is scoped to the epoch so a stale recovery (its view disposed or // replaced, epoch bumped) never blocks the replacement view's recovery. - if (this.webviewRecoveryInFlightEpoch === epoch) { - return - } + const attempt = nextRecoveryAttempt(this.webviewRecoveryAttempts, this.webviewRecoveryInFlightEpoch === epoch) + if (attempt === undefined) return this.webviewRecoveryInFlightEpoch = epoch // Count the whole attempt, including HTML-generation failures. - this.webviewRecoveryAttempts++ + this.webviewRecoveryAttempts = attempt this.lastReceivedWebviewHeartbeatAt = undefined this.webviewHealthySince = undefined // A heartbeat that arrives while the HTML is being generated means the @@ -3707,11 +3723,13 @@ export class ClineProvider // recovered, or the view hid again would either touch a dead or // unrelated webview or reload one that no longer needs recovery. if ( - this._disposed || - this.webviewRecoveryEpoch !== epoch || - this.view !== view || - this.webviewHeartbeatRevision !== heartbeatRevision || - view.visible !== true + !canCommitRecovery({ + disposed: this._disposed, + isCurrent: this.view === view, + epochMatches: this.webviewRecoveryEpoch === epoch, + heartbeatMatches: this.webviewHeartbeatRevision === heartbeatRevision, + visible: view.visible === true, + }) ) { return } @@ -3746,9 +3764,7 @@ export class ClineProvider } finally { // Clear ownership only while this completion still holds it; a // stale recovery must not release the replacement's in-flight state. - if (this.webviewRecoveryInFlightEpoch === epoch) { - this.webviewRecoveryInFlightEpoch = undefined - } + this.webviewRecoveryInFlightEpoch = releaseRecoveryOwner(this.webviewRecoveryInFlightEpoch, epoch) } } @@ -3821,7 +3837,11 @@ export class ClineProvider } catch (error) { // Only a successful focus can leave a delayed replacement pending. // Command failures must not mark a later manual open as recovery. - if (started && !sidebarFocused) this.pendingSidebarRecovery = false + this.pendingSidebarRecovery = pendingSidebarAfterCommandFailure({ + started, + focusSucceeded: sidebarFocused, + pending: this.pendingSidebarRecovery, + }) // A late discovery failure must not stop a recovered/replaced view. // Once recreation starts, disposal of the old view is expected. if ( @@ -3847,9 +3867,13 @@ export class ClineProvider if (this._disposed) return // A terminal failure still stops retries if an earlier retryable failure // already showed the notification for this incident. - if (stopRecovery) this.webviewRecoveryAttempts = ClineProvider.WEBVIEW_RECOVERY_ATTEMPT_LIMIT - if (this.webviewRecoveryFailureReported) return - this.webviewRecoveryFailureReported = true + const failure = recordRecoveryFailure( + { attempts: this.webviewRecoveryAttempts, reported: this.webviewRecoveryFailureReported }, + stopRecovery, + ) + this.webviewRecoveryAttempts = failure.attempts + this.webviewRecoveryFailureReported = failure.reported + if (!failure.notify) return this.log(`[Zoo Code] Webview recovery ${stopRecovery ? "stopped" : "will retry"}: ${reason}`) const heartbeatCount = this.webviewHeartbeatCount void Promise.resolve( @@ -3859,11 +3883,16 @@ export class ClineProvider "Show Logs", ), ) - .then(async (action) => { - if (action === "Reload Window" && !this._disposed && this.webviewHeartbeatCount === heartbeatCount) { + .then(async (choice) => { + const action = recoveryWarningAction( + choice, + this._disposed, + this.webviewHeartbeatCount === heartbeatCount, + ) + if (action === "reload") { await vscode.commands.executeCommand("workbench.action.reloadWindow") } - if (action === "Show Logs") this.outputChannel.show() + if (action === "logs") this.outputChannel.show() }) .then(undefined, (error: unknown) => { this.log(`[Zoo Code] Recovery notification failed: ${String(error)}`) diff --git a/src/core/webview/webviewRecoveryPolicy.ts b/src/core/webview/webviewRecoveryPolicy.ts new file mode 100644 index 0000000000..bf138f4748 --- /dev/null +++ b/src/core/webview/webviewRecoveryPolicy.ts @@ -0,0 +1,77 @@ +// Pure recovery decisions shared by the provider and the bounded protocol checker. +// VS Code effects, Promise settlement, and timer delivery remain in ClineProvider. +export const WEBVIEW_RECOVERY_LIMITS = { attempts: 3, healthyMs: 60_000, staleMs: 90_000 } as const + +export function preservesRecoveryIncident(inProgress: boolean, pendingSidebar: boolean): boolean { + return inProgress || pendingSidebar +} + +export function resolutionFailureAction(input: { + recoveringHost: boolean + disposed: boolean + isCurrent: boolean +}): "propagate" | "ignore" | "retry" { + if (!input.recoveringHost) return "propagate" + return input.disposed || !input.isCurrent ? "ignore" : "retry" +} + +export function canCommitRecovery(input: { + disposed: boolean + isCurrent: boolean + epochMatches: boolean + heartbeatMatches: boolean + visible: boolean +}): boolean { + return !input.disposed && input.isCurrent && input.epochMatches && input.heartbeatMatches && input.visible +} + +export function nextRecoveryAttempt(attempts: number, inFlight: boolean): number | undefined { + return inFlight || attempts >= WEBVIEW_RECOVERY_LIMITS.attempts ? undefined : attempts + 1 +} + +export function releaseRecoveryOwner(owner: number | undefined, completedEpoch: number): number | undefined { + return owner === completedEpoch ? undefined : owner +} + +export function pendingSidebarAfterCommandFailure(input: { + started: boolean + focusSucceeded: boolean + pending: boolean +}): boolean { + return input.started && !input.focusSucceeded ? false : input.pending +} + +export function recordRecoveryFailure( + state: { attempts: number; reported: boolean }, + stopRecovery: boolean, +): { attempts: number; reported: boolean; notify: boolean } { + return { + attempts: stopRecovery ? WEBVIEW_RECOVERY_LIMITS.attempts : state.attempts, + reported: true, + notify: !state.reported, + } +} + +export function recoveryHealth( + state: { attempts: number; lastHeartbeatAt: number | undefined; healthySince: number | undefined }, + now: number, +): { attempts: number; healthySince: number | undefined } { + const healthySince = + state.lastHeartbeatAt === undefined || now - state.lastHeartbeatAt > WEBVIEW_RECOVERY_LIMITS.staleMs + ? now + : state.healthySince + return { + healthySince, + attempts: + healthySince !== undefined && now - healthySince >= WEBVIEW_RECOVERY_LIMITS.healthyMs ? 0 : state.attempts, + } +} + +export function recoveryWarningAction( + choice: string | undefined, + disposed: boolean, + sameHeartbeat: boolean, +): "reload" | "logs" | "none" { + if (choice === "Show Logs") return "logs" + return choice === "Reload Window" && !disposed && sameHeartbeat ? "reload" : "none" +} From c8335eb755b324191c042923195a5fce7e704910 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 19:23:55 +0000 Subject: [PATCH 33/34] fix(webview): log terminal recovery stops after retry warnings --- apps/vscode-e2e/src/visual/heartbeat.md | 2 +- docs/architecture/webview-recovery-model.md | 8 +- scripts/check-webview-recovery.ts | 77 ++++++++++++++++--- src/core/webview/ClineProvider.ts | 12 ++- .../webview/__tests__/ClineProvider.spec.ts | 73 +++++++++++++++--- src/core/webview/webviewRecoveryPolicy.ts | 6 +- 6 files changed, 148 insertions(+), 30 deletions(-) diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index 164e5724d2..1f17408b1c 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -17,7 +17,7 @@ The message-port interceptor follows the pinned VS Code implementation. Check it A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. -A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. A later terminal failure still logs its final stop reason once, independently of warning deduplication. Subsequent watchdog ticks do not repeat the stop log. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. diff --git a/docs/architecture/webview-recovery-model.md b/docs/architecture/webview-recovery-model.md index d4f6d12836..3d6c076e30 100644 --- a/docs/architecture/webview-recovery-model.md +++ b/docs/architecture/webview-recovery-model.md @@ -11,7 +11,7 @@ Code QA runs both commands in its compile job. This is an independent webview pr ## Production connection -`src/core/webview/webviewRecoveryPolicy.ts` owns pure decisions used by both `ClineProvider` and the checker. These include attempt admission, recovery-incident preservation, resolve-failure classification, HTML commit eligibility, release of an async recovery owner, command-failure marker cleanup, failure accounting and notification deduplication, heartbeat health, and warning actions. +`src/core/webview/webviewRecoveryPolicy.ts` owns pure decisions used by both `ClineProvider` and the checker. These include attempt admission, recovery-incident preservation, resolve-failure classification, HTML commit eligibility, release of an async recovery owner, command-failure marker cleanup, failure accounting, notification deduplication and terminal-stop logging, heartbeat health, and warning actions. The provider still performs VS Code effects. It installs subscriptions, starts timers, assigns HTML, resolves views, and retains its task stack. The policy extraction does not move those effects into the model. @@ -41,15 +41,15 @@ The checker rejects: 3. duplicate in-flight recovery or attempts exceeding the cap; 4. a recovery replacement resetting its incident budget or being classified as normal startup; 5. a current recovery resolve failure losing the retry disposition; -6. retryable warnings consuming the remaining attempts, or duplicate notifications without a notification-state reset; +6. retryable warnings consuming the remaining attempts, duplicate notifications, missing terminal-stop reasons after retry warnings, or repeated terminal-stop logs without an incident reset; 7. a failed sidebar focus leaving a marker that contaminates a later manual reopen; 8. a startup heartbeat replenishing attempts before sustained health; 9. a stale Reload Window action reloading after a heartbeat or disposal; and 10. a late Show Logs action being incorrectly suppressed. -Required actions and semantic witnesses prevent a vacuous pass. They include a valid HTML commit, each invalidated completion class, a replacement owner surviving old completion, a retryable initialization failure, repeated initialization failures reaching the cap, healthy budget refill, a published sidebar timing out with attempts remaining, late publication retaining an exhausted budget, and a fresh manual open after focus failure. +Required actions and semantic witnesses prevent a vacuous pass. They include a valid HTML commit, each invalidated completion class, a replacement owner surviving old completion, a retryable initialization failure, repeated initialization failures reaching the cap, healthy budget refill, a published sidebar timing out with attempts remaining, late publication retaining an exhausted budget, and a fresh manual open after focus failure, and one terminal-stop log after an earlier retry warning. -Ten representative faulty policies are injected without editing source. Each must fail with its expected contract violation. Exploration truncation or an unrelated exception does not count as rejecting a regression. +Twelve representative faulty policies are injected without editing source. Each must fail with its expected contract violation. Exploration truncation or an unrelated exception does not count as rejecting a regression. ## Effect and integration evidence diff --git a/scripts/check-webview-recovery.ts b/scripts/check-webview-recovery.ts index 537e1d6eec..3a66840f9b 100644 --- a/scripts/check-webview-recovery.ts +++ b/scripts/check-webview-recovery.ts @@ -148,6 +148,9 @@ interface Host { phase: Phase attempts: number reported: boolean + stopLogged: boolean + terminalLogs: number + retryWarned: boolean busy: boolean pending: boolean view: "old" | "none" | "replacement" @@ -167,6 +170,9 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { phase: "idle", attempts: 0, reported: false, + stopLogged: false, + terminalLogs: 0, + retryWarned: false, busy: false, pending: false, view: "old", @@ -186,10 +192,16 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { const steps: Step[] = [] const add = (action: string, update: Partial) => steps.push({ action, state: { ...s, ...update } }) const failure = (stop: boolean): Partial => { - const next = p.recordRecoveryFailure({ attempts: s.attempts, reported: s.reported }, stop) + const next = p.recordRecoveryFailure( + { attempts: s.attempts, reported: s.reported, stopLogged: s.stopLogged }, + stop, + ) return { attempts: next.attempts, reported: next.reported, + stopLogged: next.stopLogged, + terminalLogs: s.terminalLogs + (stop && next.log ? 1 : 0), + retryWarned: s.retryWarned || (!stop && next.notify), fault: next.attempts !== (stop ? 3 : s.attempts) || next.notify === s.reported ? "failure consumed retries or repeated a warning" @@ -292,6 +304,9 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { watchdog: true, initialized: true, reported: false, + stopLogged: false, + terminalLogs: 0, + retryWarned: false, manualOpened: true, fault: recovering ? "manual open inherited a failed incident" : undefined, }) @@ -305,6 +320,9 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { add("startup-heartbeat", { healthStarted: true, reported: false, + stopLogged: false, + terminalLogs: 0, + retryWarned: false, attempts: health.attempts, fault: health.attempts !== s.attempts ? "one heartbeat refilled the budget" : undefined, }) @@ -318,6 +336,9 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { healthReset: true, attempts: health.attempts, reported: false, + stopLogged: false, + terminalLogs: 0, + retryWarned: false, fault: health.attempts !== 0 ? "healthy window failed to replenish attempts" : undefined, }) } @@ -326,10 +347,13 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { }, (s) => { assert(!s.fault, s.fault) + assert(s.terminalLogs <= 1, "duplicate terminal stop log in one incident") + assert.equal(s.stopLogged, s.terminalLogs === 1, "terminal stop log was suppressed") assert(Number.isInteger(s.attempts) && s.attempts >= 0 && s.attempts <= 3, "attempt budget") if (s.phase === "idle") assert(s.watchdog && !s.busy, "settled recovery lost watchdog or retained ownership") const marks: string[] = [] + if (s.retryWarned && s.stopLogged && s.terminalLogs === 1) marks.push("terminal-stop-after-retry-warning") if (s.phase === "idle" && !s.initialized && s.attempts === 1) marks.push("initialization-failure-retryable") if (s.phase === "stopped" && s.attempts === 3 && !s.initialized && s.view === "replacement") marks.push("failed-resolutions-bounded") @@ -342,7 +366,12 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { return marks }, ) - const required = ["initialization-failure-retryable", "failed-resolutions-bounded", "healthy-budget-refill"] + const required = [ + "initialization-failure-retryable", + "failed-resolutions-bounded", + "healthy-budget-refill", + "terminal-stop-after-retry-warning", + ] if (kind === "sidebar") required.push( "published-timeout-keeps-attempts", @@ -407,13 +436,21 @@ function policyContracts(p: Policy): void { } for (let attempts = 0; attempts <= 3; attempts++) for (const reported of [false, true]) - for (const stop of [false, true]) { - assert.deepEqual( - p.recordRecoveryFailure({ attempts, reported }, stop), - { attempts: stop ? 3 : attempts, reported: true, notify: !reported }, - "failure accounting and notification deduplication", - ) - } + for (const stopLogged of [false, true]) + for (const stop of [false, true]) { + assert.deepEqual( + p.recordRecoveryFailure({ attempts, reported, stopLogged }, stop), + { + attempts: stop ? 3 : attempts, + reported: true, + notify: !reported, + stopLogged: stopLogged || stop, + log: stop ? !stopLogged : !reported && !stopLogged, + }, + "failure accounting, notification deduplication, and terminal stop logging", + ) + } + for (const started of [false, true]) for (const focusSucceeded of [false, true]) for (const pending of [false, true]) { @@ -436,6 +473,28 @@ check(policy) // Representative policy regressions must be rejected, rather than merely // leaving a self-consistent abstract model green. These do not mutate source. const mutants: Array<[string, Policy, RegExp]> = [ + [ + "terminal stop suppressed after retry warning", + { + ...policy, + recordRecoveryFailure: (state, stop) => ({ + ...policy.recordRecoveryFailure(state, stop), + log: !state.reported, + }), + }, + /terminal stop logging/, + ], + [ + "terminal stop repeated on every tick", + { + ...policy, + recordRecoveryFailure: (state, stop) => ({ + ...policy.recordRecoveryFailure(state, stop), + log: stop || !state.reported, + }), + }, + /terminal stop logging/, + ], [ "stale notification reloads window", { ...policy, recoveryWarningAction: (choice) => (choice === "Reload Window" ? "reload" : "none") }, diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index bc6d2f6985..33264aa991 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -266,6 +266,7 @@ export class ClineProvider // Keep that repair distinct from a user-opened view until it arrives. private pendingSidebarRecovery = false private webviewRecoveryFailureReported = false + private webviewRecoveryStopLogged = false private lastReceivedWebviewHeartbeatAt: number | undefined private webviewHeartbeatCount = 0 private webviewHealthySince: number | undefined @@ -881,6 +882,7 @@ export class ClineProvider if (!preservesRecoveryIncident(this.webviewHostRecoveryInProgress, this.pendingSidebarRecovery)) { this.webviewRecoveryAttempts = 0 this.webviewRecoveryFailureReported = false + this.webviewRecoveryStopLogged = false } this.lastReceivedWebviewHeartbeatAt = undefined this.webviewHealthySince = undefined @@ -3625,6 +3627,7 @@ export class ClineProvider // A new heartbeat invalidates the old warning action. Allow a fresh warning // if this renderer stalls again, without replenishing its retry budget. this.webviewRecoveryFailureReported = false + this.webviewRecoveryStopLogged = false if (this.webviewRecoveryConfirmation) { this.log("[Zoo Code] Webview heartbeat returned after recovery") } @@ -3868,13 +3871,18 @@ export class ClineProvider // A terminal failure still stops retries if an earlier retryable failure // already showed the notification for this incident. const failure = recordRecoveryFailure( - { attempts: this.webviewRecoveryAttempts, reported: this.webviewRecoveryFailureReported }, + { + attempts: this.webviewRecoveryAttempts, + reported: this.webviewRecoveryFailureReported, + stopLogged: this.webviewRecoveryStopLogged, + }, stopRecovery, ) this.webviewRecoveryAttempts = failure.attempts this.webviewRecoveryFailureReported = failure.reported + this.webviewRecoveryStopLogged = failure.stopLogged + if (failure.log) this.log(`[Zoo Code] Webview recovery ${stopRecovery ? "stopped" : "will retry"}: ${reason}`) if (!failure.notify) return - this.log(`[Zoo Code] Webview recovery ${stopRecovery ? "stopped" : "will retry"}: ${reason}`) const heartbeatCount = this.webviewHeartbeatCount void Promise.resolve( vscode.window.showWarningMessage( diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index fa0db8d1f2..2426a38f2c 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1668,20 +1668,59 @@ describe("ClineProvider", () => { expect(provider.getCurrentTask()).toBeUndefined() }) - test("stops retries after a terminal failure even when a retry warning was already shown", async () => { - await provider.resolveWebviewView(mockWebviewView) - const html = mockWebviewView.webview.html - provider["reportWebviewRecoveryFailure"]("Initialization failed; retry pending", false) - expect(provider["webviewRecoveryAttempts"]).toBe(0) - expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + test.each([false, true])( + "logs one terminal stop and notification despite repeated ticks (prior retry warning: %s)", + async (priorWarning) => { + await provider.resolveWebviewView(mockWebviewView) + const html = mockWebviewView.webview.html + const log = vi.spyOn(provider, "log") + if (priorWarning) { + provider["reportWebviewRecoveryFailure"]("Initialization failed; retry pending", false) + expect(provider["webviewRecoveryAttempts"]).toBe(0) + expect(log).toHaveBeenCalledWith( + "[Zoo Code] Webview recovery will retry: Initialization failed; retry pending", + ) + } - provider["reportWebviewRecoveryFailure"]("Recovery is unavailable") - await vi.advanceTimersByTimeAsync(120_000) + provider["reportWebviewRecoveryFailure"]("Recovery is unavailable") + expect(log).toHaveBeenCalledWith("[Zoo Code] Webview recovery stopped: Recovery is unavailable") + await vi.advanceTimersByTimeAsync(300_000) - expect(provider["webviewRecoveryAttempts"]).toBe(3) - expect(mockWebviewView.webview.html).toBe(html) - expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) - }) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(mockWebviewView.webview.html).toBe(html) + expect(log.mock.calls.filter(([line]) => line.includes("Webview recovery stopped:"))).toEqual([ + ["[Zoo Code] Webview recovery stopped: Recovery is unavailable"], + ]) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + }, + ) + + test.each(["heartbeat", "manual reopen"] as const)( + "logs a new terminal stop after %s begins a new incident", + async (reset) => { + await provider.resolveWebviewView(mockWebviewView) + const log = vi.spyOn(provider, "log") + provider["reportWebviewRecoveryFailure"]("First incident") + if (reset === "heartbeat") provider.updateWebviewHeartbeat() + else { + disposeCallback() + await provider.resolveWebviewView({ + ...mockWebviewView, + webview: { ...mockWebviewView.webview, html: "" }, + }) + } + expect(provider["webviewRecoveryStopLogged"]).toBe(false) + + provider["reportWebviewRecoveryFailure"]("Second incident") + await vi.advanceTimersByTimeAsync(300_000) + + expect(log.mock.calls.filter(([line]) => line.includes("Webview recovery stopped:"))).toEqual([ + ["[Zoo Code] Webview recovery stopped: First incident"], + ["[Zoo Code] Webview recovery stopped: Second incident"], + ]) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(2) + }, + ) test.each(["Reload Window", "Show Logs"] as const)( "runs only the selected recovery action: %s", @@ -2651,6 +2690,16 @@ describe("ClineProvider", () => { expect(provider["webviewRecoveryAttempts"]).toBe(3) expect(provider["webviewHostRecoveryInProgress"]).toBe(false) expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) + expect( + vi + .mocked(mockOutputChannel.appendLine) + .mock.calls.filter(([line]) => line.includes("Webview recovery stopped:")), + ).toHaveLength(1) + expect(mockOutputChannel.appendLine).toHaveBeenCalledWith( + expect.stringContaining( + "Webview recovery stopped: Recovery attempt limit reached without sustained heartbeats", + ), + ) expect(provider["_disposed"]).toBe(false) expect(provider.getCurrentTask()).toBe(task) expect(task.dispose).not.toHaveBeenCalled() diff --git a/src/core/webview/webviewRecoveryPolicy.ts b/src/core/webview/webviewRecoveryPolicy.ts index bf138f4748..070c294e98 100644 --- a/src/core/webview/webviewRecoveryPolicy.ts +++ b/src/core/webview/webviewRecoveryPolicy.ts @@ -42,13 +42,15 @@ export function pendingSidebarAfterCommandFailure(input: { } export function recordRecoveryFailure( - state: { attempts: number; reported: boolean }, + state: { attempts: number; reported: boolean; stopLogged: boolean }, stopRecovery: boolean, -): { attempts: number; reported: boolean; notify: boolean } { +): { attempts: number; reported: boolean; notify: boolean; stopLogged: boolean; log: boolean } { return { attempts: stopRecovery ? WEBVIEW_RECOVERY_LIMITS.attempts : state.attempts, reported: true, notify: !state.reported, + stopLogged: state.stopLogged || stopRecovery, + log: stopRecovery ? !state.stopLogged : !state.reported && !state.stopLogged, } } From 4c8b6cf1679f9726037e7740e0ae4329008cbc6b Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 20:24:36 +0000 Subject: [PATCH 34/34] feat(webview): add explicit interface recovery retry with generation guards --- .../src/visual/heartbeat.electron.ts | 166 ++++++----- apps/vscode-e2e/src/visual/heartbeat.md | 8 +- docs/architecture/webview-recovery-model.md | 26 +- packages/types/src/vscode.ts | 1 + scripts/check-webview-recovery.ts | 144 +++++++++- .../__tests__/registerCommands.spec.ts | 20 +- src/activate/registerCommands.ts | 3 + src/core/webview/ClineProvider.ts | 122 ++++++-- .../webview/__tests__/ClineProvider.spec.ts | 265 ++++++++++++++++++ src/core/webview/webviewRecoveryPolicy.ts | 20 +- src/package.json | 5 + src/package.nls.ca.json | 1 + src/package.nls.de.json | 1 + src/package.nls.es.json | 1 + src/package.nls.fr.json | 1 + src/package.nls.hi.json | 1 + src/package.nls.id.json | 1 + src/package.nls.it.json | 1 + src/package.nls.ja.json | 1 + src/package.nls.json | 1 + src/package.nls.ko.json | 1 + src/package.nls.nl.json | 1 + src/package.nls.pl.json | 1 + src/package.nls.pt-BR.json | 1 + src/package.nls.ru.json | 1 + src/package.nls.tr.json | 1 + src/package.nls.vi.json | 1 + src/package.nls.zh-CN.json | 1 + src/package.nls.zh-TW.json | 1 + 29 files changed, 684 insertions(+), 114 deletions(-) diff --git a/apps/vscode-e2e/src/visual/heartbeat.electron.ts b/apps/vscode-e2e/src/visual/heartbeat.electron.ts index 2fa61ebfc6..e8ead5f502 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.electron.ts +++ b/apps/vscode-e2e/src/visual/heartbeat.electron.ts @@ -121,77 +121,105 @@ test("recovers when heartbeat delivery stops and resumes periodic heartbeats", a } }) -test("recovers after the isolated Zoo Code renderer is terminated", async ({ browserName: _browserName }, testInfo) => { - // Allow startup, stale detection, recovery observations, and cleanup. - test.setTimeout(8 * 60_000) - const running = await startScene( - vscodeExecutablePath, - { ...scenario, scene: "chat", startMock: true, landmark: CHAT_RESULT }, - testInfo, - ) - try { - expect(running.taskIds).toHaveLength(1) - const initial = await waitForContent(running.page, CHAT_RESULT) - await initial.evaluate(() => { - document.documentElement.dataset.heartbeatDocument = "before-crash" - }) - const processes = await running.app.evaluate(({ BrowserWindow }) => { - const mainFrame = BrowserWindow.getAllWindows()[0]?.webContents.mainFrame - if (!mainFrame) throw new Error("VS Code workbench frame not found") - const renderer = mainFrame.framesInSubtree.find( - (frame: { url: string; parent?: { url: string }; osProcessId: number }) => - frame.url.includes("/fake.html") && - frame.parent?.url.includes("extensionId=ZooCodeOrganization.zoo-code"), +for (const explicitRetry of [false, true]) + test( + explicitRetry + ? "explicit retry recovers the isolated Zoo Code renderer and retains its task" + : "recovers after the isolated Zoo Code renderer is terminated", + async ({ browserName: _browserName }, testInfo) => { + // Allow startup, stale detection, recovery observations, and cleanup. + test.setTimeout(8 * 60_000) + const running = await startScene( + vscodeExecutablePath, + { ...scenario, scene: "chat", startMock: true, landmark: CHAT_RESULT }, + testInfo, ) - if (!renderer) throw new Error("Zoo Code renderer process not found") - return { workbench: mainFrame.osProcessId, webview: renderer.osProcessId } - }) - await testInfo.attach("renderer-processes.json", { - body: JSON.stringify(processes), - contentType: "application/json", - }) - // Never terminate the workbench: this test requires a separate OOPIF process. - expect(processes.webview).toBeGreaterThan(0) - expect(processes.webview).not.toBe(processes.workbench) - process.kill(processes.webview, "SIGKILL") + try { + expect(running.taskIds).toHaveLength(1) + const initial = await waitForContent(running.page, CHAT_RESULT) + await initial.evaluate(() => { + document.documentElement.dataset.heartbeatDocument = "before-crash" + }) + if (explicitRetry) { + // Establish a fresh delivered heartbeat before the crash. Explicit + // recovery must finish before the 90-second stale threshold. + const beforeCrash = await observeHeartbeats(running.page) + await expect + .poll(() => beforeCrash.evaluate((state) => state.forwarded), { timeout: 45_000 }) + .toBeGreaterThan(0) + } + const processes = await running.app.evaluate(({ BrowserWindow }) => { + const mainFrame = BrowserWindow.getAllWindows()[0]?.webContents.mainFrame + if (!mainFrame) throw new Error("VS Code workbench frame not found") + const renderer = mainFrame.framesInSubtree.find( + (frame: { url: string; parent?: { url: string }; osProcessId: number }) => + frame.url.includes("/fake.html") && + frame.parent?.url.includes("extensionId=ZooCodeOrganization.zoo-code"), + ) + if (!renderer) throw new Error("Zoo Code renderer process not found") + return { workbench: mainFrame.osProcessId, webview: renderer.osProcessId } + }) + await testInfo.attach("renderer-processes.json", { + body: JSON.stringify(processes), + contentType: "application/json", + }) + // Never terminate the workbench: this test requires a separate OOPIF process. + expect(processes.webview).toBeGreaterThan(0) + expect(processes.webview).not.toBe(processes.workbench) + process.kill(processes.webview, "SIGKILL") + if (explicitRetry) { + await running.page.keyboard.press("F1") + const input = running.page.locator(".quick-input-widget input") + await expect(input).toBeVisible({ timeout: 10_000 }) + await input.fill(">Zoo Code: Retry Interface Recovery", { timeout: 10_000 }) + await running.page + .locator(".quick-input-list .monaco-list-row") + .filter({ hasText: "Retry Interface Recovery" }) + .first() + .click({ timeout: 10_000 }) + } - // Allow stale detection (up to 150s), confirmation and replacement waits, - // plus webview loading. A passing fast run does not bound the slow path. - await expect - .poll( - async () => { - try { - const frame = contentFrame(running.page) - if (!frame || !(await frame.getByText(CHAT_RESULT, { exact: false }).isVisible())) - return "missing" - return ( - (await frame.locator("html").getAttribute("data-heartbeat-document", { timeout: 1_000 })) ?? - "recovered" - ) - } catch { - return "unavailable" - } - }, - { timeout: 210_000, intervals: [1_000] }, - ) - .toBe("recovered") - await waitForContent(running.page, CHAT_RESULT) - await fs.writeFile(`${running.donePath}.probe`, "probe") - await expect - .poll(async () => { - const result = await fs.readFile(`${running.donePath}.probe.result`, "utf8").catch(() => "") - return result ? (JSON.parse(result) as { taskIds: string[] }) : undefined - }) - .toMatchObject({ taskIds: running.taskIds }) - const heartbeat = await observeHeartbeats(running.page) - await expect.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }).toBeGreaterThan(0) - } catch (error) { - await attachFailureArtifacts(running, testInfo) - throw error - } finally { - await stopScene(running, testInfo) - } -}) + // Allow stale detection (up to 150s), confirmation and replacement waits, + // plus webview loading. A passing fast run does not bound the slow path. + await expect + .poll( + async () => { + try { + const frame = contentFrame(running.page) + if (!frame || !(await frame.getByText(CHAT_RESULT, { exact: false }).isVisible())) + return "missing" + return ( + (await frame + .locator("html") + .getAttribute("data-heartbeat-document", { timeout: 1_000 })) ?? "recovered" + ) + } catch { + return "unavailable" + } + }, + { timeout: explicitRetry ? 45_000 : 210_000, intervals: [1_000] }, + ) + .toBe("recovered") + await waitForContent(running.page, CHAT_RESULT) + await fs.writeFile(`${running.donePath}.probe`, "probe") + await expect + .poll(async () => { + const result = await fs.readFile(`${running.donePath}.probe.result`, "utf8").catch(() => "") + return result ? (JSON.parse(result) as { taskIds: string[] }) : undefined + }) + .toMatchObject({ taskIds: running.taskIds }) + const heartbeat = await observeHeartbeats(running.page) + await expect + .poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }) + .toBeGreaterThan(0) + } catch (error) { + await attachFailureArtifacts(running, testInfo) + throw error + } finally { + await stopScene(running, testInfo) + } + }, + ) test("keeps the same document when the sidebar is hidden past the stale threshold", async ({ browserName: _browserName, diff --git a/apps/vscode-e2e/src/visual/heartbeat.md b/apps/vscode-e2e/src/visual/heartbeat.md index 1f17408b1c..48bd98b33a 100644 --- a/apps/vscode-e2e/src/visual/heartbeat.md +++ b/apps/vscode-e2e/src/visual/heartbeat.md @@ -17,8 +17,14 @@ The message-port interceptor follows the pinned VS Code implementation. Check it A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership. -A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. A later terminal failure still logs its final stop reason once, independently of warning deduplication. Subsequent watchdog ticks do not repeat the stop log. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel. +A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers non-modal Retry Interface Recovery, Reload Window, and Show Logs actions. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. A later terminal failure still logs its final stop reason once, independently of warning deduplication. Subsequent watchdog ticks do not repeat the stop log. Automatic recovery replenishes the incident budget only after sustained healthy heartbeats. Window reload is never automatic. Late Retry Interface Recovery and Reload Window actions are ignored after liveness returns or a newer explicit recovery generation starts. Show Logs still opens the output channel. Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order. The [bounded recovery model](../../../../docs/architecture/webview-recovery-model.md) checks production recovery policies and abstract interleavings. Run `pnpm webview-recovery:model-check` alongside the provider and Electron tests. It does not prove VS Code effects or recovery liveness. + +## Explicit recovery after a timeout + +Choose **Retry Interface Recovery** in the warning, or run **Zoo Code: Retry Interface Recovery** from the Command Palette. The command remains available if the warning was dismissed or the sidebar has no live view. It starts a new recovery generation, counts the requested repair as the first of three attempts, clears the old warning/log state, and retains the task stack, including an interrupted task. It does not reload the VS Code window. + +Reopening a sidebar after a timed-out focus is ambiguous: VS Code does not pass a user-versus-programmatic origin to the resolver. Such a callback keeps the pending incident and its budget. The explicit retry command supplies the missing user intent. Late command completions and notifications from older generations cannot change the renewed budget or release its operation ownership. An already-issued VS Code command cannot be cancelled by this generation guard; the guard prevents subsequent obsolete extension-side effects. diff --git a/docs/architecture/webview-recovery-model.md b/docs/architecture/webview-recovery-model.md index 3d6c076e30..399ab1e1d3 100644 --- a/docs/architecture/webview-recovery-model.md +++ b/docs/architecture/webview-recovery-model.md @@ -11,7 +11,7 @@ Code QA runs both commands in its compile job. This is an independent webview pr ## Production connection -`src/core/webview/webviewRecoveryPolicy.ts` owns pure decisions used by both `ClineProvider` and the checker. These include attempt admission, recovery-incident preservation, resolve-failure classification, HTML commit eligibility, release of an async recovery owner, command-failure marker cleanup, failure accounting, notification deduplication and terminal-stop logging, heartbeat health, and warning actions. +`src/core/webview/webviewRecoveryPolicy.ts` owns pure decisions used by both `ClineProvider` and the checker. These include attempt admission, recovery-incident preservation, resolve-failure classification, HTML commit eligibility, release of an async recovery owner, command-failure marker cleanup, failure accounting, notification deduplication and terminal-stop logging, heartbeat health, explicit retry generations, and warning actions. The provider still performs VS Code effects. It installs subscriptions, starts timers, assigns HTML, resolves views, and retains its task stack. The policy extraction does not move those effects into the model. @@ -24,11 +24,11 @@ This is production-policy-backed bounded verification, not a refinement proof of ## State spaces and bounds -The checker explores three independent state spaces to a fixed point. Each has a 100,000-state budget and a maximum discovery depth of 60. Discovering a new state beyond either bound fails the check; reaching the depth bound does not count as success. The printed state totals are diagnostics, not ratchets. +The checker explores three independent state spaces to a fixed point. Each has a 100,000-state budget and a maximum discovery depth of 100. Discovering a new state beyond either bound fails the check; reaching the depth bound does not count as success. The printed state totals are diagnostics, not ratchets. -**Async ownership** models one pending HTML operation and one possible replacement operation. Visibility can change, a heartbeat or grace reset can invalidate the pending work, the epoch can change, and the provider can be disposed or replaced before the old operation settles. The completion records which condition held at settlement, rather than inferring event order from a later state. Old completion must not release a replacement operation's owner. +**Async ownership** models one pending HTML operation and one possible replacement operation. Visibility can change, a heartbeat or grace reset can invalidate the pending work, the epoch can change, and the provider can be disposed or replaced before the old operation settles. The completion records which condition held at settlement, rather than inferring event order from a later state. Old completion must not release a replacement operation's owner. An explicit retry can invalidate the old epoch while retaining the same view, and old host-command completion must respect generation ownership. -**Sidebar and tab hosts** model recovery attempts, HTML generation success/failure, host replacement, captured recovery intent, initialization success/failure, notification state, watchdog availability, and host-operation ownership. Sidebar actions additionally include removal/focus failures, publication before or after the replacement wait, and a wait expiring during initialization. Repeated failures consume a three-attempt budget. One manual reopen and one sustained-health refill are permitted, so later incidents are reachable without an unbounded counter. +**Sidebar and tab hosts** model recovery attempts, HTML generation success/failure, host replacement, captured recovery intent, initialization success/failure, notification state, watchdog availability, and host-operation ownership. Sidebar actions additionally include removal/focus failures, publication before or after the replacement wait, and a wait expiring during initialization. Repeated failures consume a three-attempt budget. One ordinary manual reopen, one ambiguous reopen while a timeout marker is pending, one explicit user retry, and one sustained-health refill are permitted, so later incidents are reachable without an unbounded counter. The graphs are checked separately. They do not establish every possible joint interleaving between ownership, host effects, and task execution. The host graph abstracts identities as original/missing/replacement; the ownership graph checks distinct old/replacement owners. Numeric health contracts check 59,999/60,000 and 90,000/90,001 millisecond boundaries. Timer scheduling itself is an environment assumption, not a model guarantee. @@ -44,12 +44,12 @@ The checker rejects: 6. retryable warnings consuming the remaining attempts, duplicate notifications, missing terminal-stop reasons after retry warnings, or repeated terminal-stop logs without an incident reset; 7. a failed sidebar focus leaving a marker that contaminates a later manual reopen; 8. a startup heartbeat replenishing attempts before sustained health; -9. a stale Reload Window action reloading after a heartbeat or disposal; and +9. stale Reload Window or Retry Interface Recovery actions changing state after a heartbeat, disposal, or generation renewal; and 10. a late Show Logs action being incorrectly suppressed. -Required actions and semantic witnesses prevent a vacuous pass. They include a valid HTML commit, each invalidated completion class, a replacement owner surviving old completion, a retryable initialization failure, repeated initialization failures reaching the cap, healthy budget refill, a published sidebar timing out with attempts remaining, late publication retaining an exhausted budget, and a fresh manual open after focus failure, and one terminal-stop log after an earlier retry warning. +Required actions and semantic witnesses prevent a vacuous pass. They include a valid HTML commit, each invalidated completion class, a replacement owner surviving old completion, a retryable initialization failure, repeated initialization failures reaching the cap, healthy budget refill, a published sidebar timing out with attempts remaining, late publication retaining an exhausted budget, a fresh manual open after focus failure, one terminal-stop log after an earlier retry warning, preservation of the capped incident during ambiguous reopen, and a fresh budget after explicit retry. -Twelve representative faulty policies are injected without editing source. Each must fail with its expected contract violation. Exploration truncation or an unrelated exception does not count as rejecting a regression. +Fifteen representative faulty policies are injected without editing source. Each must fail with its expected contract violation. Exploration truncation or an unrelated exception does not count as rejecting a regression. ## Effect and integration evidence @@ -61,11 +61,19 @@ Twelve representative faulty policies are injected without editing source. Each - late initialization failure after the sidebar replacement wait; - disposal/replacement before an old resolve completes; - recovery ownership clearing and a second incident remaining recoverable; -- failed focus followed by a normal manual open; and +- failed focus followed by a normal manual open; +- explicit retry from a timed-out or ambiguously reopened sidebar, without evicting an interrupted task; +- old resolve, focus, and notification completion after generation renewal; and - warning actions, heartbeat invalidation, and healthy-window boundaries. These are integration checks of the handwritten effect layer, not generated trace conformance tests. The host model assumes that a retryable resolve failure installs a watchdog and that a completed host operation releases its busy flag; the provider tests verify those assumptions. The model's incident-preservation decision is used by the provider's budget reset and task-cleanup guards; the task registry itself is not modeled. -The Electron heartbeat project checks real sidebar recreation, isolated renderer termination, task-stack preservation, resumed periodic heartbeats, and hidden-sidebar retention. It does not deliberately make a real VS Code resolver reject. No model here proves renderer memory safety, eventual Promise settlement, OS sleep/resume ordering, arbitrary overlapping operations, or recovery liveness under an unfair or permanently failing environment. +The Electron heartbeat project checks Command Palette retry after renderer termination, real sidebar recreation, isolated renderer termination, task-stack preservation, resumed periodic heartbeats, and hidden-sidebar retention. It does not deliberately make a real VS Code resolver reject. No model here proves renderer memory safety, eventual Promise settlement, OS sleep/resume ordering, arbitrary overlapping operations, or recovery liveness under an unfair or permanently failing environment. When changing recovery behavior, update the production policy, model actions/contracts and required witnesses, and the narrowest provider test that checks its effects. Rerun Electron checks when a VS Code host or transport boundary changes. + +## Why explicit user intent + +The [pinned VS Code view API](https://github.com/microsoft/vscode/blob/1.100.0/src/vscode-dts/vscode.d.ts#L9567-L9616) supplies persisted state and a lifetime cancellation token, but no focus-request identifier or user-open origin. A timeout cannot establish which request caused a later callback. The implementation therefore retains the incident for ambiguous callbacks and uses an explicit command to authorize a new generation and retry budget. This follows the manual-reset approach described in [Microsoft's circuit-breaker guidance](https://learn.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker#problems-and-considerations), adapted to a local UI recovery budget. + +The model does not infer callback provenance. It explicitly allows a manual reopen while the old marker is pending and requires that this alone cannot refresh the budget. Explicit retry is a separate environment action. Generation guards protect extension-side continuations; they do not cancel a VS Code command already issued to the host. diff --git a/packages/types/src/vscode.ts b/packages/types/src/vscode.ts index fd4e31116d..27bd81a499 100644 --- a/packages/types/src/vscode.ts +++ b/packages/types/src/vscode.ts @@ -45,6 +45,7 @@ export const commandIds = [ "focusInput", "acceptInput", "focusPanel", + "retryWebviewRecovery", "toggleAutoApprove", "showRipgrepDiagnostic", diff --git a/scripts/check-webview-recovery.ts b/scripts/check-webview-recovery.ts index 3a66840f9b..7704e513ba 100644 --- a/scripts/check-webview-recovery.ts +++ b/scripts/check-webview-recovery.ts @@ -16,7 +16,7 @@ interface Node { depth: number } const MAX_STATES = 100_000 -const MAX_DEPTH = 60 +const MAX_DEPTH = 100 function explore( name: string, @@ -63,6 +63,10 @@ interface Ownership { visible: boolean owner: number | undefined completed: boolean + generation: number + hostCompleted: boolean + hostCompletedAfterRetry: boolean + hostBusy: boolean completionCase: "pending" | "valid" | "disposed" | "replaced" | "epoch" | "heartbeat" | "hidden" fault?: string } @@ -78,12 +82,33 @@ function ownershipCheck(p: Policy): void { visible: true, owner: 0, completed: false, + generation: 0, + hostCompleted: false, + hostCompletedAfterRetry: false, + hostBusy: true, completionCase: "pending", }, (state) => { const steps: Step[] = [] const add = (action: string, update: Partial) => steps.push({ action, state: { ...state, ...update } }) + if (state.generation === 0) + add("explicit-retry-invalidates-old-work", { + generation: 1, + epochMatches: false, + owner: 1, + hostBusy: true, + }) + if (!state.hostCompleted) { + const owns = p.isRecoveryGenerationCurrent(0, state.generation) + add("complete-old-host-command", { + hostCompleted: true, + hostCompletedAfterRetry: state.generation === 1, + hostBusy: owns ? false : state.hostBusy, + fault: + owns !== (state.generation === 0) ? "old host command changed a renewed incident" : undefined, + }) + } if (!state.disposed) add("dispose", { disposed: true }) if (state.isCurrent) add("replace-and-start-new-recovery", { isCurrent: false, epochMatches: false, owner: 1 }) @@ -122,6 +147,7 @@ function ownershipCheck(p: Policy): void { (s) => { assert(!s.fault, s.fault) const marks: string[] = [] + if (s.hostCompletedAfterRetry && s.hostBusy) marks.push("old-host-completion-preserves-new-owner") if (s.completionCase === "valid") marks.push("valid-html-commit") if (s.completionCase === "epoch") marks.push("epoch-invalidates-html") if (s.completionCase === "replaced" && s.owner === 1) @@ -133,6 +159,7 @@ function ownershipCheck(p: Policy): void { }, ) for (const mark of [ + "old-host-completion-preserves-new-owner", "valid-html-commit", "epoch-invalidates-html", "replacement-owner-survives-old-completion", @@ -162,6 +189,10 @@ interface Host { manualOpened: boolean healthStarted: boolean healthReset: boolean + generation: number + explicitRetried: boolean + retriedAfterAmbiguous: boolean + ambiguousOpened: boolean fault?: string } @@ -184,6 +215,10 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { manualOpened: false, healthStarted: false, healthReset: false, + generation: 0, + explicitRetried: false, + retriedAfterAmbiguous: false, + ambiguousOpened: false, } const reached = explore( `${kind} host`, @@ -208,14 +243,15 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { : undefined, } } - const publish = () => { + const publish = (manual = false) => { const recovering = p.preservesRecoveryIncident(s.busy, s.pending) - add("publish-replacement", { + add(manual ? "manual-open-with-pending-marker" : "publish-replacement", { phase: "initialize", view: "replacement", watchdog: false, initialized: false, recoveringResolve: recovering, + ambiguousOpened: s.ambiguousOpened || manual, pending: false, attempts: recovering ? s.attempts : 0, fault: !recovering @@ -273,6 +309,7 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { } if (s.phase === "wait") { publish() + if (s.waitExpired && !s.ambiguousOpened) publish(true) if (!s.waitExpired) add("timeout-before-publication", { ...failure(true), busy: false, waitExpired: true }) } @@ -294,6 +331,35 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { if (kind === "sidebar" && !s.waitExpired) add("timeout-after-publication", { ...failure(false), busy: false, waitExpired: true }) } + if (!s.explicitRetried) { + const fresh = p.beginExplicitRecovery(s.generation) + add("explicit-user-retry", { + generation: fresh.generation, + explicitRetried: true, + retriedAfterAmbiguous: s.ambiguousOpened, + attempts: fresh.attempts, + reported: fresh.reported, + stopLogged: fresh.stopLogged, + terminalLogs: 0, + retryWarned: false, + busy: true, + pending: kind === "sidebar", + healthStarted: false, + waitExpired: false, + phase: kind === "tab" ? "initialize" : s.view === "none" ? "focus" : "remove", + view: kind === "tab" ? "replacement" : s.view, + initialized: kind === "tab" ? false : s.initialized, + watchdog: kind === "tab" ? false : s.watchdog, + recoveringResolve: kind === "tab" || s.recoveringResolve, + fault: + fresh.generation !== s.generation + 1 || + fresh.attempts !== 1 || + fresh.reported || + fresh.stopLogged + ? "explicit retry failed to renew incident" + : undefined, + }) + } if (s.phase === "idle" && s.attempts === 3) add("exhaust-budget", { ...failure(true), phase: "stopped" }) if (s.phase === "stopped" && !s.pending && !s.manualOpened) { const recovering = p.preservesRecoveryIncident(s.busy, s.pending) @@ -353,6 +419,10 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { if (s.phase === "idle") assert(s.watchdog && !s.busy, "settled recovery lost watchdog or retained ownership") const marks: string[] = [] + if (s.ambiguousOpened && !s.explicitRetried && s.attempts === 3 && s.recoveringResolve) + marks.push("ambiguous-reopen-preserves-incident") + if (s.retriedAfterAmbiguous && s.attempts === 1 && s.generation === 1) + marks.push("explicit-retry-after-ambiguous-reopen") if (s.retryWarned && s.stopLogged && s.terminalLogs === 1) marks.push("terminal-stop-after-retry-warning") if (s.phase === "idle" && !s.initialized && s.attempts === 1) marks.push("initialization-failure-retryable") if (s.phase === "stopped" && s.attempts === 3 && !s.initialized && s.view === "replacement") @@ -377,10 +447,13 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { "published-timeout-keeps-attempts", "late-publication-preserves-cap", "manual-open-after-focus-failure", + "ambiguous-reopen-preserves-incident", + "explicit-retry-after-ambiguous-reopen", ) for (const mark of required) assert(reached.has(mark), `${kind} landmark missing: ${mark}`) for (const action of [ "stale-watchdog", + "explicit-user-retry", "duplicate-recovery", "html-failure", "resolve-success", @@ -392,19 +465,38 @@ function hostCheck(kind: "sidebar" | "tab", p: Policy): void { } function policyContracts(p: Policy): void { - for (const choice of [undefined, "Reload Window", "Show Logs", "dismiss"]) + for (const choice of [undefined, "Retry Interface Recovery", "Reload Window", "Show Logs", "dismiss"]) for (const disposed of [false, true]) - for (const sameHeartbeat of [false, true]) { - assert.equal( - p.recoveryWarningAction(choice, disposed, sameHeartbeat), - choice === "Show Logs" - ? "logs" - : choice === "Reload Window" && !disposed && sameHeartbeat - ? "reload" - : "none", - "notification action contract", - ) - } + for (const sameHeartbeat of [false, true]) + for (const sameGeneration of [false, true]) { + assert.equal( + p.recoveryWarningAction(choice, disposed, sameHeartbeat, sameGeneration), + choice === "Show Logs" + ? "logs" + : disposed || !sameHeartbeat || !sameGeneration + ? "none" + : choice === "Retry Interface Recovery" + ? "retry" + : choice === "Reload Window" + ? "reload" + : "none", + "notification action contract", + ) + } + for (const captured of [0, 1]) + for (const current of [0, 1]) { + assert.equal( + p.isRecoveryGenerationCurrent(captured, current), + captured === current, + "recovery generation ownership", + ) + assert.deepEqual( + p.beginExplicitRecovery(current), + { generation: current + 1, attempts: 1, reported: false, stopLogged: false }, + "explicit retry incident contract", + ) + } + for (const recoveringHost of [false, true]) for (const disposed of [false, true]) for (const isCurrent of [false, true]) { @@ -473,6 +565,28 @@ check(policy) // Representative policy regressions must be rejected, rather than merely // leaving a self-consistent abstract model green. These do not mutate source. const mutants: Array<[string, Policy, RegExp]> = [ + [ + "old work owns a new generation", + { ...policy, isRecoveryGenerationCurrent: () => true }, + /recovery generation ownership/, + ], + [ + "explicit retry inherits exhausted incident", + { + ...policy, + beginExplicitRecovery: (generation) => ({ generation, attempts: 3, reported: true, stopLogged: true }), + }, + /explicit retry incident contract/, + ], + [ + "old warning resets a new generation", + { + ...policy, + recoveryWarningAction: (choice, disposed, sameHeartbeat) => + policy.recoveryWarningAction(choice, disposed, sameHeartbeat, true), + }, + /notification action contract/, + ], [ "terminal stop suppressed after retry warning", { diff --git a/src/activate/__tests__/registerCommands.spec.ts b/src/activate/__tests__/registerCommands.spec.ts index 6ef57dacb6..50aab25b48 100644 --- a/src/activate/__tests__/registerCommands.spec.ts +++ b/src/activate/__tests__/registerCommands.spec.ts @@ -136,8 +136,13 @@ describe("getVisibleProviderOrLog", () => { describe("registerCommands handlers", () => { let mockOutputChannel: vscode.OutputChannel let mockContext: vscode.ExtensionContext - let mockVisibleProvider: { postMessageToWebview: Mock } - let mockProvider: { postMessageToWebview: Mock; webviewFocusTracker: WebviewFocusTracker } + let mockVisibleProvider: { postMessageToWebview: Mock; retryWebviewRecovery: Mock<() => Promise> } + let mockProvider: { + postMessageToWebview: Mock + webviewFocusTracker: WebviewFocusTracker + retryWebviewRecovery: Mock<() => Promise> + } + let mockVisibleProviderInstance: ClineProvider let handlers: Record unknown> beforeEach(() => { @@ -158,10 +163,14 @@ describe("registerCommands handlers", () => { mockContext = makeExtensionContext() mockVisibleProvider = { + retryWebviewRecovery: vi.fn().mockResolvedValue(undefined), postMessageToWebview: vi.fn().mockResolvedValue(undefined), } + // The visible-provider double supplies message posting and explicit recovery retry. + mockVisibleProviderInstance = mockVisibleProvider as unknown as ClineProvider mockProvider = { + retryWebviewRecovery: vi.fn().mockResolvedValue(undefined), postMessageToWebview: vi.fn().mockResolvedValue(undefined), webviewFocusTracker: new WebviewFocusTracker(), } @@ -180,6 +189,13 @@ describe("registerCommands handlers", () => { }) }) + it.each([false, true])("retries the appropriate provider when a visible provider exists: %s", async (visible) => { + vi.mocked(ClineProvider.getVisibleInstance).mockReturnValue(visible ? mockVisibleProviderInstance : undefined) + await handlers["zoo-code.retryWebviewRecovery"]() + expect(mockVisibleProvider.retryWebviewRecovery).toHaveBeenCalledTimes(visible ? 1 : 0) + expect(mockProvider.retryWebviewRecovery).toHaveBeenCalledTimes(visible ? 0 : 1) + }) + afterEach(() => { // Reset module-level panel state to prevent leakage between tests. setPanel(undefined, "sidebar") diff --git a/src/activate/registerCommands.ts b/src/activate/registerCommands.ts index 2851dac016..e4e0d855a2 100644 --- a/src/activate/registerCommands.ts +++ b/src/activate/registerCommands.ts @@ -184,6 +184,9 @@ const getCommandsMap = ({ outputChannel.appendLine(`Error focusing input: ${error}`) } }, + retryWebviewRecovery: async () => { + await (ClineProvider.getVisibleInstance() ?? provider).retryWebviewRecovery() + }, focusPanel: async () => { try { await focusPanel(tabPanel, sidebarPanel) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 33264aa991..33f932aef1 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -21,6 +21,8 @@ import { recordRecoveryFailure, recoveryHealth, recoveryWarningAction, + isRecoveryGenerationCurrent, + beginExplicitRecovery, } from "./webviewRecoveryPolicy" import { @@ -277,6 +279,7 @@ export class ClineProvider // the watched view is disposed; the reload must not reassign webview.html // afterwards. private webviewRecoveryEpoch = 0 + private webviewRecoveryRequestGeneration = 0 // Epoch that owns an in-flight recovery reload, if any. Only a recovery // from the same epoch is blocked, so disposing/replacing the view (which // bumps webviewRecoveryEpoch) never makes the replacement wait on the @@ -899,8 +902,8 @@ export class ClineProvider * view because the view was disposed or replaced (or the provider itself was * disposed) while the resolve was awaiting HTML or state hydration. */ - private isStaleViewResolve(webviewView: vscode.WebviewView | vscode.WebviewPanel): boolean { - return this._disposed || this.view !== webviewView + private isStaleViewResolve(webviewView: vscode.WebviewView | vscode.WebviewPanel, epoch: number): boolean { + return this._disposed || this.view !== webviewView || epoch !== this.webviewRecoveryEpoch } /** Drain one task's memoized cleanup without preventing the remaining provider shutdown work. */ @@ -1089,6 +1092,7 @@ export class ClineProvider } async resolveWebviewView(webviewView: vscode.WebviewView | vscode.WebviewPanel) { + const generation = this.webviewRecoveryRequestGeneration // Capture intent before yielding: host recovery can time out while this // resolve is still generating HTML or hydrating state. const recoveringHost = preservesRecoveryIncident( @@ -1098,6 +1102,7 @@ export class ClineProvider try { await this.initializeWebviewView(webviewView, recoveringHost) } catch (error) { + if (!isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) return const action = resolutionFailureAction({ recoveringHost, disposed: this._disposed, @@ -1124,6 +1129,7 @@ export class ClineProvider this.clearWebviewResources() } + const epoch = this.webviewRecoveryEpoch this.pendingSidebarRecovery = false this.view = webviewView const inTabMode = "onDidChangeViewState" in webviewView @@ -1177,7 +1183,7 @@ export class ClineProvider // The HTML load above can finish after the view was disposed or replaced. // A stale resolve must not assign HTML to a view the provider no longer owns. - if (this.isStaleViewResolve(webviewView)) { + if (this.isStaleViewResolve(webviewView, epoch)) { return } @@ -1199,6 +1205,7 @@ export class ClineProvider ttsEnabled, ttsSpeed, }) => { + if (this.isStaleViewResolve(webviewView, epoch)) return Terminal.setShellIntegrationTimeout(terminalShellIntegrationTimeout) Terminal.setShellIntegrationDisabled(terminalShellIntegrationDisabled) Terminal.setCommandDelay(terminalCommandDelay) @@ -1215,7 +1222,7 @@ export class ClineProvider // State hydration can also finish after the view was disposed or replaced. // A stale resolve must not install listeners on a view the provider no longer owns. - if (this.isStaleViewResolve(webviewView)) { + if (this.isStaleViewResolve(webviewView, epoch)) { return } @@ -3780,6 +3787,7 @@ export class ClineProvider private async recreateWebviewHostForRecovery(view: vscode.WebviewView | vscode.WebviewPanel): Promise { if (this.webviewHostRecoveryInProgress || this._disposed || this.view !== view || !view.visible) return const revision = this.webviewHeartbeatRevision + const generation = this.webviewRecoveryRequestGeneration let started = false let sidebarFocused = false try { @@ -3790,6 +3798,7 @@ export class ClineProvider commands = await vscode.commands.getCommands(true) } if ( + !isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration) || this.webviewHostRecoveryInProgress || this._disposed || this.view !== view || @@ -3824,20 +3833,15 @@ export class ClineProvider } else { this.pendingSidebarRecovery = true await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.removeView`) - if (this._disposed) return + if (this._disposed || !isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) + return await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.focus`, { preserveFocus: true }) sidebarFocused = true - await pWaitFor( - () => - this._disposed || - (this.view !== undefined && this.view !== view && this.webviewWatchdogInterval !== null), - { - interval: 50, - timeout: 10_000, - }, - ) + if (!isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) return + await this.waitForRecoveryView(view, generation) } } catch (error) { + if (!isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) return // Only a successful focus can leave a delayed replacement pending. // Command failures must not mark a later manual open as recovery. this.pendingSidebarRecovery = pendingSidebarAfterCommandFailure({ @@ -3862,7 +3866,89 @@ export class ClineProvider !replacementPublished, ) } finally { - if (started) this.webviewHostRecoveryInProgress = false + if (started && isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) + this.webviewHostRecoveryInProgress = false + } + } + + private async waitForRecoveryView( + previous: vscode.WebviewView | vscode.WebviewPanel | undefined, + generation: number, + ): Promise { + await pWaitFor( + () => + this._disposed || + !isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration) || + (this.view !== undefined && this.view !== previous && this.webviewWatchdogInterval !== null), + { interval: 50, timeout: 10_000 }, + ) + } + + /** An explicit user action, unlike an ambiguous VS Code resolve callback. */ + public async retryWebviewRecovery(expectedGeneration = this.webviewRecoveryRequestGeneration): Promise { + if (this._disposed || !isRecoveryGenerationCurrent(expectedGeneration, this.webviewRecoveryRequestGeneration)) + return + const fresh = beginExplicitRecovery(this.webviewRecoveryRequestGeneration) + const generation = fresh.generation + this.webviewRecoveryRequestGeneration = generation + const view = this.view + if (!view || !("onDidChangeViewState" in view)) this.pendingSidebarRecovery = true + this.webviewRecoveryEpoch++ + this.webviewRecoveryInFlightEpoch = undefined + this.webviewHostRecoveryInProgress = false + // Count this user-requested attempt in its fresh, bounded incident. + this.webviewRecoveryAttempts = fresh.attempts + this.webviewRecoveryFailureReported = fresh.reported + this.webviewRecoveryStopLogged = fresh.stopLogged + this.lastReceivedWebviewHeartbeatAt = undefined + this.webviewHealthySince = undefined + this.resetWebviewHeartbeatGrace() + let focused = false + try { + if (view) { + if ("onDidChangeViewState" in view) view.reveal(view.viewColumn ?? vscode.ViewColumn.Active, true) + else view.show(true) + await pWaitFor( + () => + this._disposed || + !isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration) || + this.view !== view || + view.visible, + { interval: 50, timeout: 10_000 }, + ) + if ( + this._disposed || + !isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration) || + this.view !== view + ) + return + await this.recreateWebviewHostForRecovery(view) + } else { + // The earlier timeout may have removed the only sidebar. The next + // resolve, whatever triggered it, repairs this explicitly renewed incident. + this.pendingSidebarRecovery = true + this.webviewHostRecoveryInProgress = true + await vscode.commands.executeCommand(`${ClineProvider.sideBarId}.focus`, { preserveFocus: true }) + focused = true + if (!isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) return + await this.waitForRecoveryView(undefined, generation) + } + } catch (error) { + if (this._disposed || !isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) + return + this.pendingSidebarRecovery = pendingSidebarAfterCommandFailure({ + started: true, + focusSucceeded: focused, + pending: this.pendingSidebarRecovery, + }) + const replacementPublished = this.view !== undefined && this.view !== view + this.reportWebviewRecoveryFailure( + `Explicit interface recovery failed: ${String(error)}`, + !replacementPublished, + ) + } finally { + if (isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration)) + this.webviewHostRecoveryInProgress = false } } @@ -3884,9 +3970,11 @@ export class ClineProvider if (failure.log) this.log(`[Zoo Code] Webview recovery ${stopRecovery ? "stopped" : "will retry"}: ${reason}`) if (!failure.notify) return const heartbeatCount = this.webviewHeartbeatCount + const generation = this.webviewRecoveryRequestGeneration void Promise.resolve( vscode.window.showWarningMessage( - "Zoo Code's interface is not responding. Reloading the VS Code window will restart extensions and may interrupt running tasks.", + "Zoo Code's interface is not responding. Retry interface recovery without restarting running tasks. Reloading the VS Code window will restart extensions and may interrupt running tasks.", + "Retry Interface Recovery", "Reload Window", "Show Logs", ), @@ -3896,7 +3984,9 @@ export class ClineProvider choice, this._disposed, this.webviewHeartbeatCount === heartbeatCount, + isRecoveryGenerationCurrent(generation, this.webviewRecoveryRequestGeneration), ) + if (action === "retry") await this.retryWebviewRecovery(generation) if (action === "reload") { await vscode.commands.executeCommand("workbench.action.reloadWindow") } diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 2426a38f2c..d0155955d0 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -542,6 +542,7 @@ describe("ClineProvider", () => { cspSource: "vscode-webview://test-csp-source", }, visible: true, + show: vi.fn(), onDidDispose: vi.fn().mockImplementation(() => { return { dispose: vi.fn() } }), @@ -1337,6 +1338,79 @@ describe("ClineProvider", () => { }) } + test.each([false, true])( + "explicit retry renews a timed-out sidebar incident after ambiguous reopen: %s", + async (reopened) => { + await provider.resolveWebviewView(mockWebviewView) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + let choose: (value: string) => void = () => { + throw new Error("No warning") + } + vi.mocked(vscode.window.showWarningMessage).mockImplementation( + vi.fn().mockReturnValue( + new Promise((resolve) => { + choose = resolve + }), + ), + ) + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(140_000) + expect(provider["pendingSidebarRecovery"]).toBe(true) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + if (reopened) { + await provider.resolveWebviewView({ + ...mockWebviewView, + webview: { ...mockWebviewView.webview, html: "" }, + }) + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(provider.getCurrentTask()).toBe(task) + } + const healthy = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) await provider.resolveWebviewView(healthy) + }) + + choose("Retry Interface Recovery") + await vi.advanceTimersByTimeAsync(50) + + expect(provider["view"]).toBe(healthy) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(provider["webviewRecoveryRequestGeneration"]).toBe(1) + expect(provider["webviewRecoveryFailureReported"]).toBe(false) + expect(provider["webviewRecoveryStopLogged"]).toBe(false) + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(provider.getCurrentTask()).toBe(task) + expect(remove).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") + }, + ) + + test.each(["focus failure", "replacement timeout"] as const)( + "explicit retry reports %s without an automatic retry loop", + async (failure) => { + await provider.resolveWebviewView(mockWebviewView) + removeSidebarOnCommand() + await vi.advanceTimersByTimeAsync(140_000) + if (failure === "focus failure") + vi.mocked(vscode.commands.executeCommand).mockRejectedValue(new Error("focus unavailable")) + const retry = provider.retryWebviewRecovery() + await vi.advanceTimersByTimeAsync(10_000) + await retry + expect(provider["webviewRecoveryAttempts"]).toBe(3) + expect(provider["pendingSidebarRecovery"]).toBe(failure === "replacement timeout") + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(120_000) + expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(2) + }, + ) + test("warns once after ten seconds without a replacement and releases recovery ownership", async () => { await provider.resolveWebviewView(mockWebviewView) removeSidebarOnCommand() @@ -1722,6 +1796,177 @@ describe("ClineProvider", () => { }, ) + test.each(["Retry Interface Recovery", "Reload Window", "Show Logs"] as const)( + "an old notification action cannot restart a renewed generation: %s", + async (choice) => { + await provider.resolveWebviewView(mockWebviewView) + let choose: (value: string) => void = () => { + throw new Error("No warning") + } + vi.mocked(vscode.window.showWarningMessage).mockImplementation( + vi.fn().mockReturnValue( + new Promise((resolve) => { + choose = resolve + }), + ), + ) + provider["reportWebviewRecoveryFailure"]("Old incident") + const healthy = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) await provider.resolveWebviewView(healthy) + }) + await provider.retryWebviewRecovery() + choose(choice) + await vi.advanceTimersByTimeAsync(0) + expect(provider["webviewRecoveryRequestGeneration"]).toBe(1) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(vscode.commands.executeCommand).not.toHaveBeenCalledWith("workbench.action.reloadWindow") + expect(mockOutputChannel.show).toHaveBeenCalledTimes(choice === "Show Logs" ? 1 : 0) + }, + ) + + test.each(["disposed", "stale notification"] as const)("does not renew recovery after %s", async (event) => { + await provider.resolveWebviewView(mockWebviewView) + if (event === "disposed") await provider.dispose() + else provider["webviewRecoveryRequestGeneration"] = 1 + vi.mocked(vscode.commands.executeCommand).mockClear() + await provider.retryWebviewRecovery(0) + expect(provider["webviewRecoveryRequestGeneration"]).toBe(event === "disposed" ? 0 : 1) + expect(vscode.commands.executeCommand).not.toHaveBeenCalled() + }) + + test("an old focus rejection cannot release or stop a newer explicit retry", async () => { + await provider.resolveWebviewView(mockWebviewView) + let rejectOldFocus: (error: Error) => void = () => { + throw new Error("Old focus has not started") + } + let finishNewFocus: () => void = () => { + throw new Error("New focus has not started") + } + let focusCount = 0 + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) { + focusCount++ + if (focusCount === 1) + await new Promise((_resolve, reject) => { + rejectOldFocus = reject + }) + else + await new Promise((resolve) => { + finishNewFocus = resolve + }) + } + }) + const old = provider["recreateWebviewHostForRecovery"](mockWebviewView) + await vi.advanceTimersByTimeAsync(0) + const retry = provider.retryWebviewRecovery() + await vi.advanceTimersByTimeAsync(0) + rejectOldFocus(new Error("obsolete focus rejection")) + await old + expect(provider["webviewHostRecoveryInProgress"]).toBe(true) + expect(provider["pendingSidebarRecovery"]).toBe(true) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + await provider.resolveWebviewView({ ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } }) + finishNewFocus() + await retry + expect(provider["webviewHostRecoveryInProgress"]).toBe(false) + }) + + test.each(["resolve", "reject"] as const)( + "ignores old initialization that completes on the same view after explicit retry: %s", + async (completion) => { + let finishHtml: (html: string) => void = () => { + throw new Error("HTML has not started") + } + let rejectHtml: (error: Error) => void = () => { + throw new Error("HTML has not started") + } + provider["getWebviewHtml"] = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve, reject) => { + finishHtml = resolve + rejectHtml = reject + }), + ) + .mockResolvedValue("fresh-html") + const old = provider.resolveWebviewView(mockWebviewView) + let finishDiscovery: (commands: string[]) => void = () => { + throw new Error("Discovery has not started") + } + vi.mocked(vscode.commands.getCommands).mockImplementationOnce( + () => + new Promise((resolve) => { + finishDiscovery = resolve + }), + ) + const retry = provider.retryWebviewRecovery() + await vi.advanceTimersByTimeAsync(0) + if (completion === "resolve") finishHtml("obsolete-html") + else rejectHtml(new Error("obsolete initialization failure")) + await old + expect(mockWebviewView.webview.html).toBe("") + expect(mockWebviewView.webview.onDidReceiveMessage).not.toHaveBeenCalled() + expect(vscode.window.showWarningMessage).not.toHaveBeenCalled() + const healthy = { ...mockWebviewView, webview: { ...mockWebviewView.webview, html: "" } } + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) await provider.resolveWebviewView(healthy) + }) + finishDiscovery([`${ClineProvider.sideBarId}.removeView`, `${ClineProvider.sideBarId}.focus`]) + await retry + expect(provider["view"]).toBe(healthy) + expect(healthy.webview.html).toBe("fresh-html") + }, + ) + + test("old hydration cannot change terminal settings after explicit retry renews the same view", async () => { + await provider.resolveWebviewView(mockWebviewView) + const state = await provider.getState() + provider["getWebviewHtml"] = vi.fn().mockResolvedValue("fresh-html") + let finishHydration: (value: typeof state) => void = () => { + throw new Error("Hydration has not started") + } + vi.spyOn(provider, "getState").mockImplementationOnce( + () => + new Promise((resolve) => { + finishHydration = resolve + }), + ) + const old = provider.resolveWebviewView(mockWebviewView) + await vi.advanceTimersByTimeAsync(0) + let finishDiscovery: (commands: string[]) => void = () => { + throw new Error("Discovery has not started") + } + vi.mocked(vscode.commands.getCommands).mockImplementationOnce( + () => + new Promise((resolve) => { + finishDiscovery = resolve + }), + ) + const retry = provider.retryWebviewRecovery() + await vi.advanceTimersByTimeAsync(0) + const terminal = vi.spyOn(Terminal, "setTerminalProfile") + finishHydration({ ...state, terminalProfile: "obsolete-profile" }) + await old + expect(terminal).not.toHaveBeenCalled() + vi.mocked(vscode.commands.executeCommand).mockImplementation(async (command) => { + if (command === `${ClineProvider.sideBarId}.removeView`) disposeCallback() + if (command === `${ClineProvider.sideBarId}.focus`) + await provider.resolveWebviewView({ + ...mockWebviewView, + webview: { ...mockWebviewView.webview, html: "" }, + }) + }) + finishDiscovery([`${ClineProvider.sideBarId}.removeView`, `${ClineProvider.sideBarId}.focus`]) + await retry + expect(terminal).not.toHaveBeenCalledWith("obsolete-profile") + }) + test.each(["Reload Window", "Show Logs"] as const)( "runs only the selected recovery action: %s", async (action) => { @@ -2487,6 +2732,7 @@ describe("ClineProvider", () => { return { dispose: vi.fn() } }), dispose: vi.fn(), + reveal: vi.fn(), } as unknown as vscode.WebviewPanel }) @@ -2558,6 +2804,24 @@ describe("ClineProvider", () => { }, ) + test("explicit retry replaces an exhausted tab without evicting its interrupted task", async () => { + await provider.resolveWebviewView(mockWebviewPanel) + const task = new Task(defaultTaskOptions) + task.abort = true + provider["taskRegistry"].push(task) + const remove = vi.spyOn(provider, "removeClineFromStack") + provider["reportWebviewRecoveryFailure"]("Exhausted incident") + const healthy = replacementPanel() + vi.mocked(vscode.window.createWebviewPanel).mockReturnValue(healthy) + await provider.retryWebviewRecovery() + expect(provider["view"]).toBe(healthy) + expect(provider["webviewRecoveryAttempts"]).toBe(1) + expect(provider.getCurrentTask()).toBe(task) + expect(remove).not.toHaveBeenCalled() + expect(task.dispose).not.toHaveBeenCalled() + expect(task.abortTask).not.toHaveBeenCalled() + }) + test("keeps the old tab and its task when replacement resolution fails before publication", async () => { await provider.resolveWebviewView(mockWebviewPanel) const task = new Task(defaultTaskOptions) @@ -2619,6 +2883,7 @@ describe("ClineProvider", () => { expect(vscode.window.showWarningMessage).toHaveBeenCalledTimes(1) expect(vscode.window.showWarningMessage).toHaveBeenCalledWith( expect.stringContaining("interface is not responding"), + "Retry Interface Recovery", "Reload Window", "Show Logs", ) diff --git a/src/core/webview/webviewRecoveryPolicy.ts b/src/core/webview/webviewRecoveryPolicy.ts index 070c294e98..f4e1817b01 100644 --- a/src/core/webview/webviewRecoveryPolicy.ts +++ b/src/core/webview/webviewRecoveryPolicy.ts @@ -73,7 +73,23 @@ export function recoveryWarningAction( choice: string | undefined, disposed: boolean, sameHeartbeat: boolean, -): "reload" | "logs" | "none" { + sameGeneration = true, +): "reload" | "retry" | "logs" | "none" { if (choice === "Show Logs") return "logs" - return choice === "Reload Window" && !disposed && sameHeartbeat ? "reload" : "none" + if (disposed || !sameHeartbeat || !sameGeneration) return "none" + if (choice === "Retry Interface Recovery") return "retry" + return choice === "Reload Window" ? "reload" : "none" +} + +export function isRecoveryGenerationCurrent(captured: number, current: number): boolean { + return captured === current +} + +export function beginExplicitRecovery(generation: number): { + generation: number + attempts: number + reported: boolean + stopLogged: boolean +} { + return { generation: generation + 1, attempts: 1, reported: false, stopLogged: false } } diff --git a/src/package.json b/src/package.json index c1f81f00cb..837750cec6 100644 --- a/src/package.json +++ b/src/package.json @@ -160,6 +160,11 @@ "title": "%command.acceptInput.title%", "category": "%configuration.title%" }, + { + "command": "zoo-code.retryWebviewRecovery", + "title": "%command.retryWebviewRecovery.title%", + "category": "%configuration.title%" + }, { "command": "zoo-code.showRipgrepDiagnostic", "title": "%command.showRipgrepDiagnostic.title%", diff --git a/src/package.nls.ca.json b/src/package.nls.ca.json index b1397adf71..bc13700441 100644 --- a/src/package.nls.ca.json +++ b/src/package.nls.ca.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Millorar Codi", "command.addToContext.title": "Afegir al Context", "command.openInNewTab.title": "Obrir en una Nova Pestanya", + "command.retryWebviewRecovery.title": "Torna a provar la recuperació de la interfície", "command.focusInput.title": "Enfocar Camp d'Entrada", "command.setCustomStoragePath.title": "Establir Ruta d'Emmagatzematge Personalitzada", "command.importSettings.title": "Importar Configuració", diff --git a/src/package.nls.de.json b/src/package.nls.de.json index b4cd8440ab..a3c7f8164b 100644 --- a/src/package.nls.de.json +++ b/src/package.nls.de.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Code Verbessern", "command.addToContext.title": "Zum Kontext Hinzufügen", "command.openInNewTab.title": "In Neuem Tab Öffnen", + "command.retryWebviewRecovery.title": "Wiederherstellung der Oberfläche erneut versuchen", "command.focusInput.title": "Eingabefeld Fokussieren", "command.setCustomStoragePath.title": "Benutzerdefinierten Speicherpfad Festlegen", "command.importSettings.title": "Einstellungen Importieren", diff --git a/src/package.nls.es.json b/src/package.nls.es.json index 1d86a70980..0ab9f4041c 100644 --- a/src/package.nls.es.json +++ b/src/package.nls.es.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Mejorar Código", "command.addToContext.title": "Añadir al Contexto", "command.openInNewTab.title": "Abrir en Nueva Pestaña", + "command.retryWebviewRecovery.title": "Reintentar la recuperación de la interfaz", "command.focusInput.title": "Enfocar Campo de Entrada", "command.setCustomStoragePath.title": "Establecer Ruta de Almacenamiento Personalizada", "command.importSettings.title": "Importar Configuración", diff --git a/src/package.nls.fr.json b/src/package.nls.fr.json index 85207df975..40daa050b2 100644 --- a/src/package.nls.fr.json +++ b/src/package.nls.fr.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Améliorer le Code", "command.addToContext.title": "Ajouter au Contexte", "command.openInNewTab.title": "Ouvrir dans un Nouvel Onglet", + "command.retryWebviewRecovery.title": "Réessayer la récupération de l'interface", "command.focusInput.title": "Focus sur le Champ de Saisie", "command.setCustomStoragePath.title": "Définir le Chemin de Stockage Personnalisé", "command.importSettings.title": "Importer les Paramètres", diff --git a/src/package.nls.hi.json b/src/package.nls.hi.json index f4f283bb65..8b2190f163 100644 --- a/src/package.nls.hi.json +++ b/src/package.nls.hi.json @@ -7,6 +7,7 @@ "command.improveCode.title": "कोड सुधारें", "command.addToContext.title": "संदर्भ में जोड़ें", "command.openInNewTab.title": "नए टैब में खोलें", + "command.retryWebviewRecovery.title": "इंटरफ़ेस पुनर्प्राप्ति का फिर से प्रयास करें", "command.focusInput.title": "इनपुट फ़ील्ड पर फोकस करें", "command.setCustomStoragePath.title": "कस्टम स्टोरेज पाथ सेट करें", "command.importSettings.title": "सेटिंग्स इम्पोर्ट करें", diff --git a/src/package.nls.id.json b/src/package.nls.id.json index c49e54a34e..3e25eb66b2 100644 --- a/src/package.nls.id.json +++ b/src/package.nls.id.json @@ -16,6 +16,7 @@ "command.fixCode.title": "Perbaiki Kode", "command.improveCode.title": "Tingkatkan Kode", "command.addToContext.title": "Tambahkan ke Konteks", + "command.retryWebviewRecovery.title": "Coba Lagi Pemulihan Antarmuka", "command.focusInput.title": "Fokus ke Field Input", "command.setCustomStoragePath.title": "Atur Path Penyimpanan Kustom", "command.importSettings.title": "Impor Pengaturan", diff --git a/src/package.nls.it.json b/src/package.nls.it.json index 2a4b95f954..5a2b2a4f5d 100644 --- a/src/package.nls.it.json +++ b/src/package.nls.it.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Migliora Codice", "command.addToContext.title": "Aggiungi al Contesto", "command.openInNewTab.title": "Apri in Nuova Scheda", + "command.retryWebviewRecovery.title": "Riprova il ripristino dell'interfaccia", "command.focusInput.title": "Focalizza Campo di Input", "command.setCustomStoragePath.title": "Imposta Percorso di Archiviazione Personalizzato", "command.importSettings.title": "Importa Impostazioni", diff --git a/src/package.nls.ja.json b/src/package.nls.ja.json index 0f3f4a3cee..cf0f25931d 100644 --- a/src/package.nls.ja.json +++ b/src/package.nls.ja.json @@ -16,6 +16,7 @@ "command.fixCode.title": "コードの修正", "command.improveCode.title": "コードの改善", "command.addToContext.title": "コンテキストに追加", + "command.retryWebviewRecovery.title": "インターフェイスの復旧を再試行", "command.focusInput.title": "入力フィールドにフォーカス", "command.setCustomStoragePath.title": "カスタムストレージパスの設定", "command.importSettings.title": "設定をインポート", diff --git a/src/package.nls.json b/src/package.nls.json index fe10839819..fae0d9747d 100644 --- a/src/package.nls.json +++ b/src/package.nls.json @@ -16,6 +16,7 @@ "command.fixCode.title": "Fix Code", "command.improveCode.title": "Improve Code", "command.addToContext.title": "Add To Context", + "command.retryWebviewRecovery.title": "Retry Interface Recovery", "command.focusInput.title": "Focus Input Field", "command.setCustomStoragePath.title": "Set Custom Storage Path", "command.importSettings.title": "Import Settings", diff --git a/src/package.nls.ko.json b/src/package.nls.ko.json index 99a064cd6c..e3a6f7c3ec 100644 --- a/src/package.nls.ko.json +++ b/src/package.nls.ko.json @@ -7,6 +7,7 @@ "command.improveCode.title": "코드 개선", "command.addToContext.title": "컨텍스트에 추가", "command.openInNewTab.title": "새 탭에서 열기", + "command.retryWebviewRecovery.title": "인터페이스 복구 다시 시도", "command.focusInput.title": "입력 필드 포커스", "command.setCustomStoragePath.title": "사용자 지정 저장소 경로 설정", "command.importSettings.title": "설정 가져오기", diff --git a/src/package.nls.nl.json b/src/package.nls.nl.json index 27948f933a..9ae74ade46 100644 --- a/src/package.nls.nl.json +++ b/src/package.nls.nl.json @@ -16,6 +16,7 @@ "command.fixCode.title": "Repareer Code", "command.improveCode.title": "Verbeter Code", "command.addToContext.title": "Toevoegen aan Context", + "command.retryWebviewRecovery.title": "Interfaceherstel opnieuw proberen", "command.focusInput.title": "Focus op Invoerveld", "command.setCustomStoragePath.title": "Aangepast Opslagpad Instellen", "command.importSettings.title": "Instellingen Importeren", diff --git a/src/package.nls.pl.json b/src/package.nls.pl.json index 8fb8b6eb83..adf206c347 100644 --- a/src/package.nls.pl.json +++ b/src/package.nls.pl.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Ulepsz Kod", "command.addToContext.title": "Dodaj do Kontekstu", "command.openInNewTab.title": "Otwórz w Nowej Karcie", + "command.retryWebviewRecovery.title": "Ponów odzyskiwanie interfejsu", "command.focusInput.title": "Fokus na Pole Wprowadzania", "command.setCustomStoragePath.title": "Ustaw Niestandardową Ścieżkę Przechowywania", "command.importSettings.title": "Importuj Ustawienia", diff --git a/src/package.nls.pt-BR.json b/src/package.nls.pt-BR.json index b4cae4644b..30d41068dc 100644 --- a/src/package.nls.pt-BR.json +++ b/src/package.nls.pt-BR.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Melhorar Código", "command.addToContext.title": "Adicionar ao Contexto", "command.openInNewTab.title": "Abrir em Nova Aba", + "command.retryWebviewRecovery.title": "Tentar recuperar a interface novamente", "command.focusInput.title": "Focar Campo de Entrada", "command.setCustomStoragePath.title": "Definir Caminho de Armazenamento Personalizado", "command.importSettings.title": "Importar Configurações", diff --git a/src/package.nls.ru.json b/src/package.nls.ru.json index 0e61002848..eda4ef6955 100644 --- a/src/package.nls.ru.json +++ b/src/package.nls.ru.json @@ -16,6 +16,7 @@ "command.fixCode.title": "Исправить код", "command.improveCode.title": "Улучшить код", "command.addToContext.title": "Добавить в контекст", + "command.retryWebviewRecovery.title": "Повторить восстановление интерфейса", "command.focusInput.title": "Фокус на поле ввода", "command.setCustomStoragePath.title": "Указать путь хранения", "command.importSettings.title": "Импортировать настройки", diff --git a/src/package.nls.tr.json b/src/package.nls.tr.json index 1842156d75..3a5901adc9 100644 --- a/src/package.nls.tr.json +++ b/src/package.nls.tr.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Kodu İyileştir", "command.addToContext.title": "Bağlama Ekle", "command.openInNewTab.title": "Yeni Sekmede Aç", + "command.retryWebviewRecovery.title": "Arayüz Kurtarmayı Yeniden Dene", "command.focusInput.title": "Giriş Alanına Odaklan", "command.setCustomStoragePath.title": "Özel Depolama Yolunu Ayarla", "command.importSettings.title": "Ayarları İçe Aktar", diff --git a/src/package.nls.vi.json b/src/package.nls.vi.json index c846978793..6efff8f31d 100644 --- a/src/package.nls.vi.json +++ b/src/package.nls.vi.json @@ -7,6 +7,7 @@ "command.improveCode.title": "Cải Thiện Mã", "command.addToContext.title": "Thêm vào Ngữ Cảnh", "command.openInNewTab.title": "Mở trong Tab Mới", + "command.retryWebviewRecovery.title": "Thử khôi phục giao diện lại", "command.focusInput.title": "Tập Trung vào Trường Nhập", "command.setCustomStoragePath.title": "Đặt Đường Dẫn Lưu Trữ Tùy Chỉnh", "command.importSettings.title": "Nhập Cài Đặt", diff --git a/src/package.nls.zh-CN.json b/src/package.nls.zh-CN.json index e2705e9e2f..6bd831fe84 100644 --- a/src/package.nls.zh-CN.json +++ b/src/package.nls.zh-CN.json @@ -7,6 +7,7 @@ "command.improveCode.title": "改进代码", "command.addToContext.title": "添加到上下文", "command.openInNewTab.title": "在新标签页中打开", + "command.retryWebviewRecovery.title": "重试界面恢复", "command.focusInput.title": "聚焦输入框", "command.setCustomStoragePath.title": "设置自定义存储路径", "command.importSettings.title": "导入设置", diff --git a/src/package.nls.zh-TW.json b/src/package.nls.zh-TW.json index 723d8d08ad..4ca2dd4f1e 100644 --- a/src/package.nls.zh-TW.json +++ b/src/package.nls.zh-TW.json @@ -7,6 +7,7 @@ "command.improveCode.title": "改進程式碼", "command.addToContext.title": "新增到上下文", "command.openInNewTab.title": "在新分頁中開啟", + "command.retryWebviewRecovery.title": "重試介面復原", "command.focusInput.title": "聚焦輸入框", "command.setCustomStoragePath.title": "設定自訂儲存路徑", "command.importSettings.title": "匯入設定",