diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index fc6b5f911..562b51dc5 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -46,6 +46,10 @@ interface Window { meta: import("../src/native/contracts").CompositorSharedFrameMeta, ) => void, ) => () => void; + reportRecordingOutcome?: ( + outcome: import("../src/lib/recordingSession").RecordingOutcome, + ) => Promise<{ success: boolean }>; + getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 75c3a23df..bed32c8b7 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -18,6 +18,7 @@ import { shell, systemPreferences, } from "electron"; +import { z } from "zod"; import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, @@ -88,6 +89,7 @@ import { McpController } from "../mcp/mcp-controller"; import { McpSettingsStore } from "../mcp/mcp-settings-store"; import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools"; import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server"; +import { RecordingCompletion } from "../mcp/recording-completion"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, @@ -1985,15 +1987,17 @@ export function registerIpcHandlers( ) { /** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */ let appRecording = false; + const recordingCompletion = new RecordingCompletion(); /** * Every start and stop of a take goes through here: the browser path's `set-recording-state` * and the native paths, which start and stop in this process. The MCP recorder waits on it * for a client's take to start, and the tray shows it. A native path that called * `onRecordingStateChange` itself left the MCP recorder waiting on a take already running. */ - const setAppRecording = (recording: boolean, sourceName: string) => { + const setAppRecording = (recording: boolean, sourceName: string, recordingId?: number) => { + if (recording && recordingId !== undefined) recordingCompletion.started(recordingId); appRecording = recording; - mcpAppRecorder.recordingChanged(recording); + mcpAppRecorder.recordingChanged(recording, recordingId); onRecordingStateChange?.(recording, sourceName); }; const appSettings = new AppSettingsStore(app.getPath("userData")); @@ -2782,7 +2786,7 @@ export function registerIpcHandlers( // "Screen" }`, so the tray confidently displayed the name of a // window the capture had never been told about. linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind); - setAppRecording(true, linuxNativeCaptureSourceLabel); + setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId); return { success: true, recordingId, path: outputPath }; } catch (error) { @@ -3078,7 +3082,7 @@ export function registerIpcHandlers( }); const source = selectedSource || { name: "Screen" }; - setAppRecording(true, source.name); + setAppRecording(true, source.name, recordingId); // Reported at start, not at stop: the helper decides the camera is a // lost cause during its own init — before it announces "Recording @@ -3350,7 +3354,7 @@ export function registerIpcHandlers( : 0; const source = selectedSource || { name: "Screen" }; - setAppRecording(true, source.name); + setAppRecording(true, source.name, recordingId); return { success: true, @@ -4154,7 +4158,7 @@ export function registerIpcHandlers( } const source = selectedSource || { name: "Screen" }; - setAppRecording(recording, source.name); + setAppRecording(recording, source.name, recordingId); }, ); @@ -4896,10 +4900,46 @@ export function registerIpcHandlers( setCurrentRecordingSessionState(normalizedSession); currentVideoPath = normalizedSession?.screenVideoPath ?? null; currentProjectPath = null; - if (normalizedSession) mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath); + if (normalizedSession) { + mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath); + // Every capture path stamps the session's `createdAt` with its recordingId (the + // `Date.now()` it started at), so it names the take `recordingCompletion` tracks. + // Any other session (an opened file) carries a fresh `Date.now()` and matches none. + recordingCompletion.saved(normalizedSession.createdAt, normalizedSession.screenVideoPath); + } return { success: true, session: currentRecordingSession }; }); + // A renderer acknowledgement of the actual hand-off, after the duration and + // automatic zoom saves. An imported/opened project never emits this signal. + const recordingOutcomeSchema = z.discriminatedUnion("status", [ + z.object({ + recordingId: z.number().finite(), + status: z.literal("ready"), + projectId: z.string(), + }), + z.object({ + recordingId: z.number().finite(), + status: z.enum(["failed", "cancelled"]), + message: z.string(), + }), + ]); + ipcMain.handle("recording-outcome", async (_, input: unknown) => { + const parsed = recordingOutcomeSchema.safeParse(input); + if (!parsed.success) return { success: false }; + const outcome = parsed.data; + if (outcome.status !== "ready") { + recordingCompletion.failed(outcome.recordingId, outcome.message); + mcpAppRecorder.recordingFinished(outcome.recordingId); + return { success: true }; + } + await recordingCompletion.projectReady(outcome.recordingId, outcome.projectId, (id) => + aiEditionDocuments.getProject(id), + ); + mcpAppRecorder.recordingFinished(outcome.recordingId); + return { success: true }; + }); + ipcMain.handle("get-current-recording-session", () => { if (!currentRecordingSession) { return { success: false }; @@ -5183,6 +5223,8 @@ export function registerIpcHandlers( host: mcpToolDeps.host, }); const mcpMedia = createMediaTools({ + recordingActive: () => recordingCompletion.isActive(), + waitForRecording: (timeoutMs, signal) => recordingCompletion.wait(timeoutMs, signal), transport: "http", // Recording and exporting change things too: the same switch as edits. editsAllowed: mcpToolDeps.editsAllowed, diff --git a/electron/ipc/recordingStateRouting.test.ts b/electron/ipc/recordingStateRouting.test.ts index 13c7193f1..96851dec7 100644 --- a/electron/ipc/recordingStateRouting.test.ts +++ b/electron/ipc/recordingStateRouting.test.ts @@ -36,7 +36,7 @@ describe("recording state routing", () => { const setter = source.slice(source.indexOf("const setAppRecording")); const body = setter.slice(0, setter.indexOf("};")); expect(body).toContain("appRecording = recording"); - expect(body).toContain("mcpAppRecorder.recordingChanged(recording)"); + expect(body).toContain("mcpAppRecorder.recordingChanged(recording, recordingId)"); expect(body).toContain("onRecordingStateChange?.(recording, sourceName)"); }); diff --git a/electron/mcp/app-recorder.test.ts b/electron/mcp/app-recorder.test.ts index 78b76be4e..78ebcb838 100644 --- a/electron/mcp/app-recorder.test.ts +++ b/electron/mcp/app-recorder.test.ts @@ -157,6 +157,20 @@ describe("an MCP take in the app", () => { expect(editor.applied).toEqual([]); }); + it("releases only the client take whose app lifecycle finished", async () => { + const { recorder, calls } = setup(); + const starting = recorder.start(REQUEST); + await new Promise((resolve) => setTimeout(resolve, 5)); + recorder.recordingChanged(true, 7); + await starting; + recorder.recordingFinished(6); + await expect(recorder.start(REQUEST)).rejects.toThrow("already recording"); + recorder.recordingChanged(false, 7); + recorder.recordingFinished(7); + expect(calls.filter((call) => call === "restore")).toHaveLength(1); + await expect(started(recorder, calls)).resolves.toBeDefined(); + }); + it("refuses a take while one records, or while the editor holds unsaved changes", async () => { const busy = setup({ isRecording: () => true }); await expect(busy.recorder.start(REQUEST)).rejects.toThrow("already recording"); diff --git a/electron/mcp/app-recorder.ts b/electron/mcp/app-recorder.ts index 6f2134147..dd4297f51 100644 --- a/electron/mcp/app-recorder.ts +++ b/electron/mcp/app-recorder.ts @@ -103,7 +103,9 @@ export interface AppRecorderDeps { export interface AppRecorder extends TakeRecorder { /** The app's recording state changed (`set-recording-state`). */ - recordingChanged(recording: boolean): void; + recordingChanged(recording: boolean, recordingId?: number): void; + /** A completed or failed app take no longer owns the recording bar. */ + recordingFinished(recordingId: number): void; /** The recording bar handed a saved take to the editor (`set-current-recording-session`). */ takeSaved(videoPath: string): void; } @@ -145,6 +147,7 @@ interface Deferred { } interface ClientTake { + recordingId?: number; phase: "starting" | "recording" | "saving"; /** Resolves with when capture started. */ started: Deferred; @@ -214,10 +217,16 @@ export function createAppRecorder(deps: AppRecorderDeps): AppRecorder { } return { - recordingChanged(recording) { + recordingFinished(recordingId) { + if (current?.recordingId !== recordingId) return; + current.restore(); + current = null; + }, + recordingChanged(recording, recordingId) { if (!current) return; if (recording && current.phase === "starting") { current.phase = "recording"; + current.recordingId = recordingId; current.started.resolve(Date.now()); } else if (!recording && current.phase === "recording") { // The take is captured: the bar's own settings can come back while it saves. diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index 477df437e..e2467b8e9 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -26,6 +26,7 @@ import { createToolRunner, type McpToolDeps, } from "./openscreen-mcp-server"; +import { RecordingCompletion } from "./recording-completion"; /** The runner windows: records what was asked, answers like a runner would. */ class FakeRunners { @@ -1069,3 +1070,174 @@ describe("the media tools in the app", () => { expect(exportPath.description).not.toContain("working directory, ending"); }); }); + +describe("waitForRecording through MCP", () => { + const clients: Client[] = []; + afterEach(async () => { + for (const client of clients.splice(0)) await client.close(); + }); + async function observing( + completion: RecordingCompletion, + overrides: Partial = {}, + ) { + const deps: McpToolDeps = { + transport: "http", + host: { snapshot: async () => null, apply: async () => "no-editor" }, + projects: documents, + editsAllowed: () => false, + version: "test", + }; + let subscribed: () => void = () => undefined; + const subscription = new Promise((resolve) => { + subscribed = resolve; + }); + const media = createMediaTools({ + ...stdioDeps, + transport: "http", + recordingActive: () => completion.isActive(), + editsAllowed: () => false, + waitForRecording: (timeout, signal) => { + const waiting = completion.wait(timeout, signal); + subscribed(); + return waiting; + }, + ...overrides, + }); + const server = createOpenScreenMcpServer(deps, createToolRunner(deps), media); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + await server.connect(serverSide); + const observer = new Client({ name: "observer", version: "1" }); + await observer.connect(clientSide); + clients.push(observer); + return { observer, subscription }; + } + + it("returns the stopRecording fields for a user take while MCP edits is off, without recording commands", async () => { + const project = await documents.createProject("User take"); + const completion = new RecordingCompletion(); + completion.started(42, 1_000); + const { observer, subscription } = await observing(completion); + const waiting = observer.callTool({ name: "waitForRecording", arguments: {} }); + await subscription; + completion.saved(42, videoPath); + completion.completed(42, { + projectId: project.project.id, + videoPath, + durationSec: 12, + autoZooms: 2, + }); + const result = await waiting; + expect(result.isError).toBeFalsy(); + expect(JSON.parse(textOf(result as CallToolResult))).toEqual({ + projectId: project.project.id, + videoPath, + durationSec: 12, + autoZooms: 2, + recordingStartedAt: 1_000, + checkpointId: expect.any(String), + }); + expect(runners.started).toEqual([]); + }); + + it("gives setPointerTrack the waited-for take's start, as stopRecording does", async () => { + const project = await documents.createProject("User take"); + const completion = new RecordingCompletion(); + completion.started(42, 1_000); + const sent: unknown[] = []; + const { observer, subscription } = await observing(completion, { + runTool: async (_name, args) => { + sent.push(args); + return { content: [{ type: "text", text: "ok" }] }; + }, + }); + const waiting = observer.callTool({ name: "waitForRecording", arguments: {} }); + await subscription; + completion.saved(42, videoPath); + completion.completed(42, { + projectId: project.project.id, + videoPath, + durationSec: 12, + autoZooms: 0, + }); + await waiting; + await observer.callTool({ + name: "setPointerTrack", + arguments: { projectId: project.project.id, events: [{ at: 1, x: 0.5, y: 0.5 }] }, + }); + expect(sent).toContainEqual(expect.objectContaining({ takes: { [videoPath]: 1_000 } })); + }); + + it("releases a client take after the user finishes it, so startRecording can record again", async () => { + const completion = new RecordingCompletion(); + let id = 0; + const recorder: TakeRecorder = { + start: async () => { + completion.started(++id); + return { + startedAt: id, + stop: async () => { + throw new Error("wait must never stop the take"); + }, + }; + }, + }; + const { observer, subscription } = await observing(completion, { + recorder, + editsAllowed: () => true, + }); + expect( + (await observer.callTool({ name: "startRecording", arguments: {} })).isError, + ).toBeFalsy(); + const waiting = observer.callTool({ name: "waitForRecording", arguments: {} }); + await subscription; + completion.saved(1, videoPath); + completion.completed(1, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 }); + await waiting; + expect( + (await observer.callTool({ name: "startRecording", arguments: {} })).isError, + ).toBeFalsy(); + expect(id).toBe(2); + }); + + it.each([ + 0, + -1, + 1.5, + 3601, + "10", + ])("validates timeoutSec=%s before observing", async (timeoutSec) => { + const completion = new RecordingCompletion(); + const { observer } = await observing(completion); + const result = await observer.callTool({ name: "waitForRecording", arguments: { timeoutSec } }); + expect(result.isError).toBe(true); + expect(textOf(result as CallToolResult)).toContain("validation"); + }); + + it("reports no active recording as a tool error", async () => { + const { observer } = await observing(new RecordingCompletion()); + const result = await observer.callTool({ name: "waitForRecording", arguments: {} }); + expect(result.isError).toBe(true); + expect(textOf(result as CallToolResult)).toContain("No recording"); + }); + + it("propagates MCP cancellation and leaves the user's take available", async () => { + const completion = new RecordingCompletion(); + completion.started(42); + const { observer, subscription } = await observing(completion); + const abort = new AbortController(); + const waiting = expect( + observer.callTool({ name: "waitForRecording", arguments: {} }, undefined, { + signal: abort.signal, + }), + ).rejects.toThrow(); + await subscription; + abort.abort(); + await waiting; + // Flush the MCP cancellation notification, then the same take still completes. + await new Promise((resolve) => setTimeout(resolve, 0)); + const retry = completion.wait(1000, new AbortController().signal); + completion.saved(42, videoPath); + completion.completed(42, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 }); + await expect(retry).resolves.toMatchObject({ projectId: "proj_take" }); + }); +}); diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 9aff90912..5ad8b70a6 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -99,6 +99,14 @@ export interface TakeRecorder { } export interface MediaToolDeps { + /** The app may finish a client take from its HUD, without stopRecording. */ + recordingActive?(): boolean; + /** Observes the app's current take without starting or stopping it. */ + waitForRecording?: ( + timeoutMs: number, + signal: AbortSignal, + ) => Promise; + /** The server these tools are served by: their texts say what is true on it. */ transport: McpTransport; /** Whether a client may record and export: the in-app server's "MCP edits". */ @@ -236,7 +244,7 @@ const TAKE_NOTES: Record = { export function mediaPreamble(transport: McpTransport): string { return [ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.", - "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", + "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. To observe a take the user controls on the in-app HTTP server, call waitForRecording while it is recording or saving; the user stops it. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", "To edit a video you already have instead, createProject with its videoPath; importMedia adds a video or an audio file to a project.", "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on the clicks as the editor does: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.", TAKE_NOTES[transport], @@ -623,7 +631,15 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { } catch (error) { return text(error instanceof Error ? error.message : String(error), true); } - startedAtByVideo.set(recorded.videoPath, running.startedAt); + return recordingResult(recorded, running.startedAt); + } + + /** Also the one place a take's start is kept, so setPointerTrack finds a waited-for take. */ + async function recordingResult( + recorded: RecordedTake, + recordingStartedAt: number, + ): Promise { + startedAtByVideo.set(recorded.videoPath, recordingStartedAt); const { projectId } = recorded; return text( JSON.stringify({ @@ -631,7 +647,7 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { checkpointId: await checkpointOf(projectId), videoPath: recorded.videoPath, durationSec: recorded.durationSec, - recordingStartedAt: running.startedAt, + recordingStartedAt, autoZooms: recorded.autoZooms, ...(recorded.warnings?.length ? { warnings: recorded.warnings } : {}), }), @@ -705,6 +721,7 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { }, async (args) => { if (!deps.editsAllowed()) return text(GATED_MESSAGE, true); + if (take?.running && !take.stopping && deps.recordingActive?.() === false) take = null; if (take?.stopping) { return text("The last take is still being saved. Try again in a moment.", true); } @@ -766,6 +783,43 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { }, ); + server.registerTool( + "waitForRecording", + { + description: + "Wait for the current recording in the OpenScreen app to finish and its project to be saved, including a take started by the user at the recording bar. Never starts or stops a recording. Binds to this take only, including while it saves; returns an error immediately if none is active. Returns the same recording fields as stopRecording. Timeout or request cancellation only ends the wait. Available on the in-app HTTP server; openscreen mcp cannot observe the app's recording bar.", + inputSchema: z.object({ + timeoutSec: z + .number() + .int() + .min(1) + .max(3600) + .default(300) + .describe( + "Maximum seconds to wait, including saving (1–3600, default 300). Set the client's tool-call timeout longer than this.", + ), + }), + annotations: READ_ONLY, + }, + async ({ timeoutSec }, { signal }) => { + if (!deps.waitForRecording) + return text( + "waitForRecording requires the in-app HTTP MCP server. Connect to it to observe the user's recording.", + true, + ); + const observedTake = take; + try { + const recorded = await deps.waitForRecording(timeoutSec * 1000, signal); + return recordingResult(recorded, recorded.recordingStartedAt); + } catch (error) { + return text(error instanceof Error ? error.message : String(error), true); + } finally { + if (take === observedTake && !take?.stopping && deps.recordingActive?.() === false) + take = null; + } + }, + ); + server.registerTool( "setPointerTrack", { diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts index bcdde4b26..9bfd1cf32 100644 --- a/electron/mcp/openscreen-mcp-server.test.ts +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -3,6 +3,7 @@ // DocumentService on a temp directory for the projects that are not open. import { spawnSync } from "node:child_process"; +import { getEventListeners } from "node:events"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import fsPromises from "node:fs/promises"; import os from "node:os"; @@ -18,7 +19,7 @@ import { import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools"; import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service"; import { DocumentService } from "../ai-edition/document-service"; -import { createMediaTools } from "./media-tools"; +import { createMediaTools, type MediaToolDeps } from "./media-tools"; import { LIST_PROJECTS_TOOL, MCP_ENDPOINT_PATH, @@ -29,6 +30,7 @@ import { type RunningMcpServer, startMcpHttpServer, } from "./openscreen-mcp-server"; +import { RecordingCompletion } from "./recording-completion"; const TOKEN = "test-token-0123456789"; @@ -146,6 +148,7 @@ async function connect( editsAllowed?: boolean; headers?: Record; media?: Pick; + waitForRecording?: MediaToolDeps["waitForRecording"]; } = {}, ) { const editsAllowed = () => options.editsAllowed ?? true; @@ -166,6 +169,7 @@ async function connect( deps, // As the app serves them; media-tools.test.ts calls them. media: createMediaTools({ + waitForRecording: options.waitForRecording, transport: "http", editsAllowed, recorder: { start: unused }, @@ -207,6 +211,7 @@ describe("the MCP tool surface", () => { "listSources", "startRecording", "stopRecording", + "waitForRecording", "setPointerTrack", "exportVideo", "exportAudio", @@ -990,3 +995,61 @@ describe("the HTTP guard", () => { expect((await post({ Authorization: `Bearer ${TOKEN}` }, "/other")).status).toBe(404); }); }); + +describe("HTTP recording waits", () => { + it("drops the wait's listener when its HTTP request disconnects, leaving the take alive", async () => { + const completion = new RecordingCompletion(); + completion.started(7); + let requestSignal: AbortSignal | undefined; + let subscribed: () => void = () => undefined; + const subscription = new Promise((resolve) => { + subscribed = resolve; + }); + await connect(new FakeEditor(), { + waitForRecording: (timeout, signal) => { + requestSignal = signal; + const waiting = completion.wait(timeout, signal); + subscribed(); + return waiting; + }, + }); + const controller = new AbortController(); + const request = fetch(`http://127.0.0.1:${running?.port}${MCP_ENDPOINT_PATH}`, { + method: "POST", + headers: { + Authorization: `Bearer ${TOKEN}`, + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 77, + method: "tools/call", + params: { name: "waitForRecording", arguments: { timeoutSec: 3600 } }, + }), + signal: controller.signal, + }); + const rejected = expect(request).rejects.toThrow(); + await subscription; + expect(getEventListeners(requestSignal as AbortSignal, "abort")).toHaveLength(1); + controller.abort(); + await rejected; + await vi.waitFor(() => expect(requestSignal?.aborted).toBe(true)); + expect(getEventListeners(requestSignal as AbortSignal, "abort")).toHaveLength(0); + expect(completion.isActive()).toBe(true); + completion.failed(7, "test cleanup"); + }); + + it("returns a tool timeout over HTTP without ending the recording", async () => { + const completion = new RecordingCompletion(); + completion.started(7); + const mcp = await connect(new FakeEditor(), { + waitForRecording: (timeout, signal) => completion.wait(timeout, signal), + }); + const result = await mcp.callTool({ name: "waitForRecording", arguments: { timeoutSec: 1 } }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain("Timed out"); + expect(completion.isActive()).toBe(true); + completion.failed(7, "test cleanup"); + }); +}); diff --git a/electron/mcp/recording-completion.test.ts b/electron/mcp/recording-completion.test.ts new file mode 100644 index 000000000..213d5dc90 --- /dev/null +++ b/electron/mcp/recording-completion.test.ts @@ -0,0 +1,228 @@ +import { getEventListeners } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { replaceTimeline } from "../../src/lib/ai-edition/document/timeline"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { RecordingCompletion } from "./recording-completion"; + +const RESULT = { projectId: "proj_take", videoPath: "/take.mp4", durationSec: 12, autoZooms: 2 }; +let completion: RecordingCompletion; +let abort: AbortController; +beforeEach(() => { + vi.useFakeTimers(); + completion = new RecordingCompletion(); + abort = new AbortController(); +}); +afterEach(() => { + expect(getEventListeners(abort.signal, "abort")).toHaveLength(0); + expect(vi.getTimerCount()).toBe(0); + vi.useRealTimers(); +}); + +describe("waiting on the app's recording", () => { + it("refuses idle calls and never returns an older take", async () => { + await expect(completion.wait(100, abort.signal)).rejects.toThrow("No recording"); + completion.started(1, 123); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + await expect(completion.wait(100, abort.signal)).rejects.toThrow("No recording"); + }); + + it("does not miss completion immediately after subscription, even before the first await", async () => { + completion.started(1, 123); + const waiting = completion.wait(100, abort.signal); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + expect(await waiting).toEqual({ ...RESULT, recordingStartedAt: 123 }); + }); + + it("requires that take's saved media and ignores imported projects and other recordings", async () => { + completion.started(1, 123); + completion.saved(99, "/import.mp4"); + const waiting = completion.wait(100, abort.signal); + completion.completed(1, RESULT); // no saved session yet + completion.saved(1, RESULT.videoPath); + completion.completed(99, RESULT); + completion.completed(1, { ...RESULT, videoPath: "/import.mp4" }); + expect(vi.getTimerCount()).toBe(1); + completion.completed(1, RESULT); + expect(await waiting).toMatchObject({ videoPath: RESULT.videoPath }); + }); + + it("binds late calls during saving and all concurrent waits to the same result", async () => { + completion.started(1, 123); + const first = completion.wait(100, abort.signal); + completion.saved(1, RESULT.videoPath); + const second = completion.wait(100, abort.signal); + completion.completed(1, RESULT); + expect(await first).toEqual(await second); + }); + + it("times out only the wait; a retry can still get the same take", async () => { + completion.started(1, 123); + const waiting = expect(completion.wait(10, abort.signal)).rejects.toThrow("Timed out"); + await vi.advanceTimersByTimeAsync(10); + await waiting; + const retry = completion.wait(100, abort.signal); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + expect(await retry).toMatchObject({ recordingStartedAt: 123 }); + }); + + it("cancels one request without affecting other waiters or the take", async () => { + completion.started(1); + const first = expect(completion.wait(100, abort.signal)).rejects.toThrow("cancelled"); + const other = new AbortController(); + const second = completion.wait(100, other.signal); + abort.abort(); + await first; + expect(getEventListeners(other.signal, "abort")).toHaveLength(1); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + await second; + expect(getEventListeners(other.signal, "abort")).toHaveLength(0); + }); + + it("cleans up a signal already cancelled when the call arrives", async () => { + completion.started(1); + abort.abort(); + await expect(completion.wait(100, abort.signal)).rejects.toThrow("cancelled"); + }); + + it.each([ + "Recording failed: disk full", + "The recording was cancelled or discarded.", + ])("ends all waits on %s", async (message) => { + completion.started(1); + const first = expect(completion.wait(100, abort.signal)).rejects.toThrow(message); + const second = expect(completion.wait(100, abort.signal)).rejects.toThrow(message); + completion.failed(99, "unrelated failure"); + completion.failed(1, message); + await Promise.all([first, second]); + await expect(completion.wait(100, abort.signal)).rejects.toThrow("No recording"); + }); + + it("fails a superseded target instead of waiting for the next take", async () => { + completion.started(1); + const first = expect(completion.wait(100, abort.signal)).rejects.toThrow("replaced"); + completion.started(2, 456); + await first; + const second = completion.wait(100, abort.signal); + completion.saved(1, "/old.mp4"); + completion.failed(1, "late error"); + completion.completed(1, { ...RESULT, videoPath: "/old.mp4" }); + completion.saved(2, RESULT.videoPath); + completion.completed(2, RESULT); + expect(await second).toMatchObject({ recordingStartedAt: 456 }); + }); +}); + +describe("the saved recording result", () => { + let dir: string; + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "recording-ready-")); + }); + afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); + }); + + async function savedProject(videoPath = path.join(dir, "take.mp4"), durationSec = 12) { + const base = createEmptyDocument({ projectId: "proj_saved", title: "Take" }); + const doc = replaceTimeline( + { + ...base, + project: { ...base.project, primaryAssetId: "asset_screen" }, + assets: [ + { + id: "asset_screen", + kind: "video", + label: "Take", + originalPath: videoPath, + durationSec, + cameraTrack: null, + }, + ], + }, + [{ startSec: 0, endSec: durationSec }], + "Recorded", + ); + const projectFile = path.join(dir, "project.openscreen"); + await fs.writeFile(projectFile, JSON.stringify(doc)); + return { + doc, + read: async () => JSON.parse(await fs.readFile(projectFile, "utf8")) as typeof doc, + videoPath, + }; + } + + it("reads the persisted project and verifies its actual media before resolving", async () => { + const { read, videoPath } = await savedProject(); + await fs.writeFile(videoPath, "test media"); + completion.started(1, 123); + completion.saved(1, videoPath); + const waiting = completion.wait(100, abort.signal); + await completion.projectReady(1, "proj_saved", read); + expect(await waiting).toEqual({ + projectId: "proj_saved", + videoPath, + durationSec: 12, + autoZooms: 0, + recordingStartedAt: 123, + }); + }); + + it.each([ + "missing media", + "empty media", + "unmeasured duration", + "different asset", + "unreadable project", + ])("refuses success for %s", async (condition) => { + const { read, videoPath } = await savedProject( + condition === "different asset" ? path.join(dir, "import.mp4") : undefined, + condition === "unmeasured duration" ? 60 : 12, + ); + const target = path.join(dir, "take.mp4"); + if (condition !== "missing media") + await fs.writeFile(videoPath, condition === "empty media" ? "" : "media"); + completion.started(1); + completion.saved(1, target); + const waiting = expect(completion.wait(100, abort.signal)).rejects.toThrow( + "could not be prepared", + ); + await completion.projectReady( + 1, + "proj_saved", + condition === "unreadable project" + ? async () => { + throw new Error("disk failure"); + } + : read, + ); + await waiting; + }); + + it("does not deliver an old result if a new take starts while the project is being read", async () => { + const { read, videoPath } = await savedProject(); + await fs.writeFile(videoPath, "media"); + completion.started(1); + completion.saved(1, videoPath); + const waiting = expect(completion.wait(100, abort.signal)).rejects.toThrow("replaced"); + let release: () => void = () => undefined; + const opening = completion.projectReady(1, "proj_saved", async () => { + await new Promise((resolve) => { + release = resolve; + }); + return read(); + }); + completion.started(2); + await waiting; + release(); + await opening; + const next = expect(completion.wait(1, abort.signal)).rejects.toThrow("Timed out"); + await vi.advanceTimersByTimeAsync(1); + await next; + }); +}); diff --git a/electron/mcp/recording-completion.ts b/electron/mcp/recording-completion.ts new file mode 100644 index 000000000..2f48751ff --- /dev/null +++ b/electron/mcp/recording-completion.ts @@ -0,0 +1,121 @@ +// Tracks the app's actual take, including takes started by the person at the HUD. +// No project-folder watching: only that take's saved session and editor hand-off +// can complete it. Waiters own their deadlines, never the recording's lifetime. +import fs from "node:fs/promises"; +import { clipAwaitsProbedDuration } from "../../src/lib/ai-edition/document/timeline"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import type { RecordedTake } from "./media-tools"; + +export type CompletedRecording = RecordedTake & { recordingStartedAt: number }; +type Outcome = { result: CompletedRecording } | { error: string }; +interface Take { + id: number; + startedAt: number; + videoPath?: string; + listeners: Set<(outcome: Outcome) => void>; +} + +export class RecordingCompletion { + private current: Take | null = null; + + isActive(): boolean { + return this.current !== null; + } + + started(id: number, startedAt = Date.now()): void { + if (this.current?.id === id) return; + if (this.current) this.failed(this.current.id, "The recording was replaced by another take."); + this.current = { id, startedAt, listeners: new Set() }; + } + + saved(id: number, videoPath: string): void { + if (this.current?.id === id) this.current.videoPath = videoPath; + } + + videoPath(id: number): string | undefined { + return this.current?.id === id ? this.current.videoPath : undefined; + } + + completed(id: number, result: RecordedTake): void { + const take = this.current; + if (!take || take.id !== id || take.videoPath !== result.videoPath) return; + this.finish(take, { result: { ...result, recordingStartedAt: take.startedAt } }); + } + + /** The editor has acknowledged its saves. Verify against the persisted project. */ + async projectReady( + id: number, + projectId: string, + readProject: (id: string) => Promise, + ): Promise { + const videoPath = this.videoPath(id); + if (!videoPath) return; + try { + const document = await readProject(projectId); + const asset = document.assets.find((asset) => asset.originalPath === videoPath); + if ( + !asset?.durationSec || + !document.timeline.clips.some((clip) => clip.assetId === asset.id) || + document.timeline.clips.some((clip) => clipAwaitsProbedDuration(clip, asset.id)) + ) { + throw new Error("The recording's project has no measured video clip."); + } + for (const filePath of [videoPath, asset.cameraTrack?.sourcePath]) { + if (!filePath) continue; + const stat = await fs.stat(filePath); + if (!stat.isFile() || stat.size === 0) + throw new Error("A recording media file is empty or missing."); + } + this.completed(id, { + projectId: document.project.id, + videoPath, + durationSec: asset.durationSec, + autoZooms: document.zoomRanges.length, + }); + } catch (error) { + this.failed(id, `The recording could not be prepared: ${String(error)}`); + } + } + + failed(id: number, error: string): void { + if (this.current?.id === id) this.finish(this.current, { error }); + } + + private finish(take: Take, outcome: Outcome): void { + this.current = null; + for (const listener of take.listeners) listener(outcome); + take.listeners.clear(); + } + + wait(timeoutMs: number, signal: AbortSignal): Promise { + const take = this.current; + if (!take) return Promise.reject(new Error("No recording is running or being saved.")); + return new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + take.listeners.delete(finished); + signal.removeEventListener("abort", aborted); + }; + const finished = (outcome: Outcome) => { + cleanup(); + if ("error" in outcome) reject(new Error(outcome.error)); + else resolve(outcome.result); + }; + const aborted = () => + finished({ error: "Waiting for the recording was cancelled. The recording is unchanged." }); + const timer = setTimeout( + () => + finished({ + error: + "Timed out waiting for the recording. The recording is unchanged; call waitForRecording again while it is running or being saved.", + }), + timeoutMs, + ); + // Capture the target and subscribe synchronously, with no await between them. + // Completion cannot fall between a state read and listener registration. + take.listeners.add(finished); + signal.addEventListener("abort", aborted, { once: true }); + if (signal.aborted) aborted(); + }); + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 23160bcaa..f018e8f18 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -236,6 +236,9 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, + reportRecordingOutcome: (outcome: import("../src/lib/recordingSession").RecordingOutcome) => + ipcRenderer.invoke("recording-outcome", outcome), + setRecordingState: ( recording: boolean, recordingId?: number, diff --git a/plugin/skills/openscreen-demo/SKILL.md b/plugin/skills/openscreen-demo/SKILL.md index a741f97d5..5e16b08bc 100644 --- a/plugin/skills/openscreen-demo/SKILL.md +++ b/plugin/skills/openscreen-demo/SKILL.md @@ -70,6 +70,8 @@ Write a storyboard and show it to the user in a few lines, then carry on unless - **Real mouse** (computer-use, or any tool that moves the actual pointer): OpenScreen records the pointer, draws a clean cursor and already zooms on the clicks (see *Zoom on the action* below). If your mouse tool cannot click in the app's window (some only read browsers, for one), drive it from a script instead. - **A script** (Playwright, CDP): the screen pointer never moves, so the take has no cursor and no zoom. In the script, log every action as it runs: `at` is `Date.now()`, `x` and `y` are the action's position as fractions of the window, `click` is true for a click. Hand the log to `setPointerTrack` after `stopRecording`; its description gives the formula from a page's coordinates. A script also gives the cleanest take: same timing on every run, no hesitation. +**The user records the take themselves** from the recording bar: connected to the server in the app, call `waitForRecording` while it records. It never starts or stops anything; it returns once the user has stopped and the editor has saved the take, with the same fields as `stopRecording`. Set `timeoutSec` (default 300, at most 3600) and your client's tool-call timeout longer than the take. A timeout only ends the wait: call it again while the take runs. `openscreen mcp` cannot see the recording bar. + **A video you already have** needs no recording: `createProject` with its absolute `videoPath` lays it out as a project, and returns a `projectId` and a `checkpointId` as `stopRecording` does. Check it as a take in step 5, its `videoPath` included, then go on from step 6. ## 5. Check the take diff --git a/src/components/ai-edition/recordingImport.test.ts b/src/components/ai-edition/recordingImport.test.ts index 4b1a314a6..7f8b00403 100644 --- a/src/components/ai-edition/recordingImport.test.ts +++ b/src/components/ai-edition/recordingImport.test.ts @@ -227,6 +227,52 @@ describe("what the recording import leaves on the undo stack", () => { stubElectronApi(SCREEN_PATH); }); + it("acknowledges the exact HUD take only after its measured duration is saved", async () => { + const reportRecordingOutcome = vi.fn(async () => ({ success: true })); + Object.assign(window.electronAPI, { reportRecordingOutcome }); + await importPendingRecording(); + expect(reportRecordingOutcome).not.toHaveBeenCalled(); + const loaded = useProjectStore.getState().document as AxcutDocument; + const measured = replaceTimelineOp( + { ...loaded, assets: loaded.assets.map((a) => ({ ...a, durationSec: 12 })) }, + [{ startSec: 0, endSec: 12 }], + "Measured", + ); + let release: () => void = () => undefined; + bridge.save.mockImplementationOnce(async (document: unknown) => { + await new Promise((resolve) => { + release = resolve; + }); + return { success: true, document }; + }); + const saving = useProjectStore.getState().saveDocument(measured, { history: false }); + const ready = maybeSaveFreshRecordingAutoZooms(measured, { enabled: false }); + await Promise.resolve(); + expect(reportRecordingOutcome).not.toHaveBeenCalled(); + release(); + await saving; + await ready; + expect(reportRecordingOutcome).toHaveBeenCalledExactlyOnceWith({ + recordingId: 0, + status: "ready", + projectId: PROJECT_ID, + }); + await maybeSaveFreshRecordingAutoZooms(measured); + expect(reportRecordingOutcome).toHaveBeenCalledTimes(1); + }); + + it("reports an import failure against the originating take", async () => { + const reportRecordingOutcome = vi.fn(async () => ({ success: true })); + Object.assign(window.electronAPI, { reportRecordingOutcome }); + bridge.create.mockRejectedValueOnce(new Error("disk full")); + await expect(importPendingRecording()).rejects.toThrow("disk full"); + expect(reportRecordingOutcome).toHaveBeenCalledExactlyOnceWith({ + recordingId: 0, + status: "failed", + message: expect.stringContaining("disk full"), + }); + }); + it("leaves it empty: the user has not edited anything yet", async () => { await importPendingRecording(); diff --git a/src/components/ai-edition/recordingImport.ts b/src/components/ai-edition/recordingImport.ts index b6b3e119a..69b3f421c 100644 --- a/src/components/ai-edition/recordingImport.ts +++ b/src/components/ai-edition/recordingImport.ts @@ -28,6 +28,7 @@ import { appendAutoZoomSuggestions, collectAutoZoomSuggestionsForLatestDocument, } from "@/lib/ai-edition/timeline/apply-auto-zooms"; +import type { RecordingSession } from "@/lib/recordingSession"; import { nativeBridgeClient } from "@/native/client"; // Fresh recordings used to get cursor-dwell zooms on load (legacy editor @@ -48,6 +49,40 @@ let pendingFreshRecordingAutoZoomPath: string | null = null; let appliedFreshRecordingAutoZoomProjectId: string | null = null; let freshRecordingAutoZoomSaveChain: Promise = Promise.resolve(); +// Kept only for the HUD hand-off, never for ordinary media imports or reopened +// projects. Metadata may arrive after importPendingRecording has returned. +let pendingCompletion: { session: RecordingSession; projectId: string | null } | null = null; + +async function reportReadyRecording(): Promise { + const pending = pendingCompletion; + const api = window.electronAPI; + if (!pending || !api?.reportRecordingOutcome) return; + if ((await waitForDocumentSaves()) === "timeout") return; + const state = useProjectStore.getState(); + const document = state.document; + if (!document || state.dirty || document.project.id !== pending.projectId) return; + const asset = document.assets.find( + (asset) => asset.originalPath === pending.session.screenVideoPath, + ); + if ( + !asset?.durationSec || + !document.timeline.clips.some((clip) => clip.assetId === asset.id) || + document.timeline.clips.some((clip) => clipAwaitsProbedDuration(clip, asset.id)) || + pendingFreshRecordingAutoZoom + ) + return; + try { + await api.reportRecordingOutcome({ + recordingId: pending.session.createdAt, + status: "ready", + projectId: document.project.id, + }); + if (pendingCompletion === pending) pendingCompletion = null; + } catch (error) { + console.warn("[recording] could not report saved project:", error); + } +} + export function markFreshRecordingAutoZoomPending(assetPath: string): void { pendingFreshRecordingAutoZoom = true; pendingFreshRecordingAutoZoomPath = assetPath; @@ -311,7 +346,10 @@ export async function maybeSaveFreshRecordingAutoZooms( () => undefined, () => undefined, ); - return done; + return done.then(async (saved) => { + await reportReadyRecording(); + return saved; + }); } /** @@ -336,53 +374,74 @@ export async function importPendingRecording( const result = await api.getCurrentRecordingSession(); const screenPath = result.success ? result.session?.screenVideoPath : undefined; if (!screenPath) return false; - const cursorCaptureMode = result.success ? result.session?.cursorCaptureMode : undefined; - const warning = result.success ? result.warning : undefined; - - const label = screenPath.split(/[\\/]/).pop() || "Recording"; - await useProjectStore.getState().createProject(`Recording ${new Date().toLocaleString()}`); - await useProjectStore.getState().addAsset(screenPath, label); - // Mark before the video element can fire `loadedmetadata`. The asset path is - // already on the document; waiting until the 60s seed finished let the first - // metadata pass consume nothing and the second never arrive. - // - // Except for a system-cursor take, which writes no `.cursor.json` at all: the - // toggle stays on in prefs (it is only disabled in the UI while that mode is - // picked), so without this the flag is set for a recording that can never produce - // a click. What governs is the mode THIS take was recorded in, not the current - // preference. - if (cursorCaptureMode !== "system") { - markFreshRecordingAutoZoomPending(screenPath); - } - // Consumed: the recording now lives in a project. Cleared here rather than - // after the timeline seed below so a failure down there can't hand the same - // recording to the next editor window. - await api.setCurrentRecordingSession(null); + const pending: typeof pendingCompletion = result.session + ? { session: result.session, projectId: null } + : null; + pendingCompletion = pending; + try { + const cursorCaptureMode = result.success ? result.session?.cursorCaptureMode : undefined; + const warning = result.success ? result.warning : undefined; - // ponytail: MediaRecorder WebMs ship with duration = NaN until - // fix-webm-duration patches the EBML header; until that flows through the - // asset, drop a default 60s clip into the timeline so the editor isn't stuck - // on "No clips yet" the moment the user lands in the project. Real duration - // overwrites this when handleLoadedMetadata fires with a finite value. - const doc = useProjectStore.getState().document; - if (doc && doc.timeline.clips.length === 0 && doc.assets.length > 0) { - // `history: false`. Nothing here is an edit: the user finished a recording and the - // editor built them a project around it, unattended, on mount. Recording it left a - // brand-new project sitting at `past.length === 1` before the user had touched - // anything, so their FIRST Ctrl+Z restored the state before the seed -- an empty - // timeline -- and the persist that follows an undo wrote that empty timeline to disk. - await useProjectStore + const label = screenPath.split(/[\\/]/).pop() || "Recording"; + const created = await useProjectStore .getState() - .replaceTimeline([{ startSec: 0, endSec: 60 }], "Auto-imported recording", { - history: false, - }); - } - const latest = useProjectStore.getState().document; - if (latest) { - await maybeSaveFreshRecordingAutoZooms(latest); - } - if (warning) { - onWarning?.(warning); + .createProject(`Recording ${new Date().toLocaleString()}`); + if (pending) pending.projectId = created.project.id; + await useProjectStore.getState().addAsset(screenPath, label); + // Mark before the video element can fire `loadedmetadata`. The asset path is + // already on the document; waiting until the 60s seed finished let the first + // metadata pass consume nothing and the second never arrive. + // + // Except for a system-cursor take, which writes no `.cursor.json` at all: the + // toggle stays on in prefs (it is only disabled in the UI while that mode is + // picked), so without this the flag is set for a recording that can never produce + // a click. What governs is the mode THIS take was recorded in, not the current + // preference. + if (cursorCaptureMode !== "system") { + markFreshRecordingAutoZoomPending(screenPath); + } + // Consumed: the recording now lives in a project. Cleared here rather than + // after the timeline seed below so a failure down there can't hand the same + // recording to the next editor window. + await api.setCurrentRecordingSession(null); + + // ponytail: MediaRecorder WebMs ship with duration = NaN until + // fix-webm-duration patches the EBML header; until that flows through the + // asset, drop a default 60s clip into the timeline so the editor isn't stuck + // on "No clips yet" the moment the user lands in the project. Real duration + // overwrites this when handleLoadedMetadata fires with a finite value. + const doc = useProjectStore.getState().document; + if (doc && doc.timeline.clips.length === 0 && doc.assets.length > 0) { + // `history: false`. Nothing here is an edit: the user finished a recording and the + // editor built them a project around it, unattended, on mount. Recording it left a + // brand-new project sitting at `past.length === 1` before the user had touched + // anything, so their FIRST Ctrl+Z restored the state before the seed -- an empty + // timeline -- and the persist that follows an undo wrote that empty timeline to disk. + await useProjectStore + .getState() + .replaceTimeline([{ startSec: 0, endSec: 60 }], "Auto-imported recording", { + history: false, + }); + } + const latest = useProjectStore.getState().document; + if (latest) { + await maybeSaveFreshRecordingAutoZooms(latest); + } + if (warning) { + onWarning?.(warning); + } + return true; + } catch (error) { + if (pendingCompletion === pending) pendingCompletion = null; + if (pending) { + await api + .reportRecordingOutcome?.({ + recordingId: pending.session.createdAt, + status: "failed", + message: `The recording's project could not be prepared: ${String(error)}`, + }) + .catch(() => undefined); + } + throw error; } - return true; } diff --git a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx index 928ac5886..df1f4ff3a 100644 --- a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx +++ b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx @@ -25,6 +25,7 @@ let api: Record>; */ function stubElectronAPI(overrides: Record = {}) { api = { + reportRecordingOutcome: vi.fn(async () => ({ success: true })), getRecordingPrefs: vi.fn(async () => null), getPlatform: vi.fn(() => "win32"), getSelectedSource: vi.fn(async () => SOURCE), @@ -104,6 +105,9 @@ describe("useScreenRecorder native Windows stop failure", () => { expect(view.result.current.saving).toBe(false); // The failure must not be mistaken for a successful recording. expect(api.switchToEditor).not.toHaveBeenCalled(); + expect(api.reportRecordingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ recordingId: 7, status: "failed" }), + ); // The next press starts a NEW recording instead of re-sending a stop. api.stopNativeWindowsRecording.mockClear(); @@ -122,6 +126,21 @@ describe("useScreenRecorder native Windows stop failure", () => { expect(view.result.current.recording).toBe(false); expect(toast.error).toHaveBeenCalledWith("IPC channel closed"); + expect(api.reportRecordingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ recordingId: 7, status: "failed" }), + ); + }); + + it("reports a discarded take as cancelled, never as a completed project", async () => { + api.stopNativeWindowsRecording.mockResolvedValue({ success: true, discarded: true }); + const view = renderHook(() => useScreenRecorder()); + await startNativeRecording(view); + await act(async () => view.result.current.cancelRecording()); + await settle(); + expect(api.reportRecordingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ recordingId: 7, status: "cancelled" }), + ); + expect(api.switchToEditor).not.toHaveBeenCalled(); }); it("still opens the editor when the stop succeeds", async () => { @@ -135,6 +154,7 @@ describe("useScreenRecorder native Windows stop failure", () => { await pressStop(view); expect(api.switchToEditor).toHaveBeenCalled(); + expect(api.reportRecordingOutcome).not.toHaveBeenCalled(); expect(view.result.current.recording).toBe(false); }); }); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 8c22aa9b7..12f1571c8 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -234,6 +234,20 @@ export async function finalizeWebcamAsset( } } +async function reportUnfinishedRecording(recordingId: number, cancelled: boolean) { + try { + await window.electronAPI?.reportRecordingOutcome?.({ + recordingId, + status: cancelled ? "cancelled" : "failed", + message: cancelled + ? "The recording was cancelled or discarded." + : "The recording could not be saved and opened. The recording bar says why.", + }); + } catch (error) { + console.warn("Failed to report recording outcome:", error); + } +} + export function useScreenRecorder(): UseScreenRecorderReturn { const t = useScopedT("editor"); const tLaunch = useScopedT("launch"); @@ -602,6 +616,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // store-recorded-session finalizes the streams included in a successful // save; the finally block discards everything else. let storeSucceeded = false; + let handedOver = false; let webcamIncludedInSave = false; try { const screenBlob = await activeScreenRecorder.recordedBlobPromise; @@ -674,9 +689,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; } catch (error) { console.error("Error saving recording:", error); } finally { + if (!handedOver) + await reportUnfinishedRecording( + activeRecordingId, + discardRecordingId.current === activeRecordingId, + ); // Discard any recorder whose data wasn't part of a successful save (discarded // run, failed save, or a webcam whose disk write failed while the screen still // saved) so no stream or partial file is left open or orphaned. @@ -719,6 +740,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { segmentStartedAt.current = null; }; + let handedOver = false; try { const result = await window.electronAPI.stopNativeWindowsRecording(discard); if (discard || result.discarded) { @@ -758,6 +780,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; return true; } catch (error) { console.error("Error saving native Windows recording:", error); @@ -767,6 +790,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { clearNativeRecordingState(); return true; } finally { + if (!handedOver) await reportUnfinishedRecording(activeNativeRecording.recordingId, discard); if (discardRecordingId.current === activeNativeRecording.recordingId) { discardRecordingId.current = null; } @@ -818,6 +842,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; let webcamSaved = false; + let handedOver = false; try { const result = await window.electronAPI.stopNativeMacRecording(discard); const webcamResult = await webcamResultPromise; @@ -868,6 +893,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; return true; } catch (error) { console.error("Error saving native macOS recording:", error); @@ -877,6 +903,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { clearNativeRecordingState(); return true; } finally { + if (!handedOver) + await reportUnfinishedRecording(activeNativeRecording.recordingId, discard); // A webcam stream that wasn't folded into a saved session has to be closed // and its partial file removed, or a discarded or failed take orphans a // half-written .webm now that the bytes go to disk as they arrive. @@ -936,6 +964,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; let webcamSaved = false; + let handedOver = false; try { const result = await window.electronAPI.stopNativeLinuxRecording(discard); const webcamResult = await webcamResultPromise; @@ -990,6 +1019,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; return true; } catch (error) { console.error("Error saving native Linux recording:", error); @@ -999,6 +1029,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { clearNativeRecordingState(); return true; } finally { + if (!handedOver) + await reportUnfinishedRecording(activeNativeRecording.recordingId, discard); // A webcam stream that wasn't folded into a saved session has to be closed // and its partial file removed, or a discarded or failed take orphans a // half-written .webm now that the bytes go to disk as they arrive. diff --git a/src/lib/recordingSession.ts b/src/lib/recordingSession.ts index 5fd06bc25..413ba9cbf 100644 --- a/src/lib/recordingSession.ts +++ b/src/lib/recordingSession.ts @@ -99,3 +99,8 @@ export function normalizeRecordingSession(candidate: unknown): RecordingSession : Date.now(), }; } + +/** Internal recorder/editor hand-off, never an MCP server notification. */ +export type RecordingOutcome = + | { recordingId: number; status: "ready"; projectId: string } + | { recordingId: number; status: "failed" | "cancelled"; message: string }; diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index 9a3740eed..423564398 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -12,7 +12,7 @@ The same tools are served over two transports, by two different processes: | Edits, recording, export | Off until the user turns on **MCP edits** | Always allowed | | Project open in the editor | Edited live through the editor, one undo step per call | The same while the app runs: the call is forwarded to it. With no app answering, refused ([the marker](#the-open-in-editor-marker)) | | Other projects | Read from and saved to their file, a checkpoint first | Same | -| Tools | `listProjects`, the agent's tools, `transcribe`, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `exportAudio`, `getFrames`, `createProject` | The same | +| Tools | `listProjects`, the agent's tools, `transcribe`, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `waitForRecording`, `setPointerTrack`, `exportVideo`, `exportAudio`, `getFrames`, `createProject` | The same | | A take | Recorded by the app's recording bar, as its Record button records; the editor opens it ([below](#the-media-tools)) | Recorded by a hidden runner of its own process, and made a project there | | Checkpoints | In the app's memory, shared with forwarded `openscreen mcp` calls, lost when the app quits | In the app's while it runs; otherwise in this process's, lost when it exits, which it does when the client disconnects | @@ -157,7 +157,7 @@ Three tools of the runner's own ([`project-tools.ts`](../../electron/mcp/project ## The media tools -`createMediaTools` builds the tools that make a video around the edits, once per server, and `createOpenScreenMcpServer` registers them. Both servers serve the same seven, from the same code. Their state (the take running, the export running, each take's start time, each project's last export) lives in what `createMediaTools` returns, so the in-app server, which builds a fresh `McpServer` per request, keeps it across requests and across a restart of its port. They share the runner the document tools use: the app's one runner in the app, the routed one of `stdio-server.ts` over stdio, so their document calls go to the app while it runs. +`createMediaTools` builds the tools that make a video around the edits, once per server, and `createOpenScreenMcpServer` registers them. Both servers register the same tools, from the same code; `waitForRecording` observes the GUI only on the in-app HTTP server. Over stdio it returns an unsupported-tool error explaining which server to connect to. Their state (the take running, the export running, each take's start time, each project's last export) lives in what `createMediaTools` returns, so the in-app server, which builds a fresh `McpServer` per request, keeps it across requests and across a restart of its port. They share the runner the document tools use: the app's one runner in the app, the routed one of `stdio-server.ts` over stdio, so their document calls go to the app while it runs. What differs between the two servers is who records, handed in as a `TakeRecorder`: @@ -168,6 +168,40 @@ What differs between the two servers is who records, handed in as a `TakeRecorde | `stopRecording` | Presses the HUD's Stop, as the tray does. The HUD saves the take and hands it to the editor (`set-current-recording-session`), which imports it as for any take: one clip, measured, and the zooms on its clicks when Record mode's *Auto-zoom after recording* is on. The recorder waits for the editor's document to hold the take with its real length, then for its revision to hold still for 1.5 s, and returns that project, open in the editor. A `title` is applied through the editor. | Stops the runner and makes the project here (`createProjectFromVideo`): a fresh id no editor can hold, the zooms planned from the clicks, and the webcam, when one was recorded, linked to the take's asset as the editor links it (its offset and size). | | The user's settings | The source, microphone, camera, system audio and cursor mode the user had (`TAKE_PREF_KEYS`) come back once the take is captured, or as the app quits mid-take. A request that fails before the take changes none of them. | Untouched: the runner never saves them. | +### Waiting for a user-controlled take + +On the **in-app HTTP server**, `waitForRecording({ timeoutSec: 300 })` observes the +current take, whether the user or `startRecording` began it. It never starts or stops +capture. `timeoutSec` is an integer from 1 to 3600, default 300; configure the client's +own tool-call timeout to be longer. The server's existing bearer token, Host and Origin +checks apply. Observation is read-only and works with **MCP edits** off. + +- With no active take (recording or saving), the call immediately returns `isError`. + It does not return the last recording or wait for the next one. Call it after the + HUD starts recording; the countdown is not an active take. +- `RecordingCompletion` (`electron/mcp/recording-completion.ts`) binds each wait to + the recording ID reported by the capture path. A new take, or Cancel/Restart at + the HUD, fails waits on the old one. An imported project cannot finish a wait. +- Capture finalizes the files and cursor sidecar before publishing the recording + session. The editor then acknowledges that specific session's project after its + real duration and automatic-zoom writes have settled. The main process reads the + saved project and verifies its measured clip and nonempty screen/camera files. + Unlike `stopRecording`'s quiet-period check, this acknowledgement is explicit. +- Success uses `stopRecording`'s result builder: `projectId`, `checkpointId` (null + if checkpoint creation fails), `videoPath`, `durationSec`, `recordingStartedAt`, + `autoZooms`, and optional `warnings`. It does not wait for transcription or export. +- Save/import failures and discarded takes return tool errors. A missing duration + or unfinished auto-zoom save cannot produce a success; the wait can time out. +- Timeout, MCP cancellation, HTTP disconnect, and server shutdown remove only that + wait's timer, listener and abort subscription. They leave the recording running. + Several clients may wait independently on the same take, each with its own timeout. + A timed-out caller can retry while that take is still recording or saving. + +The completed take releases the app recorder's claim. A client that used +`startRecording` and then waited for the user's stop can start another take without +an extra `stopRecording` call. The acknowledgement is an internal renderer-to-main +IPC message, not an MCP notification or a command hook. + Then both servers carry on alike: - `stopRecording` creates the take's checkpoint through the runner, and records when the take started, by its video, for `setPointerTrack`. @@ -229,7 +263,7 @@ codex mcp add openscreen -- "" mcp ## Known gaps - **`openscreen mcp` records in its own process, app or no app.** Only its document calls are forwarded; its takes, exports and frames stay in the CLI process, and its take lands in the project list rather than in the editor. The in-app server is the way to record through the app's recording bar. -- **An in-app take is ready by a quiet period.** The editor lays a take out in several saves and says none of them is the last: the recorder waits for its revision to hold for 1.5 s. A save later than that (a slow auto-zoom pass) lands after the checkpoint, so `restoreCheckpoint` takes it back with the client's edits. +- **`stopRecording` considers an in-app take ready by a quiet period.** The editor lays a take out in several saves and says none of them is the last: the recorder waits for its revision to hold for 1.5 s. A save later than that (a slow auto-zoom pass) lands after the checkpoint, so `restoreCheckpoint` takes it back with the client's edits. - **No Ctrl+Z for edits to closed projects.** Their undo is the checkpoint the write requires, which lives in memory: it is gone once its server goes away or 20 newer ones push it out, and the edits stay. - **An editor that stops answering looks closed.** A snapshot unanswered for 30 s reads as "no project open", so an edit naming the project that editor holds would go to its file, and the editor's next save would drop it. It takes a renderer hung for 30 s. - **No project deletion**, on purpose ([above](#the-project-and-its-media)). diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 2f643ec22..a07060b5d 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -16,6 +16,13 @@ flowchart LR The HUD starts and controls a recording. The source selector chooses a display or window and the countdown gives the user time to prepare. On Linux there is no in-app source selector: the compositor's portal picker takes its place. It is raised before the countdown, not during it — the helper negotiates the portal, reports `source-selected`, and then waits for `record` on stdin while the app counts down. Electron resolves the source and output paths, starts the selected capture backend, and records cursor telemetry alongside media. Stop finalizes the media and session files; the resulting paths are passed to the editor as recording assets. +The in-app MCP server can observe this lifecycle through +[`waitForRecording`](mcp-server.md#waiting-for-a-user-controlled-take). Its target is +the current recording ID, not the current project or a newly created file. The +recorder reports failure/discard explicitly; the editor acknowledges completion +after the project's duration and automatic-zoom saves. Waiting never controls +capture, and timing out or cancelling a request leaves the take running. + ## The HUD `LaunchWindow` is the recording controller. Its control row exposes recording, pause, source, microphone, system-audio, webcam, cursor, settings, and stop/restart actions through the HUD controls. The overlay is click-through, so it does not intercept the application being recorded — but it is never *born* that way: the renderer asks for it on mount over `hud-overlay-ignore-mouse-events`, and `electron/windows.ts` applies it as `setIgnoreMouseEvents(ignore)`, with no `{ forward: true }`. While the window is input-transparent Chromium delivers it no pointer events at all, so the way back out is a poll: the main process samples the OS cursor and pushes the window-relative point over `hud-overlay-cursor`, and the renderer hit-tests it to decide when the HUD's own controls temporarily opt into pointer handling. On Linux none of this runs: `setIgnoreMouseEvents` is a no-op there, so the renderer never asks for click-through and the HUD stays interactive — which is why its drag region sits on the grab handle rather than on the transparent root. (Until #385 the lift was Electron's `{ forward: true }` — a global `WH_MOUSE_LL` hook on Windows that Windows can revoke without telling the app, leaving the HUD painted and permanently dead. The poll replaced it.) diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 6841efab0..19aae9212 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -580,6 +580,7 @@ Two servers. The **in-app** one is HTTP, in *AI settings* → *MCP server*, off - [ ] In *AI settings*, confirm *Enable MCP server* and *Allow MCP clients to edit your projects, record and export* are off by default; enable the server and confirm the status reads *Running* with a *Port*, an *Access token* (Show, Copy, Regenerate) and the Claude Code and Codex snippets. Change the port, *Apply*, and confirm a client reconnects on it. - [ ] With edits not allowed, confirm an edit tool, `startRecording` and `exportVideo` are refused, naming *MCP edits*; allow them and confirm the edit lands in the open editor and undoes with `Ctrl/Cmd+Z`. +- [ ] On the **in-app HTTP MCP server**, start a take from the HUD, call `waitForRecording`, then stop it from the HUD. Verify that the returned `projectId` is the saved project in the editor and `videoPath` is its completed media. Repeat with `timeoutSec: 1` while recording: the wait returns a tool error and the HUD keeps recording; a second wait can receive that same take after the user stops it. While idle the tool must immediately return a tool error. Cancel/Restart must fail waits on the discarded take; test an MCP request cancellation independently. This tool does not observe the HUD through the stdio server. - [ ] From a client of the **in-app** server, `startRecording` on a window: the editor closes and the recording bar records, its countdown first, with no second recorder process. Click in the window, `stopRecording`: the editor opens on the take, laid out and zoomed as after a take started by hand, and the returned `projectId` is that project. An edit to it lands live and undoes with `Ctrl/Cmd+Z`; `exportVideo` to an absolute path and `getFrames` on it work. Confirm the recording bar's microphone, camera, system audio and cursor mode are back to what they were before the take, and that `startRecording` is refused while the editor has unsaved changes. - [ ] Confirm `"" help` lists `mcp`, register the stdio server in an MCP client, and confirm `listSources` lists displays, windows, microphones and cameras. - [ ] Run `startRecording` on a window, click inside it a few times, `stopRecording`, and confirm the returned project is in the editor's project list with one clip and the zooms its `autoZooms` count announces. Clicks close in time (3 s or less) and on screen share one zoom, and a click in the first seconds or the stopping click gets none, so there can be fewer zooms than clicks. On macOS 15.2+, confirm Apple's picker never appears; on Linux, confirm the portal asks. @@ -1002,3 +1003,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | | 2026-10-22 | Dev `d8bf2d81` + issue #1260 patch (`fix/agent-stop-1260`), Electron 41.2.1; Vite `127.0.0.1:5186`, private `openscreen1260-ui-*` userData, local controlled OpenAI-compatible provider | macOS 15.8, Apple Silicon | Superseded — pre-`0cb26e07` Agent Stop check | **Superseded by `0cb26e07`:** the two-Undo observation below records the old per-tool behavior. The current implementation applies the accumulated document once on completion or Stop, with one save and one undo step per turn. Automated regressions cover normal and stopped turns with 60 tool edits; this desktop check has not been rerun on the new head. **Historical run:** Real Electron operated through computer-use: sent a first turn, observed Stop while the provider held its SSE response open, applied two `setVocabulary` edits, clicked Stop, saw the stopped status and Send return, confirmed the provider connection closed and the partial answer plus `stopped: true` reached the chat sidecar. Document bytes stayed at `Second edit`; one Undo restored `First edit`, a second restored the initial document. A new turn completed with `Next turn works.` Screenshots captured for running and continued states. Skipped: paid/remote providers, Windows/Linux desktop UI, capture, media preview and export (this patch changes Agent chat only). | | 2026-10-23 | `fix/mac-mic-silent-with-system-audio` on `main` @ `264975d`, dev build (`npm run dev`), capture helper built locally with Xcode 27 and copied from `out/Products/Release` by hand | macOS 26.6.2 (25G83), MacBook Pro M2 Pro; built-in microphone and a USB Audio CODEC (08bb:2902) | **Partial: fix slice only, pass** | #1173, Apple's picker, system audio on, output device idle (after `sudo killall coreaudiod`, or checked idle). Before the fix: the microphone delivered nothing until another app played a sound, 56 s (built-in) and 14 s (USB) into the take; without system audio it started at 0.17 s. With the tap merely started after the stream, four takes (built-in twice, USB twice) recorded the microphone from 0.17–0.18 s, but only 12–30 ms ahead of the tap. With this branch's version, the tap waiting for the first microphone buffer: one USB take recorded the microphone from 0.18 s and the tap started 27 ms after it; one take with the microphone off. In both, a sound played mid-take was in system audio. Measured with temporary per-source buffer counters in the helper, not shipped. **Not covered:** this branch's version with the built-in microphone, a legacy-picker take, macOS 15, an Intel Mac, a microphone that never delivers (the 1 s fallback), pause before the first microphone buffer, a CI-built artifact. | +| 2026-10-22 | #1188 local worktree, `feat/wait-for-recording-1188`, based on `main` @ `d8bf2d81`, development build (Electron 41.2.1) | macOS 15.8 arm64 | **Partial — HTTP idle behavior passed; real capture blocked** | Launched this worktree with Vite port 5188, MCP port 47888, CDP port 9388 and isolated userData `/tmp/os1188-3cvs73hn/profile`; `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`, `OPENSCREEN_MAC_SOURCE_PICKER=legacy`, `DO_NOT_TRACK=1`. Computer-use selected Screen 1 from the HUD and clicked Start recording. The app displayed **Native macOS capture helper is not available.** The full borrowed native library directory has the compositor and ffmpeg but no capture helper; `npm run build:native:mac` cannot build it because only Command Line Tools is active, not full Xcode. A real authenticated HTTP `waitForRecording` call returned HTTP 200 with `isError: true` and **No recording is running or being saved.** in 679 ms. **Skipped:** successful HUD recording → wait → manual stop → returned recording, and timeout while a real recording keeps running; no real recording began. Unit/SDK/HTTP fixture tests are separate evidence and do not count as real capture. Editor, export, webcam, other platforms and the rest of this checklist were not exercised. Quit only this isolated app and stopped only this worktree's Vite process. | diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.md index d7e1647c6..a79a48f05 100644 --- a/website/docs/ai-editing.md +++ b/website/docs/ai-editing.md @@ -76,7 +76,7 @@ Checkpoints are kept in OpenScreen's memory: they are lost when it quits or when ## Letting an agent make the whole video -An agent can make a whole video on its own, from the recording to the export. Besides the editing tools, both ways of adding OpenScreen to it give it `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `exportAudio`, `getFrames` and `createProject`. A take can also record your webcam and the system's own cursor, as the recording bar can, and an export takes the Export dialog's options: quality and frame rate for an MP4; frame rate, size and looping for a GIF. `exportAudio` writes the project's audio, or one imported track alone (a voiceover added with `importMedia` and `addAudio`), to a WAV or an M4A, exactly as the MP4 mixes it. +An agent can make a whole video on its own, from the recording to the export. Besides the editing tools, both ways of adding OpenScreen to it give it `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `exportAudio`, `getFrames` and `createProject`. On the server in the app, `waitForRecording` also waits for a take you record yourself from the recording bar, and returns it as `stopRecording` does. A take can also record your webcam and the system's own cursor, as the recording bar can, and an export takes the Export dialog's options: quality and frame rate for an MP4; frame rate, size and looping for a GIF. `exportAudio` writes the project's audio, or one imported track alone (a voiceover added with `importMedia` and `addAudio`), to a WAV or an M4A, exactly as the MP4 mixes it. **Through the server above**, a take is recorded by OpenScreen itself, as with its Record button: the editor closes, the recording bar counts down and records, and the take opens in the editor as a new project, where you watch each edit land. It needs **MCP edits** on, and the recording bar gets your own settings back once the take ends. diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md index 737485741..33928ac1b 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ Wiederherstellungspunkte liegen im Speicher von OpenScreen: Sie gehen verloren, ## Einen Agenten das ganze Video erstellen lassen {#letting-an-agent-make-the-whole-video} -Ein Agent kann ein ganzes Video selbstständig erstellen, von der Aufnahme bis zum Export. Neben den Bearbeitungswerkzeugen geben ihm beide Wege, OpenScreen hinzuzufügen, `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` und `createProject`. Eine Aufnahme kann wie mit der Aufnahmeleiste auch Ihre Webcam und den eigenen Mauszeiger des Systems aufnehmen, und ein Export übernimmt die Optionen des Export-Dialogs: Qualität und Bildrate für ein MP4; Bildrate, Größe und Wiederholung für ein GIF. +Ein Agent kann ein ganzes Video selbstständig erstellen, von der Aufnahme bis zum Export. Neben den Bearbeitungswerkzeugen geben ihm beide Wege, OpenScreen hinzuzufügen, `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` und `createProject`. Über den Server in der App wartet `waitForRecording` außerdem auf eine Aufnahme, die Sie selbst mit der Aufnahmeleiste machen, und gibt sie zurück wie `stopRecording`. Eine Aufnahme kann wie mit der Aufnahmeleiste auch Ihre Webcam und den eigenen Mauszeiger des Systems aufnehmen, und ein Export übernimmt die Optionen des Export-Dialogs: Qualität und Bildrate für ein MP4; Bildrate, Größe und Wiederholung für ein GIF. **Über den Server oben** nimmt OpenScreen eine Aufnahme selbst auf, wie mit seinem Aufnahme-Button: Der Editor schließt sich, die Aufnahmeleiste zählt herunter und nimmt auf, und die Aufnahme öffnet sich im Editor als neues Projekt, wo du jede Änderung ankommen siehst. Dafür muss **MCP-Änderungen** an sein, und die Aufnahmeleiste bekommt am Ende der Aufnahme deine eigenen Einstellungen zurück. diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md index ee2cd6f03..1d4d0c4f0 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ Los puntos de restauración se guardan en la memoria de OpenScreen: se pierden c ## Dejar que un agente haga todo el video {#letting-an-agent-make-the-whole-video} -Un agente puede hacer un video completo por sí solo, desde la grabación hasta la exportación. Además de las herramientas de edición, las dos formas de agregar OpenScreen al agente le dan `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` y `createProject`. Una toma también puede grabar tu cámara web y el cursor propio del sistema, como la barra de grabación, y una exportación acepta las opciones del diálogo de exportación: calidad y velocidad de fotogramas para un MP4; velocidad de fotogramas, tamaño y bucle para un GIF. +Un agente puede hacer un video completo por sí solo, desde la grabación hasta la exportación. Además de las herramientas de edición, las dos formas de agregar OpenScreen al agente le dan `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` y `createProject`. En el servidor de la app, `waitForRecording` también espera una toma que grabas tú mismo desde la barra de grabación y la devuelve como `stopRecording`. Una toma también puede grabar tu cámara web y el cursor propio del sistema, como la barra de grabación, y una exportación acepta las opciones del diálogo de exportación: calidad y velocidad de fotogramas para un MP4; velocidad de fotogramas, tamaño y bucle para un GIF. **Con el servidor de arriba**, la toma la graba el propio OpenScreen, como con su botón Grabar: el editor se cierra, la barra de grabación hace su cuenta regresiva y graba, y la toma se abre en el editor como un proyecto nuevo, donde ves llegar cada edición. Necesita **Ediciones por MCP** activado, y la barra de grabación recupera tus propios ajustes al terminar la toma. diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md index 46e43b614..e110d946f 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ Les points de restauration sont gardés dans la mémoire d'OpenScreen : ils son ## Laisser un agent faire toute la vidéo {#letting-an-agent-make-the-whole-video} -Un agent peut faire une vidéo entière tout seul, de l'enregistrement à l'export. En plus des outils d'édition, les deux façons d'ajouter OpenScreen à l'agent lui donnent `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` et `createProject`. Une prise peut aussi enregistrer votre webcam et le curseur du système, comme la barre d'enregistrement, et un export prend les options de la boîte d'export : qualité et fréquence d'images pour un MP4 ; fréquence d'images, taille et boucle pour un GIF. +Un agent peut faire une vidéo entière tout seul, de l'enregistrement à l'export. En plus des outils d'édition, les deux façons d'ajouter OpenScreen à l'agent lui donnent `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` et `createProject`. Sur le serveur de l'app, `waitForRecording` attend aussi une prise que vous enregistrez vous-même depuis la barre d'enregistrement, et la rend comme `stopRecording`. Une prise peut aussi enregistrer votre webcam et le curseur du système, comme la barre d'enregistrement, et un export prend les options de la boîte d'export : qualité et fréquence d'images pour un MP4 ; fréquence d'images, taille et boucle pour un GIF. **Par le serveur ci-dessus**, une prise est enregistrée par OpenScreen lui-même, comme avec son bouton Enregistrer : l'éditeur se ferme, la barre d'enregistrement fait son décompte et enregistre, et la prise s'ouvre dans l'éditeur comme nouveau projet, où vous voyez chaque modification arriver. Il faut que **Modifications par MCP** soit activé, et la barre d'enregistrement retrouve vos propres réglages à la fin de la prise. diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md index 5f2dc43b9..c758ff921 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ Claude Code や Codex などの AI コーディングエージェントをすで ## エージェントに動画全体を作らせる {#letting-an-agent-make-the-whole-video} -エージェントは、録画からエクスポートまで、動画全体を自力で作れます。どちらの方法で OpenScreen を追加しても、編集ツールに加えて `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames`、`createProject` を使えます。録画バーと同じく、テイクには Web カメラやシステム自身のカーソルも録画でき、エクスポートでは書き出しダイアログのオプションを使えます。MP4 なら画質とフレームレート、GIF ならフレームレート、サイズ、ループです。 +エージェントは、録画からエクスポートまで、動画全体を自力で作れます。どちらの方法で OpenScreen を追加しても、編集ツールに加えて `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames`、`createProject` を使えます。アプリ内のサーバーでは、`waitForRecording` で、あなた自身が録画バーから録画するテイクの完了を待ち、`stopRecording` と同じ形で受け取ることもできます。録画バーと同じく、テイクには Web カメラやシステム自身のカーソルも録画でき、エクスポートでは書き出しダイアログのオプションを使えます。MP4 なら画質とフレームレート、GIF ならフレームレート、サイズ、ループです。 **上のサーバーを使う場合**、録画は OpenScreen 自身が、録画ボタンを押したときと同じように行います。エディターが閉じ、録画バーがカウントダウンして録画し、録画は新しいプロジェクトとしてエディターで開きます。そこで各編集が反映されるのを確認できます。**MCP による編集**をオンにする必要があり、録画が終わると録画バーの設定はあなた自身のものに戻ります。 diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md index 169573336..28dbbdbfc 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ Os pontos de restauração ficam na memória do OpenScreen: eles se perdem quand ## Como deixar um agente fazer o vídeo inteiro {#letting-an-agent-make-the-whole-video} -Um agente pode fazer um vídeo inteiro sozinho, da gravação à exportação. Além das ferramentas de edição, as duas formas de adicionar o OpenScreen ao agente lhe dão `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` e `createProject`. Uma gravação também pode incluir sua webcam e o cursor do próprio sistema, como a barra de gravação, e uma exportação aceita as opções da caixa de exportação: qualidade e taxa de quadros para um MP4; taxa de quadros, tamanho e repetição para um GIF. +Um agente pode fazer um vídeo inteiro sozinho, da gravação à exportação. Além das ferramentas de edição, as duas formas de adicionar o OpenScreen ao agente lhe dão `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` e `createProject`. No servidor do app, `waitForRecording` também espera uma gravação que você faz pela barra de gravação e a devolve como `stopRecording`. Uma gravação também pode incluir sua webcam e o cursor do próprio sistema, como a barra de gravação, e uma exportação aceita as opções da caixa de exportação: qualidade e taxa de quadros para um MP4; taxa de quadros, tamanho e repetição para um GIF. **Pelo servidor acima**, a gravação é feita pelo próprio OpenScreen, como com o botão Gravar: o editor fecha, a barra de gravação faz a contagem regressiva e grava, e a gravação abre no editor como um novo projeto, onde você vê cada edição chegar. É preciso ativar **Edições via MCP**, e a barra de gravação volta às suas próprias configurações quando a gravação termina. diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md index 44811efbf..e48600ada 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 ## 让 AI 代理制作整段视频 {#letting-an-agent-make-the-whole-video} -AI 代理可以从录制到导出独立完成一整段视频。无论以哪种方式添加 OpenScreen，除编辑工具外，AI 代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` 和 `createProject`。与录制栏一样，录制时也可以录下你的摄像头和系统自带的光标；导出则接受导出对话框的选项：MP4 的画质和帧率，GIF 的帧率、尺寸和循环。 +AI 代理可以从录制到导出独立完成一整段视频。无论以哪种方式添加 OpenScreen，除编辑工具外，AI 代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` 和 `createProject`。在应用内的服务器上，`waitForRecording` 还能等待你自己用录制栏录下的一段录制，并像 `stopRecording` 一样返回它。与录制栏一样，录制时也可以录下你的摄像头和系统自带的光标；导出则接受导出对话框的选项：MP4 的画质和帧率，GIF 的帧率、尺寸和循环。 **通过上面的服务器**，录制由 OpenScreen 自己完成，就像按下它的录制按钮一样：编辑器关闭，录制栏倒计时后开始录制，录好的内容作为新项目在编辑器中打开，你可以看到每项编辑实时生效。这需要开启**MCP 编辑**，录制结束后，录制栏会恢复你自己的设置。 diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md index 0ffce8f09..8cdc4fe88 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md @@ -76,7 +76,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 ## 讓代理製作整支影片 {#letting-an-agent-make-the-whole-video} -代理可以從錄影到匯出自行製作一整支影片。無論以哪種方式加入 OpenScreen，除了剪輯工具之外，代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` 與 `createProject`。與錄影列一樣，錄影時也能錄下你的網路攝影機與系統本身的游標；匯出則接受匯出對話框的選項：MP4 的畫質與影格速率，GIF 的影格速率、尺寸與循環播放。 +代理可以從錄影到匯出自行製作一整支影片。無論以哪種方式加入 OpenScreen，除了剪輯工具之外，代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` 與 `createProject`。在應用程式內的伺服器上，`waitForRecording` 還能等待你自己用錄影列錄下的一段錄影，並像 `stopRecording` 一樣傳回它。與錄影列一樣，錄影時也能錄下你的網路攝影機與系統本身的游標；匯出則接受匯出對話框的選項：MP4 的畫質與影格速率，GIF 的影格速率、尺寸與循環播放。 **透過上述伺服器**，錄影由 OpenScreen 自己完成，就像按下它的錄影按鈕一樣：編輯器會關閉，錄影列倒數後開始錄影，錄好的內容會以新專案在編輯器中開啟，你可以看到每項編輯即時生效。這需要開啟**MCP 編輯**，錄影結束後，錄影列會恢復你自己的設定。