diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 7717d1a5b1..db0f4720f2 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -30,9 +30,9 @@ import { resolveDaemonPaths, type DaemonCapability, type DaemonOwnership, - type ObservedDaemonIdentity, type DaemonPaths, type DaemonServerMode, + type ObservedDaemonIdentity, } from './daemon-resolution.ts'; import { findUnrecoveredRepairCommitFailure } from './session-repair-tombstone.ts'; import { @@ -44,6 +44,7 @@ import { import { buildDaemonShutdownReport, resolveDaemonShutdownReportPath, + type DaemonShutdownCompletion, type DaemonShutdownOutcome, } from './daemon-shutdown-report.ts'; import { sweepDeadDaemonStartupPhasesFiles } from './daemon-startup-phases.ts'; @@ -61,6 +62,7 @@ export type DaemonRegistrationFields = Readonly<{ capabilities?: readonly DaemonCapability[]; /** The provider plugin inventory, so a client can read what this daemon registered. */ plugins?: readonly PluginInventoryEntry[]; + /** Who owns this process; see {@link DaemonOwnership}. Absent records are detached daemons. */ ownership?: DaemonOwnership; }>; type DaemonRegistrationRemoval = @@ -68,7 +70,10 @@ type DaemonRegistrationRemoval = | Exclude; export type DaemonRegistrationOwner = Readonly<{ publish(fields: DaemonRegistrationFields): void; - finish(outcome?: DaemonShutdownOutcome): Promise; + finish( + outcome?: DaemonShutdownOutcome, + close?: Readonly<{ retainRegistration: boolean; unfinished: readonly string[] }>, + ): Promise>; }>; export async function tryAcquireDaemonRegistration( @@ -93,6 +98,7 @@ export async function tryAcquireDaemonRegistration( } catch (error) { await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } + let published: Pick | undefined; return { status: 'acquired', owner: Object.freeze({ @@ -118,20 +124,72 @@ export async function tryAcquireDaemonRegistration( ), mode: 0o600, }); + published = { ownership: fields.ownership, token: fields.token }; }, - async finish(outcome?: DaemonShutdownOutcome) { - let removal: DaemonRegistrationRemoval; + async finish( + outcome?: DaemonShutdownOutcome, + close?: Readonly<{ retainRegistration: boolean; unfinished: readonly string[] }>, + ) { + let removal: DaemonRegistrationRemoval | Readonly<{ state: 'retained' }>; try { - if (outcome) writeShutdownReport(boundPaths.baseDir, outcome, acquisition); - removal = removeRegistrationUnderLock(boundPaths.infoPath, identity, acquisition); + acquisition.assertHeld(); + removal = close?.retainRegistration + ? { state: 'retained' } + : removeRegistrationUnderLock(boundPaths.infoPath, identity, acquisition); } catch (error) { + await writeShutdownRecord( + 'retained', + outcome, + close?.unfinished ?? ['Daemon shutdown: registration removal failed.'], + ); + if (published?.ownership === 'embedded') throw error; return await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } - await acquisition.release(); + const released = removal.state === 'removed' || removal.state === 'absent'; + await writeShutdownRecord( + released ? 'released' : 'retained', + outcome, + close?.unfinished ?? ['Daemon shutdown: close details are unavailable.'], + ); + if (removal.state !== 'retained' && (released || published?.ownership !== 'embedded')) { + await acquisition.release(); + } return removal; }, }), }; + + async function writeShutdownRecord( + disposition: DaemonShutdownCompletion['outcome'], + outcome: DaemonShutdownOutcome | undefined, + unfinished: readonly string[], + ): Promise { + const completion = + published?.ownership === 'embedded' && identity.startTime !== null + ? { + pid: identity.pid, + processStartTime: identity.startTime, + token: published.token, + outcome: disposition, + unfinished, + } + : undefined; + if (!outcome && !completion) return; + try { + acquisition.assertHeld(); + publishFileSync({ + destination: resolveDaemonShutdownReportPath(boundPaths.baseDir), + contents: `${JSON.stringify({ ...(outcome ? buildDaemonShutdownReport(outcome) : {}), ...(completion ? { completion } : {}) })}\n`, + mode: 0o600, + }); + } catch (error) { + await recordRegistrationWarning( + boundPaths.logPath, + 'daemon_shutdown_record_failed', + error, + ).catch(() => {}); + } + } } function removeRegistrationUnderLock( @@ -159,26 +217,15 @@ async function releaseRegistrationAfterFailure( try { await acquisition.release(); } catch (releaseError) { - await recordRegistrationWarning(logPath, 'daemon_registration_release_failed', releaseError); + await recordRegistrationWarning( + logPath, + 'daemon_registration_release_failed', + releaseError, + ).catch(() => {}); } throw error; } -function writeShutdownReport( - stateDir: string, - outcome: DaemonShutdownOutcome, - acquisition: ProcessLockAcquisition, -): void { - const filePath = resolveDaemonShutdownReportPath(stateDir); - const contents = `${JSON.stringify(buildDaemonShutdownReport(outcome))}\n`; - acquisition.assertHeld(); - try { - publishFileSync({ destination: filePath, contents, mode: 0o600 }); - } catch { - return; - } -} - /** * The daemon's stdout and stderr append to this inode, so it is emptied in place, never * replaced. On POSIX the append handle that creates the file also permits truncation, and diff --git a/packages/daemon-contracts/src/daemon-registration.ts b/packages/daemon-contracts/src/daemon-registration.ts index 12dfbc8402..f781e58988 100644 --- a/packages/daemon-contracts/src/daemon-registration.ts +++ b/packages/daemon-contracts/src/daemon-registration.ts @@ -209,6 +209,7 @@ export type DaemonStopIdentity = Readonly<{ /** `absent`: no record. `unreadable`: a record whose contents cannot be used, unreadable or malformed. */ ownership: DaemonOwnership | 'absent' | 'unreadable'; identity: OwnerIdentity | null; + record: DaemonRecord | null; }>; /** @@ -219,8 +220,12 @@ export type DaemonStopIdentity = Readonly<{ export function readDaemonStopIdentity(infoPath: string): DaemonStopIdentity { const reading = readDaemonRegistration(infoPath); if (reading.status !== 'present') - return { ownership: reading.status === 'absent' ? 'absent' : 'unreadable', identity: null }; - return { ownership: reading.ownership, identity: reading.identity }; + return { + ownership: reading.status === 'absent' ? 'absent' : 'unreadable', + identity: null, + record: null, + }; + return { ownership: reading.ownership, identity: reading.identity, record: reading.record }; } /** diff --git a/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts b/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts index b603a631bb..2cff5adbc4 100644 --- a/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts +++ b/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts @@ -47,9 +47,9 @@ test('a fence names an integer pid and a start time or null, nothing else', () = assert.equal(readDaemonLifecycleFence('7'), undefined); }); -test('a quiesce carries the original instance fence', () => { +test('both lifecycle requests carry one fence shape under the command of their mode', () => { const fence = { pid: 7, processStartTime: 'Fri Oct 17 09:00:00 2026' }; - for (const mode of ['quiesce'] as const) { + for (const mode of ['quiesce', 'shutdown'] as const) { const request = buildDaemonLifecycleRequest({ mode, token: 't0ken', fence, requestId: 'r-1' }); assert.equal(request.command, DAEMON_LIFECYCLE_COMMANDS[mode]); assert.deepEqual(request.input, fence); diff --git a/packages/daemon-contracts/src/daemon-resolution.ts b/packages/daemon-contracts/src/daemon-resolution.ts index d4c0ddd818..690e48b3eb 100644 --- a/packages/daemon-contracts/src/daemon-resolution.ts +++ b/packages/daemon-contracts/src/daemon-resolution.ts @@ -147,6 +147,12 @@ export function resolveDaemonTransportPreference( return 'auto'; } +// The two ways a process asks a daemon to leave, and the fence that addresses the exact instance. +// Client and daemon share this vocabulary, so neither owns it. `quiesce` asks an idle daemon to +// leave so a compatible one can take its state directory; `shutdown` asks a runtime whose owner is a +// host process to close itself, work in hand included. Both ride one request envelope, one fence, +// and one admission rule, and differ only in what the daemon may still hold when it agrees. + /** What a daemon can do beyond the base RPC contract, published in its `daemon.json`. */ export const DAEMON_CAPABILITIES = ['quiesce'] as const; export type DaemonCapability = (typeof DAEMON_CAPABILITIES)[number]; @@ -158,7 +164,7 @@ export type DaemonCapability = (typeof DAEMON_CAPABILITIES)[number]; * predates the field names. * - `embedded`: the host that started the runtime owns the process. Nothing in the central * lifecycle signals that PID, not even hung recovery or a forced stop: the host closes the - * runtime through its own close, and an external stop refuses it. + * runtime through its own close, and a stop from outside reaches it only as a shutdown request. * - `unknown`: the record states an owner this build does not recognize, so no process may be * signaled on its strength. Signalling on a guess could end a host process, and a value this * build cannot read is exactly the case where the safe answer is to do nothing. @@ -197,10 +203,15 @@ export function readDaemonOwnership(value: unknown): DaemonOwnership { : 'unknown'; } -/** The command carrying the fenced lifecycle request. */ +/** + * The lifecycle requests, each named by the command that carries it. Both are the same envelope, + * fence and admission rule; they differ only in what the daemon may hold while it agrees. + */ export const DAEMON_LIFECYCLE_COMMANDS = { /** Stop if idle: refuses while the daemon holds a request, a session, or a lease. */ quiesce: 'daemon_quiesce', + /** Stop and drain: ends what the daemon holds, and only an embedder-owned runtime agrees. */ + shutdown: 'daemon_shutdown', } as const; export type DaemonLifecycleMode = keyof typeof DAEMON_LIFECYCLE_COMMANDS; @@ -212,6 +223,8 @@ export function readDaemonLifecycleMode(command: string): DaemonLifecycleMode | switch (command) { case DAEMON_LIFECYCLE_COMMANDS.quiesce: return 'quiesce'; + case DAEMON_LIFECYCLE_COMMANDS.shutdown: + return 'shutdown'; default: return undefined; } @@ -228,6 +241,8 @@ export const DAEMON_LIFECYCLE_REASONS = { instanceMismatch: 'daemon-instance-mismatch', /** A quiesce was refused because a host owns this runtime and ends it through its own close. */ ownerEmbedded: 'daemon-owner-embedded', + /** A shutdown was refused because a detached daemon's owner ends it by signalling its process. */ + notEmbedded: 'daemon-not-embedded', } as const; /** @@ -246,7 +261,9 @@ export type DaemonLifecycleFence = Readonly<{ /** * What a daemon holds, and the census a busy refusal carries so a caller knows what to wait for. - * A quiesce leaves only when all four counts are zero and it is not still starting up. + * A quiesce leaves only when all four counts are zero and it is not still starting up, because before + * that it has no drain path to run. A shutdown drains whatever it finds, and refuses with this census + * only while the daemon is still starting; its acknowledgement carries the one `draining` outcome. */ export type DaemonLifecycleCensus = Readonly<{ /** True from process start until the daemon can drain. */ diff --git a/packages/daemon-contracts/src/daemon-shutdown-report-completion.test.ts b/packages/daemon-contracts/src/daemon-shutdown-report-completion.test.ts new file mode 100644 index 0000000000..62902a7e0c --- /dev/null +++ b/packages/daemon-contracts/src/daemon-shutdown-report-completion.test.ts @@ -0,0 +1,58 @@ +import fs from 'node:fs'; +import { expect, test } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { + daemonShutdownCompletionAnswers, + readDaemonShutdownRecord, + readDaemonShutdownReport, + resolveDaemonShutdownReportPath, +} from './daemon-shutdown-report.ts'; + +const completion = { + pid: 4242, + processStartTime: 'birth', + token: 'runtime-a', + outcome: 'retained' as const, + unfinished: ['1 in-flight request ignored cancellation'], +}; + +test('one record reads the legacy report and completion independently', () => { + const dir = mkdtempForTestSync('agent-device-shutdown-record-'); + const file = resolveDaemonShutdownReportPath(dir); + const report = { + providerReleases: { released: [{ leaseId: 'lease-1' }], pending: [] }, + claims: { released: [], orphaned: [], superseded: [], unattributable: [] }, + }; + for (const present of [false, true]) { + fs.writeFileSync(file, JSON.stringify({ ...(present ? report : {}), completion })); + expect(readDaemonShutdownRecord(dir)).toEqual({ report: present ? report : null, completion }); + expect(readDaemonShutdownReport(dir)).toEqual(present ? report : null); + } + fs.writeFileSync(file, JSON.stringify({ ...report, completion: { ...completion, token: null } })); + expect(readDaemonShutdownRecord(dir)).toEqual({ report, completion: null }); +}); + +test.each([ + ['absent', undefined], + ['malformed', '{not json'], + ['empty', '{}'], + [ + 'missing lifetime', + JSON.stringify({ completion: { ...completion, processStartTime: undefined } }), + ], + ['missing token', JSON.stringify({ completion: { ...completion, token: undefined } })], + ['invalid outcome', JSON.stringify({ completion: { ...completion, outcome: 'maybe' } })], +])('unusable %s completion supplies no cleanup evidence', (_label, raw) => { + const dir = mkdtempForTestSync('agent-device-shutdown-record-'); + if (raw !== undefined) fs.writeFileSync(resolveDaemonShutdownReportPath(dir), raw); + expect(readDaemonShutdownRecord(dir)).toEqual({ report: null, completion: null }); +}); + +test.each([ + [completion, true], + [{ ...completion, pid: 9090 }, false], + [{ ...completion, processStartTime: 'another birth' }, false], + [{ ...completion, token: 'runtime-b' }, false], +])('completion matches the original runtime identity: %j', (query, matches) => { + expect(daemonShutdownCompletionAnswers(completion, query)).toBe(matches); +}); diff --git a/packages/daemon-contracts/src/daemon-shutdown-report.ts b/packages/daemon-contracts/src/daemon-shutdown-report.ts index 5208ad11ba..6b8d9fbd62 100644 --- a/packages/daemon-contracts/src/daemon-shutdown-report.ts +++ b/packages/daemon-contracts/src/daemon-shutdown-report.ts @@ -64,20 +64,31 @@ export function buildDaemonShutdownReport(outcome: DaemonShutdownOutcome): Daemo }; } -export function readDaemonShutdownReport(stateDir: string): DaemonShutdownReport | null { +export type DaemonShutdownRecord = Readonly<{ + report: DaemonShutdownReport | null; + completion: DaemonShutdownCompletion | null; +}>; + +export function readDaemonShutdownRecord(stateDir: string): DaemonShutdownRecord { try { - const parsed = JSON.parse( + const parsed: unknown = JSON.parse( fs.readFileSync(resolveDaemonShutdownReportPath(stateDir), 'utf8'), - ) as unknown; - if (!isProviderReleaseReport(parsed)) return null; - // A report left behind by a daemon that predates claim reporting still - // describes its provider releases honestly; it just knows nothing of claims. - return { ...parsed, claims: readClaimSection(parsed) }; + ); + return { + report: isProviderReleaseReport(parsed) + ? { providerReleases: parsed.providerReleases, claims: readClaimSection(parsed) } + : null, + completion: readCompletion(parsed), + }; } catch { - return null; + return { report: null, completion: null }; } } +export function readDaemonShutdownReport(stateDir: string): DaemonShutdownReport | null { + return readDaemonShutdownRecord(stateDir).report; +} + export function resolveDaemonShutdownReportPath(stateDir: string): string { return path.join(stateDir, SHUTDOWN_REPORT_FILE); } @@ -147,3 +158,45 @@ function isProviderReleaseRecord(value: unknown): value is ProviderReleaseRecord (record.provider === undefined || typeof record.provider === 'string') ); } + +/** The registration disposition proved before releasing its claim; lock release can still fail. */ +export type DaemonShutdownCompletion = Readonly<{ + pid: number; + processStartTime: string; + token: string; + outcome: 'released' | 'retained'; + unfinished: readonly string[]; +}>; + +export type DaemonShutdownCompletionQuery = Pick< + DaemonShutdownCompletion, + 'pid' | 'processStartTime' | 'token' +>; + +export function daemonShutdownCompletionAnswers( + completion: DaemonShutdownCompletion, + query: DaemonShutdownCompletionQuery, +): boolean { + return ( + completion.pid === query.pid && + completion.processStartTime === query.processStartTime && + completion.token === query.token + ); +} + +function readCompletion(value: unknown): DaemonShutdownCompletion | null { + if (!value || typeof value !== 'object') return null; + const record = (value as { completion?: Partial }).completion; + return record && + Number.isSafeInteger(record.pid) && + Number(record.pid) > 0 && + typeof record.processStartTime === 'string' && + record.processStartTime.trim().length > 0 && + typeof record.token === 'string' && + record.token.length > 0 && + (record.outcome === 'released' || record.outcome === 'retained') && + Array.isArray(record.unfinished) && + record.unfinished.every((line) => typeof line === 'string') + ? (record as DaemonShutdownCompletion) + : null; +} diff --git a/packages/daemon-contracts/src/daemon-stop.ts b/packages/daemon-contracts/src/daemon-stop.ts index 3158c1ef14..aa3862f2b7 100644 --- a/packages/daemon-contracts/src/daemon-stop.ts +++ b/packages/daemon-contracts/src/daemon-stop.ts @@ -153,7 +153,7 @@ function daemonRetirementError( ); } -function daemonNotRunningStopResult(): DaemonStopResult { +export function daemonNotRunningStopResult(): DaemonStopResult { return { stopped: false, mode: 'not-running', diff --git a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts index c4f289ec7a..5800e23526 100644 --- a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts @@ -5,12 +5,19 @@ import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/tes import { buildDaemonLifecycleRequest, resolveDaemonPaths, + type DaemonRuntimeOwnership, } from '@agent-device/daemon-contracts/daemon-resolution'; +import { + daemonShutdownCompletionAnswers, + readDaemonShutdownRecord, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import * as shutdown from './server-shutdown.ts'; -// A quiesce is the daemon's own answer to "may I leave?", so these tests boot the real runtime and -// ask over its real socket: the census, the closing of admission and the drain are one flow. +// A lifecycle request is the daemon's own answer to "may I leave?", so these tests boot the real +// runtime and ask over its real socket: the census, the closing of admission, the drain and the +// completion record a stop waits for are one flow. const probe = vi.hoisted(() => ({ /** Holds daemon startup after its servers listen, until released. */ startupGate: undefined as Promise | undefined, @@ -18,7 +25,21 @@ const probe = vi.hoisted(() => ({ registry: undefined as import('../../resources/lease-registry.ts').LeaseRegistry | undefined, held: undefined as Promise | undefined, entered: undefined as (() => void) | undefined, + diagnosticFailure: undefined as Error | undefined, })); +vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + emitDiagnostic: (...args: Parameters) => { + if (probe.diagnosticFailure) { + if (args[0].phase === 'daemon_fatal') throw probe.diagnosticFailure; + throw new Error('secondary diagnostic failure'); + } + return actual.emitDiagnostic(...args); + }, + }; +}); vi.mock('@agent-device/daemon-contracts/device-claims', async (importOriginal) => { const actual = @@ -72,7 +93,11 @@ vi.mock('../request-router.ts', async (importOriginal) => { }; }); -import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { + applicationLifecycleResourceFixture, + daemonRuntimeDepsFixture, +} from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; import { makeRepairArmedSession, makeSession } from '../../__tests__/support/session-factories.ts'; import { startDaemonRuntime } from './daemon-runtime.ts'; @@ -82,31 +107,39 @@ afterEach(() => { probe.entered = undefined; probe.store = undefined; probe.registry = undefined; + probe.diagnosticFailure = undefined; + vi.restoreAllMocks(); }); type Booted = { runtime: NonNullable>>; stateDir: string; exit: Promise; - quiesce(overrides?: { - token?: string; - pid?: number; - processStartTime?: string | null; - }): Promise; + quiesce(overrides?: LifecycleAddress): Promise; + shutdown(overrides?: LifecycleAddress): Promise; sessionList(): Promise; }; +type LifecycleAddress = { + token?: string; + pid?: number; + processStartTime?: string | null; + requestId?: string; + /** Sends the request with no `meta` at all, the shape a caller that tracks no id produces. */ + omitMeta?: boolean; +}; + async function bootDaemon( options: { serverMode?: 'socket' | 'dual'; - ownership?: 'embedded' | 'detached'; + ownership?: DaemonRuntimeOwnership; idleTimeoutMs?: number; + requestDrainMs?: number; } = {}, ): Promise { - const stateDir = mkdtempForTestSync('agent-device-daemon-quiesce-'); + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-'); const paths = resolveDaemonPaths(stateDir); const runtime = await startDaemonRuntime({ - ownership: options.ownership, composeDeps: daemonRuntimeDepsFixture(), host: createTestHostEnvironment({ env: { @@ -116,6 +149,9 @@ async function bootDaemon( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: String(options.idleTimeoutMs ?? 0), }, }), + ownership: options.ownership, + shutdownBudgets: + options.requestDrainMs === undefined ? undefined : { requestDrainMs: options.requestDrainMs }, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -127,25 +163,27 @@ async function bootDaemon( }; const socketPort = runtime.socketPort; const send = (request: object) => sendSocketRequest(socketPort, request); + const sendLifecycle = (mode: 'quiesce' | 'shutdown', overrides: LifecycleAddress) => { + const request = buildDaemonLifecycleRequest({ + mode, + token: overrides.token ?? runtime.token, + fence: { + pid: overrides.pid ?? info.pid, + processStartTime: + overrides.processStartTime !== undefined + ? overrides.processStartTime + : (info.processStartTime ?? null), + }, + requestId: overrides.requestId ?? `${mode}-test`, + }); + return overrides.omitMeta ? send({ ...request, meta: undefined }) : send(request); + }; return { runtime, stateDir, exit, - quiesce: (overrides = {}) => - send( - buildDaemonLifecycleRequest({ - mode: 'quiesce', - token: overrides.token ?? runtime.token, - fence: { - pid: overrides.pid ?? info.pid, - processStartTime: - overrides.processStartTime !== undefined - ? overrides.processStartTime - : (info.processStartTime ?? null), - }, - requestId: 'quiesce-test', - }), - ), + quiesce: (overrides = {}) => sendLifecycle('quiesce', overrides), + shutdown: (overrides = {}) => sendLifecycle('shutdown', overrides), sessionList: () => send({ token: runtime.token, @@ -326,8 +364,322 @@ test('a local HTTP quiesce drains an idle daemon like a socket one', async () => expect(await daemon.exit).toBe(0); }); +test('an embedded runtime records its close for every stop of that instance', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + const paths = resolveDaemonPaths(daemon.stateDir); + + expect(await daemon.shutdown({ requestId: 'embedded-stop-1' })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await daemon.exit).toBe(0); + + // The registration is gone by the time a close has a report, so this record is the only answer a + // stop of an embedded runtime can read. + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect(completion?.outcome).toBe('released'); + expect(completion?.unfinished).toEqual([]); + // The record names the instance that wrote it, so every stop addressed to that instance — including + // a second one that arrived after the first was answered — reads this same answer. + expect(completion?.pid).toBe(process.pid); + const stopped = { + pid: completion!.pid, + processStartTime: completion!.processStartTime, + token: completion!.token, + }; + expect(daemonShutdownCompletionAnswers(completion!, stopped)).toBe(true); + // A different instance is not answered by this record, so it cannot be told somebody else's shutdown. + expect(daemonShutdownCompletionAnswers(completion!, { ...stopped, pid: 999_999 })).toBe(false); + expect(fs.existsSync(paths.infoPath)).toBe(false); +}); + +test('a shutdown carrying no request id is accepted and still writes a record a stop can read', async () => { + // The answer is fenced to the instance, so nothing about the close depends on the caller having an + // id: a host that tracks none gets the same completion record as one that does. + const daemon = await bootDaemon({ ownership: 'embedded' }); + const paths = resolveDaemonPaths(daemon.stateDir); + + expect(await daemon.shutdown({ omitMeta: true })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await daemon.exit).toBe(0); + + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect(completion).toMatchObject({ pid: process.pid, outcome: 'released', unfinished: [] }); + expect( + daemonShutdownCompletionAnswers(completion!, { + pid: completion!.pid, + processStartTime: completion!.processStartTime, + token: completion!.token, + }), + ).toBe(true); +}); + +test('direct host close publishes the successful result once', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + await Promise.all([daemon.runtime.close(), daemon.runtime.close()]); + expect(readDaemonShutdownRecord(daemon.stateDir).completion).toMatchObject({ + outcome: 'released', + unfinished: [], + }); +}); + +test('a host that closes its own runtime with work in hand still records what it kept', async () => { + const daemon = await bootDaemon({ ownership: 'embedded', requestDrainMs: 50 }); + const paths = resolveDaemonPaths(daemon.stateDir); + let release: () => void = () => {}; + const entered = new Promise((resolve) => { + probe.entered = resolve; + }); + probe.held = new Promise((resolve) => { + release = resolve; + }); + daemon.sessionList().catch(() => undefined); + await entered; + + try { + await daemon.runtime.close(); + + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect(completion).toMatchObject({ pid: process.pid, outcome: 'retained' }); + expect(completion?.unfinished.join('\n')).toContain('ignored cancellation'); + expect(fs.existsSync(paths.infoPath)).toBe(true); + } finally { + release(); + } +}); + +test('a diagnostic failure starts listener close, retains unproved work and preserves the primary error', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + const paths = resolveDaemonPaths(daemon.stateDir); + const listeners = vi.spyOn(shutdown, 'closeDaemonServers'); + const primary = new Error('fatal diagnostic failed'); + probe.diagnosticFailure = primary; + const exited = expect(daemon.exit).rejects.toBe(primary); + await expect(daemon.runtime.close({ cause: new Error('shutdown cause') })).rejects.toBe(primary); + await exited; + expect(listeners).toHaveBeenCalledTimes(1); + expect(fs.existsSync(paths.infoPath)).toBe(true); + expect(fs.existsSync(paths.lockPath)).toBe(true); + expect(readDaemonShutdownRecord(paths.baseDir)).toMatchObject({ + report: null, + completion: { + outcome: 'retained', + unfinished: [expect.stringContaining('did not finish leaving')], + }, + }); +}); + +test('a close whose step rejects still answers the stop that asked for it', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-close-failure-'); + const paths = resolveDaemonPaths(stateDir); + const runtime = await startDaemonRuntime({ + composeDeps: daemonRuntimeDepsFixture(() => ({ + deviceRuntimeGateway: { + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture(), + shutdown: async () => { + throw new Error('gateway teardown rejected'); + }, + }, + })), + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'socket', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }), + ownership: 'embedded', + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + port: number; + token: string; + pid: number; + processStartTime?: string; + }; + const request = buildDaemonLifecycleRequest({ + mode: 'shutdown', + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId: 'failing-stop', + }); + + expect(await sendSocketRequest(info.port, request)).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + await expect(runtime.closed).rejects.toThrow('gateway teardown rejected'); + + // A stop would otherwise sit out its whole budget on a close that had already failed, so the record + // says what happened instead of saying nothing. + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect(completion?.pid).toBe(info.pid); + // A platform owner whose teardown rejected may still be driving devices, so the close keeps the + // registration and lock until this process exits — releasing them would let a second daemon take the + // same devices. The record says `retained` because that is what the state directory shows. + expect(completion?.outcome).toBe('retained'); + expect(completion?.unfinished.join('\n')).toContain('did not finish leaving'); + expect(fs.existsSync(paths.infoPath)).toBe(true); +}); + +test('a shutdown that drains work it could not cancel records what it kept', async () => { + const daemon = await bootDaemon({ ownership: 'embedded', requestDrainMs: 50 }); + const paths = resolveDaemonPaths(daemon.stateDir); + let release: () => void = () => {}; + const entered = new Promise((resolve) => { + probe.entered = resolve; + }); + // One request that answers nothing. Its own connection is still open when the listeners go away, so + // nothing about it is knowable from the client side: that is work a drain has to give up on, which is + // the one outcome a stop must be told about. + probe.held = new Promise((resolve) => { + release = resolve; + }); + daemon.sessionList().catch(() => undefined); + await entered; + + try { + expect(await daemon.shutdown({ requestId: 'draining-stop' })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + await daemon.runtime.close(); + + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect(completion?.pid).toBe(process.pid); + expect(completion?.outcome).toBe('retained'); + expect(completion?.unfinished.join('\n')).toContain('ignored cancellation'); + // The abandoned work holds the registration: releasing it would let a second daemon drive the same + // devices while this process may still be running one. + expect(fs.existsSync(paths.infoPath)).toBe(true); + } finally { + release(); + } +}); + +test('a detached daemon refuses a shutdown request and keeps serving', async () => { + const daemon = await bootDaemon(); + expect(reasonOf(await daemon.shutdown())).toBe('daemon-not-embedded'); + expect((await daemon.sessionList()).ok).toBe(true); + expect(readDaemonShutdownRecord(resolveDaemonPaths(daemon.stateDir).baseDir).completion).toBe( + null, + ); + await daemon.runtime.close(); + expect(readDaemonShutdownRecord(resolveDaemonPaths(daemon.stateDir).baseDir).completion).toBe( + null, + ); +}); + +test('an embedded runtime outlives the idle window a detached daemon reaps itself on', async () => { + // Booted first on purpose: both get the same window, so the embedded one's window ends no later than + // the detached one's. Booting it second would let this assertion run while its own window still had + // time left, which would prove nothing about the rule. + const embedded = await bootDaemon({ ownership: 'embedded', idleTimeoutMs: 40 }); + const detached = await bootDaemon({ idleTimeoutMs: 40 }); + const detachedPaths = resolveDaemonPaths(detached.stateDir); + try { + // Idle reap releases the state directory it reaped, so that is the fact to wait for. By then the + // embedded runtime has been idle at least as long, so a window that ended one daemon's life is + // proof enough that it did not end the other's. + await vi.waitFor(() => expect(fs.existsSync(detachedPaths.infoPath)).toBe(false), { + timeout: 5_000, + }); + // A gap between requests is not the owner asking to be shut down. + expect((await embedded.sessionList()).ok).toBe(true); + } finally { + await detached.runtime.close(); + await embedded.runtime.close(); + } +}); + +test('an embedded runtime refuses a quiesce, because its host decides when it leaves', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + expect(reasonOf(await daemon.quiesce())).toBe('daemon-owner-embedded'); + // Nothing was closed and no admission was left closed: the host's runtime still serves. + expect((await daemon.sessionList()).ok).toBe(true); + expect(await daemon.shutdown({ requestId: 'asked-to-leave' })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await daemon.exit).toBe(0); +}); + +test('a shutdown that arrives while the runtime is starting is refused, then accepted once it can drain', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-shutdown-startup-'); + const paths = resolveDaemonPaths(stateDir); + let openGate: () => void = () => {}; + probe.startupGate = new Promise((resolve) => { + openGate = resolve; + }); + const starting = startDaemonRuntime({ + composeDeps: daemonRuntimeDepsFixture(), + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'socket', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }), + ownership: 'embedded', + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + try { + for (let polls = 0; !fs.existsSync(paths.infoPath); polls += 1) { + if (polls >= 500) throw new Error('daemon did not publish daemon.json during startup'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + port: number; + token: string; + pid: number; + processStartTime?: string; + ownership?: string; + }; + // Published while it still had no drain path, and recorded as owned by a host. + expect(info.ownership).toBe('embedded'); + const shutdownRequest = buildDaemonLifecycleRequest({ + mode: 'shutdown', + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId: 'startup-shutdown', + }); + const refused = await sendSocketRequest(info.port, shutdownRequest); + expect(reasonOf(refused)).toBe('daemon-busy'); + + openGate(); + const runtime = await starting; + const exit = runtime.closed.then(() => 0); + expect(await sendSocketRequest(info.port, shutdownRequest)).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await exit).toBe(0); + // The record answers the instance the stop was fenced to, which is the one the record named before + // the close removed it. + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect( + completion && + daemonShutdownCompletionAnswers(completion, { + pid: info.pid, + processStartTime: info.processStartTime ?? '', + token: info.token, + }), + ).toBe(true); + } finally { + openGate(); + await starting.then((runtime) => runtime?.close()); + } +}); + test('a quiesce asked while the daemon is still starting up is refused as busy, then accepted', async () => { - const stateDir = mkdtempForTestSync('agent-device-daemon-quiesce-startup-'); + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-startup-'); const paths = resolveDaemonPaths(stateDir); let openGate: () => void = () => {}; probe.startupGate = new Promise((resolve) => { @@ -386,46 +738,3 @@ test('a quiesce asked while the daemon is still starting up is refused as busy, await starting.then((runtime) => runtime?.close()); } }); - -test('an embedded runtime outlives the idle window a detached daemon reaps itself on', async () => { - // Booted first on purpose: both get the same window, so the embedded one's window ends no later than - // the detached one's. Booting it second would let this assertion run while its own window still had - // time left, which would prove nothing about the rule. - const embedded = await bootDaemon({ ownership: 'embedded', idleTimeoutMs: 40 }); - const detached = await bootDaemon({ idleTimeoutMs: 40 }); - const detachedPaths = resolveDaemonPaths(detached.stateDir); - try { - // Idle reap releases the state directory it reaped, so that is the fact to wait for. By then the - // embedded runtime has been idle at least as long, so a window that ended one daemon's life is - // proof enough that it did not end the other's. - await vi.waitFor(() => expect(fs.existsSync(detachedPaths.infoPath)).toBe(false), { - timeout: 5_000, - }); - // A gap between requests is not the owner asking to be shut down. - expect((await embedded.sessionList()).ok).toBe(true); - } finally { - await detached.runtime.close(); - await embedded.runtime.close(); - } -}); - -test('an embedded runtime refuses a quiesce and remains serving until direct host close', async () => { - const daemon = await bootDaemon({ ownership: 'embedded' }); - const paths = resolveDaemonPaths(daemon.stateDir); - expect(JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')).ownership).toBe('embedded'); - expect(reasonOf(await daemon.quiesce())).toBe('daemon-owner-embedded'); - const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')); - const shutdown = await sendSocketRequest(info.port, { - ...buildDaemonLifecycleRequest({ - mode: 'quiesce', - token: info.token, - fence: { pid: info.pid, processStartTime: info.processStartTime }, - requestId: 'not-exposed', - }), - command: 'daemon_shutdown', - }); - expect(shutdown.ok).toBe(false); - expect((await daemon.sessionList()).ok).toBe(true); - await daemon.runtime.close(); - expect(fs.existsSync(paths.infoPath)).toBe(false); -}); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 11d4d929db..d60c4e9335 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -48,6 +48,7 @@ import { type WritableOutput, createPluginClose, createStartupRollback, + describeUnfinishedShutdownWork, settleWithinBudget, type ShutdownReport, type StartupRollback, @@ -88,6 +89,7 @@ import { import { tryAcquireDaemonRegistration, DAEMON_STARTUP_EXIT_CODES, + type DaemonRegistrationOwner, } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { recordDaemonStartupMarker, @@ -219,7 +221,12 @@ export type DaemonRuntimeOptions = { composeDeps: ComposeDaemonRuntimeDeps; /** Shutdown time budgets, for hosts and tests that need other than the defaults. */ shutdownBudgets?: { requestDrainMs?: number; pluginMs?: number; ownerMs?: number }; - /** The process owner. An embedded runtime ends through its host's direct close. */ + /** + * Who owns this process. A host that started this runtime in its own process passes `embedded`: the + * record says so, no idle timer ends the runtime, and it answers a shutdown request instead of a + * quiesce. Defaults to a detached daemon. `unknown` is not offered: it is what a reader answers for a + * record it cannot place, and a runtime always knows which of these two it is. + */ ownership?: DaemonRuntimeOwnership; }; @@ -514,6 +521,8 @@ async function startDaemonRuntimeOnHost( // well after this closure captures it. let shuttingDown = false; const inFlightRequests = new Set>(); + // Who owns this process decides what it publishes and which lifecycle request it answers. A host + // that starts the runtime passes `embedded`; the daemon executable leaves it detached. const ownership = options.ownership ?? 'detached'; const idleReap = createDaemonIdleReap({ sessionStore, @@ -532,12 +541,11 @@ async function startDaemonRuntimeOnHost( leaseSessionEnd.cancel(); }); - // The quiesce request is answered before dispatch and never joins `inFlightRequests`, so the - // count it reads needs no allowance for it. The servers listen and `daemon.json` is published - // before `shutdown` exists, and a daemon answers `draining` only once it has a drain path to run. + // Both lifecycle requests are answered before dispatch and never join `inFlightRequests`, so the + // count they read needs no allowance for them. The servers listen and `daemon.json` is published + // before `close` exists, and a daemon answers `draining` only once it has a drain path to run. let startingUp = true; - const quiesce = createDaemonLifecycleAdmission({ - ownership: () => ownership, + const lifecycleAdmission = createDaemonLifecycleAdmission({ token, identity: { pid: daemonIdentity.pid, processStartTime: daemonIdentity.startTime }, sessionStore, @@ -547,9 +555,10 @@ async function startDaemonRuntimeOnHost( drain: () => { void close(); }, + ownership: () => ownership, }); - const handleRequest: DaemonInvokeFn = quiesce.admit(async (req) => { + const handleRequest: DaemonInvokeFn = lifecycleAdmission.admit(async (req) => { if (shuttingDown) return daemonShuttingDownResponse(); idleReap.cancel(); const dispatch = runtimeContext.run(() => dispatchRequest(req)); @@ -597,7 +606,12 @@ async function startDaemonRuntimeOnHost( await deps.ownerLifecycle.clearDaemonLockConfiguration(); }); rollback.onRollback(async () => { - await finishDaemonRegistration({ registration, infoPath, logPath }); + await finishDaemonRegistration({ + registration, + infoPath, + logPath, + close: { retainRegistration: false, unfinished: ['Daemon startup rolled back.'] }, + }); }); const startupPhasesPath = resolveDaemonStartupPhasesPath(baseDir, process.pid); const recordStartupPhase = (phase: Parameters[1]): void => @@ -744,13 +758,10 @@ async function startDaemonRuntimeOnHost( }).catch(() => {}); }); - let registrationFinished = false; - let lockConfigurationCleared = false; - // Work that ignored cancellation may still be running, so this process keeps the registration - // and lock it holds: releasing them would let a second daemon drive the same devices. let abandonedRequests = 0; - let platformOwnersPending = false; + let platformOwnersPending = true; const hasUnfinishedWork = () => abandonedRequests > 0 || platformOwnersPending; + let shutdownOutcome: Parameters[0]; const drainInFlightRequests = async (): Promise => { const total = inFlightRequests.size; if (total === 0) return { drained: 0, abandoned: 0 }; @@ -772,6 +783,7 @@ async function startDaemonRuntimeOnHost( } return { drained: total - abandonedRequests, abandoned: abandonedRequests }; }; + const closeSteps = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); @@ -779,12 +791,10 @@ async function startDaemonRuntimeOnHost( clearInterval(leaseExpirySweep); shuttingDown = true; stopMetadataLossWatch(); - if (closeOptions.cause) { - await emitFatalDiagnostic(closeOptions.cause); - } // Cancellation and the listener close run together: a connection held by a request that ignores // cancellation is force-closed at the same budget the drain gives it. const serversClosed = closeDaemonServers(servers, requestDrainTimeoutMs); + if (closeOptions.cause) await emitFatalDiagnostic(closeOptions.cause); const requests = await drainInFlightRequests(); await serversClosed; await leaseSessionEnd.drain(); @@ -833,7 +843,6 @@ async function startDaemonRuntimeOnHost( }, }); expiredProviderLeaseReleaser.shutdown(); - platformOwnersPending = true; const ownerClose = await settleWithinBudget( () => deviceRuntimeGateway.shutdown(), options.shutdownBudgets?.ownerMs ?? DAEMON_PLATFORM_OWNER_SHUTDOWN_TIMEOUT_MS, @@ -849,24 +858,8 @@ async function startDaemonRuntimeOnHost( }); } const stoppedRunners = await applicationLifecycle.finalizeDaemonShutdown(); - // Best effort: stop the PNG worker so an in-flight job cannot delay exit. - await Promise.race([ - releasePngWorker().catch(() => {}), - sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), - ]); - if (!hasUnfinishedWork()) { - registrationFinished = true; - await finishDaemonRegistration({ - registration, - infoPath, - logPath, - outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, - }); - lockConfigurationCleared = true; - await deps.ownerLifecycle.clearDaemonLockConfiguration(); - } const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); - return { + const report: ShutdownReport = { providerReleases: { completed: providerReleaseDrain.released.map((lease) => lease.leaseId), pending: providerReleaseDrain.pending @@ -880,25 +873,46 @@ async function startDaemonRuntimeOnHost( sessions, plugins, }; + shutdownOutcome = { + providerReleases: providerReleaseDrain, + claims: shutdownClaimLedger.claims, + }; + return report; }; - // A step that rejects must not strand what the later steps release: the PNG worker hold, the - // provider runtimes, the registration and lock, and the runtime's own state. const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { + let report: ShutdownReport | undefined; try { - return await closeSteps(closeOptions); + report = await closeSteps(closeOptions); + return report; } finally { await Promise.allSettled([ - releasePngWorker(), - pluginClose.close(), - registrationFinished || hasUnfinishedWork() - ? undefined - : finishDaemonRegistration({ registration, infoPath, logPath }), - lockConfigurationCleared || hasUnfinishedWork() - ? undefined - : deps.ownerLifecycle.clearDaemonLockConfiguration(), + Promise.race([ + Promise.resolve().then(releasePngWorker), + sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), + ]), + Promise.resolve().then(() => pluginClose.close()), ]); - if (!hasUnfinishedWork()) runtimeContext.dispose(); + await Promise.allSettled([ + finishDaemonRegistration({ + registration, + infoPath, + logPath, + outcome: shutdownOutcome, + close: { + retainRegistration: hasUnfinishedWork(), + unfinished: report + ? describeUnfinishedShutdownWork(report) + : ['Daemon shutdown: a close step failed, so the runtime did not finish leaving.'], + }, + }), + ]); + if (!hasUnfinishedWork()) { + try { + await deps.ownerLifecycle.clearDaemonLockConfiguration(); + } catch {} + runtimeContext.dispose(); + } } }; diff --git a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts index ba5ccbc2d9..b47c2a73cc 100644 --- a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts +++ b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts @@ -37,9 +37,10 @@ export async function finishDaemonRegistration(params: { infoPath: string; logPath: string; outcome?: Parameters[0]; + close?: Parameters[1]; }): Promise { try { - const removal = await params.registration.finish(params.outcome); + const removal = await params.registration.finish(params.outcome, params.close); if (removal.state !== 'removed' && removal.state !== 'absent') { await emitDaemonDiagnostic(params.logPath, 'daemon_info_removal_declined', { infoPath: params.infoPath, diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap-lifecycle.test.ts b/packages/daemon/src/sessions/server/daemon-idle-reap-lifecycle.test.ts new file mode 100644 index 0000000000..78f6a5fa0f --- /dev/null +++ b/packages/daemon/src/sessions/server/daemon-idle-reap-lifecycle.test.ts @@ -0,0 +1,251 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, test, vi } from 'vitest'; +import { + buildDaemonLifecycleRequest, + DAEMON_LIFECYCLE_COMMANDS, + type DaemonLifecycleMode, + type DaemonRuntimeOwnership, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { makeIosSession } from '../../__tests__/support/session-factories.ts'; +import { SessionStore } from '../../core/session-store.ts'; +import type { DaemonRequest } from '../../core/daemon-request.ts'; +import { createDaemonLifecycleAdmission, daemonShuttingDownResponse } from './daemon-idle-reap.ts'; + +// Both lifecycle requests are answered by one seam, so these tests ask that seam and check what it +// did to admission: the two modes share the token check, the instance fence, and the rule that an +// answer and the closing of admission are one step. + +const TOKEN = 'daemon-token'; +const FENCE = { pid: 4242, processStartTime: 'Fri Oct 17 09:00:00 2026' }; + +let stateDir: string; +let sessionStore: SessionStore; + +beforeEach(() => { + stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-admission-'); + sessionStore = new SessionStore(path.join(stateDir, 'sessions')); +}); + +afterEach(() => { + fs.rmSync(stateDir, { recursive: true, force: true }); +}); + +type Seam = { + admit: (handler: Handler) => (request: DaemonRequest) => Promise; + drained: ReturnType; + served: MockHandler; +}; + +type Handler = (request: DaemonRequest) => Promise; +type MockHandler = Handler & ReturnType; + +function makeSeam( + options: { + ownership?: DaemonRuntimeOwnership; + identity?: { pid: number; processStartTime: string | null }; + startingUp?: boolean; + } = {}, +): Seam { + const drained = vi.fn(); + const admission = createDaemonLifecycleAdmission({ + token: TOKEN, + identity: options.identity ?? FENCE, + sessionStore, + leaseRegistry: { listActiveLeases: () => [] }, + inFlightRequestCount: () => 0, + startingUp: () => options.startingUp ?? false, + drain: drained, + ownership: () => options.ownership ?? 'detached', + }); + return { + admit: admission.admit, + drained, + served: vi.fn(async () => ({ ok: true })) as MockHandler, + }; +} + +function lifecycleRequest( + mode: DaemonLifecycleMode, + overrides: { + token?: string; + fence?: { pid: number; processStartTime: string | null }; + requestId?: string; + } = {}, +): DaemonRequest { + return buildDaemonLifecycleRequest({ + mode, + token: overrides.token ?? TOKEN, + fence: overrides.fence ?? FENCE, + requestId: overrides.requestId ?? `${mode}-test`, + }); +} + +function ordinaryRequest(): DaemonRequest { + return { token: TOKEN, session: 'default', command: 'devices', positionals: [] }; +} + +/** One open session, which is work a stop-if-idle must not walk away from. */ +function openOneSession(): void { + sessionStore.publish('busy-session', makeIosSession('busy-session')); +} + +function reasonOf(response: DaemonResponse): unknown { + return response.ok ? undefined : response.error.details?.reason; +} + +async function settled(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + +test('a lifecycle request is answered at the seam and never reaches the handler it guards', async () => { + const { admit, drained, served } = makeSeam(); + assert.deepEqual(await admit(served)(lifecycleRequest('quiesce')), { + ok: true, + data: { outcome: 'draining' }, + }); + await settled(); + assert.equal(drained.mock.calls.length, 1); + assert.equal(served.mock.calls.length, 0); +}); + +test('a detached daemon refuses a shutdown, and a host-owned runtime refuses a quiesce', async () => { + for (const [ownership, refused, accepted] of [ + ['detached', 'shutdown', 'quiesce'], + ['embedded', 'quiesce', 'shutdown'], + ] as const) { + const { admit, drained, served } = makeSeam({ ownership }); + assert.equal( + reasonOf(await admit(served)(lifecycleRequest(refused))), + ownership === 'detached' ? 'daemon-not-embedded' : 'daemon-owner-embedded', + ownership, + ); + await settled(); + assert.equal(drained.mock.calls.length, 0, `${ownership}: a refusal never drained`); + // A refusal is not a retirement: the daemon keeps serving, and the other mode still works. + assert.deepEqual(await admit(served)(lifecycleRequest(accepted)), { + ok: true, + data: { outcome: 'draining' }, + }); + } +}); + +test('a host-owned runtime drains work in hand, where a detached daemon refuses it', async () => { + openOneSession(); + + const held = makeSeam({ ownership: 'detached' }); + const refusal = await held.admit(held.served)(lifecycleRequest('quiesce')); + assert.equal(reasonOf(refusal), 'daemon-busy'); + if (!refusal.ok) assert.equal(refusal.error.details?.sessions, 1); + await settled(); + assert.equal(held.drained.mock.calls.length, 0); + // The refusal reopened admission, so this daemon is still serving its sessions. + assert.deepEqual(await held.admit(held.served)(ordinaryRequest()), { ok: true }); + + const draining = makeSeam({ ownership: 'embedded' }); + assert.deepEqual(await draining.admit(draining.served)(lifecycleRequest('shutdown')), { + ok: true, + data: { outcome: 'draining' }, + }); + await settled(); + assert.equal(draining.drained.mock.calls.length, 1); +}); + +test('a runtime still starting accepts neither mode and keeps serving while it starts', async () => { + for (const mode of ['quiesce', 'shutdown'] as const) { + const { admit, drained, served } = makeSeam({ + ownership: mode === 'shutdown' ? 'embedded' : 'detached', + startingUp: true, + }); + const refusal = await admit(served)(lifecycleRequest(mode)); + assert.equal(reasonOf(refusal), 'daemon-busy', mode); + if (!refusal.ok) assert.equal(refusal.error.details?.startingUp, true, mode); + await settled(); + assert.equal(drained.mock.calls.length, 0, mode); + assert.deepEqual( + await admit(served)(ordinaryRequest()), + { ok: true }, + `${mode}: a refusal keeps admission open`, + ); + } +}); + +test('both modes refuse a wrong token, a remote origin, and any other instance', async () => { + for (const mode of ['quiesce', 'shutdown'] as const) { + const ownership = mode === 'shutdown' ? 'embedded' : 'detached'; + + const seam = makeSeam({ ownership }); + const wrongToken = await seam.admit(seam.served)(lifecycleRequest(mode, { token: 'other' })); + assert.equal(!wrongToken.ok && wrongToken.error.code, 'UNAUTHORIZED', mode); + assert.equal(seam.drained.mock.calls.length, 0, mode); + + const remote = await seam.admit(seam.served)({ + ...lifecycleRequest(mode), + internal: { publicNetworkOnly: true }, + }); + assert.equal(!remote.ok && remote.error.code, 'UNAUTHORIZED', mode); + + for (const [label, fence] of [ + ['recycled pid', { pid: FENCE.pid + 1, processStartTime: FENCE.processStartTime }], + ['other lifetime', { pid: FENCE.pid, processStartTime: 'Sat Oct 18 09:00:00 2026' }], + ['no recorded birth', { pid: FENCE.pid, processStartTime: null }], + ] as const) { + const addressed = makeSeam({ ownership }); + const response = await addressed.admit(addressed.served)(lifecycleRequest(mode, { fence })); + assert.equal(reasonOf(response), 'daemon-instance-mismatch', `${mode} ${label}`); + await settled(); + assert.equal(addressed.drained.mock.calls.length, 0, `${mode} ${label}`); + } + } + // A runtime that cannot prove its own birth cannot be addressed at all, in either mode: a pid alone + // cannot tell this lifetime from the process that recycled its number, and a drain would end a + // stranger's daemon. Admission stays open, so that daemon is still serving. + for (const mode of ['quiesce', 'shutdown'] as const) { + const unprovable = makeSeam({ + ownership: mode === 'shutdown' ? 'embedded' : 'detached', + identity: { ...FENCE, processStartTime: null }, + }); + const refused = unprovable.served; + assert.equal( + reasonOf(await unprovable.admit(refused)(lifecycleRequest(mode))), + 'daemon-instance-mismatch', + mode, + ); + await settled(); + assert.equal(unprovable.drained.mock.calls.length, 0, `${mode}: a refusal never drained`); + assert.deepEqual( + await unprovable.admit(refused)(ordinaryRequest()), + { ok: true }, + `${mode}: an unaddressable identity still serves its own requests`, + ); + } +}); + +test('a second lifecycle request after one was accepted is answered as draining, not re-drained', async () => { + const { admit, drained } = makeSeam({ ownership: 'embedded' }); + assert.deepEqual(await admit(async () => ({ ok: true }))(lifecycleRequest('shutdown')), { + ok: true, + data: { outcome: 'draining' }, + }); + assert.deepEqual( + await admit(async () => ({ ok: true }))(lifecycleRequest('shutdown', { requestId: 'second' })), + { ok: true, data: { outcome: 'draining' } }, + ); + await settled(); + assert.equal(drained.mock.calls.length, 1); +}); + +test('every request that is not a lifecycle one is refused once a drain was accepted', async () => { + const { admit, served } = makeSeam({ ownership: 'embedded' }); + await admit(served)(lifecycleRequest('shutdown')); + assert.deepEqual(await admit(served)(ordinaryRequest()), daemonShuttingDownResponse()); + assert.equal(served.mock.calls.length, 0); +}); + +test('the two requests ride distinct commands', () => { + assert.equal(DAEMON_LIFECYCLE_COMMANDS.quiesce, 'daemon_quiesce'); + assert.equal(DAEMON_LIFECYCLE_COMMANDS.shutdown, 'daemon_shutdown'); +}); diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts b/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts index e7b0b83e25..9e51d5e792 100644 --- a/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts +++ b/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts @@ -7,7 +7,6 @@ import { SessionStore } from '../../core/session-store.ts'; import type { SessionState } from '../../core/session-state.ts'; import { createDaemonIdleReap, - createDaemonLifecycleAdmission, hasActiveRecording, hasReapBlockingOpenSessions, isDaemonIdle, @@ -327,37 +326,3 @@ test("a host that owns the runtime's lifetime never has it reaped on a timer", a assert.equal(reaped, 0); assert.equal(idleReap.idleMs, 0); }); - -test('a daemon that cannot prove its start time refuses every quiesce', async () => { - const drain = vi.fn(); - const quiesce = createDaemonLifecycleAdmission({ - ownership: () => 'detached', - token: 'daemon-token', - identity: { pid: process.pid, processStartTime: null }, - sessionStore, - leaseRegistry: { listActiveLeases: () => [] }, - inFlightRequestCount: () => 0, - startingUp: () => false, - drain, - }); - const admitted = vi.fn(async () => ({ ok: true as const })); - - const response = await quiesce.admit(admitted)({ - token: 'daemon-token', - session: 'default', - command: 'daemon_quiesce', - positionals: [], - input: { pid: process.pid, processStartTime: null }, - }); - - assert.equal(response.ok, false); - assert.equal(!response.ok && response.error.details?.reason, 'daemon-instance-mismatch'); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(drain.mock.calls.length, 0); - // Admission stays open: a refusal never leaves the daemon closed. - assert.deepEqual(await quiesce.admit(admitted)(makeRequest('devices')), { ok: true }); -}); - -function makeRequest(command: string) { - return { token: 'daemon-token', session: 'default', command, positionals: [] }; -} diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap.ts b/packages/daemon/src/sessions/server/daemon-idle-reap.ts index db102c2858..45df16799b 100644 --- a/packages/daemon/src/sessions/server/daemon-idle-reap.ts +++ b/packages/daemon/src/sessions/server/daemon-idle-reap.ts @@ -18,17 +18,28 @@ import type { SessionStore } from '../../core/session-store.ts'; import type { SessionState } from '../../core/session-state.ts'; import { isUncommittedRepairSession } from '../../core/session-replay-transaction.ts'; -// Bounds the daemon's own lifetime when nothing is using it. Each -// AGENT_DEVICE_STATE_DIR spawns a dedicated daemon that otherwise never exits -// on its own, so short-lived sandboxes (deleted codex/claude worktrees, one-off -// CLI invocations) leave orphaned daemon processes behind indefinitely - -// observed accumulating (10+) in production. A stale daemon left running this -// way also keeps holding its iOS runner lease, which blocks a fresh daemon for -// the same device (see runner-lease.ts stale-lease takeover). +// Whether this daemon may leave now, and when it will. Two things live here because they answer one +// question from one census: // -// The window mirrors the runner idle-stop default. Idle retained runners stop with their daemon: -// their timer and listener observation cannot survive a handoff. Active eligible runners retain -// the graceful-shutdown adoption policy. +// - `createDaemonIdleReap` bounds the daemon's own lifetime when nothing is using it. Each +// AGENT_DEVICE_STATE_DIR spawns a dedicated daemon that otherwise never exits on its own, so +// short-lived sandboxes (deleted codex/claude worktrees, one-off CLI invocations) leave orphaned +// daemon processes behind indefinitely - observed accumulating (10+) in production. A stale daemon +// left running this way also keeps holding its iOS runner lease, which blocks a fresh daemon for the +// same device (see runner-lease.ts stale-lease takeover). +// +// The window mirrors the runner idle-stop default. Idle retained runners stop with their daemon: +// their timer and listener observation cannot survive a handoff. Active eligible runners retain the +// graceful-shutdown adoption policy. +// +// - `createDaemonLifecycleAdmission` answers, in place of dispatch, the two requests that ask this +// daemon to leave: `daemon_quiesce` (stop if idle, for a daemon a client may replace) and +// `daemon_shutdown` (drain, for a runtime an embedder owns). Both decide from the same census the +// idle-reap rule reads, so a stop cannot be granted by one owner and refused by the other. +// +// TODO(#3757): the file is named for the older half. A rename to a lifecycle-named module is a +// follow-up, so this PR does not spend a gate's rename tolerance on it. + const DAEMON_IDLE_REAP_DEFAULT_MS = 5 * 60_000; // AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS overrides the window; 0 disables idle @@ -159,7 +170,21 @@ export function createDaemonIdleReap(params: { return { noteActivity, cancel, idleMs }; } -/** Answers a fenced quiesce before dispatch and closes admission atomically. */ +/** + * The one seam that answers both ways a process asks this daemon to leave, in place of dispatch. + * + * ADR 0006: a daemon leaves for a newer one only when it holds nothing, and it decides that itself. + * A client that read "idle" and then signaled would race every other client of the state directory; + * the daemon is the one place the answer and the closing of admission can be one step. + * + * `quiesce` is that stop-if-idle verdict, and it is what a detached daemon answers: a host owns an + * embedded runtime and ends it itself, so a replacement asks the host instead of retiring it. + * `shutdown` is the explicit stop of an embedded runtime, where draining is the point: work in hand + * is canceled rather than refused, because the caller owns this process and asked for it to close. + * + * Both modes share the token check, the instance fence, the answer-before-dispatch rule, and the one + * `drain` the runtime owns, so a stop can never reach teardown by a route the host does not control. + */ export type DaemonLifecycleAdmission = Readonly<{ /** * Puts the lifecycle request and the closed admission it leaves behind in front of `handler`: a @@ -191,7 +216,7 @@ export function createDaemonLifecycleAdmission(params: { startingUp: () => boolean; /** Starts the drain once, after sending the response. */ drain: () => void; - /** A host-owned runtime refuses a quiesce. */ + /** Who owns this process, and so which of the two requests it may answer. */ ownership: () => DaemonRuntimeOwnership; }): DaemonLifecycleAdmission { let admitting = true; @@ -221,6 +246,7 @@ export function createDaemonLifecycleAdmission(params: { } // Each mode belongs to one owner: a host ends an embedded runtime it started, and the process that // started a detached daemon ends it by signalling its pid. Neither asks a stranger. + if (mode === 'shutdown' && params.ownership() !== 'embedded') return refuse(notEmbedded()); if (mode === 'quiesce' && params.ownership() === 'embedded') return refuse(ownerEmbedded()); // Admission closes and the census is read in the same synchronous step, so no request can be // admitted between the two; a refusal reopens admission in that same step. @@ -229,12 +255,17 @@ export function createDaemonLifecycleAdmission(params: { const census = readCensus(); // A daemon that is still starting has no drain path yet, so it accepts nothing, and a quiesce // that finds work in hand refuses too: replacing that daemon would end another client's session. - const held = isHeld(census); + // A shutdown drains instead of refusing — the caller owns this process and asked for it to + // close, canceling its work included — but only once a drain path exists. The census says which + // of the two applied, so a caller knows whether to retry or to ask again later. + const held = mode === 'quiesce' ? isHeld(census) : census.startingUp; if (held) { admitting = true; return refuse(busy(census)); } setImmediate(() => params.drain()); + // One ack shape for both modes, as quiesce's shipped one already is: the census belongs in a + // refusal that a caller must act on, and in the completion record, not in an agreement. return { ok: true, data: { outcome: 'draining' } }; }; @@ -270,6 +301,9 @@ function isHeld(census: DaemonLifecycleCensus): boolean { } function busy(census: DaemonLifecycleCensus): AppError { + // The refusal's shape is the one quiesce already shipped: the reason plus the census that explains + // it. A shutdown refuses for one of those counts (`startingUp`) and a stop reads that count rather + // than a second field naming the mode it already asked for. return new AppError('COMMAND_FAILED', 'Daemon is busy and was not asked to leave.', { reason: DAEMON_LIFECYCLE_REASONS.busy, ...census, @@ -282,6 +316,14 @@ function instanceMismatch(): AppError { }); } +function notEmbedded(): AppError { + return new AppError( + 'COMMAND_FAILED', + 'This daemon is not embedder-owned, so it does not accept a shutdown request.', + { reason: DAEMON_LIFECYCLE_REASONS.notEmbedded }, + ); +} + function ownerEmbedded(): AppError { return new AppError( 'COMMAND_FAILED', diff --git a/src/__tests__/daemon-registration-embedded-owner.test.ts b/src/__tests__/daemon-registration-embedded-owner.test.ts index 707fa332d1..fcf8d02bd6 100644 --- a/src/__tests__/daemon-registration-embedded-owner.test.ts +++ b/src/__tests__/daemon-registration-embedded-owner.test.ts @@ -1,12 +1,16 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import { afterEach, test, vi } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process'; -import { stopAndRetireDaemon } from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { + tryAcquireDaemonRegistration, + stopAndRetireDaemon, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import { stopRecordedDaemon } from '../daemon-client/daemon-client-stop.ts'; -import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { + readDaemonShutdownRecord, + resolveDaemonShutdownReportPath, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { mkdtempForTestSync } from '@agent-device/test-kit'; // What the registration owner may do to a record a host process owns: prove the lifetime is gone @@ -38,6 +42,13 @@ vi.mock('@agent-device/daemon-contracts/daemon-process', async (importOriginal) }; }); +const fields = { + socketPort: 4210, + token: 'token', + version: '0.0.0-test', + codeOrigin: 'checkout' as const, + codeSignature: 'signature', +}; const ownIdentity = readCurrentOwnerIdentity(); const deadIdentity = { pid: 999_999_999, startTime: 'dead-start' }; @@ -78,6 +89,16 @@ function writeEmbeddedRecord(paths: ReturnType, ident writeRecord(paths, identity); } +/** A live acquisition of a state directory, so the caller holds the claim it judges. */ +async function acquire( + paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-embedded-claim-')), +) { + const attempt = await tryAcquireDaemonRegistration(paths); + assert.equal(attempt.status, 'acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + return { paths, owner: attempt.owner }; +} + // A host owns an embedded runtime and ends it with its own close, so retiring that record from outside // would take the state directory out from under a process that is still serving. test('retirement of an embedded record proves the lifetime is gone and signals nothing', async () => { @@ -99,14 +120,6 @@ test('retirement of an embedded record proves the lifetime is gone and signals n assert.equal(result.termination?.status, 'retained'); } - await assert.rejects( - stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), paths), - (error: unknown) => - error instanceof AppError && - error.details?.reason === 'daemon-owner-embedded' && - /Close the host/.test(String(error.details.hint)), - ); - // The row that gives this file its teeth: the stub above accepts this pid for the signal path, so // only the record's owner stands between this stop and a SIGTERM to the test worker itself. assert.deepEqual(signalPath.attempted, [], 'an embedded owner is never reached for a signal'); @@ -180,12 +193,6 @@ test('retirement refuses to signal a record whose owner it cannot recognize', as // The reason differs from the embedded one: nothing here proves the pid is a daemon of its own, so // an operator fixes this by inspecting or updating the record, not by closing a host. assert.equal(result.reason, 'owner-unproven', ownership); - if (ownership === 'sideways') - await assert.rejects( - stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), paths), - (error: unknown) => - error instanceof AppError && error.details?.reason === 'daemon-owner-unrecognized', - ); assert.deepEqual( signalPath.attempted, [], @@ -219,3 +226,95 @@ test.each([ownIdentity, deadIdentity])( assert.equal(fs.existsSync(paths.infoPath), identity === ownIdentity); }, ); + +test('a close answers its stop through the release, and an acquisition clears the answer', async () => { + const { paths, owner } = await acquire(); + owner.publish({ ...fields, ownership: 'embedded' }); + + await owner.finish(undefined, { + retainRegistration: false, + unfinished: ['1 provider release left pending'], + }); + const record = readDaemonShutdownRecord(paths.baseDir).completion; + // The disposition is the removal this call performed, not the caller's guess about it. + assert.equal(record?.outcome, 'released'); + assert.deepEqual(record?.unfinished, ['1 provider release left pending']); + assert.equal(record?.pid, process.pid, 'the record names the instance a stop is fenced to'); + // This is the production write, so it is where the file mode is a fact rather than a fixture's + // choice: a state-directory file a stranger could read leaks what a daemon said about its own close. + assert.equal( + fs.statSync(resolveDaemonShutdownReportPath(paths.baseDir)).mode & 0o777, + 0o600, + 'the completion record is published at 0600', + ); + + const before = fs.readFileSync(resolveDaemonShutdownReportPath(paths.baseDir), 'utf8'); + await assert.rejects(owner.finish(undefined, { retainRegistration: true, unfinished: ['late'] })); + assert.equal(fs.readFileSync(resolveDaemonShutdownReportPath(paths.baseDir), 'utf8'), before); + + const next = await acquire(paths); + assert.equal( + readDaemonShutdownRecord(paths.baseDir).completion, + null, + "a new instance must not inherit an answer to somebody else's stop", + ); + await next.owner.finish(); +}); + +test.each(['intentional', 'unlink failure', 'publication failure'] as const)( + 'embedded finalization reports actual registration disposition: %s', + async (scenario) => { + const { paths, owner } = await acquire(); + owner.publish({ ...fields, ownership: 'embedded' }); + const primary = new Error('registration unlink failed'); + const originalUnlink = fs.unlinkSync; + const originalRename = fs.renameSync; + const recordPath = resolveDaemonShutdownReportPath(paths.baseDir); + let registrationPresentAtPublish: boolean | undefined; + vi.spyOn(fs, 'unlinkSync').mockImplementation((target) => { + if (target === paths.infoPath && scenario === 'unlink failure') throw primary; + return originalUnlink(target); + }); + vi.spyOn(fs, 'renameSync').mockImplementation((source, target) => { + if (target === recordPath) { + assert.equal(fs.existsSync(paths.lockPath), true); + registrationPresentAtPublish = fs.existsSync(paths.infoPath); + if (scenario === 'publication failure') throw new Error('publication failed'); + } + return originalRename(source, target); + }); + const finished = owner.finish(undefined, { + retainRegistration: scenario === 'intentional', + unfinished: ['unfinished work'], + }); + if (scenario === 'unlink failure') await assert.rejects(finished, (error) => error === primary); + else await finished; + const retained = scenario !== 'publication failure'; + assert.equal(registrationPresentAtPublish, retained); + assert.equal(fs.existsSync(paths.infoPath), retained); + assert.equal(fs.existsSync(paths.lockPath), retained); + assert.equal(readDaemonShutdownRecord(paths.baseDir).report, null); + assert.equal( + readDaemonShutdownRecord(paths.baseDir).completion?.outcome ?? null, + retained ? 'retained' : null, + ); + const next = await tryAcquireDaemonRegistration(paths); + assert.equal(next.status, retained ? 'busy' : 'acquired'); + if (next.status === 'acquired') await next.owner.finish(); + }, +); + +test('a spent embedded owner cannot overwrite a same-host successor', async () => { + const { paths, owner } = await acquire(); + owner.publish({ ...fields, ownership: 'embedded' }); + await owner.finish(undefined, { retainRegistration: false, unfinished: [] }); + const next = await acquire(paths); + next.owner.publish({ ...fields, ownership: 'embedded', token: 'next-runtime' }); + await next.owner.finish(undefined, { retainRegistration: true, unfinished: ['next work'] }); + const before = fs.readFileSync(resolveDaemonShutdownReportPath(paths.baseDir), 'utf8'); + await assert.rejects(owner.finish(undefined, { retainRegistration: false, unfinished: [] })); + assert.equal(fs.readFileSync(resolveDaemonShutdownReportPath(paths.baseDir), 'utf8'), before); + assert.equal(readDaemonShutdownRecord(paths.baseDir).completion?.token, 'next-runtime'); + assert.equal(fs.existsSync(paths.infoPath), true); + assert.equal(fs.existsSync(paths.lockPath), true); +}); diff --git a/src/cli/commands/__tests__/daemon.test.ts b/src/cli/commands/__tests__/daemon.test.ts index e9e9bc50b1..069eb7b05a 100644 --- a/src/cli/commands/__tests__/daemon.test.ts +++ b/src/cli/commands/__tests__/daemon.test.ts @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import { afterEach, expect, test, vi } from 'vitest'; import type { DaemonStopResult } from '@agent-device/daemon-contracts/daemon-stop'; -import { AppError } from '@agent-device/kernel/errors'; import { mkdtempForTestSync, bindTestCommandHost, @@ -331,14 +330,14 @@ test.each([ true, JSON.stringify({ pid: 555, processStartTime: 'start', ownership: 'embedded' }), 'daemon-clean-not-detached', - /Close the host/, + /Stop it without --clean/, ], [ 'unknown', true, JSON.stringify({ pid: 555, processStartTime: 'start', ownership: 'sideways' }), 'daemon-clean-not-detached', - /remove .*daemon\.json/, + /Stop it without --clean/, ], ['unreadable', true, '{not json', 'daemon-clean-not-detached', /remove .*daemon\.json/], ['unreadable', false, '{not json', 'daemon-record-unreadable', /Inspect or remove that record/], @@ -386,22 +385,18 @@ test('daemon status names a record whose owner it cannot recognize', async () => expect(render()).toContain('pid 321, owner-not-recognized'); }); -// A host-owned record reaches the central retirement through the one router, and at this layer the -// owner's rule is the contract: the refusal the retirement raises is what `daemon stop` reports, -// unchanged. What the real retirement raises for a live embedded owner, without signaling it, is -// pinned at that seam by src/__tests__/daemon-registration-embedded-owner.test.ts. -test('daemon stop routes a host-owned record through the one router and reports its refusal', async () => { +// A host-owned record never reaches the signal path a detached stop uses: it is answered by a shutdown +// request over the record's own transport, so a record that states no usable transport is refused before +// anything is asked or signaled, and the record stays. What a reachable embedded stop answers, and what +// the retirement raises for a live owner without signaling it, are pinned at those seams by +// src/daemon-client-embedded-stop.test.ts and src/__tests__/daemon-registration-embedded-owner.test.ts. +test('daemon stop routes a host-owned record through the one router and never the signal path', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-embedded-'); fs.writeFileSync( `${stateDir}/daemon.json`, JSON.stringify({ pid: 555, processStartTime: 'start-time', ownership: 'embedded' }), { mode: 0o600 }, ); - mocks.stopDaemon.mockRejectedValue( - new AppError('COMMAND_FAILED', 'The daemon was not stopped, because its owner ends it.', { - reason: 'daemon-owner-embedded', - }), - ); try { await expect( @@ -411,13 +406,10 @@ test('daemon stop routes a host-owned record through the one router and reports flags: { help: false, json: false, stateDir, version: false }, client: {} as never, }), - ).rejects.toMatchObject({ details: { reason: 'daemon-owner-embedded' } }); + ).rejects.toMatchObject({ details: { reason: 'daemon-embedded-stop-unreachable' } }); - // The refusal travels with the lifetime and owner from the one read, and nothing was cleaned. - expect(mocks.stopDaemon).toHaveBeenCalledWith({ - paths: expect.objectContaining({ baseDir: stateDir }), - observed: { pid: 555, startTime: 'start-time', ownership: 'embedded' }, - }); + // The embedded route owns this record end to end: no detached retirement, no cleanup, no output. + expect(mocks.stopDaemon).not.toHaveBeenCalled(); expect(mocks.cleanupDaemonOwner).not.toHaveBeenCalled(); expect(mocks.writeCommandOutput).not.toHaveBeenCalled(); expect(fs.readFileSync(`${stateDir}/daemon.json`, 'utf8')).toContain('"embedded"'); diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index b32b278c1e..85786b8c3e 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -46,7 +46,7 @@ export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, if (flags.clean === true && record.ownership !== 'absent' && record.ownership !== 'detached') { throw daemonCleanRefusedForUnprovenOwner(paths, record.ownership); } - const result = await stopRecordedDaemon(record, paths); + const result = await stopRecordedDaemon(record, { paths, host }); const cleanupOwner = readStopCleanupOwner(record); const shouldClean = flags.clean === true && cleanupOwner !== null && result.stopped; if (shouldClean) { @@ -60,7 +60,16 @@ export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, return true; }; -/** Refuses PID-wide cleanup when the registration cannot prove a detached daemon. */ +/** + * `--clean` walks the owner's whole process tree, which for a host process includes everything else + * that host runs. An embedded runtime's own close is the only boundary a stop may cross, so this + * refuses instead of reaching past it, and the stop without `--clean` still works. + * + * A record whose owner cannot be read is refused too: the tree this would walk may be a host's, and + * the reason cleanup is allowed for a detached daemon is that its record proves the process is a + * daemon of its own. This is not a bad flag — the flag is fine, the record is the problem — so it + * reports as a failed command, like the other ownership refusals. + */ function daemonCleanRefusedForUnprovenOwner( paths: DaemonPaths, record: DaemonStopIdentity['ownership'], @@ -77,12 +86,23 @@ function daemonCleanRefusedForUnprovenOwner( ); } +/** + * What to do next, which depends on whether a plain `daemon stop` has a route left for this record. + * A readable record naming a host or an unfamiliar owner does: that stop asks the running process to + * leave and never signals it. A record that cannot be read has none — the same stop refuses it for the + * same reason — so its hint names the record repair instead of sending the operator to a second refusal. + */ function daemonCleanRefusalHint( paths: DaemonPaths, record: DaemonStopIdentity['ownership'], ): string { - if (record === 'embedded') return 'Close the host that started this runtime.'; - return `Nothing was signaled. Check what owns that process, then repair the record: stop it by its own means and remove ${shellQuoteIfNeeded(`${paths.baseDir}/daemon.json`)} so a daemon can register again.`; + if (record === 'unreadable') { + return `Nothing was signaled. Check what owns that process, then repair the record: stop it by its own means and remove ${shellQuoteIfNeeded(`${paths.baseDir}/daemon.json`)} so a daemon can register again.`; + } + if (record === 'embedded') { + return 'Stop it without --clean, which drains it through its own close, or let the host that started it close it.'; + } + return 'Stop it without --clean, which asks the running process to leave and reports what it answers.'; } function renderDaemonStop( diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index d5e247aeaa..f8136ade2c 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -514,7 +514,7 @@ Runner and daemon lifecycle (applies to simulators too): Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. - A host-owned runtime records ownership and daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime. Takeover, stop, drain, and timeout recovery all ask the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one so the state directory stays usable. A host closes its own live runtime directly. daemon stop refuses a live host-owned or unrecognized owner without signaling it; --clean also requires a proven detached owner and refuses host-owned, unrecognized and unreadable records with daemon-clean-not-detached. Detached stop and cleanup use the lifetime and ownership from one registration read. Without --clean, an unreadable daemon.json fails stop with daemon-record-unreadable. + daemon stop --state-dir stops a detached daemon with process signals after verifying its identity. For a host-owned runtime, it sends an authenticated shutdown request over the registered local transport and waits for the close report without signaling the host. daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime. Takeover, drain, and timeout recovery ask the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one so the state directory stays usable. Stop can also ask a readable record with unrecognized ownership: an embedded runtime can close itself, but a detached answer fails with details.reason daemon-not-embedded and a record-repair hint, without signaling. For these shutdown requests, a record without usable transport or token fails with daemon-embedded-stop-unreachable. An unconfirmed close, including after connection failure, fails with daemon-shutdown-unconfirmed. Without --clean, an unreadable daemon.json fails stop with daemon-record-unreadable. Detached stop and cleanup use the lifetime and ownership from one registration read. A retained close returns stopped: true with warnings and cleanupConfidence: unknown (exit 0): it stopped serving, but resource cleanup is unconfirmed. --clean requires proven detached ownership; host-owned, unrecognized and unreadable records fail with daemon-clean-not-detached because that cleanup walks the owner's whole process tree. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client-embedded-stop.test.ts b/src/daemon-client-embedded-stop.test.ts new file mode 100644 index 0000000000..d4b8267b07 --- /dev/null +++ b/src/daemon-client-embedded-stop.test.ts @@ -0,0 +1,680 @@ +// One stop, end to end: a runtime a host process owns, the record it publishes, and the stop an outside +// caller performs. The host here is the test process itself, which is what makes the strongest assertion +// in this file: a stop that ever reached for a signal would take the test runner down with it. + +import fs from 'node:fs'; +import net from 'node:net'; +import { once } from 'node:events'; +import { spawn } from 'node:child_process'; +import { afterEach, expect, test, vi } from 'vitest'; +import { startDaemonRuntime } from '@agent-device/daemon'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { stopDaemon } from '@agent-device/daemon-contracts/daemon-stop'; +import { + daemonShutdownCompletionAnswers, + readDaemonShutdownRecord, + resolveDaemonShutdownReportPath, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { readProcessStartTime } from '@agent-device/host-kit/process'; +import { + closeLoopbackServer, + createTestHostEnvironment, + listenOnLoopback, + mkdtempForTestSync, +} from '@agent-device/test-kit'; +import { createDaemonRuntimeDeps } from './daemon-composition.ts'; +import { stopRecordedDaemon } from './daemon-client/daemon-client-stop.ts'; +import { readDaemonInfo } from './daemon-client/daemon-client-metadata.ts'; +import { sendRequest } from './daemon-client/daemon-client-transport.ts'; +import * as transport from './daemon-client/daemon-client-transport.ts'; +import * as processWait from '@agent-device/daemon-contracts/daemon-process'; +import { AppError } from '@agent-device/kernel/errors'; + +const stopProbe = vi.hoisted(() => ({ + request: undefined as Promise | undefined, + owner: undefined as Promise | undefined, + entered: undefined as (() => void) | undefined, +})); +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + stopProbe.request = undefined; + stopProbe.owner = undefined; + stopProbe.entered = undefined; +}); + +vi.mock('./platform-runtime.ts', () => ({ + androidObservation: {}, + createRequestPlatformProviders: () => ({ + run: async (_context: unknown, task: () => Promise) => await task(), + }), + createPlatformRuntimeGateway: () => ({ + applicationLifecycle: { + recoverStartupResources: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], + }, + inspectFacts: async () => { + throw new Error('unused'); + }, + bind: async () => { + throw new Error('unused'); + }, + shutdown: async () => { + await stopProbe.owner; + return []; + }, + }), + createPlatformDeviceInventoryGateways: () => ({}), +})); + +vi.mock('./provider-device-runtimes.ts', async () => ({ + createDaemonProviderRuntimeComposition: async () => ({ + runtimes: [], + platformModules: [], + // The composition is the one plugin-cleanup owner: its close answers with every plugin's outcomes. + shutdown: async () => [], + inventory: await ( + await import('./__tests__/test-utils/builtin-plugin-inventory.ts') + ).builtinPluginInventory(), + }), +})); + +/** + * A record whose owner field this build cannot place — the state a newer build's vocabulary, or a hand + * edit, produces. A running runtime is never in this state: it always knows whether it owns itself, so + * `unknown` is a reader's verdict about a file. The record is rewritten under a live runtime to produce + * exactly that, with everything else about the instance left intact. + */ +async function bootRuntimeWithUnplaceableRecordOwner(ownership: 'embedded' | 'detached') { + const booted = await bootRuntime(ownership); + const record = JSON.parse(fs.readFileSync(booted.paths.infoPath, 'utf8')) as Record< + string, + unknown + >; + record.ownership = 'sideways'; + fs.writeFileSync(booted.paths.infoPath, JSON.stringify(record), { mode: 0o600 }); + expect(readDaemonStopIdentity(booted.paths.infoPath).ownership).toBe('unknown'); + return booted; +} + +async function bootRuntime(ownership: 'embedded' | 'detached' = 'embedded') { + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-'); + const runtime = await startDaemonRuntime({ + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }), + ownership, + shutdownBudgets: { requestDrainMs: 25, ownerMs: 25 }, + stderr: { write: () => {} }, + stdout: { write: () => {} }, + composeDeps: async (input) => { + const deps = await createDaemonRuntimeDeps(input); + return { + ...deps, + requestRouter: { + ...deps.requestRouter, + providerDeviceRuntimeScope: async (task) => { + if (stopProbe.request) { + stopProbe.entered?.(); + await stopProbe.request; + } + return await task(); + }, + }, + }; + }, + }); + return { stateDir, paths: resolveDaemonPaths(stateDir), runtime }; +} + +/** The stop as the CLI performs it: read the record once, then route on what it states. */ +function stopViaSocket(paths: ReturnType) { + return stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { + paths, + host: createTestHostEnvironment(), + }); +} + +/** A port that was bound and given back, so a request to it is refused rather than served. */ +async function closedLoopbackPort(): Promise { + const server = net.createServer(); + const port = await listenOnLoopback(server); + await closeLoopbackServer(server); + return port; +} + +/** The lifetime a record can name for a child that has only just been spawned. */ +async function startTimeOf(pid: number): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + const startTime = readProcessStartTime(pid); + if (startTime?.trim()) return startTime; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error(`child ${pid} never became readable`); +} + +/** + * The signals this stop sent at a process. `0` is the liveness probe and is not one; a `kill(pid)` with + * no signal argument is SIGTERM, so an omitted signal counts as the signal it really is. + */ +function hostSignals(killSpy: { mock: { calls: Array } }) { + return killSpy.mock.calls.filter(([, signal]) => signal !== 0); +} + +test('a host-owned runtime publishes the owner a stop has to respect', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + try { + const record = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + pid: number; + ownership?: string; + }; + // The record names the host's own process, which is why the signal path is closed here: stopping + // this runtime means asking it, not ending the program that asked. + expect(record.pid).toBe(process.pid); + expect(record.ownership).toBe('embedded'); + expect(readDaemonStopIdentity(paths.infoPath).ownership).toBe('embedded'); + expect(readDaemonInfo(paths.infoPath)?.ownership).toBe('embedded'); + } finally { + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('stopping a host-owned runtime asks it to leave and reports what its close said', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + + try { + // The host here stays alive the whole time, which is the case a signal-based stop cannot answer: the + // runtime closes inside a process that outlives it, so the only proof of the close is the record that + // close writes. Reading it is what ends this wait, not a timeout. + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + warnings: [], + }); + expect(Date.now() - startedAt).toBeLessThan(10_000); + // The runtime left by its own close, and the record it wrote is what the stop waited for. + await expect(runtime.closed).resolves.toMatchObject({ requests: { abandoned: 0 } }); + const completion = readDaemonShutdownRecord(paths.baseDir).completion; + expect(completion?.outcome).toBe('released'); + expect(completion?.unfinished).toEqual([]); + // Every stop addressed to that instance reads this answer, and a stop addressed to another one + // reads nothing — a recycled pid must not inherit an answer it was not part of. + expect( + daemonShutdownCompletionAnswers(completion!, { + pid: completion!.pid, + processStartTime: completion!.processStartTime, + token: completion!.token, + }), + ).toBe(true); + expect( + daemonShutdownCompletionAnswers(completion!, { + pid: 42, + processStartTime: 'x', + token: 'other', + }), + ).toBe(false); + expect(fs.existsSync(paths.infoPath)).toBe(false); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('a stop performed without the shutdown route is refused and leaves the runtime serving', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + + try { + // A caller that goes straight to the retirement funnel skips the shutdown request the embedded route + // sends, and what it has left is a signal — exactly what must never reach a host process. `observed` + // carries the owner the caller read, and the funnel refuses on it rather than proving an exit it + // cannot prove. + const observed = { + pid: process.pid, + startTime: readProcessStartTime(process.pid), + ownership: 'embedded' as const, + }; + const refusal = await stopDaemon({ paths, observed }).then( + () => null, + (error: unknown) => error, + ); + expect(refusal).toMatchObject({ + code: 'COMMAND_FAILED', + details: { retirement: { reason: 'embedded-owner' } }, + }); + + expect(hostSignals(signals)).toEqual([]); + // The refusal changed nothing: the runtime the host owns is still there to close. + expect(readDaemonInfo(paths.infoPath)?.pid).toBe(process.pid); + expect(await runtime.close()).toMatchObject({ requests: { abandoned: 0 } }); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('two stops of one runtime both get the answer its close gives', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + + try { + // Started in the same tick, so both stops read the same embedded record and neither can be told it + // is already gone. The runtime admits one shutdown and answers the other with the same + // `draining` ack, so both are waiting on the one completion record this instance writes. + const [first, second] = await Promise.all([stopViaSocket(paths), stopViaSocket(paths)]); + + for (const [label, stopped] of [ + ['first', first], + ['second', second], + ] as const) { + expect(stopped, label).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + }); + expect(stopped.warnings, label).toEqual([]); + } + await expect(runtime.closed).resolves.toMatchObject({ requests: { abandoned: 0 } }); + expect(fs.existsSync(paths.infoPath)).toBe(false); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test.each(['request', 'owner'] as const)( + 'direct close retaining %s work answers two later stops with an old timestamp field', + async (kind) => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + let release = () => {}; + const held = new Promise((resolve) => { + release = resolve; + }); + let pending: Promise | undefined; + if (kind === 'request') { + stopProbe.request = held; + const entered = new Promise((resolve) => { + stopProbe.entered = resolve; + }); + pending = sendRequest( + readDaemonInfo(paths.infoPath)!, + { token: runtime.token, command: 'session_list', positionals: [], session: 'default' }, + 'http', + paths, + 5_000, + { host: createTestHostEnvironment(), timeoutRecovery: 'none' }, + ).catch(() => {}); + await entered; + } else { + stopProbe.owner = held; + } + try { + await runtime.close(); + const file = resolveDaemonShutdownReportPath(stateDir); + const record = JSON.parse(fs.readFileSync(file, 'utf8')); + expect(record.completion.outcome).toBe('retained'); + fs.writeFileSync( + file, + JSON.stringify({ + ...record, + completion: { ...record.completion, writtenAtMs: Date.now() - 61_000 }, + }), + ); + for (let attempt = 0; attempt < 2; attempt++) { + const stopped = await stopViaSocket(paths); + expect(stopped).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'unknown', + providerReleases: { status: 'completed' }, + }); + expect(stopped.warnings.join(' ')).toContain( + kind === 'request' ? 'ignored cancellation' : 'platform owners', + ); + expect(stopped.warnings.join(' ')).toContain('keeps its registration'); + } + expect(fs.existsSync(paths.infoPath)).toBe(true); + expect(fs.existsSync(paths.lockPath)).toBe(true); + expect(hostSignals(signals)).toEqual([]); + } finally { + release(); + await pending; + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }, +); + +/** + * A record naming a child process that is about to be killed, with a port nothing listens on: the shape a + * host leaves when it dies, or when its age makes its record the only thing a later stop can read. + */ +async function plantDeadWriterRecord(paths: ReturnType) { + const writer = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 60_000)'], { + stdio: 'ignore', + }); + const pid = writer.pid ?? -1; + const startTime = await startTimeOf(pid); + fs.writeFileSync( + paths.infoPath, + JSON.stringify({ + pid, + processStartTime: startTime, + token: 'local-secret', + // A port nothing listens on: the ask cannot reach a runtime that is already gone. + port: await closedLoopbackPort(), + ownership: 'embedded', + }), + ); + writer.kill('SIGKILL'); + await once(writer, 'exit'); + return { pid, processStartTime: startTime, token: 'local-secret' }; +} + +test('a stop whose runtime died before writing its report does not wait on a dead writer', async () => { + // A host killed between removing the registration and publishing its completion record leaves a stop + // with no writer. The lifetime is the second piece of evidence, and it has to answer: sitting out the + // whole budget to report an unconfirmed failure would describe a close that had plainly ended. + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-dead-writer-'); + const paths = resolveDaemonPaths(stateDir); + await plantDeadWriterRecord(paths); + const signals = vi.spyOn(process, 'kill'); + + try { + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject({ stopped: false, mode: 'not-running' }); + // Proof, not a timeout: the budget this stop would otherwise sit out is a minute long. + expect(Date.now() - startedAt).toBeLessThan(10_000); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test.each(['retained', 'released'] as const)( + 'a dead writer with %s completion and an old timestamp field reports its actual close', + async (outcome) => { + // A retained record answers a later stop while the lifetime that wrote it is still live. Once that + // process is gone, nothing holds the state directory any more, and reporting a graceful stop would + // send the operator away from the one thing left to do: the takeover path that clears the stale record. + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-dead-retention-'); + const paths = resolveDaemonPaths(stateDir); + const instance = await plantDeadWriterRecord(paths); + fs.writeFileSync( + resolveDaemonShutdownReportPath(paths.baseDir), + `${JSON.stringify({ + completion: { + ...instance, + outcome, + unfinished: ['1 in-flight request ignored cancellation'], + }, + writtenAtMs: Date.now() - 60_000 - 1000, + })}\n`, + { mode: 0o600 }, + ); + const signals = vi.spyOn(process, 'kill'); + + try { + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject( + outcome === 'retained' + ? { stopped: false, mode: 'not-running' } + : { stopped: true, mode: 'graceful', cleanupConfidence: 'unknown' }, + ); + expect(Date.now() - startedAt).toBeLessThan(10_000); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }, +); + +// A record can claim a host-owned runtime and still be unusable: a file that is not JSON, or one that +// names no transport. Neither is an absence, and `not-running` would tell the operator to move on while +// a host may be serving on that very state directory. The two refusals are separate because the fixes +// are: a file that cannot be parsed says nothing about who owns the process, so nothing is even addressed. +test('a stop of a record it cannot address refuses instead of reporting nothing runs', async () => { + for (const [label, record, reason, recordState] of [ + ['unreadable', '{not json', 'daemon-record-unreadable', undefined], + [ + 'no transport', + JSON.stringify({ + pid: process.pid, + processStartTime: readProcessStartTime(process.pid), + ownership: 'embedded', + }), + 'daemon-embedded-stop-unreachable', + 'incomplete', + ], + ] as const) { + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-record-'); + const paths = resolveDaemonPaths(stateDir); + fs.mkdirSync(paths.baseDir, { recursive: true }); + fs.writeFileSync(paths.infoPath, record, { mode: 0o600 }); + const signals = vi.spyOn(process, 'kill'); + + try { + const refusal = await stopViaSocket(paths).then( + () => null, + (error: unknown) => error, + ); + expect(refusal, label).toMatchObject({ + code: 'COMMAND_FAILED', + details: { + reason, + stateDir: paths.baseDir, + ...(recordState ? { record: recordState } : {}), + }, + }); + expect(hostSignals(signals), label).toEqual([]); + // Refusing changes nothing: whatever owns that record still does. + expect(fs.readFileSync(paths.infoPath, 'utf8'), label).toBe(record); + } finally { + signals.mockRestore(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } + } +}); + +// A record this build cannot place still gets the authenticated request rather than a signal: asking is +// free, and the runtime's own answer says whose process this is. What the answer must not do is name +// `daemon stop`, because that command refuses to signal a record whose owner it cannot prove either — the +// operator needs the record repair, which is the only fix for a file that states nothing usable. +test('a detached answer against an unreadable owner names the record repair, not daemon stop', async () => { + const { stateDir, paths, runtime } = await bootRuntimeWithUnplaceableRecordOwner('detached'); + const signals = vi.spyOn(process, 'kill'); + + try { + const refusal = await stopViaSocket(paths).then( + () => null, + (error: unknown) => error, + ); + // The runtime knows its own owner, and it is not a host: it answers the shutdown request the way a + // detached daemon does. + expect(refusal).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'daemon-not-embedded' }, + }); + const hint = String((refusal as { details: { hint: string } }).details.hint); + expect(hint).toMatch(/daemon\.json/); + expect(hint).not.toMatch(/daemon stop/); + expect(hostSignals(signals)).toEqual([]); + expect(await runtime.close()).toMatchObject({ requests: { abandoned: 0 } }); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// The same rewritten record under a runtime that really is host-owned. Asking is still the right move: +// the runtime answers for itself, agrees, and closes its own process. Nothing here signals, and the stop +// does not need the record to say who owns it — the answer is the proof. +test('an unrecognized owner under a host-owned runtime leaves by its own shutdown request', async () => { + const { stateDir, paths, runtime } = await bootRuntimeWithUnplaceableRecordOwner('embedded'); + const signals = vi.spyOn(process, 'kill'); + + try { + expect(await stopViaSocket(paths)).toMatchObject({ stopped: true, mode: 'graceful' }); + await expect(runtime.closed).resolves.toBeTruthy(); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('a second stop of a host-owned runtime reports nothing left to stop', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + + try { + expect(await stopViaSocket(paths)).toMatchObject({ stopped: true, mode: 'graceful' }); + await expect(runtime.closed).resolves.toBeTruthy(); + + // The instance is gone, so this is not a refused stop or an unconfirmed one: there is nothing there. + expect(await stopViaSocket(paths)).toMatchObject({ + stopped: false, + mode: 'not-running', + }); + } finally { + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test.each(['no answer', 'same-host successor'] as const)( + 'a live host with %s never supplies false completion', + async (scenario) => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-stop-fenced-')); + const original = { + pid: process.pid, + processStartTime: readProcessStartTime(process.pid)!, + token: 'runtime-a', + port: 4210, + ownership: 'embedded', + }; + fs.writeFileSync(paths.infoPath, JSON.stringify(original)); + const observed = readDaemonStopIdentity(paths.infoPath); + const successor = { ...original, token: 'runtime-b' }; + fs.writeFileSync(paths.infoPath, JSON.stringify(successor)); + vi.spyOn(processWait, 'waitForDaemonExit').mockResolvedValue({ exited: false, elapsedMs: 0 }); + const send = vi.spyOn(transport, 'sendRequest').mockImplementation(async (info, request) => { + expect(info.token).toBe('runtime-a'); + expect(request.token).toBe('runtime-a'); + if (scenario === 'same-host successor') + fs.writeFileSync( + resolveDaemonShutdownReportPath(paths.baseDir), + JSON.stringify({ + completion: { ...successor, outcome: 'released', unfinished: [] }, + providerReleases: { released: [{ leaseId: 'successor-only' }], pending: [] }, + }), + ); + throw new AppError('COMMAND_FAILED', 'connection refused', { + reason: 'daemon_transport_failure', + }); + }); + const signals = vi.spyOn(process, 'kill'); + vi.useFakeTimers(); + const stop = stopRecordedDaemon(observed, { paths, host: createTestHostEnvironment() }); + const result = expect(stop).rejects.toMatchObject({ + details: { reason: 'daemon-shutdown-unconfirmed' }, + }); + await vi.advanceTimersByTimeAsync(60_000); + await result; + expect(send).toHaveBeenCalledTimes(1); + expect(JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')).token).toBe('runtime-b'); + expect(hostSignals(signals)).toEqual([]); + }, +); + +test.each([true, false])('startup retry requires the typed busy reason (%s)', async (typed) => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-stop-startup-')); + const original = { + pid: process.pid, + processStartTime: readProcessStartTime(process.pid)!, + token: 'starting-runtime', + port: 4210, + ownership: 'embedded', + }; + fs.writeFileSync(paths.infoPath, JSON.stringify(original)); + const error = new AppError('COMMAND_FAILED', 'The daemon is still starting up.', { + ...(typed ? { reason: 'daemon-busy' } : {}), + startingUp: true, + inFlightRequests: 0, + sessions: 0, + repairSessions: 0, + leases: 0, + }); + const send = vi.spyOn(transport, 'sendRequest').mockImplementation(async () => { + fs.writeFileSync( + resolveDaemonShutdownReportPath(paths.baseDir), + JSON.stringify({ + completion: { ...original, outcome: 'released', unfinished: [] }, + providerReleases: { released: [{ leaseId: 'original-close' }], pending: [] }, + }), + ); + throw error; + }); + vi.useFakeTimers(); + const stop = stopViaSocket(paths); + const checked = typed + ? expect(stop).resolves.toMatchObject({ + stopped: true, + mode: 'graceful', + providerReleases: { released: [{ leaseId: 'original-close' }] }, + }) + : expect(stop).rejects.toBe(error); + await vi.advanceTimersByTimeAsync(250); + await checked; + expect(send).toHaveBeenCalledTimes(1); +}); + +test('an unreachable embedded record without a birth fails promptly without guessing its lifetime', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-stop-no-birth-')); + const record = JSON.stringify({ + pid: process.pid, + token: 'unreachable-runtime', + httpPort: await closedLoopbackPort(), + ownership: 'embedded', + }); + fs.writeFileSync(paths.infoPath, record); + const signals = vi.spyOn(process, 'kill'); + const startedAt = Date.now(); + try { + await expect(stopViaSocket(paths)).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'daemon-shutdown-unconfirmed' }, + }); + expect(Date.now() - startedAt).toBeLessThan(5_000); + expect(fs.readFileSync(paths.infoPath, 'utf8')).toBe(record); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + } +}); diff --git a/src/daemon-client-released-close.test.ts b/src/daemon-client-released-close.test.ts new file mode 100644 index 0000000000..689b27725c --- /dev/null +++ b/src/daemon-client-released-close.test.ts @@ -0,0 +1,153 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import { expect, test, vi } from 'vitest'; +import { startDaemonRuntime } from '@agent-device/daemon'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { resolveDaemonShutdownReportPath } from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { isProcessAlive } from '@agent-device/host-kit/process'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { createDaemonRuntimeDeps } from './daemon-composition.ts'; +import { readDaemonInfo } from './daemon-client/daemon-client-metadata.ts'; +import { stopRecordedDaemon } from './daemon-client/daemon-client-stop.ts'; +import * as transport from './daemon-client/daemon-client-transport.ts'; + +const closeStep = vi.hoisted(() => ({ + gate: undefined as Promise | undefined, + entered: undefined as (() => void) | undefined, +})); +vi.mock('./platform-runtime.ts', () => ({ + androidObservation: {}, + createRequestPlatformProviders: () => ({ + run: async (_: unknown, task: () => Promise) => await task(), + }), + createPlatformDeviceInventoryGateways: () => ({}), + createPlatformRuntimeGateway: () => ({ + applicationLifecycle: { + recoverStartupResources: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], + }, + inspectFacts: async () => { + throw new Error('unused'); + }, + bind: async () => { + throw new Error('unused'); + }, + shutdown: async () => { + closeStep.entered?.(); + await closeStep.gate; + return []; + }, + }), +})); +vi.mock('./provider-device-runtimes.ts', () => ({ + createDaemonProviderRuntimeComposition: async () => ({ + runtimes: [], + platformModules: [], + shutdown: async () => [], + inventory: [], + }), +})); + +test('two stops after direct host close loses its listener receive its released cleanup', async () => { + const stateDir = mkdtempForTestSync('agent-device-direct-released-'); + const paths = resolveDaemonPaths(stateDir); + const host = createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }); + let release = () => {}; + closeStep.gate = new Promise((resolve) => { + release = resolve; + }); + const entered = new Promise((resolve) => { + closeStep.entered = resolve; + }); + const runtime = await startDaemonRuntime({ + host, + ownership: 'embedded', + composeDeps: createDaemonRuntimeDeps, + shutdownBudgets: { ownerMs: 10_000 }, + stdout: { write: () => {} }, + stderr: { write: () => {} }, + }); + const info = readDaemonInfo(paths.infoPath)!; + const original = fs.readFileSync(paths.infoPath, 'utf8'); + const signals = vi.spyOn(process, 'kill'); + const closing = runtime.close(); + try { + await entered; + await expect( + new Promise((resolve, reject) => { + const socket = net.createConnection({ host: '127.0.0.1', port: info.httpPort! }); + socket.on('connect', () => { + socket.destroy(); + resolve('still listening'); + }); + socket.on('error', reject); + }), + ).rejects.toMatchObject({ code: 'ECONNREFUSED' }); + expect(fs.readFileSync(paths.infoPath, 'utf8')).toBe(original); + const failures: unknown[] = []; + const send = transport.sendRequest; + vi.spyOn(transport, 'sendRequest').mockImplementation(async (...args) => { + try { + return await send(...args); + } catch (error) { + failures.push(error); + throw error; + } + }); + let settled = 0; + const started = Date.now(); + const stops = [0, 1].map(() => + stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { paths, host }).then((result) => { + settled++; + return result; + }), + ); + await vi.waitFor(() => expect(failures).toHaveLength(2)); + for (const failure of failures) + expect(failure).toMatchObject({ details: { reason: 'daemon_transport_failure' } }); + expect(settled).toBe(0); + expect(fs.readFileSync(paths.infoPath, 'utf8')).toBe(original); + release(); + const report = await closing; + const results = await Promise.all(stops); + expect(Date.now() - started).toBeLessThan(10_000); + expect(report).toMatchObject({ requests: { abandoned: 0 }, providerReleases: { pending: [] } }); + for (const result of results) + expect(result).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + warnings: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + }); + expect( + JSON.parse(fs.readFileSync(resolveDaemonShutdownReportPath(stateDir), 'utf8')).completion, + ).toMatchObject({ + token: info.token, + pid: info.pid, + processStartTime: info.processStartTime, + outcome: 'released', + unfinished: [], + }); + expect(fs.existsSync(paths.infoPath)).toBe(false); + expect(fs.existsSync(paths.lockPath)).toBe(false); + expect(isProcessAlive(process.pid)).toBe(true); + expect(signals.mock.calls.filter(([, signal]) => signal !== 0)).toEqual([]); + } finally { + release(); + await closing; + vi.restoreAllMocks(); + closeStep.gate = undefined; + closeStep.entered = undefined; + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); diff --git a/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts index a66e5d6d48..c1b16d37c7 100644 --- a/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts @@ -108,7 +108,7 @@ async function crashedStaleRegistration(ownership: 'embedded' | 'unrecognized' | } for (const [ownership, reason, hint] of [ - ['embedded', DAEMON_LIFECYCLE_REASONS.ownerEmbedded, /Close the host/], + ['embedded', DAEMON_LIFECYCLE_REASONS.ownerEmbedded, /Close the host that started it/], ['sideways', 'daemon-owner-unrecognized', /Update agent-device/], ] as const) test(`a managed client preserves a ${ownership} owner`, async (t) => { diff --git a/src/daemon-client/daemon-client-lifecycle-request.ts b/src/daemon-client/daemon-client-lifecycle-request.ts index 0e8771231b..b01226712c 100644 --- a/src/daemon-client/daemon-client-lifecycle-request.ts +++ b/src/daemon-client/daemon-client-lifecycle-request.ts @@ -15,7 +15,14 @@ import { daemonBusyError, daemonNotRetirableError } from './daemon-lifecycle-err const LIFECYCLE_REQUEST_TIMEOUT_MS = 10_000; -/** The bounded close budget after an idle daemon accepts a quiesce. */ +/** + * How long a daemon or runtime that agreed to leave gets to finish leaving. Its own shutdown is bounded + * in phases (server close, session and lease teardown, plugin disposal, provider release), so this is + * the longest a healthy close can take with margin, not a polling interval. Both waits a client keeps + * after that agreement — `ensureDaemon` replacing a detached daemon, and a stop of an embedder-owned + * runtime — use it, because the same close runs either way: a shorter one would report an unconfirmed + * failure for a close that was still succeeding. + */ export const DAEMON_LEAVE_WAIT_MS = 60_000; /** What a lifecycle request needs besides the daemon it addresses: where and how to reach it. */ @@ -30,10 +37,23 @@ function isConnectionFailure(error: unknown): boolean { return error instanceof AppError && error.details?.reason === 'daemon_transport_failure'; } -/** Whether the instance accepted the quiesce or is no longer there to answer. */ +/** + * `draining`: the daemon accepted and is leaving, which is all its own close needs to know — what a + * stop waits for afterwards is the completion record, and that answers by instance, not by request. + * `gone`: the daemon the request was addressed to is no longer there to answer, because it left, is + * leaving already, or was replaced. + */ export type DaemonLifecycleOutcome = Readonly<{ outcome: 'draining' | 'gone' }>; -/** Asks the fenced instance to quiesce without timeout recovery or signals. */ +/** + * Asks the daemon `info` names to leave, by the rule `mode` states, and reports its answer. + * + * The request is fenced to that instance, so a replacement refuses it. A quiesce a daemon that holds + * work refuses with `daemon-busy`, and that refusal is the answer: nothing here signals it. A + * shutdown a detached daemon refuses names its owner instead, because a detached daemon is stopped by + * signalling its process. A timeout is not recovered here either; the caller decides what an + * unresponsive daemon deserves. + */ export async function requestDaemonLifecycle( info: DaemonInfo, target: DaemonLifecycleTarget, @@ -62,7 +82,7 @@ export async function requestDaemonLifecycle( if (isDaemonTransportUnavailableError(error) || isConnectionFailure(error)) return { outcome: 'gone' }; // Over HTTP a daemon's refusal arrives as an error carrying its reason, not as the response that - // carries it. The ownership refusal is judged here so an embedded runtime says the same + // carries it. The three ownership refusals are judged here so an embedded runtime says the same // thing whichever transport it serves; every other reason keeps the path it shipped with. const refusal = readOwnerRefusal(error); const judged = refusal && judgeRefusedAnswer(refusal, info, target); @@ -79,8 +99,15 @@ export async function requestDaemonLifecycle( /** A daemon's refusal, as read from either transport's shape of it. */ type RefusedDaemonAnswer = Readonly<{ reason: unknown; details: unknown }>; -/** Ownership refusals have the same meaning over HTTP and sockets. */ -const OWNER_REFUSALS: readonly string[] = [DAEMON_LIFECYCLE_REASONS.ownerEmbedded]; +/** + * The refusals that name who owns the process. Quiesce never answers with one, so judging them on the + * error path cannot change a shipped refusal; it only gives the shutdown request the same answer a + * socket client already got. + */ +const OWNER_REFUSALS: readonly string[] = [ + DAEMON_LIFECYCLE_REASONS.ownerEmbedded, + DAEMON_LIFECYCLE_REASONS.notEmbedded, +]; function readOwnerRefusal(error: unknown): RefusedDaemonAnswer | undefined { if (!(error instanceof AppError)) return undefined; @@ -102,8 +129,11 @@ function judgeRefusedAnswer( // A detached daemon has an owner that can signal it, and a host owns an embedded runtime: either // way, this client is not the one that ends it. The daemon's own answer says which owner is on the // other side of that line, so it is trusted over this process's possibly-stale read of the record. + if (refusal.reason === DAEMON_LIFECYCLE_REASONS.notEmbedded) { + throw daemonNotRetirableError({ info, stateDir, owner: 'detached' }); + } if (refusal.reason === DAEMON_LIFECYCLE_REASONS.ownerEmbedded) { - throw daemonNotRetirableError({ info, stateDir }); + throw daemonNotRetirableError({ info, stateDir, owner: 'embedded' }); } if ( refusal.reason === DAEMON_LIFECYCLE_REASONS.instanceMismatch || diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index 33c90989f3..02c1a6c3a3 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -50,7 +50,7 @@ export function readDaemonInfo(infoPath: string): DaemonInfo | null { return read.status === 'present' ? daemonInfoFromRecord(read.record) : null; } -function daemonInfoFromRecord(parsed: Partial): DaemonInfo | null { +export function daemonInfoFromRecord(parsed: Partial): DaemonInfo | null { const token = readRequiredDaemonToken(parsed); const ports = readDaemonInfoPorts(parsed); return token && ports ? readDaemonInfoFromRecord(parsed, token, ports) : null; diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index 2dfcbb8736..997cca5568 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -150,6 +150,7 @@ export async function readReusableLocalDaemon( } if (!settings.retireIncompatibleContender) throw daemonContendedError(); + if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); await retireDaemonForTakeover(existing, settings.paths); diff --git a/src/daemon-client/daemon-client-stop.ts b/src/daemon-client/daemon-client-stop.ts index 0d76637097..539ef908c8 100644 --- a/src/daemon-client/daemon-client-stop.ts +++ b/src/daemon-client/daemon-client-stop.ts @@ -1,43 +1,188 @@ -import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { - observedDaemonIdentityOfRegistration, - type DaemonStopIdentity, -} from '@agent-device/daemon-contracts/daemon-registration'; + daemonShutdownCompletionAnswers, + readDaemonShutdownRecord, + readDaemonShutdownReport, + type DaemonShutdownRecord, + type DaemonShutdownCompletionQuery, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process'; import { + DAEMON_LIFECYCLE_REASONS, + readDaemonLifecycleCensus, + type DaemonPaths, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import type { DaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { + daemonNotRunningStopResult, stopDaemon, mergeDaemonShutdownReport, type DaemonStopResult, } from '@agent-device/daemon-contracts/daemon-stop'; -import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { AppError } from '@agent-device/kernel/errors'; +import { sleep } from '@agent-device/host-kit/retry'; +import { daemonInfoFromRecord, type DaemonInfo } from './daemon-client-metadata.ts'; +import { + DAEMON_LEAVE_WAIT_MS, + requestDaemonLifecycle, + type DaemonLifecycleTarget, +} from './daemon-client-lifecycle-request.ts'; + +const EMBEDDED_STOP_ASK_BUDGET_MS = 5_000; +const EMBEDDED_STOP_ASK_GAP_MS = 250; +const EMBEDDED_STOP_POLL_MS = 100; +const EMBEDDED_STOP_LIFETIME_POLL_MS = 1_000; -/** - * Stops the daemon one registration read names. Ownership travels with the lifetime it read, and the - * central retirement judges it: an embedded runtime or an unrecognized owner comes back as the refusal - * that names its owner, with nothing signaled. - */ export async function stopRecordedDaemon( record: DaemonStopIdentity, - paths: DaemonPaths, + target: DaemonLifecycleTarget, ): Promise { - if (record.ownership === 'unreadable') { - throw new AppError( - 'COMMAND_FAILED', - `The daemon record on ${paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`, - { - reason: 'daemon-record-unreadable', - stateDir: paths.baseDir, - hint: 'Nothing was signaled. Inspect or remove that record, then retry.', - }, - ); + if (record.ownership === 'unreadable') throw daemonRecordUnreadableError(target.paths); + if (record.ownership === 'embedded' || record.ownership === 'unknown') { + const info = record.record && daemonInfoFromRecord(record.record); + if (!info) throw daemonEmbeddedRecordUnaddressableError(target.paths); + return await stopEmbeddedDaemon(info, target); } - // The retirement judges the owner this one read stated; nothing here decides it. - const stopped = await stopDaemon({ - paths, - observed: observedDaemonIdentityOfRegistration(record), - }); + const observed = record.identity ? { ...record.identity, ownership: 'detached' as const } : null; + const stopped = await stopDaemon({ paths: target.paths, observed }); return mergeDaemonShutdownReport( stopped, - stopped.mode === 'graceful' ? readDaemonShutdownReport(paths.baseDir) : null, + stopped.mode === 'graceful' ? readDaemonShutdownReport(target.paths.baseDir) : null, + ); +} + +function daemonRecordUnreadableError(paths: DaemonPaths): AppError { + return new AppError( + 'COMMAND_FAILED', + `The daemon record on ${paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`, + { + reason: 'daemon-record-unreadable', + stateDir: paths.baseDir, + hint: 'Nothing was signaled. Inspect or remove that record, then retry.', + }, + ); +} + +async function stopEmbeddedDaemon( + info: DaemonInfo, + target: DaemonLifecycleTarget, +): Promise { + const instance = { + pid: info.pid, + processStartTime: info.processStartTime ?? '', + token: info.token, + }; + const deadlineMs = Date.now() + DAEMON_LEAVE_WAIT_MS; + const early = await askToLeave(info, target, instance); + if (early?.completion?.outcome === 'retained' && (await instanceEnded(instance))) + return daemonNotRunningStopResult(); + const waited = early ?? (await waitForClose(target.paths, instance, deadlineMs)); + if (waited === 'instance-gone') return daemonNotRunningStopResult(); + if (!waited) throw daemonShutdownUnconfirmedError(info, target.paths); + return stopResultFromCompletion(waited); +} + +async function askToLeave( + info: DaemonInfo, + target: DaemonLifecycleTarget, + instance: DaemonShutdownCompletionQuery, +): Promise { + const askUntilMs = Date.now() + EMBEDDED_STOP_ASK_BUDGET_MS; + for (;;) { + const record = readDaemonShutdownRecord(target.paths.baseDir); + if (record.completion && daemonShutdownCompletionAnswers(record.completion, instance)) + return record; + try { + const asked = await requestDaemonLifecycle(info, target, 'shutdown'); + if (asked.outcome === 'gone' && !instance.processStartTime.trim()) + throw daemonShutdownUnconfirmedError(info, target.paths); + return null; + } catch (error) { + if (!refusedWhileStartingUp(error) || Date.now() >= askUntilMs) throw error; + } + await sleep(EMBEDDED_STOP_ASK_GAP_MS); + } +} + +function refusedWhileStartingUp(error: unknown): boolean { + return ( + error instanceof AppError && + error.details?.reason === DAEMON_LIFECYCLE_REASONS.busy && + readDaemonLifecycleCensus(error.details)?.startingUp === true ); } + +async function waitForClose( + paths: DaemonPaths, + instance: DaemonShutdownCompletionQuery, + deadlineMs: number, +): Promise { + let nextLifetimeCheckMs = 0; + while (Date.now() < deadlineMs) { + const record = readDaemonShutdownRecord(paths.baseDir); + const matching = + record.completion && daemonShutdownCompletionAnswers(record.completion, instance); + if (matching && record.completion?.outcome === 'released') return record; + if (Date.now() >= nextLifetimeCheckMs || matching) { + nextLifetimeCheckMs = Date.now() + EMBEDDED_STOP_LIFETIME_POLL_MS; + if (await instanceEnded(instance)) return 'instance-gone'; + } + if (matching) return record; + await sleep(Math.min(EMBEDDED_STOP_POLL_MS, Math.max(0, deadlineMs - Date.now()))); + } + return null; +} + +async function instanceEnded(instance: DaemonShutdownCompletionQuery): Promise { + if (!instance.processStartTime.trim()) return false; + const identity = { pid: instance.pid, startTime: instance.processStartTime }; + return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited; +} + +function stopResultFromCompletion(record: DaemonShutdownRecord): DaemonStopResult { + const completion = record.completion!; + const released = completion.outcome === 'released'; + return mergeDaemonShutdownReport( + { + stopped: true, + mode: 'graceful', + cleanupConfidence: released && completion.unfinished.length === 0 ? 'known' : 'unknown', + claimsReleased: [], + claimsOrphaned: [], + claimsSuperseded: [], + claimsUnattributable: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + warnings: [ + ...completion.unfinished, + ...(released + ? [] + : [ + 'The runtime stopped serving and keeps its registration: a host process still owns it, so nothing outside that process may signal it.', + ]), + ], + }, + record.report, + ); +} + +function daemonEmbeddedRecordUnaddressableError(paths: DaemonPaths): AppError { + return new AppError( + 'COMMAND_FAILED', + `The record on ${paths.baseDir} does not name a transport and a token, so the runtime it describes cannot be reached.`, + { + reason: 'daemon-embedded-stop-unreachable', + stateDir: paths.baseDir, + record: 'incomplete', + hint: 'Nothing was signaled. Inspect the record, or close the host that owns this runtime.', + }, + ); +} + +function daemonShutdownUnconfirmedError(info: DaemonInfo, paths: DaemonPaths): AppError { + return new AppError('COMMAND_FAILED', 'The runtime did not report that it finished leaving.', { + reason: 'daemon-shutdown-unconfirmed', + pid: info.pid, + ...(info.processStartTime ? { processStartTime: info.processStartTime } : {}), + stateDir: paths.baseDir, + hint: 'It was not signaled, and it may still be serving. Ask the host that owns it to close it, or retry the stop.', + }); +} diff --git a/src/daemon-client/daemon-lifecycle-errors.ts b/src/daemon-client/daemon-lifecycle-errors.ts index 09e2913bbb..5313d6dc37 100644 --- a/src/daemon-client/daemon-lifecycle-errors.ts +++ b/src/daemon-client/daemon-lifecycle-errors.ts @@ -3,6 +3,7 @@ import { DAEMON_LIFECYCLE_REASONS, DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, type DaemonLifecycleCensus, + type DaemonOwnership, type DaemonPaths, } from '@agent-device/daemon-contracts/daemon-resolution'; import type { DaemonRetirementResult } from '@agent-device/daemon-contracts/daemon-registration-owner'; @@ -97,7 +98,7 @@ export function daemonTakeoverRetirementError( paths: DaemonPaths, ): AppError { if (retirement.reason === 'embedded-owner') - return daemonNotRetirableError({ info, stateDir: paths.baseDir }); + return daemonNotRetirableError({ info, stateDir: paths.baseDir, owner: 'embedded' }); if (retirement.reason === 'owner-unproven') return daemonRecordOwnerUnrecognizedError({ info, stateDir: paths.baseDir }); return daemonRetirementUnconfirmedError(retirement, paths); @@ -176,16 +177,41 @@ function daemonRecordOwnerUnrecognizedError(params: { ); } -/** Refuses replacement of a runtime whose host owns its lifetime. */ -export function daemonNotRetirableError(params: { info: DaemonInfo; stateDir: string }): AppError { +/** + * A caller wanted this daemon gone, and its owner's rule says otherwise: a host process owns an + * embedded runtime and ends it, and a detached daemon is stopped by signalling its own process, which is + * what `daemon stop` does for a record that proves that owner. + * + * Both cases mean the same thing to a caller: this daemon is not yours to retire. `owner` is the + * daemon's own answer about itself, so it is trusted over this process's read of the record — except for + * the hint, which reads the record too. A detached answer against a record this build cannot place is a + * corrupt state directory, and `daemon stop` refuses that record for the same reason this client just + * did, so its hint names the record repair instead of sending the operator to a second refusal. + */ +export function daemonNotRetirableError(params: { + info: DaemonInfo; + stateDir: string; + owner: Exclude; +}): AppError { + const { info, stateDir, owner } = params; + const embedded = owner === 'embedded'; + const recordUnreadable = !embedded && info.ownership === 'unknown'; return new AppError( 'COMMAND_FAILED', - 'A host process owns this daemon, so it was not replaced.', + embedded + ? 'A host process owns this daemon, so it was not replaced.' + : 'This daemon is not embedder-owned, so a shutdown request is not how it is stopped.', { - reason: DAEMON_LIFECYCLE_REASONS.ownerEmbedded, - stateDir: params.stateDir, - daemon: daemonIdentityOf(params.info), - hint: 'Close the host that started this runtime.', + reason: embedded + ? DAEMON_LIFECYCLE_REASONS.ownerEmbedded + : DAEMON_LIFECYCLE_REASONS.notEmbedded, + stateDir, + daemon: daemonIdentityOf(info), + hint: embedded + ? `Close the host that started it, or ask this daemon to leave with ${daemonStopCommand(stateDir)}, which sends it a shutdown request instead of a signal.` + : recordUnreadable + ? `The daemon says it is a daemon of its own, so the owner its record states is wrong. Nothing was signaled. Check what owns pid ${info.pid}, then repair the record: stop that process by its own means and remove ${shellQuoteIfNeeded(stateDir)}/daemon.json so a daemon can register again.` + : `Stop it with ${daemonStopCommand(stateDir)}.`, }, ); } diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 69d33b3f45..9174343b84 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -90,7 +90,9 @@ if ('cloudArtifacts' in result) { A client talks to one local daemon per state directory. `daemonLifecycle` says who owns that daemon. -A host-owned runtime records its ownership. A managed client reuses it while it matches. Every takeover, stop, drain, and timeout recovery asks the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one, so the state directory stays usable. The host closes its live runtime directly. `daemon stop` refuses a live host-owned or unrecognized owner without signaling it; `--clean` also requires a proven detached owner and refuses host-owned, unrecognized and unreadable records with `daemon-clean-not-detached`. Without `--clean`, an unreadable `daemon.json` fails stop with `daemon-record-unreadable`. +A host-owned daemon ends through its host's own close or an authenticated shutdown request over its registered local transport. Clients never signal the host. A `'managed'` client reuses a matching host-owned daemon. Takeover, drain, and timeout recovery ask the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one, so the state directory stays usable. `agent-device daemon stop --state-dir ` sends the shutdown request and waits for the close report. It can also ask a readable record with unrecognized ownership: an embedded runtime can close itself, but a detached answer fails with `daemon-not-embedded` and a record-repair hint, without signaling. A host's direct close also records its result, so stops already waiting can confirm a clean release. + +For these shutdown requests, stop fails with `daemon-embedded-stop-unreachable` when the record lacks usable transport or token, `daemon-shutdown-unconfirmed` when the close cannot be confirmed (including after connection failure), or, without `--clean`, `daemon-record-unreadable` when `daemon.json` cannot be read. These are `error.details.reason` values. A retained close returns `stopped: true`, warnings and `cleanupConfidence: 'unknown'` with exit 0: the runtime stopped serving, but resource cleanup is unconfirmed. Later stops can read that retained answer while the same host holds the registration. | Mode | What the client does with the daemon | | --- | --- | @@ -121,6 +123,8 @@ These failures carry a typed `error.details.reason`: | `daemon-incompatible` | The running daemon does not match the client. `details.daemon` and `details.client` hold both identities. Stop it with `agent-device daemon stop --state-dir `, then call `ensureDaemon()`. A daemon from a release that predates quiescing cannot be asked to leave, so `ensureDaemon()` reports it the same way. | | `daemon-busy` | `ensureDaemon()` found an incompatible daemon that still holds work. `details` counts `inFlightRequests`, `sessions`, `repairSessions` and `leases`. Wait for them to end and retry. | | `daemon-drain-timeout` | A daemon accepted the request to leave but was still running 60 s later and could not be force-retired, either because it still answers or because its process cannot be proven to be a daemon (an embedded host). It was not signaled. Retry once it has exited, or stop it with `agent-device daemon stop --state-dir `. | +| `daemon-owner-embedded` | A host process owns the daemon on `stateDir`, so this client neither replaced it nor retired it, and signaled nothing. A daemon that matches the client is still usable. Close that host, or stop it with `agent-device daemon stop --state-dir `. | +| `daemon-owner-unrecognized` | The record on `stateDir` states an owner this version cannot read, so this client replaced nothing and signaled nothing. Update agent-device if the record comes from a newer build. Otherwise check what owns that process: a host that started it closes it with its own close, and otherwise remove `daemon.json` once that process is gone. | `ensureDaemon()` only ever manages the local daemon on `stateDir`. A client with `daemonBaseUrl` never starts or stops a local daemon in either mode, and `daemonLifecycle` does not change `daemonTransport` or `daemonServerMode`. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 74e5533c7e..6aaa3fcbb7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -85,7 +85,8 @@ Device claims and the daemon: - `device status` lists host-local device claims without starting or contacting a daemon; `--stale` shows the stale claims it hides by default (owner exited, PID reused, state directory deleted, or daemon replaced), some of whose owners may still be running. Scope it with `--platform` plus `--udid` (Apple) or `--serial` (Android). A claim held by another live or uncertain owner blocks `open`, unless the device rebooted after the claim was taken: then `open` settles that owner's durable resources, takes the claim, and reports the release in `warnings`. - `device release --stale` settles the durable resources of an owner that provably can't release its claim (its process exited or a newer daemon replaced it), then clears the claim. Live, uncertain, PID-reused, corrupt, and deleted-state-directory claims are refused with the reason. See [Sessions](/docs/sessions#release-a-device-another-session-or-workspace-holds). - A command that needs the shared local daemon starts it for its state directory when none runs; `device status` and `device release --stale` never contact one. When the running daemon comes from another version or build, the command replaces it only if it reports no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise the command fails with `daemon-busy` and leaves the daemon alone. A daemon from a release that predates this check is never replaced, and fails with `daemon-incompatible`. End a busy or older daemon deliberately with `daemon stop`. Node.js clients choose this per client; see [Daemon lifecycle](/docs/client-api#daemon-lifecycle). -- `daemon stop --state-dir ` stops a detached local daemon after verifying its identity. Stop and `--clean` use the lifetime and ownership from one registration read. A live host-owned or unrecognized owner is refused without signaling; its host closes the runtime directly. Without `--clean`, an unreadable `daemon.json` fails stop with `daemon-record-unreadable`; with `--clean`, host-owned, unrecognized and unreadable records fail with `daemon-clean-not-detached`. `daemon status` shows `host-owned` or `owner-not-recognized`. `--clean` also removes the Apple runner processes and leases that daemon retained. +- `daemon stop --state-dir ` stops a detached local daemon with process signals after verifying its identity. `--clean` also removes the Apple runner processes and leases that daemon retained. +- For a host-owned runtime, `daemon stop` sends an authenticated shutdown request over its registered local transport and waits for the close report without signaling the host. It can also ask a record with unrecognized ownership: an embedded runtime can close itself, but a detached answer fails with `details.reason: daemon-not-embedded` and a record-repair hint, without signaling. For these shutdown requests, a record without usable transport or token fails with `daemon-embedded-stop-unreachable`. An unconfirmed close, including after connection failure, fails with `daemon-shutdown-unconfirmed`. Without `--clean`, an unreadable `daemon.json` fails stop with `daemon-record-unreadable`. A retained close returns `stopped: true`, warnings and `cleanupConfidence: unknown` (exit 0): it stopped serving, but resource cleanup is unconfirmed. `--clean` requires proven detached ownership; host-owned, unrecognized and unreadable records fail with `daemon-clean-not-detached` because that cleanup walks the owner's whole process tree. `daemon status` shows `host-owned` or `owner-not-recognized`. See [Daemon lifecycle](/docs/client-api#daemon-lifecycle). - `daemon status [--state-dir ] [--json]` reads the daemon's record without contacting it. For a running daemon it lists every provider it registered, built in or installed, and the reason any plugin failed to register. See [provider plugins](/docs/plugins). Open apps and links: