From 42f2f8a83d2e8be439e37164e7d6e6743cacaa38 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:16:49 -0800 Subject: [PATCH] Accept an idempotency key when arming a fault An arm may carry idempotencyKey. Another arm with the same key and the same fault stores nothing and returns the fault first stored for that key, so a caller that lost an answer can send the arm again. Each key is remembered with the instance's state, through consumption, removal, clearing, reset and Durable Object eviction, so a late copy cannot arm its fault again. The same key with a different fault is refused with 409. Arms without a key behave as before. The faults list says the emulator honors keys. --- README.md | 2 + apps/web/app/docs/ledger/page.mdx | 2 + .../cloudflare/src/__tests__/worker.test.ts | 48 ++++++ .../core/src/__tests__/fault-keys.test.ts | 156 ++++++++++++++++++ packages/@emulators/core/src/control-plane.ts | 12 +- packages/@emulators/core/src/faults.ts | 90 +++++++++- packages/@emulators/core/src/index.ts | 3 + packages/emulate/src/index.ts | 3 +- 8 files changed, 306 insertions(+), 10 deletions(-) create mode 100644 packages/@emulators/core/src/__tests__/fault-keys.test.ts diff --git a/README.md b/README.md index bd689114f..4ba47cd83 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ curl -s -X POST "$EMULATOR_URL/_emulate/faults" \ -d '{"match":{"method":"GET","pathPattern":"/v1/*"},"response":{"status":503,"body":{"error":"temporary"}}}' ``` +An arm may carry an `idempotencyKey` (1 to 255 characters, chosen by the caller). Another arm with the same key and the same fault stores nothing and returns the fault first stored for that key, with `replayed: true`, so a caller that lost an answer can send the arm again safely. The instance remembers each key for as long as its state lives, through consumption, removal by id, clearing and reset, so a late copy of an arm whose fault is already gone returns that fault with `remaining: 0` instead of arming it again. The same key with a different fault is refused with `409` and `idempotency_key_conflict`. Arms without a key are stored as before. `GET /_emulate/faults` answers `idempotencyKeys: true` so a caller can tell an emulator that honors keys. + Credential creation follows each service's real shape. For example, GitHub can mint a bearer token for a user, Spotify creates a client credentials app, Google/Microsoft/Apple/Okta/Clerk create OAuth/OIDC clients, Stripe and Resend create API-key style credentials, and AWS advertises provider-specific SDK credentials instead of pretending to be OAuth. ### MCP OAuth compliance scenarios diff --git a/apps/web/app/docs/ledger/page.mdx b/apps/web/app/docs/ledger/page.mdx index 2fe74c193..ba8fce6fd 100644 --- a/apps/web/app/docs/ledger/page.mdx +++ b/apps/web/app/docs/ledger/page.mdx @@ -96,6 +96,8 @@ Set `X-Correlation-Id` (or `X-Request-Id`) on a request and the emulator stores `POST /_emulate/faults` arms a one-shot or counted response for matching provider requests. Match criteria are combined: `operationId` when the emulator can resolve one from the service manifest, `method`, and `pathPattern`. `pathPattern` is a glob where `*` matches any characters in the request path. A matched request short-circuits with the configured response, decrements the fault's remaining count, and is recorded here with `faulted: true` and `faultId`. +An arm may carry an `idempotencyKey` (1 to 255 characters, chosen by the caller). Another arm with the same key and the same fault stores nothing and returns the fault first stored for that key, with `replayed: true`, so a caller that lost an answer can send the arm again safely. The instance remembers each key for as long as its state lives, through consumption, removal by id, clearing and reset, so a late copy of an arm whose fault is already gone returns that fault with `remaining: 0` instead of arming it again. The same key with a different fault is refused with `409` and `idempotency_key_conflict`. Arms without a key are stored as before. `GET /_emulate/faults` answers `idempotencyKeys: true` so a caller can tell an emulator that honors keys. + ## Persistence On the hosted Cloudflare surface the ledger is persistent. Each instance is backed by a Durable Object that snapshots state and the ledger to storage, so records survive eviction. Locally and in process the ledger is in memory and is cleared on reset. See [Deployment](/docs/deployment) for the instance model. diff --git a/packages/@emulators/cloudflare/src/__tests__/worker.test.ts b/packages/@emulators/cloudflare/src/__tests__/worker.test.ts index 099ba10e6..696705bbe 100644 --- a/packages/@emulators/cloudflare/src/__tests__/worker.test.ts +++ b/packages/@emulators/cloudflare/src/__tests__/worker.test.ts @@ -650,6 +650,54 @@ describe("cloudflare durable object control plane", () => { expect((await pending).status).toBe(503); }); + // A caller that lost an arm's answer sends it again with the same key. The original copy can + // reach the instance at any later moment, after the instance was replaced, the fault consumed + // and the instance reset; it must never arm a second fault. + it("keeps fault idempotency keys across eviction, consumption and reset", async () => { + const { state } = makeState(); + const url = "https://github.my-run.emulators.dev"; + const arm = async (durableObject: EmulatorDurableObject) => { + const res = await durableObject.fetch( + new Request(`${url}/_emulate/faults`, { + method: "POST", + headers: idHeaders({ "content-type": "application/json" }), + body: JSON.stringify({ + match: { method: "GET", pathPattern: "/repos/*" }, + response: { status: 503, body: { message: "synthetic outage" } }, + idempotencyKey: "arm-7f3a", + }), + }), + ); + expect(res.status).toBe(200); + return (await res.json()) as { fault: { id: string; remaining: number }; replayed: boolean }; + }; + const listed = async (durableObject: EmulatorDurableObject) => { + const res = await durableObject.fetch(new Request(`${url}/_emulate/faults`, { headers: idHeaders() })); + return ((await res.json()) as { faults: Array<{ id: string }> }).faults.map((fault) => fault.id); + }; + const repo = (durableObject: EmulatorDurableObject) => + durableObject.fetch(new Request(`${url}/repos/acme/widget`, { headers: idHeaders() })); + + const retry = await arm(new EmulatorDurableObject(state, {})); + expect(retry.replayed).toBe(false); + const evicted = new EmulatorDurableObject(state, {}); + const original = await arm(evicted); + expect(original).toMatchObject({ replayed: true, fault: { id: retry.fault.id, remaining: 1 } }); + expect(await listed(evicted)).toEqual([retry.fault.id]); + expect((await repo(evicted)).status).toBe(503); + + const afterConsumed = new EmulatorDurableObject(state, {}); + expect(await arm(afterConsumed)).toMatchObject({ replayed: true, fault: { id: retry.fault.id, remaining: 0 } }); + const reset = await afterConsumed.fetch( + new Request(`${url}/_emulate/reset`, { method: "POST", headers: idHeaders() }), + ); + expect(reset.status).toBe(200); + const afterReset = new EmulatorDurableObject(state, {}); + expect((await arm(afterReset)).replayed).toBe(true); + expect(await listed(afterReset)).toEqual([]); + expect((await repo(afterReset)).status).not.toBe(503); + }); + it("reports the real instance id in the manifest", async () => { const { state } = makeState(); const durableObject = new EmulatorDurableObject(state, {}); diff --git a/packages/@emulators/core/src/__tests__/fault-keys.test.ts b/packages/@emulators/core/src/__tests__/fault-keys.test.ts new file mode 100644 index 000000000..c5dc951da --- /dev/null +++ b/packages/@emulators/core/src/__tests__/fault-keys.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest"; +import { createServer } from "../server.js"; +import { FaultRegistry } from "../faults.js"; +import type { ServicePlugin } from "../plugin.js"; + +const plugin: ServicePlugin = { + name: "demo", + register(app) { + app.get("/things", (c) => c.json({ things: [] })); + }, +}; + +const fault = { + match: { method: "get", pathPattern: "/things" }, + response: { status: 503, body: { error: "planned" } }, +}; + +type Armed = { id: string; remaining: number; idempotencyKey?: string }; + +function setup() { + const server = createServer(plugin, { + baseUrl: "https://demo.instance.emulators.dev", + reset() { + server.ledger.clear(); + }, + }); + const { app } = server; + const arm = async (body: Record) => { + const res = await app.request("/_emulate/faults", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { status: res.status, body: (await res.json()) as { fault: Armed; replayed?: boolean; error?: string } }; + }; + const list = async () => + (await (await app.request("/_emulate/faults")).json()) as { faults: Armed[]; idempotencyKeys?: boolean }; + const things = async () => (await app.request("/things")).status; + return { server, app, arm, list, things }; +} + +describe("fault idempotency keys", () => { + it("returns the fault first stored for a key to a second arm sent after it, and stores nothing", async () => { + const { arm, list, things } = setup(); + const first = await arm({ ...fault, idempotencyKey: "arm-1" }); + const second = await arm({ ...fault, idempotencyKey: "arm-1" }); + expect(first.status).toBe(200); + expect(first.body.replayed).toBe(false); + expect(second.status).toBe(200); + expect(second.body.replayed).toBe(true); + expect(second.body.fault.id).toBe(first.body.fault.id); + expect(second.body.fault.idempotencyKey).toBe("arm-1"); + expect((await list()).faults.map((f) => f.id)).toEqual([first.body.fault.id]); + expect(await things()).toBe(503); + expect(await things()).toBe(200); + }); + + it("stores one fault for two arms with one key sent at once", async () => { + const { arm, list } = setup(); + const both = await Promise.all([ + arm({ ...fault, idempotencyKey: "arm-1" }), + arm({ ...fault, idempotencyKey: "arm-1" }), + ]); + expect(both.map((r) => r.status)).toEqual([200, 200]); + expect(new Set(both.map((r) => r.body.fault.id)).size).toBe(1); + expect(both.map((r) => r.body.replayed).sort()).toEqual([false, true]); + expect((await list()).faults).toHaveLength(1); + }); + + it("does not re-arm when a late copy arrives after its fault was consumed, removed, cleared or reset", async () => { + const { app, arm, list, things } = setup(); + // Consumed. + const consumed = await arm({ ...fault, idempotencyKey: "consumed" }); + expect(await things()).toBe(503); + const lateConsumed = await arm({ ...fault, idempotencyKey: "consumed" }); + expect(lateConsumed.body.replayed).toBe(true); + expect(lateConsumed.body.fault).toMatchObject({ id: consumed.body.fault.id, remaining: 0 }); + expect((await list()).faults).toEqual([]); + expect(await things()).toBe(200); + + // Removed by id. + const removed = await arm({ ...fault, idempotencyKey: "removed" }); + await app.request(`/_emulate/faults/${removed.body.fault.id}`, { method: "DELETE" }); + const lateRemoved = await arm({ ...fault, idempotencyKey: "removed" }); + expect(lateRemoved.body).toMatchObject({ replayed: true, fault: { id: removed.body.fault.id, remaining: 0 } }); + expect((await list()).faults).toEqual([]); + expect(await things()).toBe(200); + + // Every fault cleared, then the instance reset. + await arm({ ...fault, idempotencyKey: "cleared" }); + await app.request("/_emulate/faults", { method: "DELETE" }); + await arm({ ...fault, idempotencyKey: "reset" }); + await app.request("/_emulate/reset", { method: "POST" }); + expect((await arm({ ...fault, idempotencyKey: "cleared" })).body.replayed).toBe(true); + expect((await arm({ ...fault, idempotencyKey: "reset" })).body.replayed).toBe(true); + expect((await list()).faults).toEqual([]); + expect(await things()).toBe(200); + }); + + it("refuses a key used again for a different fault, and keeps the first", async () => { + const { arm, list } = setup(); + const first = await arm({ ...fault, idempotencyKey: "arm-1" }); + const other = await arm({ ...fault, times: 2, idempotencyKey: "arm-1" }); + expect(other.status).toBe(409); + expect(other.body.error).toBe("idempotency_key_conflict"); + const otherStatus = await arm({ ...fault, response: { status: 500 }, idempotencyKey: "arm-1" }); + expect(otherStatus.status).toBe(409); + expect((await list()).faults.map((f) => f.id)).toEqual([first.body.fault.id]); + // The same fault written differently (method case, key order, `times` defaulted) is the same fault. + const same = await arm({ + response: { body: { error: "planned" }, status: 503 }, + match: { pathPattern: "/things", method: "GET" }, + times: 1, + idempotencyKey: "arm-1", + }); + expect(same.status).toBe(200); + expect(same.body.fault.id).toBe(first.body.fault.id); + expect((await arm({ ...fault, idempotencyKey: "" })).status).toBe(400); + }); + + it("stores every arm without a key as before, and says it honors keys", async () => { + const { arm, list } = setup(); + const keyed = await arm({ ...fault, idempotencyKey: "arm-1" }); + const a = await arm(fault); + const b = await arm(fault); + expect(a.body).not.toHaveProperty("replayed"); + expect(a.body.fault).not.toHaveProperty("idempotencyKey"); + expect(new Set([keyed.body.fault.id, a.body.fault.id, b.body.fault.id]).size).toBe(3); + const listed = await list(); + expect(listed.faults).toHaveLength(3); + expect(listed.idempotencyKeys).toBe(true); + }); + + it("keeps the record of keys through a snapshot and restore, consumed faults included", () => { + const registry = new FaultRegistry(); + const live = registry.arm({ ...fault, idempotencyKey: "live" }); + const spent = registry.arm({ ...fault, idempotencyKey: "spent" }); + registry.clear(spent.id); + const restored = new FaultRegistry(); + restored.restore(JSON.parse(JSON.stringify(registry.serialize()))); + expect(restored.armOnce({ ...fault, idempotencyKey: "live" })).toMatchObject({ + replayed: true, + fault: { id: live.id, remaining: 1 }, + }); + expect(restored.armOnce({ ...fault, idempotencyKey: "spent" })).toMatchObject({ + replayed: true, + fault: { id: spent.id, remaining: 0 }, + }); + expect(restored.list().map((f) => f.id)).toEqual([live.id]); + expect(() => restored.armOnce({ ...fault, times: 3, idempotencyKey: "spent" })).toThrow(/different fault/); + // A snapshot stored before keys existed restores with none. + const old = new FaultRegistry(); + old.restore({ counter: 4, faults: [] }); + expect(old.arm(fault).id).toBe("fault_4"); + }); +}); diff --git a/packages/@emulators/core/src/control-plane.ts b/packages/@emulators/core/src/control-plane.ts index 55c05841a..82ffe4e83 100644 --- a/packages/@emulators/core/src/control-plane.ts +++ b/packages/@emulators/core/src/control-plane.ts @@ -7,7 +7,7 @@ import type { ConnectionVars, EmulatorInstanceInfo, ResolvedConnection, ServiceM import { coverageReport, enrichManifest, resolveConnections } from "./manifest.js"; import type { TokenMap } from "./middleware/auth.js"; import { escapeHtml, renderCardPage } from "./ui.js"; -import type { FaultArmInput, FaultRegistry } from "./faults.js"; +import { FaultKeyConflictError, type FaultArmInput, type FaultRegistry } from "./faults.js"; export interface CredentialRequest { type?: string; @@ -160,13 +160,17 @@ export function registerControlPlane(app: Hono, options: ControlPlaneOpt ledger.clear(); return c.json({ ok: true }); }); - app.get("/_emulate/faults", (c) => c.json({ faults: faults.list() })); + // `idempotencyKeys` tells a caller this emulator honors `idempotencyKey` on an arm. + app.get("/_emulate/faults", (c) => c.json({ faults: faults.list(), idempotencyKeys: true })); app.post("/_emulate/faults", async (c) => { const body = (await c.req.json().catch(() => undefined)) as FaultArmInput | undefined; try { - const fault = faults.arm(body as FaultArmInput); - return c.json({ fault }); + const { fault, replayed } = faults.armOnce(body as FaultArmInput); + return c.json(body?.idempotencyKey === undefined ? { fault } : { fault, replayed }); } catch (err) { + if (err instanceof FaultKeyConflictError) { + return c.json({ error: "idempotency_key_conflict", message: err.message }, 409); + } return c.json({ error: "invalid_fault", message: err instanceof Error ? err.message : "Invalid fault." }, 400); } }); diff --git a/packages/@emulators/core/src/faults.ts b/packages/@emulators/core/src/faults.ts index d88e19aec..50207fa27 100644 --- a/packages/@emulators/core/src/faults.ts +++ b/packages/@emulators/core/src/faults.ts @@ -20,6 +20,12 @@ export interface FaultArmInput { response: FaultResponse; times?: number; delayMs?: number; + /** + * A caller-chosen key. Arming again with the same key and the same fault stores nothing and + * returns the fault first stored for it, so a caller that lost an answer can send the same arm + * again safely. The same key with a different fault is refused. + */ + idempotencyKey?: string; } export interface ArmedFault { @@ -30,6 +36,28 @@ export interface ArmedFault { remaining: number; delayMs?: number; createdAt: string; + idempotencyKey?: string; +} + +/** The result of an arm: the fault, and whether a key named one stored by an earlier arm. */ +export interface FaultArmResult { + fault: ArmedFault; + replayed: boolean; +} + +/** An arm whose key was already used for a different fault. */ +export class FaultKeyConflictError extends Error {} + +/** + * The fault each key armed, kept for as long as the registry's state: through consumption, + * removal, clearing and reset. A key forgotten when its fault went away would let a late copy of + * its arm store the fault again. + */ +export interface FaultKeyRecord { + key: string; + /** The arm's fault as stored (match normalized, `times` defaulted), in a canonical JSON form. */ + definition: string; + fault: ArmedFault; } export interface FaultLedgerMarker { @@ -41,25 +69,44 @@ export interface FaultLedgerMarker { export interface FaultRegistrySnapshot { counter: number; faults: ArmedFault[]; + /** Absent in snapshots stored before keys existed. */ + keys?: FaultKeyRecord[]; } export class FaultRegistry { private faults: ArmedFault[] = []; private counter = 1; + private keys = new Map(); serialize(): FaultRegistrySnapshot { - return { counter: this.counter, faults: this.faults.map(cloneFault) }; + return { + counter: this.counter, + faults: this.faults.map(cloneFault), + keys: [...this.keys.values()].map((record) => ({ ...record, fault: cloneFault(record.fault) })), + }; } restore(snapshot: FaultRegistrySnapshot | undefined): void { this.faults = (snapshot?.faults ?? []).map(cloneFault); this.counter = snapshot?.counter ?? 1; + this.keys = new Map( + (snapshot?.keys ?? []).map((record) => [record.key, { ...record, fault: cloneFault(record.fault) }]), + ); } arm(input: FaultArmInput): ArmedFault { + return this.armOnce(input).fault; + } + + /** + * Arm a fault, or with a key already used for the same fault, return that key's fault (as it is + * now, or with nothing remaining once it was consumed or removed) and store nothing. Checking + * the key and storing happen in one synchronous step, so concurrent arms with a key cannot both + * store. + */ + armOnce(input: FaultArmInput): FaultArmResult { validateFaultInput(input); - const fault: ArmedFault = { - id: `fault_${this.counter++}`, + const definition: Omit = { match: normalizeMatch(input.match), response: { status: input.response.status, @@ -67,12 +114,30 @@ export class FaultRegistry { ...(input.response.headers ? { headers: input.response.headers } : {}), }, times: input.times ?? 1, - remaining: input.times ?? 1, ...(input.delayMs === undefined ? {} : { delayMs: input.delayMs }), + }; + const key = input.idempotencyKey; + const canonical = key === undefined ? "" : canonicalJson(definition); + if (key !== undefined) { + const record = this.keys.get(key); + if (record) { + if (record.definition !== canonical) { + throw new FaultKeyConflictError(`Fault idempotencyKey "${key}" was already used for a different fault.`); + } + const live = this.faults.find((fault) => fault.id === record.fault.id); + return { fault: cloneFault(live ?? { ...record.fault, remaining: 0 }), replayed: true }; + } + } + const fault: ArmedFault = { + id: `fault_${this.counter++}`, + ...definition, + remaining: definition.times, createdAt: new Date().toISOString(), + ...(key === undefined ? {} : { idempotencyKey: key }), }; this.faults.push(fault); - return cloneFault(fault); + if (key !== undefined) this.keys.set(key, { key, definition: canonical, fault: cloneFault(fault) }); + return { fault: cloneFault(fault), replayed: false }; } list(): ArmedFault[] { @@ -187,6 +252,21 @@ function validateFaultInput(input: FaultArmInput): void { if (input.delayMs !== undefined && (!Number.isInteger(input.delayMs) || input.delayMs < 0)) { throw new Error("Fault delayMs must be a non-negative integer."); } + if ( + input.idempotencyKey !== undefined && + (typeof input.idempotencyKey !== "string" || input.idempotencyKey.length < 1 || input.idempotencyKey.length > 255) + ) { + throw new Error("Fault idempotencyKey must be a string of 1 to 255 characters."); + } +} + +/** JSON with object keys sorted, so two copies of one fault compare equal. */ +function canonicalJson(value: unknown): string { + return JSON.stringify(value, (_key, item: unknown) => + item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries(Object.entries(item as Record).sort(([a], [b]) => (a < b ? -1 : 1))) + : item, + ); } function normalizeMatch(match: FaultMatch): FaultMatch { diff --git a/packages/@emulators/core/src/index.ts b/packages/@emulators/core/src/index.ts index a8d1e69d0..9ad701600 100644 --- a/packages/@emulators/core/src/index.ts +++ b/packages/@emulators/core/src/index.ts @@ -73,10 +73,13 @@ export { type LedgerWebhookDelivery, } from "./ledger.js"; export { + FaultKeyConflictError, FaultRegistry, createFaultMiddleware, type ArmedFault, type FaultArmInput, + type FaultArmResult, + type FaultKeyRecord, type FaultLedgerMarker, type FaultMatch, type FaultRegistrySnapshot, diff --git a/packages/emulate/src/index.ts b/packages/emulate/src/index.ts index ee703012b..f9b9fd609 100644 --- a/packages/emulate/src/index.ts +++ b/packages/emulate/src/index.ts @@ -43,7 +43,8 @@ Control plane (under /_emulate on each service): GET /_emulate/mcp MCP surface (when supported) GET /_emulate/state current emulator state GET /_emulate/ledger request ledger (DELETE to clear) - GET /_emulate/faults armed one-shot faults (POST to arm, DELETE to clear) + GET /_emulate/faults armed one-shot faults (POST to arm, with an optional + idempotencyKey; DELETE to clear) GET /_emulate/logs webhook deliveries and recent requests POST /_emulate/instances create an instance (server-generated unguessable name; optional {"instance"} body adds a prefix)