From 85c4ff150cdde791ccdd64bf300fc85224064c0e Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 17:15:48 -0800 Subject: [PATCH 1/5] Guard kept-order reads against concurrent cache changes --- packages/sdk/src/implementation/tool-search-store.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/implementation/tool-search-store.ts b/packages/sdk/src/implementation/tool-search-store.ts index 4c2b9229d5..6097239f5e 100644 --- a/packages/sdk/src/implementation/tool-search-store.ts +++ b/packages/sdk/src/implementation/tool-search-store.ts @@ -213,8 +213,9 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { orders.delete(key); ordersBytes -= kept.order.bytes; }; - /** Mark a kept order read now, as the most recently read. */ + /** Mark an order read now, unless it was evicted or replaced while its page was read. */ const touch = (key: string, order: Kept, used: number) => { + if (orders.get(key)?.order !== order) return; orders.delete(key); orders.set(key, { order, used }); }; @@ -727,7 +728,8 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { yield* Effect.annotateCurrentSpan("executor.search.order", "kept"); return page; } - forget(key); + // Only discard the order this read found stale, not one admitted while it waited. + if (orders.get(key)?.order === remembered) forget(key); } if (evictable(keptOrderBytes, yield* monotonicMillis) === undefined) { // Every kept search is in use: rank this page as before rather than evict one. From 42863d304f45983a8bde7b875247c6bb1010053a Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 17:19:34 -0800 Subject: [PATCH 2/5] Exercise kept-order interleavings in the self-host test server --- .../hosted/testing/monotonic-clock-fixture.ts | 45 +++ apps/hosted/testing/self-host.ts | 4 + apps/hosted/testing/statement-hold-fixture.ts | 36 ++- e2e/test-plan.ts | 30 ++ e2e/tests/mcp-search-kept-order-races.spec.ts | 298 ++++++++++++++++++ 5 files changed, 403 insertions(+), 10 deletions(-) create mode 100644 apps/hosted/testing/monotonic-clock-fixture.ts create mode 100644 e2e/tests/mcp-search-kept-order-races.spec.ts diff --git a/apps/hosted/testing/monotonic-clock-fixture.ts b/apps/hosted/testing/monotonic-clock-fixture.ts new file mode 100644 index 0000000000..543287ceab --- /dev/null +++ b/apps/hosted/testing/monotonic-clock-fixture.ts @@ -0,0 +1,45 @@ +/** + * Test host only: advances Effect's monotonic clock so a scenario can age a kept search order + * while a read waits. Wall time, sleeps, and deadlines based on them use the live clock. + * Production entry points never provide it. + */ +import { Clock, Effect, Layer, Schema } from "effect"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; + +const path = "/api/devtools/monotonic-clock"; +const Advance = Schema.Struct({ advanceMillis: Schema.Int.check(Schema.isGreaterThan(0)) }); + +export const monotonicClockFixture = Effect.gen(function* () { + const live = yield* Clock.Clock; + let offset = 0n; + const nanos = () => live.monotonicTimeNanosUnsafe() + offset; + const clock = Layer.succeed(Clock.Clock, { + currentTimeMillisUnsafe: () => live.currentTimeMillisUnsafe(), + currentTimeMillis: live.currentTimeMillis, + currentTimeNanosUnsafe: () => live.currentTimeNanosUnsafe(), + currentTimeNanos: live.currentTimeNanos, + sleep: (duration) => live.sleep(duration), + monotonicTimeNanosUnsafe: nanos, + monotonicTimeNanos: Effect.sync(nanos), + }); + const routes = Layer.mergeAll( + HttpRouter.add( + "POST", + path, + Effect.gen(function* () { + const { advanceMillis } = yield* HttpServerRequest.schemaBodyJson(Advance); + offset += BigInt(advanceMillis) * 1_000_000n; + return HttpServerResponse.jsonUnsafe({ advanced: true }); + }).pipe(Effect.orDie), + ), + HttpRouter.add( + "DELETE", + path, + Effect.sync(() => { + offset = 0n; + return HttpServerResponse.jsonUnsafe({ reset: true }); + }), + ), + ); + return { clock, routes }; +}); diff --git a/apps/hosted/testing/self-host.ts b/apps/hosted/testing/self-host.ts index 2836bd27a1..f81879d628 100644 --- a/apps/hosted/testing/self-host.ts +++ b/apps/hosted/testing/self-host.ts @@ -16,6 +16,7 @@ import { devAppName, freePort } from "../../../scripts/dev-host.ts"; import { developmentSettings, developmentSignIn } from "./development.ts"; import { accessCheckFixture } from "./access-check-fixture.ts"; import { statementHoldFixture } from "./statement-hold-fixture.ts"; +import { monotonicClockFixture } from "./monotonic-clock-fixture.ts"; import { storageFaultFixture } from "./storage-fault-fixture.ts"; import { apiDocumentFixture } from "./api-document-fixture.ts"; @@ -54,6 +55,7 @@ const command = Command.make("test-self-host", { const target = yield* developmentSettings; const directory = yield* Config.NonEmptyString("EXECUTOR_DATA_DIR"); const storageFault = yield* storageFaultFixture; + const monotonic = yield* monotonicClockFixture; const server = Layer.unwrap( Effect.gen(function* () { const development = yield* developmentSignIn(target, organization); @@ -63,6 +65,7 @@ const command = Command.make("test-self-host", { HttpRouter.add("GET", "/api/devtools", development.status), HttpRouter.add("POST", "/api/devtools/operator", development.signIn), statementHold.routes, + monotonic.routes, storageFault.routes, yield* apiDocumentFixture, product, @@ -88,6 +91,7 @@ const command = Command.make("test-self-host", { yield* Console.log(`Starting local test server at ${target.origin}/login`); // Fibers inherit the reference, so every statement the server runs reaches the fault. return yield* Layer.launch(server).pipe( + Effect.provide(monotonic.clock), Effect.provideService(Statement.CurrentTransformer, storageFault.transformer), ); }).pipe(Effect.provideServiceEffect(ConfigProvider.ConfigProvider, testServerConfiguration)), diff --git a/apps/hosted/testing/statement-hold-fixture.ts b/apps/hosted/testing/statement-hold-fixture.ts index 1e13339be3..4cbde59c6a 100644 --- a/apps/hosted/testing/statement-hold-fixture.ts +++ b/apps/hosted/testing/statement-hold-fixture.ts @@ -3,12 +3,12 @@ * personal access token check an MCP tool call makes, until the scenario releases it, so a * scenario can change saved state while that check waits. It can instead hold a request just after * its check of the caller's access to one app or one account has answered, so a scenario can - * revoke that access between the check and the write the request makes next. It can instead fail - * the next read of account rows an MCP tool call makes, as a dropped connection would, so a - * scenario can see what the call does with a read that failed for storage. PostgreSQL reads take no - * row locks, so no lock on rows a scenario created can hold them; the test host holds the statement - * before it reaches the database, or after its reply, instead. Production entry points never - * provide it. + * revoke that access between the check and the write the request makes next. It can hold a kept + * search page after SQL answers so a scenario can change the order cache before the read resumes. + * It can instead fail the next read of account rows an MCP tool call makes, as a dropped + * connection would, to see how the call handles a storage failure. PostgreSQL reads take no row + * locks, so no lock on rows a scenario created can hold them; the test host holds the statement + * before it reaches the database, or after its reply, instead. Production entry points never provide it. */ import { Deferred, Effect, Layer, Option, Schema, type Tracer } from "effect"; import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; @@ -34,6 +34,8 @@ const answeredChecks = { "app-access": "left join hosted_app_access p on", "account-access": "select p.account_id as account", } as const; +/** The row lookup by ID in a kept-order page, not the ranking statement. */ +const keptOrderPage = "where s.id in (select value from json_array_elements_text("; const path = "/api/devtools/statement-hold"; const Arm = Schema.Union([ // `toolCall` holds only a check made inside an MCP tool call, not one made by discovery. @@ -43,6 +45,7 @@ const Arm = Schema.Union([ after: Schema.Literals(["app-access", "account-access"]), id: Schema.NonEmptyString, }), + Schema.Struct({ after: Schema.Literal("kept-order") }), Schema.Struct({ authentication: Schema.Literal("tool-call") }), // Fails the next read of account rows inside a tool call instead of holding it. Schema.Struct({ fail: Schema.Literal("tool-call-accounts") }), @@ -64,8 +67,11 @@ const inToolCall = Effect.map(Effect.currentParentSpan, (span) => { interface Hold { /** The app whose access check to hold, or the next PAT check inside a tool call. */ readonly app: string | undefined; - /** Hold the next answered check of this kind that binds this ID, instead. */ - readonly after: { readonly check: keyof typeof answeredChecks; readonly id: string } | undefined; + /** Hold the next answered check binding this ID, or the next answered kept-order page. */ + readonly after: + | { readonly check: keyof typeof answeredChecks; readonly id: string } + | { readonly check: "kept-order" } + | undefined; /** Hold the app's check only inside a tool call. */ readonly toolCall: boolean; /** Fail the next read of account rows inside a tool call; nothing is held. */ @@ -97,7 +103,12 @@ export const statementHoldFixture = Effect.gen(function* () { if (hold.after !== undefined) { const statement: Statement.Statement = Reflect.apply(target, self, args); const [text, bound] = statement.compile(); - if (!text.includes(answeredChecks[hold.after.check]) || !bound.includes(hold.after.id)) + if (hold.after.check === "kept-order") { + if (!text.includes(keptOrderPage)) return statement; + } else if ( + !text.includes(answeredChecks[hold.after.check]) || + !bound.includes(hold.after.id) + ) return statement; // Disarmed by the first such check to answer, not to be built: one built and never run // must not take the hold. @@ -163,7 +174,12 @@ export const statementHoldFixture = Effect.gen(function* () { yield* release; armed = current = { app: "app" in arm ? arm.app : undefined, - after: "after" in arm ? { check: arm.after, id: arm.id } : undefined, + after: + "id" in arm + ? { check: arm.after, id: arm.id } + : "after" in arm + ? { check: arm.after } + : undefined, toolCall: "app" in arm && arm.toolCall === true, failAccounts: "fail" in arm, reached: yield* Deferred.make(), diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 1466b740f8..5be9989041 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -8181,6 +8181,36 @@ export const scenarios = plan({ local: na("Hosted MCP scenario"), }, }, + mcpSearchKeptOrderEvicted: { + fixtures: "actors", + file: "mcp-search-kept-order-races.spec.ts", + title: "A held kept-order read cannot resurrect an evicted order or skew admission", + targets: { + "self-host": scheduled, + cloud: na("Needs the self-host test entry point's SQL hold and monotonic clock."), + local: na("Needs the self-host test entry point's SQL hold and monotonic clock."), + }, + }, + mcpSearchKeptOrderReplaced: { + fixtures: "actors", + file: "mcp-search-kept-order-races.spec.ts", + title: "A held kept-order read cannot overwrite a fresher order", + targets: { + "self-host": scheduled, + cloud: na("Needs the self-host test entry point's SQL hold."), + local: na("Needs the self-host test entry point's SQL hold."), + }, + }, + mcpSearchKeptOrderStale: { + fixtures: "actors", + file: "mcp-search-kept-order-races.spec.ts", + title: "A held kept-order read with a stale stamp cannot discard a fresher order", + targets: { + "self-host": scheduled, + cloud: na("Needs the self-host test entry point's SQL hold."), + local: na("Needs the self-host test entry point's SQL hold."), + }, + }, mcpSearchStatements: { fixtures: "actors", file: "mcp-search-statements.spec.ts", diff --git a/e2e/tests/mcp-search-kept-order-races.spec.ts b/e2e/tests/mcp-search-kept-order-races.spec.ts new file mode 100644 index 0000000000..2b6484ec9c --- /dev/null +++ b/e2e/tests/mcp-search-kept-order-races.spec.ts @@ -0,0 +1,298 @@ +/** + * Hold a page at its SQL reply, then evict or replace the order it read. The page still answers + * from the right listing, but must not put an evicted order back, overwrite the replacement, + * or discard it on a stamp miss. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Fiber, Layer, Schedule, Schema } from "effect"; +import { Api, body, SessionClients } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { Evidence } from "../support/evidence.ts"; +import { startDevelopmentServer } from "../support/managed-server.ts"; +import { McpClient } from "../support/mcp-client.ts"; +import { Target } from "../support/platform.ts"; +import { throttlingCollector } from "../support/throttling-collector.ts"; +import { scenarios } from "../test-plan.ts"; + +const holdPath = "/api/devtools/statement-hold"; +const clockPath = "/api/devtools/monotonic-clock"; +const Held = Schema.Struct({ held: Schema.Boolean }); +const Users = Schema.Struct({ + users: Schema.Array(Schema.Struct({ id: Schema.String, email: Schema.String })), +}); +const Organizations = Schema.Array(Schema.Struct({ id: Schema.String })); +const Token = Schema.Struct({ key: Schema.RedactedFromValue(Schema.String), id: Schema.String }); +const Ran = Schema.Struct({ + structuredContent: Schema.Struct({ + status: Schema.Literal("completed"), + execution: Schema.Struct({ ok: Schema.Literal(true), value: Schema.Unknown }), + }), +}); +const Page = Schema.Struct({ + items: Schema.Array(Schema.Struct({ path: Schema.String, description: Schema.String })), +}); + +/** The revision changes on invalidation without changing the deployment or listing ID. */ +const source = `import { defineApp, mutation, object, query, router, string } from "apps"; +export default defineApp({}, async (ctx) => { + const revision = await ctx.cache.get({ + key: "revision", schema: string(), freshFor: "1 day", load: async () => crypto.randomUUID(), + }); + const tools = { + bump: mutation({ input: object({}) }, async () => { + await ctx.cache.invalidate("revision"); + return true; + }), + }; + for (let i = 0; i < 40; i++) { + tools["probe" + i] = query( + { description: "Kept probe " + i + " revision " + revision, input: object({}) }, + async () => i, + ); + } + return { tools: router(tools) }; +});`; + +/** Test the byte limit below the count limit: each of these keys holds about 1.52 MB. */ +const heavyQuery = (index: number) => ` + "kept probe " + Array.from({ length: 190 }, (_, i) => + ("word" + ${index} + "x" + i).padEnd(4000, "w") + ).join(" ")`; + +const raceFixture = Effect.gen(function* () { + const target = yield* Target; + const evidence = yield* Evidence; + const collector = yield* throttlingCollector; + // Only the development test entry point mounts either control. + const origin = yield* startDevelopmentServer(target, { + OTEL_EXPORTER_OTLP_ENDPOINT: collector.endpoint, + }); + const api = yield* Api.pipe( + Effect.provide(Layer.fresh(Api.layer)), + Effect.provide(Layer.fresh(SessionClients.layer)), + Effect.provideService(Target, { ...target, metadata: { ...target.metadata, origin } }), + ); + const owner = yield* api.session(); + const send = (method: "GET" | "POST" | "DELETE", path: string, payload?: unknown) => + api + .request(owner, method, `${origin}${path}`, payload, { origin }) + .pipe( + Effect.tap((response) => + Effect.sync(() => expect(response.status, `${method} ${path}`).toBe(200)), + ), + ); + yield* send("POST", "/api/devtools/operator", {}); + const directory = yield* body(Users, yield* send("GET", "/api/auth/admin/list-users")); + const developer = directory.users.find((user) => user.email === "agent-agent@example.test"); + if (developer === undefined) return yield* Effect.die("Development owner is missing"); + yield* send("POST", "/api/auth/admin/impersonate-user", { userId: developer.id }); + const [org] = yield* body(Organizations, yield* send("GET", "/api/auth/organization/list")); + if (org === undefined) return yield* Effect.die("Development organization is missing"); + const prefix = `/api/organizations/${org.id}`; + const app = yield* body( + App, + yield* send("POST", `${prefix}/apps/deploy`, { + name: "Kept order race", + files: [{ path: "index.ts", content: source }, appsManifest], + }), + ); + const token = yield* body( + Token, + yield* send("POST", "/api/auth/api-key/create", { name: "Kept order race" }), + ); + yield* Effect.addFinalizer(() => + send("POST", "/api/auth/api-key/delete", { keyId: token.id }).pipe(Effect.orDie), + ); + yield* Effect.addFinalizer(() => send("DELETE", clockPath).pipe(Effect.orDie)); + const mcp = yield* McpClient; + const connect = (label: string) => + Effect.gen(function* () { + const client = yield* mcp.connect(token.key, label, { origin, organization: org.id }); + const execute = (step: string, code: string) => + client + .use(step, (client, signal) => + client.callTool({ name: "execute", arguments: { code } }, undefined, { signal }), + ) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Ran)), + Effect.map((ran) => ran.structuredContent.execution.value), + ); + return { + /** Each connection reads one program at a time; the held page uses its own connection. */ + search: (step: string, query: string, offset: number) => + Effect.gen(function* () { + // The large query leaves room for only one item in the page budget. Explicit offsets + // re-read the same rows without returning that query in the program's result. + const code = `const page = await tools.search({ query: ${query}, offset: ${offset}, limit: 10 }); + return { items: page.items };`; + const page = yield* Schema.decodeUnknownEffect(Page)(yield* execute(step, code)); + // The writer finishes before the held reader is released, so no two replies race. + const response = (yield* evidence.requests).findLast( + ({ method, path }) => method === "POST" && path === "/mcp", + ); + if (response === undefined) return yield* Effect.die("Search response trace missing"); + const orders = yield* collector.query(response.traceId).pipe( + Effect.flatMap(({ data }) => { + const orders = data + .map(({ span }) => span) + .filter((span) => span.operationName === "sdk.tools.search") + .map((span) => span.tags["executor.search.order"]); + return orders.length === 1 + ? Effect.succeed(orders) + : Effect.fail(new Error(`${step}: missing search spans`)); + }), + Effect.retry({ schedule: Schedule.spaced("50 millis"), times: 200 }), + ); + expect(page.items.length, step).toBeGreaterThan(0); + return { ...page, orders }; + }), + }; + }); + const main = yield* connect("orders"); + const reader = yield* connect("held-reader"); + // Start with the revision populated and the listing written before keeping any order. + expect( + (yield* main.search("Read the first page to warm the listing", '"kept probe"', 0)).orders, + ).toEqual(["ranked"]); + const page = (step: string, query: string) => main.search(step, query, 20); + const keep = (step: string, query = '"kept probe"') => + Effect.gen(function* () { + const second = yield* main.search(step, query, 10); + expect(second.orders, step).toEqual(["ordered"]); + return query; + }); + const replaceListing = send("POST", `${prefix}/apps/${app.id}/tools/call`, { + tool: "bump", + kind: "mutation", + input: {}, + }).pipe(Effect.tap((response) => Effect.sync(() => expect(response.body).toBe(true)))); + + const holdPage = (query: string, meanwhile: Effect.Effect) => + Effect.gen(function* () { + yield* send("POST", holdPath, { after: "kept-order" }); + const fiber = yield* Effect.forkChild(reader.search("Read the held page", query, 20)); + yield* send("GET", holdPath).pipe( + Effect.flatMap((response) => body(Held, response)), + Effect.filterOrFail( + ({ held }) => held, + () => new Error("The kept-order read was not held"), + ), + Effect.retry({ schedule: Schedule.spaced("50 millis"), times: 200 }), + ); + yield* meanwhile; + yield* send("DELETE", holdPath); + return yield* Fiber.join(fiber); + }).pipe(Effect.ensuring(send("DELETE", holdPath).pipe(Effect.orDie))); + + return { + page, + keep, + holdPage, + replaceListing, + ageOrders: send("POST", clockPath, { advanceMillis: 31_000 }), + }; +}); + +layer(HostedLive, { excludeTestServices: true })("MCP kept-order races", (it) => { + it.effect(scenarios.mcpSearchKeptOrderEvicted.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const { page, keep, holdPage, ageOrders } = yield* raceFixture; + const a = yield* keep("Keep A", heavyQuery(0)); + const before = yield* page("Read A before eviction", a); + expect(before.orders).toEqual(["kept"]); + // A must be least recently read at the first admission. + const b = yield* keep("Keep B", heavyQuery(1)); + const c = yield* keep("Keep C", heavyQuery(2)); + const retained = [b, c]; + const held = yield* holdPage( + a, + Effect.gen(function* () { + yield* ageOrders; + // Four orders do not fit in 6 MB. D's admission evicts idle A. + retained.push(yield* keep("Admit D, evicting A", heavyQuery(3))); + }), + ); + expect(held.orders).toEqual(["kept"]); + expect(held.items).toEqual(before.items); + // On old code the held reply resurrects A without its bytes. After aging again and + // refreshing B/C/D, room for another order exists only if the pre-check can subtract + // idle A's uncounted bytes. Fixed code has no A to subtract, so E must only rank. + yield* ageOrders; + for (const [i, next] of retained.entries()) + expect((yield* page(`Refresh retained order ${i + 1}`, next)).orders).toEqual(["kept"]); + const probe = yield* page("Try to admit E", '"kept probe extra"'); + expect( + probe.orders, + "A resurrected order must not make room for another admission", + ).toEqual(["ranked"]); + }).pipe(Effect.provide(McpClient.layer)), + ), + ); + + it.effect(scenarios.mcpSearchKeptOrderReplaced.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const { page, keep, holdPage, replaceListing } = yield* raceFixture; + const next = yield* keep("Keep O1"); + const old = yield* page("Read O1", next); + const held = yield* holdPage( + next, + replaceListing.pipe( + Effect.andThen(page("Admit O2", next)), + Effect.tap((fresh) => + Effect.sync(() => { + expect(fresh.orders).toEqual(["ordered"]); + expect(fresh.items).not.toEqual(old.items); + }), + ), + ), + ); + expect(held.orders).toEqual(["kept"]); + expect(held.items).toEqual(old.items); + const after = yield* page("Read the replacement after O1 finishes", next); + expect(after.items).not.toEqual(old.items); + expect(after.orders, "O1 must not overwrite O2").toEqual(["kept"]); + }).pipe(Effect.provide(McpClient.layer)), + ), + ); + + it.effect(scenarios.mcpSearchKeptOrderStale.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const { page, keep, holdPage, replaceListing } = yield* raceFixture; + yield* keep("Fill the cache with B", heavyQuery(1)); + yield* keep("Fill the cache with C", heavyQuery(2)); + const next = yield* keep("Keep O1", heavyQuery(0)); + const old = yield* page("Read O1", next); + yield* replaceListing; + // O1's read has a stale stamp. A second miss drops O1 and admits O2. The three + // active orders now leave less than 2 MB free, so the held miss must rank without + // admitting another order. If it drops O2 first, it finds room and orders instead. + const held = yield* holdPage( + next, + page("Admit O2 while O1's stamp miss waits", next).pipe( + Effect.tap((fresh) => + Effect.sync(() => { + expect(fresh.orders).toEqual(["ordered"]); + expect(fresh.items).not.toEqual(old.items); + }), + ), + ), + ); + const after = yield* page("Read the replacement after O1's stamp miss", next); + expect(held.items).toEqual(after.items); + expect(held.items).not.toEqual(old.items); + expect(held.orders, "O1's stamp miss must not discard O2 and admit another order").toEqual([ + "ranked", + ]); + expect(after.orders).toEqual(["kept"]); + }).pipe(Effect.provide(McpClient.layer)), + ), + ); +}); From 421f49c22f934b4d3b2771b8b2c0ef7cbc955ed7 Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 17:19:34 -0800 Subject: [PATCH 3/5] Run kept-order and search walk scenarios in fork CI --- .github/workflows/fork-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml index cfa63ab317..facd5ed272 100644 --- a/.github/workflows/fork-ci.yml +++ b/.github/workflows/fork-ci.yml @@ -131,3 +131,5 @@ jobs: - name: MCP search scenarios run: | bun run e2e:local --test-name 'MCP search resolves namespace-relative dotted paths|A broad MCP tool search returns concise pages' + - name: MCP search kept-order scenarios + run: bun run e2e:self-host --test-name '^A search being paged keeps its order|^An MCP search without a query pages|^A held kept-order read' --workers 1 From ea2f675bfe9ac099feacc216cae3810ee4b2ba6f Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 17:19:34 -0800 Subject: [PATCH 4/5] Document the kept-order pre-ranking threshold --- packages/sdk/src/implementation/tool-search-store.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/implementation/tool-search-store.ts b/packages/sdk/src/implementation/tool-search-store.ts index 6097239f5e..0dd85ae466 100644 --- a/packages/sdk/src/implementation/tool-search-store.ts +++ b/packages/sdk/src/implementation/tool-search-store.ts @@ -147,7 +147,8 @@ const oversizedBytes = 2_000_000; * evicted. While every kept search is being paged, a new one is not kept: its pages rank as * before, so more concurrent walks than {@link keptOrders} cost what they did before orders were * kept, instead of evicting each other. Once 32 searches have each been read within 30 s, or - * their orders fill {@link keptBytes}, every further walk goes uncached until one of them idles. + * orders read within 30 s hold more than `keptBytes - keptOrderBytes`, new walks go uncached + * until one of those orders idles. * Times are monotonic, so a wall-clock step neither evicts a search in use nor keeps an idle one. */ const keptIdleMillis = 30_000; From f3d0fed9178975f028d165c8bd1f5d9286a4165f Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 17:30:40 -0800 Subject: [PATCH 5/5] Refresh retained orders before the stale-stamp race --- apps/hosted/testing/statement-hold-fixture.ts | 1 + e2e/tests/mcp-search-kept-order-races.spec.ts | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/hosted/testing/statement-hold-fixture.ts b/apps/hosted/testing/statement-hold-fixture.ts index 4cbde59c6a..35d10d3289 100644 --- a/apps/hosted/testing/statement-hold-fixture.ts +++ b/apps/hosted/testing/statement-hold-fixture.ts @@ -45,6 +45,7 @@ const Arm = Schema.Union([ after: Schema.Literals(["app-access", "account-access"]), id: Schema.NonEmptyString, }), + // Holds the next kept-order page after SQL answers; only tool search runs that statement. Schema.Struct({ after: Schema.Literal("kept-order") }), Schema.Struct({ authentication: Schema.Literal("tool-call") }), // Fails the next read of account rows inside a tool call instead of holding it. diff --git a/e2e/tests/mcp-search-kept-order-races.spec.ts b/e2e/tests/mcp-search-kept-order-races.spec.ts index 2b6484ec9c..73c0eac5cd 100644 --- a/e2e/tests/mcp-search-kept-order-races.spec.ts +++ b/e2e/tests/mcp-search-kept-order-races.spec.ts @@ -266,10 +266,12 @@ layer(HostedLive, { excludeTestServices: true })("MCP kept-order races", (it) => context, Effect.gen(function* () { const { page, keep, holdPage, replaceListing } = yield* raceFixture; - yield* keep("Fill the cache with B", heavyQuery(1)); - yield* keep("Fill the cache with C", heavyQuery(2)); + const b = yield* keep("Fill the cache with B", heavyQuery(1)); + const c = yield* keep("Fill the cache with C", heavyQuery(2)); const next = yield* keep("Keep O1", heavyQuery(0)); const old = yield* page("Read O1", next); + expect((yield* page("Refresh retained order B", b)).orders).toEqual(["kept"]); + expect((yield* page("Refresh retained order C", c)).orders).toEqual(["kept"]); yield* replaceListing; // O1's read has a stale stamp. A second miss drops O1 and admits O2. The three // active orders now leave less than 2 MB free, so the held miss must rank without