diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 81cb0d2203..38751a86d0 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -8135,6 +8135,17 @@ export const scenarios = plan({ local: na("Local has no hosted organization policies."), }, }, + mcpSearchWalk: { + fixtures: "actors", + file: "mcp-search-walk.spec.ts", + title: + "An MCP search without a query pages through 8,000 tools ranking them once, each later page reading only its own rows", + targets: { + "self-host": scheduled, + cloud: na("This scenario reads SQL spans from the self-host Motel collector."), + local: na("Hosted MCP scenario"), + }, + }, mcpSearchStatements: { fixtures: "actors", file: "mcp-search-statements.spec.ts", diff --git a/e2e/tests/mcp-search-walk.spec.ts b/e2e/tests/mcp-search-walk.spec.ts new file mode 100644 index 0000000000..5486240b7b --- /dev/null +++ b/e2e/tests/mcp-search-walk.spec.ts @@ -0,0 +1,217 @@ +/** + * Paging `tools.search` through the whole catalog, with neither query nor namespace, ranked every + * row the catalog holds again for each page, so a walk of n tools cost n squared over the page + * size on the product's one thread. On Nov 18 2026 three such walks over 11,467 tools held a + * self-hosted product at full CPU for minutes. A search now ranks once and keeps the order for its + * later pages: past the second, a page reads only its own rows, however large the catalog. + */ +import { expect, layer } from "@effect/vitest"; +import { Console, Effect, Schedule, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { Evidence, Telemetry } from "../support/evidence.ts"; +import { McpClient } from "../support/mcp-client.ts"; +import { scenarios } from "../test-plan.ts"; + +const apps = 16; +const toolsPerApp = 500; +const pageItems = 150; + +/** An app of `toolsPerApp` small tools, generated when it evaluates. */ +const walkSource = ( + index: number, +) => `import { defineApp, object, query, router, string } from "apps"; +export default defineApp({}, async () => { + const tools = {}; + for (let t = 0; t < ${toolsPerApp}; t++) + tools["walkprobe" + t] = query( + { description: "Walk probe " + t + " of app ${index}", input: object({ id: string() }) }, + async () => t, + ); + return { tools: router(tools) }; +});`; + +/** Every page of a search with neither query nor namespace, counting the probes it lists. */ +const walk = `let next = { limit: ${pageItems} }; +let seen = 0; +let pages = 0; +while (next !== null) { + const page = await tools.search(next); + pages += 1; + seen += page.items.filter((item) => item.path.includes("walkprobe")).length; + next = page.next; +} +return { seen, pages };`; + +const Token = Schema.Struct({ key: Schema.RedactedFromValue(Schema.String), id: Schema.String }); +const Walked = Schema.Struct({ + structuredContent: Schema.Struct({ + status: Schema.Literal("completed"), + execution: Schema.Struct({ + ok: Schema.Literal(true), + value: Schema.Struct({ seen: Schema.Number, pages: Schema.Number }), + }), + }), +}); + +layer(HostedLive, { excludeTestServices: true })("MCP search walk", (it) => { + it.effect(scenarios.mcpSearchWalk.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence, + telemetry = yield* Telemetry, + mcp = yield* McpClient; + const organization = actors.organization.id; + const prefix = `/api/organizations/${organization}`; + const run = randomUUID().slice(0, 8); + yield* Effect.forEach( + Array.from({ length: apps }, (_, index) => index), + (index) => + Effect.gen(function* () { + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: `Search walk ${index} ${run}`, + files: [{ path: "index.ts", content: walkSource(index) }, appsManifest], + }); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe( + Effect.map((response) => expect(response.status).toBe(200)), + Effect.orDie, + ), + ); + }), + { concurrency: 4, discard: true }, + ); + const created = yield* api.request(actors.owner, "POST", "/api/auth/api-key/create", { + name: "MCP search walk", + }); + expect(created.status).toBe(200); + const token = yield* body(Token, created); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "POST", "/api/auth/api-key/delete", { keyId: token.id }) + .pipe(Effect.orDie), + ); + const client = yield* mcp.connect(token.key, "pat", { organization }); + + /** Walk the whole catalog in one execute, and read each page's search span and SQL time. */ + const walkOnce = (operation: string) => + Effect.gen(function* () { + const started = Date.now(); + const called = yield* client.use(operation, (client, signal) => + client.callTool({ name: "execute", arguments: { code: walk } }, undefined, { + signal, + }), + ); + const elapsed = Date.now() - started; + const walked = yield* Schema.decodeUnknownEffect(Walked)(called); + const request = (yield* evidence.requests) + .filter((request) => request.path === "/mcp") + .at(-1); + if (request === undefined) + return yield* Effect.die(new Error("MCP request evidence missing")); + const { pages } = walked.structuredContent.execution.value; + const searches = yield* telemetry.query(request.traceId).pipe( + Effect.flatMap((result) => { + if (result.meta?.truncated === true) + return Effect.die(new Error("The walk's trace must reach Motel whole")); + const byId = new Map(result.data.map(({ span }) => [span.spanId, span])); + const searchOf = (spanId: string | null) => { + const visited = new Set(); + for (let parent = spanId; parent !== null && !visited.has(parent);) { + visited.add(parent); + const span = byId.get(parent); + if (span === undefined) return undefined; + if (span.operationName === "sdk.tools.search") return span.spanId; + parent = span.parentSpanId; + } + return undefined; + }; + const sqlMs = new Map(); + for (const { span } of result.data) { + if (span.operationName !== "sql.execute") continue; + const search = searchOf(span.parentSpanId); + if (search !== undefined) + sqlMs.set(search, (sqlMs.get(search) ?? 0) + span.durationMs); + } + const searches = result.data + .filter(({ span }) => span.operationName === "sdk.tools.search") + .map(({ span }) => ({ + startTime: span.startTime, + order: span.tags["executor.search.order"] ?? "", + durationMs: span.durationMs, + sqlMs: sqlMs.get(span.spanId) ?? 0, + })) + .sort((a, b) => a.startTime.localeCompare(b.startTime)); + return searches.length === pages + ? Effect.succeed(searches) + : Effect.fail(new Error("Every page's search span must reach Motel")); + }), + Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), + Effect.timeout("25 seconds"), + ); + return { ...walked.structuredContent.execution.value, elapsed, searches }; + }); + + // The first walk ranks the catalog for its first page, and again for its second, keeping + // that order; every later page reads its rows by it. The second walk, over the same + // listings, ranks its first page and reads every later one by the order the first kept. + const firstWalk = yield* walkOnce("Walk the whole catalog for the first time"); + const secondWalk = yield* walkOnce("Walk the whole catalog again"); + const summarize = (walked: typeof firstWalk) => { + const [first, second, ...rest] = walked.searches; + return { + pages: walked.pages, + elapsed: walked.elapsed, + firstPageSqlMs: first?.sqlMs ?? 0, + secondPageSqlMs: second?.sqlMs ?? 0, + laterPagesSqlMs: rest.reduce((sum, search) => sum + search.sqlMs, 0), + medianLaterPageSqlMs: + rest.map((search) => search.sqlMs).sort((x, y) => x - y)[ + Math.floor(rest.length / 2) + ] ?? 0, + slowestLaterPageSqlMs: Math.max(0, ...rest.map((search) => search.sqlMs)), + totalSqlMs: walked.searches.reduce((sum, search) => sum + search.sqlMs, 0), + orders: walked.searches.map((search) => search.order), + }; + }; + const summary = { + tools: apps * toolsPerApp, + first: summarize(firstWalk), + again: summarize(secondWalk), + }; + yield* Console.log(JSON.stringify({ label: "Whole-catalog walk", ...summary })); + yield* evidence.json("mcp-search-walk.json", { + summary, + first: firstWalk.searches, + again: secondWalk.searches, + }); + yield* evidence.measure("Whole-catalog walk", firstWalk.elapsed); + yield* evidence.measure("Whole-catalog walk again", secondWalk.elapsed); + + for (const walked of [firstWalk, secondWalk]) { + expect(walked.seen, "The walk lists every tool once").toBe(apps * toolsPerApp); + expect(walked.pages).toBeGreaterThan(20); + } + const kept = (count: number) => Array.from({ length: count }, () => "kept"); + expect(summary.first.orders).toEqual(["ranked", "ordered", ...kept(firstWalk.pages - 2)]); + expect(summary.again.orders).toEqual(["ranked", ...kept(secondWalk.pages - 1)]); + // A kept page reads only its own rows: the typical one costs well under the ranking the + // first page does, where every page used to cost a ranking. + for (const walked of [summary.first, summary.again]) + expect( + walked.medianLaterPageSqlMs, + `The median kept page read ${walked.medianLaterPageSqlMs} ms of SQL; the first page, which ranks the catalog, ${walked.firstPageSqlMs} ms`, + ).toBeLessThan(walked.firstPageSqlMs / 2); + }).pipe(Effect.provide(McpClient.layer)), + ), + ); +}); diff --git a/packages/sdk/src/implementation/tool-search-store.ts b/packages/sdk/src/implementation/tool-search-store.ts index a720d29171..992907f7ae 100644 --- a/packages/sdk/src/implementation/tool-search-store.ts +++ b/packages/sdk/src/implementation/tool-search-store.ts @@ -87,6 +87,89 @@ const Ranked = Schema.Struct({ definitions: Schema.Array(Schema.Tuple([Schema.String, Schema.Int, Json])), }); export type RankedPage = typeof Ranked.Type; +/** A ranked page with the order of every item it ranked; see {@link Kept}. */ +const RankedInOrder = Schema.Struct({ + ...Ranked.fields, + stamp: Schema.String, + /** + * Every item in rank order as `target.position[.ordinal...]`, comma-separated: the target it is + * at, the position of its first row in that target's listing, and every other target it merges. + */ + ranking: Schema.NullOr(Schema.String), +}); +/** The rows of a page of a kept order, read with the stamp of the listings they belong to. */ +const Shown = Schema.Struct({ + stamp: Schema.String, + items: Schema.Array( + Schema.Struct({ + id: Schema.String, + listing: Schema.String, + tool: StoredTool, + router: Schema.NullOr(ToolRouter), + }), + ), + definitions: Ranked.fields.definitions, +}); + +/** + * A search's order, kept for its later pages. A page past the first reads its rows by the kept + * order instead of ranking every row the targets hold again, so paging through a catalog of n + * tools costs one ranking and n rows, not one ranking per page. The order is used only while + * every listing it ranked still has the digest it had (`stamp`), so a page never mixes two + * rankings. + * + * Items are kept as integers: `cells` holds three per item, its target, the position of its first + * row in that target's listing (row ids are `listing:position`) and the end of its other merged + * targets in `extras`. An item takes 12 bytes, so an order of 11,467 tools about 140 KB. + */ +interface Kept { + readonly total: number; + readonly missing: ReadonlyArray; + readonly stamp: string; + readonly cells: Int32Array; + readonly extras: Int32Array; + /** What the entry holds, its key included, as counted against {@link keptBytes}. */ + readonly bytes: number; +} +/** A search whose order is larger than {@link keptOrderBytes}: its pages rank as before. */ +interface Oversized { + readonly bytes: number; +} +/** Most searches kept at once. */ +const keptOrders = 32; +/** Most bytes all kept searches hold together; the least recently used go first. */ +const keptBytes = 6_000_000; +/** Most bytes one kept search holds: about 160,000 tools. A larger one is not kept. */ +const keptOrderBytes = 2_000_000; +/** + * How long a kept search goes unread before a new one may evict it. While every kept search is + * being paged, a new one is not kept: its pages rank as before, so more concurrent walks than + * {@link keptOrders} cost what they did before orders were kept, instead of evicting each other. + */ +const keptIdleMillis = 30_000; +/** Bytes an entry takes beyond its arrays: its key, stamp and object headers. */ +const entryBytes = (key: string, stamp: string, missing: number) => + 256 + 2 * (key.length + stamp.length) + 8 * missing; + +/** The cells and extras of an order as the ranking statement writes it, or nothing if malformed. */ +const parseOrder = (order: string, total: number) => { + const cells = new Int32Array(total * 3); + const extras: Array = []; + let item = 0; + for (const encoded of order === "" ? [] : order.split(",")) { + const parts = encoded.split("."); + if (item >= total || parts.length < 2) return undefined; + for (const [index, part] of parts.entries()) { + const value = Number(part); + if (!Number.isInteger(value) || value < 0 || value > 0x7fffffff) return undefined; + if (index < 2) cells[item * 3 + index] = value; + else extras.push(value); + } + cells[item * 3 + 2] = extras.length; + item += 1; + } + return item === total ? { cells, extras: Int32Array.from(extras) } : undefined; +}; /** * Rows per insert statement: nine parameters each stays well inside Postgres's bound. A batch @@ -108,6 +191,31 @@ export const supersededRetentionMillis = 60 * 60_000; const BM25 = { k1: 1.2, b: 0.75 } as const; export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { + /** Kept orders by search, least recently used first; see {@link Kept}. */ + const orders = new Map(); + let ordersBytes = 0; + const forget = (key: string) => { + const kept = orders.get(key); + if (kept === undefined) return; + orders.delete(key); + ordersBytes -= kept.entry.bytes; + }; + /** Keep an entry as the most recently used, evicting the least recently used past the bounds. */ + const keep = (key: string, entry: Kept | Oversized, used: number) => { + forget(key); + orders.set(key, { entry, used }); + ordersBytes += entry.bytes; + for (const old of orders.keys()) { + if (orders.size <= keptOrders && ordersBytes <= keptBytes) break; + forget(old); + } + }; + /** Whether a new order fits without evicting a search read in the last `keptIdleMillis`. */ + const room = (now: number) => { + if (orders.size < keptOrders && ordersBytes + keptOrderBytes <= keptBytes) return true; + const [oldest] = orders.values(); + return oldest !== undefined && now - oldest.used >= keptIdleMillis; + }; const failed = (effect: Effect.Effect) => effect.pipe(Effect.mapError(() => new StorageError())); /** A tool is in a scope when it is read-only under a read-only scope or the scope names it. */ @@ -368,9 +476,12 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { * their rows matches, the item is at their first permitted target, lists every other * permitted target with that tool, and ranks by the best matching row. * - * Ranking and reading the page's tools, definitions and routers are one statement, so they - * read one snapshot: a page never holds a row the scopes did not rank, even while another - * evaluation replaces the listing's rows. `missing` names targets whose listing has no rows. + * The first page ranks and reads the page's tools, definitions and routers in one statement, + * so they read one snapshot: a page never holds a row the scopes did not rank, even while + * another evaluation replaces the listing's rows. A later page reads its rows by the order the + * search kept (see `keptOrders`) and checks, in the same statement, that every listing still + * has the digest it was ranked with; a listing written since is ranked again. + * `missing` names targets whose listing has no rows. */ page: (input: { readonly targets: ReadonlyArray; @@ -408,12 +519,12 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { + ${BM25.b}::float8 * hits.length / stats.avgdl)))::float8 as score from hits join df using (term) cross join stats group by hits.ordinal, hits.id, hits.name, hits.signature)`; - const [page] = - yield* sql`with t(ordinal, listing, app, target, labels, length) as (values ${values}), + const listed = sql`t(ordinal, listing, app, target, labels, length) as (values ${values}), present as ( - select t.ordinal from t + select t.ordinal, l.digest from t join executor_tool_search_listings l - on l.listing = t.listing and l.app = t.app and l.target = t.target), + on l.listing = t.listing and l.app = t.app and l.target = t.target)`; + const merged = sql`${listed}, scope as materialized ( select t.ordinal, t.app, s.id, s.name, s.signature, string_to_array(s.terms, ' ') as own, string_to_array(t.labels, ' ') as labels, s.length + t.length as length @@ -434,7 +545,33 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { join scope on scope.app = best.app and scope.name = best.name and scope.signature = best.signature - group by best.app, best.name, best.signature, best.score), + group by best.app, best.name, best.signature, best.score)`; + const missing = sql`coalesce((select json_agg(t.ordinal order by t.ordinal) from t + where not exists (select 1 from present where present.ordinal = t.ordinal)), + '[]'::json)`; + const stamp = sql`coalesce((select json_agg(json_build_array(ordinal, digest) + order by ordinal) from present), '[]'::json)::text`; + /** A page's tool's router and the definitions its tools name, from `shown`. */ + const router = sql`(select router from json_array_elements(shown.routers) as r(router) + where router->>'path' = shown.tool->>'router' limit 1)`; + const definitions = sql`coalesce((select json_agg(json_build_array(d.listing, d.position, d.definition)) + from executor_tool_search_definitions d + where d.id in ( + select shown.listing || ':' || p.position + from shown, json_array_elements_text(shown.definitions) as p(position))), + '[]'::json)`; + const pageOf = (row: typeof Ranked.Type | undefined): RankedPage => ({ + total: row?.total ?? 0, + missing: row?.missing ?? [], + items: row?.items ?? [], + definitions: row?.definitions ?? [], + }); + /** + * Rank and read one page in one statement; with `order`, also the stamp and the order of + * every item ranked, for {@link Kept}. + */ + const statement = (order: boolean) => + sql`with ${merged}, page as ( select id, target, ordinals, row_number() over (order by score desc, target, name collate "C") as rank @@ -447,25 +584,132 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { from page join executor_tool_search s on s.id = page.id join executor_tool_search_listings l on l.listing = s.listing) - select (select count(*) from merged)::int as total, - coalesce((select json_agg(t.ordinal order by t.ordinal) from t - where not exists (select 1 from present where present.ordinal = t.ordinal)), - '[]'::json) as missing, + select (select count(*) from merged)::int as total, ${missing} as missing, coalesce((select json_agg(json_build_object( 'target', shown.target, 'ordinals', to_json(shown.ordinals), - 'listing', shown.listing, 'tool', shown.tool, - 'router', (select router from json_array_elements(shown.routers) as r(router) - where router->>'path' = shown.tool->>'router' limit 1)) + 'listing', shown.listing, 'tool', shown.tool, 'router', ${router}) order by shown.rank) from shown), '[]'::json) as items, - coalesce((select json_agg(json_build_array(d.listing, d.position, d.definition)) - from executor_tool_search_definitions d - where d.id in ( - select shown.listing || ':' || p.position - from shown, json_array_elements_text(shown.definitions) as p(position))), - '[]'::json) as definitions`.pipe( + ${definitions} as definitions${ + order + ? sql`, ${stamp} as stamp, + (select string_agg(concat_ws('.', merged.target, substr(merged.id, length(t.listing) + 2), + nullif(array_to_string(merged.ordinals[2:], '.'), '')), ',' + order by merged.score desc, merged.target, merged.name collate "C") + from merged join t on t.ordinal = merged.target) as ranking` + : sql`` + }`; + const ranked = () => + statement(false).pipe( Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Ranked))), + Effect.map(([row]) => row), + ); + const rankedInOrder = () => + statement(true).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(RankedInOrder))), + Effect.map(([row]) => row), ); - return page ?? { total: 0, missing: [], items: [], definitions: [] }; + /** A page of a kept order, or nothing when a listing it ranked has changed since. */ + const kept = (order: Kept) => + Effect.gen(function* () { + const from = Math.min(input.offset, order.total); + const to = Math.min(input.offset + input.limit, order.total); + const slice: Array<{ id: string; target: number; ordinals: Array }> = []; + for (let item = from; item < to; item++) { + const target = order.cells[item * 3]!; + const listing = input.targets[target]?.listing; + if (listing === undefined) return undefined; + const start = item === 0 ? 0 : order.cells[item * 3 - 1]!; + slice.push({ + id: `${listing}:${order.cells[item * 3 + 1]}`, + target, + ordinals: [target, ...order.extras.subarray(start, order.cells[item * 3 + 2])], + }); + } + const [shown] = yield* sql`with ${listed}, + shown as ( + select s.id, s.listing, s.tool, s.definitions, l.routers + from executor_tool_search s + join executor_tool_search_listings l on l.listing = s.listing + where s.id in (select value from json_array_elements_text( + ${JSON.stringify(slice.map(({ id }) => id))}::json))) + select ${stamp} as stamp, + coalesce((select json_agg(json_build_object('id', shown.id, + 'listing', shown.listing, 'tool', shown.tool, 'router', ${router})) + from shown), '[]'::json) as items, + ${definitions} as definitions`.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Shown))), + ); + if (shown === undefined || shown.stamp !== order.stamp) return undefined; + const rows = new Map(shown.items.map((item) => [item.id, item])); + const items: Array = []; + for (const { id, target, ordinals } of slice) { + const row = rows.get(id); + if (row === undefined) return undefined; + items.push({ + target, + ordinals, + listing: row.listing, + tool: row.tool, + router: row.router, + }); + } + return { + total: order.total, + missing: order.missing, + items, + definitions: shown.definitions, + } satisfies RankedPage; + }); + // The first page, which most searches end at, ranks and reads in one statement. + if (input.offset === 0) { + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return pageOf(yield* ranked()); + } + const key = JSON.stringify([input.targets, input.terms]); + const now = yield* Clock.currentTimeMillis; + const remembered = orders.get(key)?.entry; + if (remembered !== undefined && !("cells" in remembered)) { + // Too large to keep: each page ranks, as before. + keep(key, remembered, now); + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return pageOf(yield* ranked()); + } + if (remembered !== undefined) { + const page = yield* kept(remembered); + if (page !== undefined) { + keep(key, remembered, now); + yield* Effect.annotateCurrentSpan("executor.search.order", "kept"); + return page; + } + forget(key); + } + if (!room(now)) { + // Every kept search is in use: rank this page as before rather than evict one. + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return pageOf(yield* ranked()); + } + // No order kept, or a listing changed since: rank, read the page from that ranking in the + // same statement, and keep the order for the pages after it. + const row = yield* rankedInOrder(); + yield* Effect.annotateCurrentSpan("executor.search.order", "ordered"); + if (row === undefined) return pageOf(row); + const overhead = entryBytes(key, row.stamp, row.missing.length); + const parsed = + overhead + 12 * row.total > keptOrderBytes + ? undefined + : parseOrder(row.ranking ?? "", row.total); + const bytes = + parsed === undefined + ? undefined + : overhead + parsed.cells.byteLength + parsed.extras.byteLength; + keep( + key, + parsed === undefined || bytes === undefined || bytes > keptOrderBytes + ? { bytes: entryBytes(key, "", 0) } + : { total: row.total, missing: row.missing, stamp: row.stamp, ...parsed, bytes }, + now, + ); + return pageOf(row); }).pipe(failed), }; };