From f5995c3248eafd72034aad20f28e70a0765f5ea3 Mon Sep 17 00:00:00 2001 From: "Executor Owner (Grok Bot)" Date: Wed, 18 Nov 2026 17:14:45 -0800 Subject: [PATCH 1/2] Keep a search's order for its later pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paging tools.search ranked every row its targets hold again for each page (tool-search-store page()), so walking a catalog of n tools in pages of k cost n²/k on the product's one thread. On Nov 18 three walks with neither query nor namespace over 11,467 tools held a self-hosted product at full CPU for minutes. A page past the first now ranks once, keeps the order (eight searches, least recently used dropped) and reads later pages' rows by id, in a statement that also checks every ranked listing still has its digest; a listing written since is ranked again in one statement, as before. The first page, which most searches end at, is unchanged. Paging, offsets and totals keep their contract. mcp-search-walk walks 8,000 tools twice: SQL across 55 pages falls from 1,848 ms to 153 ms, and a later page from ~34 ms to ~3 ms. --- e2e/test-plan.ts | 11 + e2e/tests/mcp-search-walk.spec.ts | 217 ++++++++++++++++++ .../src/implementation/tool-search-store.ts | 166 ++++++++++++-- 3 files changed, 372 insertions(+), 22 deletions(-) create mode 100644 e2e/tests/mcp-search-walk.spec.ts diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 81cb0d2203..38751a86d0 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -8135,6 +8135,17 @@ export const scenarios = plan({ local: na("Local has no hosted organization policies."), }, }, + mcpSearchWalk: { + fixtures: "actors", + file: "mcp-search-walk.spec.ts", + title: + "An MCP search without a query pages through 8,000 tools ranking them once, each later page reading only its own rows", + targets: { + "self-host": scheduled, + cloud: na("This scenario reads SQL spans from the self-host Motel collector."), + local: na("Hosted MCP scenario"), + }, + }, mcpSearchStatements: { fixtures: "actors", file: "mcp-search-statements.spec.ts", diff --git a/e2e/tests/mcp-search-walk.spec.ts b/e2e/tests/mcp-search-walk.spec.ts new file mode 100644 index 0000000000..5486240b7b --- /dev/null +++ b/e2e/tests/mcp-search-walk.spec.ts @@ -0,0 +1,217 @@ +/** + * Paging `tools.search` through the whole catalog, with neither query nor namespace, ranked every + * row the catalog holds again for each page, so a walk of n tools cost n squared over the page + * size on the product's one thread. On Nov 18 2026 three such walks over 11,467 tools held a + * self-hosted product at full CPU for minutes. A search now ranks once and keeps the order for its + * later pages: past the second, a page reads only its own rows, however large the catalog. + */ +import { expect, layer } from "@effect/vitest"; +import { Console, Effect, Schedule, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { Evidence, Telemetry } from "../support/evidence.ts"; +import { McpClient } from "../support/mcp-client.ts"; +import { scenarios } from "../test-plan.ts"; + +const apps = 16; +const toolsPerApp = 500; +const pageItems = 150; + +/** An app of `toolsPerApp` small tools, generated when it evaluates. */ +const walkSource = ( + index: number, +) => `import { defineApp, object, query, router, string } from "apps"; +export default defineApp({}, async () => { + const tools = {}; + for (let t = 0; t < ${toolsPerApp}; t++) + tools["walkprobe" + t] = query( + { description: "Walk probe " + t + " of app ${index}", input: object({ id: string() }) }, + async () => t, + ); + return { tools: router(tools) }; +});`; + +/** Every page of a search with neither query nor namespace, counting the probes it lists. */ +const walk = `let next = { limit: ${pageItems} }; +let seen = 0; +let pages = 0; +while (next !== null) { + const page = await tools.search(next); + pages += 1; + seen += page.items.filter((item) => item.path.includes("walkprobe")).length; + next = page.next; +} +return { seen, pages };`; + +const Token = Schema.Struct({ key: Schema.RedactedFromValue(Schema.String), id: Schema.String }); +const Walked = Schema.Struct({ + structuredContent: Schema.Struct({ + status: Schema.Literal("completed"), + execution: Schema.Struct({ + ok: Schema.Literal(true), + value: Schema.Struct({ seen: Schema.Number, pages: Schema.Number }), + }), + }), +}); + +layer(HostedLive, { excludeTestServices: true })("MCP search walk", (it) => { + it.effect(scenarios.mcpSearchWalk.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence, + telemetry = yield* Telemetry, + mcp = yield* McpClient; + const organization = actors.organization.id; + const prefix = `/api/organizations/${organization}`; + const run = randomUUID().slice(0, 8); + yield* Effect.forEach( + Array.from({ length: apps }, (_, index) => index), + (index) => + Effect.gen(function* () { + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: `Search walk ${index} ${run}`, + files: [{ path: "index.ts", content: walkSource(index) }, appsManifest], + }); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe( + Effect.map((response) => expect(response.status).toBe(200)), + Effect.orDie, + ), + ); + }), + { concurrency: 4, discard: true }, + ); + const created = yield* api.request(actors.owner, "POST", "/api/auth/api-key/create", { + name: "MCP search walk", + }); + expect(created.status).toBe(200); + const token = yield* body(Token, created); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "POST", "/api/auth/api-key/delete", { keyId: token.id }) + .pipe(Effect.orDie), + ); + const client = yield* mcp.connect(token.key, "pat", { organization }); + + /** Walk the whole catalog in one execute, and read each page's search span and SQL time. */ + const walkOnce = (operation: string) => + Effect.gen(function* () { + const started = Date.now(); + const called = yield* client.use(operation, (client, signal) => + client.callTool({ name: "execute", arguments: { code: walk } }, undefined, { + signal, + }), + ); + const elapsed = Date.now() - started; + const walked = yield* Schema.decodeUnknownEffect(Walked)(called); + const request = (yield* evidence.requests) + .filter((request) => request.path === "/mcp") + .at(-1); + if (request === undefined) + return yield* Effect.die(new Error("MCP request evidence missing")); + const { pages } = walked.structuredContent.execution.value; + const searches = yield* telemetry.query(request.traceId).pipe( + Effect.flatMap((result) => { + if (result.meta?.truncated === true) + return Effect.die(new Error("The walk's trace must reach Motel whole")); + const byId = new Map(result.data.map(({ span }) => [span.spanId, span])); + const searchOf = (spanId: string | null) => { + const visited = new Set(); + for (let parent = spanId; parent !== null && !visited.has(parent);) { + visited.add(parent); + const span = byId.get(parent); + if (span === undefined) return undefined; + if (span.operationName === "sdk.tools.search") return span.spanId; + parent = span.parentSpanId; + } + return undefined; + }; + const sqlMs = new Map(); + for (const { span } of result.data) { + if (span.operationName !== "sql.execute") continue; + const search = searchOf(span.parentSpanId); + if (search !== undefined) + sqlMs.set(search, (sqlMs.get(search) ?? 0) + span.durationMs); + } + const searches = result.data + .filter(({ span }) => span.operationName === "sdk.tools.search") + .map(({ span }) => ({ + startTime: span.startTime, + order: span.tags["executor.search.order"] ?? "", + durationMs: span.durationMs, + sqlMs: sqlMs.get(span.spanId) ?? 0, + })) + .sort((a, b) => a.startTime.localeCompare(b.startTime)); + return searches.length === pages + ? Effect.succeed(searches) + : Effect.fail(new Error("Every page's search span must reach Motel")); + }), + Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), + Effect.timeout("25 seconds"), + ); + return { ...walked.structuredContent.execution.value, elapsed, searches }; + }); + + // The first walk ranks the catalog for its first page, and again for its second, keeping + // that order; every later page reads its rows by it. The second walk, over the same + // listings, ranks its first page and reads every later one by the order the first kept. + const firstWalk = yield* walkOnce("Walk the whole catalog for the first time"); + const secondWalk = yield* walkOnce("Walk the whole catalog again"); + const summarize = (walked: typeof firstWalk) => { + const [first, second, ...rest] = walked.searches; + return { + pages: walked.pages, + elapsed: walked.elapsed, + firstPageSqlMs: first?.sqlMs ?? 0, + secondPageSqlMs: second?.sqlMs ?? 0, + laterPagesSqlMs: rest.reduce((sum, search) => sum + search.sqlMs, 0), + medianLaterPageSqlMs: + rest.map((search) => search.sqlMs).sort((x, y) => x - y)[ + Math.floor(rest.length / 2) + ] ?? 0, + slowestLaterPageSqlMs: Math.max(0, ...rest.map((search) => search.sqlMs)), + totalSqlMs: walked.searches.reduce((sum, search) => sum + search.sqlMs, 0), + orders: walked.searches.map((search) => search.order), + }; + }; + const summary = { + tools: apps * toolsPerApp, + first: summarize(firstWalk), + again: summarize(secondWalk), + }; + yield* Console.log(JSON.stringify({ label: "Whole-catalog walk", ...summary })); + yield* evidence.json("mcp-search-walk.json", { + summary, + first: firstWalk.searches, + again: secondWalk.searches, + }); + yield* evidence.measure("Whole-catalog walk", firstWalk.elapsed); + yield* evidence.measure("Whole-catalog walk again", secondWalk.elapsed); + + for (const walked of [firstWalk, secondWalk]) { + expect(walked.seen, "The walk lists every tool once").toBe(apps * toolsPerApp); + expect(walked.pages).toBeGreaterThan(20); + } + const kept = (count: number) => Array.from({ length: count }, () => "kept"); + expect(summary.first.orders).toEqual(["ranked", "ordered", ...kept(firstWalk.pages - 2)]); + expect(summary.again.orders).toEqual(["ranked", ...kept(secondWalk.pages - 1)]); + // A kept page reads only its own rows: the typical one costs well under the ranking the + // first page does, where every page used to cost a ranking. + for (const walked of [summary.first, summary.again]) + expect( + walked.medianLaterPageSqlMs, + `The median kept page read ${walked.medianLaterPageSqlMs} ms of SQL; the first page, which ranks the catalog, ${walked.firstPageSqlMs} ms`, + ).toBeLessThan(walked.firstPageSqlMs / 2); + }).pipe(Effect.provide(McpClient.layer)), + ), + ); +}); diff --git a/packages/sdk/src/implementation/tool-search-store.ts b/packages/sdk/src/implementation/tool-search-store.ts index a720d29171..62d1cbe0b4 100644 --- a/packages/sdk/src/implementation/tool-search-store.ts +++ b/packages/sdk/src/implementation/tool-search-store.ts @@ -87,6 +87,38 @@ const Ranked = Schema.Struct({ definitions: Schema.Array(Schema.Tuple([Schema.String, Schema.Int, Json])), }); export type RankedPage = typeof Ranked.Type; +/** + * Every item a search ranks, in rank order, as a page names it: its first row, the target it is + * at and every target it merges. `stamp` names the digest of each target's listing it ranked. + */ +const Ordered = Schema.Struct({ + total: Schema.Int, + missing: Schema.Array(Schema.Int), + stamp: Schema.String, + ranked: Schema.Array(Schema.Tuple([Schema.String, Schema.Int, Schema.Array(Schema.Int)])), +}); +type Ordered = typeof Ordered.Type; +/** The rows of a page of a kept order, read with the stamp of the listings they belong to. */ +const Shown = Schema.Struct({ + stamp: Schema.String, + items: Schema.Array( + Schema.Struct({ + id: Schema.String, + listing: Schema.String, + tool: StoredTool, + router: Schema.NullOr(ToolRouter), + }), + ), + definitions: Ranked.fields.definitions, +}); + +/** + * Searches whose order is kept for their later pages. A page past the first reads its rows by the + * kept order instead of ranking every row the targets hold again, so paging through a catalog of + * n tools costs one ranking and n rows, not one ranking per page. The order is used only while + * every listing it ranked still has the digest it had, so a page never mixes two rankings. + */ +const keptOrders = 8; /** * Rows per insert statement: nine parameters each stays well inside Postgres's bound. A batch @@ -108,6 +140,8 @@ export const supersededRetentionMillis = 60 * 60_000; const BM25 = { k1: 1.2, b: 0.75 } as const; export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { + /** Kept orders by search, least recently used first; see `keptOrders`. */ + const orders = new Map(); const failed = (effect: Effect.Effect) => effect.pipe(Effect.mapError(() => new StorageError())); /** A tool is in a scope when it is read-only under a read-only scope or the scope names it. */ @@ -368,9 +402,12 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { * their rows matches, the item is at their first permitted target, lists every other * permitted target with that tool, and ranks by the best matching row. * - * Ranking and reading the page's tools, definitions and routers are one statement, so they - * read one snapshot: a page never holds a row the scopes did not rank, even while another - * evaluation replaces the listing's rows. `missing` names targets whose listing has no rows. + * The first page ranks and reads the page's tools, definitions and routers in one statement, + * so they read one snapshot: a page never holds a row the scopes did not rank, even while + * another evaluation replaces the listing's rows. A later page reads its rows by the order the + * search kept (see `keptOrders`) and checks, in the same statement, that every listing still + * has the digest it was ranked with; a listing written since is ranked again. + * `missing` names targets whose listing has no rows. */ page: (input: { readonly targets: ReadonlyArray; @@ -408,12 +445,12 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { + ${BM25.b}::float8 * hits.length / stats.avgdl)))::float8 as score from hits join df using (term) cross join stats group by hits.ordinal, hits.id, hits.name, hits.signature)`; - const [page] = - yield* sql`with t(ordinal, listing, app, target, labels, length) as (values ${values}), + const listed = sql`t(ordinal, listing, app, target, labels, length) as (values ${values}), present as ( - select t.ordinal from t + select t.ordinal, l.digest from t join executor_tool_search_listings l - on l.listing = t.listing and l.app = t.app and l.target = t.target), + on l.listing = t.listing and l.app = t.app and l.target = t.target)`; + const merged = sql`${listed}, scope as materialized ( select t.ordinal, t.app, s.id, s.name, s.signature, string_to_array(s.terms, ' ') as own, string_to_array(t.labels, ' ') as labels, s.length + t.length as length @@ -434,7 +471,24 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { join scope on scope.app = best.app and scope.name = best.name and scope.signature = best.signature - group by best.app, best.name, best.signature, best.score), + group by best.app, best.name, best.signature, best.score)`; + const missing = sql`coalesce((select json_agg(t.ordinal order by t.ordinal) from t + where not exists (select 1 from present where present.ordinal = t.ordinal)), + '[]'::json)`; + const stamp = sql`coalesce((select json_agg(json_build_array(ordinal, digest) + order by ordinal) from present), '[]'::json)::text`; + /** A page's tool's router and the definitions its tools name, from `shown`. */ + const router = sql`(select router from json_array_elements(shown.routers) as r(router) + where router->>'path' = shown.tool->>'router' limit 1)`; + const definitions = sql`coalesce((select json_agg(json_build_array(d.listing, d.position, d.definition)) + from executor_tool_search_definitions d + where d.id in ( + select shown.listing || ':' || p.position + from shown, json_array_elements_text(shown.definitions) as p(position))), + '[]'::json)`; + /** Rank and read one page in one statement. */ + const ranked = () => + sql`with ${merged}, page as ( select id, target, ordinals, row_number() over (order by score desc, target, name collate "C") as rank @@ -447,25 +501,93 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { from page join executor_tool_search s on s.id = page.id join executor_tool_search_listings l on l.listing = s.listing) - select (select count(*) from merged)::int as total, - coalesce((select json_agg(t.ordinal order by t.ordinal) from t - where not exists (select 1 from present where present.ordinal = t.ordinal)), - '[]'::json) as missing, + select (select count(*) from merged)::int as total, ${missing} as missing, coalesce((select json_agg(json_build_object( 'target', shown.target, 'ordinals', to_json(shown.ordinals), - 'listing', shown.listing, 'tool', shown.tool, - 'router', (select router from json_array_elements(shown.routers) as r(router) - where router->>'path' = shown.tool->>'router' limit 1)) + 'listing', shown.listing, 'tool', shown.tool, 'router', ${router}) order by shown.rank) from shown), '[]'::json) as items, - coalesce((select json_agg(json_build_array(d.listing, d.position, d.definition)) - from executor_tool_search_definitions d - where d.id in ( - select shown.listing || ':' || p.position - from shown, json_array_elements_text(shown.definitions) as p(position))), - '[]'::json) as definitions`.pipe( + ${definitions} as definitions`.pipe( Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Ranked))), + Effect.map(([page]) => page ?? { total: 0, missing: [], items: [], definitions: [] }), ); - return page ?? { total: 0, missing: [], items: [], definitions: [] }; + /** A page of a kept order, or nothing when a listing it ranked has changed since. */ + const kept = (order: Ordered) => + Effect.gen(function* () { + const slice = order.ranked.slice(input.offset, input.offset + input.limit); + const [shown] = yield* sql`with ${listed}, + shown as ( + select s.id, s.listing, s.tool, s.definitions, l.routers + from executor_tool_search s + join executor_tool_search_listings l on l.listing = s.listing + where s.id in (select value from json_array_elements_text( + ${JSON.stringify(slice.map(([id]) => id))}::json))) + select ${stamp} as stamp, + coalesce((select json_agg(json_build_object('id', shown.id, + 'listing', shown.listing, 'tool', shown.tool, 'router', ${router})) + from shown), '[]'::json) as items, + ${definitions} as definitions`.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Shown))), + ); + if (shown === undefined || shown.stamp !== order.stamp) return undefined; + const rows = new Map(shown.items.map((item) => [item.id, item])); + const items: Array = []; + for (const [id, target, ordinals] of slice) { + const row = rows.get(id); + if (row === undefined) return undefined; + items.push({ + target, + ordinals, + listing: row.listing, + tool: row.tool, + router: row.router, + }); + } + return { + total: order.total, + missing: order.missing, + items, + definitions: shown.definitions, + } satisfies RankedPage; + }); + // The first page, which most searches end at, ranks and reads in one statement. + if (input.offset === 0) { + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return yield* ranked(); + } + const key = JSON.stringify([input.targets, input.terms]); + const remembered = orders.get(key); + if (remembered !== undefined) { + const page = yield* kept(remembered); + if (page !== undefined) { + orders.delete(key); + orders.set(key, remembered); + yield* Effect.annotateCurrentSpan("executor.search.order", "kept"); + return page; + } + orders.delete(key); + } + const [order] = yield* sql`with ${merged} + select (select count(*) from merged)::int as total, ${missing} as missing, + ${stamp} as stamp, + coalesce((select json_agg(json_build_array(id, target, ordinals) + order by score desc, target, name collate "C") from merged), '[]'::json) as ranked`.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Ordered))), + ); + if (order !== undefined) { + const page = yield* kept(order); + if (page !== undefined) { + orders.set(key, order); + for (const old of orders.keys()) { + if (orders.size <= keptOrders) break; + orders.delete(old); + } + yield* Effect.annotateCurrentSpan("executor.search.order", "ordered"); + return page; + } + } + // A listing changed between ranking and reading: rank and read in one statement. + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return yield* ranked(); }).pipe(failed), }; }; From 08fabf93bf1b8ed72ab89f4ed7993892f4f21f54 Mon Sep 17 00:00:00 2001 From: "Executor Owner (Grok Bot)" Date: Wed, 18 Nov 2026 17:55:13 -0800 Subject: [PATCH 2/2] Keep search orders as integers under a byte bound A kept order held each item as a [row id, target, ordinals] array: about 5 MB per order at 11,467 tools on V8 and 28 MB at 60,000, eight of them with no byte cap. An item is now three Int32 cells (target, row position in its listing, end of its other merged targets), about 140 KB per order at 11,467 tools. Kept orders are bounded at 32 searches and 6 MB together, 2 MB each; a larger order is not kept and its pages rank as before. A miss reads its page from the same statement that ranks, so it no longer makes a second read. A new order evicts only a search unread for 30 s: while every kept search is in use, a new one's pages rank as before, so more concurrent walks than the cap cost what they did before, instead of evicting each other on every page. --- .../src/implementation/tool-search-store.ts | 222 ++++++++++++++---- 1 file changed, 172 insertions(+), 50 deletions(-) diff --git a/packages/sdk/src/implementation/tool-search-store.ts b/packages/sdk/src/implementation/tool-search-store.ts index 62d1cbe0b4..992907f7ae 100644 --- a/packages/sdk/src/implementation/tool-search-store.ts +++ b/packages/sdk/src/implementation/tool-search-store.ts @@ -87,17 +87,16 @@ const Ranked = Schema.Struct({ definitions: Schema.Array(Schema.Tuple([Schema.String, Schema.Int, Json])), }); export type RankedPage = typeof Ranked.Type; -/** - * Every item a search ranks, in rank order, as a page names it: its first row, the target it is - * at and every target it merges. `stamp` names the digest of each target's listing it ranked. - */ -const Ordered = Schema.Struct({ - total: Schema.Int, - missing: Schema.Array(Schema.Int), +/** A ranked page with the order of every item it ranked; see {@link Kept}. */ +const RankedInOrder = Schema.Struct({ + ...Ranked.fields, stamp: Schema.String, - ranked: Schema.Array(Schema.Tuple([Schema.String, Schema.Int, Schema.Array(Schema.Int)])), + /** + * Every item in rank order as `target.position[.ordinal...]`, comma-separated: the target it is + * at, the position of its first row in that target's listing, and every other target it merges. + */ + ranking: Schema.NullOr(Schema.String), }); -type Ordered = typeof Ordered.Type; /** The rows of a page of a kept order, read with the stamp of the listings they belong to. */ const Shown = Schema.Struct({ stamp: Schema.String, @@ -113,12 +112,64 @@ const Shown = Schema.Struct({ }); /** - * Searches whose order is kept for their later pages. A page past the first reads its rows by the - * kept order instead of ranking every row the targets hold again, so paging through a catalog of - * n tools costs one ranking and n rows, not one ranking per page. The order is used only while - * every listing it ranked still has the digest it had, so a page never mixes two rankings. + * A search's order, kept for its later pages. A page past the first reads its rows by the kept + * order instead of ranking every row the targets hold again, so paging through a catalog of n + * tools costs one ranking and n rows, not one ranking per page. The order is used only while + * every listing it ranked still has the digest it had (`stamp`), so a page never mixes two + * rankings. + * + * Items are kept as integers: `cells` holds three per item, its target, the position of its first + * row in that target's listing (row ids are `listing:position`) and the end of its other merged + * targets in `extras`. An item takes 12 bytes, so an order of 11,467 tools about 140 KB. */ -const keptOrders = 8; +interface Kept { + readonly total: number; + readonly missing: ReadonlyArray; + readonly stamp: string; + readonly cells: Int32Array; + readonly extras: Int32Array; + /** What the entry holds, its key included, as counted against {@link keptBytes}. */ + readonly bytes: number; +} +/** A search whose order is larger than {@link keptOrderBytes}: its pages rank as before. */ +interface Oversized { + readonly bytes: number; +} +/** Most searches kept at once. */ +const keptOrders = 32; +/** Most bytes all kept searches hold together; the least recently used go first. */ +const keptBytes = 6_000_000; +/** Most bytes one kept search holds: about 160,000 tools. A larger one is not kept. */ +const keptOrderBytes = 2_000_000; +/** + * How long a kept search goes unread before a new one may evict it. While every kept search is + * being paged, a new one is not kept: its pages rank as before, so more concurrent walks than + * {@link keptOrders} cost what they did before orders were kept, instead of evicting each other. + */ +const keptIdleMillis = 30_000; +/** Bytes an entry takes beyond its arrays: its key, stamp and object headers. */ +const entryBytes = (key: string, stamp: string, missing: number) => + 256 + 2 * (key.length + stamp.length) + 8 * missing; + +/** The cells and extras of an order as the ranking statement writes it, or nothing if malformed. */ +const parseOrder = (order: string, total: number) => { + const cells = new Int32Array(total * 3); + const extras: Array = []; + let item = 0; + for (const encoded of order === "" ? [] : order.split(",")) { + const parts = encoded.split("."); + if (item >= total || parts.length < 2) return undefined; + for (const [index, part] of parts.entries()) { + const value = Number(part); + if (!Number.isInteger(value) || value < 0 || value > 0x7fffffff) return undefined; + if (index < 2) cells[item * 3 + index] = value; + else extras.push(value); + } + cells[item * 3 + 2] = extras.length; + item += 1; + } + return item === total ? { cells, extras: Int32Array.from(extras) } : undefined; +}; /** * Rows per insert statement: nine parameters each stays well inside Postgres's bound. A batch @@ -140,8 +191,31 @@ export const supersededRetentionMillis = 60 * 60_000; const BM25 = { k1: 1.2, b: 0.75 } as const; export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { - /** Kept orders by search, least recently used first; see `keptOrders`. */ - const orders = new Map(); + /** Kept orders by search, least recently used first; see {@link Kept}. */ + const orders = new Map(); + let ordersBytes = 0; + const forget = (key: string) => { + const kept = orders.get(key); + if (kept === undefined) return; + orders.delete(key); + ordersBytes -= kept.entry.bytes; + }; + /** Keep an entry as the most recently used, evicting the least recently used past the bounds. */ + const keep = (key: string, entry: Kept | Oversized, used: number) => { + forget(key); + orders.set(key, { entry, used }); + ordersBytes += entry.bytes; + for (const old of orders.keys()) { + if (orders.size <= keptOrders && ordersBytes <= keptBytes) break; + forget(old); + } + }; + /** Whether a new order fits without evicting a search read in the last `keptIdleMillis`. */ + const room = (now: number) => { + if (orders.size < keptOrders && ordersBytes + keptOrderBytes <= keptBytes) return true; + const [oldest] = orders.values(); + return oldest !== undefined && now - oldest.used >= keptIdleMillis; + }; const failed = (effect: Effect.Effect) => effect.pipe(Effect.mapError(() => new StorageError())); /** A tool is in a scope when it is read-only under a read-only scope or the scope names it. */ @@ -486,8 +560,17 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { select shown.listing || ':' || p.position from shown, json_array_elements_text(shown.definitions) as p(position))), '[]'::json)`; - /** Rank and read one page in one statement. */ - const ranked = () => + const pageOf = (row: typeof Ranked.Type | undefined): RankedPage => ({ + total: row?.total ?? 0, + missing: row?.missing ?? [], + items: row?.items ?? [], + definitions: row?.definitions ?? [], + }); + /** + * Rank and read one page in one statement; with `order`, also the stamp and the order of + * every item ranked, for {@link Kept}. + */ + const statement = (order: boolean) => sql`with ${merged}, page as ( select id, target, ordinals, @@ -506,21 +589,49 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { 'target', shown.target, 'ordinals', to_json(shown.ordinals), 'listing', shown.listing, 'tool', shown.tool, 'router', ${router}) order by shown.rank) from shown), '[]'::json) as items, - ${definitions} as definitions`.pipe( + ${definitions} as definitions${ + order + ? sql`, ${stamp} as stamp, + (select string_agg(concat_ws('.', merged.target, substr(merged.id, length(t.listing) + 2), + nullif(array_to_string(merged.ordinals[2:], '.'), '')), ',' + order by merged.score desc, merged.target, merged.name collate "C") + from merged join t on t.ordinal = merged.target) as ranking` + : sql`` + }`; + const ranked = () => + statement(false).pipe( Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Ranked))), - Effect.map(([page]) => page ?? { total: 0, missing: [], items: [], definitions: [] }), + Effect.map(([row]) => row), + ); + const rankedInOrder = () => + statement(true).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(RankedInOrder))), + Effect.map(([row]) => row), ); /** A page of a kept order, or nothing when a listing it ranked has changed since. */ - const kept = (order: Ordered) => + const kept = (order: Kept) => Effect.gen(function* () { - const slice = order.ranked.slice(input.offset, input.offset + input.limit); + const from = Math.min(input.offset, order.total); + const to = Math.min(input.offset + input.limit, order.total); + const slice: Array<{ id: string; target: number; ordinals: Array }> = []; + for (let item = from; item < to; item++) { + const target = order.cells[item * 3]!; + const listing = input.targets[target]?.listing; + if (listing === undefined) return undefined; + const start = item === 0 ? 0 : order.cells[item * 3 - 1]!; + slice.push({ + id: `${listing}:${order.cells[item * 3 + 1]}`, + target, + ordinals: [target, ...order.extras.subarray(start, order.cells[item * 3 + 2])], + }); + } const [shown] = yield* sql`with ${listed}, shown as ( select s.id, s.listing, s.tool, s.definitions, l.routers from executor_tool_search s join executor_tool_search_listings l on l.listing = s.listing where s.id in (select value from json_array_elements_text( - ${JSON.stringify(slice.map(([id]) => id))}::json))) + ${JSON.stringify(slice.map(({ id }) => id))}::json))) select ${stamp} as stamp, coalesce((select json_agg(json_build_object('id', shown.id, 'listing', shown.listing, 'tool', shown.tool, 'router', ${router})) @@ -531,7 +642,7 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { if (shown === undefined || shown.stamp !== order.stamp) return undefined; const rows = new Map(shown.items.map((item) => [item.id, item])); const items: Array = []; - for (const [id, target, ordinals] of slice) { + for (const { id, target, ordinals } of slice) { const row = rows.get(id); if (row === undefined) return undefined; items.push({ @@ -552,42 +663,53 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { // The first page, which most searches end at, ranks and reads in one statement. if (input.offset === 0) { yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); - return yield* ranked(); + return pageOf(yield* ranked()); } const key = JSON.stringify([input.targets, input.terms]); - const remembered = orders.get(key); + const now = yield* Clock.currentTimeMillis; + const remembered = orders.get(key)?.entry; + if (remembered !== undefined && !("cells" in remembered)) { + // Too large to keep: each page ranks, as before. + keep(key, remembered, now); + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return pageOf(yield* ranked()); + } if (remembered !== undefined) { const page = yield* kept(remembered); if (page !== undefined) { - orders.delete(key); - orders.set(key, remembered); + keep(key, remembered, now); yield* Effect.annotateCurrentSpan("executor.search.order", "kept"); return page; } - orders.delete(key); + forget(key); } - const [order] = yield* sql`with ${merged} - select (select count(*) from merged)::int as total, ${missing} as missing, - ${stamp} as stamp, - coalesce((select json_agg(json_build_array(id, target, ordinals) - order by score desc, target, name collate "C") from merged), '[]'::json) as ranked`.pipe( - Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Ordered))), - ); - if (order !== undefined) { - const page = yield* kept(order); - if (page !== undefined) { - orders.set(key, order); - for (const old of orders.keys()) { - if (orders.size <= keptOrders) break; - orders.delete(old); - } - yield* Effect.annotateCurrentSpan("executor.search.order", "ordered"); - return page; - } + if (!room(now)) { + // Every kept search is in use: rank this page as before rather than evict one. + yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); + return pageOf(yield* ranked()); } - // A listing changed between ranking and reading: rank and read in one statement. - yield* Effect.annotateCurrentSpan("executor.search.order", "ranked"); - return yield* ranked(); + // No order kept, or a listing changed since: rank, read the page from that ranking in the + // same statement, and keep the order for the pages after it. + const row = yield* rankedInOrder(); + yield* Effect.annotateCurrentSpan("executor.search.order", "ordered"); + if (row === undefined) return pageOf(row); + const overhead = entryBytes(key, row.stamp, row.missing.length); + const parsed = + overhead + 12 * row.total > keptOrderBytes + ? undefined + : parseOrder(row.ranking ?? "", row.total); + const bytes = + parsed === undefined + ? undefined + : overhead + parsed.cells.byteLength + parsed.extras.byteLength; + keep( + key, + parsed === undefined || bytes === undefined || bytes > keptOrderBytes + ? { bytes: entryBytes(key, "", 0) } + : { total: row.total, missing: row.missing, stamp: row.stamp, ...parsed, bytes }, + now, + ); + return pageOf(row); }).pipe(failed), }; };