Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions e2e/test-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8157,6 +8157,17 @@ export const scenarios = plan({
local: na("Hosted MCP scenario"),
},
},
mcpSearchKeptOrders: {
fixtures: "actors",
file: "mcp-search-kept-orders.spec.ts",
title:
"A search being paged keeps its order beside 40 new searches at once and beside a search too large to keep",
targets: {
"self-host": scheduled,
cloud: na("This scenario reads search spans from the self-host Motel collector."),
local: na("Hosted MCP scenario"),
},
},
mcpSearchStatements: {
fixtures: "actors",
file: "mcp-search-statements.spec.ts",
Expand Down
35 changes: 34 additions & 1 deletion e2e/tests/mcp-execute-step-budget.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
* operators that walk or coerce a whole value. A value a call returns that the program already
* holds, such as a group `Map.get` returns, is not charged, so grouping many rows into few groups
* completes. Storing a value walks it once to reject cycles; that walk is charged, and a value
* holding no objects is not walked again, so attaching one shared value to every row completes.
* whose graph was walked is not walked again until something changes inside it, so attaching one
* shared value to every row completes, even one of 1,000 records, while storing a row into that
* shared value is still rejected as a cycle.
* Storing the value a property already holds skips that walk, so grouping row objects with
* `g[k] = g[k] || []` completes. Joining strings with `+` or a template is charged by the length
* of the result, so a loop that joins a megabyte string stops in seconds.
Expand Down Expand Up @@ -160,6 +162,26 @@ const rows = Array.from({ length: 20000 }, (_, i) => ({ id: i }));
for (const r of rows) { r.meta = meta }
return rows.length;`;

/**
* Attaches one shared value of 1,000 records, each an object, to each of 20,000 rows. Each store
* walked all 1,000 records, about 500 steps, and the program stopped at the budget halfway.
*/
const sharedRecordsMeta = `
const meta = { records: Array.from({ length: 1000 }, (_, i) => ({ id: 'm' + i, label: 'L' + i, w: i })) };
const rows = [];
for (let i = 0; i < 20000; i++) rows.push({ id: i, meta });
return rows.length;`;

/**
* Stores rows holding the shared value, then a row into one of its records: the walk must not stop
* at the shared value it walked before, because the record it changes is inside it.
*/
const sharedMetaCycle = `
const meta = { records: Array.from({ length: 1000 }, (_, i) => ({ id: 'm' + i })) };
const rows = [];
for (let i = 0; i < 10; i++) rows.push({ id: i, meta });
try { meta.records[5].back = rows[3]; return 'stored'; } catch (e) { return String(e) }`;

/**
* The review's proof that string `+` escaped the budget: joining a megabyte string to a number is
* one step, and reading the result copies the megabyte. Uncharged, it ran to the wall clock.
Expand Down Expand Up @@ -325,6 +347,16 @@ layer(HostedLive, { excludeTestServices: true })("Hosted MCP execute step budget
const shared = yield* execute(runner, "shared-meta", sharedMeta);
expect(shared.execution).toMatchObject({ ok: true, value: 20000 });
expect(shared.ms).toBeLessThan(20_000);
// Also when that value holds 1,000 records, each an object.
const sharedRecords = yield* execute(runner, "shared-records-meta", sharedRecordsMeta);
expect(sharedRecords.execution).toMatchObject({ ok: true, value: 20000 });
expect(sharedRecords.ms).toBeLessThan(5_000);
// Storing a row into the shared value it already walked is still a cycle.
const cycle = yield* execute(runner, "shared-meta-cycle", sharedMetaCycle);
expect(cycle.execution).toMatchObject({
ok: true,
value: expect.stringContaining("contains a circular value"),
});

// Grouping row objects by storing each group back into itself completes, in both forms.
const groupedRows = yield* execute(runner, "group-rows-or-push", groupRowsOrPush);
Expand Down Expand Up @@ -382,6 +414,7 @@ layer(HostedLive, { excludeTestServices: true })("Hosted MCP execute step budget
yield* evidence.measure("group-by-50-ms", grouped50.ms);
yield* evidence.measure("group-by-object-ms", groupedObject.ms);
yield* evidence.measure("shared-meta-ms", shared.ms);
yield* evidence.measure("shared-records-meta-ms", sharedRecords.ms);
yield* evidence.measure("concat-runaway-ms", concat.ms);
yield* evidence.measure("group-rows-or-push-ms", groupedRows.ms);
yield* evidence.measure("group-rows-reduce-ms", reducedRows.ms);
Expand Down
182 changes: 182 additions & 0 deletions e2e/tests/mcp-search-kept-orders.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
/**
* A search past its first page keeps its order for its later pages (see mcp-search-walk), in a
* cache of at most 32 searches and 6 MB. Two ways a new search could push out one still being
* paged: many new searches ranking at once each found room before their ranking and evicted the
* least recently read search after it, idle or not; and a search with a key too large to keep was
* remembered in the same cache with its key's bytes, past the 6 MB, which evicted every kept
* search. A search read in the last 30 s is now never evicted, and a search too large to keep is
* remembered apart from kept orders, or not at all when its key alone is too large.
*/
import { expect, layer } from "@effect/vitest";
import { Effect, Schedule, Schema } from "effect";
import { randomUUID } from "node:crypto";
import { Actors } from "../support/actors.ts";
import { Api, body } from "../support/api.ts";
import { appsManifest } from "../support/apps-release.ts";
import { HostedLive, withHostedCase } from "../support/case.ts";
import { App } from "../support/contracts.ts";
import { Evidence, Telemetry } from "../support/evidence.ts";
import { McpClient } from "../support/mcp-client.ts";
import { scenarios } from "../test-plan.ts";

const apps = 2;
const toolsPerApp = 60;

/** An app of `toolsPerApp` small tools, generated when it evaluates. */
const probeSource = (
index: number,
) => `import { defineApp, object, query, router, string } from "apps";
export default defineApp({}, async () => {
const tools = {};
for (let t = 0; t < ${toolsPerApp}; t++)
tools["keptprobe" + t] = query(
{ description: "Kept probe " + t + " of app ${index}", input: object({ id: string() }) },
async () => t,
);
return { tools: router(tools) };
});`;

/** Pages a search to its second page, so its order is kept, and returns the third page's request. */
const secondPage = (query: string) => `
const first = await tools.search({ query: ${JSON.stringify(query)}, limit: 10 });
const second = await tools.search(first.next);
return second.next;`;

/** Reads the page a request returned by {@link secondPage} names. */
const thirdPage = (next: unknown) => `
const third = await tools.search(${JSON.stringify(next)});
return third.items.length;`;

/**
* Pages one search to its second page, then pages a search whose 500 distinct 8,000-character
* words make a key of about 4 million characters, then reads the first search's third page.
*/
const huge = `
const first = await tools.search({ query: 'kept probe again', limit: 10 });
const second = await tools.search(first.next);
const words = Array.from({ length: 500 }, (_, i) => String(i).padStart(8000, 'w'));
const big = await tools.search({ query: 'kept probe ' + words.join(' '), limit: 10 });
await tools.search(big.next);
const third = await tools.search(second.next);
return third.items.length;`;

const Token = Schema.Struct({ key: Schema.RedactedFromValue(Schema.String), id: Schema.String });
const Ran = Schema.Struct({
structuredContent: Schema.Struct({
status: Schema.Literal("completed"),
execution: Schema.Struct({ ok: Schema.Literal(true), value: Schema.Unknown }),
}),
});

layer(HostedLive, { excludeTestServices: true })("MCP search kept orders", (it) => {
it.effect(scenarios.mcpSearchKeptOrders.title, (context) =>
withHostedCase(
context,
Effect.gen(function* () {
const api = yield* Api,
actors = yield* Actors,
evidence = yield* Evidence,
telemetry = yield* Telemetry,
mcp = yield* McpClient;
const organization = actors.organization.id;
const prefix = `/api/organizations/${organization}`;
const run = randomUUID().slice(0, 8);
yield* Effect.forEach(
Array.from({ length: apps }, (_, index) => index),
(index) =>
Effect.gen(function* () {
const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, {
name: `Kept orders ${index} ${run}`,
files: [{ path: "index.ts", content: probeSource(index) }, appsManifest],
});
expect(deployed.status, JSON.stringify(deployed.body)).toBe(200);
const app = yield* body(App, deployed);
yield* Effect.addFinalizer(() =>
api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe(
Effect.map((response) => expect(response.status).toBe(200)),
Effect.orDie,
),
);
}),
{ concurrency: 2, discard: true },
);
const created = yield* api.request(actors.owner, "POST", "/api/auth/api-key/create", {
name: "MCP search kept orders",
});
expect(created.status).toBe(200);
const token = yield* body(Token, created);
yield* Effect.addFinalizer(() =>
api
.request(actors.owner, "POST", "/api/auth/api-key/delete", { keyId: token.id })
.pipe(Effect.orDie),
);
const client = yield* mcp.connect(token.key, "pat", { organization });

/** Run a program and return its value. */
const execute = (operation: string, code: string) =>
client
.use(operation, (client, signal) =>
client.callTool({ name: "execute", arguments: { code } }, undefined, { signal }),
)
.pipe(
Effect.flatMap(Schema.decodeUnknownEffect(Ran)),
Effect.map((ran) => ran.structuredContent.execution.value),
);
/**
* Run a program alone and read its value and how each of its searches got its page, in
* order, from the trace of the last MCP request.
*/
const orders = (operation: string, code: string, searches: number) =>
Effect.gen(function* () {
const value = yield* execute(operation, code);
const request = (yield* evidence.requests)
.filter((request) => request.path === "/mcp")
.at(-1);
if (request === undefined)
return yield* Effect.die(new Error("MCP request evidence missing"));
const traced = yield* telemetry.query(request.traceId).pipe(
Effect.flatMap((result) => {
const spans = result.data
.map(({ span }) => span)
.filter((span) => span.operationName === "sdk.tools.search")
.sort((a, b) => a.startTime.localeCompare(b.startTime))
.map((span) => span.tags["executor.search.order"] ?? "");
return spans.length === searches
? Effect.succeed(spans)
: Effect.fail(new Error("Every search span must reach Motel"));
}),
Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }),
Effect.timeout("25 seconds"),
);
return { value, orders: traced };
});

// A search with a key too large to keep ranks each page and evicts nothing.
const large = yield* orders("Page a search beside one too large to keep", huge, 5);
yield* evidence.json("huge-orders.json", large.orders);
expect(large.value).toBe(10);
expect(large.orders).toEqual(["ranked", "ordered", "ranked", "ranked", "kept"]);

// Forty new searches keeping their orders at once leave the one being paged kept.
const paged = yield* orders(
"Page a search to its second page",
secondPage("kept probe"),
2,
);
expect(paged.orders).toEqual(["ranked", "ordered"]);
yield* Effect.forEach(
Array.from({ length: 40 }, (_, index) => index),
(index) => execute(`Page new search ${index}`, secondPage(`kept probe z${index}`)),
{ concurrency: "unbounded", discard: true },
);
const third = yield* orders(
"Read the paged search's third page",
thirdPage(paged.value),
1,
);
expect(third.value).toBe(10);
expect(third.orders, "The search being paged keeps its order").toEqual(["kept"]);
}).pipe(Effect.provide(McpClient.layer)),
),
);
});
Loading
Loading