From 6fd6fad0b6bb3accee212c57c386cd3db49093eb Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 00:51:01 +0900 Subject: [PATCH 01/12] fix(openai-native): send body prompt_cache_key for cache affinity What: the OpenAI-native handler now sends a task-scoped `prompt_cache_key` in the Responses request body on both body builders (streaming and completePrompt), and no longer sends the `session_id` request header. The key is `metadata.taskId`, falling back to the handler session id, clamped to OpenAI's 64-character limit, and omitted for models whose catalog entry has `supportsPromptCache: false`. Why: OpenAI keys prompt-cache routing off the body `prompt_cache_key` field. The `session_id` header we were sending is not part of the prompt-caching API and is ignored on the public Responses API, so every OpenAI-native task was leaving cache hit-rate on the table. The header also carries an underscore in its name, which hardened gateways (Envoy/nginx `underscores_in_headers`) reject outright, so it was a latent 400 for anyone behind such a proxy with a custom base URL. `prompt_cache_retention: "24h"` is untouched: it is the supported lifetime control on the pre-5.6 models that set it in the catalog, and the existing retention specs pass unmodified. Impact: cache affinity now follows the documented body field, stable across turns of one task and distinct across tasks. No usage accounting, cost, or other request field changes. openai-codex.ts and xai.ts are untouched: codex already sends a body key, and xai never had the header. --- .../__tests__/openai-native-usage.spec.ts | 64 +++++++++++++++ .../providers/__tests__/openai-native.spec.ts | 78 +++++++++++++++++++ src/api/providers/openai-native.ts | 64 ++++++++++++--- 3 files changed, 197 insertions(+), 9 deletions(-) diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 8922c2cf9c..2b05ab5b2b 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -440,6 +440,70 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { }) }) + describe("OpenAiNativeHandler - prompt cache key", () => { + let handler: OpenAiNativeHandler + + beforeEach(() => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + }) + }) + + const buildRequestBody = (modelId: string, taskId?: string) => { + handler["options"].apiModelId = modelId + const model = handler.getModel() + const metadata = taskId === undefined ? undefined : { taskId } + return handler["buildRequestBody"](model, [], "", model.verbosity, undefined, metadata) + } + + it("should send the task id as the body prompt_cache_key", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + }) + + it("should keep the key stable across turns of the same task", () => { + const firstTurn = buildRequestBody("gpt-5.1", "task-stable") + const secondTurn = buildRequestBody("gpt-5.1", "task-stable") + expect(firstTurn.prompt_cache_key).toBe("task-stable") + expect(secondTurn.prompt_cache_key).toBe(firstTurn.prompt_cache_key) + }) + + it("should give distinct keys to distinct tasks", () => { + const first = buildRequestBody("gpt-5.1", "task-one") + const second = buildRequestBody("gpt-5.1", "task-two") + expect(first.prompt_cache_key).toBe("task-one") + expect(second.prompt_cache_key).toBe("task-two") + expect(first.prompt_cache_key).not.toBe(second.prompt_cache_key) + }) + + it("should fall back to the handler session id when no task id is supplied", () => { + const body = buildRequestBody("gpt-5.1") + expect(body.prompt_cache_key).toBe(handler["sessionId"]) + expect(body.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the key when the model does not support prompt caching", () => { + const modelId = "codex-mini-latest" + expect(openAiNativeModels[modelId as keyof typeof openAiNativeModels].supportsPromptCache).toBe(false) + + const body = buildRequestBody(modelId, "task-abc") + expect(body.prompt_cache_key).toBeUndefined() + }) + + it("should clamp an over-length task id to OpenAI's 64-character limit", () => { + const longTaskId = "t".repeat(80) + const body = buildRequestBody("gpt-5.1", longTaskId) + expect(body.prompt_cache_key).toHaveLength(64) + expect(body.prompt_cache_key).toBe(longTaskId.slice(0, 64)) + }) + + it("should keep prompt_cache_retention alongside the key for 24h models", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + expect(body.prompt_cache_retention).toBe("24h") + }) + }) + describe("cost calculation", () => { it.each([ { tier: OpenAiServiceTier.Default, expectedCost: 4.525 }, diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 8a086b48db..b7485bb65b 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -534,6 +534,26 @@ describe("OpenAiNativeHandler", () => { expect(result).toBe("") }) + + it("should scope the prompt_cache_key to the handler session", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request.prompt_cache_key).toBe(handler["sessionId"]) + expect(request.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the prompt_cache_key when the model does not support prompt caching", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + handler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "codex-mini-latest" }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request).not.toHaveProperty("prompt_cache_key") + }) }) describe("getModel", () => { @@ -1446,6 +1466,64 @@ describe("OpenAiNativeHandler", () => { expect(callBody.previous_response_id).toBeUndefined() }) + it("sends a task-scoped prompt_cache_key in the body and no session_id header", async () => { + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + // Force the hand-built SSE path so the wire body and headers are inspectable. + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["task-cache-one", "task-cache-one", "task-cache-two"]) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(3) + expect(bodies[0].prompt_cache_key).toBe("task-cache-one") + // Same task, second turn: the key must not drift, or the growing prefix loses affinity. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + // A different task must not share the bucket. + expect(bodies[2].prompt_cache_key).toBe("task-cache-two") + expect(bodies[2].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + + // The retention control is untouched by this change. + expect(bodies[0].prompt_cache_retention).toBe("24h") + + // The affinity header is gone: OpenAI ignores it for caching and hardened + // gateways reject underscore header names outright. + for (const [, init] of mockFetch.mock.calls) { + expect(init.headers).not.toHaveProperty("session_id") + } + }) + + it("sends the task-scoped prompt_cache_key through the SDK path", async () => { + mockResponsesCreate.mockResolvedValue(asyncStreamFrom([])) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["sdk-task-one", "sdk-task-one", "sdk-task-two"]) { + await collectStream(cacheHandler.createMessage(systemPrompt, messages, { taskId })) + } + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies[0].prompt_cache_key).toBe("sdk-task-one") + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + expect(bodies[2].prompt_cache_key).toBe("sdk-task-two") + expect(bodies[0].prompt_cache_retention).toBe("24h") + expect(mockResponsesCreate.mock.calls[0][1].headers).not.toHaveProperty("session_id") + }) + it("should provide helpful error messages for different error codes", async () => { const testCases = [ { status: 400, expectedMessage: "Invalid request to Responses API" }, diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index fbbecf5760..91f514ffd5 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -34,6 +34,12 @@ import { sanitizeOpenAiCallId } from "../../utils/tool-id" export type OpenAiNativeModel = ReturnType +/** + * OpenAI caps `prompt_cache_key` at 64 characters; longer values are rejected with a 400. + * Task ids are uuidv7 today (36 chars), so this is a guard rather than a routine truncation. + */ +const PROMPT_CACHE_KEY_MAX_LENGTH = 64 + export class OpenAiNativeHandler extends BaseProvider implements SingleCompletionHandler { protected options: ApiHandlerOptions private client: OpenAI @@ -96,14 +102,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio this.options.enableResponsesReasoningSummary = true } const apiKey = this.options.openAiNativeApiKey ?? NOT_PROVIDED - // Include originator, session_id, and User-Agent headers for API tracking and debugging + // Include originator and User-Agent headers for API tracking and debugging. + // Cache affinity rides on the body `prompt_cache_key`, not on a header: OpenAI documents + // the `session_id` header as ignored on the public Responses API, and underscore header + // names are rejected outright by hardened gateways (Envoy/nginx `underscores_in_headers`). const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` this.client = new OpenAI({ baseURL: this.options.openAiNativeBaseUrl || undefined, apiKey, defaultHeaders: { originator: "zoo-code", - session_id: this.sessionId, "User-Agent": userAgent, }, timeout: this.timeoutMs, @@ -330,6 +338,11 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio include?: string[] /** Prompt cache retention policy: "in_memory" (default) or "24h" for extended caching */ prompt_cache_retention?: "in_memory" | "24h" + /** + * Cache routing affinity key. OpenAI documents this body field as the routing lever; + * the `session_id` request header is not part of the prompt-caching API. + */ + prompt_cache_key?: string tools?: Array<{ type: "function" name: string @@ -347,6 +360,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Decide whether to enable extended prompt cache retention for this request const promptCacheRetention = this.getPromptCacheRetention(model) + // Cache routing affinity: a stable, task-scoped key lets OpenAI route every turn of the + // same task to the machine holding its cached prefix. + const promptCacheKey = this.getPromptCacheKey(model, metadata) + const body: ResponsesRequestBody = { model: model.id, input: formattedInput, @@ -379,6 +396,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Enable extended prompt cache retention for models that support it. // This uses the OpenAI Responses API `prompt_cache_retention` parameter. ...(promptCacheRetention ? { prompt_cache_retention: promptCacheRetention } : {}), + // Cache routing affinity key (body field, not the inert `session_id` header). + ...(promptCacheKey ? { prompt_cache_key: promptCacheKey } : {}), tools: (metadata?.tools ?? []) .filter((tool) => tool.type === "function") .map((tool) => { @@ -418,12 +437,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` const requestHeaders: Record = { originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, } @@ -557,7 +574,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio private async *makeResponsesApiRequest( requestBody: any, model: OpenAiNativeModel, - metadata?: ApiHandlerCreateMessageMetadata, + _metadata?: ApiHandlerCreateMessageMetadata, _systemPrompt?: string, _messages?: Anthropic.Messages.MessageParam[], ): ApiStream { @@ -568,8 +585,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` try { @@ -579,7 +595,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, }, body: JSON.stringify(requestBody), @@ -1420,6 +1435,30 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio return undefined } + /** + * Returns the prompt cache routing key for this request, if the model caches prompts. + * + * OpenAI keys cache affinity off the `prompt_cache_key` body field; the `session_id` + * request header is not part of the prompt-caching API. The key must stay identical + * across every turn of one task (so its growing prefix keeps landing on the same + * machine) and differ between tasks (so unrelated prefixes do not compete for one + * routing bucket). `metadata.taskId` is the longest-lived identifier the handler + * receives: it is the persisted task id, stable across turns, restarts and + * rehydration, whereas `this.sessionId` is regenerated per handler instance. + * The handler session id is the fallback for callers that send no task id. + */ + private getPromptCacheKey( + model: OpenAiNativeModel, + metadata?: ApiHandlerCreateMessageMetadata, + ): string | undefined { + if (!model.info.supportsPromptCache) return undefined + + const key = metadata?.taskId || this.sessionId + if (!key) return undefined + + return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? key.slice(0, PROMPT_CACHE_KEY_MAX_LENGTH) : key + } + /** * Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available. * If no tier or no overrides exist, the original ModelInfo is returned. @@ -1531,6 +1570,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio requestBody[SERVICE_TIER_KEY] = serviceTier } + // Cache routing affinity key. `completePrompt` has no task metadata, so the handler + // session id scopes the key: one bucket per handler instance, never shared across tasks. + const promptCacheKey = this.getPromptCacheKey(model) + if (promptCacheKey) { + requestBody.prompt_cache_key = promptCacheKey + } + // Add reasoning if supported if (reasoningEffort) { requestBody.reasoning = { From 0f52a75e5fecfe896c0fe68aed04281b52c1fc7a Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 02:08:04 +0900 Subject: [PATCH 02/12] fix(cost): account cache writes per TTL class instead of one aggregate rate What: - Add `cacheWritesPrice1h` / `cacheWritesPrice12h` to the ModelInfo schema plus `deriveCacheWritesPrice1h` / `deriveCacheWritesPrice12h`, so the vendor multiplier relationship is stated once in model.ts. - Bill three write classes in `calculateApiCostInternal`: the 5m remainder is derived as total - 1h - 12h (clamped at 0), so the classes always sum to the aggregate and no token is billed twice or dropped. - Scale the TTL rates with `cacheWritesPriceMultiplier` in long-context pricing. - Carry `cacheWrite1hTokens` / `cacheWrite12hTokens` on `ApiStreamUsageChunk` and read them from Anthropic `usage.cache_creation.ephemeral_1h_input_tokens` (anthropic, minimax, anthropic-vertex) and OpenAI Responses `input_tokens_details.cache_write_tokens_details.ttl_12h_tokens` (openai-native `normalizeUsage` only). - Thread the split through the Task usage accumulator, including the cancel-path cost recomputation that runs when a provider yields no totalCost. - Populate the rates: 22 Anthropic entries, 19 Bedrock Anthropic-family entries, 7 OpenAI GPT-5.6/6-class entries. Why: Anthropic bills a 1h cache write at 2x input versus 1.25x for 5m, and OpenAI bills a 12h write at 2.5x versus 1.25x for 30m. The repo read only the aggregate `cache_creation_input_tokens` and priced every write at the short rate, so a 1h write under-reported by 37.5% and a 12h write by 50%. The vendors already report the per-TTL split; it was being discarded. Impact: Reporting only. No request shape changes: the repo emits no `ttl` field, so live traffic still reports 0/absent long-lived writes and pricing is byte-identical to before (asserted against the historical cost literals). When the TTL knob lands, a 1h write can no longer be billed at the 5m rate. 18 new tests fail on the base blob; the new cost code has no surviving mutants. --- .../__tests__/cache-write-ttl-pricing.test.ts | 84 ++++++ .../types/src/__tests__/openai-models.test.ts | 2 + packages/types/src/model.ts | 35 +++ packages/types/src/providers/anthropic.ts | 22 ++ packages/types/src/providers/bedrock.ts | 19 ++ packages/types/src/providers/openai.ts | 7 + src/api/providers/__tests__/anthropic.spec.ts | 99 ++++++- .../__tests__/openai-native-usage.spec.ts | 120 +++++++- src/api/providers/anthropic-vertex.ts | 5 + src/api/providers/anthropic.ts | 9 + src/api/providers/minimax.ts | 9 + src/api/providers/openai-native.ts | 14 + src/api/transform/stream.ts | 11 + src/core/task/Task.ts | 23 ++ src/shared/cost.ts | 63 +++- src/utils/__tests__/cost.spec.ts | 270 ++++++++++++++++++ 16 files changed, 788 insertions(+), 4 deletions(-) create mode 100644 packages/types/src/__tests__/cache-write-ttl-pricing.test.ts diff --git a/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts new file mode 100644 index 0000000000..e62f658016 --- /dev/null +++ b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts @@ -0,0 +1,84 @@ +// npx vitest run src/__tests__/cache-write-ttl-pricing.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { bedrockModels } from "../providers/bedrock.js" +import { openAiNativeModels } from "../providers/openai.js" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + deriveCacheWritesPrice12h, + deriveCacheWritesPrice1h, + type ModelInfo, +} from "../model.js" + +/** + * Catalog invariant for the TTL accounting program (ADR-0006 phase 1). + * + * A model that advertises prompt caching on a provider which can report a + * long-lived cache write must carry the matching per-TTL write rate. Without it + * the cost layer can only fall back to the aggregate write rate, which + * under-reports every long-lived write (37.5% for Anthropic 1h, 50% for OpenAI 12h). + */ +describe("cache-write TTL pricing catalog invariant", () => { + const anthropicFamilyEntries = Object.entries(anthropicModels as Record) + + const bedrockAnthropicEntries = Object.entries(bedrockModels as Record).filter(([id]) => + id.startsWith("anthropic."), + ) + + const openAiLongLivedEntries = Object.entries(openAiNativeModels as Record).filter( + ([id]) => id.startsWith("gpt-5.6") || id.startsWith("gpt-6"), + ) + + it("every Anthropic catalog entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = anthropicFamilyEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every Bedrock Anthropic-family entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = bedrockAnthropicEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every OpenAI 5.6/6-class entry defines cacheWritesPrice12h", () => { + expect(openAiLongLivedEntries.length).toBeGreaterThan(0) + + const missing = openAiLongLivedEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice12h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("Anthropic 1h rates match the documented 2x input-price relationship", () => { + for (const [id, info] of anthropicFamilyEntries) { + if (info.cacheWritesPrice1h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice1h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_1H, 9) + } + }) + + it("OpenAI 12h rates match the documented 2.5x input-price relationship", () => { + for (const [id, info] of openAiLongLivedEntries) { + if (info.cacheWritesPrice12h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice12h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_12H, 9) + } + }) + + it("deriveCacheWritesPrice1h scales inputPrice by 2x and stays undefined without one", () => { + expect(deriveCacheWritesPrice1h(3.0)).toBe(6.0) + expect(deriveCacheWritesPrice1h(undefined)).toBeUndefined() + }) + + it("deriveCacheWritesPrice12h scales inputPrice by 2.5x and stays undefined without one", () => { + expect(deriveCacheWritesPrice12h(4.0)).toBe(10.0) + expect(deriveCacheWritesPrice12h(undefined)).toBeUndefined() + }) +}) diff --git a/packages/types/src/__tests__/openai-models.test.ts b/packages/types/src/__tests__/openai-models.test.ts index 78a3259f09..e1e0f1e323 100644 --- a/packages/types/src/__tests__/openai-models.test.ts +++ b/packages/types/src/__tests__/openai-models.test.ts @@ -122,6 +122,7 @@ describe("OpenAI native models", () => { inputPrice: 2, outputPrice: 10, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -166,6 +167,7 @@ describe("OpenAI native models", () => { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 9ddc4ea182..40c00bdab3 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -136,6 +136,13 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL cache-write rates. Vendors bill a longer cache lifetime at a higher + // write rate than the default 5m/30m class, so a single `cacheWritesPrice` + // under-reports whenever the response reports a long-lived write. Omitted + // means "no long-lived writes are expected on this model"; the cost layer + // derives a fail-closed rate from `inputPrice` if one still arrives. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), longContextPricing: z .object({ @@ -188,6 +195,34 @@ export const modelInfoSchema = z.object({ export type ModelInfo = z.infer +/** + * Multipliers vendors apply to the base input price for a cache write, by TTL class. + * Anthropic: 5m = 1.25x, 1h = 2x. OpenAI: 30m = 1.25x, 12h = 2.5x. + */ +export const CACHE_WRITE_MULTIPLIER_5M = 1.25 +export const CACHE_WRITE_MULTIPLIER_1H = 2.0 +export const CACHE_WRITE_MULTIPLIER_12H = 2.5 + +/** + * Derive the 1h cache-write rate from the base input price. + * + * Stated once here so every Anthropic-family catalog entry and every runtime + * fallback agrees on the 2x relationship. Returns `undefined` when the model + * has no input price to scale, which keeps the cost layer fail-closed: it will + * fall back to `cacheWritesPrice` rather than invent a rate. + */ +export function deriveCacheWritesPrice1h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_1H +} + +/** + * Derive the 12h cache-write rate from the base input price (OpenAI 12h class). + * Same fail-closed contract as {@link deriveCacheWritesPrice1h}. + */ +export function deriveCacheWritesPrice12h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_12H +} + export type ModelRecord = Record export type RouterModels = Record diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index 567f6268be..cc50bfb190 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -16,6 +16,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -37,6 +38,7 @@ export const anthropicModels = { inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) // Sonnet 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Fable 5 on the direct Anthropic provider path. Manual @@ -56,6 +58,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -77,6 +80,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -98,6 +102,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag) @@ -119,6 +124,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Keep the hybrid-reasoning capability so Anthropic token-cap handling and // stored max-token overrides behave the same as before. @@ -136,6 +142,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens (regular tier) outputPrice: 25.0, // $25 per million output tokens (regular tier) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // 4.8 inherits the adaptive-thinking model introduced in 4.7 — no breaking // API changes. supportsReasoningBudget is kept true so the existing token-cap @@ -154,6 +161,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Opus 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Sonnet 5 on the direct Anthropic provider path. Manual @@ -172,6 +180,7 @@ export const anthropicModels = { inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // Opus 5.5 always uses adaptive thinking and rejects manual budget_tokens. supportsReasoningBinary: true, @@ -192,6 +201,7 @@ export const anthropicModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // Haiku 5.5 uses the same adaptive-thinking / binary-toggle convention as // Sonnet 5 and Opus 5 on the direct Anthropic provider path. Manual @@ -211,6 +221,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, // $0.25 per million tokens (2.5% of base input price) // Fable 5.1 only accepts adaptive thinking (or an omitted thinking field). // Manual budget_tokens and non-default sampling parameters return a 400. @@ -228,6 +239,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens // Fable 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ on the direct Anthropic provider path. @@ -245,6 +257,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, }, @@ -256,6 +269,7 @@ export const anthropicModels = { inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -267,6 +281,7 @@ export const anthropicModels = { inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -278,6 +293,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, requiredReasoningBudget: true, @@ -290,6 +306,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-sonnet-20241022": { @@ -300,6 +317,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-haiku-20241022": { @@ -310,6 +328,7 @@ export const anthropicModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, }, "claude-3-opus-20240229": { @@ -320,6 +339,7 @@ export const anthropicModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, }, "claude-3-haiku-20240307": { @@ -330,6 +350,7 @@ export const anthropicModels = { inputPrice: 0.25, outputPrice: 1.25, cacheWritesPrice: 0.3, + cacheWritesPrice1h: 0.5, // $0.50 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.03, }, "claude-haiku-4-5-20251001": { @@ -340,6 +361,7 @@ export const anthropicModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, supportsReasoningBudget: true, description: diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index 1e89f3e6ba..696cc04a99 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -22,6 +22,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -36,6 +37,7 @@ export const bedrockModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -63,6 +65,7 @@ export const bedrockModels = { inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -143,6 +146,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -157,6 +161,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -171,6 +176,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -195,6 +201,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -219,6 +226,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -248,6 +256,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -268,6 +277,7 @@ export const bedrockModels = { inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Opus 5.5 (matching the Fable 5.1 entry below). @@ -297,6 +307,7 @@ export const bedrockModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Haiku 5.5 (matching the Opus 5.5 entry above). @@ -317,6 +328,7 @@ export const bedrockModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, minTokensPerCachePoint: 512, maxCachePoints: 4, @@ -335,6 +347,7 @@ export const bedrockModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -351,6 +364,7 @@ export const bedrockModels = { inputPrice: 5.0, outputPrice: 25.0, cacheWritesPrice: 6.25, + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -365,6 +379,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -379,6 +394,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -392,6 +408,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -405,6 +422,7 @@ export const bedrockModels = { inputPrice: 0.8, outputPrice: 4.0, cacheWritesPrice: 1.0, + cacheWritesPrice1h: 1.6, // $1.60 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.08, minTokensPerCachePoint: 2048, maxCachePoints: 4, @@ -419,6 +437,7 @@ export const bedrockModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, // 5m cache writes + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, // cache hits / refreshes minTokensPerCachePoint: 2048, maxCachePoints: 4, diff --git a/packages/types/src/providers/openai.ts b/packages/types/src/providers/openai.ts index 41d00c7a04..89a24fe2ef 100644 --- a/packages/types/src/providers/openai.ts +++ b/packages/types/src/providers/openai.ts @@ -31,6 +31,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.1, longContextPricing: { thresholdTokens: 272_000, @@ -75,6 +76,7 @@ export const openAiNativeModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice12h: 25, // $25.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 1.0, longContextPricing: { thresholdTokens: 272_000, @@ -117,6 +119,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -160,6 +163,7 @@ export const openAiNativeModels = { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, // $0.25 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, @@ -203,6 +207,7 @@ export const openAiNativeModels = { inputPrice: 4.0, outputPrice: 20.0, cacheWritesPrice: 5.0, + cacheWritesPrice12h: 10, // $10.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.4, longContextPricing: { thresholdTokens: 272_000, @@ -246,6 +251,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 12.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -289,6 +295,7 @@ export const openAiNativeModels = { inputPrice: 0.2, outputPrice: 1.2, cacheWritesPrice: 0.25, + cacheWritesPrice12h: 0.5, // $0.50 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.02, longContextPricing: { thresholdTokens: 272_000, diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index b97f309dcc..5bca7e6b30 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -5,7 +5,7 @@ import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { ApiHandlerOptions } from "../../../shared/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" -import type { ApiStreamChunk } from "../../../api/transform/stream" +import type { ApiStreamChunk, ApiStreamUsageChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" // Mock TelemetryService @@ -1045,6 +1045,103 @@ describe("AnthropicHandler", () => { "fine-grained-tool-streaming-2025-05-14,prompt-caching-2024-07-31", ) }) + + it("should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + + it("should leave the 1h split undefined when cache_creation is absent", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(20) + expect(usageChunk?.cacheWrite1hTokens).toBeUndefined() + }) + + it("should price a 1h write at the 1h rate in the final cost chunk", async () => { + // claude-sonnet-4-5: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + const sonnetHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + }) + + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 0, + ephemeral_1h_input_tokens: 100_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = sonnetHandler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const costChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.totalCost !== undefined, + ) + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour charged the 5m rate: 0.375 (37.5% low). + expect(costChunk?.totalCost).toBeCloseTo(0.6, 9) + expect(costChunk?.totalCost).not.toBeCloseTo(0.375, 9) + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 2b05ab5b2b..3725a801bc 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach } from "vitest" import { OpenAiNativeHandler } from "../openai-native" -import { OpenAiServiceTier, openAiNativeModels } from "@roo-code/types" +import { OpenAiServiceTier, openAiNativeModels, type ModelInfo } from "@roo-code/types" +import type { ApiStreamUsageChunk } from "../../transform/stream" describe("OpenAiNativeHandler - normalizeUsage", () => { let handler: OpenAiNativeHandler @@ -615,4 +616,121 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split (Responses API)", () => { + // `normalizeUsage` is private. A narrow structural cast keeps the call typed + // without widening the handler to `any`; the return is asserted non-null because + // every call site below passes a usage object that yields a chunk. + const normalizeUsage = ( + h: OpenAiNativeHandler, + usage: unknown, + model: { id: string; info: ModelInfo }, + ): ApiStreamUsageChunk => { + const result = ( + h as unknown as { + normalizeUsage(u: unknown, m: { id: string; info: ModelInfo }): ApiStreamUsageChunk | undefined + } + ).normalizeUsage(usage, model) + expect(result).toBeDefined() + return result as ApiStreamUsageChunk + } + + it("should read ttl_12h_tokens and bill it at the 12h rate", () => { + // gpt-5.6-sol: input 4.0, cacheWritesPrice 5.0 (30m class), cacheWritesPrice12h 10.0. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 0, + ttl_12h_tokens: 100_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 100_000, + }) + + // gpt-6-astra: input 10.0, 12h rate 25.0. + // Base input: (10.0 / 1_000_000) * (100_000 - 100_000) = 0 + // 12h writes: (25.0 / 1_000_000) * 100_000 = 2.5 + // The pre-split behaviour charged the 30m rate: 12.5 * 0.1 = 1.25 (50% low). + expect(result.totalCost).toBeCloseTo(2.5, 6) + expect(result.totalCost).not.toBeCloseTo(1.25, 6) + }) + + it("should split a mixed 30m/12h write total without double counting", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 60_000, + ttl_12h_tokens: 40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 40_000, + }) + + // 30m remainder: 60_000 at 12.5 => 0.75 + // 12h: 40_000 at 25.0 => 1.0 + // Total: 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + + it("should leave the split undefined when the response omits it", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result.cacheWrite12hTokens).toBeUndefined() + // All writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) + + it("should clamp a 12h split that exceeds the write total", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 50_000, + cache_write_tokens_details: { + ttl_12h_tokens: 80_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // Clamped to the write total so no more 12h tokens are billed than written. + expect(result.cacheWrite12hTokens).toBe(50_000) + // Base input: (10.0 / 1_000_000) * (100_000 - 50_000) = 0.5 + // 12h writes: (25.0 / 1_000_000) * 50_000 = 1.25 + // Total: 0.5 + 1.25 = 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + }) }) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 87da090346..c8a4407950 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -145,6 +145,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple case "message_start": { const usage = chunk.message!.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const cacheWrite1hTokens = usage.cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens; surfaced for per-turn // thinking telemetry (billing already includes it in output_tokens). @@ -153,6 +157,7 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0, cacheWriteTokens: usage.cache_creation_input_tokens || undefined, + cacheWrite1hTokens: cacheWrite1hTokens || undefined, cacheReadTokens: usage.cache_read_input_tokens || undefined, ...(typeof usage.output_tokens_details?.thinking_tokens === "number" ? { reasoningTokens: usage.output_tokens_details!.thinking_tokens } diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index c65eb4beba..677f38bbf5 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -256,6 +256,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -266,9 +267,14 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // The per-TTL breakdown decomposes `cache_creation_input_tokens`; it is + // absent for providers that do not report it, which leaves the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens (0 ⇒ the model skipped // thinking this turn). `output_tokens` already includes them for @@ -280,6 +286,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, ...(typeof startThinkingTokens === "number" ? { reasoningTokens: startThinkingTokens } : {}), } @@ -287,6 +294,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -381,6 +389,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { diff --git a/src/api/providers/minimax.ts b/src/api/providers/minimax.ts index 52c42a4138..12d3cafe37 100644 --- a/src/api/providers/minimax.ts +++ b/src/api/providers/minimax.ts @@ -137,6 +137,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -147,20 +148,27 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + yield { type: "usage", inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, } inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -244,6 +252,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index 91f514ffd5..bae737e9cc 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -132,6 +132,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const writesFromDetails = typeof inputDetails?.cache_write_tokens === "number" ? inputDetails.cache_write_tokens : 0 + // The Responses API splits cache writes by TTL class under + // `input_tokens_details.cache_write_tokens_details`. The installed SDK types + // (openai@5.23.2) do not declare it yet, so it is read defensively; the + // fields are a decomposition of `cache_write_tokens`, not an addition. + const writeDetails = inputDetails?.cache_write_tokens_details + const writes12hFromDetails = typeof writeDetails?.ttl_12h_tokens === "number" ? writeDetails.ttl_12h_tokens : 0 + // If total input tokens are missing but we have details, derive from them let totalInputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0 if ( @@ -152,6 +159,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0 + // Clamp the 12h subset to the write total so a malformed response cannot bill + // more 12h tokens than were written at all. + const cacheWrite12hTokens = Math.min(writes12hFromDetails, cacheWriteTokens) + // Resolve effective tier: prefer actual tier from response; otherwise requested tier const effectiveTier = this.lastServiceTier || (this.options.openAiNativeServiceTier as ServiceTier | undefined) || undefined @@ -166,6 +177,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio cacheWriteTokens, cacheReadTokens, effectiveTier, + undefined, + cacheWrite12hTokens, ) const reasoningTokens = @@ -179,6 +192,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio inputTokens: totalInputTokens, outputTokens: totalOutputTokens, cacheWriteTokens, + cacheWrite12hTokens: cacheWrite12hTokens || undefined, cacheReadTokens, ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}), totalCost, diff --git a/src/api/transform/stream.ts b/src/api/transform/stream.ts index 40efeb3bb0..a3d4b533e7 100644 --- a/src/api/transform/stream.ts +++ b/src/api/transform/stream.ts @@ -59,7 +59,18 @@ export interface ApiStreamUsageChunk { type: "usage" inputTokens: number outputTokens: number + /** Total cache-write tokens, across every TTL class. */ cacheWriteTokens?: number + /** + * The 1h-TTL subset of `cacheWriteTokens` (Anthropic `ephemeral_1h_input_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite1hTokens?: number + /** + * The 12h-TTL subset of `cacheWriteTokens` (OpenAI `ttl_12h_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite12hTokens?: number cacheReadTokens?: number reasoningTokens?: number totalCost?: number diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 3202358c09..af6f9b87fb 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -4021,6 +4021,8 @@ export class Task extends EventEmitter implements TaskLike { try { let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 + let cacheWrite12hTokens = 0 let cacheReadTokens = 0 let inputTokens = 0 let outputTokens = 0 @@ -4056,6 +4058,8 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, + cacheWrite12hTokens, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4063,6 +4067,9 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + undefined, + cacheWrite1hTokens, + cacheWrite12hTokens, ) this.clineMessages[lastApiReqIndex].text = JSON.stringify({ @@ -4210,6 +4217,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens += chunk.inputTokens outputTokens += chunk.outputTokens cacheWriteTokens += chunk.cacheWriteTokens ?? 0 + cacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + cacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 cacheReadTokens += chunk.cacheReadTokens ?? 0 totalCost = chunk.totalCost break @@ -4450,6 +4459,8 @@ export class Task extends EventEmitter implements TaskLike { input: inputTokens, output: outputTokens, cacheWrite: cacheWriteTokens, + cacheWrite1h: cacheWrite1hTokens, + cacheWrite12h: cacheWrite12hTokens, cacheRead: cacheReadTokens, total: totalCost, } @@ -4466,6 +4477,8 @@ export class Task extends EventEmitter implements TaskLike { let bgInputTokens = currentTokens.input let bgOutputTokens = currentTokens.output let bgCacheWriteTokens = currentTokens.cacheWrite + let bgCacheWrite1hTokens = currentTokens.cacheWrite1h + let bgCacheWrite12hTokens = currentTokens.cacheWrite12h let bgCacheReadTokens = currentTokens.cacheRead let bgTotalCost = currentTokens.total @@ -4475,6 +4488,8 @@ export class Task extends EventEmitter implements TaskLike { input: number output: number cacheWrite: number + cacheWrite1h: number + cacheWrite12h: number cacheRead: number total?: number }, @@ -4491,6 +4506,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens = tokens.input outputTokens = tokens.output cacheWriteTokens = tokens.cacheWrite + cacheWrite1hTokens = tokens.cacheWrite1h + cacheWrite12hTokens = tokens.cacheWrite12h cacheReadTokens = tokens.cacheRead totalCost = tokens.total @@ -4568,6 +4585,8 @@ export class Task extends EventEmitter implements TaskLike { bgInputTokens += chunk.inputTokens bgOutputTokens += chunk.outputTokens bgCacheWriteTokens += chunk.cacheWriteTokens ?? 0 + bgCacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + bgCacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 bgCacheReadTokens += chunk.cacheReadTokens ?? 0 bgTotalCost = chunk.totalCost } @@ -4586,6 +4605,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, @@ -4611,6 +4632,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 754a2d84df..7fcc4e0d13 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -1,4 +1,10 @@ -import { OpenAiServiceTier, type ModelInfo, type ServiceTier } from "@roo-code/types" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + OpenAiServiceTier, + type ModelInfo, + type ServiceTier, +} from "@roo-code/types" export interface ApiCostResult { totalInputTokens: number @@ -6,6 +12,24 @@ export interface ApiCostResult { totalCost: number } +/** + * Resolve the per-million rate for a TTL-specific cache-write class. + * + * Precedence is fail-closed: an explicit catalog rate wins, then the vendor + * multiplier applied to the base input price, and only then the aggregate + * `cacheWritesPrice`. The last step is the pre-split behaviour, reachable only + * when a model has neither a 1h/12h rate nor an input price to derive one from. + */ +function resolveCacheWriteRate( + modelInfo: ModelInfo, + explicit: number | undefined, + multiplier: number, +): number | undefined { + if (explicit !== undefined) return explicit + if (modelInfo.inputPrice !== undefined) return modelInfo.inputPrice * multiplier + return modelInfo.cacheWritesPrice +} + function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, serviceTier?: ServiceTier): ModelInfo { const pricing = modelInfo.longContextPricing if (!pricing || totalInputTokens <= pricing.thresholdTokens) { @@ -31,6 +55,16 @@ function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, modelInfo.cacheWritesPrice !== undefined && pricing.cacheWritesPriceMultiplier !== undefined ? modelInfo.cacheWritesPrice * pricing.cacheWritesPriceMultiplier : modelInfo.cacheWritesPrice, + // The TTL-specific write rates scale with the same multiplier as the aggregate + // write rate; leaving them unscaled would under-bill long-context 1h writes. + cacheWritesPrice1h: + modelInfo.cacheWritesPrice1h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice1h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice1h, + cacheWritesPrice12h: + modelInfo.cacheWritesPrice12h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice12h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice12h, cacheReadsPrice: modelInfo.cacheReadsPrice !== undefined && pricing.cacheReadsPriceMultiplier !== undefined ? modelInfo.cacheReadsPrice * pricing.cacheReadsPriceMultiplier @@ -46,8 +80,25 @@ function calculateApiCostInternal( cacheReadInputTokens: number, totalInputTokens: number, totalOutputTokens: number, + cacheCreation1hInputTokens = 0, + cacheCreation12hInputTokens = 0, ): ApiCostResult { - const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens + // The per-TTL counts are a decomposition of `cacheCreationInputTokens`, never + // an addition to it. Deriving the 5m remainder this way keeps the aggregate + // path byte-identical when a vendor reports no split (both terms are 0). + const writes1h = Math.max(0, cacheCreation1hInputTokens) + const writes12h = Math.max(0, cacheCreation12hInputTokens) + const writes5m = Math.max(0, cacheCreationInputTokens - writes1h - writes12h) + + const writes5mRate = modelInfo.cacheWritesPrice || 0 + const writes1hRate = resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice1h, CACHE_WRITE_MULTIPLIER_1H) || 0 + const writes12hRate = + resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice12h, CACHE_WRITE_MULTIPLIER_12H) || 0 + + const cacheWritesCost = + (writes5mRate / 1_000_000) * writes5m + + (writes1hRate / 1_000_000) * writes1h + + (writes12hRate / 1_000_000) * writes12h const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens const outputCost = ((modelInfo.outputPrice || 0) / 1_000_000) * outputTokens @@ -68,6 +119,8 @@ export function calculateApiCostAnthropic( outputTokens: number, cacheCreationInputTokens?: number, cacheReadInputTokens?: number, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreation = cacheCreationInputTokens || 0 const cacheRead = cacheReadInputTokens || 0 @@ -84,6 +137,8 @@ export function calculateApiCostAnthropic( cacheRead, totalInputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } @@ -95,6 +150,8 @@ export function calculateApiCostOpenAI( cacheCreationInputTokens?: number, cacheReadInputTokens?: number, serviceTier?: ServiceTier, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreationInputTokensNum = cacheCreationInputTokens || 0 const cacheReadInputTokensNum = cacheReadInputTokens || 0 @@ -111,6 +168,8 @@ export function calculateApiCostOpenAI( cacheReadInputTokensNum, inputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b369a0405b..b27eb4100a 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -303,4 +303,274 @@ describe("Cost Utility", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split accounting", () => { + const anthropicModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, // 1.25x input + cacheWritesPrice1h: 6.0, // 2x input + cacheReadsPrice: 0.3, + } + + const openAiModel: ModelInfo = { + maxTokens: 128_000, + contextWindow: 1_050_000, + supportsPromptCache: true, + inputPrice: 4.0, + outputPrice: 20.0, + cacheWritesPrice: 5.0, // 1.25x input (30m class) + cacheWritesPrice12h: 10.0, // 2.5x input + cacheReadsPrice: 0.4, + } + + it("bills 100k 1h tokens at the 1h rate, not the 5m rate (37.5% tripwire)", () => { + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 100_000) + + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour would have charged the 5m rate: + // (3.75 / 1_000_000) * 100_000 = 0.375, under-reporting by 37.5%. + expect(result.totalCost).toBeCloseTo(0.6, 9) + expect(result.totalCost).not.toBeCloseTo(0.375, 9) + }) + + it("bills a mixed 5m/1h split per class (0.465 case)", () => { + // 60k of the 100k writes are 1h, 40k remain at the 5m rate. + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 60_000) + + // 5m writes: (3.75 / 1_000_000) * 40_000 = 0.15 + // 1h writes: (6.0 / 1_000_000) * 60_000 = 0.36 + // Total: 0.15 + 0.36 = 0.51 + expect(result.totalCost).toBeCloseTo(0.51, 9) + }) + + it("derives the 5m remainder as total minus 1h minus 12h", () => { + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 30_000, 20_000) + + // 5m remainder: 100k - 30k - 20k = 50k at 3.75 => 0.1875 + // 1h: 30k at 6.0 => 0.18 + // 12h: 20k at 7.5 => 0.15 + // Total: 0.1875 + 0.18 + 0.15 = 0.5175 + expect(result.totalCost).toBeCloseTo(0.5175, 9) + }) + + it("clamps the 5m remainder at zero when the split exceeds the total", () => { + // A malformed response claiming more 1h tokens than the write total must + // not produce negative 5m writes (which would refund cost). + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 150_000) + + // 5m remainder clamps to 0; 1h is capped by the clamp on the remainder only, + // so the 1h class still bills its reported 150k at 6.0 => 0.9 + expect(result.totalCost).toBeCloseTo(0.9, 9) + expect(result.totalCost).toBeGreaterThan(0) + }) + + it("is byte-identical to the pre-split cost when no split is reported", () => { + // Invariant I1: absent split fields must reproduce today's pricing exactly. + const withSplitArgs = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000) + const explicitZeros = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000, 0, 0) + + expect(withSplitArgs.totalCost).toBe(0.0189) + expect(explicitZeros.totalCost).toBe(withSplitArgs.totalCost) + }) + + it("falls back to the 5m rate when a model has no 1h rate and no input price", () => { + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + } + + // Fail-closed: without a derivable 1h rate the write is billed at the + // aggregate rate rather than dropped or invented. + const result = calculateApiCostAnthropic(bareModel, 0, 0, 100_000, 0, 100_000) + expect(result.totalCost).toBeCloseTo(0.375, 9) + }) + + it("derives the 1h rate from inputPrice when the catalog omits it", () => { + const modelWithout1h: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, + } + + const result = calculateApiCostAnthropic(modelWithout1h, 0, 0, 100_000, 0, 100_000) + // Derived 1h rate = 3.0 * 2 = 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) + }) + + it("bills OpenAI 12h writes at the 12h rate, not the 30m rate", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 100_000) + + // 12h writes: (10.0 / 1_000_000) * 100_000 = 1.0 + // The pre-split behaviour would have charged the 30m rate: + // (5.0 / 1_000_000) * 100_000 = 0.5, under-reporting by 50%. + expect(result.totalCost).toBeCloseTo(1.0, 9) + expect(result.totalCost).not.toBeCloseTo(0.5, 9) + }) + + it("bills a mixed OpenAI 30m/12h split per class", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 40_000) + + // 30m remainder: 100k - 40k = 60k at 5.0 => 0.3 + // 12h: 40k at 10.0 => 0.4 + // Total: 0.7 + expect(result.totalCost).toBeCloseTo(0.7, 9) + }) + + it("scales the 1h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + // calculateApiCostOpenAI applies long-context pricing; total input 300k > 200k. + const result = calculateApiCostOpenAI(modelWithLongContext, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input: (3.0 / 1_000_000) * (300_000 - 100_000) = 0.6 + // 1h writes scale 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8. Unscaled 1h would have given 0.6 + 0.6 = 1.2. + expect(result.totalCost).toBeCloseTo(1.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.2, 9) + }) + + it("scales the 12h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...openAiModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithLongContext, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (4.0 / 1_000_000) * (300_000 - 100_000) = 0.8 + // 12h writes scale 10.0 -> 20.0: (20.0 / 1_000_000) * 100_000 = 2.0 + // Total: 0.8 + 2.0 = 2.8. Unscaled 12h would have given 0.8 + 1.0 = 1.8. + expect(result.totalCost).toBeCloseTo(2.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.8, 9) + }) + + it("leaves the 1h rate undefined when neither a catalog rate nor an input price exists", () => { + // Covers the `cacheWritesPrice1h !== undefined` false branch of long-context scaling. + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(bareModel, 300_000, 0, 100_000, 0, undefined, 100_000) + + // No input price => no derivable 1h rate => falls back to the scaled aggregate + // write rate (3.75 * 2 = 7.5): (7.5 / 1_000_000) * 100_000 = 0.75 + expect(result.totalCost).toBeCloseTo(0.75, 9) + }) + + it("does not scale TTL rates when long-context pricing has no write multiplier", () => { + // Covers the `cacheWritesPriceMultiplier !== undefined` false branch for both + // TTL classes: without a multiplier the rates must pass through unchanged. + const modelWithoutMultiplier: ModelInfo = { + ...anthropicModel, + cacheWritesPrice12h: 7.5, + longContextPricing: { + thresholdTokens: 200_000, + inputPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithoutMultiplier, + 300_000, + 0, + 100_000, + 0, + undefined, + 50_000, + 50_000, + ) + + // Base input scales: (3.0 * 2 / 1_000_000) * (300_000 - 100_000) = 1.2 + // 1h unscaled: (6.0 / 1_000_000) * 50_000 = 0.3 + // 12h unscaled: (7.5 / 1_000_000) * 50_000 = 0.375 + // Total: 1.2 + 0.3 + 0.375 = 1.875 + expect(result.totalCost).toBeCloseTo(1.875, 9) + }) + + it("keeps an undefined 12h rate undefined when a write multiplier is present", () => { + // Covers the `cacheWritesPrice12h !== undefined` false branch: a model with a + // long-context write multiplier but no 12h rate must not gain a fabricated one. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(modelWithout12h, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input (no inputPriceMultiplier, so unscaled): (3.0 / 1_000_000) * 200_000 = 0.6 + // 1h scales 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8 + expect(result.totalCost).toBeCloseTo(1.8, 9) + }) + + it("derives the 12h rate from inputPrice when a write multiplier is present but no 12h rate", () => { + // Distinguishes the guarded branch from an unguarded multiply: an unguarded + // `undefined * multiplier` yields NaN, which the cost layer reads as 0 and + // would silently drop the 12h write cost entirely. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithout12h, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (3.0 / 1_000_000) * 200_000 = 0.6 + // No 12h rate in the catalog, so it derives from inputPrice * 2.5 = 7.5 + // (the long-context multiplier does not apply to a rate that is not there): + // (7.5 / 1_000_000) * 100_000 = 0.75 + // Total: 0.6 + 0.75 = 1.35 + expect(result.totalCost).toBeCloseTo(1.35, 9) + expect(result.totalCost).not.toBeCloseTo(0.6, 9) + }) + }) }) From b647ec7d8f746ae7a10d338536ddc5d9796ef863 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:12:08 +0900 Subject: [PATCH 03/12] fix(cost): bill the TTL split on the telemetry recompute and in tier blocks What: - Pass `cacheWrite1h`/`cacheWrite12h` to the cost call inside `captureUsageData`. The bag already carried them and fed the shared accumulator vars, but the telemetry recompute passed only the aggregate write count, so a cancelled request's telemetry billed every write at the short-TTL rate. - Add `cacheWritesPrice1h`/`cacheWritesPrice12h` to the `tiers` schema and to the four Anthropic and four Bedrock 1M-context tier blocks, derived from each tier's own input price. - Carry the tier's TTL rates through all three tier-merge seams (anthropic, anthropic-vertex, bedrock) and derive them from the tier input price in the OpenAI service-tier seam, so a flex/priority or 1M-context write is not billed at the base tier's TTL rate. Why: Two under-reports survived the first pass. `captureUsageData` recomputes cost for telemetry when a provider yields no final cost chunk (the cancelled-stream path), and it dropped the split: 100k 1h tokens at claude-3-5-sonnet billed 0.375 instead of 0.6. Separately, the 1M-context tier merge overrode `inputPrice` but not the TTL rate, so the entry-level rate derived from the 200K input price survived: a 1M-context 1h write billed 6.0 instead of the tier's 12.0, a 50% under-report. Impact: Both paths now bill at the correct TTL rate. Three new tests fail on the pre-fix code (0.375 vs 0.6, and 6 vs 12); a fourth asserts the tier rate reaches the cost function. No request shape changes. --- packages/types/src/model.ts | 5 ++ packages/types/src/providers/anthropic.ts | 4 ++ packages/types/src/providers/bedrock.ts | 4 ++ .../__tests__/anthropic-vertex.spec.ts | 36 ++++++++++++++ src/api/providers/__tests__/anthropic.spec.ts | 33 +++++++++++++ src/api/providers/__tests__/minimax.spec.ts | 32 +++++++++++++ src/api/providers/anthropic-vertex.ts | 4 ++ src/api/providers/anthropic.ts | 5 ++ src/api/providers/bedrock.ts | 4 ++ src/api/providers/openai-native.ts | 13 +++++ src/core/task/Task.ts | 10 +++- src/core/task/__tests__/Task.spec.ts | 47 +++++++++++++++++++ 12 files changed, 196 insertions(+), 1 deletion(-) diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 40c00bdab3..0ac2a4c339 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -187,6 +187,11 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL write rates for this tier. A tier that overrides inputPrice + // must also override these, otherwise a long-context or flex/priority + // write bills at the base tier's TTL rate. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), }), ) diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index cc50bfb190..7bf55238ac 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -26,6 +26,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -68,6 +69,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -90,6 +92,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -112,6 +115,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index 696cc04a99..faef150a2a 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -49,6 +49,7 @@ export const bedrockModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -188,6 +189,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -213,6 +215,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -240,6 +243,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 34788df89b..94b339fc58 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -10,6 +10,7 @@ import { AnthropicVertexHandler } from "../anthropic-vertex" import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" +import type { ApiStreamChunk } from "../../../api/transform/stream" vitest.mock("../utils/timeout-config", () => ({ getApiRequestTimeout: vitest.fn().mockReturnValue(300_000), @@ -453,6 +454,41 @@ describe("VertexHandler", () => { ) }) + it("should surface the 1h cache-write split from cache_creation", async () => { + const mockStream = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ] + + const mockCreate = vitest.fn().mockResolvedValue(asyncStreamFrom(mockStream)) + // Typed structural cast rather than `as any`: the client is a private field and + // this file already has 31 suppressed `any` uses, a budget this test must not grow. + ;(handler["client"].messages as unknown as { create: typeof mockCreate }).create = mockCreate + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is Extract => + chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + it("should handle cache-related usage metrics for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 5bca7e6b30..317a897385 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -7,6 +7,7 @@ import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" import type { ApiStreamChunk, ApiStreamUsageChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" +import { calculateApiCostAnthropic } from "../../../shared/cost" // Mock TelemetryService vitest.mock("@roo-code/telemetry", () => ({ @@ -1538,6 +1539,38 @@ describe("AnthropicHandler", () => { expect(model.info.outputPrice).toBe(22.5) }) + it("carries the tier's own cacheWritesPrice1h when the 1M context tier applies", () => { + // glm P2: the tier merge overrode inputPrice but not the TTL rate, so the + // entry-level 1h rate (derived from the 200K input price) survived and a + // 1M-context 1h write billed 6.0 instead of the tier's 12.0 (50% under). + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + // Tier input price 6.0 => 1h rate must be 2x that, not the 200K entry's 6.0. + expect(model.info.cacheWritesPrice1h).toBe(12.0) + expect(model.info.cacheWritesPrice1h).not.toBe(6.0) + }) + + it("bills a 1M-context 1h cache write at the tier rate, not the 200K rate", () => { + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + const { totalCost } = calculateApiCostAnthropic(model.info, 0, 0, 100_000, 0, 100_000) + + // Tier 1h rate 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // The pre-fix behaviour billed the 200K 1h rate: 0.6 (50% under). + expect(totalCost).toBeCloseTo(1.2, 9) + expect(totalCost).not.toBeCloseTo(0.6, 9) + }) + it("should honor a custom/unrecognized model ID instead of silently falling back to anthropicDefaultModelId", () => { const handler = new AnthropicHandler({ apiKey: "test-api-key", diff --git a/src/api/providers/__tests__/minimax.spec.ts b/src/api/providers/__tests__/minimax.spec.ts index 5d87ef3de8..e33193a6d5 100644 --- a/src/api/providers/__tests__/minimax.spec.ts +++ b/src/api/providers/__tests__/minimax.spec.ts @@ -373,6 +373,38 @@ describe("MiniMaxHandler", () => { expect(firstChunk.value).toEqual({ type: "usage", inputTokens: 10, outputTokens: 20 }) }) + it("createMessage should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockResolvedValueOnce( + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage("system prompt", []) + const firstChunk = await stream.next() + + expect(firstChunk.done).toBe(false) + expect(firstChunk.value).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 60_000, + }) + }) + it("createMessage should pass correct parameters to MiniMax client", async () => { const modelId: MinimaxModelId = "MiniMax-M2" const modelInfo = minimaxModels[modelId] diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index c8a4407950..02fb7ef4ae 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -275,6 +275,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from + // the 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 677f38bbf5..8811645fa8 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -442,6 +442,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // The tier's own TTL rate must come along: the entry-level rate was + // derived from the 200K input price, so keeping it here would bill a + // 1M-context 1h write at half its tier rate. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index e09cc0b17b..d11f5f7dba 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -1295,6 +1295,10 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH inputPrice: tier?.inputPrice ?? modelConfig.info.inputPrice, outputPrice: tier?.outputPrice ?? modelConfig.info.outputPrice, cacheWritesPrice: tier?.cacheWritesPrice ?? modelConfig.info.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from the + // 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier?.cacheWritesPrice1h ?? modelConfig.info.cacheWritesPrice1h, + cacheWritesPrice12h: tier?.cacheWritesPrice12h ?? modelConfig.info.cacheWritesPrice12h, cacheReadsPrice: tier?.cacheReadsPrice ?? modelConfig.info.cacheReadsPrice, } } diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index bae737e9cc..be0a1bc12a 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -16,6 +16,8 @@ import { SERVICE_TIER_KEY, type ServiceTier, ApiProviderError, + CACHE_WRITE_MULTIPLIER_1H, + CACHE_WRITE_MULTIPLIER_12H, } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" @@ -1490,6 +1492,17 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio outputPrice: tierInfo.outputPrice ?? info.outputPrice, cacheReadsPrice: tierInfo.cacheReadsPrice ?? info.cacheReadsPrice, cacheWritesPrice: tierInfo.cacheWritesPrice ?? info.cacheWritesPrice, + // A service tier that overrides the write rate must also override the TTL + // rate, otherwise a flex/priority 12h write bills at the standard-tier 12h + // rate. Derived from the tier's own input price so the 2.5x ratio holds. + cacheWritesPrice12h: + tierInfo.cacheWritesPrice12h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_12H : undefined) ?? + info.cacheWritesPrice12h, + cacheWritesPrice1h: + tierInfo.cacheWritesPrice1h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_1H : undefined) ?? + info.cacheWritesPrice1h, } } diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index af6f9b87fb..8a904e94d6 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -4529,7 +4529,10 @@ export class Task extends EventEmitter implements TaskLike { modelId, ) - // Use the appropriate cost function based on the API protocol + // Use the appropriate cost function based on the API protocol. + // The per-TTL split must be passed here too: without it this + // telemetry recompute bills every write at the short-TTL rate, + // which is exactly the under-report this PR exists to fix. const costResult = apiProtocol === "anthropic" ? calculateApiCostAnthropic( @@ -4538,6 +4541,8 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4545,6 +4550,9 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + undefined, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) TelemetryService.instance.captureLlmCompletion(this.taskId, { diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index bbf77a85e9..122d1af8bc 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -534,6 +534,53 @@ describe("Cline", () => { }) }) + describe("cache-write TTL split in telemetry", () => { + it("bills a 1h cache write at the 1h rate when usage is captured after a cancelled stream", async () => { + // glm P1: the telemetry cost recompute inside captureUsageData must receive + // the per-TTL split. Before the fix it passed only the aggregate write count, + // so a 1h write was billed at the 5m rate (0.375 instead of 0.6 for 100k + // tokens at claude-3-5-sonnet's 3.0 input price). + const task = await createTaskWithManualRetries() + const captureLlmCompletion = vi.spyOn(TelemetryService.instance, "captureLlmCompletion") + + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) + // A stream that reports the split but never yields a final cost chunk, which is + // the shape a cancelled request leaves behind. The second call throws so the + // request loop terminates the way the neighbouring tests do. + vi.spyOn(task, "attemptApiRequest") + .mockImplementationOnce(() => + stream([ + { + type: "usage", + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 100_000, + cacheReadTokens: 0, + }, + { type: "text", text: "partial response" }, + ]), + ) + .mockImplementation(() => { + throw new Error("stop after usage capture") + }) + + await task.recursivelyMakeClineRequests([{ type: "text", text: "original user request" }]) + + // Allow the fire-and-forget drain to settle. + await new Promise((resolve) => setTimeout(resolve, 50)) + + const call = captureLlmCompletion.mock.calls.find(([, payload]) => payload?.cacheWriteTokens === 100_000) + expect(call).toBeDefined() + + // claude-3-5-sonnet-20241022: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-fix lumped behaviour billed (3.75 / 1_000_000) * 100_000 = 0.375. + expect(call![1].cost).toBeCloseTo(0.6, 9) + expect(call![1].cost).not.toBeCloseTo(0.375, 9) + }) + }) + describe("request continuation", () => { it("sends the no-tool-use reminder in a second request when the first response has no tool use", async () => { const task = await createTaskWithManualRetries() From 025ff9c59d2580fb384e79fe656eaae1d336a0b9 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:12:34 +0900 Subject: [PATCH 04/12] test(cost): correct the mixed-split test label to match its arithmetic What: - Rename "bills a mixed 5m/1h split per class (0.465 case)" to "(60/40 split, 0.51)". Why: The label cited a figure the test does not assert. A 60/40 split of 100k writes at 3.75/6.0 costs 0.15 + 0.36 = 0.51, not 0.465. The assertion was already correct; only the name was misleading, and a reviewer reading the label would have expected a different number. Impact: Test name only, no behaviour change. --- src/utils/__tests__/cost.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b27eb4100a..304a1fb21b 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -337,7 +337,7 @@ describe("Cost Utility", () => { expect(result.totalCost).not.toBeCloseTo(0.375, 9) }) - it("bills a mixed 5m/1h split per class (0.465 case)", () => { + it("bills a mixed 5m/1h split per class (60/40 split, 0.51)", () => { // 60k of the 100k writes are 1h, 40k remain at the 5m rate. const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 60_000) From 282ee30628167e6b4e809aec239f2efa6dc13bc4 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:15:45 +0900 Subject: [PATCH 05/12] test(openai-native): cover the 12h rate under each service tier What: - Add a per-tier case asserting the 12h write rate follows the tier's own input price (default 25.0, flex 12.5, priority 50.0 for gpt-6-astra). Why: The service-tier seam was changed to derive the TTL rates from the tier input price, but nothing pinned that behaviour. Without a test, a future edit could revert to the standard-tier 12h rate and a flex/priority 12h write would be mis-billed by 2x in either direction. Impact: Test only. Fails without the tier derivation (flex bills 2.5 instead of 1.25, priority 2.5 instead of 5.0). --- .../__tests__/openai-native-usage.spec.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 3725a801bc..44ac2deaf1 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -528,6 +528,35 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(expectedCost, 6) }) + it.each([ + // gpt-6-astra: default input 10.0 / 12h 25.0; flex input 5.0; priority input 20.0. + // The 12h rate must follow the tier's own input price (2.5x), not the + // standard-tier 12h rate, or a flex/priority 12h write is mis-billed. + { tier: OpenAiServiceTier.Default, expectedCost: 2.5 }, + { tier: OpenAiServiceTier.Flex, expectedCost: 1.25 }, + { tier: OpenAiServiceTier.Priority, expectedCost: 5.0 }, + ])("derives the 12h rate from the $tier tier's own input price", ({ tier, expectedCost }) => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + openAiNativeServiceTier: tier, + }) + const result = handler["normalizeUsage"]( + { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { ttl_12h_tokens: 100_000 }, + }, + }, + getGpt6AstraModel(), + ) + + if (!result) throw new Error("Expected usage") + // All input is consumed by the write, so the whole cost is the 12h class. + expect(result.totalCost).toBeCloseTo(expectedCost, 6) + }) + it("should pass total input tokens to calculateApiCostOpenAI", () => { const usage = { input_tokens: 100, From 65ac86200585008f1a13013d18570fead277e73b Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 04:08:35 +0900 Subject: [PATCH 06/12] feat(api): emit an opt-in prompt cache TTL gated by catalog capability What: add a `promptCacheTtl` provider-settings knob ("5m" | "1h"), a `ModelInfo.promptCacheTtlSupport` capability array, and a shared `resolveCacheTtl`/`cacheControl` helper that the anthropic, vertex and vercel-ai-gateway caching transforms plus the anthropic, anthropic-vertex, openrouter, vercel-ai-gateway and zoo-gateway providers thread through. Why: ADR-0006 phase 2. C5a (#2260) landed per-TTL write accounting, so a 1h cache write is billed at 2x instead of the 5m rate. This is the matching request-side mechanism: once the settings UI ships in C6, a user selecting 1h causes Anthropic-shaped providers to emit `cache_control: { ttl: "1h" }`. Impact: no behavior change by default. An unset knob resolves to `undefined` and the marker stays the byte-identical `{ type: "ephemeral" }` literal, so no `ttl` key is emitted unless the user opts in AND the resolved ModelInfo advertises the capability. Absent capability means 5m only (I2), capability is read from ModelInfo rather than id substrings, and every marker in a request shares one resolved TTL (I4). Dynamic catalogs (OpenRouter, the gateways) carry no capability data and therefore stay at 5m, as the ADR requires. --- .../src/__tests__/prompt-cache-ttl.test.ts | 103 +++++++++ packages/types/src/model.ts | 16 ++ .../types/src/provider-settings/common.ts | 7 +- packages/types/src/providers/anthropic.ts | 25 ++ .../__tests__/anthropic-vertex.spec.ts | 102 +++++++++ src/api/providers/__tests__/anthropic.spec.ts | 72 ++++++ .../providers/__tests__/openrouter.spec.ts | 109 +++++++++ .../__tests__/vercel-ai-gateway.spec.ts | 17 ++ src/api/providers/anthropic-vertex.ts | 19 +- src/api/providers/anthropic.ts | 13 +- src/api/providers/openrouter.ts | 6 +- src/api/providers/vercel-ai-gateway.ts | 3 +- src/api/providers/zoo-gateway.ts | 3 +- .../transform/caching/__tests__/ttl.spec.ts | 216 ++++++++++++++++++ src/api/transform/caching/anthropic.ts | 16 +- src/api/transform/caching/ttl.ts | 32 +++ .../transform/caching/vercel-ai-gateway.ts | 16 +- src/api/transform/caching/vertex.ts | 22 +- 18 files changed, 774 insertions(+), 23 deletions(-) create mode 100644 packages/types/src/__tests__/prompt-cache-ttl.test.ts create mode 100644 src/api/transform/caching/__tests__/ttl.spec.ts create mode 100644 src/api/transform/caching/ttl.ts diff --git a/packages/types/src/__tests__/prompt-cache-ttl.test.ts b/packages/types/src/__tests__/prompt-cache-ttl.test.ts new file mode 100644 index 0000000000..05b76245ee --- /dev/null +++ b/packages/types/src/__tests__/prompt-cache-ttl.test.ts @@ -0,0 +1,103 @@ +// npx vitest run src/__tests__/prompt-cache-ttl.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { vertexModels } from "../providers/vertex.js" +import { openRouterDefaultModelInfo } from "../providers/openrouter.js" +import { + vercelAiGatewayDefaultModelId, + VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS, +} from "../providers/vercel-ai-gateway.js" +import { + DEFAULT_PROMPT_CACHE_TTL, + modelInfoSchema, + promptCacheTtlSchema, + promptCacheTtlValues, + type ModelInfo, +} from "../model.js" +import { anthropicProviderDefinition } from "../provider-settings/anthropic.js" +import { baseProviderSettingsShape } from "../provider-settings/common.js" +import { providerIdentifiers } from "../provider-identifiers.js" +import { globalSettingsSchema } from "../global-settings.js" + +/** + * Catalog invariant for the request-side TTL knob (ADR-0006 phase 2, PR-C5b). + * + * Emission is gated by `ModelInfo.promptCacheTtlSupport`, so a model that can + * report a 1h cache write but carries no capability array would silently never + * emit the TTL the user asked for. These tests pin both directions: capable + * catalogs advertise it, and catalogs that cannot know stay absent (fail closed). + */ +describe("prompt cache TTL schema", () => { + it("accepts exactly the 5m and 1h classes", () => { + expect(promptCacheTtlValues).toEqual(["5m", "1h"]) + expect(promptCacheTtlSchema.parse("5m")).toBe("5m") + expect(promptCacheTtlSchema.parse("1h")).toBe("1h") + expect(promptCacheTtlSchema.safeParse("30m").success).toBe(false) + expect(promptCacheTtlSchema.safeParse("12h").success).toBe(false) + }) + + it("defaults to undefined so an unset knob emits no ttl key", () => { + expect(DEFAULT_PROMPT_CACHE_TTL).toBeUndefined() + }) + + it("is optional on ModelInfo", () => { + const parsed = modelInfoSchema.parse({ contextWindow: 200_000, supportsPromptCache: true }) + expect(parsed.promptCacheTtlSupport).toBeUndefined() + + const withCapability = modelInfoSchema.parse({ + contextWindow: 200_000, + supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], + }) + expect(withCapability.promptCacheTtlSupport).toEqual(["5m", "1h"]) + }) +}) + +describe("prompt cache TTL knob location", () => { + it("lives in provider settings so handlers can see it", () => { + // ApiHandlerOptions = Omit, so only a + // provider-settings key reaches a handler (ADR-0006 decision item 2). + expect(baseProviderSettingsShape.promptCacheTtl).toBeDefined() + + const parsed = anthropicProviderDefinition.schema.safeParse({ + apiProvider: providerIdentifiers.anthropic, + promptCacheTtl: "1h", + }) + expect(parsed.success).toBe(true) + expect(parsed.success && parsed.data.promptCacheTtl).toBe("1h") + }) + + it("is not a global setting", () => { + const shape = globalSettingsSchema.shape as Record + expect(shape.promptCacheTtl).toBeUndefined() + }) +}) + +describe("prompt cache TTL capability catalog", () => { + it("every Anthropic entry with prompt caching advertises both TTL classes", () => { + const missing = Object.entries(anthropicModels as Record) + .filter(([, info]) => info.supportsPromptCache && !info.promptCacheTtlSupport?.includes("1h")) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("leaves the Vertex catalog without capability so it fails closed", () => { + // PR-C5b scopes the catalog capability to the Anthropic entries; the Vertex + // catalog is populated separately, so its models stay 5m-only for now. + const withCapability = Object.entries(vertexModels as Record) + .filter(([, info]) => info.promptCacheTtlSupport !== undefined) + .map(([id]) => id) + + expect(withCapability).toEqual([]) + }) + + it("leaves dynamic-catalog defaults without capability so they fail closed", () => { + // OpenRouter and the gateway catalogs are fetched at runtime and carry no TTL + // capability data, which ADR-0006 maps to "5m only" (invariant I2). + expect(openRouterDefaultModelInfo.promptCacheTtlSupport).toBeUndefined() + expect(VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(vercelAiGatewayDefaultModelId)).toBe(true) + }) +}) diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 0ac2a4c339..b3c2b8151d 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -95,6 +95,20 @@ export type ModelParameter = z.infer export const isModelParameter = (value: string): value is ModelParameter => modelParameters.includes(value as ModelParameter) +/** + * Prompt cache TTL classes a request-side cache marker can carry. `"5m"` is the + * vendor default and is never emitted explicitly, so only `"1h"` changes the wire + * shape and an unset knob stays byte-identical to a pre-knob build (ADR-0006 I3). + */ +export const promptCacheTtlValues = ["5m", "1h"] as const + +export const promptCacheTtlSchema = z.enum(promptCacheTtlValues) + +export type PromptCacheTtl = z.infer + +/** Deliberately `undefined`, not `"5m"`: emitting `ttl: "5m"` would change every request's bytes. */ +export const DEFAULT_PROMPT_CACHE_TTL: PromptCacheTtl | undefined = undefined + /** * ModelInfo */ @@ -112,6 +126,8 @@ export const modelInfoSchema = z.object({ // When set to "24h", extended prompt caching will be requested; when omitted // or set to "in_memory", the default in‑memory cache is used. promptCacheRetention: z.enum(["in_memory", "24h"]).optional(), + // Cache TTLs this model accepts. Absent means "5m only" (ADR-0006 I2, fail closed). + promptCacheTtlSupport: z.array(promptCacheTtlSchema).optional(), // Capability flag to indicate whether the model supports an output verbosity parameter supportsVerbosity: z.boolean().optional(), // Capability flag to indicate whether the model exposes a user-configurable max output diff --git a/packages/types/src/provider-settings/common.ts b/packages/types/src/provider-settings/common.ts index ca78c3a25d..ca908d0c60 100644 --- a/packages/types/src/provider-settings/common.ts +++ b/packages/types/src/provider-settings/common.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { reasoningEffortSettingSchema, verbosityLevelsSchema } from "../model.js" +import { reasoningEffortSettingSchema, verbosityLevelsSchema, promptCacheTtlSchema } from "../model.js" import type { ProviderIdentifier } from "../provider-identifiers.js" export const API_PROVIDER_FIELD = "apiProvider" @@ -33,6 +33,11 @@ export const baseProviderSettingsShape = { modelMaxTokens: z.number().optional(), modelMaxThinkingTokens: z.number().optional(), verbosity: verbosityLevelsSchema.optional(), + // Opt-in prompt cache TTL for this provider profile. Lives here, not in global + // settings, because `ApiHandlerOptions = Omit` + // (src/shared/api.ts) makes a global key invisible to handlers. Unset emits no + // ttl key at all, not "5m" (ADR-0006 I3). + promptCacheTtl: promptCacheTtlSchema.optional(), } export const apiModelIdProviderModelShape = { diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index 7bf55238ac..bd90938931 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -3,6 +3,9 @@ import type { ModelInfo } from "../model.js" // https://docs.anthropic.com/en/docs/about-claude/models // https://platform.claude.com/docs/en/about-claude/pricing +// Every entry here is an Anthropic-hosted Claude model, and 1h cache writes are GA +// on all active Claude models, so each one advertises both TTL classes. The 5m class +// is the vendor default and is never emitted explicitly (ADR-0006 I3). export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-sonnet-4-5" export const ANTHROPIC_API_PROTOCOL = "anthropic" @@ -13,6 +16,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -36,6 +40,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) @@ -56,6 +61,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -79,6 +85,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -102,6 +109,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -125,6 +133,7 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -143,6 +152,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required, same as 4.7) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (regular tier) outputPrice: 25.0, // $25 per million output tokens (regular tier) cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -162,6 +172,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -181,6 +192,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens @@ -197,6 +209,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], // Standard pricing (prompts of 100,000 tokens or fewer) from Anthropic's // pricing page. Prompts over 100,000 tokens bill at tiered rates // ($0.50 in / $2.50 out / $0.625 5m cache write / $0.05 cache read per @@ -222,6 +235,7 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache) @@ -240,6 +254,7 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens @@ -258,6 +273,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -270,6 +286,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens @@ -282,6 +299,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens @@ -294,6 +312,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -307,6 +326,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -318,6 +338,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -329,6 +350,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: false, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, @@ -340,6 +362,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, @@ -351,6 +374,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 0.25, outputPrice: 1.25, cacheWritesPrice: 0.3, @@ -362,6 +386,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 94b339fc58..f4860e9914 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -260,6 +260,108 @@ describe("VertexHandler", () => { ) }) + describe("prompt cache TTL", () => { + const minimalStream = asyncStreamFrom([ + { + type: "message_start", + message: { usage: { input_tokens: 10, output_tokens: 0 } }, + }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // The handler's client is private; reach it once here rather than casting at + // each call site. `messages.create` is replaced with a plain vitest mock. + const stubCreate = (mockCreate: ReturnType) => { + const client = handler["client"] as unknown as { messages: { create: unknown } } + client.messages.create = mockCreate + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on the system prompt and every user marker when the model is capable", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + // The Vertex catalog carries no TTL capability yet, so inject one to + // exercise the emission path this PR adds. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + // System + the user marker the vertex transform emits. + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls.length).toBeGreaterThanOrEqual(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("fails closed when the Vertex catalog advertises no TTL capability", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should handle multiple content blocks with line breaks for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 317a897385..d839003e5d 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -191,6 +191,78 @@ describe("AnthropicHandler", () => { expect(mockCreate).toHaveBeenCalled() }) + describe("prompt cache TTL", () => { + const threeTurnMessages = () => [ + { role: "user" as const, content: [{ type: "text" as const, text: "First message" }] }, + { role: "assistant" as const, content: [{ type: "text" as const, text: "Response" }] }, + { role: "user" as const, content: [{ type: "text" as const, text: "Second message" }] }, + ] + + /** Every cache_control object in the serialized request body. */ + const requestCacheControls = () => { + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const serialized = JSON.parse(JSON.stringify(body)) + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(serialized) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + await collectStream(handler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on all three marker sites when the knob is 1h", async () => { + const ttlHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "1h", + }) + + await collectStream(ttlHandler.createMessage(systemPrompt, threeTurnMessages())) + + // System + last two user messages: a single missed site would show up + // here as a mixed-TTL request (ADR-0006 I4). + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(new Set(controls.map((control) => control.ttl))).toEqual(new Set(["1h"])) + }) + + it("emits no ttl key for an explicit 5m knob", async () => { + const fiveMinuteHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "5m", + }) + + await collectStream(fiveMinuteHandler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should include 1M context beta header for Claude Sonnet 4.6 when enabled", async () => { const sonnet46Handler = new AnthropicHandler({ apiKey: "test-api-key", diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index 1bf6669324..fec3e4a2e2 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -485,6 +485,115 @@ describe("OpenRouterHandler", () => { ) }) + describe("prompt cache TTL", () => { + const mockStreamWithUsage = () => + asyncStreamFrom([ + { id: "test-id", choices: [{ delta: { content: "test response" } }] }, + { id: "test-id", choices: [{ delta: {} }], usage: { prompt_tokens: 10, completion_tokens: 20 } }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // `openai` is mocked for this file, so the prototype is a plain object here. + // Reaching it once keeps the casts out of each test body. + const stubChatCreate = (mockCreate: ReturnType) => { + const proto = OpenAI.prototype as unknown as { chat: { completions: { create: unknown } } } + proto.chat = { completions: { create: mockCreate } } + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ ...mockOptions, openRouterModelId: "anthropic/claude-sonnet-4" }), + ) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("fails closed when the resolved model advertises no TTL capability", async () => { + // OpenRouter models come from a dynamic catalog that carries no TTL + // capability data, so a 1h knob must not reach the wire (ADR-0006 I2, + // and the ADR's stated risk for dynamic catalogs). + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on every marker when the resolved model advertises it", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + // Simulate a catalog that does carry capability data. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + }) + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockStream = asyncStreamFrom([{ error: { message: "API Error", code: 500 } }]) diff --git a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts index e69240c1da..687607d305 100644 --- a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts +++ b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts @@ -449,6 +449,23 @@ describe("VercelAiGatewayHandler", () => { expect(addCacheBreakpoints).toHaveBeenCalled() }) + it("does not emit a ttl for a gateway model with no TTL capability", async () => { + // The gateway catalog is fetched at runtime and carries no TTL capability, + // so a 1h knob must fail closed (ADR-0006 I2). + const handler = new VercelAiGatewayHandler( + makeApiHandlerOptions({ + ...mockOptions, + vercelAiGatewayModelId: "anthropic/claude-3.5-haiku", + promptCacheTtl: "1h", + }), + ) + + await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next() + + const call = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0] + expect(JSON.stringify(call)).not.toContain("ttl") + }) + it("sets correct max_completion_tokens", async () => { const handler = new VercelAiGatewayHandler(mockOptions) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 02fb7ef4ae..9334e436c3 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -15,6 +15,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { addCacheBreakpoints } from "../transform/caching/vertex" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" @@ -79,6 +80,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple const { supportsPromptCache } = info + // One TTL for the whole request (ADR-0006 I4). + const cacheTtl = resolveCacheTtl(this.options, info) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -116,9 +120,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple thinking, // Cache the system prompt if caching is enabled. system: supportsPromptCache - ? [{ text: systemPrompt, type: "text" as const, cache_control: { type: "ephemeral" } }] + ? [{ text: systemPrompt, type: "text" as const, cache_control: cacheControl(cacheTtl) }] : systemPrompt, - messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages) : sanitizedMessages, + messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages, cacheTtl) : sanitizedMessages, stream: true, ...nativeToolParams, } as Anthropic.Messages.MessageCreateParamsStreaming @@ -322,11 +326,12 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple try { const { id, - info: { supportsPromptCache }, + info, temperature, maxTokens = ANTHROPIC_DEFAULT_MAX_TOKENS, reasoning: thinking, } = this.getModel() + const { supportsPromptCache } = info const params = { model: id, @@ -337,7 +342,13 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple { role: "user", content: supportsPromptCache - ? [{ type: "text" as const, text: prompt, cache_control: { type: "ephemeral" } }] + ? [ + { + type: "text" as const, + text: prompt, + cache_control: cacheControl(resolveCacheTtl(this.options, info)), + }, + ] : prompt, }, ], diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 8811645fa8..45bef00096 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -19,6 +19,7 @@ import { ApiStream } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { BaseProvider } from "./base-provider" import { bridgeExternalAbortSignal, hasUsableTimeout, toSdkTimeoutMs } from "./utils/abort-signal" @@ -64,7 +65,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { let stream: AnthropicStream - const cacheControl: CacheControlEphemeral = { type: "ephemeral" } const { id: modelId, betas = ["fine-grained-tool-streaming-2025-05-14"], @@ -79,6 +79,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa settings: this.options, }) + // One TTL for the whole request (ADR-0006 I4): every marker below reuses + // this single object, so a mixed-TTL request is unrepresentable. + const cacheTtl = resolveCacheTtl(this.options, info) + const marker: CacheControlEphemeral = cacheControl(cacheTtl) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -169,17 +174,17 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa temperature, thinking, // Setting cache breakpoint for system prompt so new tasks can reuse it. - system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + system: [{ text: systemPrompt, type: "text", cache_control: marker }], messages: sanitizedMessages.map((message, index) => { if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { return { ...message, content: typeof message.content === "string" - ? [{ type: "text", text: message.content, cache_control: cacheControl }] + ? [{ type: "text", text: message.content, cache_control: marker }] : message.content.map((content, contentIndex) => contentIndex === message.content.length - 1 - ? { ...content, cache_control: cacheControl } + ? { ...content, cache_control: marker } : content, ), } diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 36e09aa883..6d2057a7a3 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -26,6 +26,7 @@ import { ApiStreamChunk } from "../transform/stream" import { convertToR1Format } from "../transform/r1-format" import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic" import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { OpenRouterReasoningParams } from "../transform/reasoning" import { getModelParams } from "../transform/model-params" @@ -306,9 +307,12 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH // TODO: Add a `promptCacheStratey` field to `ModelInfo`. if (OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId)) { if (modelId.startsWith("google")) { + // Gemini markers keep their default TTL: ADR-0006 scopes the knob to + // Anthropic-shaped markers only, and Gemini has no 1h class. addGeminiCacheBreakpoints(systemPrompt, openAiMessages) } else { - addAnthropicCacheBreakpoints(systemPrompt, openAiMessages) + // Capability comes from the resolved ModelInfo, never from the id. + addAnthropicCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, model.info)) } } diff --git a/src/api/providers/vercel-ai-gateway.ts b/src/api/providers/vercel-ai-gateway.ts index 2f1586b928..135ac8bef6 100644 --- a/src/api/providers/vercel-ai-gateway.ts +++ b/src/api/providers/vercel-ai-gateway.ts @@ -17,6 +17,7 @@ import { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -82,7 +83,7 @@ export class VercelAiGatewayHandler extends RouterProvider implements SingleComp ] if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } const supportsTemperature = info.supportsTemperature !== false && this.supportsTemperature(modelId) diff --git a/src/api/providers/zoo-gateway.ts b/src/api/providers/zoo-gateway.ts index 66ccac5067..5f33ee48fe 100644 --- a/src/api/providers/zoo-gateway.ts +++ b/src/api/providers/zoo-gateway.ts @@ -19,6 +19,7 @@ import { t } from "../../i18n" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -208,7 +209,7 @@ export class ZooGatewayHandler extends RouterProvider implements SingleCompletio // Apply prompt caching for models that support it // Zoo Gateway serves the same models as Vercel AI Gateway, so caching support is identical if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } // Build request headers with enrichment metadata diff --git a/src/api/transform/caching/__tests__/ttl.spec.ts b/src/api/transform/caching/__tests__/ttl.spec.ts new file mode 100644 index 0000000000..ab11d64b4d --- /dev/null +++ b/src/api/transform/caching/__tests__/ttl.spec.ts @@ -0,0 +1,216 @@ +// npx vitest run src/api/transform/caching/__tests__/ttl.spec.ts + +import OpenAI from "openai" + +import { addCacheBreakpoints as addAnthropicBreakpoints } from "../anthropic" +import { addCacheBreakpoints as addVertexBreakpoints } from "../vertex" +import { addCacheBreakpoints as addGatewayBreakpoints } from "../vercel-ai-gateway" +import { cacheControl, resolveCacheTtl } from "../ttl" + +/** + * Collect every `cache_control` object reachable in a serialized request. + * + * Counting occurrences rather than spot-checking one marker is the point: a + * transform that forgot one of its marker sites would still pass a single-site + * assertion, which is exactly the failure mode ADR-0006 I4 is meant to exclude. + */ +const collectCacheControls = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +/** The serialized request body, so assertions run on the bytes that would be sent. */ +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) + +describe("resolveCacheTtl", () => { + it("returns undefined when the knob is unset, even for a capable model", () => { + expect(resolveCacheTtl({}, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + expect(resolveCacheTtl(undefined, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns undefined for an explicit 5m so the wire shape stays unchanged", () => { + expect(resolveCacheTtl({ promptCacheTtl: "5m" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns 1h when the knob asks for it and the model advertises it", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBe("1h") + }) + + it("fails closed when the model advertises no capability at all", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, {})).toBeUndefined() + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, undefined)).toBeUndefined() + }) + + it("fails closed when the model advertises 5m only", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m"] })).toBeUndefined() + }) +}) + +describe("cacheControl", () => { + it("is the historical literal when no TTL is resolved", () => { + expect(cacheControl(undefined)).toEqual({ type: "ephemeral" }) + expect(Object.keys(cacheControl(undefined))).toEqual(["type"]) + }) + + it("carries the ttl key only when a TTL is resolved", () => { + expect(cacheControl("1h")).toEqual({ type: "ephemeral", ttl: "1h" }) + }) +}) + +describe("caching transforms under a prompt cache TTL", () => { + const systemPrompt = "You are a helpful assistant." + + const buildOpenAiMessages = (): OpenAI.Chat.ChatCompletionMessageParam[] => [ + { role: "system", content: systemPrompt }, + { role: "user", content: "First message" }, + { role: "assistant", content: "First response" }, + { role: "user", content: "Second message" }, + ] + + const buildAnthropicMessages = () => [ + { role: "user" as const, content: "First message" }, + { role: "assistant" as const, content: "First response" }, + { role: "user" as const, content: "Second message" }, + ] + + describe("default parity (ADR-0006 I3)", () => { + it("emits no ttl key anywhere when the TTL argument is omitted", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages) + + const serialized = serialize(messages) + const rawJson = JSON.stringify(messages) + + expect(rawJson).not.toContain("ttl") + expect(serialized).not.toContain("ttl") + + const controls = collectCacheControls(serialized) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("is byte-identical to the pre-knob literal for every transform", () => { + const anthropicMessages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, anthropicMessages) + + const gatewayMessages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, gatewayMessages) + + const vertexMessages = addVertexBreakpoints(buildAnthropicMessages()) + + expect(serialize(anthropicMessages)).toEqual( + serialize([ + { + role: "system", + content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], + }, + { + role: "user", + content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], + }, + { role: "assistant", content: "First response" }, + { + role: "user", + content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], + }, + ]), + ) + + expect(serialize(gatewayMessages)).toEqual( + serialize([ + { role: "system", content: systemPrompt, cache_control: { type: "ephemeral" } }, + { + role: "user", + content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], + }, + { role: "assistant", content: "First response" }, + { + role: "user", + content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], + }, + ]), + ) + + expect(serialize(vertexMessages)).toEqual( + serialize([ + { + role: "user", + content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], + }, + { role: "assistant", content: "First response" }, + { + role: "user", + content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], + }, + ]), + ) + }) + }) + + describe("opt-in uniformity (ADR-0006 I4)", () => { + it("puts ttl 1h on every anthropic marker, not just the first", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(controls.every((control) => control.type === "ephemeral")).toBe(true) + }) + + it("puts ttl 1h on every gateway marker", () => { + const messages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("puts ttl 1h on every vertex marker", () => { + const messages = addVertexBreakpoints(buildAnthropicMessages(), "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("never mixes TTLs within one request", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const ttls = new Set(collectCacheControls(serialize(messages)).map((control) => control.ttl)) + expect(ttls).toEqual(new Set(["1h"])) + }) + }) + + describe("signature is additive-optional", () => { + it("keeps the two-argument call compiling and byte-identical", () => { + const withDefault = buildOpenAiMessages() + const withExplicitUndefined = buildOpenAiMessages() + + addAnthropicBreakpoints(systemPrompt, withDefault) + addAnthropicBreakpoints(systemPrompt, withExplicitUndefined, undefined) + + expect(serialize(withDefault)).toEqual(serialize(withExplicitUndefined)) + }) + }) +}) diff --git a/src/api/transform/caching/anthropic.ts b/src/api/transform/caching/anthropic.ts index cff671a56c..3bdcba4ba1 100644 --- a/src/api/transform/caching/anthropic.ts +++ b/src/api/transform/caching/anthropic.ts @@ -1,10 +1,20 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + messages[0] = { role: "system", // @ts-ignore-next-line - content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], + content: [{ type: "text", text: systemPrompt, cache_control: control }], } // Ensure all user messages have content in array format first @@ -35,7 +45,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. } // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } }) } diff --git a/src/api/transform/caching/ttl.ts b/src/api/transform/caching/ttl.ts new file mode 100644 index 0000000000..4ba2652fe2 --- /dev/null +++ b/src/api/transform/caching/ttl.ts @@ -0,0 +1,32 @@ +import type { ModelInfo, PromptCacheTtl } from "@roo-code/types" +import type { CacheControlEphemeral } from "@anthropic-ai/sdk/resources" + +/** + * Resolve the TTL a request may emit, gated by the provider-profile knob and the + * model's catalog capability. Both gates fail closed: absent capability means + * "5m only" (ADR-0006 I2), and capability is read from the resolved `ModelInfo` + * rather than inferred from the model id. + * + * `"5m"` resolves to `undefined` on purpose. It is the vendor default, so emitting + * it would add bytes without changing behavior; only `"1h"` may change the wire, + * which keeps default parity (I3) checkable by a single grep. + */ +export function resolveCacheTtl( + settings: { promptCacheTtl?: PromptCacheTtl } | undefined, + info: Pick | undefined, +): PromptCacheTtl | undefined { + if (settings?.promptCacheTtl !== "1h") { + return undefined + } + + return info?.promptCacheTtlSupport?.includes("1h") ? "1h" : undefined +} + +/** + * Build the `cache_control` object for one marker. Every marker in a request goes + * through this function, which makes one uniform TTL per request (I4) structural + * instead of a convention each call site has to remember. + */ +export function cacheControl(ttl: PromptCacheTtl | undefined): CacheControlEphemeral { + return ttl ? { type: "ephemeral", ttl } : { type: "ephemeral" } +} diff --git a/src/api/transform/caching/vercel-ai-gateway.ts b/src/api/transform/caching/vercel-ai-gateway.ts index 0d3834bbe5..7272de34d7 100644 --- a/src/api/transform/caching/vercel-ai-gateway.ts +++ b/src/api/transform/caching/vercel-ai-gateway.ts @@ -1,12 +1,22 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + // Apply cache_control to system message at the message level messages[0] = { role: "system", content: systemPrompt, // @ts-ignore-next-line - cache_control: { type: "ephemeral" }, + cache_control: control, } // Add cache_control to the last two user messages for conversation context caching @@ -23,7 +33,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. if (lastTextPart && lastTextPart.text && lastTextPart.text.length > 0) { // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } } }) diff --git a/src/api/transform/caching/vertex.ts b/src/api/transform/caching/vertex.ts index 48bf261587..e223bb4d11 100644 --- a/src/api/transform/caching/vertex.ts +++ b/src/api/transform/caching/vertex.ts @@ -1,6 +1,13 @@ import { Anthropic } from "@anthropic-ai/sdk" -export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + messages: Anthropic.Messages.MessageParam[], + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam[] { // Find indices of user messages that we want to cache. // We only cache the last two user messages to stay within the 4-block limit // (1 block for system + 1 block each for last two user messages = 3 total). @@ -12,18 +19,23 @@ export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) return messages.map((message, index) => message.role !== "assistant" && (index === lastIndex || index === secondLastIndex) - ? cachedMessage(message) + ? cachedMessage(message, ttl) : message, ) } -function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Messages.MessageParam { +function cachedMessage( + message: Anthropic.Messages.MessageParam, + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam { + const control = cacheControl(ttl) + // For string content, we convert to array format with optional cache control. if (typeof message.content === "string") { return { ...message, // For string content, we only have one block so it's always the last block. - content: [{ type: "text" as const, text: message.content, cache_control: { type: "ephemeral" } }], + content: [{ type: "text" as const, text: message.content, cache_control: control }], } } @@ -41,7 +53,7 @@ function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Mess ? { ...content, // Check if this is the last text block using our pre-calculated index. - ...(index === lastTextBlockIndex && { cache_control: { type: "ephemeral" } }), + ...(index === lastTextBlockIndex && { cache_control: control }), } : content, ), From 7a06eac8af7982639a35b1a6549c1f1e55385f89 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 05:14:15 +0900 Subject: [PATCH 07/12] feat(bedrock): emit the opt-in cache TTL only for 1h-capable Converse models What: - Add `promptCacheTtlSupport: ["5m", "1h"]` to the 13 Bedrock catalog rows whose AWS "Supported TTL" table entry lists 1h. The remaining rows stay absent, which is the fail-closed default. - Thread one resolved TTL through `CacheStrategyConfig.cacheTtl` into `CacheStrategy.createCachePoint`, and make that factory public so the provider's message-level emitter reuses it instead of building its own literal. - Resolve the TTL once in `createMessage` via the shared `resolveCacheTtl` helper and pass it to `convertToBedrockConverseMessages`, so both cache-point emitters (system/message blocks from the strategy, and the `messages[i].content` literal that used to hardcode `{ cachePoint: { type: "default" } }`) carry the same value. - Read the 1h share of `usage.cacheDetails` on the Converse stream and report it as `cacheWrite1hTokens`, clamped to the reported aggregate. Why: ADR-0006 PR-C7, the last request-side step of the TTL program. C5a (#2260) made a 1h cache write bill at 2x instead of the 5m rate and C5b (#2264) added the knob plus the capability field, but no Bedrock request could carry a TTL, so the knob was inert on the one provider that hard-fails on a wrong one. Impact: No behavior change by default: an unset or "5m" knob resolves to `undefined` and every cache point stays the byte-identical `{ type: "default" }`. A 1h request is emitted only when the resolved base model id is exactly listed, so a custom ARN, an inference profile naming an unlisted model, or any Nova row keeps today's bytes. Bedrock rejects an explicit ttl for a 5m-only model with a ValidationException that kills the turn, so the gate is capability-based and never error-driven: no catch-400-retry path exists. One TTL per request also makes the AWS "longer-before-shorter across tools -> system -> messages" ordering hazard unrepresentable rather than merely avoided. Cache-point placement is untouched. --- packages/types/src/providers/bedrock.ts | 21 + .../__tests__/bedrock-cache-ttl.spec.ts | 487 ++++++++++++++++++ src/api/providers/bedrock.ts | 43 +- .../__tests__/cache-strategy-ttl.spec.ts | 121 +++++ .../transform/cache-strategy/base-strategy.ts | 17 +- src/api/transform/cache-strategy/types.ts | 9 + 6 files changed, 691 insertions(+), 7 deletions(-) create mode 100644 src/api/providers/__tests__/bedrock-cache-ttl.spec.ts create mode 100644 src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index faef150a2a..e825b418a7 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -12,12 +12,21 @@ export const bedrockDefaultPromptRouterModelId: BedrockModelId = "anthropic.clau // https://aws.amazon.com/bedrock/pricing, including older models that are part // of the default prompt routers AWS enabled for GA of the promot router // feature. +// +// `promptCacheTtlSupport` marks the models whose Converse cache points accept an +// explicit `ttl` (ADR-0006 PR-C7). The 1h boundary is not a version rule: Opus 4.1 +// and Opus 4 are 5m-only while Opus 4.6/4.7/4.8 are 1h-capable, so the flag is set +// per exact model id from the AWS "Supported TTL" table and never inferred from a +// pattern. Rows without it are 5m-only, which makes Bedrock reject an explicit ttl +// (ValidationException), so absence must stay the fail-closed default. +// https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html export const bedrockModels = { "anthropic.claude-sonnet-4-5-20250929-v1:0": { maxTokens: 64_000, contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 3.0, outputPrice: 15.0, @@ -33,6 +42,7 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) @@ -60,6 +70,7 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -173,6 +184,7 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) @@ -199,6 +211,7 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console @@ -225,6 +238,7 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console @@ -254,6 +268,7 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -275,6 +290,7 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -300,6 +316,7 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -326,6 +343,7 @@ export const bedrockModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -345,6 +363,7 @@ export const bedrockModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -364,6 +383,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, outputPrice: 25.0, @@ -437,6 +457,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 1.0, outputPrice: 5.0, diff --git a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts new file mode 100644 index 0000000000..a2a69706d6 --- /dev/null +++ b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts @@ -0,0 +1,487 @@ +// npx vitest run src/api/providers/__tests__/bedrock-cache-ttl.spec.ts + +const mockCaptureException = vi.fn() + +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + instance: { + captureException: (...args: unknown[]) => mockCaptureException(...args), + }, + }, +})) + +vi.mock("@aws-sdk/credential-providers", () => { + const mockFromIni = vi.fn().mockReturnValue({ + accessKeyId: "profile-access-key", + secretAccessKey: "profile-secret-key", + }) + return { fromIni: mockFromIni } +}) + +vi.mock("../../../utils/networkProxy", () => ({ + getSystemProxyUrl: vi.fn().mockReturnValue(undefined), +})) + +vi.mock("@smithy/node-http-handler", () => ({ + NodeHttpHandler: vi.fn(), +})) + +vi.mock("http-proxy-agent", () => ({ + HttpProxyAgent: vi.fn(), +})) + +vi.mock("https-proxy-agent", () => ({ + HttpsProxyAgent: vi.fn(), +})) + +vi.mock("@aws-sdk/client-bedrock-runtime", async () => { + const actual = await vi.importActual( + "@aws-sdk/client-bedrock-runtime", + ) + const mockSend = vi.fn().mockResolvedValue({ stream: [] }) + const mockConverseStreamCommand = vi.fn() + + return { + ...actual, + BedrockRuntimeClient: vi.fn().mockImplementation(function () { + return { send: mockSend } + }), + ConverseStreamCommand: mockConverseStreamCommand, + ConverseCommand: vi.fn(), + } +}) + +import { AwsBedrockHandler } from "../bedrock" +import { ConverseStreamCommand } from "@aws-sdk/client-bedrock-runtime" +import { bedrockModels, BEDROCK_CUSTOM_ARN_OTHER_BASE_MODEL, type ProviderSettings } from "@roo-code/types" + +import type { Anthropic } from "@anthropic-ai/sdk" +import { clearAllMocks } from "../../../test-utils/reset" +import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" + +const mockConverseStreamCommand = vi.mocked(ConverseStreamCommand) + +/** + * Bedrock Converse cache-point TTL emission (ADR-0006 PR-C7). + * + * Every assertion here runs on the serialized `ConverseStreamCommand` input, i.e. + * the bytes that would reach AWS. Bedrock rejects an explicit `ttl` for a model + * that does not support it (`ValidationException`, 400, no fallback), and it also + * rejects a shorter TTL appearing before a longer one across tools -> system -> + * messages, so both "did we emit it everywhere" and "did we emit it nowhere" have + * to be checked by counting cache points rather than by spot-checking one. + */ + +/** Collect every `cachePoint` object reachable in a serialized request. */ +const collectCachePoints = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cachePoint" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +/** True when any `ttl` key exists anywhere in the serialized request. */ +const containsTtlKey = (value: unknown): boolean => { + if (Array.isArray(value)) return value.some(containsTtlKey) + if (value && typeof value === "object") { + return Object.entries(value as Record).some( + ([key, child]) => key === "ttl" || containsTtlKey(child), + ) + } + return false +} + +/** The serialized command input, so assertions run on the bytes that would be sent. */ +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) as Record + +// The system prompt must clear minTokensPerCachePoint (1024 for the Claude rows) so +// the strategy actually places a system cache point, and the message tail must clear +// it too so a message-level cache point is placed. Both emitters then appear in one +// request, which is the case a single-emitter assertion would miss. +const longSystemPrompt = "You are a helpful assistant. ".repeat(200) +const longUserMessage = "Please review this file. ".repeat(200) + +const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: longUserMessage }, + { role: "assistant", content: "Here is the review. ".repeat(200) }, + { role: "user", content: longUserMessage }, +] + +const createHandler = (overrides: Partial = {}) => + new AwsBedrockHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + ...overrides, + }) + +/** Run one createMessage and return the serialized ConverseStreamCommand input. */ +const captureConverseInput = async (handler: AwsBedrockHandler): Promise> => { + mockConverseStreamCommand.mockClear() + const stream = handler.createMessage(longSystemPrompt, messages) + await collectStream(stream) + + expect(mockConverseStreamCommand).toHaveBeenCalledTimes(1) + return serialize(mockConverseStreamCommand.mock.calls[0][0]) +} + +describe("Bedrock cache-point TTL emission", () => { + beforeEach(() => { + clearAllMocks() + mockCaptureException.mockClear() + }) + + describe("1h-capable model with the knob at 1h", () => { + it("carries ttl 1h on every cache point in the request", async () => { + const handler = createHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + const cachePoints = collectCachePoints(input) + + // Both emitters must be present, otherwise the "every cache point" claim is vacuous. + const systemCachePoints = collectCachePoints(input.system) + const messageCachePoints = collectCachePoints(input.messages) + expect(systemCachePoints.length).toBeGreaterThan(0) + expect(messageCachePoints.length).toBeGreaterThan(0) + expect(cachePoints.length).toBe(systemCachePoints.length + messageCachePoints.length) + + for (const cachePoint of cachePoints) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + + it("emits one uniform TTL per request, so the tools -> system -> messages ordering hazard stays unreachable", async () => { + const handler = createHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + const ttls = new Set(collectCachePoints(input).map((cachePoint) => cachePoint.ttl)) + + // A set of size 1 is the whole invariant: with one value there is no + // "shorter before longer" pair to reject, regardless of placement order. + expect(ttls).toEqual(new Set(["1h"])) + }) + + it("keeps the cache-point placement identical to the unset-knob request", async () => { + const capable = { apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" } as const + + const withoutKnob = createHandler(capable) + withoutKnob["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const baseInput = await captureConverseInput(withoutKnob) + + const withKnob = createHandler({ ...capable, promptCacheTtl: "1h" }) + withKnob["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const ttlInput = await captureConverseInput(withKnob) + + // Strip the ttl the knob adds and the two requests must be byte-identical: + // adding a TTL must not move a single cache point. + const stripTtl = (value: unknown): unknown => + JSON.parse(JSON.stringify(value, (key, child) => (key === "ttl" ? undefined : child))) + + expect(stripTtl(ttlInput)).toEqual(baseInput) + expect(collectCachePoints(baseInput).length).toBe(collectCachePoints(ttlInput).length) + }) + }) + + describe("5m-only models with the knob at 1h (ValidationException tripwire)", () => { + // AWS rejects an explicit ttl for these with a 400 that kills the turn, so the + // assertion is that no `ttl` key exists anywhere in the serialized request. + it.each([ + ["Opus 4.1 (4.x generation, 5m-only)", "anthropic.claude-opus-4-1-20250805-v1:0"], + ["Opus 4 (pre-4.1, 5m-only)", "anthropic.claude-opus-4-20250514-v1:0"], + ["Claude 3.7 Sonnet (3.x, 5m-only)", "anthropic.claude-3-7-sonnet-20250219-v1:0"], + ["Nova Pro (non-Anthropic, 5m-only)", "amazon.nova-pro-v1:0"], + ])("emits no ttl for %s", async (_label, apiModelId) => { + const handler = createHandler({ apiModelId, promptCacheTtl: "1h" }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(containsTtlKey(input)).toBe(false) + // The cache points themselves must still be there: the gate suppresses the + // ttl, not the caching. + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default" }) + } + }) + }) + + describe("knob unset (default parity)", () => { + it("serializes a byte-identical request to a pre-knob build for a capable model", async () => { + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(containsTtlKey(input)).toBe(false) + expect(collectCachePoints(input).length).toBeGreaterThan(0) + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default" }) + } + }) + + it("treats an explicit 5m knob as unset so the wire shape does not change", async () => { + const capable = { apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" } as const + + const unset = createHandler(capable) + unset["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const unsetInput = await captureConverseInput(unset) + + const explicit5m = createHandler({ ...capable, promptCacheTtl: "5m" }) + explicit5m["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const explicit5mInput = await captureConverseInput(explicit5m) + + expect(explicit5mInput).toEqual(unsetInput) + expect(containsTtlKey(explicit5mInput)).toBe(false) + }) + }) + + describe("custom ARN / inference-profile resolution (fail closed)", () => { + it("emits 1h for an inference profile whose resolved base id is exactly listed", async () => { + // `us.` is a cross-region prefix, not part of the catalog id, so this only + // works if the prefix is stripped before the capability lookup. + const handler = createHandler({ + apiModelId: "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(collectCachePoints(input).length).toBeGreaterThan(0) + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + + it("emits no ttl for an inference-profile ARN that names a 5m-only model", async () => { + // The ARN names Opus 4.1, which is 5m-only despite being a 4.x generation. + // Resolution succeeds (so cache points are placed), but the capability + // lookup must still refuse the TTL. + const handler = createHandler({ + awsCustomArn: + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-opus-4-1-20250805-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + // Cache points exist, so the assertion is about the gate rather than about + // caching being off for an unrelated reason. + expect(collectCachePoints(input).length).toBeGreaterThan(0) + expect(containsTtlKey(input)).toBe(false) + }) + + it("places no cache points at all when the ARN resolves to no catalog model", async () => { + // The deepest fail-closed path: an application inference profile the user + // marked as "other" resolves to no catalog entry, so the guessed ModelInfo + // reports no prompt-cache support and no capability can exist. + const handler = createHandler({ + awsCustomArn: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + awsCustomArnBaseModelId: BEDROCK_CUSTOM_ARN_OTHER_BASE_MODEL, + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(collectCachePoints(input)).toEqual([]) + expect(containsTtlKey(input)).toBe(false) + }) + + it("emits 1h for a foundation-model ARN that names a listed model", async () => { + const handler = createHandler({ + awsCustomArn: "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(collectCachePoints(input).length).toBeGreaterThan(0) + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + }) + + describe("usage cacheDetails split", () => { + it("reports the 1h share of the cache writes when Converse returns a split", async () => { + const handler = createHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 3000, + cacheDetails: [ + { ttl: "1h", inputTokens: 2000 }, + { ttl: "5m", inputTokens: 1000 }, + ], + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ + type: "usage", + cacheWriteTokens: 3000, + cacheWrite1hTokens: 2000, + }) + }) + + it("omits the 1h share when the response carries no cacheDetails", async () => { + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { metadata: { usage: { inputTokens: 10, outputTokens: 5, cacheWriteInputTokens: 3000 } } }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 3000 }) + expect((usage as { cacheWrite1hTokens?: number }).cacheWrite1hTokens).toBeUndefined() + }) + + it("clamps the 1h share to the reported aggregate so a malformed split cannot over-bill", async () => { + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 500, + cacheDetails: [{ ttl: "1h", inputTokens: 900 }], + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 500, cacheWrite1hTokens: 500 }) + }) + }) +}) + +describe("Bedrock cache TTL capability catalog", () => { + const oneHourIds = Object.entries(bedrockModels) + .filter(([, info]) => (info as { promptCacheTtlSupport?: string[] }).promptCacheTtlSupport?.includes("1h")) + .map(([id]) => id) + .sort() + + it("marks exactly the 1h-capable Claude rows the AWS TTL table lists", () => { + // Pinned by exact id, not by pattern: Opus 4.1 and Opus 4 are 5m-only while + // Opus 4.6/4.7/4.8 are 1h-capable, so a version heuristic mis-classifies the + // boundary in both directions. Adding a row here means adding it to the AWS + // table first. + expect(oneHourIds).toEqual( + [ + "anthropic.claude-fable-5", + "anthropic.claude-fable-5-1", + "anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-haiku-5-5", + "anthropic.claude-opus-4-5-20251101-v1:0", + "anthropic.claude-opus-4-6-v1", + "anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "anthropic.claude-opus-5", + "anthropic.claude-opus-5-5", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-5", + ].sort(), + ) + }) + + it("never marks a Nova row, because Nova rejects an extended TTL outright", () => { + // The observed error for a Nova model is "Extended TTL prompt caching is only + // supported for Anthropic models", and Nova Premier is unresolved in the AWS + // table, so no Nova row may claim 1h. + const novaRows = Object.entries(bedrockModels) + .filter(([id]) => id.startsWith("amazon.nova")) + .filter(([, info]) => (info as { promptCacheTtlSupport?: string[] }).promptCacheTtlSupport !== undefined) + .map(([id]) => id) + + expect(novaRows).toEqual([]) + }) + + it("leaves the 5m-only Claude rows without capability so they fail closed", () => { + const fiveMinuteOnly = [ + "anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic.claude-opus-4-1-20250805-v1:0", + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + ] + + for (const id of fiveMinuteOnly) { + const info = (bedrockModels as Record)[id] + expect(info, id).toBeDefined() + expect(info.promptCacheTtlSupport, id).toBeUndefined() + } + }) + + it("keys every 1h cache-write rate off the same tier's input price", () => { + // The tier-shadow bug class: a tier that overrides inputPrice must override + // cacheWritesPrice1h too, otherwise a long-context 1h write bills at the base + // tier's rate. Checked per entry and per tier, not just on the top-level row. + for (const [id, info] of Object.entries(bedrockModels)) { + const entry = info as { + inputPrice?: number + cacheWritesPrice1h?: number + tiers?: Array<{ inputPrice?: number; cacheWritesPrice1h?: number }> + } + if (entry.cacheWritesPrice1h !== undefined && entry.inputPrice !== undefined) { + expect(entry.cacheWritesPrice1h, id).toBeCloseTo(entry.inputPrice * 2, 9) + } + + for (const [index, tier] of (entry.tiers ?? []).entries()) { + if (tier.cacheWritesPrice1h === undefined) continue + expect(tier.inputPrice, `${id} tier ${index}`).toBeDefined() + expect(tier.cacheWritesPrice1h, `${id} tier ${index}`).toBeCloseTo((tier.inputPrice as number) * 2, 9) + } + } + }) +}) diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index d11f5f7dba..38d45c5946 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -3,12 +3,13 @@ import { ConverseStreamCommand, ConverseCommand, BedrockRuntimeClientConfig, - ContentBlock, Message, SystemContentBlock, Tool, ToolConfiguration, ToolChoice, + CacheTTL, + type CacheDetail, } from "@aws-sdk/client-bedrock-runtime" import { NodeHttpHandler } from "@smithy/node-http-handler" import OpenAI from "openai" @@ -22,6 +23,7 @@ import { type ProviderSettings, type BedrockModelId, type BedrockServiceTier, + type PromptCacheTtl, bedrockDefaultModelId, bedrockModels, bedrockDefaultPromptRouterModelId, @@ -47,6 +49,7 @@ import { logger } from "../../utils/logging" import { Package } from "../../shared/package" import { MultiPointStrategy } from "../transform/cache-strategy/multi-point-strategy" import { ModelInfo as CacheModelInfo } from "../transform/cache-strategy/types" +import { resolveCacheTtl } from "../transform/caching/ttl" import { convertToBedrockConverseMessages as sharedConverter } from "../transform/bedrock-converse-format" import { getModelParams } from "../transform/model-params" import { shouldUseReasoningBudget } from "../../shared/api" @@ -179,6 +182,9 @@ export interface StreamEvent { cacheWriteInputTokens?: number cacheReadInputTokenCount?: number cacheWriteInputTokenCount?: number + // Per-TTL split of the cache writes. Converse returns one entry per TTL + // class that received writes, longest TTL first. + cacheDetails?: CacheDetail[] } metrics?: { latencyMs: number @@ -210,6 +216,25 @@ export type UsageType = { cacheWriteInputTokens?: number cacheReadInputTokenCount?: number cacheWriteInputTokenCount?: number + cacheDetails?: CacheDetail[] +} + +/** + * Read the 1h share of a Converse `cacheDetails` split. + * + * Best effort by construction: the field is optional in the pinned SDK and absent + * on every pre-1h response, so a missing split means "no long-lived write was + * reported" and the cost layer prices the whole aggregate at the 5m rate. The + * counts are a decomposition of `cacheWriteInputTokens`, never an addition to it, + * so the value is clamped to the aggregate to keep a malformed response from + * billing more writes than were reported. + */ +function readCacheWrite1hTokens(usage: UsageType, cacheWriteTokens: number): number | undefined { + const oneHour = usage.cacheDetails?.find((detail) => detail.ttl === CacheTTL.ONE_HOUR)?.inputTokens + if (!oneHour || oneHour <= 0) { + return undefined + } + return Math.min(oneHour, cacheWriteTokens) } /************************************************************************************ @@ -448,6 +473,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH usePromptCache, modelConfig.info, conversationId, + resolveCacheTtl(this.options, modelConfig.info), ) let additionalModelRequestFields: BedrockAdditionalModelFields | undefined @@ -642,6 +668,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH outputTokens: usage.outputTokens || 0, cacheReadTokens: cacheReadTokens, cacheWriteTokens: cacheWriteTokens, + cacheWrite1hTokens: readCacheWrite1hTokens(usage, cacheWriteTokens), } continue } @@ -974,7 +1001,11 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH } /** - * Convert Anthropic messages to Bedrock Converse format + * Convert Anthropic messages to Bedrock Converse format. + * + * `cacheTtl` is resolved once by the caller and threaded into the strategy, so + * both cache-point emitters in this method (the strategy's system/message + * blocks and the message-level literal below) carry the same value. */ private convertToBedrockConverseMessages( anthropicMessages: Anthropic.Messages.MessageParam[] | { role: string; content: string }[], @@ -982,6 +1013,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH usePromptCache: boolean = false, modelInfo?: any, conversationId?: string, // Optional conversation ID to track cache points across messages + cacheTtl?: PromptCacheTtl, ): { system: SystemContentBlock[]; messages: Message[] } { // First convert messages using shared converter for proper image handling const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[]) @@ -1017,6 +1049,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH messages: anthropicMessages as Anthropic.Messages.MessageParam[], usePromptCache, previousCachePointPlacements: previousPlacements, + cacheTtl, } // Get cache point placements @@ -1028,13 +1061,15 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH this.previousCachePointPlacements[conversationId] = cacheResult.messageCachePointPlacements } - // Apply cache points to the properly converted messages + // Apply cache points to the properly converted messages. The strategy owns the + // cache-point shape, so the message-level emitter asks it rather than repeating + // the literal: that is what keeps both emitters on one TTL. const messagesWithCache = convertedMessages.map((msg, index) => { const placement = cacheResult.messageCachePointPlacements?.find((p) => p.index === index) if (placement) { return { ...msg, - content: [...(msg.content || []), { cachePoint: { type: "default" } } as ContentBlock], + content: [...(msg.content || []), strategy.createCachePoint()], } } return msg diff --git a/src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts b/src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts new file mode 100644 index 0000000000..e295d9ef6a --- /dev/null +++ b/src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts @@ -0,0 +1,121 @@ +// npx vitest run src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts + +import type { ContentBlock, SystemContentBlock } from "@aws-sdk/client-bedrock-runtime" + +import { MultiPointStrategy } from "../multi-point-strategy" +import type { CacheStrategyConfig, ModelInfo } from "../types" + +/** + * TTL threading through the cache strategy (ADR-0006 PR-C7). + * + * The strategy is the single owner of the cache-point shape, so these tests pin the + * two properties the provider relies on: a configured TTL reaches every cache point + * it emits, and an unset TTL leaves the block byte-identical to a pre-knob build. + */ + +const modelInfo: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + maxCachePoints: 4, + minTokensPerCachePoint: 50, + cachableFields: ["system", "messages", "tools"], +} + +// The strategy estimates tokens by counting whitespace-separated words, so the +// fixtures need real words rather than one long run of characters. +const longText = (tokens: number) => "word ".repeat(tokens) + +const createConfig = (overrides: Partial = {}): CacheStrategyConfig => ({ + modelInfo, + systemPrompt: longText(200), + messages: [ + { role: "user", content: longText(200) }, + { role: "assistant", content: longText(200) }, + { role: "user", content: longText(200) }, + ], + usePromptCache: true, + ...overrides, +}) + +const collectCachePoints = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cachePoint" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) + +describe("MultiPointStrategy cache TTL", () => { + it("carries the configured TTL on the system and message cache points", () => { + const strategy = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })) + const result = strategy.determineOptimalCachePoints() + + const cachePoints = collectCachePoints(serialize({ system: result.system, messages: result.messages })) + + expect(collectCachePoints(serialize(result.system)).length).toBeGreaterThan(0) + expect(collectCachePoints(serialize(result.messages)).length).toBeGreaterThan(0) + for (const cachePoint of cachePoints) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + + it("omits the ttl key entirely when no TTL is configured", () => { + const strategy = new MultiPointStrategy(createConfig()) + const result = strategy.determineOptimalCachePoints() + + const cachePoints = collectCachePoints(serialize({ system: result.system, messages: result.messages })) + + expect(cachePoints.length).toBeGreaterThan(0) + for (const cachePoint of cachePoints) { + // Deep-equality against the pre-knob literal, not just "no ttl": a stray + // extra key would change the request bytes just as surely. + expect(cachePoint).toEqual({ type: "default" }) + } + }) + + it("places the same cache points with and without a TTL", () => { + const withoutTtl = new MultiPointStrategy(createConfig()).determineOptimalCachePoints() + const withTtl = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })).determineOptimalCachePoints() + + expect(withTtl.messageCachePointPlacements).toEqual(withoutTtl.messageCachePointPlacements) + expect(collectCachePoints(serialize(withTtl.system)).length).toBe( + collectCachePoints(serialize(withoutTtl.system)).length, + ) + }) + + it("keeps createCachePoint public so the provider's second emitter can reuse it", () => { + // The provider's message-level emitter used to build its own literal, which is + // how two emitters drift apart. Asserting the method is reachable from outside + // the class is the cheap structural guard against it going back to private. + const strategy = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })) + const block = strategy.createCachePoint() as unknown as { cachePoint: Record } + + expect(block.cachePoint).toEqual({ type: "default", ttl: "1h" }) + expect(typeof (block as unknown as ContentBlock)).toBe("object") + }) + + it("returns an untyped-compatible block for the system union as well", () => { + const strategy = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })) + const block = strategy.createCachePoint() as unknown as SystemContentBlock + + expect(collectCachePoints(serialize(block))).toEqual([{ type: "default", ttl: "1h" }]) + }) +}) diff --git a/src/api/transform/cache-strategy/base-strategy.ts b/src/api/transform/cache-strategy/base-strategy.ts index 1bc05cdb84..740940d805 100644 --- a/src/api/transform/cache-strategy/base-strategy.ts +++ b/src/api/transform/cache-strategy/base-strategy.ts @@ -48,10 +48,21 @@ export abstract class CacheStrategy { } /** - * Create a cache point content block + * Create a cache point content block. + * + * `ttl` is omitted, not set to "5m", when no TTL was resolved: AWS documents + * omitting the field as the way to get the default 5-minute behaviour, and an + * explicit "5m" would change the bytes of every existing request. + * + * Public so the provider's message-level emitter can reuse it instead of + * building its own literal: one factory means one TTL per request (I4) and + * makes a mixed-TTL request unrepresentable rather than merely unlikely. */ - protected createCachePoint(): ContentBlock { - return { cachePoint: { type: "default" } } as unknown as ContentBlock + public createCachePoint(): ContentBlock { + const { cacheTtl } = this.config + return { + cachePoint: cacheTtl ? { type: "default", ttl: cacheTtl } : { type: "default" }, + } as ContentBlock } /** diff --git a/src/api/transform/cache-strategy/types.ts b/src/api/transform/cache-strategy/types.ts index 4215295d6a..94a02f9c44 100644 --- a/src/api/transform/cache-strategy/types.ts +++ b/src/api/transform/cache-strategy/types.ts @@ -1,5 +1,6 @@ import { Anthropic } from "@anthropic-ai/sdk" import { SystemContentBlock, Message } from "@aws-sdk/client-bedrock-runtime" +import type { PromptCacheTtl } from "@roo-code/types" /** * Information about a model's capabilities and constraints @@ -57,4 +58,12 @@ export interface CacheStrategyConfig { usePromptCache: boolean /** Previous cache point placements (for maintaining consistency across consecutive messages) */ previousCachePointPlacements?: CachePointPlacement[] + /** + * Cache TTL every cache point in this request carries, already resolved against + * the model's catalog capability. One value per request is the point: Bedrock + * Converse rejects a shorter TTL appearing before a longer one across + * tools -> system -> messages, so a per-point TTL would be a 400 waiting for the + * first mixed placement. Leaving it unset omits `ttl` from the wire entirely. + */ + cacheTtl?: PromptCacheTtl } From bed36d8922784ba076c5de9294cc97bdf029d9a1 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 07:21:59 +0900 Subject: [PATCH 08/12] fix(api,cost): hash long cache keys, clamp TTL splits, harden the TTL specs What: - openai-native: hash over-long task ids with sha256 hex instead of slicing to 64 chars. A slice drops the distinguishing tail, so two ids sharing a 64-char prefix collapsed into one routing bucket. sha256 hex is exactly 64 chars, so the result still fits OpenAI's cap. - openai-native: clamp the 12h write subset at zero as well as at the write total. A negative `ttl_12h_tokens` from a misbehaving gateway used to reach the Task accumulator and the webview as a negative count. - cost: cap the 1h and 12h write subsets at the remaining aggregate write total. A malformed split (1h + 12h > total) billed the excess; 150k reported 1h writes against a 100k total cost 0.9 instead of 0.6. - Task.spec: replace the fixed 50ms sleep with `vi.waitFor` on the captured telemetry call. The drain is fire-and-forget and touches the filesystem, so a fixed sleep races it on a loaded worker. - vercel-ai-gateway.spec: assert the TTL argument of the `addCacheBreakpoints` mock call. The transform is file-mocked, so `JSON.stringify(mockCreate)` can never contain `cache_control` and the old assertion could not fail. - ttl.spec: compare raw `JSON.stringify` output against literal expected strings in the byte-identical test. `toEqual` on round-tripped objects ignores key insertion order, so a key reorder passed while the wire bytes changed. Why: Seven CodeRabbit "Minor" threads on the C5a/C5b stack. Three are production correctness (cache-key collision, negative token count, overbilled split); three are assertions that could not fail. The seventh thread's premise, forwarding the TTL split into the telemetry recompute, already landed in b647ec7d8. Impact: Long task ids keep distinct cache buckets; malformed TTL splits can no longer over- or under-bill or leak negative counts. No request shape changes for the default path: the aggregate-only case still derives both subsets as zero. --- .../__tests__/openai-native-usage.spec.ts | 30 ++++++++++- .../providers/__tests__/openai-native.spec.ts | 36 +++++++++++++ .../__tests__/vercel-ai-gateway.spec.ts | 10 +++- src/api/providers/openai-native.ts | 12 +++-- .../transform/caching/__tests__/ttl.spec.ts | 51 ++++--------------- src/core/task/__tests__/Task.spec.ts | 10 +++- src/shared/cost.ts | 9 ++-- src/utils/__tests__/cost.spec.ts | 27 ++++++++-- 8 files changed, 128 insertions(+), 57 deletions(-) diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 44ac2deaf1..b79145efdd 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach } from "vitest" +import { createHash } from "crypto" import { OpenAiNativeHandler } from "../openai-native" import { OpenAiServiceTier, openAiNativeModels, type ModelInfo } from "@roo-code/types" import type { ApiStreamUsageChunk } from "../../transform/stream" @@ -495,7 +496,9 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { const longTaskId = "t".repeat(80) const body = buildRequestBody("gpt-5.1", longTaskId) expect(body.prompt_cache_key).toHaveLength(64) - expect(body.prompt_cache_key).toBe(longTaskId.slice(0, 64)) + // A hash, not a slice: truncation would collapse ids that share a prefix. + expect(body.prompt_cache_key).toBe(createHash("sha256").update(longTaskId).digest("hex")) + expect(body.prompt_cache_key).not.toBe(longTaskId.slice(0, 64)) }) it("should keep prompt_cache_retention alongside the key for 24h models", () => { @@ -761,5 +764,30 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { // Total: 0.5 + 1.25 = 1.75 expect(result.totalCost).toBeCloseTo(1.75, 6) }) + + it("should clamp a negative 12h split at zero instead of emitting a negative count", () => { + // A misbehaving gateway can report a negative ttl_12h_tokens. Math.min alone + // leaves it negative, so the negative count reaches the Task accumulator and + // the webview. Shared costing clamps, but the emitted chunk must not lie. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_12h_tokens: -40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // A zero split is indistinguishable from "no split", so the field stays unset. + expect(result.cacheWrite12hTokens).toBeUndefined() + expect(result.cacheWrite12hTokens ?? 0).toBeGreaterThanOrEqual(0) + // All 100k writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) }) }) diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index b7485bb65b..32099e7818 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -1507,6 +1507,42 @@ describe("OpenAiNativeHandler", () => { } }) + it("keeps two over-long task ids distinct instead of truncating them to one bucket", async () => { + // Task ids are uuidv7 today (36 chars), so this guards the wire contract for + // any caller that sends a longer id: a 64-char slice drops the distinguishing + // tail, so two ids sharing a 64-char prefix would land in one routing bucket. + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + // 70 characters: identical for the first 64, different in the tail. + const sharedPrefix = "a".repeat(64) + const longTaskIds = [`${sharedPrefix}-one`, `${sharedPrefix}-two`] + + for (const taskId of longTaskIds) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(2) + // OpenAI rejects keys over 64 characters, so the key must still fit. + expect(bodies[0].prompt_cache_key.length).toBeLessThanOrEqual(64) + expect(bodies[1].prompt_cache_key.length).toBeLessThanOrEqual(64) + // ...but the two tasks must not collide on that shortened form. + expect(bodies[1].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + }) + it("sends the task-scoped prompt_cache_key through the SDK path", async () => { mockResponsesCreate.mockResolvedValue(asyncStreamFrom([])) diff --git a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts index 687607d305..0231d34fd1 100644 --- a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts +++ b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts @@ -452,6 +452,7 @@ describe("VercelAiGatewayHandler", () => { it("does not emit a ttl for a gateway model with no TTL capability", async () => { // The gateway catalog is fetched at runtime and carries no TTL capability, // so a 1h knob must fail closed (ADR-0006 I2). + const { addCacheBreakpoints } = await import("../../transform/caching/vercel-ai-gateway") const handler = new VercelAiGatewayHandler( makeApiHandlerOptions({ ...mockOptions, @@ -462,8 +463,13 @@ describe("VercelAiGatewayHandler", () => { await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next() - const call = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0] - expect(JSON.stringify(call)).not.toContain("ttl") + // `addCacheBreakpoints` is file-mocked, so the request body never receives + // `cache_control` and `JSON.stringify(mockCreate...)` cannot see the TTL. + // The TTL only exists as the third argument to the mocked call. + expect(addCacheBreakpoints).toHaveBeenCalled() + const lastCall = vitest.mocked(addCacheBreakpoints).mock.calls.at(-1) + expect(lastCall).toBeDefined() + expect(lastCall![2]).toBeUndefined() }) it("sets correct max_completion_tokens", async () => { diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index be0a1bc12a..d4c5f070cd 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -1,4 +1,5 @@ import * as os from "os" +import { createHash } from "crypto" import { v7 as uuidv7 } from "uuid" import { Anthropic } from "@anthropic-ai/sdk" import OpenAI from "openai" @@ -162,8 +163,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0 // Clamp the 12h subset to the write total so a malformed response cannot bill - // more 12h tokens than were written at all. - const cacheWrite12hTokens = Math.min(writes12hFromDetails, cacheWriteTokens) + // more 12h tokens than were written at all. The lower clamp matters too: a + // negative `ttl_12h_tokens` from a misbehaving gateway would otherwise reach the + // Task accumulator and the webview as a negative count. + const cacheWrite12hTokens = Math.max(0, Math.min(writes12hFromDetails, cacheWriteTokens)) // Resolve effective tier: prefer actual tier from response; otherwise requested tier const effectiveTier = @@ -1472,7 +1475,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const key = metadata?.taskId || this.sessionId if (!key) return undefined - return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? key.slice(0, PROMPT_CACHE_KEY_MAX_LENGTH) : key + // Hash rather than slice: a 64-char prefix drops the distinguishing tail, so two + // distinct task ids that share a long prefix would land in one routing bucket. + // sha256 hex is 64 characters, so the result fits the cap without truncation. + return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? createHash("sha256").update(key).digest("hex") : key } /** diff --git a/src/api/transform/caching/__tests__/ttl.spec.ts b/src/api/transform/caching/__tests__/ttl.spec.ts index ab11d64b4d..7c4253320c 100644 --- a/src/api/transform/caching/__tests__/ttl.spec.ts +++ b/src/api/transform/caching/__tests__/ttl.spec.ts @@ -116,51 +116,20 @@ describe("caching transforms under a prompt cache TTL", () => { const vertexMessages = addVertexBreakpoints(buildAnthropicMessages()) - expect(serialize(anthropicMessages)).toEqual( - serialize([ - { - role: "system", - content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], - }, - { - role: "user", - content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], - }, - { role: "assistant", content: "First response" }, - { - role: "user", - content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], - }, - ]), + // Compare the raw serialized bytes, not parsed objects: `toEqual` ignores key + // insertion order, so a transform that reorders request keys would pass while + // the bytes on the wire changed. The expected strings are literals for the + // same reason: building them from the same helper would re-derive the order. + expect(JSON.stringify(anthropicMessages)).toBe( + '[{"role":"system","content":[{"type":"text","text":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}}]},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', ) - expect(serialize(gatewayMessages)).toEqual( - serialize([ - { role: "system", content: systemPrompt, cache_control: { type: "ephemeral" } }, - { - role: "user", - content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], - }, - { role: "assistant", content: "First response" }, - { - role: "user", - content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], - }, - ]), + expect(JSON.stringify(gatewayMessages)).toBe( + '[{"role":"system","content":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', ) - expect(serialize(vertexMessages)).toEqual( - serialize([ - { - role: "user", - content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], - }, - { role: "assistant", content: "First response" }, - { - role: "user", - content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], - }, - ]), + expect(JSON.stringify(vertexMessages)).toBe( + '[{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', ) }) }) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 122d1af8bc..dab89538fa 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -567,8 +567,14 @@ describe("Cline", () => { await task.recursivelyMakeClineRequests([{ type: "text", text: "original user request" }]) - // Allow the fire-and-forget drain to settle. - await new Promise((resolve) => setTimeout(resolve, 50)) + // The drain is fire-and-forget and reaches `captureLlmCompletion` after several + // awaits, some of which touch the filesystem. A fixed sleep races that work on a + // loaded worker, so poll for the captured call instead. + await vi.waitFor(() => + expect( + captureLlmCompletion.mock.calls.some(([, payload]) => payload?.cacheWriteTokens === 100_000), + ).toBe(true), + ) const call = captureLlmCompletion.mock.calls.find(([, payload]) => payload?.cacheWriteTokens === 100_000) expect(call).toBeDefined() diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 7fcc4e0d13..bc1f2f64d8 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -86,9 +86,12 @@ function calculateApiCostInternal( // The per-TTL counts are a decomposition of `cacheCreationInputTokens`, never // an addition to it. Deriving the 5m remainder this way keeps the aggregate // path byte-identical when a vendor reports no split (both terms are 0). - const writes1h = Math.max(0, cacheCreation1hInputTokens) - const writes12h = Math.max(0, cacheCreation12hInputTokens) - const writes5m = Math.max(0, cacheCreationInputTokens - writes1h - writes12h) + // Each subset also caps at what the aggregate has left, so a malformed split + // (1h + 12h > total) cannot bill more writes than the vendor reported. + const writesTotal = Math.max(0, cacheCreationInputTokens) + const writes1h = Math.min(Math.max(0, cacheCreation1hInputTokens), writesTotal) + const writes12h = Math.min(Math.max(0, cacheCreation12hInputTokens), writesTotal - writes1h) + const writes5m = Math.max(0, writesTotal - writes1h - writes12h) const writes5mRate = modelInfo.cacheWritesPrice || 0 const writes1hRate = resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice1h, CACHE_WRITE_MULTIPLIER_1H) || 0 diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index 304a1fb21b..6782de5e6f 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -358,15 +358,32 @@ describe("Cost Utility", () => { expect(result.totalCost).toBeCloseTo(0.5175, 9) }) - it("clamps the 5m remainder at zero when the split exceeds the total", () => { + it("caps each TTL class at the aggregate write total", () => { // A malformed response claiming more 1h tokens than the write total must - // not produce negative 5m writes (which would refund cost). + // not bill the excess. The 5m remainder already clamps at zero, but the 1h + // and 12h classes were billed at their full reported count. const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 150_000) - // 5m remainder clamps to 0; 1h is capped by the clamp on the remainder only, - // so the 1h class still bills its reported 150k at 6.0 => 0.9 - expect(result.totalCost).toBeCloseTo(0.9, 9) + // 1h writes cap at the 100k aggregate; the 5m remainder clamps to 0. + // 100k at 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) expect(result.totalCost).toBeGreaterThan(0) + // The pre-fix behaviour billed all 150k at 6.0 => 0.9. + expect(result.totalCost).not.toBeCloseTo(0.9, 9) + }) + + it("caps the 12h class at the remainder left after the 1h class", () => { + // 1h and 12h are a decomposition of one aggregate, so they cannot both + // claim the whole total. 1h wins the ordering; 12h gets what is left. + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 60_000, 80_000) + + // 1h: 60k at 6.0 => 0.36 + // 12h: capped at 100k - 60k = 40k at 7.5 => 0.3 + // 5m: 100k - 60k - 40k = 0 + // Total: 0.66 (the pre-fix behaviour billed 80k 12h => 0.36 + 0.6 = 0.96) + expect(result.totalCost).toBeCloseTo(0.66, 9) + expect(result.totalCost).not.toBeCloseTo(0.96, 9) }) it("is byte-identical to the pre-split cost when no split is reported", () => { From 6fde2ef492c3f1fd5d57695ad3d68a18fe11275e Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 07:28:05 +0900 Subject: [PATCH 09/12] test(bedrock): kill the cacheDetails reader mutants the gate found surviving What: - Add a 3-row `it.each` covering the malformed split shapes the reader must absorb: a 5m entry with no 1h entry, a negative 1h count, and a zero 1h count. - Add a split ordered 5m-first so the reader is proven to match on the ttl value rather than trusting the entry's position. Why: CI's mutation-diff run reported four survivors inside `readCacheWrite1hTokens`. Re-run of Stryker on the same span confirmed all four, and three are real gaps rather than equivalent mutants: the optional chain before `.inputTokens` was never exercised without a 1h entry, the `ttl === ONE_HOUR` predicate was only ever fed a 1h-first array (so "return the first entry" passed), and the `<= 0` guard was never fed a negative count. The fourth (`<= 0` -> `< 0`) is equivalent because `!oneHour` already rejects 0. Impact: Test-only. `readCacheWrite1hTokens` is unchanged and still clamps to the reported aggregate. Stryker on the same span now kills 14/15, with the single survivor being the equivalent `<= 0` -> `< 0` mutant. No production line, price row, or suppression count changes. --- .../__tests__/bedrock-cache-ttl.spec.ts | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts index a2a69706d6..9a2d6eaac6 100644 --- a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts +++ b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts @@ -379,6 +379,66 @@ describe("Bedrock cache-point TTL emission", () => { expect((usage as { cacheWrite1hTokens?: number }).cacheWrite1hTokens).toBeUndefined() }) + it.each([ + ["a 5m entry and no 1h entry", [{ ttl: "5m", inputTokens: 1000 }]], + ["a negative 1h count", [{ ttl: "1h", inputTokens: -5 }]], + ["a zero 1h count", [{ ttl: "1h", inputTokens: 0 }]], + ])("omits the 1h share when the split carries %s", async (_label, cacheDetails) => { + // Each row is a distinct malformed shape the reader must absorb rather than + // bill: an absent 1h entry (optional chain), a negative count (the `<= 0` + // guard, which `!oneHour` alone does not cover), and a zero count. + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 3000, + cacheDetails, + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 3000 }) + expect((usage as { cacheWrite1hTokens?: number }).cacheWrite1hTokens).toBeUndefined() + }) + + it("reads the 1h entry wherever it sits in the split, not just the first entry", async () => { + // Converse orders the split longest-TTL-first today, but the reader must match + // on the ttl value rather than trusting position: a 5m entry ahead of the 1h + // entry would otherwise report the 5m count as the 1h share. + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 3000, + cacheDetails: [ + { ttl: "5m", inputTokens: 1000 }, + { ttl: "1h", inputTokens: 2000 }, + ], + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 3000, cacheWrite1hTokens: 2000 }) + }) + it("clamps the 1h share to the reported aggregate so a malformed split cannot over-bill", async () => { const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) handler["client"].send = vi.fn().mockResolvedValue({ From 072cdc721690804b987feb9d145944fc39081684 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 08:58:40 +0900 Subject: [PATCH 10/12] fix(api,core): key prompt enhancement off the task id, not the handler session `completePrompt` built its `prompt_cache_key` from `this.sessionId`, which is regenerated for every handler instance. Prompt enhancement builds a fresh handler per request (`singleCompletionHandler`), so repeated enhancements of one task each started a new cache bucket and could not reuse the previous prefix. `CompletePromptOptions` now carries an optional `taskId`. `MessageEnhancer` sets it from the open task, `webviewMessageHandler` passes `currentCline?.taskId`, and the OpenAI native handler feeds it through the existing `getPromptCacheKey`, which already prefers `metadata.taskId` and falls back to the session id. Callers with no task keep the old behaviour. Tests: two new provider cases (task-scoped key across handler instances, session fallback without a task id) and two enhancer cases (task id forwarded, options omitted when no task is open). The provider case fails on the pre-fix code with the session uuid in place of the task id. --- src/api/index.ts | 8 ++++ .../providers/__tests__/openai-native.spec.ts | 30 +++++++++++++++ src/api/providers/openai-native.ts | 12 ++++-- .../webview/__tests__/messageEnhancer.test.ts | 38 ++++++++++++++++++- src/core/webview/messageEnhancer.ts | 13 ++++++- src/core/webview/webviewMessageHandler.ts | 1 + 6 files changed, 95 insertions(+), 7 deletions(-) diff --git a/src/api/index.ts b/src/api/index.ts index d61eea033f..e737ebd4bd 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -61,6 +61,14 @@ import { NativeOllamaHandler } from "./providers/native-ollama" export interface CompletePromptOptions extends Pick { /** Optional timeout override (ms) — falls back to provider default if omitted */ timeoutMs?: number + /** + * Task the completion belongs to, when it belongs to one. + * + * Providers that key cache affinity off a body field use this to route repeated + * completions for one task to the same bucket. Omit it when the completion is not + * scoped to a task; providers then fall back to a per-handler key. + */ + taskId?: string } export interface SingleCompletionHandler { diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 32099e7818..c7e3663efb 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -1560,6 +1560,36 @@ describe("OpenAiNativeHandler", () => { expect(mockResponsesCreate.mock.calls[0][1].headers).not.toHaveProperty("session_id") }) + it("keys completePrompt off the task id when the caller supplies one", async () => { + // Prompt enhancement builds a fresh handler per request, so a session-scoped key + // would start a new cache bucket every time. The task id keeps repeated + // enhancements of one task on the same bucket. + mockResponsesCreate.mockResolvedValue({ output: [] }) + + const first = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + const second = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + await first.completePrompt("enhance me", { taskId: "task-enhance" }) + await second.completePrompt("enhance me", { taskId: "task-enhance" }) + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies).toHaveLength(2) + expect(bodies[0].prompt_cache_key).toBe("task-enhance") + // Different handler instances, same task: the key must not drift. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + }) + + it("falls back to the handler session id when completePrompt gets no task id", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + await cacheHandler.completePrompt("enhance me") + + const [body] = mockResponsesCreate.mock.calls[0] + expect(body.prompt_cache_key).toBe(cacheHandler["sessionId"]) + }) + it("should provide helpful error messages for different error codes", async () => { const testCases = [ { status: 400, expectedMessage: "Invalid request to Responses API" }, diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index d4c5f070cd..6feb0ae75f 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -1572,7 +1572,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio getResponseId(): string | undefined { return this.lastResponseId } - async completePrompt(prompt: string, _options?: CompletePromptOptions): Promise { + async completePrompt(prompt: string, options?: CompletePromptOptions): Promise { try { this.abortController = new AbortController() @@ -1603,9 +1603,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio requestBody[SERVICE_TIER_KEY] = serviceTier } - // Cache routing affinity key. `completePrompt` has no task metadata, so the handler - // session id scopes the key: one bucket per handler instance, never shared across tasks. - const promptCacheKey = this.getPromptCacheKey(model) + // Cache routing affinity key. A caller that knows the task passes it, so repeated + // completions for one task share a bucket across handler instances; otherwise the + // handler session id scopes the key to this instance. + const promptCacheKey = this.getPromptCacheKey( + model, + options?.taskId ? { taskId: options.taskId } : undefined, + ) if (promptCacheKey) { requestBody.prompt_cache_key = promptCacheKey } diff --git a/src/core/webview/__tests__/messageEnhancer.test.ts b/src/core/webview/__tests__/messageEnhancer.test.ts index 964c4ebdef..6bf16b66e8 100644 --- a/src/core/webview/__tests__/messageEnhancer.test.ts +++ b/src/core/webview/__tests__/messageEnhancer.test.ts @@ -77,6 +77,7 @@ describe("MessageEnhancer", () => { expect(mockSingleCompletionHandler).toHaveBeenCalledWith( mockApiConfiguration, expect.stringContaining("Write a function to calculate fibonacci"), + undefined, ) }) @@ -99,7 +100,7 @@ describe("MessageEnhancer", () => { apiKey: "enhancement-key", apiModelId: "claude-3", } - expect(mockSingleCompletionHandler).toHaveBeenCalledWith(expectedConfig, expect.any(String)) + expect(mockSingleCompletionHandler).toHaveBeenCalledWith(expectedConfig, expect.any(String), undefined) }) it("should include task history when enabled", async () => { @@ -238,7 +239,40 @@ describe("MessageEnhancer", () => { }) // Should use the default config - expect(mockSingleCompletionHandler).toHaveBeenCalledWith(mockApiConfiguration, expect.any(String)) + expect(mockSingleCompletionHandler).toHaveBeenCalledWith( + mockApiConfiguration, + expect.any(String), + undefined, + ) + }) + + it("passes the task id through to the completion so the cache key stays task-scoped", async () => { + await MessageEnhancer.enhanceMessage({ + text: "Test", + apiConfiguration: mockApiConfiguration, + listApiConfigMeta: mockListApiConfigMeta, + providerSettingsManager: mockProviderSettingsManager, + taskId: "task-123", + }) + + expect(mockSingleCompletionHandler).toHaveBeenCalledWith(mockApiConfiguration, expect.any(String), { + taskId: "task-123", + }) + }) + + it("omits the completion options when no task is open", async () => { + await MessageEnhancer.enhanceMessage({ + text: "Test", + apiConfiguration: mockApiConfiguration, + listApiConfigMeta: mockListApiConfigMeta, + providerSettingsManager: mockProviderSettingsManager, + }) + + expect(mockSingleCompletionHandler).toHaveBeenCalledWith( + mockApiConfiguration, + expect.any(String), + undefined, + ) }) it("should handle empty task history gracefully", async () => { diff --git a/src/core/webview/messageEnhancer.ts b/src/core/webview/messageEnhancer.ts index 90c1f3b37d..6994e57291 100644 --- a/src/core/webview/messageEnhancer.ts +++ b/src/core/webview/messageEnhancer.ts @@ -13,6 +13,12 @@ export interface MessageEnhancerOptions { includeTaskHistoryInEnhance?: boolean currentClineMessages?: ClineMessage[] providerSettingsManager: ProviderSettingsManager + /** + * Task the enhancement belongs to, when one is open. Passed through to the provider so + * repeated enhancements of one task reuse a single prompt-cache bucket instead of + * starting a new one per handler instance. + */ + taskId?: string } export interface MessageEnhancerResult { @@ -41,6 +47,7 @@ export class MessageEnhancer { includeTaskHistoryInEnhance, currentClineMessages, providerSettingsManager, + taskId, } = options // Determine which API configuration to use @@ -76,7 +83,11 @@ export class MessageEnhancer { ) // Call the single completion handler to get the enhanced prompt - const enhancedText = await singleCompletionHandler(configToUse, enhancementPrompt) + const enhancedText = await singleCompletionHandler( + configToUse, + enhancementPrompt, + taskId ? { taskId } : undefined, + ) return { success: true, diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index bd7c589958..5da8019187 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -2252,6 +2252,7 @@ export const webviewMessageHandler = async ( includeTaskHistoryInEnhance, currentClineMessages: currentCline?.clineMessages, providerSettingsManager: provider.providerSettingsManager, + taskId: currentCline?.taskId, }) if (result.success && result.enhancedText) { From f5aa3e1d64021f46bf9f8ad7f0d6e1eeb6c871d1 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 09:01:16 +0900 Subject: [PATCH 11/12] fix(bedrock): fail closed on unlisted models and unpriced 1h tiers What: - bedrock: clear `promptCacheTtlSupport` in the `guessModelInfoFromId` fallback. The fallback spreads `bedrockModels[bedrockDefaultModelId]`, which is 1h-capable, and then overlays the guess, so an id that resolves to no catalog row inherited `["5m","1h"]` and could emit `ttl: "1h"` for a model we cannot prove accepts it. An unlisted id is now 5m-only (ADR-0006 I2). - bedrock-cache-ttl.spec: add the regression case for that path, and make the tier-rate invariant fail when a tier reprices `inputPrice` without a `cacheWritesPrice1h` instead of skipping it. Why: CodeRabbit round 1 on PR #2270, threads KJYW (Major) and KJYK (Minor). The KJYW premise is real: a custom inference-profile ARN naming an unlisted model (for example `us.anthropic.claude-3-5-sonnet-20241022-v1:0`, which has no catalog row) produced `ttlSupport: ["5m","1h"]` on head. Bedrock answers an unsupported ttl with a ValidationException that kills the turn, so this was a turn-breaking path rather than a pricing nit. Impact: No change for the 48 catalog rows or for any listed ARN, inference profile, or prompt-router path: only the unresolvable-id fallback loses capability. The tier guard is a test-only tightening that currently passes on all four tiers. Red-on-head: with the production hunk reverted and the new test kept, `bedrock-cache-ttl.spec.ts` fails exactly one case ("emits no ttl for an unlisted model that the heuristic fallback guesses") and passes the other 24. With the tier guard reverted and a tier 1h rate temporarily removed from the catalog, the catalog invariant fails on `anthropic.claude-sonnet-4-6 tier 0`. --- .../__tests__/bedrock-cache-ttl.spec.ts | 34 +++++++++++++++++-- src/api/providers/bedrock.ts | 14 ++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts index 9a2d6eaac6..b3dcb86929 100644 --- a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts +++ b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts @@ -314,6 +314,26 @@ describe("Bedrock cache-point TTL emission", () => { expect(containsTtlKey(input)).toBe(false) }) + it("emits no ttl for an unlisted model that the heuristic fallback guesses", async () => { + // The fallback spreads the default row (which is 1h-capable) and then overlays + // the guess, so the capability is inherited unless it is cleared. An unlisted + // id is exactly the case where we cannot prove the model accepts 1h, and Bedrock + // answers an unsupported ttl with a ValidationException that kills the turn. + const handler = createHandler({ + awsCustomArn: + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + // The guess still reports caching support, so cache points exist and the + // assertion is about the capability gate rather than caching being off. + expect(collectCachePoints(input).length).toBeGreaterThan(0) + expect(containsTtlKey(input)).toBe(false) + }) + it("emits 1h for a foundation-model ARN that names a listed model", async () => { const handler = createHandler({ awsCustomArn: "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0", @@ -538,9 +558,17 @@ describe("Bedrock cache TTL capability catalog", () => { } for (const [index, tier] of (entry.tiers ?? []).entries()) { - if (tier.cacheWritesPrice1h === undefined) continue - expect(tier.inputPrice, `${id} tier ${index}`).toBeDefined() - expect(tier.cacheWritesPrice1h, `${id} tier ${index}`).toBeCloseTo((tier.inputPrice as number) * 2, 9) + // A tier that reprices input without repricing the 1h write is exactly the + // shadow this test exists to catch, so it must fail rather than be skipped. + if (tier.inputPrice === undefined) { + expect( + tier.cacheWritesPrice1h, + `${id} tier ${index} prices a 1h write without an input price`, + ).toBeUndefined() + continue + } + expect(tier.cacheWritesPrice1h, `${id} tier ${index} reprices input but not the 1h write`).toBeDefined() + expect(tier.cacheWritesPrice1h, `${id} tier ${index}`).toBeCloseTo(tier.inputPrice * 2, 9) } } }) diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index 38d45c5946..c72dc8f360 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -1247,13 +1247,15 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH } else { // Use heuristics for model info, then allow overrides from ProviderSettings const guessed = this.guessModelInfoFromId(modelId) - model = { - id: bedrockDefaultModelId, - info: { - ...JSON.parse(JSON.stringify(bedrockModels[bedrockDefaultModelId])), - ...guessed, - }, + const info: ModelInfo = { + ...JSON.parse(JSON.stringify(bedrockModels[bedrockDefaultModelId])), + ...guessed, } + // The default row is 1h-capable, but an id we cannot resolve is not. Inheriting + // the capability would send `ttl: "1h"` to a model that may reject it with a + // ValidationException that kills the turn, so an unlisted id fails closed (I2). + delete info.promptCacheTtlSupport + model = { id: bedrockDefaultModelId, info } } // Always allow user to override detected/guessed maxTokens and contextWindow From 9f1f7301a56b853bc56f4cbf090da803e9a31d35 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 09:02:29 +0900 Subject: [PATCH 12/12] test(bedrock): scope the tier 1h guard to rows that price a 1h write What: - Narrow the tier invariant: a tier that omits `cacheWritesPrice1h` now fails only when the entry itself prices a 1h write, so a 5m-only row's tier has no rate to shadow and is exempt. Why: The first cut required a 1h rate on every tier that repriced `inputPrice`, which would have failed a future 5m-only model that gains a long-context tier. CodeRabbit's inline suggestion on thread KJYK named the correct exemption. Impact: Test-only. Still fails on the shadow it targets: removing the 1h rate from `anthropic.claude-sonnet-4-6`'s 1M tier fails the case with "reprices input but not the 1h write". All four current tiers pass. --- .../providers/__tests__/bedrock-cache-ttl.spec.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts index b3dcb86929..45a7a7db46 100644 --- a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts +++ b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts @@ -560,15 +560,17 @@ describe("Bedrock cache TTL capability catalog", () => { for (const [index, tier] of (entry.tiers ?? []).entries()) { // A tier that reprices input without repricing the 1h write is exactly the // shadow this test exists to catch, so it must fail rather than be skipped. - if (tier.inputPrice === undefined) { + // The exemption is only for rows that price no 1h write at all: a 5m-only + // model's tier has no 1h rate to shadow. + if (tier.cacheWritesPrice1h === undefined) { expect( - tier.cacheWritesPrice1h, - `${id} tier ${index} prices a 1h write without an input price`, - ).toBeUndefined() + tier.inputPrice === undefined || entry.cacheWritesPrice1h === undefined, + `${id} tier ${index} reprices input but not the 1h write`, + ).toBe(true) continue } - expect(tier.cacheWritesPrice1h, `${id} tier ${index} reprices input but not the 1h write`).toBeDefined() - expect(tier.cacheWritesPrice1h, `${id} tier ${index}`).toBeCloseTo(tier.inputPrice * 2, 9) + expect(tier.inputPrice, `${id} tier ${index}`).toBeDefined() + expect(tier.cacheWritesPrice1h, `${id} tier ${index}`).toBeCloseTo((tier.inputPrice as number) * 2, 9) } } })