From 6fd6fad0b6bb3accee212c57c386cd3db49093eb Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 00:51:01 +0900 Subject: [PATCH 01/14] fix(openai-native): send body prompt_cache_key for cache affinity What: the OpenAI-native handler now sends a task-scoped `prompt_cache_key` in the Responses request body on both body builders (streaming and completePrompt), and no longer sends the `session_id` request header. The key is `metadata.taskId`, falling back to the handler session id, clamped to OpenAI's 64-character limit, and omitted for models whose catalog entry has `supportsPromptCache: false`. Why: OpenAI keys prompt-cache routing off the body `prompt_cache_key` field. The `session_id` header we were sending is not part of the prompt-caching API and is ignored on the public Responses API, so every OpenAI-native task was leaving cache hit-rate on the table. The header also carries an underscore in its name, which hardened gateways (Envoy/nginx `underscores_in_headers`) reject outright, so it was a latent 400 for anyone behind such a proxy with a custom base URL. `prompt_cache_retention: "24h"` is untouched: it is the supported lifetime control on the pre-5.6 models that set it in the catalog, and the existing retention specs pass unmodified. Impact: cache affinity now follows the documented body field, stable across turns of one task and distinct across tasks. No usage accounting, cost, or other request field changes. openai-codex.ts and xai.ts are untouched: codex already sends a body key, and xai never had the header. --- .../__tests__/openai-native-usage.spec.ts | 64 +++++++++++++++ .../providers/__tests__/openai-native.spec.ts | 78 +++++++++++++++++++ src/api/providers/openai-native.ts | 64 ++++++++++++--- 3 files changed, 197 insertions(+), 9 deletions(-) diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 8922c2cf9c..2b05ab5b2b 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -440,6 +440,70 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { }) }) + describe("OpenAiNativeHandler - prompt cache key", () => { + let handler: OpenAiNativeHandler + + beforeEach(() => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + }) + }) + + const buildRequestBody = (modelId: string, taskId?: string) => { + handler["options"].apiModelId = modelId + const model = handler.getModel() + const metadata = taskId === undefined ? undefined : { taskId } + return handler["buildRequestBody"](model, [], "", model.verbosity, undefined, metadata) + } + + it("should send the task id as the body prompt_cache_key", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + }) + + it("should keep the key stable across turns of the same task", () => { + const firstTurn = buildRequestBody("gpt-5.1", "task-stable") + const secondTurn = buildRequestBody("gpt-5.1", "task-stable") + expect(firstTurn.prompt_cache_key).toBe("task-stable") + expect(secondTurn.prompt_cache_key).toBe(firstTurn.prompt_cache_key) + }) + + it("should give distinct keys to distinct tasks", () => { + const first = buildRequestBody("gpt-5.1", "task-one") + const second = buildRequestBody("gpt-5.1", "task-two") + expect(first.prompt_cache_key).toBe("task-one") + expect(second.prompt_cache_key).toBe("task-two") + expect(first.prompt_cache_key).not.toBe(second.prompt_cache_key) + }) + + it("should fall back to the handler session id when no task id is supplied", () => { + const body = buildRequestBody("gpt-5.1") + expect(body.prompt_cache_key).toBe(handler["sessionId"]) + expect(body.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the key when the model does not support prompt caching", () => { + const modelId = "codex-mini-latest" + expect(openAiNativeModels[modelId as keyof typeof openAiNativeModels].supportsPromptCache).toBe(false) + + const body = buildRequestBody(modelId, "task-abc") + expect(body.prompt_cache_key).toBeUndefined() + }) + + it("should clamp an over-length task id to OpenAI's 64-character limit", () => { + const longTaskId = "t".repeat(80) + const body = buildRequestBody("gpt-5.1", longTaskId) + expect(body.prompt_cache_key).toHaveLength(64) + expect(body.prompt_cache_key).toBe(longTaskId.slice(0, 64)) + }) + + it("should keep prompt_cache_retention alongside the key for 24h models", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + expect(body.prompt_cache_retention).toBe("24h") + }) + }) + describe("cost calculation", () => { it.each([ { tier: OpenAiServiceTier.Default, expectedCost: 4.525 }, diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 8a086b48db..b7485bb65b 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -534,6 +534,26 @@ describe("OpenAiNativeHandler", () => { expect(result).toBe("") }) + + it("should scope the prompt_cache_key to the handler session", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request.prompt_cache_key).toBe(handler["sessionId"]) + expect(request.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the prompt_cache_key when the model does not support prompt caching", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + handler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "codex-mini-latest" }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request).not.toHaveProperty("prompt_cache_key") + }) }) describe("getModel", () => { @@ -1446,6 +1466,64 @@ describe("OpenAiNativeHandler", () => { expect(callBody.previous_response_id).toBeUndefined() }) + it("sends a task-scoped prompt_cache_key in the body and no session_id header", async () => { + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + // Force the hand-built SSE path so the wire body and headers are inspectable. + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["task-cache-one", "task-cache-one", "task-cache-two"]) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(3) + expect(bodies[0].prompt_cache_key).toBe("task-cache-one") + // Same task, second turn: the key must not drift, or the growing prefix loses affinity. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + // A different task must not share the bucket. + expect(bodies[2].prompt_cache_key).toBe("task-cache-two") + expect(bodies[2].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + + // The retention control is untouched by this change. + expect(bodies[0].prompt_cache_retention).toBe("24h") + + // The affinity header is gone: OpenAI ignores it for caching and hardened + // gateways reject underscore header names outright. + for (const [, init] of mockFetch.mock.calls) { + expect(init.headers).not.toHaveProperty("session_id") + } + }) + + it("sends the task-scoped prompt_cache_key through the SDK path", async () => { + mockResponsesCreate.mockResolvedValue(asyncStreamFrom([])) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["sdk-task-one", "sdk-task-one", "sdk-task-two"]) { + await collectStream(cacheHandler.createMessage(systemPrompt, messages, { taskId })) + } + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies[0].prompt_cache_key).toBe("sdk-task-one") + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + expect(bodies[2].prompt_cache_key).toBe("sdk-task-two") + expect(bodies[0].prompt_cache_retention).toBe("24h") + expect(mockResponsesCreate.mock.calls[0][1].headers).not.toHaveProperty("session_id") + }) + it("should provide helpful error messages for different error codes", async () => { const testCases = [ { status: 400, expectedMessage: "Invalid request to Responses API" }, diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index fbbecf5760..91f514ffd5 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -34,6 +34,12 @@ import { sanitizeOpenAiCallId } from "../../utils/tool-id" export type OpenAiNativeModel = ReturnType +/** + * OpenAI caps `prompt_cache_key` at 64 characters; longer values are rejected with a 400. + * Task ids are uuidv7 today (36 chars), so this is a guard rather than a routine truncation. + */ +const PROMPT_CACHE_KEY_MAX_LENGTH = 64 + export class OpenAiNativeHandler extends BaseProvider implements SingleCompletionHandler { protected options: ApiHandlerOptions private client: OpenAI @@ -96,14 +102,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio this.options.enableResponsesReasoningSummary = true } const apiKey = this.options.openAiNativeApiKey ?? NOT_PROVIDED - // Include originator, session_id, and User-Agent headers for API tracking and debugging + // Include originator and User-Agent headers for API tracking and debugging. + // Cache affinity rides on the body `prompt_cache_key`, not on a header: OpenAI documents + // the `session_id` header as ignored on the public Responses API, and underscore header + // names are rejected outright by hardened gateways (Envoy/nginx `underscores_in_headers`). const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` this.client = new OpenAI({ baseURL: this.options.openAiNativeBaseUrl || undefined, apiKey, defaultHeaders: { originator: "zoo-code", - session_id: this.sessionId, "User-Agent": userAgent, }, timeout: this.timeoutMs, @@ -330,6 +338,11 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio include?: string[] /** Prompt cache retention policy: "in_memory" (default) or "24h" for extended caching */ prompt_cache_retention?: "in_memory" | "24h" + /** + * Cache routing affinity key. OpenAI documents this body field as the routing lever; + * the `session_id` request header is not part of the prompt-caching API. + */ + prompt_cache_key?: string tools?: Array<{ type: "function" name: string @@ -347,6 +360,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Decide whether to enable extended prompt cache retention for this request const promptCacheRetention = this.getPromptCacheRetention(model) + // Cache routing affinity: a stable, task-scoped key lets OpenAI route every turn of the + // same task to the machine holding its cached prefix. + const promptCacheKey = this.getPromptCacheKey(model, metadata) + const body: ResponsesRequestBody = { model: model.id, input: formattedInput, @@ -379,6 +396,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Enable extended prompt cache retention for models that support it. // This uses the OpenAI Responses API `prompt_cache_retention` parameter. ...(promptCacheRetention ? { prompt_cache_retention: promptCacheRetention } : {}), + // Cache routing affinity key (body field, not the inert `session_id` header). + ...(promptCacheKey ? { prompt_cache_key: promptCacheKey } : {}), tools: (metadata?.tools ?? []) .filter((tool) => tool.type === "function") .map((tool) => { @@ -418,12 +437,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` const requestHeaders: Record = { originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, } @@ -557,7 +574,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio private async *makeResponsesApiRequest( requestBody: any, model: OpenAiNativeModel, - metadata?: ApiHandlerCreateMessageMetadata, + _metadata?: ApiHandlerCreateMessageMetadata, _systemPrompt?: string, _messages?: Anthropic.Messages.MessageParam[], ): ApiStream { @@ -568,8 +585,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` try { @@ -579,7 +595,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, }, body: JSON.stringify(requestBody), @@ -1420,6 +1435,30 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio return undefined } + /** + * Returns the prompt cache routing key for this request, if the model caches prompts. + * + * OpenAI keys cache affinity off the `prompt_cache_key` body field; the `session_id` + * request header is not part of the prompt-caching API. The key must stay identical + * across every turn of one task (so its growing prefix keeps landing on the same + * machine) and differ between tasks (so unrelated prefixes do not compete for one + * routing bucket). `metadata.taskId` is the longest-lived identifier the handler + * receives: it is the persisted task id, stable across turns, restarts and + * rehydration, whereas `this.sessionId` is regenerated per handler instance. + * The handler session id is the fallback for callers that send no task id. + */ + private getPromptCacheKey( + model: OpenAiNativeModel, + metadata?: ApiHandlerCreateMessageMetadata, + ): string | undefined { + if (!model.info.supportsPromptCache) return undefined + + const key = metadata?.taskId || this.sessionId + if (!key) return undefined + + return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? key.slice(0, PROMPT_CACHE_KEY_MAX_LENGTH) : key + } + /** * Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available. * If no tier or no overrides exist, the original ModelInfo is returned. @@ -1531,6 +1570,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio requestBody[SERVICE_TIER_KEY] = serviceTier } + // Cache routing affinity key. `completePrompt` has no task metadata, so the handler + // session id scopes the key: one bucket per handler instance, never shared across tasks. + const promptCacheKey = this.getPromptCacheKey(model) + if (promptCacheKey) { + requestBody.prompt_cache_key = promptCacheKey + } + // Add reasoning if supported if (reasoningEffort) { requestBody.reasoning = { From 0f52a75e5fecfe896c0fe68aed04281b52c1fc7a Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 02:08:04 +0900 Subject: [PATCH 02/14] fix(cost): account cache writes per TTL class instead of one aggregate rate What: - Add `cacheWritesPrice1h` / `cacheWritesPrice12h` to the ModelInfo schema plus `deriveCacheWritesPrice1h` / `deriveCacheWritesPrice12h`, so the vendor multiplier relationship is stated once in model.ts. - Bill three write classes in `calculateApiCostInternal`: the 5m remainder is derived as total - 1h - 12h (clamped at 0), so the classes always sum to the aggregate and no token is billed twice or dropped. - Scale the TTL rates with `cacheWritesPriceMultiplier` in long-context pricing. - Carry `cacheWrite1hTokens` / `cacheWrite12hTokens` on `ApiStreamUsageChunk` and read them from Anthropic `usage.cache_creation.ephemeral_1h_input_tokens` (anthropic, minimax, anthropic-vertex) and OpenAI Responses `input_tokens_details.cache_write_tokens_details.ttl_12h_tokens` (openai-native `normalizeUsage` only). - Thread the split through the Task usage accumulator, including the cancel-path cost recomputation that runs when a provider yields no totalCost. - Populate the rates: 22 Anthropic entries, 19 Bedrock Anthropic-family entries, 7 OpenAI GPT-5.6/6-class entries. Why: Anthropic bills a 1h cache write at 2x input versus 1.25x for 5m, and OpenAI bills a 12h write at 2.5x versus 1.25x for 30m. The repo read only the aggregate `cache_creation_input_tokens` and priced every write at the short rate, so a 1h write under-reported by 37.5% and a 12h write by 50%. The vendors already report the per-TTL split; it was being discarded. Impact: Reporting only. No request shape changes: the repo emits no `ttl` field, so live traffic still reports 0/absent long-lived writes and pricing is byte-identical to before (asserted against the historical cost literals). When the TTL knob lands, a 1h write can no longer be billed at the 5m rate. 18 new tests fail on the base blob; the new cost code has no surviving mutants. --- .../__tests__/cache-write-ttl-pricing.test.ts | 84 ++++++ .../types/src/__tests__/openai-models.test.ts | 2 + packages/types/src/model.ts | 35 +++ packages/types/src/providers/anthropic.ts | 22 ++ packages/types/src/providers/bedrock.ts | 19 ++ packages/types/src/providers/openai.ts | 7 + src/api/providers/__tests__/anthropic.spec.ts | 99 ++++++- .../__tests__/openai-native-usage.spec.ts | 120 +++++++- src/api/providers/anthropic-vertex.ts | 5 + src/api/providers/anthropic.ts | 9 + src/api/providers/minimax.ts | 9 + src/api/providers/openai-native.ts | 14 + src/api/transform/stream.ts | 11 + src/core/task/Task.ts | 23 ++ src/shared/cost.ts | 63 +++- src/utils/__tests__/cost.spec.ts | 270 ++++++++++++++++++ 16 files changed, 788 insertions(+), 4 deletions(-) create mode 100644 packages/types/src/__tests__/cache-write-ttl-pricing.test.ts diff --git a/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts new file mode 100644 index 0000000000..e62f658016 --- /dev/null +++ b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts @@ -0,0 +1,84 @@ +// npx vitest run src/__tests__/cache-write-ttl-pricing.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { bedrockModels } from "../providers/bedrock.js" +import { openAiNativeModels } from "../providers/openai.js" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + deriveCacheWritesPrice12h, + deriveCacheWritesPrice1h, + type ModelInfo, +} from "../model.js" + +/** + * Catalog invariant for the TTL accounting program (ADR-0006 phase 1). + * + * A model that advertises prompt caching on a provider which can report a + * long-lived cache write must carry the matching per-TTL write rate. Without it + * the cost layer can only fall back to the aggregate write rate, which + * under-reports every long-lived write (37.5% for Anthropic 1h, 50% for OpenAI 12h). + */ +describe("cache-write TTL pricing catalog invariant", () => { + const anthropicFamilyEntries = Object.entries(anthropicModels as Record) + + const bedrockAnthropicEntries = Object.entries(bedrockModels as Record).filter(([id]) => + id.startsWith("anthropic."), + ) + + const openAiLongLivedEntries = Object.entries(openAiNativeModels as Record).filter( + ([id]) => id.startsWith("gpt-5.6") || id.startsWith("gpt-6"), + ) + + it("every Anthropic catalog entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = anthropicFamilyEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every Bedrock Anthropic-family entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = bedrockAnthropicEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every OpenAI 5.6/6-class entry defines cacheWritesPrice12h", () => { + expect(openAiLongLivedEntries.length).toBeGreaterThan(0) + + const missing = openAiLongLivedEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice12h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("Anthropic 1h rates match the documented 2x input-price relationship", () => { + for (const [id, info] of anthropicFamilyEntries) { + if (info.cacheWritesPrice1h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice1h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_1H, 9) + } + }) + + it("OpenAI 12h rates match the documented 2.5x input-price relationship", () => { + for (const [id, info] of openAiLongLivedEntries) { + if (info.cacheWritesPrice12h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice12h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_12H, 9) + } + }) + + it("deriveCacheWritesPrice1h scales inputPrice by 2x and stays undefined without one", () => { + expect(deriveCacheWritesPrice1h(3.0)).toBe(6.0) + expect(deriveCacheWritesPrice1h(undefined)).toBeUndefined() + }) + + it("deriveCacheWritesPrice12h scales inputPrice by 2.5x and stays undefined without one", () => { + expect(deriveCacheWritesPrice12h(4.0)).toBe(10.0) + expect(deriveCacheWritesPrice12h(undefined)).toBeUndefined() + }) +}) diff --git a/packages/types/src/__tests__/openai-models.test.ts b/packages/types/src/__tests__/openai-models.test.ts index 78a3259f09..e1e0f1e323 100644 --- a/packages/types/src/__tests__/openai-models.test.ts +++ b/packages/types/src/__tests__/openai-models.test.ts @@ -122,6 +122,7 @@ describe("OpenAI native models", () => { inputPrice: 2, outputPrice: 10, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -166,6 +167,7 @@ describe("OpenAI native models", () => { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 9ddc4ea182..40c00bdab3 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -136,6 +136,13 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL cache-write rates. Vendors bill a longer cache lifetime at a higher + // write rate than the default 5m/30m class, so a single `cacheWritesPrice` + // under-reports whenever the response reports a long-lived write. Omitted + // means "no long-lived writes are expected on this model"; the cost layer + // derives a fail-closed rate from `inputPrice` if one still arrives. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), longContextPricing: z .object({ @@ -188,6 +195,34 @@ export const modelInfoSchema = z.object({ export type ModelInfo = z.infer +/** + * Multipliers vendors apply to the base input price for a cache write, by TTL class. + * Anthropic: 5m = 1.25x, 1h = 2x. OpenAI: 30m = 1.25x, 12h = 2.5x. + */ +export const CACHE_WRITE_MULTIPLIER_5M = 1.25 +export const CACHE_WRITE_MULTIPLIER_1H = 2.0 +export const CACHE_WRITE_MULTIPLIER_12H = 2.5 + +/** + * Derive the 1h cache-write rate from the base input price. + * + * Stated once here so every Anthropic-family catalog entry and every runtime + * fallback agrees on the 2x relationship. Returns `undefined` when the model + * has no input price to scale, which keeps the cost layer fail-closed: it will + * fall back to `cacheWritesPrice` rather than invent a rate. + */ +export function deriveCacheWritesPrice1h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_1H +} + +/** + * Derive the 12h cache-write rate from the base input price (OpenAI 12h class). + * Same fail-closed contract as {@link deriveCacheWritesPrice1h}. + */ +export function deriveCacheWritesPrice12h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_12H +} + export type ModelRecord = Record export type RouterModels = Record diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index 567f6268be..cc50bfb190 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -16,6 +16,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -37,6 +38,7 @@ export const anthropicModels = { inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) // Sonnet 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Fable 5 on the direct Anthropic provider path. Manual @@ -56,6 +58,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -77,6 +80,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -98,6 +102,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag) @@ -119,6 +124,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Keep the hybrid-reasoning capability so Anthropic token-cap handling and // stored max-token overrides behave the same as before. @@ -136,6 +142,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens (regular tier) outputPrice: 25.0, // $25 per million output tokens (regular tier) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // 4.8 inherits the adaptive-thinking model introduced in 4.7 — no breaking // API changes. supportsReasoningBudget is kept true so the existing token-cap @@ -154,6 +161,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Opus 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Sonnet 5 on the direct Anthropic provider path. Manual @@ -172,6 +180,7 @@ export const anthropicModels = { inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // Opus 5.5 always uses adaptive thinking and rejects manual budget_tokens. supportsReasoningBinary: true, @@ -192,6 +201,7 @@ export const anthropicModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // Haiku 5.5 uses the same adaptive-thinking / binary-toggle convention as // Sonnet 5 and Opus 5 on the direct Anthropic provider path. Manual @@ -211,6 +221,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, // $0.25 per million tokens (2.5% of base input price) // Fable 5.1 only accepts adaptive thinking (or an omitted thinking field). // Manual budget_tokens and non-default sampling parameters return a 400. @@ -228,6 +239,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens // Fable 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ on the direct Anthropic provider path. @@ -245,6 +257,7 @@ export const anthropicModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, }, @@ -256,6 +269,7 @@ export const anthropicModels = { inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -267,6 +281,7 @@ export const anthropicModels = { inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -278,6 +293,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, requiredReasoningBudget: true, @@ -290,6 +306,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-sonnet-20241022": { @@ -300,6 +317,7 @@ export const anthropicModels = { inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-haiku-20241022": { @@ -310,6 +328,7 @@ export const anthropicModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, }, "claude-3-opus-20240229": { @@ -320,6 +339,7 @@ export const anthropicModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, }, "claude-3-haiku-20240307": { @@ -330,6 +350,7 @@ export const anthropicModels = { inputPrice: 0.25, outputPrice: 1.25, cacheWritesPrice: 0.3, + cacheWritesPrice1h: 0.5, // $0.50 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.03, }, "claude-haiku-4-5-20251001": { @@ -340,6 +361,7 @@ export const anthropicModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, supportsReasoningBudget: true, description: diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index 1e89f3e6ba..696cc04a99 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -22,6 +22,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -36,6 +37,7 @@ export const bedrockModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -63,6 +65,7 @@ export const bedrockModels = { inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -143,6 +146,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -157,6 +161,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -171,6 +176,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -195,6 +201,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -219,6 +226,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -248,6 +256,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -268,6 +277,7 @@ export const bedrockModels = { inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Opus 5.5 (matching the Fable 5.1 entry below). @@ -297,6 +307,7 @@ export const bedrockModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Haiku 5.5 (matching the Opus 5.5 entry above). @@ -317,6 +328,7 @@ export const bedrockModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, minTokensPerCachePoint: 512, maxCachePoints: 4, @@ -335,6 +347,7 @@ export const bedrockModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -351,6 +364,7 @@ export const bedrockModels = { inputPrice: 5.0, outputPrice: 25.0, cacheWritesPrice: 6.25, + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -365,6 +379,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -379,6 +394,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -392,6 +408,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -405,6 +422,7 @@ export const bedrockModels = { inputPrice: 0.8, outputPrice: 4.0, cacheWritesPrice: 1.0, + cacheWritesPrice1h: 1.6, // $1.60 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.08, minTokensPerCachePoint: 2048, maxCachePoints: 4, @@ -419,6 +437,7 @@ export const bedrockModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, // 5m cache writes + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, // cache hits / refreshes minTokensPerCachePoint: 2048, maxCachePoints: 4, diff --git a/packages/types/src/providers/openai.ts b/packages/types/src/providers/openai.ts index 41d00c7a04..89a24fe2ef 100644 --- a/packages/types/src/providers/openai.ts +++ b/packages/types/src/providers/openai.ts @@ -31,6 +31,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.1, longContextPricing: { thresholdTokens: 272_000, @@ -75,6 +76,7 @@ export const openAiNativeModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice12h: 25, // $25.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 1.0, longContextPricing: { thresholdTokens: 272_000, @@ -117,6 +119,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -160,6 +163,7 @@ export const openAiNativeModels = { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, // $0.25 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, @@ -203,6 +207,7 @@ export const openAiNativeModels = { inputPrice: 4.0, outputPrice: 20.0, cacheWritesPrice: 5.0, + cacheWritesPrice12h: 10, // $10.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.4, longContextPricing: { thresholdTokens: 272_000, @@ -246,6 +251,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 12.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -289,6 +295,7 @@ export const openAiNativeModels = { inputPrice: 0.2, outputPrice: 1.2, cacheWritesPrice: 0.25, + cacheWritesPrice12h: 0.5, // $0.50 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.02, longContextPricing: { thresholdTokens: 272_000, diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index b97f309dcc..5bca7e6b30 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -5,7 +5,7 @@ import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { ApiHandlerOptions } from "../../../shared/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" -import type { ApiStreamChunk } from "../../../api/transform/stream" +import type { ApiStreamChunk, ApiStreamUsageChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" // Mock TelemetryService @@ -1045,6 +1045,103 @@ describe("AnthropicHandler", () => { "fine-grained-tool-streaming-2025-05-14,prompt-caching-2024-07-31", ) }) + + it("should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + + it("should leave the 1h split undefined when cache_creation is absent", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(20) + expect(usageChunk?.cacheWrite1hTokens).toBeUndefined() + }) + + it("should price a 1h write at the 1h rate in the final cost chunk", async () => { + // claude-sonnet-4-5: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + const sonnetHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + }) + + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 0, + ephemeral_1h_input_tokens: 100_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = sonnetHandler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const costChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.totalCost !== undefined, + ) + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour charged the 5m rate: 0.375 (37.5% low). + expect(costChunk?.totalCost).toBeCloseTo(0.6, 9) + expect(costChunk?.totalCost).not.toBeCloseTo(0.375, 9) + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 2b05ab5b2b..3725a801bc 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach } from "vitest" import { OpenAiNativeHandler } from "../openai-native" -import { OpenAiServiceTier, openAiNativeModels } from "@roo-code/types" +import { OpenAiServiceTier, openAiNativeModels, type ModelInfo } from "@roo-code/types" +import type { ApiStreamUsageChunk } from "../../transform/stream" describe("OpenAiNativeHandler - normalizeUsage", () => { let handler: OpenAiNativeHandler @@ -615,4 +616,121 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split (Responses API)", () => { + // `normalizeUsage` is private. A narrow structural cast keeps the call typed + // without widening the handler to `any`; the return is asserted non-null because + // every call site below passes a usage object that yields a chunk. + const normalizeUsage = ( + h: OpenAiNativeHandler, + usage: unknown, + model: { id: string; info: ModelInfo }, + ): ApiStreamUsageChunk => { + const result = ( + h as unknown as { + normalizeUsage(u: unknown, m: { id: string; info: ModelInfo }): ApiStreamUsageChunk | undefined + } + ).normalizeUsage(usage, model) + expect(result).toBeDefined() + return result as ApiStreamUsageChunk + } + + it("should read ttl_12h_tokens and bill it at the 12h rate", () => { + // gpt-5.6-sol: input 4.0, cacheWritesPrice 5.0 (30m class), cacheWritesPrice12h 10.0. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 0, + ttl_12h_tokens: 100_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 100_000, + }) + + // gpt-6-astra: input 10.0, 12h rate 25.0. + // Base input: (10.0 / 1_000_000) * (100_000 - 100_000) = 0 + // 12h writes: (25.0 / 1_000_000) * 100_000 = 2.5 + // The pre-split behaviour charged the 30m rate: 12.5 * 0.1 = 1.25 (50% low). + expect(result.totalCost).toBeCloseTo(2.5, 6) + expect(result.totalCost).not.toBeCloseTo(1.25, 6) + }) + + it("should split a mixed 30m/12h write total without double counting", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 60_000, + ttl_12h_tokens: 40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 40_000, + }) + + // 30m remainder: 60_000 at 12.5 => 0.75 + // 12h: 40_000 at 25.0 => 1.0 + // Total: 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + + it("should leave the split undefined when the response omits it", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result.cacheWrite12hTokens).toBeUndefined() + // All writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) + + it("should clamp a 12h split that exceeds the write total", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 50_000, + cache_write_tokens_details: { + ttl_12h_tokens: 80_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // Clamped to the write total so no more 12h tokens are billed than written. + expect(result.cacheWrite12hTokens).toBe(50_000) + // Base input: (10.0 / 1_000_000) * (100_000 - 50_000) = 0.5 + // 12h writes: (25.0 / 1_000_000) * 50_000 = 1.25 + // Total: 0.5 + 1.25 = 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + }) }) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 87da090346..c8a4407950 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -145,6 +145,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple case "message_start": { const usage = chunk.message!.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const cacheWrite1hTokens = usage.cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens; surfaced for per-turn // thinking telemetry (billing already includes it in output_tokens). @@ -153,6 +157,7 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0, cacheWriteTokens: usage.cache_creation_input_tokens || undefined, + cacheWrite1hTokens: cacheWrite1hTokens || undefined, cacheReadTokens: usage.cache_read_input_tokens || undefined, ...(typeof usage.output_tokens_details?.thinking_tokens === "number" ? { reasoningTokens: usage.output_tokens_details!.thinking_tokens } diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index c65eb4beba..677f38bbf5 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -256,6 +256,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -266,9 +267,14 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // The per-TTL breakdown decomposes `cache_creation_input_tokens`; it is + // absent for providers that do not report it, which leaves the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens (0 ⇒ the model skipped // thinking this turn). `output_tokens` already includes them for @@ -280,6 +286,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, ...(typeof startThinkingTokens === "number" ? { reasoningTokens: startThinkingTokens } : {}), } @@ -287,6 +294,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -381,6 +389,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { diff --git a/src/api/providers/minimax.ts b/src/api/providers/minimax.ts index 52c42a4138..12d3cafe37 100644 --- a/src/api/providers/minimax.ts +++ b/src/api/providers/minimax.ts @@ -137,6 +137,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -147,20 +148,27 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + yield { type: "usage", inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, } inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -244,6 +252,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index 91f514ffd5..bae737e9cc 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -132,6 +132,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const writesFromDetails = typeof inputDetails?.cache_write_tokens === "number" ? inputDetails.cache_write_tokens : 0 + // The Responses API splits cache writes by TTL class under + // `input_tokens_details.cache_write_tokens_details`. The installed SDK types + // (openai@5.23.2) do not declare it yet, so it is read defensively; the + // fields are a decomposition of `cache_write_tokens`, not an addition. + const writeDetails = inputDetails?.cache_write_tokens_details + const writes12hFromDetails = typeof writeDetails?.ttl_12h_tokens === "number" ? writeDetails.ttl_12h_tokens : 0 + // If total input tokens are missing but we have details, derive from them let totalInputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0 if ( @@ -152,6 +159,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0 + // Clamp the 12h subset to the write total so a malformed response cannot bill + // more 12h tokens than were written at all. + const cacheWrite12hTokens = Math.min(writes12hFromDetails, cacheWriteTokens) + // Resolve effective tier: prefer actual tier from response; otherwise requested tier const effectiveTier = this.lastServiceTier || (this.options.openAiNativeServiceTier as ServiceTier | undefined) || undefined @@ -166,6 +177,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio cacheWriteTokens, cacheReadTokens, effectiveTier, + undefined, + cacheWrite12hTokens, ) const reasoningTokens = @@ -179,6 +192,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio inputTokens: totalInputTokens, outputTokens: totalOutputTokens, cacheWriteTokens, + cacheWrite12hTokens: cacheWrite12hTokens || undefined, cacheReadTokens, ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}), totalCost, diff --git a/src/api/transform/stream.ts b/src/api/transform/stream.ts index 40efeb3bb0..a3d4b533e7 100644 --- a/src/api/transform/stream.ts +++ b/src/api/transform/stream.ts @@ -59,7 +59,18 @@ export interface ApiStreamUsageChunk { type: "usage" inputTokens: number outputTokens: number + /** Total cache-write tokens, across every TTL class. */ cacheWriteTokens?: number + /** + * The 1h-TTL subset of `cacheWriteTokens` (Anthropic `ephemeral_1h_input_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite1hTokens?: number + /** + * The 12h-TTL subset of `cacheWriteTokens` (OpenAI `ttl_12h_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite12hTokens?: number cacheReadTokens?: number reasoningTokens?: number totalCost?: number diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 3202358c09..af6f9b87fb 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -4021,6 +4021,8 @@ export class Task extends EventEmitter implements TaskLike { try { let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 + let cacheWrite12hTokens = 0 let cacheReadTokens = 0 let inputTokens = 0 let outputTokens = 0 @@ -4056,6 +4058,8 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, + cacheWrite12hTokens, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4063,6 +4067,9 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + undefined, + cacheWrite1hTokens, + cacheWrite12hTokens, ) this.clineMessages[lastApiReqIndex].text = JSON.stringify({ @@ -4210,6 +4217,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens += chunk.inputTokens outputTokens += chunk.outputTokens cacheWriteTokens += chunk.cacheWriteTokens ?? 0 + cacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + cacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 cacheReadTokens += chunk.cacheReadTokens ?? 0 totalCost = chunk.totalCost break @@ -4450,6 +4459,8 @@ export class Task extends EventEmitter implements TaskLike { input: inputTokens, output: outputTokens, cacheWrite: cacheWriteTokens, + cacheWrite1h: cacheWrite1hTokens, + cacheWrite12h: cacheWrite12hTokens, cacheRead: cacheReadTokens, total: totalCost, } @@ -4466,6 +4477,8 @@ export class Task extends EventEmitter implements TaskLike { let bgInputTokens = currentTokens.input let bgOutputTokens = currentTokens.output let bgCacheWriteTokens = currentTokens.cacheWrite + let bgCacheWrite1hTokens = currentTokens.cacheWrite1h + let bgCacheWrite12hTokens = currentTokens.cacheWrite12h let bgCacheReadTokens = currentTokens.cacheRead let bgTotalCost = currentTokens.total @@ -4475,6 +4488,8 @@ export class Task extends EventEmitter implements TaskLike { input: number output: number cacheWrite: number + cacheWrite1h: number + cacheWrite12h: number cacheRead: number total?: number }, @@ -4491,6 +4506,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens = tokens.input outputTokens = tokens.output cacheWriteTokens = tokens.cacheWrite + cacheWrite1hTokens = tokens.cacheWrite1h + cacheWrite12hTokens = tokens.cacheWrite12h cacheReadTokens = tokens.cacheRead totalCost = tokens.total @@ -4568,6 +4585,8 @@ export class Task extends EventEmitter implements TaskLike { bgInputTokens += chunk.inputTokens bgOutputTokens += chunk.outputTokens bgCacheWriteTokens += chunk.cacheWriteTokens ?? 0 + bgCacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + bgCacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 bgCacheReadTokens += chunk.cacheReadTokens ?? 0 bgTotalCost = chunk.totalCost } @@ -4586,6 +4605,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, @@ -4611,6 +4632,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 754a2d84df..7fcc4e0d13 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -1,4 +1,10 @@ -import { OpenAiServiceTier, type ModelInfo, type ServiceTier } from "@roo-code/types" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + OpenAiServiceTier, + type ModelInfo, + type ServiceTier, +} from "@roo-code/types" export interface ApiCostResult { totalInputTokens: number @@ -6,6 +12,24 @@ export interface ApiCostResult { totalCost: number } +/** + * Resolve the per-million rate for a TTL-specific cache-write class. + * + * Precedence is fail-closed: an explicit catalog rate wins, then the vendor + * multiplier applied to the base input price, and only then the aggregate + * `cacheWritesPrice`. The last step is the pre-split behaviour, reachable only + * when a model has neither a 1h/12h rate nor an input price to derive one from. + */ +function resolveCacheWriteRate( + modelInfo: ModelInfo, + explicit: number | undefined, + multiplier: number, +): number | undefined { + if (explicit !== undefined) return explicit + if (modelInfo.inputPrice !== undefined) return modelInfo.inputPrice * multiplier + return modelInfo.cacheWritesPrice +} + function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, serviceTier?: ServiceTier): ModelInfo { const pricing = modelInfo.longContextPricing if (!pricing || totalInputTokens <= pricing.thresholdTokens) { @@ -31,6 +55,16 @@ function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, modelInfo.cacheWritesPrice !== undefined && pricing.cacheWritesPriceMultiplier !== undefined ? modelInfo.cacheWritesPrice * pricing.cacheWritesPriceMultiplier : modelInfo.cacheWritesPrice, + // The TTL-specific write rates scale with the same multiplier as the aggregate + // write rate; leaving them unscaled would under-bill long-context 1h writes. + cacheWritesPrice1h: + modelInfo.cacheWritesPrice1h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice1h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice1h, + cacheWritesPrice12h: + modelInfo.cacheWritesPrice12h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice12h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice12h, cacheReadsPrice: modelInfo.cacheReadsPrice !== undefined && pricing.cacheReadsPriceMultiplier !== undefined ? modelInfo.cacheReadsPrice * pricing.cacheReadsPriceMultiplier @@ -46,8 +80,25 @@ function calculateApiCostInternal( cacheReadInputTokens: number, totalInputTokens: number, totalOutputTokens: number, + cacheCreation1hInputTokens = 0, + cacheCreation12hInputTokens = 0, ): ApiCostResult { - const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens + // The per-TTL counts are a decomposition of `cacheCreationInputTokens`, never + // an addition to it. Deriving the 5m remainder this way keeps the aggregate + // path byte-identical when a vendor reports no split (both terms are 0). + const writes1h = Math.max(0, cacheCreation1hInputTokens) + const writes12h = Math.max(0, cacheCreation12hInputTokens) + const writes5m = Math.max(0, cacheCreationInputTokens - writes1h - writes12h) + + const writes5mRate = modelInfo.cacheWritesPrice || 0 + const writes1hRate = resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice1h, CACHE_WRITE_MULTIPLIER_1H) || 0 + const writes12hRate = + resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice12h, CACHE_WRITE_MULTIPLIER_12H) || 0 + + const cacheWritesCost = + (writes5mRate / 1_000_000) * writes5m + + (writes1hRate / 1_000_000) * writes1h + + (writes12hRate / 1_000_000) * writes12h const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens const outputCost = ((modelInfo.outputPrice || 0) / 1_000_000) * outputTokens @@ -68,6 +119,8 @@ export function calculateApiCostAnthropic( outputTokens: number, cacheCreationInputTokens?: number, cacheReadInputTokens?: number, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreation = cacheCreationInputTokens || 0 const cacheRead = cacheReadInputTokens || 0 @@ -84,6 +137,8 @@ export function calculateApiCostAnthropic( cacheRead, totalInputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } @@ -95,6 +150,8 @@ export function calculateApiCostOpenAI( cacheCreationInputTokens?: number, cacheReadInputTokens?: number, serviceTier?: ServiceTier, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreationInputTokensNum = cacheCreationInputTokens || 0 const cacheReadInputTokensNum = cacheReadInputTokens || 0 @@ -111,6 +168,8 @@ export function calculateApiCostOpenAI( cacheReadInputTokensNum, inputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b369a0405b..b27eb4100a 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -303,4 +303,274 @@ describe("Cost Utility", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split accounting", () => { + const anthropicModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, // 1.25x input + cacheWritesPrice1h: 6.0, // 2x input + cacheReadsPrice: 0.3, + } + + const openAiModel: ModelInfo = { + maxTokens: 128_000, + contextWindow: 1_050_000, + supportsPromptCache: true, + inputPrice: 4.0, + outputPrice: 20.0, + cacheWritesPrice: 5.0, // 1.25x input (30m class) + cacheWritesPrice12h: 10.0, // 2.5x input + cacheReadsPrice: 0.4, + } + + it("bills 100k 1h tokens at the 1h rate, not the 5m rate (37.5% tripwire)", () => { + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 100_000) + + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour would have charged the 5m rate: + // (3.75 / 1_000_000) * 100_000 = 0.375, under-reporting by 37.5%. + expect(result.totalCost).toBeCloseTo(0.6, 9) + expect(result.totalCost).not.toBeCloseTo(0.375, 9) + }) + + it("bills a mixed 5m/1h split per class (0.465 case)", () => { + // 60k of the 100k writes are 1h, 40k remain at the 5m rate. + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 60_000) + + // 5m writes: (3.75 / 1_000_000) * 40_000 = 0.15 + // 1h writes: (6.0 / 1_000_000) * 60_000 = 0.36 + // Total: 0.15 + 0.36 = 0.51 + expect(result.totalCost).toBeCloseTo(0.51, 9) + }) + + it("derives the 5m remainder as total minus 1h minus 12h", () => { + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 30_000, 20_000) + + // 5m remainder: 100k - 30k - 20k = 50k at 3.75 => 0.1875 + // 1h: 30k at 6.0 => 0.18 + // 12h: 20k at 7.5 => 0.15 + // Total: 0.1875 + 0.18 + 0.15 = 0.5175 + expect(result.totalCost).toBeCloseTo(0.5175, 9) + }) + + it("clamps the 5m remainder at zero when the split exceeds the total", () => { + // A malformed response claiming more 1h tokens than the write total must + // not produce negative 5m writes (which would refund cost). + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 150_000) + + // 5m remainder clamps to 0; 1h is capped by the clamp on the remainder only, + // so the 1h class still bills its reported 150k at 6.0 => 0.9 + expect(result.totalCost).toBeCloseTo(0.9, 9) + expect(result.totalCost).toBeGreaterThan(0) + }) + + it("is byte-identical to the pre-split cost when no split is reported", () => { + // Invariant I1: absent split fields must reproduce today's pricing exactly. + const withSplitArgs = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000) + const explicitZeros = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000, 0, 0) + + expect(withSplitArgs.totalCost).toBe(0.0189) + expect(explicitZeros.totalCost).toBe(withSplitArgs.totalCost) + }) + + it("falls back to the 5m rate when a model has no 1h rate and no input price", () => { + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + } + + // Fail-closed: without a derivable 1h rate the write is billed at the + // aggregate rate rather than dropped or invented. + const result = calculateApiCostAnthropic(bareModel, 0, 0, 100_000, 0, 100_000) + expect(result.totalCost).toBeCloseTo(0.375, 9) + }) + + it("derives the 1h rate from inputPrice when the catalog omits it", () => { + const modelWithout1h: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, + } + + const result = calculateApiCostAnthropic(modelWithout1h, 0, 0, 100_000, 0, 100_000) + // Derived 1h rate = 3.0 * 2 = 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) + }) + + it("bills OpenAI 12h writes at the 12h rate, not the 30m rate", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 100_000) + + // 12h writes: (10.0 / 1_000_000) * 100_000 = 1.0 + // The pre-split behaviour would have charged the 30m rate: + // (5.0 / 1_000_000) * 100_000 = 0.5, under-reporting by 50%. + expect(result.totalCost).toBeCloseTo(1.0, 9) + expect(result.totalCost).not.toBeCloseTo(0.5, 9) + }) + + it("bills a mixed OpenAI 30m/12h split per class", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 40_000) + + // 30m remainder: 100k - 40k = 60k at 5.0 => 0.3 + // 12h: 40k at 10.0 => 0.4 + // Total: 0.7 + expect(result.totalCost).toBeCloseTo(0.7, 9) + }) + + it("scales the 1h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + // calculateApiCostOpenAI applies long-context pricing; total input 300k > 200k. + const result = calculateApiCostOpenAI(modelWithLongContext, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input: (3.0 / 1_000_000) * (300_000 - 100_000) = 0.6 + // 1h writes scale 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8. Unscaled 1h would have given 0.6 + 0.6 = 1.2. + expect(result.totalCost).toBeCloseTo(1.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.2, 9) + }) + + it("scales the 12h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...openAiModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithLongContext, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (4.0 / 1_000_000) * (300_000 - 100_000) = 0.8 + // 12h writes scale 10.0 -> 20.0: (20.0 / 1_000_000) * 100_000 = 2.0 + // Total: 0.8 + 2.0 = 2.8. Unscaled 12h would have given 0.8 + 1.0 = 1.8. + expect(result.totalCost).toBeCloseTo(2.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.8, 9) + }) + + it("leaves the 1h rate undefined when neither a catalog rate nor an input price exists", () => { + // Covers the `cacheWritesPrice1h !== undefined` false branch of long-context scaling. + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(bareModel, 300_000, 0, 100_000, 0, undefined, 100_000) + + // No input price => no derivable 1h rate => falls back to the scaled aggregate + // write rate (3.75 * 2 = 7.5): (7.5 / 1_000_000) * 100_000 = 0.75 + expect(result.totalCost).toBeCloseTo(0.75, 9) + }) + + it("does not scale TTL rates when long-context pricing has no write multiplier", () => { + // Covers the `cacheWritesPriceMultiplier !== undefined` false branch for both + // TTL classes: without a multiplier the rates must pass through unchanged. + const modelWithoutMultiplier: ModelInfo = { + ...anthropicModel, + cacheWritesPrice12h: 7.5, + longContextPricing: { + thresholdTokens: 200_000, + inputPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithoutMultiplier, + 300_000, + 0, + 100_000, + 0, + undefined, + 50_000, + 50_000, + ) + + // Base input scales: (3.0 * 2 / 1_000_000) * (300_000 - 100_000) = 1.2 + // 1h unscaled: (6.0 / 1_000_000) * 50_000 = 0.3 + // 12h unscaled: (7.5 / 1_000_000) * 50_000 = 0.375 + // Total: 1.2 + 0.3 + 0.375 = 1.875 + expect(result.totalCost).toBeCloseTo(1.875, 9) + }) + + it("keeps an undefined 12h rate undefined when a write multiplier is present", () => { + // Covers the `cacheWritesPrice12h !== undefined` false branch: a model with a + // long-context write multiplier but no 12h rate must not gain a fabricated one. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(modelWithout12h, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input (no inputPriceMultiplier, so unscaled): (3.0 / 1_000_000) * 200_000 = 0.6 + // 1h scales 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8 + expect(result.totalCost).toBeCloseTo(1.8, 9) + }) + + it("derives the 12h rate from inputPrice when a write multiplier is present but no 12h rate", () => { + // Distinguishes the guarded branch from an unguarded multiply: an unguarded + // `undefined * multiplier` yields NaN, which the cost layer reads as 0 and + // would silently drop the 12h write cost entirely. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithout12h, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (3.0 / 1_000_000) * 200_000 = 0.6 + // No 12h rate in the catalog, so it derives from inputPrice * 2.5 = 7.5 + // (the long-context multiplier does not apply to a rate that is not there): + // (7.5 / 1_000_000) * 100_000 = 0.75 + // Total: 0.6 + 0.75 = 1.35 + expect(result.totalCost).toBeCloseTo(1.35, 9) + expect(result.totalCost).not.toBeCloseTo(0.6, 9) + }) + }) }) From b647ec7d8f746ae7a10d338536ddc5d9796ef863 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:12:08 +0900 Subject: [PATCH 03/14] fix(cost): bill the TTL split on the telemetry recompute and in tier blocks What: - Pass `cacheWrite1h`/`cacheWrite12h` to the cost call inside `captureUsageData`. The bag already carried them and fed the shared accumulator vars, but the telemetry recompute passed only the aggregate write count, so a cancelled request's telemetry billed every write at the short-TTL rate. - Add `cacheWritesPrice1h`/`cacheWritesPrice12h` to the `tiers` schema and to the four Anthropic and four Bedrock 1M-context tier blocks, derived from each tier's own input price. - Carry the tier's TTL rates through all three tier-merge seams (anthropic, anthropic-vertex, bedrock) and derive them from the tier input price in the OpenAI service-tier seam, so a flex/priority or 1M-context write is not billed at the base tier's TTL rate. Why: Two under-reports survived the first pass. `captureUsageData` recomputes cost for telemetry when a provider yields no final cost chunk (the cancelled-stream path), and it dropped the split: 100k 1h tokens at claude-3-5-sonnet billed 0.375 instead of 0.6. Separately, the 1M-context tier merge overrode `inputPrice` but not the TTL rate, so the entry-level rate derived from the 200K input price survived: a 1M-context 1h write billed 6.0 instead of the tier's 12.0, a 50% under-report. Impact: Both paths now bill at the correct TTL rate. Three new tests fail on the pre-fix code (0.375 vs 0.6, and 6 vs 12); a fourth asserts the tier rate reaches the cost function. No request shape changes. --- packages/types/src/model.ts | 5 ++ packages/types/src/providers/anthropic.ts | 4 ++ packages/types/src/providers/bedrock.ts | 4 ++ .../__tests__/anthropic-vertex.spec.ts | 36 ++++++++++++++ src/api/providers/__tests__/anthropic.spec.ts | 33 +++++++++++++ src/api/providers/__tests__/minimax.spec.ts | 32 +++++++++++++ src/api/providers/anthropic-vertex.ts | 4 ++ src/api/providers/anthropic.ts | 5 ++ src/api/providers/bedrock.ts | 4 ++ src/api/providers/openai-native.ts | 13 +++++ src/core/task/Task.ts | 10 +++- src/core/task/__tests__/Task.spec.ts | 47 +++++++++++++++++++ 12 files changed, 196 insertions(+), 1 deletion(-) diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 40c00bdab3..0ac2a4c339 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -187,6 +187,11 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL write rates for this tier. A tier that overrides inputPrice + // must also override these, otherwise a long-context or flex/priority + // write bills at the base tier's TTL rate. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), }), ) diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index cc50bfb190..7bf55238ac 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -26,6 +26,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -68,6 +69,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -90,6 +92,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -112,6 +115,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index 696cc04a99..faef150a2a 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -49,6 +49,7 @@ export const bedrockModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -188,6 +189,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -213,6 +215,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -240,6 +243,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 34788df89b..94b339fc58 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -10,6 +10,7 @@ import { AnthropicVertexHandler } from "../anthropic-vertex" import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" +import type { ApiStreamChunk } from "../../../api/transform/stream" vitest.mock("../utils/timeout-config", () => ({ getApiRequestTimeout: vitest.fn().mockReturnValue(300_000), @@ -453,6 +454,41 @@ describe("VertexHandler", () => { ) }) + it("should surface the 1h cache-write split from cache_creation", async () => { + const mockStream = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ] + + const mockCreate = vitest.fn().mockResolvedValue(asyncStreamFrom(mockStream)) + // Typed structural cast rather than `as any`: the client is a private field and + // this file already has 31 suppressed `any` uses, a budget this test must not grow. + ;(handler["client"].messages as unknown as { create: typeof mockCreate }).create = mockCreate + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is Extract => + chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + it("should handle cache-related usage metrics for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 5bca7e6b30..317a897385 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -7,6 +7,7 @@ import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" import type { ApiStreamChunk, ApiStreamUsageChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" +import { calculateApiCostAnthropic } from "../../../shared/cost" // Mock TelemetryService vitest.mock("@roo-code/telemetry", () => ({ @@ -1538,6 +1539,38 @@ describe("AnthropicHandler", () => { expect(model.info.outputPrice).toBe(22.5) }) + it("carries the tier's own cacheWritesPrice1h when the 1M context tier applies", () => { + // glm P2: the tier merge overrode inputPrice but not the TTL rate, so the + // entry-level 1h rate (derived from the 200K input price) survived and a + // 1M-context 1h write billed 6.0 instead of the tier's 12.0 (50% under). + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + // Tier input price 6.0 => 1h rate must be 2x that, not the 200K entry's 6.0. + expect(model.info.cacheWritesPrice1h).toBe(12.0) + expect(model.info.cacheWritesPrice1h).not.toBe(6.0) + }) + + it("bills a 1M-context 1h cache write at the tier rate, not the 200K rate", () => { + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + const { totalCost } = calculateApiCostAnthropic(model.info, 0, 0, 100_000, 0, 100_000) + + // Tier 1h rate 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // The pre-fix behaviour billed the 200K 1h rate: 0.6 (50% under). + expect(totalCost).toBeCloseTo(1.2, 9) + expect(totalCost).not.toBeCloseTo(0.6, 9) + }) + it("should honor a custom/unrecognized model ID instead of silently falling back to anthropicDefaultModelId", () => { const handler = new AnthropicHandler({ apiKey: "test-api-key", diff --git a/src/api/providers/__tests__/minimax.spec.ts b/src/api/providers/__tests__/minimax.spec.ts index 5d87ef3de8..e33193a6d5 100644 --- a/src/api/providers/__tests__/minimax.spec.ts +++ b/src/api/providers/__tests__/minimax.spec.ts @@ -373,6 +373,38 @@ describe("MiniMaxHandler", () => { expect(firstChunk.value).toEqual({ type: "usage", inputTokens: 10, outputTokens: 20 }) }) + it("createMessage should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockResolvedValueOnce( + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage("system prompt", []) + const firstChunk = await stream.next() + + expect(firstChunk.done).toBe(false) + expect(firstChunk.value).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 60_000, + }) + }) + it("createMessage should pass correct parameters to MiniMax client", async () => { const modelId: MinimaxModelId = "MiniMax-M2" const modelInfo = minimaxModels[modelId] diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index c8a4407950..02fb7ef4ae 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -275,6 +275,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from + // the 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 677f38bbf5..8811645fa8 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -442,6 +442,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // The tier's own TTL rate must come along: the entry-level rate was + // derived from the 200K input price, so keeping it here would bill a + // 1M-context 1h write at half its tier rate. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index e09cc0b17b..d11f5f7dba 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -1295,6 +1295,10 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH inputPrice: tier?.inputPrice ?? modelConfig.info.inputPrice, outputPrice: tier?.outputPrice ?? modelConfig.info.outputPrice, cacheWritesPrice: tier?.cacheWritesPrice ?? modelConfig.info.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from the + // 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier?.cacheWritesPrice1h ?? modelConfig.info.cacheWritesPrice1h, + cacheWritesPrice12h: tier?.cacheWritesPrice12h ?? modelConfig.info.cacheWritesPrice12h, cacheReadsPrice: tier?.cacheReadsPrice ?? modelConfig.info.cacheReadsPrice, } } diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index bae737e9cc..be0a1bc12a 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -16,6 +16,8 @@ import { SERVICE_TIER_KEY, type ServiceTier, ApiProviderError, + CACHE_WRITE_MULTIPLIER_1H, + CACHE_WRITE_MULTIPLIER_12H, } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" @@ -1490,6 +1492,17 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio outputPrice: tierInfo.outputPrice ?? info.outputPrice, cacheReadsPrice: tierInfo.cacheReadsPrice ?? info.cacheReadsPrice, cacheWritesPrice: tierInfo.cacheWritesPrice ?? info.cacheWritesPrice, + // A service tier that overrides the write rate must also override the TTL + // rate, otherwise a flex/priority 12h write bills at the standard-tier 12h + // rate. Derived from the tier's own input price so the 2.5x ratio holds. + cacheWritesPrice12h: + tierInfo.cacheWritesPrice12h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_12H : undefined) ?? + info.cacheWritesPrice12h, + cacheWritesPrice1h: + tierInfo.cacheWritesPrice1h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_1H : undefined) ?? + info.cacheWritesPrice1h, } } diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index af6f9b87fb..8a904e94d6 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -4529,7 +4529,10 @@ export class Task extends EventEmitter implements TaskLike { modelId, ) - // Use the appropriate cost function based on the API protocol + // Use the appropriate cost function based on the API protocol. + // The per-TTL split must be passed here too: without it this + // telemetry recompute bills every write at the short-TTL rate, + // which is exactly the under-report this PR exists to fix. const costResult = apiProtocol === "anthropic" ? calculateApiCostAnthropic( @@ -4538,6 +4541,8 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4545,6 +4550,9 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + undefined, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) TelemetryService.instance.captureLlmCompletion(this.taskId, { diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index bbf77a85e9..122d1af8bc 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -534,6 +534,53 @@ describe("Cline", () => { }) }) + describe("cache-write TTL split in telemetry", () => { + it("bills a 1h cache write at the 1h rate when usage is captured after a cancelled stream", async () => { + // glm P1: the telemetry cost recompute inside captureUsageData must receive + // the per-TTL split. Before the fix it passed only the aggregate write count, + // so a 1h write was billed at the 5m rate (0.375 instead of 0.6 for 100k + // tokens at claude-3-5-sonnet's 3.0 input price). + const task = await createTaskWithManualRetries() + const captureLlmCompletion = vi.spyOn(TelemetryService.instance, "captureLlmCompletion") + + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) + // A stream that reports the split but never yields a final cost chunk, which is + // the shape a cancelled request leaves behind. The second call throws so the + // request loop terminates the way the neighbouring tests do. + vi.spyOn(task, "attemptApiRequest") + .mockImplementationOnce(() => + stream([ + { + type: "usage", + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 100_000, + cacheReadTokens: 0, + }, + { type: "text", text: "partial response" }, + ]), + ) + .mockImplementation(() => { + throw new Error("stop after usage capture") + }) + + await task.recursivelyMakeClineRequests([{ type: "text", text: "original user request" }]) + + // Allow the fire-and-forget drain to settle. + await new Promise((resolve) => setTimeout(resolve, 50)) + + const call = captureLlmCompletion.mock.calls.find(([, payload]) => payload?.cacheWriteTokens === 100_000) + expect(call).toBeDefined() + + // claude-3-5-sonnet-20241022: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-fix lumped behaviour billed (3.75 / 1_000_000) * 100_000 = 0.375. + expect(call![1].cost).toBeCloseTo(0.6, 9) + expect(call![1].cost).not.toBeCloseTo(0.375, 9) + }) + }) + describe("request continuation", () => { it("sends the no-tool-use reminder in a second request when the first response has no tool use", async () => { const task = await createTaskWithManualRetries() From 025ff9c59d2580fb384e79fe656eaae1d336a0b9 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:12:34 +0900 Subject: [PATCH 04/14] test(cost): correct the mixed-split test label to match its arithmetic What: - Rename "bills a mixed 5m/1h split per class (0.465 case)" to "(60/40 split, 0.51)". Why: The label cited a figure the test does not assert. A 60/40 split of 100k writes at 3.75/6.0 costs 0.15 + 0.36 = 0.51, not 0.465. The assertion was already correct; only the name was misleading, and a reviewer reading the label would have expected a different number. Impact: Test name only, no behaviour change. --- src/utils/__tests__/cost.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b27eb4100a..304a1fb21b 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -337,7 +337,7 @@ describe("Cost Utility", () => { expect(result.totalCost).not.toBeCloseTo(0.375, 9) }) - it("bills a mixed 5m/1h split per class (0.465 case)", () => { + it("bills a mixed 5m/1h split per class (60/40 split, 0.51)", () => { // 60k of the 100k writes are 1h, 40k remain at the 5m rate. const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 60_000) From 282ee30628167e6b4e809aec239f2efa6dc13bc4 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:15:45 +0900 Subject: [PATCH 05/14] test(openai-native): cover the 12h rate under each service tier What: - Add a per-tier case asserting the 12h write rate follows the tier's own input price (default 25.0, flex 12.5, priority 50.0 for gpt-6-astra). Why: The service-tier seam was changed to derive the TTL rates from the tier input price, but nothing pinned that behaviour. Without a test, a future edit could revert to the standard-tier 12h rate and a flex/priority 12h write would be mis-billed by 2x in either direction. Impact: Test only. Fails without the tier derivation (flex bills 2.5 instead of 1.25, priority 2.5 instead of 5.0). --- .../__tests__/openai-native-usage.spec.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 3725a801bc..44ac2deaf1 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -528,6 +528,35 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(expectedCost, 6) }) + it.each([ + // gpt-6-astra: default input 10.0 / 12h 25.0; flex input 5.0; priority input 20.0. + // The 12h rate must follow the tier's own input price (2.5x), not the + // standard-tier 12h rate, or a flex/priority 12h write is mis-billed. + { tier: OpenAiServiceTier.Default, expectedCost: 2.5 }, + { tier: OpenAiServiceTier.Flex, expectedCost: 1.25 }, + { tier: OpenAiServiceTier.Priority, expectedCost: 5.0 }, + ])("derives the 12h rate from the $tier tier's own input price", ({ tier, expectedCost }) => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + openAiNativeServiceTier: tier, + }) + const result = handler["normalizeUsage"]( + { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { ttl_12h_tokens: 100_000 }, + }, + }, + getGpt6AstraModel(), + ) + + if (!result) throw new Error("Expected usage") + // All input is consumed by the write, so the whole cost is the 12h class. + expect(result.totalCost).toBeCloseTo(expectedCost, 6) + }) + it("should pass total input tokens to calculateApiCostOpenAI", () => { const usage = { input_tokens: 100, From 65ac86200585008f1a13013d18570fead277e73b Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 04:08:35 +0900 Subject: [PATCH 06/14] feat(api): emit an opt-in prompt cache TTL gated by catalog capability What: add a `promptCacheTtl` provider-settings knob ("5m" | "1h"), a `ModelInfo.promptCacheTtlSupport` capability array, and a shared `resolveCacheTtl`/`cacheControl` helper that the anthropic, vertex and vercel-ai-gateway caching transforms plus the anthropic, anthropic-vertex, openrouter, vercel-ai-gateway and zoo-gateway providers thread through. Why: ADR-0006 phase 2. C5a (#2260) landed per-TTL write accounting, so a 1h cache write is billed at 2x instead of the 5m rate. This is the matching request-side mechanism: once the settings UI ships in C6, a user selecting 1h causes Anthropic-shaped providers to emit `cache_control: { ttl: "1h" }`. Impact: no behavior change by default. An unset knob resolves to `undefined` and the marker stays the byte-identical `{ type: "ephemeral" }` literal, so no `ttl` key is emitted unless the user opts in AND the resolved ModelInfo advertises the capability. Absent capability means 5m only (I2), capability is read from ModelInfo rather than id substrings, and every marker in a request shares one resolved TTL (I4). Dynamic catalogs (OpenRouter, the gateways) carry no capability data and therefore stay at 5m, as the ADR requires. --- .../src/__tests__/prompt-cache-ttl.test.ts | 103 +++++++++ packages/types/src/model.ts | 16 ++ .../types/src/provider-settings/common.ts | 7 +- packages/types/src/providers/anthropic.ts | 25 ++ .../__tests__/anthropic-vertex.spec.ts | 102 +++++++++ src/api/providers/__tests__/anthropic.spec.ts | 72 ++++++ .../providers/__tests__/openrouter.spec.ts | 109 +++++++++ .../__tests__/vercel-ai-gateway.spec.ts | 17 ++ src/api/providers/anthropic-vertex.ts | 19 +- src/api/providers/anthropic.ts | 13 +- src/api/providers/openrouter.ts | 6 +- src/api/providers/vercel-ai-gateway.ts | 3 +- src/api/providers/zoo-gateway.ts | 3 +- .../transform/caching/__tests__/ttl.spec.ts | 216 ++++++++++++++++++ src/api/transform/caching/anthropic.ts | 16 +- src/api/transform/caching/ttl.ts | 32 +++ .../transform/caching/vercel-ai-gateway.ts | 16 +- src/api/transform/caching/vertex.ts | 22 +- 18 files changed, 774 insertions(+), 23 deletions(-) create mode 100644 packages/types/src/__tests__/prompt-cache-ttl.test.ts create mode 100644 src/api/transform/caching/__tests__/ttl.spec.ts create mode 100644 src/api/transform/caching/ttl.ts diff --git a/packages/types/src/__tests__/prompt-cache-ttl.test.ts b/packages/types/src/__tests__/prompt-cache-ttl.test.ts new file mode 100644 index 0000000000..05b76245ee --- /dev/null +++ b/packages/types/src/__tests__/prompt-cache-ttl.test.ts @@ -0,0 +1,103 @@ +// npx vitest run src/__tests__/prompt-cache-ttl.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { vertexModels } from "../providers/vertex.js" +import { openRouterDefaultModelInfo } from "../providers/openrouter.js" +import { + vercelAiGatewayDefaultModelId, + VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS, +} from "../providers/vercel-ai-gateway.js" +import { + DEFAULT_PROMPT_CACHE_TTL, + modelInfoSchema, + promptCacheTtlSchema, + promptCacheTtlValues, + type ModelInfo, +} from "../model.js" +import { anthropicProviderDefinition } from "../provider-settings/anthropic.js" +import { baseProviderSettingsShape } from "../provider-settings/common.js" +import { providerIdentifiers } from "../provider-identifiers.js" +import { globalSettingsSchema } from "../global-settings.js" + +/** + * Catalog invariant for the request-side TTL knob (ADR-0006 phase 2, PR-C5b). + * + * Emission is gated by `ModelInfo.promptCacheTtlSupport`, so a model that can + * report a 1h cache write but carries no capability array would silently never + * emit the TTL the user asked for. These tests pin both directions: capable + * catalogs advertise it, and catalogs that cannot know stay absent (fail closed). + */ +describe("prompt cache TTL schema", () => { + it("accepts exactly the 5m and 1h classes", () => { + expect(promptCacheTtlValues).toEqual(["5m", "1h"]) + expect(promptCacheTtlSchema.parse("5m")).toBe("5m") + expect(promptCacheTtlSchema.parse("1h")).toBe("1h") + expect(promptCacheTtlSchema.safeParse("30m").success).toBe(false) + expect(promptCacheTtlSchema.safeParse("12h").success).toBe(false) + }) + + it("defaults to undefined so an unset knob emits no ttl key", () => { + expect(DEFAULT_PROMPT_CACHE_TTL).toBeUndefined() + }) + + it("is optional on ModelInfo", () => { + const parsed = modelInfoSchema.parse({ contextWindow: 200_000, supportsPromptCache: true }) + expect(parsed.promptCacheTtlSupport).toBeUndefined() + + const withCapability = modelInfoSchema.parse({ + contextWindow: 200_000, + supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], + }) + expect(withCapability.promptCacheTtlSupport).toEqual(["5m", "1h"]) + }) +}) + +describe("prompt cache TTL knob location", () => { + it("lives in provider settings so handlers can see it", () => { + // ApiHandlerOptions = Omit, so only a + // provider-settings key reaches a handler (ADR-0006 decision item 2). + expect(baseProviderSettingsShape.promptCacheTtl).toBeDefined() + + const parsed = anthropicProviderDefinition.schema.safeParse({ + apiProvider: providerIdentifiers.anthropic, + promptCacheTtl: "1h", + }) + expect(parsed.success).toBe(true) + expect(parsed.success && parsed.data.promptCacheTtl).toBe("1h") + }) + + it("is not a global setting", () => { + const shape = globalSettingsSchema.shape as Record + expect(shape.promptCacheTtl).toBeUndefined() + }) +}) + +describe("prompt cache TTL capability catalog", () => { + it("every Anthropic entry with prompt caching advertises both TTL classes", () => { + const missing = Object.entries(anthropicModels as Record) + .filter(([, info]) => info.supportsPromptCache && !info.promptCacheTtlSupport?.includes("1h")) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("leaves the Vertex catalog without capability so it fails closed", () => { + // PR-C5b scopes the catalog capability to the Anthropic entries; the Vertex + // catalog is populated separately, so its models stay 5m-only for now. + const withCapability = Object.entries(vertexModels as Record) + .filter(([, info]) => info.promptCacheTtlSupport !== undefined) + .map(([id]) => id) + + expect(withCapability).toEqual([]) + }) + + it("leaves dynamic-catalog defaults without capability so they fail closed", () => { + // OpenRouter and the gateway catalogs are fetched at runtime and carry no TTL + // capability data, which ADR-0006 maps to "5m only" (invariant I2). + expect(openRouterDefaultModelInfo.promptCacheTtlSupport).toBeUndefined() + expect(VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(vercelAiGatewayDefaultModelId)).toBe(true) + }) +}) diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 0ac2a4c339..b3c2b8151d 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -95,6 +95,20 @@ export type ModelParameter = z.infer export const isModelParameter = (value: string): value is ModelParameter => modelParameters.includes(value as ModelParameter) +/** + * Prompt cache TTL classes a request-side cache marker can carry. `"5m"` is the + * vendor default and is never emitted explicitly, so only `"1h"` changes the wire + * shape and an unset knob stays byte-identical to a pre-knob build (ADR-0006 I3). + */ +export const promptCacheTtlValues = ["5m", "1h"] as const + +export const promptCacheTtlSchema = z.enum(promptCacheTtlValues) + +export type PromptCacheTtl = z.infer + +/** Deliberately `undefined`, not `"5m"`: emitting `ttl: "5m"` would change every request's bytes. */ +export const DEFAULT_PROMPT_CACHE_TTL: PromptCacheTtl | undefined = undefined + /** * ModelInfo */ @@ -112,6 +126,8 @@ export const modelInfoSchema = z.object({ // When set to "24h", extended prompt caching will be requested; when omitted // or set to "in_memory", the default in‑memory cache is used. promptCacheRetention: z.enum(["in_memory", "24h"]).optional(), + // Cache TTLs this model accepts. Absent means "5m only" (ADR-0006 I2, fail closed). + promptCacheTtlSupport: z.array(promptCacheTtlSchema).optional(), // Capability flag to indicate whether the model supports an output verbosity parameter supportsVerbosity: z.boolean().optional(), // Capability flag to indicate whether the model exposes a user-configurable max output diff --git a/packages/types/src/provider-settings/common.ts b/packages/types/src/provider-settings/common.ts index ca78c3a25d..ca908d0c60 100644 --- a/packages/types/src/provider-settings/common.ts +++ b/packages/types/src/provider-settings/common.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { reasoningEffortSettingSchema, verbosityLevelsSchema } from "../model.js" +import { reasoningEffortSettingSchema, verbosityLevelsSchema, promptCacheTtlSchema } from "../model.js" import type { ProviderIdentifier } from "../provider-identifiers.js" export const API_PROVIDER_FIELD = "apiProvider" @@ -33,6 +33,11 @@ export const baseProviderSettingsShape = { modelMaxTokens: z.number().optional(), modelMaxThinkingTokens: z.number().optional(), verbosity: verbosityLevelsSchema.optional(), + // Opt-in prompt cache TTL for this provider profile. Lives here, not in global + // settings, because `ApiHandlerOptions = Omit` + // (src/shared/api.ts) makes a global key invisible to handlers. Unset emits no + // ttl key at all, not "5m" (ADR-0006 I3). + promptCacheTtl: promptCacheTtlSchema.optional(), } export const apiModelIdProviderModelShape = { diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index 7bf55238ac..bd90938931 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -3,6 +3,9 @@ import type { ModelInfo } from "../model.js" // https://docs.anthropic.com/en/docs/about-claude/models // https://platform.claude.com/docs/en/about-claude/pricing +// Every entry here is an Anthropic-hosted Claude model, and 1h cache writes are GA +// on all active Claude models, so each one advertises both TTL classes. The 5m class +// is the vendor default and is never emitted explicitly (ADR-0006 I3). export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-sonnet-4-5" export const ANTHROPIC_API_PROTOCOL = "anthropic" @@ -13,6 +16,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -36,6 +40,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) @@ -56,6 +61,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -79,6 +85,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -102,6 +109,7 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -125,6 +133,7 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -143,6 +152,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required, same as 4.7) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (regular tier) outputPrice: 25.0, // $25 per million output tokens (regular tier) cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -162,6 +172,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -181,6 +192,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens @@ -197,6 +209,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], // Standard pricing (prompts of 100,000 tokens or fewer) from Anthropic's // pricing page. Prompts over 100,000 tokens bill at tiered rates // ($0.50 in / $2.50 out / $0.625 5m cache write / $0.05 cache read per @@ -222,6 +235,7 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache) @@ -240,6 +254,7 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens @@ -258,6 +273,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens @@ -270,6 +286,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens @@ -282,6 +299,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens @@ -294,6 +312,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -307,6 +326,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -318,6 +338,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens @@ -329,6 +350,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: false, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, @@ -340,6 +362,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, @@ -351,6 +374,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 0.25, outputPrice: 1.25, cacheWritesPrice: 0.3, @@ -362,6 +386,7 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 94b339fc58..f4860e9914 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -260,6 +260,108 @@ describe("VertexHandler", () => { ) }) + describe("prompt cache TTL", () => { + const minimalStream = asyncStreamFrom([ + { + type: "message_start", + message: { usage: { input_tokens: 10, output_tokens: 0 } }, + }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // The handler's client is private; reach it once here rather than casting at + // each call site. `messages.create` is replaced with a plain vitest mock. + const stubCreate = (mockCreate: ReturnType) => { + const client = handler["client"] as unknown as { messages: { create: unknown } } + client.messages.create = mockCreate + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on the system prompt and every user marker when the model is capable", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + // The Vertex catalog carries no TTL capability yet, so inject one to + // exercise the emission path this PR adds. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + // System + the user marker the vertex transform emits. + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls.length).toBeGreaterThanOrEqual(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("fails closed when the Vertex catalog advertises no TTL capability", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should handle multiple content blocks with line breaks for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 317a897385..d839003e5d 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -191,6 +191,78 @@ describe("AnthropicHandler", () => { expect(mockCreate).toHaveBeenCalled() }) + describe("prompt cache TTL", () => { + const threeTurnMessages = () => [ + { role: "user" as const, content: [{ type: "text" as const, text: "First message" }] }, + { role: "assistant" as const, content: [{ type: "text" as const, text: "Response" }] }, + { role: "user" as const, content: [{ type: "text" as const, text: "Second message" }] }, + ] + + /** Every cache_control object in the serialized request body. */ + const requestCacheControls = () => { + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const serialized = JSON.parse(JSON.stringify(body)) + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(serialized) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + await collectStream(handler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on all three marker sites when the knob is 1h", async () => { + const ttlHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "1h", + }) + + await collectStream(ttlHandler.createMessage(systemPrompt, threeTurnMessages())) + + // System + last two user messages: a single missed site would show up + // here as a mixed-TTL request (ADR-0006 I4). + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(new Set(controls.map((control) => control.ttl))).toEqual(new Set(["1h"])) + }) + + it("emits no ttl key for an explicit 5m knob", async () => { + const fiveMinuteHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "5m", + }) + + await collectStream(fiveMinuteHandler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should include 1M context beta header for Claude Sonnet 4.6 when enabled", async () => { const sonnet46Handler = new AnthropicHandler({ apiKey: "test-api-key", diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index 1bf6669324..fec3e4a2e2 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -485,6 +485,115 @@ describe("OpenRouterHandler", () => { ) }) + describe("prompt cache TTL", () => { + const mockStreamWithUsage = () => + asyncStreamFrom([ + { id: "test-id", choices: [{ delta: { content: "test response" } }] }, + { id: "test-id", choices: [{ delta: {} }], usage: { prompt_tokens: 10, completion_tokens: 20 } }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // `openai` is mocked for this file, so the prototype is a plain object here. + // Reaching it once keeps the casts out of each test body. + const stubChatCreate = (mockCreate: ReturnType) => { + const proto = OpenAI.prototype as unknown as { chat: { completions: { create: unknown } } } + proto.chat = { completions: { create: mockCreate } } + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ ...mockOptions, openRouterModelId: "anthropic/claude-sonnet-4" }), + ) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("fails closed when the resolved model advertises no TTL capability", async () => { + // OpenRouter models come from a dynamic catalog that carries no TTL + // capability data, so a 1h knob must not reach the wire (ADR-0006 I2, + // and the ADR's stated risk for dynamic catalogs). + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on every marker when the resolved model advertises it", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + // Simulate a catalog that does carry capability data. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + }) + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockStream = asyncStreamFrom([{ error: { message: "API Error", code: 500 } }]) diff --git a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts index e69240c1da..687607d305 100644 --- a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts +++ b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts @@ -449,6 +449,23 @@ describe("VercelAiGatewayHandler", () => { expect(addCacheBreakpoints).toHaveBeenCalled() }) + it("does not emit a ttl for a gateway model with no TTL capability", async () => { + // The gateway catalog is fetched at runtime and carries no TTL capability, + // so a 1h knob must fail closed (ADR-0006 I2). + const handler = new VercelAiGatewayHandler( + makeApiHandlerOptions({ + ...mockOptions, + vercelAiGatewayModelId: "anthropic/claude-3.5-haiku", + promptCacheTtl: "1h", + }), + ) + + await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next() + + const call = mockCreate.mock.calls[mockCreate.mock.calls.length - 1][0] + expect(JSON.stringify(call)).not.toContain("ttl") + }) + it("sets correct max_completion_tokens", async () => { const handler = new VercelAiGatewayHandler(mockOptions) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 02fb7ef4ae..9334e436c3 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -15,6 +15,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { addCacheBreakpoints } from "../transform/caching/vertex" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" @@ -79,6 +80,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple const { supportsPromptCache } = info + // One TTL for the whole request (ADR-0006 I4). + const cacheTtl = resolveCacheTtl(this.options, info) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -116,9 +120,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple thinking, // Cache the system prompt if caching is enabled. system: supportsPromptCache - ? [{ text: systemPrompt, type: "text" as const, cache_control: { type: "ephemeral" } }] + ? [{ text: systemPrompt, type: "text" as const, cache_control: cacheControl(cacheTtl) }] : systemPrompt, - messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages) : sanitizedMessages, + messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages, cacheTtl) : sanitizedMessages, stream: true, ...nativeToolParams, } as Anthropic.Messages.MessageCreateParamsStreaming @@ -322,11 +326,12 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple try { const { id, - info: { supportsPromptCache }, + info, temperature, maxTokens = ANTHROPIC_DEFAULT_MAX_TOKENS, reasoning: thinking, } = this.getModel() + const { supportsPromptCache } = info const params = { model: id, @@ -337,7 +342,13 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple { role: "user", content: supportsPromptCache - ? [{ type: "text" as const, text: prompt, cache_control: { type: "ephemeral" } }] + ? [ + { + type: "text" as const, + text: prompt, + cache_control: cacheControl(resolveCacheTtl(this.options, info)), + }, + ] : prompt, }, ], diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 8811645fa8..45bef00096 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -19,6 +19,7 @@ import { ApiStream } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { BaseProvider } from "./base-provider" import { bridgeExternalAbortSignal, hasUsableTimeout, toSdkTimeoutMs } from "./utils/abort-signal" @@ -64,7 +65,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { let stream: AnthropicStream - const cacheControl: CacheControlEphemeral = { type: "ephemeral" } const { id: modelId, betas = ["fine-grained-tool-streaming-2025-05-14"], @@ -79,6 +79,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa settings: this.options, }) + // One TTL for the whole request (ADR-0006 I4): every marker below reuses + // this single object, so a mixed-TTL request is unrepresentable. + const cacheTtl = resolveCacheTtl(this.options, info) + const marker: CacheControlEphemeral = cacheControl(cacheTtl) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -169,17 +174,17 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa temperature, thinking, // Setting cache breakpoint for system prompt so new tasks can reuse it. - system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + system: [{ text: systemPrompt, type: "text", cache_control: marker }], messages: sanitizedMessages.map((message, index) => { if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { return { ...message, content: typeof message.content === "string" - ? [{ type: "text", text: message.content, cache_control: cacheControl }] + ? [{ type: "text", text: message.content, cache_control: marker }] : message.content.map((content, contentIndex) => contentIndex === message.content.length - 1 - ? { ...content, cache_control: cacheControl } + ? { ...content, cache_control: marker } : content, ), } diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 36e09aa883..6d2057a7a3 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -26,6 +26,7 @@ import { ApiStreamChunk } from "../transform/stream" import { convertToR1Format } from "../transform/r1-format" import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic" import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { OpenRouterReasoningParams } from "../transform/reasoning" import { getModelParams } from "../transform/model-params" @@ -306,9 +307,12 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH // TODO: Add a `promptCacheStratey` field to `ModelInfo`. if (OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId)) { if (modelId.startsWith("google")) { + // Gemini markers keep their default TTL: ADR-0006 scopes the knob to + // Anthropic-shaped markers only, and Gemini has no 1h class. addGeminiCacheBreakpoints(systemPrompt, openAiMessages) } else { - addAnthropicCacheBreakpoints(systemPrompt, openAiMessages) + // Capability comes from the resolved ModelInfo, never from the id. + addAnthropicCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, model.info)) } } diff --git a/src/api/providers/vercel-ai-gateway.ts b/src/api/providers/vercel-ai-gateway.ts index 2f1586b928..135ac8bef6 100644 --- a/src/api/providers/vercel-ai-gateway.ts +++ b/src/api/providers/vercel-ai-gateway.ts @@ -17,6 +17,7 @@ import { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -82,7 +83,7 @@ export class VercelAiGatewayHandler extends RouterProvider implements SingleComp ] if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } const supportsTemperature = info.supportsTemperature !== false && this.supportsTemperature(modelId) diff --git a/src/api/providers/zoo-gateway.ts b/src/api/providers/zoo-gateway.ts index 66ccac5067..5f33ee48fe 100644 --- a/src/api/providers/zoo-gateway.ts +++ b/src/api/providers/zoo-gateway.ts @@ -19,6 +19,7 @@ import { t } from "../../i18n" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -208,7 +209,7 @@ export class ZooGatewayHandler extends RouterProvider implements SingleCompletio // Apply prompt caching for models that support it // Zoo Gateway serves the same models as Vercel AI Gateway, so caching support is identical if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } // Build request headers with enrichment metadata diff --git a/src/api/transform/caching/__tests__/ttl.spec.ts b/src/api/transform/caching/__tests__/ttl.spec.ts new file mode 100644 index 0000000000..ab11d64b4d --- /dev/null +++ b/src/api/transform/caching/__tests__/ttl.spec.ts @@ -0,0 +1,216 @@ +// npx vitest run src/api/transform/caching/__tests__/ttl.spec.ts + +import OpenAI from "openai" + +import { addCacheBreakpoints as addAnthropicBreakpoints } from "../anthropic" +import { addCacheBreakpoints as addVertexBreakpoints } from "../vertex" +import { addCacheBreakpoints as addGatewayBreakpoints } from "../vercel-ai-gateway" +import { cacheControl, resolveCacheTtl } from "../ttl" + +/** + * Collect every `cache_control` object reachable in a serialized request. + * + * Counting occurrences rather than spot-checking one marker is the point: a + * transform that forgot one of its marker sites would still pass a single-site + * assertion, which is exactly the failure mode ADR-0006 I4 is meant to exclude. + */ +const collectCacheControls = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +/** The serialized request body, so assertions run on the bytes that would be sent. */ +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) + +describe("resolveCacheTtl", () => { + it("returns undefined when the knob is unset, even for a capable model", () => { + expect(resolveCacheTtl({}, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + expect(resolveCacheTtl(undefined, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns undefined for an explicit 5m so the wire shape stays unchanged", () => { + expect(resolveCacheTtl({ promptCacheTtl: "5m" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns 1h when the knob asks for it and the model advertises it", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBe("1h") + }) + + it("fails closed when the model advertises no capability at all", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, {})).toBeUndefined() + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, undefined)).toBeUndefined() + }) + + it("fails closed when the model advertises 5m only", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m"] })).toBeUndefined() + }) +}) + +describe("cacheControl", () => { + it("is the historical literal when no TTL is resolved", () => { + expect(cacheControl(undefined)).toEqual({ type: "ephemeral" }) + expect(Object.keys(cacheControl(undefined))).toEqual(["type"]) + }) + + it("carries the ttl key only when a TTL is resolved", () => { + expect(cacheControl("1h")).toEqual({ type: "ephemeral", ttl: "1h" }) + }) +}) + +describe("caching transforms under a prompt cache TTL", () => { + const systemPrompt = "You are a helpful assistant." + + const buildOpenAiMessages = (): OpenAI.Chat.ChatCompletionMessageParam[] => [ + { role: "system", content: systemPrompt }, + { role: "user", content: "First message" }, + { role: "assistant", content: "First response" }, + { role: "user", content: "Second message" }, + ] + + const buildAnthropicMessages = () => [ + { role: "user" as const, content: "First message" }, + { role: "assistant" as const, content: "First response" }, + { role: "user" as const, content: "Second message" }, + ] + + describe("default parity (ADR-0006 I3)", () => { + it("emits no ttl key anywhere when the TTL argument is omitted", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages) + + const serialized = serialize(messages) + const rawJson = JSON.stringify(messages) + + expect(rawJson).not.toContain("ttl") + expect(serialized).not.toContain("ttl") + + const controls = collectCacheControls(serialized) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("is byte-identical to the pre-knob literal for every transform", () => { + const anthropicMessages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, anthropicMessages) + + const gatewayMessages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, gatewayMessages) + + const vertexMessages = addVertexBreakpoints(buildAnthropicMessages()) + + expect(serialize(anthropicMessages)).toEqual( + serialize([ + { + role: "system", + content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], + }, + { + role: "user", + content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], + }, + { role: "assistant", content: "First response" }, + { + role: "user", + content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], + }, + ]), + ) + + expect(serialize(gatewayMessages)).toEqual( + serialize([ + { role: "system", content: systemPrompt, cache_control: { type: "ephemeral" } }, + { + role: "user", + content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], + }, + { role: "assistant", content: "First response" }, + { + role: "user", + content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], + }, + ]), + ) + + expect(serialize(vertexMessages)).toEqual( + serialize([ + { + role: "user", + content: [{ type: "text", text: "First message", cache_control: { type: "ephemeral" } }], + }, + { role: "assistant", content: "First response" }, + { + role: "user", + content: [{ type: "text", text: "Second message", cache_control: { type: "ephemeral" } }], + }, + ]), + ) + }) + }) + + describe("opt-in uniformity (ADR-0006 I4)", () => { + it("puts ttl 1h on every anthropic marker, not just the first", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(controls.every((control) => control.type === "ephemeral")).toBe(true) + }) + + it("puts ttl 1h on every gateway marker", () => { + const messages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("puts ttl 1h on every vertex marker", () => { + const messages = addVertexBreakpoints(buildAnthropicMessages(), "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("never mixes TTLs within one request", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const ttls = new Set(collectCacheControls(serialize(messages)).map((control) => control.ttl)) + expect(ttls).toEqual(new Set(["1h"])) + }) + }) + + describe("signature is additive-optional", () => { + it("keeps the two-argument call compiling and byte-identical", () => { + const withDefault = buildOpenAiMessages() + const withExplicitUndefined = buildOpenAiMessages() + + addAnthropicBreakpoints(systemPrompt, withDefault) + addAnthropicBreakpoints(systemPrompt, withExplicitUndefined, undefined) + + expect(serialize(withDefault)).toEqual(serialize(withExplicitUndefined)) + }) + }) +}) diff --git a/src/api/transform/caching/anthropic.ts b/src/api/transform/caching/anthropic.ts index cff671a56c..3bdcba4ba1 100644 --- a/src/api/transform/caching/anthropic.ts +++ b/src/api/transform/caching/anthropic.ts @@ -1,10 +1,20 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + messages[0] = { role: "system", // @ts-ignore-next-line - content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], + content: [{ type: "text", text: systemPrompt, cache_control: control }], } // Ensure all user messages have content in array format first @@ -35,7 +45,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. } // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } }) } diff --git a/src/api/transform/caching/ttl.ts b/src/api/transform/caching/ttl.ts new file mode 100644 index 0000000000..4ba2652fe2 --- /dev/null +++ b/src/api/transform/caching/ttl.ts @@ -0,0 +1,32 @@ +import type { ModelInfo, PromptCacheTtl } from "@roo-code/types" +import type { CacheControlEphemeral } from "@anthropic-ai/sdk/resources" + +/** + * Resolve the TTL a request may emit, gated by the provider-profile knob and the + * model's catalog capability. Both gates fail closed: absent capability means + * "5m only" (ADR-0006 I2), and capability is read from the resolved `ModelInfo` + * rather than inferred from the model id. + * + * `"5m"` resolves to `undefined` on purpose. It is the vendor default, so emitting + * it would add bytes without changing behavior; only `"1h"` may change the wire, + * which keeps default parity (I3) checkable by a single grep. + */ +export function resolveCacheTtl( + settings: { promptCacheTtl?: PromptCacheTtl } | undefined, + info: Pick | undefined, +): PromptCacheTtl | undefined { + if (settings?.promptCacheTtl !== "1h") { + return undefined + } + + return info?.promptCacheTtlSupport?.includes("1h") ? "1h" : undefined +} + +/** + * Build the `cache_control` object for one marker. Every marker in a request goes + * through this function, which makes one uniform TTL per request (I4) structural + * instead of a convention each call site has to remember. + */ +export function cacheControl(ttl: PromptCacheTtl | undefined): CacheControlEphemeral { + return ttl ? { type: "ephemeral", ttl } : { type: "ephemeral" } +} diff --git a/src/api/transform/caching/vercel-ai-gateway.ts b/src/api/transform/caching/vercel-ai-gateway.ts index 0d3834bbe5..7272de34d7 100644 --- a/src/api/transform/caching/vercel-ai-gateway.ts +++ b/src/api/transform/caching/vercel-ai-gateway.ts @@ -1,12 +1,22 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + // Apply cache_control to system message at the message level messages[0] = { role: "system", content: systemPrompt, // @ts-ignore-next-line - cache_control: { type: "ephemeral" }, + cache_control: control, } // Add cache_control to the last two user messages for conversation context caching @@ -23,7 +33,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. if (lastTextPart && lastTextPart.text && lastTextPart.text.length > 0) { // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } } }) diff --git a/src/api/transform/caching/vertex.ts b/src/api/transform/caching/vertex.ts index 48bf261587..e223bb4d11 100644 --- a/src/api/transform/caching/vertex.ts +++ b/src/api/transform/caching/vertex.ts @@ -1,6 +1,13 @@ import { Anthropic } from "@anthropic-ai/sdk" -export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + messages: Anthropic.Messages.MessageParam[], + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam[] { // Find indices of user messages that we want to cache. // We only cache the last two user messages to stay within the 4-block limit // (1 block for system + 1 block each for last two user messages = 3 total). @@ -12,18 +19,23 @@ export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) return messages.map((message, index) => message.role !== "assistant" && (index === lastIndex || index === secondLastIndex) - ? cachedMessage(message) + ? cachedMessage(message, ttl) : message, ) } -function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Messages.MessageParam { +function cachedMessage( + message: Anthropic.Messages.MessageParam, + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam { + const control = cacheControl(ttl) + // For string content, we convert to array format with optional cache control. if (typeof message.content === "string") { return { ...message, // For string content, we only have one block so it's always the last block. - content: [{ type: "text" as const, text: message.content, cache_control: { type: "ephemeral" } }], + content: [{ type: "text" as const, text: message.content, cache_control: control }], } } @@ -41,7 +53,7 @@ function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Mess ? { ...content, // Check if this is the last text block using our pre-calculated index. - ...(index === lastTextBlockIndex && { cache_control: { type: "ephemeral" } }), + ...(index === lastTextBlockIndex && { cache_control: control }), } : content, ), From 1e2601247413626cd0ee94cc35f584982daa67f7 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 07:14:00 +0900 Subject: [PATCH 07/14] feat(settings): offer the prompt cache TTL control on Anthropic profiles What: adds the 1h cache TTL selector to the Anthropic provider panel, gated on the model's catalog capability, plus the four locale keys in all 18 locales. Why: ADR-0006 decision item 3 ships the knob last, after the accounting (C5a) and the mechanism (C5b) so no request can emit a 1h write that is not priced. The control is the user-facing half; it renders only where promptCacheTtlSupport advertises 1h (invariant I2), and selecting the default writes DEFAULT_PROMPT_CACHE_TTL (undefined) rather than a literal "5m" so an unset knob keeps emitting no ttl key (invariant I3). Impact: Anthropic profiles gain an opt-in control; default behavior and wire bytes are unchanged for every profile that leaves it alone. Test: webview-ui Anthropic.spec.tsx 8/8. --- .../settings/providers/Anthropic.tsx | 38 +++++ .../providers/__tests__/Anthropic.spec.tsx | 135 ++++++++++++++++++ webview-ui/src/i18n/locales/ca/settings.json | 4 + webview-ui/src/i18n/locales/de/settings.json | 4 + webview-ui/src/i18n/locales/en/settings.json | 4 + webview-ui/src/i18n/locales/es/settings.json | 4 + webview-ui/src/i18n/locales/fr/settings.json | 4 + webview-ui/src/i18n/locales/hi/settings.json | 4 + webview-ui/src/i18n/locales/id/settings.json | 4 + webview-ui/src/i18n/locales/it/settings.json | 4 + webview-ui/src/i18n/locales/ja/settings.json | 4 + webview-ui/src/i18n/locales/ko/settings.json | 4 + webview-ui/src/i18n/locales/nl/settings.json | 4 + webview-ui/src/i18n/locales/pl/settings.json | 4 + .../src/i18n/locales/pt-BR/settings.json | 4 + webview-ui/src/i18n/locales/ru/settings.json | 4 + webview-ui/src/i18n/locales/tr/settings.json | 4 + webview-ui/src/i18n/locales/vi/settings.json | 4 + .../src/i18n/locales/zh-CN/settings.json | 4 + .../src/i18n/locales/zh-TW/settings.json | 4 + 20 files changed, 245 insertions(+) create mode 100644 webview-ui/src/components/settings/providers/__tests__/Anthropic.spec.tsx diff --git a/webview-ui/src/components/settings/providers/Anthropic.tsx b/webview-ui/src/components/settings/providers/Anthropic.tsx index f1849d7c0d..05358133ec 100644 --- a/webview-ui/src/components/settings/providers/Anthropic.tsx +++ b/webview-ui/src/components/settings/providers/Anthropic.tsx @@ -3,10 +3,12 @@ import { Checkbox } from "vscrui" import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react" import type { ProviderSettings } from "@roo-code/types" +import { DEFAULT_PROMPT_CACHE_TTL } from "@roo-code/types" import { useAppTranslation } from "@src/i18n/TranslationContext" import { VSCodeButtonLink } from "@src/components/common/VSCodeButtonLink" import { useSelectedModel } from "@src/components/ui/hooks/useSelectedModel" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@src/components/ui" import { inputEventTransform, noTransform } from "../transforms" @@ -29,6 +31,16 @@ export const Anthropic = ({ apiConfiguration, setApiConfigurationField }: Anthro selectedModel?.id === "claude-sonnet-4-6" || selectedModel?.id === "claude-opus-4-6" + // The 1h cache TTL is only offered where the catalog advertises it, so the + // control cannot produce a value the provider would reject. Absent capability + // metadata means 5m only (ADR-0006 I2). + const supportsPromptCacheTtl1h = selectedModel?.info?.promptCacheTtlSupport?.includes("1h") ?? false + + // Unset must stay absent so the request carries no ttl key at all (ADR-0006 I3), + // which is exactly what the shared default says. The control still needs a + // visible selection, so an unset knob renders as the vendor's own 5m default. + const promptCacheTtl = apiConfiguration.promptCacheTtl ?? "5m" + const handleInputChange = useCallback( ( field: K, @@ -103,6 +115,32 @@ export const Anthropic = ({ apiConfiguration, setApiConfigurationField }: Anthro )} + {supportsPromptCacheTtl1h && ( +
+ + +
+ {t("settings:providers.promptCacheTtlDescription")} +
+
+ )} ) } diff --git a/webview-ui/src/components/settings/providers/__tests__/Anthropic.spec.tsx b/webview-ui/src/components/settings/providers/__tests__/Anthropic.spec.tsx new file mode 100644 index 0000000000..2229adb404 --- /dev/null +++ b/webview-ui/src/components/settings/providers/__tests__/Anthropic.spec.tsx @@ -0,0 +1,135 @@ +// pnpm --filter @roo-code/vscode-webview test src/components/settings/providers/__tests__/Anthropic.spec.tsx + +import React from "react" + +import { providerIdentifiers, type ModelInfo, type ProviderSettings } from "@roo-code/types" + +import { fireEvent, render, screen } from "@/utils/test-utils" + +import { Anthropic } from "../Anthropic" + +const selectedModelInfo = vi.hoisted(() => ({ value: undefined as ModelInfo | undefined })) + +vi.mock("@src/components/ui/hooks/useSelectedModel", () => ({ + useSelectedModel: () => ({ id: "claude-sonnet-4-5", info: selectedModelInfo.value }), +})) + +vi.mock("@src/i18n/TranslationContext", () => ({ + useAppTranslation: () => ({ t: (key: string) => key }), +})) + +vi.mock("vscrui", () => ({ + Checkbox: ({ children, checked, onChange }: any) => ( + + ), +})) + +vi.mock("@vscode/webview-ui-toolkit/react", () => ({ + VSCodeTextField: ({ children, value, onInput, type }: any) => ( +
+ {children} + onInput(event)} /> +
+ ), +})) + +vi.mock("@src/components/common/VSCodeButtonLink", () => ({ + VSCodeButtonLink: ({ children }: any) => {children}, +})) + +// Radix Select renders its content in a portal and needs pointer events JSDOM +// does not provide, so the mock keeps the controlled value and the option list +// addressable through plain DOM. +vi.mock("@src/components/ui", () => ({ + Select: ({ children, value, onValueChange }: any) => ( + + ), + SelectContent: ({ children }: any) => <>{children}, + SelectItem: ({ children, value }: any) => , + SelectTrigger: () => null, + SelectValue: () => null, +})) + +const capableModel: ModelInfo = { + contextWindow: 200_000, + supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], +} + +const fiveMinuteOnlyModel: ModelInfo = { + contextWindow: 200_000, + supportsPromptCache: true, + promptCacheTtlSupport: ["5m"], +} + +const baseConfiguration: ProviderSettings = { + apiProvider: providerIdentifiers.anthropic, + apiKey: "test-api-key", +} + +const renderPanel = (apiConfiguration: ProviderSettings = baseConfiguration) => { + const setApiConfigurationField = vi.fn() + + render() + + return { setApiConfigurationField } +} + +describe("Anthropic prompt cache TTL control", () => { + beforeEach(() => { + selectedModelInfo.value = capableModel + }) + + it("renders the control for a model that advertises the 1h class", () => { + renderPanel() + + expect(screen.getByTestId("prompt-cache-ttl")).toBeInTheDocument() + expect(screen.getAllByRole("option").map((option) => option.getAttribute("value"))).toEqual(["5m", "1h"]) + }) + + it.each([ + ["the catalog advertises 5m only", fiveMinuteOnlyModel], + ["the catalog carries no capability metadata", { contextWindow: 200_000, supportsPromptCache: true }], + ["no model info resolved", undefined], + ])("hides the control when %s", (_label, info) => { + selectedModelInfo.value = info + renderPanel() + + expect(screen.queryByTestId("prompt-cache-ttl")).not.toBeInTheDocument() + }) + + it("shows the 5 minute default when the knob is unset", () => { + renderPanel() + + expect(screen.getByRole("combobox", { name: "prompt-cache-ttl" })).toHaveValue("5m") + }) + + it("shows the stored 1h value rather than the default", () => { + renderPanel({ ...baseConfiguration, promptCacheTtl: "1h" }) + + expect(screen.getByRole("combobox", { name: "prompt-cache-ttl" })).toHaveValue("1h") + }) + + it("writes 1h when the user selects it", () => { + const { setApiConfigurationField } = renderPanel() + + fireEvent.change(screen.getByRole("combobox", { name: "prompt-cache-ttl" }), { target: { value: "1h" } }) + + expect(setApiConfigurationField).toHaveBeenCalledWith("promptCacheTtl", "1h") + }) + + it("clears the knob instead of storing an explicit 5m", () => { + // An explicit "5m" would be a second way to say "default" and would make + // the wire-shape guarantee depend on the resolver rather than on absence. + const { setApiConfigurationField } = renderPanel({ ...baseConfiguration, promptCacheTtl: "1h" }) + + fireEvent.change(screen.getByRole("combobox", { name: "prompt-cache-ttl" }), { target: { value: "5m" } }) + + expect(setApiConfigurationField).toHaveBeenCalledWith("promptCacheTtl", undefined) + }) +}) diff --git a/webview-ui/src/i18n/locales/ca/settings.json b/webview-ui/src/i18n/locales/ca/settings.json index d39d8edd6a..7e6464a271 100644 --- a/webview-ui/src/i18n/locales/ca/settings.json +++ b/webview-ui/src/i18n/locales/ca/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Passar la clau API d'Anthropic com a capçalera d'autorització en lloc de X-Api-Key", "anthropic1MContextBetaLabel": "Activa la finestra de context d'1M (Beta)", "anthropic1MContextBetaDescription": "Amplia la finestra de context a 1 milió de tokens per a Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Durada de la memòria cau de prompts", + "promptCacheTtl5m": "5 minuts (per defecte)", + "promptCacheTtl1h": "1 hora", + "promptCacheTtlDescription": "Quant de temps els prefixos de prompt emmagatzemats a la memòria cau romanen vàlids. La durada d'1 hora costa més per escriptura a la memòria cau però sobreviu a intervals més llargs entre sol·licituds. Deixa-ho en 5 minuts tret que tornis a enviar el mateix context llarg repetidament.", "awsBedrock1MContextBetaLabel": "Activa la finestra de context d'1M (Beta)", "awsBedrock1MContextBetaDescription": "Amplia la finestra de context a 1 milió de tokens per a Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Activa la finestra de context d'1M (Beta)", diff --git a/webview-ui/src/i18n/locales/de/settings.json b/webview-ui/src/i18n/locales/de/settings.json index 983db1df5d..3567fda6c0 100644 --- a/webview-ui/src/i18n/locales/de/settings.json +++ b/webview-ui/src/i18n/locales/de/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Anthropic API-Schlüssel als Authorization-Header anstelle von X-Api-Key übergeben", "anthropic1MContextBetaLabel": "1M Kontextfenster aktivieren (Beta)", "anthropic1MContextBetaDescription": "Erweitert das Kontextfenster für Claude Sonnet 4.x / Claude Opus 4.6 auf 1 Million Token", + "promptCacheTtlLabel": "Lebensdauer des Prompt-Caches", + "promptCacheTtl5m": "5 Minuten (Standard)", + "promptCacheTtl1h": "1 Stunde", + "promptCacheTtlDescription": "Wie lange zwischengespeicherte Prompt-Präfixe gültig bleiben. Die Lebensdauer von 1 Stunde kostet mehr pro Cache-Schreibvorgang, übersteht aber längere Pausen zwischen Anfragen. Lass sie bei 5 Minuten, es sei denn, du sendest denselben langen Kontext wiederholt erneut.", "awsBedrock1MContextBetaLabel": "1M Kontextfenster aktivieren (Beta)", "awsBedrock1MContextBetaDescription": "Erweitert das Kontextfenster für Claude Sonnet 4.x / Claude Opus 4.6 auf 1 Million Token", "vertex1MContextBetaLabel": "1M Kontextfenster aktivieren (Beta)", diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index ea924ce79e..6576bccf6c 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -575,6 +575,10 @@ "anthropicUseAuthToken": "Pass Anthropic API Key as Authorization header instead of X-Api-Key", "anthropic1MContextBetaLabel": "Enable 1M context window (Beta)", "anthropic1MContextBetaDescription": "Extends context window to 1 million tokens for Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Prompt cache lifetime", + "promptCacheTtl5m": "5 minutes (default)", + "promptCacheTtl1h": "1 hour", + "promptCacheTtlDescription": "How long cached prompt prefixes stay valid. The 1 hour lifetime costs more per cache write but survives longer gaps between requests. Leave it on 5 minutes unless you repeatedly resend the same long context.", "awsBedrock1MContextBetaLabel": "Enable 1M context window (Beta)", "awsBedrock1MContextBetaDescription": "Extends context window to 1 million tokens for Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Enable 1M context window (Beta)", diff --git a/webview-ui/src/i18n/locales/es/settings.json b/webview-ui/src/i18n/locales/es/settings.json index 57612bf3e5..ecf12a2a86 100644 --- a/webview-ui/src/i18n/locales/es/settings.json +++ b/webview-ui/src/i18n/locales/es/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Pasar la clave API de Anthropic como encabezado de autorización en lugar de X-Api-Key", "anthropic1MContextBetaLabel": "Habilitar ventana de contexto de 1M (Beta)", "anthropic1MContextBetaDescription": "Amplía la ventana de contexto a 1 millón de tokens para Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Duración de la caché de prompts", + "promptCacheTtl5m": "5 minutos (predeterminado)", + "promptCacheTtl1h": "1 hora", + "promptCacheTtlDescription": "Cuánto tiempo los prefijos de prompt en caché permanecen válidos. La duración de 1 hora cuesta más por escritura en caché pero sobrevive a intervalos más largos entre solicitudes. Déjalo en 5 minutos a menos que reenvíes repetidamente el mismo contexto largo.", "awsBedrock1MContextBetaLabel": "Habilitar ventana de contexto de 1M (Beta)", "awsBedrock1MContextBetaDescription": "Amplía la ventana de contexto a 1 millón de tokens para Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Habilitar ventana de contexto de 1M (Beta)", diff --git a/webview-ui/src/i18n/locales/fr/settings.json b/webview-ui/src/i18n/locales/fr/settings.json index a91a5137d3..d039530606 100644 --- a/webview-ui/src/i18n/locales/fr/settings.json +++ b/webview-ui/src/i18n/locales/fr/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Passer la clé API Anthropic comme en-tête d'autorisation au lieu de X-Api-Key", "anthropic1MContextBetaLabel": "Activer la fenêtre de contexte de 1M (Bêta)", "anthropic1MContextBetaDescription": "Étend la fenêtre de contexte à 1 million de tokens pour Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Durée du cache de prompts", + "promptCacheTtl5m": "5 minutes (par défaut)", + "promptCacheTtl1h": "1 heure", + "promptCacheTtlDescription": "Combien de temps les préfixes de prompt mis en cache restent valides. La durée d'1 heure coûte plus cher par écriture en cache mais survit à des intervalles plus longs entre les requêtes. Laissez-la sur 5 minutes sauf si vous renvoyez plusieurs fois le même contexte long.", "awsBedrock1MContextBetaLabel": "Activer la fenêtre de contexte de 1M (Bêta)", "awsBedrock1MContextBetaDescription": "Étend la fenêtre de contexte à 1 million de tokens pour Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Activer la fenêtre de contexte de 1M (Bêta)", diff --git a/webview-ui/src/i18n/locales/hi/settings.json b/webview-ui/src/i18n/locales/hi/settings.json index bc6e973623..9fb43298c0 100644 --- a/webview-ui/src/i18n/locales/hi/settings.json +++ b/webview-ui/src/i18n/locales/hi/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "X-Api-Key के बजाय Anthropic API कुंजी को Authorization हेडर के रूप में पास करें", "anthropic1MContextBetaLabel": "1M संदर्भ विंडो सक्षम करें (बीटा)", "anthropic1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6 के लिए संदर्भ विंडो को 1 मिलियन टोकन तक बढ़ाता है", + "promptCacheTtlLabel": "प्रॉम्प्ट कैश जीवनकाल", + "promptCacheTtl5m": "5 मिनट (डिफ़ॉल्ट)", + "promptCacheTtl1h": "1 घंटा", + "promptCacheTtlDescription": "कैश किए गए प्रॉम्प्ट प्रीफ़िक्स कितने समय तक मान्य रहते हैं। 1 घंटे का जीवनकाल प्रति कैश लेखन में अधिक खर्च करता है लेकिन अनुरोधों के बीच लंबे अंतराल में भी बना रहता है। इसे 5 मिनट पर ही रखें जब तक कि आप बार-बार वही लंबा संदर्भ न भेजें।", "awsBedrock1MContextBetaLabel": "1M संदर्भ विंडो सक्षम करें (बीटा)", "awsBedrock1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6 के लिए संदर्भ विंडो को 1 मिलियन टोकन तक बढ़ाता है", "vertex1MContextBetaLabel": "1M संदर्भ विंडो सक्षम करें (बीटा)", diff --git a/webview-ui/src/i18n/locales/id/settings.json b/webview-ui/src/i18n/locales/id/settings.json index 3ed57f373e..793d2818e1 100644 --- a/webview-ui/src/i18n/locales/id/settings.json +++ b/webview-ui/src/i18n/locales/id/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Kirim Anthropic API Key sebagai Authorization header alih-alih X-Api-Key", "anthropic1MContextBetaLabel": "Aktifkan jendela konteks 1M (Beta)", "anthropic1MContextBetaDescription": "Memperluas jendela konteks menjadi 1 juta token untuk Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Masa berlaku cache prompt", + "promptCacheTtl5m": "5 menit (bawaan)", + "promptCacheTtl1h": "1 jam", + "promptCacheTtlDescription": "Berapa lama prefiks prompt yang di-cache tetap valid. Masa berlaku 1 jam lebih mahal per penulisan cache tetapi bertahan melewati jeda yang lebih panjang antar permintaan. Biarkan pada 5 menit kecuali Anda berulang kali mengirim ulang konteks panjang yang sama.", "awsBedrock1MContextBetaLabel": "Aktifkan jendela konteks 1M (Beta)", "awsBedrock1MContextBetaDescription": "Memperluas jendela konteks menjadi 1 juta token untuk Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Aktifkan jendela konteks 1M (Beta)", diff --git a/webview-ui/src/i18n/locales/it/settings.json b/webview-ui/src/i18n/locales/it/settings.json index b708304e63..0d9b20fbe7 100644 --- a/webview-ui/src/i18n/locales/it/settings.json +++ b/webview-ui/src/i18n/locales/it/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Passa la chiave API Anthropic come header di autorizzazione invece di X-Api-Key", "anthropic1MContextBetaLabel": "Abilita finestra di contesto da 1M (Beta)", "anthropic1MContextBetaDescription": "Estende la finestra di contesto a 1 milione di token per Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Durata della cache dei prompt", + "promptCacheTtl5m": "5 minuti (predefinito)", + "promptCacheTtl1h": "1 ora", + "promptCacheTtlDescription": "Per quanto tempo i prefissi dei prompt memorizzati nella cache rimangono validi. La durata di 1 ora costa di più per scrittura in cache ma sopravvive a intervalli più lunghi tra le richieste. Lasciala su 5 minuti a meno che tu non invii ripetutamente lo stesso contesto lungo.", "awsBedrock1MContextBetaLabel": "Abilita finestra di contesto da 1M (Beta)", "awsBedrock1MContextBetaDescription": "Estende la finestra di contesto a 1 milione di token per Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Abilita finestra di contesto da 1M (Beta)", diff --git a/webview-ui/src/i18n/locales/ja/settings.json b/webview-ui/src/i18n/locales/ja/settings.json index f6959b50dc..94c4c967eb 100644 --- a/webview-ui/src/i18n/locales/ja/settings.json +++ b/webview-ui/src/i18n/locales/ja/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Anthropic APIキーをX-Api-Keyの代わりにAuthorizationヘッダーとして渡す", "anthropic1MContextBetaLabel": "1Mコンテキストウィンドウを有効にする（ベータ版）", "anthropic1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6のコンテキストウィンドウを100万トークンに拡張します", + "promptCacheTtlLabel": "プロンプトキャッシュの有効期間", + "promptCacheTtl5m": "5分（デフォルト）", + "promptCacheTtl1h": "1時間", + "promptCacheTtlDescription": "キャッシュされたプロンプトプレフィックスが有効な期間です。1時間の有効期間はキャッシュ書き込みあたりのコストが高くなりますが、リクエスト間の長い間隔に耐えられます。同じ長いコンテキストを繰り返し送信しない限り、5分のままにしてください。", "awsBedrock1MContextBetaLabel": "1Mコンテキストウィンドウを有効にする（ベータ版）", "awsBedrock1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6のコンテキストウィンドウを100万トークンに拡張します", "vertex1MContextBetaLabel": "1Mコンテキストウィンドウを有効にする（ベータ版）", diff --git a/webview-ui/src/i18n/locales/ko/settings.json b/webview-ui/src/i18n/locales/ko/settings.json index 6cadbd8a3b..63da101265 100644 --- a/webview-ui/src/i18n/locales/ko/settings.json +++ b/webview-ui/src/i18n/locales/ko/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "X-Api-Key 대신 Authorization 헤더로 Anthropic API 키 전달", "anthropic1MContextBetaLabel": "1M 컨텍스트 창 활성화 (베타)", "anthropic1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6의 컨텍스트 창을 100만 토큰으로 확장", + "promptCacheTtlLabel": "프롬프트 캐시 수명", + "promptCacheTtl5m": "5분 (기본값)", + "promptCacheTtl1h": "1시간", + "promptCacheTtlDescription": "캐시된 프롬프트 접두사가 유효하게 유지되는 기간입니다. 1시간 수명은 캐시 쓰기당 비용이 더 높지만 요청 사이의 긴 간격에도 유지됩니다. 동일한 긴 컨텍스트를 반복해서 다시 보내지 않는 한 5분으로 두세요.", "awsBedrock1MContextBetaLabel": "1M 컨텍스트 창 활성화 (베타)", "awsBedrock1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6의 컨텍스트 창을 100만 토큰으로 확장", "vertex1MContextBetaLabel": "1M 컨텍스트 창 활성화 (베타)", diff --git a/webview-ui/src/i18n/locales/nl/settings.json b/webview-ui/src/i18n/locales/nl/settings.json index 992e9c53e0..a7d009eb60 100644 --- a/webview-ui/src/i18n/locales/nl/settings.json +++ b/webview-ui/src/i18n/locales/nl/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Anthropic API-sleutel als Authorization-header doorgeven in plaats van X-Api-Key", "anthropic1MContextBetaLabel": "1M contextvenster inschakelen (bèta)", "anthropic1MContextBetaDescription": "Breidt het contextvenster uit tot 1 miljoen tokens voor Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Levensduur van promptcache", + "promptCacheTtl5m": "5 minuten (standaard)", + "promptCacheTtl1h": "1 uur", + "promptCacheTtlDescription": "Hoe lang gecachte promptprefixen geldig blijven. De levensduur van 1 uur kost meer per cacheschrijfactie maar overleeft langere tussenpozen tussen verzoeken. Laat het op 5 minuten staan tenzij je herhaaldelijk dezelfde lange context opnieuw verstuurt.", "awsBedrock1MContextBetaLabel": "1M contextvenster inschakelen (bèta)", "awsBedrock1MContextBetaDescription": "Breidt het contextvenster uit tot 1 miljoen tokens voor Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "1M contextvenster inschakelen (bèta)", diff --git a/webview-ui/src/i18n/locales/pl/settings.json b/webview-ui/src/i18n/locales/pl/settings.json index 459675b0e1..218e20f5cb 100644 --- a/webview-ui/src/i18n/locales/pl/settings.json +++ b/webview-ui/src/i18n/locales/pl/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Przekaż klucz API Anthropic jako nagłówek Authorization zamiast X-Api-Key", "anthropic1MContextBetaLabel": "Włącz okno kontekstowe 1M (Beta)", "anthropic1MContextBetaDescription": "Rozszerza okno kontekstowe do 1 miliona tokenów dla Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Czas życia pamięci podręcznej promptów", + "promptCacheTtl5m": "5 minut (domyślnie)", + "promptCacheTtl1h": "1 godzina", + "promptCacheTtlDescription": "Jak długo buforowane prefiksy promptów pozostają ważne. Czas życia 1 godziny kosztuje więcej za zapis do pamięci podręcznej, ale przetrwa dłuższe przerwy między żądaniami. Pozostaw 5 minut, chyba że wielokrotnie wysyłasz ten sam długi kontekst.", "awsBedrock1MContextBetaLabel": "Włącz okno kontekstowe 1M (Beta)", "awsBedrock1MContextBetaDescription": "Rozszerza okno kontekstowe do 1 miliona tokenów dla Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Włącz okno kontekstowe 1M (Beta)", diff --git a/webview-ui/src/i18n/locales/pt-BR/settings.json b/webview-ui/src/i18n/locales/pt-BR/settings.json index 500babaaa8..81626e7286 100644 --- a/webview-ui/src/i18n/locales/pt-BR/settings.json +++ b/webview-ui/src/i18n/locales/pt-BR/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Passar a chave de API Anthropic como cabeçalho Authorization em vez de X-Api-Key", "anthropic1MContextBetaLabel": "Ativar janela de contexto de 1M (Beta)", "anthropic1MContextBetaDescription": "Estende a janela de contexto para 1 milhão de tokens para o Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Duração do cache de prompts", + "promptCacheTtl5m": "5 minutos (padrão)", + "promptCacheTtl1h": "1 hora", + "promptCacheTtlDescription": "Por quanto tempo os prefixos de prompt em cache permanecem válidos. A duração de 1 hora custa mais por gravação em cache mas sobrevive a intervalos mais longos entre solicitações. Deixe em 5 minutos a menos que você reenvie repetidamente o mesmo contexto longo.", "awsBedrock1MContextBetaLabel": "Ativar janela de contexto de 1M (Beta)", "awsBedrock1MContextBetaDescription": "Estende a janela de contexto para 1 milhão de tokens para o Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Ativar janela de contexto de 1M (Beta)", diff --git a/webview-ui/src/i18n/locales/ru/settings.json b/webview-ui/src/i18n/locales/ru/settings.json index 053ff0684d..429ef13831 100644 --- a/webview-ui/src/i18n/locales/ru/settings.json +++ b/webview-ui/src/i18n/locales/ru/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Передавать Anthropic API-ключ как Authorization-заголовок вместо X-Api-Key", "anthropic1MContextBetaLabel": "Включить контекстное окно 1M (бета)", "anthropic1MContextBetaDescription": "Расширяет контекстное окно до 1 миллиона токенов для Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Время жизни кеша промптов", + "promptCacheTtl5m": "5 минут (по умолчанию)", + "promptCacheTtl1h": "1 час", + "promptCacheTtlDescription": "Как долго кешированные префиксы промптов остаются действительными. Время жизни 1 час стоит дороже за запись в кеш, но переживает более длинные паузы между запросами. Оставьте 5 минут, если вы не отправляете повторно один и тот же длинный контекст.", "awsBedrock1MContextBetaLabel": "Включить контекстное окно 1M (бета)", "awsBedrock1MContextBetaDescription": "Расширяет контекстное окно до 1 миллиона токенов для Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Включить контекстное окно 1M (бета)", diff --git a/webview-ui/src/i18n/locales/tr/settings.json b/webview-ui/src/i18n/locales/tr/settings.json index b37f28d44d..08db4f2aae 100644 --- a/webview-ui/src/i18n/locales/tr/settings.json +++ b/webview-ui/src/i18n/locales/tr/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Anthropic API Anahtarını X-Api-Key yerine Authorization başlığı olarak geçir", "anthropic1MContextBetaLabel": "1M bağlam penceresini etkinleştir (Beta)", "anthropic1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6 için bağlam penceresini 1 milyon token'a genişletir", + "promptCacheTtlLabel": "İstem önbelleği ömrü", + "promptCacheTtl5m": "5 dakika (varsayılan)", + "promptCacheTtl1h": "1 saat", + "promptCacheTtlDescription": "Önbelleğe alınmış istem öneklerinin ne kadar süre geçerli kaldığı. 1 saatlik ömür, önbellek yazma başına daha pahalıdır ancak istekler arasındaki daha uzun aralıklara dayanır. Aynı uzun bağlamı tekrar tekrar göndermiyorsanız 5 dakikada bırakın.", "awsBedrock1MContextBetaLabel": "1M bağlam penceresini etkinleştir (Beta)", "awsBedrock1MContextBetaDescription": "Claude Sonnet 4.x / Claude Opus 4.6 için bağlam penceresini 1 milyon token'a genişletir", "vertex1MContextBetaLabel": "1M bağlam penceresini etkinleştir (Beta)", diff --git a/webview-ui/src/i18n/locales/vi/settings.json b/webview-ui/src/i18n/locales/vi/settings.json index 94843217ae..07976bfd92 100644 --- a/webview-ui/src/i18n/locales/vi/settings.json +++ b/webview-ui/src/i18n/locales/vi/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "Truyền khóa API Anthropic dưới dạng tiêu đề Authorization thay vì X-Api-Key", "anthropic1MContextBetaLabel": "Bật cửa sổ ngữ cảnh 1M (Beta)", "anthropic1MContextBetaDescription": "Mở rộng cửa sổ ngữ cảnh lên 1 triệu token cho Claude Sonnet 4.x / Claude Opus 4.6", + "promptCacheTtlLabel": "Thời hạn bộ nhớ đệm prompt", + "promptCacheTtl5m": "5 phút (mặc định)", + "promptCacheTtl1h": "1 giờ", + "promptCacheTtlDescription": "Tiền tố prompt được lưu trong bộ nhớ đệm còn hiệu lực trong bao lâu. Thời hạn 1 giờ tốn nhiều chi phí hơn cho mỗi lần ghi bộ nhớ đệm nhưng chịu được khoảng cách dài hơn giữa các yêu cầu. Hãy để 5 phút trừ khi bạn gửi lại cùng một ngữ cảnh dài nhiều lần.", "awsBedrock1MContextBetaLabel": "Bật cửa sổ ngữ cảnh 1M (Beta)", "awsBedrock1MContextBetaDescription": "Mở rộng cửa sổ ngữ cảnh lên 1 triệu token cho Claude Sonnet 4.x / Claude Opus 4.6", "vertex1MContextBetaLabel": "Bật cửa sổ ngữ cảnh 1M (Beta)", diff --git a/webview-ui/src/i18n/locales/zh-CN/settings.json b/webview-ui/src/i18n/locales/zh-CN/settings.json index 8be0fe1532..56e35cea62 100644 --- a/webview-ui/src/i18n/locales/zh-CN/settings.json +++ b/webview-ui/src/i18n/locales/zh-CN/settings.json @@ -495,6 +495,10 @@ "anthropicUseAuthToken": "将 Anthropic API 密钥作为 Authorization 标头传递，而不是 X-Api-Key", "anthropic1MContextBetaLabel": "启用 1M 上下文窗口 (Beta)", "anthropic1MContextBetaDescription": "为 Claude Sonnet 4.x / Claude Opus 4.6 将上下文窗口扩展至 100 万个 token", + "promptCacheTtlLabel": "提示缓存有效期", + "promptCacheTtl5m": "5 分钟（默认）", + "promptCacheTtl1h": "1 小时", + "promptCacheTtlDescription": "缓存的提示前缀保持有效的时间。1 小时的有效期每次缓存写入费用更高，但能承受请求之间更长的间隔。除非你反复重新发送相同的长上下文，否则请保持 5 分钟。", "awsBedrock1MContextBetaLabel": "启用 1M 上下文窗口 (Beta)", "awsBedrock1MContextBetaDescription": "为 Claude Sonnet 4.x / Claude Opus 4.6 将上下文窗口扩展至 100 万个 token", "vertex1MContextBetaLabel": "启用 1M 上下文窗口 (Beta)", diff --git a/webview-ui/src/i18n/locales/zh-TW/settings.json b/webview-ui/src/i18n/locales/zh-TW/settings.json index 65eb5c67a5..e3f561ccaa 100644 --- a/webview-ui/src/i18n/locales/zh-TW/settings.json +++ b/webview-ui/src/i18n/locales/zh-TW/settings.json @@ -522,6 +522,10 @@ "anthropicUseAuthToken": "將 Anthropic API 金鑰作為 Authorization 標頭傳遞，而非使用 X-Api-Key", "anthropic1MContextBetaLabel": "啟用 1M 上下文視窗 (Beta)", "anthropic1MContextBetaDescription": "為 Claude Sonnet 4.x / Claude Opus 4.6 將上下文視窗擴展至 100 萬個 token", + "promptCacheTtlLabel": "提示快取有效期", + "promptCacheTtl5m": "5 分鐘（預設）", + "promptCacheTtl1h": "1 小時", + "promptCacheTtlDescription": "快取的提示前綴保持有效的時間。1 小時的有效期每次快取寫入費用更高，但能承受請求之間更長的間隔。除非你反覆重新傳送相同的長上下文，否則請保持 5 分鐘。", "awsBedrock1MContextBetaLabel": "啟用 1M 上下文視窗 (Beta)", "awsBedrock1MContextBetaDescription": "為 Claude Sonnet 4.x / Claude Opus 4.6 將上下文視窗擴展至 100 萬個 token", "vertex1MContextBetaLabel": "啟用 1M 上下文視窗 (Beta)", From d1fcbeb09d9590f7bd9d7dc5579779a1a6364a4d Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 07:14:20 +0900 Subject: [PATCH 08/14] test(settings): cover the prompt cache TTL settings round trip What: adds the round-trip coverage the Persisted Setting Checklist asks for: extension-side survival through getStateToPostToWebview (set and unset), import/export schema preservation of both "1h" and "5m", and the SettingsView cachedState buffer (restore, buffer-until-Save, and the absence-vs-explicit-5m distinction). Why: the knob rides apiConfiguration, so a missing schema field or a missing state-post field would silently revert the control without any type error. The import/export case is falsified by removing the schema entry (test goes red), so it guards the field rather than the fixture. Impact: test-only; no production behavior change. Test: src 504/504 across the six affected suites, webview-ui 2235/2235. --- .../config/__tests__/importExport.spec.ts | 59 ++++++++++++++ .../webview/__tests__/ClineProvider.spec.ts | 59 ++++++++++++++ .../settings/__tests__/SettingsView.spec.tsx | 78 +++++++++++++++++++ 3 files changed, 196 insertions(+) diff --git a/src/core/config/__tests__/importExport.spec.ts b/src/core/config/__tests__/importExport.spec.ts index 90db81d8a1..6c66655325 100644 --- a/src/core/config/__tests__/importExport.spec.ts +++ b/src/core/config/__tests__/importExport.spec.ts @@ -357,6 +357,65 @@ describe("importExport", () => { expect(mockContextProxy.setValues).not.toHaveBeenCalled() }) + it("round-trips a prompt cache TTL through export and import", async () => { + // The knob rides the provider profile, so the export/import schema is the + // only thing that can lose it. Both classes are exercised because a + // schema that only accepted "1h" would still pass a one-sided test. + const profiles = { + currentApiConfigName: "cache", + apiConfigs: { + cache: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + id: "cache-id", + promptCacheTtl: "1h" as const, + }, + explicit: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + id: "explicit-id", + promptCacheTtl: "5m" as const, + }, + }, + } + + ;(vscode.window.showSaveDialog as Mock).mockResolvedValue({ fsPath: "/mock/path/export.json" }) + mockProviderSettingsManager.export.mockResolvedValue(profiles) + mockContextProxy.export.mockResolvedValue({ mode: "code" }) + + await exportSettings({ + providerSettingsManager: mockProviderSettingsManager, + contextProxy: mockContextProxy, + }) + + const exported = (safeWriteJson as Mock).mock.calls.at(-1)?.[1] + expect(exported.providerProfiles.apiConfigs.cache.promptCacheTtl).toBe("1h") + expect(exported.providerProfiles.apiConfigs.explicit.promptCacheTtl).toBe("5m") + + ;(vscode.window.showOpenDialog as Mock).mockResolvedValue([{ fsPath: "/mock/path/export.json" }]) + ;(fs.readFile as Mock).mockResolvedValue(JSON.stringify(exported)) + mockProviderSettingsManager.export.mockResolvedValue({ currentApiConfigName: "default", apiConfigs: {} }) + mockProviderSettingsManager.listConfig.mockResolvedValue([]) + + const result = await importSettings({ + providerSettingsManager: mockProviderSettingsManager, + contextProxy: mockContextProxy, + customModesManager: mockCustomModesManager, + }) + + expect(result.success).toBe(true) + expect(mockProviderSettingsManager.import).toHaveBeenCalledWith( + expect.objectContaining({ + apiConfigs: expect.objectContaining({ + cache: expect.objectContaining({ promptCacheTtl: "1h" }), + explicit: expect.objectContaining({ promptCacheTtl: "5m" }), + }), + }), + ) + }) + it("should import settings successfully when globalSettings key is missing", async () => { ;(vscode.window.showOpenDialog as Mock).mockResolvedValue([{ fsPath: "/mock/path/settings.json" }]) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 779313d385..73a9ad8964 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -2111,6 +2111,65 @@ describe("ClineProvider", () => { expect(state.allowedReadFiles).toEqual(["notes.md"]) }) + describe("prompt cache TTL round trip", () => { + // The provider-settings route: the knob rides `apiConfiguration`, so the + // round trip is the profile write plus the state post, with no dedicated + // message type and no settings-specific branch in the message handler. + test("a saved 1h survives a fresh getStateToPostToWebview payload", async () => { + await provider.resolveWebviewView(mockWebviewView) + + await provider.upsertProviderProfile("ttl-profile", { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + promptCacheTtl: "1h", + }) + + // A fresh read, not the in-memory value the save path just wrote: this is + // the "saved control reverts visually" case the checklist calls out. + const posted = await provider.getStateToPostToWebview() + + expect(posted.apiConfiguration.promptCacheTtl).toBe("1h") + expect((await provider.getState()).apiConfiguration.promptCacheTtl).toBe("1h") + }) + + test("an unset knob stays absent rather than becoming an explicit 5m", async () => { + await provider.resolveWebviewView(mockWebviewView) + + await provider.upsertProviderProfile("default-ttl-profile", { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + }) + + const posted = await provider.getStateToPostToWebview() + + // Consumers apply the vendor default, so the stored value must be absent + // rather than a literal "5m" that would look like a user choice. + expect(posted.apiConfiguration.promptCacheTtl).toBeUndefined() + expect(posted.apiConfiguration).not.toHaveProperty("promptCacheTtl", "5m") + }) + + test("a hand-edited 1h for a model without the capability is stored, not silently rewritten", async () => { + await provider.resolveWebviewView(mockWebviewView) + + // MiniMax caches prompts but advertises no TTL class. The profile can only + // have been written by hand or imported, and the save path must keep what + // the user stored: capability is enforced where the request is built (the + // emitter fails closed), so a save-time rewrite would be a second, silent + // source of truth for the same rule. + await provider.upsertProviderProfile("hand-edited-profile", { + apiProvider: providerIdentifiers.minimax, + apiKey: "test-key", + promptCacheTtl: "1h", + }) + + const posted = await provider.getStateToPostToWebview() + + expect(posted.apiConfiguration.promptCacheTtl).toBe("1h") + }) + }) + test("getStateToPostToWebview defaults allowed read files to an empty list", async () => { await provider.resolveWebviewView(mockWebviewView) diff --git a/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx b/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx index 0835f255f9..14a3100057 100644 --- a/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx @@ -33,6 +33,12 @@ vi.mock("../ApiOptions", () => ({ data-testid="set-other-field" onClick={() => setApiConfigurationField("openAiBaseUrl", "https://example.test")} /> + {String(apiConfiguration?.promptCacheTtl)} +