diff --git a/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts new file mode 100644 index 0000000000..e62f658016 --- /dev/null +++ b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts @@ -0,0 +1,84 @@ +// npx vitest run src/__tests__/cache-write-ttl-pricing.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { bedrockModels } from "../providers/bedrock.js" +import { openAiNativeModels } from "../providers/openai.js" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + deriveCacheWritesPrice12h, + deriveCacheWritesPrice1h, + type ModelInfo, +} from "../model.js" + +/** + * Catalog invariant for the TTL accounting program (ADR-0006 phase 1). + * + * A model that advertises prompt caching on a provider which can report a + * long-lived cache write must carry the matching per-TTL write rate. Without it + * the cost layer can only fall back to the aggregate write rate, which + * under-reports every long-lived write (37.5% for Anthropic 1h, 50% for OpenAI 12h). + */ +describe("cache-write TTL pricing catalog invariant", () => { + const anthropicFamilyEntries = Object.entries(anthropicModels as Record) + + const bedrockAnthropicEntries = Object.entries(bedrockModels as Record).filter(([id]) => + id.startsWith("anthropic."), + ) + + const openAiLongLivedEntries = Object.entries(openAiNativeModels as Record).filter( + ([id]) => id.startsWith("gpt-5.6") || id.startsWith("gpt-6"), + ) + + it("every Anthropic catalog entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = anthropicFamilyEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every Bedrock Anthropic-family entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = bedrockAnthropicEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every OpenAI 5.6/6-class entry defines cacheWritesPrice12h", () => { + expect(openAiLongLivedEntries.length).toBeGreaterThan(0) + + const missing = openAiLongLivedEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice12h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("Anthropic 1h rates match the documented 2x input-price relationship", () => { + for (const [id, info] of anthropicFamilyEntries) { + if (info.cacheWritesPrice1h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice1h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_1H, 9) + } + }) + + it("OpenAI 12h rates match the documented 2.5x input-price relationship", () => { + for (const [id, info] of openAiLongLivedEntries) { + if (info.cacheWritesPrice12h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice12h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_12H, 9) + } + }) + + it("deriveCacheWritesPrice1h scales inputPrice by 2x and stays undefined without one", () => { + expect(deriveCacheWritesPrice1h(3.0)).toBe(6.0) + expect(deriveCacheWritesPrice1h(undefined)).toBeUndefined() + }) + + it("deriveCacheWritesPrice12h scales inputPrice by 2.5x and stays undefined without one", () => { + expect(deriveCacheWritesPrice12h(4.0)).toBe(10.0) + expect(deriveCacheWritesPrice12h(undefined)).toBeUndefined() + }) +}) diff --git a/packages/types/src/__tests__/openai-models.test.ts b/packages/types/src/__tests__/openai-models.test.ts index 78a3259f09..e1e0f1e323 100644 --- a/packages/types/src/__tests__/openai-models.test.ts +++ b/packages/types/src/__tests__/openai-models.test.ts @@ -122,6 +122,7 @@ describe("OpenAI native models", () => { inputPrice: 2, outputPrice: 10, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -166,6 +167,7 @@ describe("OpenAI native models", () => { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, diff --git a/packages/types/src/__tests__/prompt-cache-ttl.test.ts b/packages/types/src/__tests__/prompt-cache-ttl.test.ts new file mode 100644 index 0000000000..05b76245ee --- /dev/null +++ b/packages/types/src/__tests__/prompt-cache-ttl.test.ts @@ -0,0 +1,103 @@ +// npx vitest run src/__tests__/prompt-cache-ttl.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { vertexModels } from "../providers/vertex.js" +import { openRouterDefaultModelInfo } from "../providers/openrouter.js" +import { + vercelAiGatewayDefaultModelId, + VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS, +} from "../providers/vercel-ai-gateway.js" +import { + DEFAULT_PROMPT_CACHE_TTL, + modelInfoSchema, + promptCacheTtlSchema, + promptCacheTtlValues, + type ModelInfo, +} from "../model.js" +import { anthropicProviderDefinition } from "../provider-settings/anthropic.js" +import { baseProviderSettingsShape } from "../provider-settings/common.js" +import { providerIdentifiers } from "../provider-identifiers.js" +import { globalSettingsSchema } from "../global-settings.js" + +/** + * Catalog invariant for the request-side TTL knob (ADR-0006 phase 2, PR-C5b). + * + * Emission is gated by `ModelInfo.promptCacheTtlSupport`, so a model that can + * report a 1h cache write but carries no capability array would silently never + * emit the TTL the user asked for. These tests pin both directions: capable + * catalogs advertise it, and catalogs that cannot know stay absent (fail closed). + */ +describe("prompt cache TTL schema", () => { + it("accepts exactly the 5m and 1h classes", () => { + expect(promptCacheTtlValues).toEqual(["5m", "1h"]) + expect(promptCacheTtlSchema.parse("5m")).toBe("5m") + expect(promptCacheTtlSchema.parse("1h")).toBe("1h") + expect(promptCacheTtlSchema.safeParse("30m").success).toBe(false) + expect(promptCacheTtlSchema.safeParse("12h").success).toBe(false) + }) + + it("defaults to undefined so an unset knob emits no ttl key", () => { + expect(DEFAULT_PROMPT_CACHE_TTL).toBeUndefined() + }) + + it("is optional on ModelInfo", () => { + const parsed = modelInfoSchema.parse({ contextWindow: 200_000, supportsPromptCache: true }) + expect(parsed.promptCacheTtlSupport).toBeUndefined() + + const withCapability = modelInfoSchema.parse({ + contextWindow: 200_000, + supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], + }) + expect(withCapability.promptCacheTtlSupport).toEqual(["5m", "1h"]) + }) +}) + +describe("prompt cache TTL knob location", () => { + it("lives in provider settings so handlers can see it", () => { + // ApiHandlerOptions = Omit, so only a + // provider-settings key reaches a handler (ADR-0006 decision item 2). + expect(baseProviderSettingsShape.promptCacheTtl).toBeDefined() + + const parsed = anthropicProviderDefinition.schema.safeParse({ + apiProvider: providerIdentifiers.anthropic, + promptCacheTtl: "1h", + }) + expect(parsed.success).toBe(true) + expect(parsed.success && parsed.data.promptCacheTtl).toBe("1h") + }) + + it("is not a global setting", () => { + const shape = globalSettingsSchema.shape as Record + expect(shape.promptCacheTtl).toBeUndefined() + }) +}) + +describe("prompt cache TTL capability catalog", () => { + it("every Anthropic entry with prompt caching advertises both TTL classes", () => { + const missing = Object.entries(anthropicModels as Record) + .filter(([, info]) => info.supportsPromptCache && !info.promptCacheTtlSupport?.includes("1h")) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("leaves the Vertex catalog without capability so it fails closed", () => { + // PR-C5b scopes the catalog capability to the Anthropic entries; the Vertex + // catalog is populated separately, so its models stay 5m-only for now. + const withCapability = Object.entries(vertexModels as Record) + .filter(([, info]) => info.promptCacheTtlSupport !== undefined) + .map(([id]) => id) + + expect(withCapability).toEqual([]) + }) + + it("leaves dynamic-catalog defaults without capability so they fail closed", () => { + // OpenRouter and the gateway catalogs are fetched at runtime and carry no TTL + // capability data, which ADR-0006 maps to "5m only" (invariant I2). + expect(openRouterDefaultModelInfo.promptCacheTtlSupport).toBeUndefined() + expect(VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(vercelAiGatewayDefaultModelId)).toBe(true) + }) +}) diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 9ddc4ea182..b3c2b8151d 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -95,6 +95,20 @@ export type ModelParameter = z.infer export const isModelParameter = (value: string): value is ModelParameter => modelParameters.includes(value as ModelParameter) +/** + * Prompt cache TTL classes a request-side cache marker can carry. `"5m"` is the + * vendor default and is never emitted explicitly, so only `"1h"` changes the wire + * shape and an unset knob stays byte-identical to a pre-knob build (ADR-0006 I3). + */ +export const promptCacheTtlValues = ["5m", "1h"] as const + +export const promptCacheTtlSchema = z.enum(promptCacheTtlValues) + +export type PromptCacheTtl = z.infer + +/** Deliberately `undefined`, not `"5m"`: emitting `ttl: "5m"` would change every request's bytes. */ +export const DEFAULT_PROMPT_CACHE_TTL: PromptCacheTtl | undefined = undefined + /** * ModelInfo */ @@ -112,6 +126,8 @@ export const modelInfoSchema = z.object({ // When set to "24h", extended prompt caching will be requested; when omitted // or set to "in_memory", the default in‑memory cache is used. promptCacheRetention: z.enum(["in_memory", "24h"]).optional(), + // Cache TTLs this model accepts. Absent means "5m only" (ADR-0006 I2, fail closed). + promptCacheTtlSupport: z.array(promptCacheTtlSchema).optional(), // Capability flag to indicate whether the model supports an output verbosity parameter supportsVerbosity: z.boolean().optional(), // Capability flag to indicate whether the model exposes a user-configurable max output @@ -136,6 +152,13 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL cache-write rates. Vendors bill a longer cache lifetime at a higher + // write rate than the default 5m/30m class, so a single `cacheWritesPrice` + // under-reports whenever the response reports a long-lived write. Omitted + // means "no long-lived writes are expected on this model"; the cost layer + // derives a fail-closed rate from `inputPrice` if one still arrives. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), longContextPricing: z .object({ @@ -180,6 +203,11 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL write rates for this tier. A tier that overrides inputPrice + // must also override these, otherwise a long-context or flex/priority + // write bills at the base tier's TTL rate. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), }), ) @@ -188,6 +216,34 @@ export const modelInfoSchema = z.object({ export type ModelInfo = z.infer +/** + * Multipliers vendors apply to the base input price for a cache write, by TTL class. + * Anthropic: 5m = 1.25x, 1h = 2x. OpenAI: 30m = 1.25x, 12h = 2.5x. + */ +export const CACHE_WRITE_MULTIPLIER_5M = 1.25 +export const CACHE_WRITE_MULTIPLIER_1H = 2.0 +export const CACHE_WRITE_MULTIPLIER_12H = 2.5 + +/** + * Derive the 1h cache-write rate from the base input price. + * + * Stated once here so every Anthropic-family catalog entry and every runtime + * fallback agrees on the 2x relationship. Returns `undefined` when the model + * has no input price to scale, which keeps the cost layer fail-closed: it will + * fall back to `cacheWritesPrice` rather than invent a rate. + */ +export function deriveCacheWritesPrice1h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_1H +} + +/** + * Derive the 12h cache-write rate from the base input price (OpenAI 12h class). + * Same fail-closed contract as {@link deriveCacheWritesPrice1h}. + */ +export function deriveCacheWritesPrice12h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_12H +} + export type ModelRecord = Record export type RouterModels = Record diff --git a/packages/types/src/provider-settings/common.ts b/packages/types/src/provider-settings/common.ts index ca78c3a25d..ca908d0c60 100644 --- a/packages/types/src/provider-settings/common.ts +++ b/packages/types/src/provider-settings/common.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { reasoningEffortSettingSchema, verbosityLevelsSchema } from "../model.js" +import { reasoningEffortSettingSchema, verbosityLevelsSchema, promptCacheTtlSchema } from "../model.js" import type { ProviderIdentifier } from "../provider-identifiers.js" export const API_PROVIDER_FIELD = "apiProvider" @@ -33,6 +33,11 @@ export const baseProviderSettingsShape = { modelMaxTokens: z.number().optional(), modelMaxThinkingTokens: z.number().optional(), verbosity: verbosityLevelsSchema.optional(), + // Opt-in prompt cache TTL for this provider profile. Lives here, not in global + // settings, because `ApiHandlerOptions = Omit` + // (src/shared/api.ts) makes a global key invisible to handlers. Unset emits no + // ttl key at all, not "5m" (ADR-0006 I3). + promptCacheTtl: promptCacheTtlSchema.optional(), } export const apiModelIdProviderModelShape = { diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index 567f6268be..bd90938931 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -3,6 +3,9 @@ import type { ModelInfo } from "../model.js" // https://docs.anthropic.com/en/docs/about-claude/models // https://platform.claude.com/docs/en/about-claude/pricing +// Every entry here is an Anthropic-hosted Claude model, and 1h cache writes are GA +// on all active Claude models, so each one advertises both TTL classes. The 5m class +// is the vendor default and is never emitted explicitly (ADR-0006 I3). export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-sonnet-4-5" export const ANTHROPIC_API_PROTOCOL = "anthropic" @@ -13,9 +16,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -25,6 +30,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -34,9 +40,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) // Sonnet 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Fable 5 on the direct Anthropic provider path. Manual @@ -53,9 +61,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -65,6 +75,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -74,9 +85,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -86,6 +99,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -95,9 +109,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag) @@ -107,6 +123,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -116,9 +133,11 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Keep the hybrid-reasoning capability so Anthropic token-cap handling and // stored max-token overrides behave the same as before. @@ -133,9 +152,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required, same as 4.7) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (regular tier) outputPrice: 25.0, // $25 per million output tokens (regular tier) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // 4.8 inherits the adaptive-thinking model introduced in 4.7 — no breaking // API changes. supportsReasoningBudget is kept true so the existing token-cap @@ -151,9 +172,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Opus 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Sonnet 5 on the direct Anthropic provider path. Manual @@ -169,9 +192,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // Opus 5.5 always uses adaptive thinking and rejects manual budget_tokens. supportsReasoningBinary: true, @@ -184,6 +209,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], // Standard pricing (prompts of 100,000 tokens or fewer) from Anthropic's // pricing page. Prompts over 100,000 tokens bill at tiered rates // ($0.50 in / $2.50 out / $0.625 5m cache write / $0.05 cache read per @@ -192,6 +218,7 @@ export const anthropicModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // Haiku 5.5 uses the same adaptive-thinking / binary-toggle convention as // Sonnet 5 and Opus 5 on the direct Anthropic provider path. Manual @@ -208,9 +235,11 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, // $0.25 per million tokens (2.5% of base input price) // Fable 5.1 only accepts adaptive thinking (or an omitted thinking field). // Manual budget_tokens and non-default sampling parameters return a 400. @@ -225,9 +254,11 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens // Fable 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ on the direct Anthropic provider path. @@ -242,9 +273,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, }, @@ -253,9 +286,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -264,9 +299,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -275,9 +312,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, requiredReasoningBudget: true, @@ -287,9 +326,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-sonnet-20241022": { @@ -297,9 +338,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-haiku-20241022": { @@ -307,9 +350,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: false, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, }, "claude-3-opus-20240229": { @@ -317,9 +362,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, }, "claude-3-haiku-20240307": { @@ -327,9 +374,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 0.25, outputPrice: 1.25, cacheWritesPrice: 0.3, + cacheWritesPrice1h: 0.5, // $0.50 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.03, }, "claude-haiku-4-5-20251001": { @@ -337,9 +386,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, supportsReasoningBudget: true, description: diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index 1e89f3e6ba..faef150a2a 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -22,6 +22,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -36,6 +37,7 @@ export const bedrockModels = { inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -47,6 +49,7 @@ export const bedrockModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -63,6 +66,7 @@ export const bedrockModels = { inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -143,6 +147,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -157,6 +162,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -171,6 +177,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -182,6 +189,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -195,6 +203,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -206,6 +215,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -219,6 +229,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -232,6 +243,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -248,6 +260,7 @@ export const bedrockModels = { inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -268,6 +281,7 @@ export const bedrockModels = { inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Opus 5.5 (matching the Fable 5.1 entry below). @@ -297,6 +311,7 @@ export const bedrockModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Haiku 5.5 (matching the Opus 5.5 entry above). @@ -317,6 +332,7 @@ export const bedrockModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, minTokensPerCachePoint: 512, maxCachePoints: 4, @@ -335,6 +351,7 @@ export const bedrockModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -351,6 +368,7 @@ export const bedrockModels = { inputPrice: 5.0, outputPrice: 25.0, cacheWritesPrice: 6.25, + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -365,6 +383,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -379,6 +398,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -392,6 +412,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -405,6 +426,7 @@ export const bedrockModels = { inputPrice: 0.8, outputPrice: 4.0, cacheWritesPrice: 1.0, + cacheWritesPrice1h: 1.6, // $1.60 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.08, minTokensPerCachePoint: 2048, maxCachePoints: 4, @@ -419,6 +441,7 @@ export const bedrockModels = { inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, // 5m cache writes + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, // cache hits / refreshes minTokensPerCachePoint: 2048, maxCachePoints: 4, diff --git a/packages/types/src/providers/openai.ts b/packages/types/src/providers/openai.ts index 41d00c7a04..89a24fe2ef 100644 --- a/packages/types/src/providers/openai.ts +++ b/packages/types/src/providers/openai.ts @@ -31,6 +31,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.1, longContextPricing: { thresholdTokens: 272_000, @@ -75,6 +76,7 @@ export const openAiNativeModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice12h: 25, // $25.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 1.0, longContextPricing: { thresholdTokens: 272_000, @@ -117,6 +119,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -160,6 +163,7 @@ export const openAiNativeModels = { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, // $0.25 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, @@ -203,6 +207,7 @@ export const openAiNativeModels = { inputPrice: 4.0, outputPrice: 20.0, cacheWritesPrice: 5.0, + cacheWritesPrice12h: 10, // $10.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.4, longContextPricing: { thresholdTokens: 272_000, @@ -246,6 +251,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 12.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -289,6 +295,7 @@ export const openAiNativeModels = { inputPrice: 0.2, outputPrice: 1.2, cacheWritesPrice: 0.25, + cacheWritesPrice12h: 0.5, // $0.50 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.02, longContextPricing: { thresholdTokens: 272_000, diff --git a/src/api/index.ts b/src/api/index.ts index d61eea033f..e737ebd4bd 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -61,6 +61,14 @@ import { NativeOllamaHandler } from "./providers/native-ollama" export interface CompletePromptOptions extends Pick { /** Optional timeout override (ms) — falls back to provider default if omitted */ timeoutMs?: number + /** + * Task the completion belongs to, when it belongs to one. + * + * Providers that key cache affinity off a body field use this to route repeated + * completions for one task to the same bucket. Omit it when the completion is not + * scoped to a task; providers then fall back to a per-handler key. + */ + taskId?: string } export interface SingleCompletionHandler { diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 34788df89b..f4860e9914 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -10,6 +10,7 @@ import { AnthropicVertexHandler } from "../anthropic-vertex" import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" +import type { ApiStreamChunk } from "../../../api/transform/stream" vitest.mock("../utils/timeout-config", () => ({ getApiRequestTimeout: vitest.fn().mockReturnValue(300_000), @@ -259,6 +260,108 @@ describe("VertexHandler", () => { ) }) + describe("prompt cache TTL", () => { + const minimalStream = asyncStreamFrom([ + { + type: "message_start", + message: { usage: { input_tokens: 10, output_tokens: 0 } }, + }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // The handler's client is private; reach it once here rather than casting at + // each call site. `messages.create` is replaced with a plain vitest mock. + const stubCreate = (mockCreate: ReturnType) => { + const client = handler["client"] as unknown as { messages: { create: unknown } } + client.messages.create = mockCreate + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on the system prompt and every user marker when the model is capable", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + // The Vertex catalog carries no TTL capability yet, so inject one to + // exercise the emission path this PR adds. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + // System + the user marker the vertex transform emits. + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls.length).toBeGreaterThanOrEqual(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("fails closed when the Vertex catalog advertises no TTL capability", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should handle multiple content blocks with line breaks for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", @@ -453,6 +556,41 @@ describe("VertexHandler", () => { ) }) + it("should surface the 1h cache-write split from cache_creation", async () => { + const mockStream = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ] + + const mockCreate = vitest.fn().mockResolvedValue(asyncStreamFrom(mockStream)) + // Typed structural cast rather than `as any`: the client is a private field and + // this file already has 31 suppressed `any` uses, a budget this test must not grow. + ;(handler["client"].messages as unknown as { create: typeof mockCreate }).create = mockCreate + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is Extract => + chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + it("should handle cache-related usage metrics for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 3eedcfc906..4f15564323 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -1,14 +1,13 @@ // npx vitest run src/api/providers/__tests__/anthropic.spec.ts -import { type ApiStreamUsageChunk } from "../../../api/transform/stream" import { AnthropicHandler } from "../anthropic" import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { ApiHandlerOptions } from "../../../shared/api" -import { calculateApiCostAnthropic } from "../../../shared/cost" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" -import type { ApiStreamChunk } from "../../../api/transform/stream" +import type { ApiStreamChunk, ApiStreamUsageChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" +import { calculateApiCostAnthropic } from "../../../shared/cost" // Mock TelemetryService vitest.mock("@roo-code/telemetry", () => ({ @@ -241,6 +240,78 @@ describe("AnthropicHandler", () => { expect(usageChunks[0]?.totalCost).not.toBe(calculateApiCostAnthropic(info, 100, 1).totalCost) }) + describe("prompt cache TTL", () => { + const threeTurnMessages = () => [ + { role: "user" as const, content: [{ type: "text" as const, text: "First message" }] }, + { role: "assistant" as const, content: [{ type: "text" as const, text: "Response" }] }, + { role: "user" as const, content: [{ type: "text" as const, text: "Second message" }] }, + ] + + /** Every cache_control object in the serialized request body. */ + const requestCacheControls = () => { + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const serialized = JSON.parse(JSON.stringify(body)) + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(serialized) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + await collectStream(handler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on all three marker sites when the knob is 1h", async () => { + const ttlHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "1h", + }) + + await collectStream(ttlHandler.createMessage(systemPrompt, threeTurnMessages())) + + // System + last two user messages: a single missed site would show up + // here as a mixed-TTL request (ADR-0006 I4). + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(new Set(controls.map((control) => control.ttl))).toEqual(new Set(["1h"])) + }) + + it("emits no ttl key for an explicit 5m knob", async () => { + const fiveMinuteHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "5m", + }) + + await collectStream(fiveMinuteHandler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should include 1M context beta header for Claude Sonnet 4.6 when enabled", async () => { const sonnet46Handler = new AnthropicHandler({ apiKey: "test-api-key", @@ -1096,6 +1167,103 @@ describe("AnthropicHandler", () => { "fine-grained-tool-streaming-2025-05-14,prompt-caching-2024-07-31", ) }) + + it("should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + + it("should leave the 1h split undefined when cache_creation is absent", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(20) + expect(usageChunk?.cacheWrite1hTokens).toBeUndefined() + }) + + it("should price a 1h write at the 1h rate in the final cost chunk", async () => { + // claude-sonnet-4-5: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + const sonnetHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + }) + + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 0, + ephemeral_1h_input_tokens: 100_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = sonnetHandler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const costChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.totalCost !== undefined, + ) + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour charged the 5m rate: 0.375 (37.5% low). + expect(costChunk?.totalCost).toBeCloseTo(0.6, 9) + expect(costChunk?.totalCost).not.toBeCloseTo(0.375, 9) + }) }) describe("completePrompt", () => { @@ -1492,6 +1660,38 @@ describe("AnthropicHandler", () => { expect(model.info.outputPrice).toBe(22.5) }) + it("carries the tier's own cacheWritesPrice1h when the 1M context tier applies", () => { + // glm P2: the tier merge overrode inputPrice but not the TTL rate, so the + // entry-level 1h rate (derived from the 200K input price) survived and a + // 1M-context 1h write billed 6.0 instead of the tier's 12.0 (50% under). + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + // Tier input price 6.0 => 1h rate must be 2x that, not the 200K entry's 6.0. + expect(model.info.cacheWritesPrice1h).toBe(12.0) + expect(model.info.cacheWritesPrice1h).not.toBe(6.0) + }) + + it("bills a 1M-context 1h cache write at the tier rate, not the 200K rate", () => { + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + const { totalCost } = calculateApiCostAnthropic(model.info, 0, 0, 100_000, 0, 100_000) + + // Tier 1h rate 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // The pre-fix behaviour billed the 200K 1h rate: 0.6 (50% under). + expect(totalCost).toBeCloseTo(1.2, 9) + expect(totalCost).not.toBeCloseTo(0.6, 9) + }) + it("should honor a custom/unrecognized model ID instead of silently falling back to anthropicDefaultModelId", () => { const handler = new AnthropicHandler({ apiKey: "test-api-key", diff --git a/src/api/providers/__tests__/minimax.spec.ts b/src/api/providers/__tests__/minimax.spec.ts index 278168ff54..29edebdbfa 100644 --- a/src/api/providers/__tests__/minimax.spec.ts +++ b/src/api/providers/__tests__/minimax.spec.ts @@ -375,6 +375,38 @@ describe("MiniMaxHandler", () => { expect(firstChunk.value).toEqual({ type: "usage", inputTokens: 10, outputTokens: 20 }) }) + it("createMessage should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockResolvedValueOnce( + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage("system prompt", []) + const firstChunk = await stream.next() + + expect(firstChunk.done).toBe(false) + expect(firstChunk.value).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 60_000, + }) + }) + it("createMessage should adopt cumulative message_delta output tokens for the final totalCost", async () => { mockCreate.mockResolvedValueOnce( asyncStreamFrom([ diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 8922c2cf9c..b79145efdd 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -1,6 +1,8 @@ import { describe, it, expect, beforeEach } from "vitest" +import { createHash } from "crypto" import { OpenAiNativeHandler } from "../openai-native" -import { OpenAiServiceTier, openAiNativeModels } from "@roo-code/types" +import { OpenAiServiceTier, openAiNativeModels, type ModelInfo } from "@roo-code/types" +import type { ApiStreamUsageChunk } from "../../transform/stream" describe("OpenAiNativeHandler - normalizeUsage", () => { let handler: OpenAiNativeHandler @@ -440,6 +442,72 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { }) }) + describe("OpenAiNativeHandler - prompt cache key", () => { + let handler: OpenAiNativeHandler + + beforeEach(() => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + }) + }) + + const buildRequestBody = (modelId: string, taskId?: string) => { + handler["options"].apiModelId = modelId + const model = handler.getModel() + const metadata = taskId === undefined ? undefined : { taskId } + return handler["buildRequestBody"](model, [], "", model.verbosity, undefined, metadata) + } + + it("should send the task id as the body prompt_cache_key", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + }) + + it("should keep the key stable across turns of the same task", () => { + const firstTurn = buildRequestBody("gpt-5.1", "task-stable") + const secondTurn = buildRequestBody("gpt-5.1", "task-stable") + expect(firstTurn.prompt_cache_key).toBe("task-stable") + expect(secondTurn.prompt_cache_key).toBe(firstTurn.prompt_cache_key) + }) + + it("should give distinct keys to distinct tasks", () => { + const first = buildRequestBody("gpt-5.1", "task-one") + const second = buildRequestBody("gpt-5.1", "task-two") + expect(first.prompt_cache_key).toBe("task-one") + expect(second.prompt_cache_key).toBe("task-two") + expect(first.prompt_cache_key).not.toBe(second.prompt_cache_key) + }) + + it("should fall back to the handler session id when no task id is supplied", () => { + const body = buildRequestBody("gpt-5.1") + expect(body.prompt_cache_key).toBe(handler["sessionId"]) + expect(body.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the key when the model does not support prompt caching", () => { + const modelId = "codex-mini-latest" + expect(openAiNativeModels[modelId as keyof typeof openAiNativeModels].supportsPromptCache).toBe(false) + + const body = buildRequestBody(modelId, "task-abc") + expect(body.prompt_cache_key).toBeUndefined() + }) + + it("should clamp an over-length task id to OpenAI's 64-character limit", () => { + const longTaskId = "t".repeat(80) + const body = buildRequestBody("gpt-5.1", longTaskId) + expect(body.prompt_cache_key).toHaveLength(64) + // A hash, not a slice: truncation would collapse ids that share a prefix. + expect(body.prompt_cache_key).toBe(createHash("sha256").update(longTaskId).digest("hex")) + expect(body.prompt_cache_key).not.toBe(longTaskId.slice(0, 64)) + }) + + it("should keep prompt_cache_retention alongside the key for 24h models", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + expect(body.prompt_cache_retention).toBe("24h") + }) + }) + describe("cost calculation", () => { it.each([ { tier: OpenAiServiceTier.Default, expectedCost: 4.525 }, @@ -463,6 +531,35 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(expectedCost, 6) }) + it.each([ + // gpt-6-astra: default input 10.0 / 12h 25.0; flex input 5.0; priority input 20.0. + // The 12h rate must follow the tier's own input price (2.5x), not the + // standard-tier 12h rate, or a flex/priority 12h write is mis-billed. + { tier: OpenAiServiceTier.Default, expectedCost: 2.5 }, + { tier: OpenAiServiceTier.Flex, expectedCost: 1.25 }, + { tier: OpenAiServiceTier.Priority, expectedCost: 5.0 }, + ])("derives the 12h rate from the $tier tier's own input price", ({ tier, expectedCost }) => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + openAiNativeServiceTier: tier, + }) + const result = handler["normalizeUsage"]( + { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { ttl_12h_tokens: 100_000 }, + }, + }, + getGpt6AstraModel(), + ) + + if (!result) throw new Error("Expected usage") + // All input is consumed by the write, so the whole cost is the 12h class. + expect(result.totalCost).toBeCloseTo(expectedCost, 6) + }) + it("should pass total input tokens to calculateApiCostOpenAI", () => { const usage = { input_tokens: 100, @@ -551,4 +648,146 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split (Responses API)", () => { + // `normalizeUsage` is private. A narrow structural cast keeps the call typed + // without widening the handler to `any`; the return is asserted non-null because + // every call site below passes a usage object that yields a chunk. + const normalizeUsage = ( + h: OpenAiNativeHandler, + usage: unknown, + model: { id: string; info: ModelInfo }, + ): ApiStreamUsageChunk => { + const result = ( + h as unknown as { + normalizeUsage(u: unknown, m: { id: string; info: ModelInfo }): ApiStreamUsageChunk | undefined + } + ).normalizeUsage(usage, model) + expect(result).toBeDefined() + return result as ApiStreamUsageChunk + } + + it("should read ttl_12h_tokens and bill it at the 12h rate", () => { + // gpt-5.6-sol: input 4.0, cacheWritesPrice 5.0 (30m class), cacheWritesPrice12h 10.0. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 0, + ttl_12h_tokens: 100_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 100_000, + }) + + // gpt-6-astra: input 10.0, 12h rate 25.0. + // Base input: (10.0 / 1_000_000) * (100_000 - 100_000) = 0 + // 12h writes: (25.0 / 1_000_000) * 100_000 = 2.5 + // The pre-split behaviour charged the 30m rate: 12.5 * 0.1 = 1.25 (50% low). + expect(result.totalCost).toBeCloseTo(2.5, 6) + expect(result.totalCost).not.toBeCloseTo(1.25, 6) + }) + + it("should split a mixed 30m/12h write total without double counting", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 60_000, + ttl_12h_tokens: 40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 40_000, + }) + + // 30m remainder: 60_000 at 12.5 => 0.75 + // 12h: 40_000 at 25.0 => 1.0 + // Total: 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + + it("should leave the split undefined when the response omits it", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result.cacheWrite12hTokens).toBeUndefined() + // All writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) + + it("should clamp a 12h split that exceeds the write total", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 50_000, + cache_write_tokens_details: { + ttl_12h_tokens: 80_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // Clamped to the write total so no more 12h tokens are billed than written. + expect(result.cacheWrite12hTokens).toBe(50_000) + // Base input: (10.0 / 1_000_000) * (100_000 - 50_000) = 0.5 + // 12h writes: (25.0 / 1_000_000) * 50_000 = 1.25 + // Total: 0.5 + 1.25 = 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + + it("should clamp a negative 12h split at zero instead of emitting a negative count", () => { + // A misbehaving gateway can report a negative ttl_12h_tokens. Math.min alone + // leaves it negative, so the negative count reaches the Task accumulator and + // the webview. Shared costing clamps, but the emitted chunk must not lie. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_12h_tokens: -40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // A zero split is indistinguishable from "no split", so the field stays unset. + expect(result.cacheWrite12hTokens).toBeUndefined() + expect(result.cacheWrite12hTokens ?? 0).toBeGreaterThanOrEqual(0) + // All 100k writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) + }) }) diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 8a086b48db..c7e3663efb 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -534,6 +534,26 @@ describe("OpenAiNativeHandler", () => { expect(result).toBe("") }) + + it("should scope the prompt_cache_key to the handler session", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request.prompt_cache_key).toBe(handler["sessionId"]) + expect(request.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the prompt_cache_key when the model does not support prompt caching", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + handler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "codex-mini-latest" }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request).not.toHaveProperty("prompt_cache_key") + }) }) describe("getModel", () => { @@ -1446,6 +1466,130 @@ describe("OpenAiNativeHandler", () => { expect(callBody.previous_response_id).toBeUndefined() }) + it("sends a task-scoped prompt_cache_key in the body and no session_id header", async () => { + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + // Force the hand-built SSE path so the wire body and headers are inspectable. + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["task-cache-one", "task-cache-one", "task-cache-two"]) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(3) + expect(bodies[0].prompt_cache_key).toBe("task-cache-one") + // Same task, second turn: the key must not drift, or the growing prefix loses affinity. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + // A different task must not share the bucket. + expect(bodies[2].prompt_cache_key).toBe("task-cache-two") + expect(bodies[2].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + + // The retention control is untouched by this change. + expect(bodies[0].prompt_cache_retention).toBe("24h") + + // The affinity header is gone: OpenAI ignores it for caching and hardened + // gateways reject underscore header names outright. + for (const [, init] of mockFetch.mock.calls) { + expect(init.headers).not.toHaveProperty("session_id") + } + }) + + it("keeps two over-long task ids distinct instead of truncating them to one bucket", async () => { + // Task ids are uuidv7 today (36 chars), so this guards the wire contract for + // any caller that sends a longer id: a 64-char slice drops the distinguishing + // tail, so two ids sharing a 64-char prefix would land in one routing bucket. + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + // 70 characters: identical for the first 64, different in the tail. + const sharedPrefix = "a".repeat(64) + const longTaskIds = [`${sharedPrefix}-one`, `${sharedPrefix}-two`] + + for (const taskId of longTaskIds) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(2) + // OpenAI rejects keys over 64 characters, so the key must still fit. + expect(bodies[0].prompt_cache_key.length).toBeLessThanOrEqual(64) + expect(bodies[1].prompt_cache_key.length).toBeLessThanOrEqual(64) + // ...but the two tasks must not collide on that shortened form. + expect(bodies[1].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + }) + + it("sends the task-scoped prompt_cache_key through the SDK path", async () => { + mockResponsesCreate.mockResolvedValue(asyncStreamFrom([])) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["sdk-task-one", "sdk-task-one", "sdk-task-two"]) { + await collectStream(cacheHandler.createMessage(systemPrompt, messages, { taskId })) + } + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies[0].prompt_cache_key).toBe("sdk-task-one") + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + expect(bodies[2].prompt_cache_key).toBe("sdk-task-two") + expect(bodies[0].prompt_cache_retention).toBe("24h") + expect(mockResponsesCreate.mock.calls[0][1].headers).not.toHaveProperty("session_id") + }) + + it("keys completePrompt off the task id when the caller supplies one", async () => { + // Prompt enhancement builds a fresh handler per request, so a session-scoped key + // would start a new cache bucket every time. The task id keeps repeated + // enhancements of one task on the same bucket. + mockResponsesCreate.mockResolvedValue({ output: [] }) + + const first = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + const second = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + await first.completePrompt("enhance me", { taskId: "task-enhance" }) + await second.completePrompt("enhance me", { taskId: "task-enhance" }) + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies).toHaveLength(2) + expect(bodies[0].prompt_cache_key).toBe("task-enhance") + // Different handler instances, same task: the key must not drift. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + }) + + it("falls back to the handler session id when completePrompt gets no task id", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + await cacheHandler.completePrompt("enhance me") + + const [body] = mockResponsesCreate.mock.calls[0] + expect(body.prompt_cache_key).toBe(cacheHandler["sessionId"]) + }) + it("should provide helpful error messages for different error codes", async () => { const testCases = [ { status: 400, expectedMessage: "Invalid request to Responses API" }, diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index 1bf6669324..fec3e4a2e2 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -485,6 +485,115 @@ describe("OpenRouterHandler", () => { ) }) + describe("prompt cache TTL", () => { + const mockStreamWithUsage = () => + asyncStreamFrom([ + { id: "test-id", choices: [{ delta: { content: "test response" } }] }, + { id: "test-id", choices: [{ delta: {} }], usage: { prompt_tokens: 10, completion_tokens: 20 } }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // `openai` is mocked for this file, so the prototype is a plain object here. + // Reaching it once keeps the casts out of each test body. + const stubChatCreate = (mockCreate: ReturnType) => { + const proto = OpenAI.prototype as unknown as { chat: { completions: { create: unknown } } } + proto.chat = { completions: { create: mockCreate } } + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ ...mockOptions, openRouterModelId: "anthropic/claude-sonnet-4" }), + ) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("fails closed when the resolved model advertises no TTL capability", async () => { + // OpenRouter models come from a dynamic catalog that carries no TTL + // capability data, so a 1h knob must not reach the wire (ADR-0006 I2, + // and the ADR's stated risk for dynamic catalogs). + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on every marker when the resolved model advertises it", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + // Simulate a catalog that does carry capability data. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + }) + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockStream = asyncStreamFrom([{ error: { message: "API Error", code: 500 } }]) diff --git a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts index e69240c1da..5bafa25c78 100644 --- a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts +++ b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts @@ -98,6 +98,22 @@ vitest.mock("../fetchers/modelCache", () => ({ cacheReadsPrice: 0.1, description: "Claude 3.5 Haiku", }, + // The real gateway catalog never sets `promptCacheTtlSupport`, so this row + // exists only to inject the capability the accept path needs. It is kept + // separate from the shared haiku row so the reject-path test keeps a model + // with no TTL capability. + "anthropic/claude-3.5-sonnet": { + maxTokens: 8192, + contextWindow: 200000, + supportsImages: true, + supportsPromptCache: true, + promptCacheTtlSupport: ["1h"], + inputPrice: 3, + outputPrice: 15, + cacheWritesPrice: 3.75, + cacheReadsPrice: 0.3, + description: "Claude 3.5 Sonnet", + }, "openai/gpt-4o": { maxTokens: 16000, contextWindow: 128000, @@ -449,6 +465,47 @@ describe("VercelAiGatewayHandler", () => { expect(addCacheBreakpoints).toHaveBeenCalled() }) + it("does not emit a ttl for a gateway model with no TTL capability", async () => { + // The gateway catalog is fetched at runtime and carries no TTL capability, + // so a 1h knob must fail closed (ADR-0006 I2). + const { addCacheBreakpoints } = await import("../../transform/caching/vercel-ai-gateway") + const handler = new VercelAiGatewayHandler( + makeApiHandlerOptions({ + ...mockOptions, + vercelAiGatewayModelId: "anthropic/claude-3.5-haiku", + promptCacheTtl: "1h", + }), + ) + + await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next() + + // `addCacheBreakpoints` is file-mocked, so the request body never receives + // `cache_control` and `JSON.stringify(mockCreate...)` cannot see the TTL. + // The TTL only exists as the third argument to the mocked call. + expect(addCacheBreakpoints).toHaveBeenCalled() + const lastCall = vitest.mocked(addCacheBreakpoints).mock.calls.at(-1) + expect(lastCall).toBeDefined() + expect(lastCall![2]).toBeUndefined() + }) + + it("passes the resolved ttl for a gateway model with 1h TTL capability", async () => { + // The runtime catalog never sets `promptCacheTtlSupport`, so the accept path is + // only reachable with an injected capability. The dedicated catalog row keeps + // the reject-path model capability-free. + const { addCacheBreakpoints } = await import("../../transform/caching/vercel-ai-gateway") + const handler = new VercelAiGatewayHandler( + makeApiHandlerOptions({ + ...mockOptions, + vercelAiGatewayModelId: "anthropic/claude-3.5-sonnet", + promptCacheTtl: "1h", + }), + ) + + await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next() + + expect(vitest.mocked(addCacheBreakpoints).mock.calls.at(-1)![2]).toBe("1h") + }) + it("sets correct max_completion_tokens", async () => { const handler = new VercelAiGatewayHandler(mockOptions) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 87da090346..9334e436c3 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -15,6 +15,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { addCacheBreakpoints } from "../transform/caching/vertex" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" @@ -79,6 +80,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple const { supportsPromptCache } = info + // One TTL for the whole request (ADR-0006 I4). + const cacheTtl = resolveCacheTtl(this.options, info) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -116,9 +120,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple thinking, // Cache the system prompt if caching is enabled. system: supportsPromptCache - ? [{ text: systemPrompt, type: "text" as const, cache_control: { type: "ephemeral" } }] + ? [{ text: systemPrompt, type: "text" as const, cache_control: cacheControl(cacheTtl) }] : systemPrompt, - messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages) : sanitizedMessages, + messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages, cacheTtl) : sanitizedMessages, stream: true, ...nativeToolParams, } as Anthropic.Messages.MessageCreateParamsStreaming @@ -145,6 +149,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple case "message_start": { const usage = chunk.message!.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const cacheWrite1hTokens = usage.cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens; surfaced for per-turn // thinking telemetry (billing already includes it in output_tokens). @@ -153,6 +161,7 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0, cacheWriteTokens: usage.cache_creation_input_tokens || undefined, + cacheWrite1hTokens: cacheWrite1hTokens || undefined, cacheReadTokens: usage.cache_read_input_tokens || undefined, ...(typeof usage.output_tokens_details?.thinking_tokens === "number" ? { reasoningTokens: usage.output_tokens_details!.thinking_tokens } @@ -270,6 +279,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from + // the 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } @@ -313,11 +326,12 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple try { const { id, - info: { supportsPromptCache }, + info, temperature, maxTokens = ANTHROPIC_DEFAULT_MAX_TOKENS, reasoning: thinking, } = this.getModel() + const { supportsPromptCache } = info const params = { model: id, @@ -328,7 +342,13 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple { role: "user", content: supportsPromptCache - ? [{ type: "text" as const, text: prompt, cache_control: { type: "ephemeral" } }] + ? [ + { + type: "text" as const, + text: prompt, + cache_control: cacheControl(resolveCacheTtl(this.options, info)), + }, + ] : prompt, }, ], diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index d56a476a8f..977d4934ab 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -19,6 +19,7 @@ import { ApiStream } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { BaseProvider } from "./base-provider" import { bridgeExternalAbortSignal, hasUsableTimeout, toSdkTimeoutMs } from "./utils/abort-signal" @@ -64,7 +65,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { let stream: AnthropicStream - const cacheControl: CacheControlEphemeral = { type: "ephemeral" } const { id: modelId, betas = ["fine-grained-tool-streaming-2025-05-14"], @@ -79,6 +79,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa settings: this.options, }) + // One TTL for the whole request (ADR-0006 I4): every marker below reuses + // this single object, so a mixed-TTL request is unrepresentable. + const cacheTtl = resolveCacheTtl(this.options, info) + const marker: CacheControlEphemeral = cacheControl(cacheTtl) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -169,17 +174,17 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa temperature, thinking, // Setting cache breakpoint for system prompt so new tasks can reuse it. - system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + system: [{ text: systemPrompt, type: "text", cache_control: marker }], messages: sanitizedMessages.map((message, index) => { if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { return { ...message, content: typeof message.content === "string" - ? [{ type: "text", text: message.content, cache_control: cacheControl }] + ? [{ type: "text", text: message.content, cache_control: marker }] : message.content.map((content, contentIndex) => contentIndex === message.content.length - 1 - ? { ...content, cache_control: cacheControl } + ? { ...content, cache_control: marker } : content, ), } @@ -256,6 +261,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -266,9 +272,14 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // The per-TTL breakdown decomposes `cache_creation_input_tokens`; it is + // absent for providers that do not report it, which leaves the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens (0 ⇒ the model skipped // thinking this turn). `output_tokens` already includes them for @@ -280,6 +291,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, ...(typeof startThinkingTokens === "number" ? { reasoningTokens: startThinkingTokens } : {}), } @@ -287,6 +299,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -385,6 +398,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { @@ -437,6 +451,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // The tier's own TTL rate must come along: the entry-level rate was + // derived from the 200K input price, so keeping it here would bill a + // 1M-context 1h write at half its tier rate. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index e09cc0b17b..d11f5f7dba 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -1295,6 +1295,10 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH inputPrice: tier?.inputPrice ?? modelConfig.info.inputPrice, outputPrice: tier?.outputPrice ?? modelConfig.info.outputPrice, cacheWritesPrice: tier?.cacheWritesPrice ?? modelConfig.info.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from the + // 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier?.cacheWritesPrice1h ?? modelConfig.info.cacheWritesPrice1h, + cacheWritesPrice12h: tier?.cacheWritesPrice12h ?? modelConfig.info.cacheWritesPrice12h, cacheReadsPrice: tier?.cacheReadsPrice ?? modelConfig.info.cacheReadsPrice, } } diff --git a/src/api/providers/minimax.ts b/src/api/providers/minimax.ts index 88c8a443a1..2addaf1a39 100644 --- a/src/api/providers/minimax.ts +++ b/src/api/providers/minimax.ts @@ -137,6 +137,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -147,20 +148,27 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + yield { type: "usage", inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, } inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -247,6 +255,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index fbbecf5760..6feb0ae75f 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -1,4 +1,5 @@ import * as os from "os" +import { createHash } from "crypto" import { v7 as uuidv7 } from "uuid" import { Anthropic } from "@anthropic-ai/sdk" import OpenAI from "openai" @@ -16,6 +17,8 @@ import { SERVICE_TIER_KEY, type ServiceTier, ApiProviderError, + CACHE_WRITE_MULTIPLIER_1H, + CACHE_WRITE_MULTIPLIER_12H, } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" @@ -34,6 +37,12 @@ import { sanitizeOpenAiCallId } from "../../utils/tool-id" export type OpenAiNativeModel = ReturnType +/** + * OpenAI caps `prompt_cache_key` at 64 characters; longer values are rejected with a 400. + * Task ids are uuidv7 today (36 chars), so this is a guard rather than a routine truncation. + */ +const PROMPT_CACHE_KEY_MAX_LENGTH = 64 + export class OpenAiNativeHandler extends BaseProvider implements SingleCompletionHandler { protected options: ApiHandlerOptions private client: OpenAI @@ -96,14 +105,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio this.options.enableResponsesReasoningSummary = true } const apiKey = this.options.openAiNativeApiKey ?? NOT_PROVIDED - // Include originator, session_id, and User-Agent headers for API tracking and debugging + // Include originator and User-Agent headers for API tracking and debugging. + // Cache affinity rides on the body `prompt_cache_key`, not on a header: OpenAI documents + // the `session_id` header as ignored on the public Responses API, and underscore header + // names are rejected outright by hardened gateways (Envoy/nginx `underscores_in_headers`). const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` this.client = new OpenAI({ baseURL: this.options.openAiNativeBaseUrl || undefined, apiKey, defaultHeaders: { originator: "zoo-code", - session_id: this.sessionId, "User-Agent": userAgent, }, timeout: this.timeoutMs, @@ -124,6 +135,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const writesFromDetails = typeof inputDetails?.cache_write_tokens === "number" ? inputDetails.cache_write_tokens : 0 + // The Responses API splits cache writes by TTL class under + // `input_tokens_details.cache_write_tokens_details`. The installed SDK types + // (openai@5.23.2) do not declare it yet, so it is read defensively; the + // fields are a decomposition of `cache_write_tokens`, not an addition. + const writeDetails = inputDetails?.cache_write_tokens_details + const writes12hFromDetails = typeof writeDetails?.ttl_12h_tokens === "number" ? writeDetails.ttl_12h_tokens : 0 + // If total input tokens are missing but we have details, derive from them let totalInputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0 if ( @@ -144,6 +162,12 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0 + // Clamp the 12h subset to the write total so a malformed response cannot bill + // more 12h tokens than were written at all. The lower clamp matters too: a + // negative `ttl_12h_tokens` from a misbehaving gateway would otherwise reach the + // Task accumulator and the webview as a negative count. + const cacheWrite12hTokens = Math.max(0, Math.min(writes12hFromDetails, cacheWriteTokens)) + // Resolve effective tier: prefer actual tier from response; otherwise requested tier const effectiveTier = this.lastServiceTier || (this.options.openAiNativeServiceTier as ServiceTier | undefined) || undefined @@ -158,6 +182,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio cacheWriteTokens, cacheReadTokens, effectiveTier, + undefined, + cacheWrite12hTokens, ) const reasoningTokens = @@ -171,6 +197,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio inputTokens: totalInputTokens, outputTokens: totalOutputTokens, cacheWriteTokens, + cacheWrite12hTokens: cacheWrite12hTokens || undefined, cacheReadTokens, ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}), totalCost, @@ -330,6 +357,11 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio include?: string[] /** Prompt cache retention policy: "in_memory" (default) or "24h" for extended caching */ prompt_cache_retention?: "in_memory" | "24h" + /** + * Cache routing affinity key. OpenAI documents this body field as the routing lever; + * the `session_id` request header is not part of the prompt-caching API. + */ + prompt_cache_key?: string tools?: Array<{ type: "function" name: string @@ -347,6 +379,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Decide whether to enable extended prompt cache retention for this request const promptCacheRetention = this.getPromptCacheRetention(model) + // Cache routing affinity: a stable, task-scoped key lets OpenAI route every turn of the + // same task to the machine holding its cached prefix. + const promptCacheKey = this.getPromptCacheKey(model, metadata) + const body: ResponsesRequestBody = { model: model.id, input: formattedInput, @@ -379,6 +415,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Enable extended prompt cache retention for models that support it. // This uses the OpenAI Responses API `prompt_cache_retention` parameter. ...(promptCacheRetention ? { prompt_cache_retention: promptCacheRetention } : {}), + // Cache routing affinity key (body field, not the inert `session_id` header). + ...(promptCacheKey ? { prompt_cache_key: promptCacheKey } : {}), tools: (metadata?.tools ?? []) .filter((tool) => tool.type === "function") .map((tool) => { @@ -418,12 +456,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` const requestHeaders: Record = { originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, } @@ -557,7 +593,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio private async *makeResponsesApiRequest( requestBody: any, model: OpenAiNativeModel, - metadata?: ApiHandlerCreateMessageMetadata, + _metadata?: ApiHandlerCreateMessageMetadata, _systemPrompt?: string, _messages?: Anthropic.Messages.MessageParam[], ): ApiStream { @@ -568,8 +604,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` try { @@ -579,7 +614,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, }, body: JSON.stringify(requestBody), @@ -1420,6 +1454,33 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio return undefined } + /** + * Returns the prompt cache routing key for this request, if the model caches prompts. + * + * OpenAI keys cache affinity off the `prompt_cache_key` body field; the `session_id` + * request header is not part of the prompt-caching API. The key must stay identical + * across every turn of one task (so its growing prefix keeps landing on the same + * machine) and differ between tasks (so unrelated prefixes do not compete for one + * routing bucket). `metadata.taskId` is the longest-lived identifier the handler + * receives: it is the persisted task id, stable across turns, restarts and + * rehydration, whereas `this.sessionId` is regenerated per handler instance. + * The handler session id is the fallback for callers that send no task id. + */ + private getPromptCacheKey( + model: OpenAiNativeModel, + metadata?: ApiHandlerCreateMessageMetadata, + ): string | undefined { + if (!model.info.supportsPromptCache) return undefined + + const key = metadata?.taskId || this.sessionId + if (!key) return undefined + + // Hash rather than slice: a 64-char prefix drops the distinguishing tail, so two + // distinct task ids that share a long prefix would land in one routing bucket. + // sha256 hex is 64 characters, so the result fits the cap without truncation. + return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? createHash("sha256").update(key).digest("hex") : key + } + /** * Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available. * If no tier or no overrides exist, the original ModelInfo is returned. @@ -1437,6 +1498,17 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio outputPrice: tierInfo.outputPrice ?? info.outputPrice, cacheReadsPrice: tierInfo.cacheReadsPrice ?? info.cacheReadsPrice, cacheWritesPrice: tierInfo.cacheWritesPrice ?? info.cacheWritesPrice, + // A service tier that overrides the write rate must also override the TTL + // rate, otherwise a flex/priority 12h write bills at the standard-tier 12h + // rate. Derived from the tier's own input price so the 2.5x ratio holds. + cacheWritesPrice12h: + tierInfo.cacheWritesPrice12h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_12H : undefined) ?? + info.cacheWritesPrice12h, + cacheWritesPrice1h: + tierInfo.cacheWritesPrice1h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_1H : undefined) ?? + info.cacheWritesPrice1h, } } @@ -1500,7 +1572,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio getResponseId(): string | undefined { return this.lastResponseId } - async completePrompt(prompt: string, _options?: CompletePromptOptions): Promise { + async completePrompt(prompt: string, options?: CompletePromptOptions): Promise { try { this.abortController = new AbortController() @@ -1531,6 +1603,17 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio requestBody[SERVICE_TIER_KEY] = serviceTier } + // Cache routing affinity key. A caller that knows the task passes it, so repeated + // completions for one task share a bucket across handler instances; otherwise the + // handler session id scopes the key to this instance. + const promptCacheKey = this.getPromptCacheKey( + model, + options?.taskId ? { taskId: options.taskId } : undefined, + ) + if (promptCacheKey) { + requestBody.prompt_cache_key = promptCacheKey + } + // Add reasoning if supported if (reasoningEffort) { requestBody.reasoning = { diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 36e09aa883..6d2057a7a3 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -26,6 +26,7 @@ import { ApiStreamChunk } from "../transform/stream" import { convertToR1Format } from "../transform/r1-format" import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic" import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { OpenRouterReasoningParams } from "../transform/reasoning" import { getModelParams } from "../transform/model-params" @@ -306,9 +307,12 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH // TODO: Add a `promptCacheStratey` field to `ModelInfo`. if (OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId)) { if (modelId.startsWith("google")) { + // Gemini markers keep their default TTL: ADR-0006 scopes the knob to + // Anthropic-shaped markers only, and Gemini has no 1h class. addGeminiCacheBreakpoints(systemPrompt, openAiMessages) } else { - addAnthropicCacheBreakpoints(systemPrompt, openAiMessages) + // Capability comes from the resolved ModelInfo, never from the id. + addAnthropicCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, model.info)) } } diff --git a/src/api/providers/vercel-ai-gateway.ts b/src/api/providers/vercel-ai-gateway.ts index 2f1586b928..135ac8bef6 100644 --- a/src/api/providers/vercel-ai-gateway.ts +++ b/src/api/providers/vercel-ai-gateway.ts @@ -17,6 +17,7 @@ import { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -82,7 +83,7 @@ export class VercelAiGatewayHandler extends RouterProvider implements SingleComp ] if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } const supportsTemperature = info.supportsTemperature !== false && this.supportsTemperature(modelId) diff --git a/src/api/providers/zoo-gateway.ts b/src/api/providers/zoo-gateway.ts index 66ccac5067..5f33ee48fe 100644 --- a/src/api/providers/zoo-gateway.ts +++ b/src/api/providers/zoo-gateway.ts @@ -19,6 +19,7 @@ import { t } from "../../i18n" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -208,7 +209,7 @@ export class ZooGatewayHandler extends RouterProvider implements SingleCompletio // Apply prompt caching for models that support it // Zoo Gateway serves the same models as Vercel AI Gateway, so caching support is identical if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } // Build request headers with enrichment metadata diff --git a/src/api/transform/caching/__tests__/ttl.spec.ts b/src/api/transform/caching/__tests__/ttl.spec.ts new file mode 100644 index 0000000000..7c4253320c --- /dev/null +++ b/src/api/transform/caching/__tests__/ttl.spec.ts @@ -0,0 +1,185 @@ +// npx vitest run src/api/transform/caching/__tests__/ttl.spec.ts + +import OpenAI from "openai" + +import { addCacheBreakpoints as addAnthropicBreakpoints } from "../anthropic" +import { addCacheBreakpoints as addVertexBreakpoints } from "../vertex" +import { addCacheBreakpoints as addGatewayBreakpoints } from "../vercel-ai-gateway" +import { cacheControl, resolveCacheTtl } from "../ttl" + +/** + * Collect every `cache_control` object reachable in a serialized request. + * + * Counting occurrences rather than spot-checking one marker is the point: a + * transform that forgot one of its marker sites would still pass a single-site + * assertion, which is exactly the failure mode ADR-0006 I4 is meant to exclude. + */ +const collectCacheControls = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +/** The serialized request body, so assertions run on the bytes that would be sent. */ +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) + +describe("resolveCacheTtl", () => { + it("returns undefined when the knob is unset, even for a capable model", () => { + expect(resolveCacheTtl({}, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + expect(resolveCacheTtl(undefined, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns undefined for an explicit 5m so the wire shape stays unchanged", () => { + expect(resolveCacheTtl({ promptCacheTtl: "5m" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns 1h when the knob asks for it and the model advertises it", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBe("1h") + }) + + it("fails closed when the model advertises no capability at all", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, {})).toBeUndefined() + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, undefined)).toBeUndefined() + }) + + it("fails closed when the model advertises 5m only", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m"] })).toBeUndefined() + }) +}) + +describe("cacheControl", () => { + it("is the historical literal when no TTL is resolved", () => { + expect(cacheControl(undefined)).toEqual({ type: "ephemeral" }) + expect(Object.keys(cacheControl(undefined))).toEqual(["type"]) + }) + + it("carries the ttl key only when a TTL is resolved", () => { + expect(cacheControl("1h")).toEqual({ type: "ephemeral", ttl: "1h" }) + }) +}) + +describe("caching transforms under a prompt cache TTL", () => { + const systemPrompt = "You are a helpful assistant." + + const buildOpenAiMessages = (): OpenAI.Chat.ChatCompletionMessageParam[] => [ + { role: "system", content: systemPrompt }, + { role: "user", content: "First message" }, + { role: "assistant", content: "First response" }, + { role: "user", content: "Second message" }, + ] + + const buildAnthropicMessages = () => [ + { role: "user" as const, content: "First message" }, + { role: "assistant" as const, content: "First response" }, + { role: "user" as const, content: "Second message" }, + ] + + describe("default parity (ADR-0006 I3)", () => { + it("emits no ttl key anywhere when the TTL argument is omitted", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages) + + const serialized = serialize(messages) + const rawJson = JSON.stringify(messages) + + expect(rawJson).not.toContain("ttl") + expect(serialized).not.toContain("ttl") + + const controls = collectCacheControls(serialized) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("is byte-identical to the pre-knob literal for every transform", () => { + const anthropicMessages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, anthropicMessages) + + const gatewayMessages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, gatewayMessages) + + const vertexMessages = addVertexBreakpoints(buildAnthropicMessages()) + + // Compare the raw serialized bytes, not parsed objects: `toEqual` ignores key + // insertion order, so a transform that reorders request keys would pass while + // the bytes on the wire changed. The expected strings are literals for the + // same reason: building them from the same helper would re-derive the order. + expect(JSON.stringify(anthropicMessages)).toBe( + '[{"role":"system","content":[{"type":"text","text":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}}]},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', + ) + + expect(JSON.stringify(gatewayMessages)).toBe( + '[{"role":"system","content":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', + ) + + expect(JSON.stringify(vertexMessages)).toBe( + '[{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', + ) + }) + }) + + describe("opt-in uniformity (ADR-0006 I4)", () => { + it("puts ttl 1h on every anthropic marker, not just the first", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(controls.every((control) => control.type === "ephemeral")).toBe(true) + }) + + it("puts ttl 1h on every gateway marker", () => { + const messages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("puts ttl 1h on every vertex marker", () => { + const messages = addVertexBreakpoints(buildAnthropicMessages(), "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("never mixes TTLs within one request", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const ttls = new Set(collectCacheControls(serialize(messages)).map((control) => control.ttl)) + expect(ttls).toEqual(new Set(["1h"])) + }) + }) + + describe("signature is additive-optional", () => { + it("keeps the two-argument call compiling and byte-identical", () => { + const withDefault = buildOpenAiMessages() + const withExplicitUndefined = buildOpenAiMessages() + + addAnthropicBreakpoints(systemPrompt, withDefault) + addAnthropicBreakpoints(systemPrompt, withExplicitUndefined, undefined) + + expect(serialize(withDefault)).toEqual(serialize(withExplicitUndefined)) + }) + }) +}) diff --git a/src/api/transform/caching/anthropic.ts b/src/api/transform/caching/anthropic.ts index cff671a56c..3bdcba4ba1 100644 --- a/src/api/transform/caching/anthropic.ts +++ b/src/api/transform/caching/anthropic.ts @@ -1,10 +1,20 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + messages[0] = { role: "system", // @ts-ignore-next-line - content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], + content: [{ type: "text", text: systemPrompt, cache_control: control }], } // Ensure all user messages have content in array format first @@ -35,7 +45,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. } // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } }) } diff --git a/src/api/transform/caching/ttl.ts b/src/api/transform/caching/ttl.ts new file mode 100644 index 0000000000..4ba2652fe2 --- /dev/null +++ b/src/api/transform/caching/ttl.ts @@ -0,0 +1,32 @@ +import type { ModelInfo, PromptCacheTtl } from "@roo-code/types" +import type { CacheControlEphemeral } from "@anthropic-ai/sdk/resources" + +/** + * Resolve the TTL a request may emit, gated by the provider-profile knob and the + * model's catalog capability. Both gates fail closed: absent capability means + * "5m only" (ADR-0006 I2), and capability is read from the resolved `ModelInfo` + * rather than inferred from the model id. + * + * `"5m"` resolves to `undefined` on purpose. It is the vendor default, so emitting + * it would add bytes without changing behavior; only `"1h"` may change the wire, + * which keeps default parity (I3) checkable by a single grep. + */ +export function resolveCacheTtl( + settings: { promptCacheTtl?: PromptCacheTtl } | undefined, + info: Pick | undefined, +): PromptCacheTtl | undefined { + if (settings?.promptCacheTtl !== "1h") { + return undefined + } + + return info?.promptCacheTtlSupport?.includes("1h") ? "1h" : undefined +} + +/** + * Build the `cache_control` object for one marker. Every marker in a request goes + * through this function, which makes one uniform TTL per request (I4) structural + * instead of a convention each call site has to remember. + */ +export function cacheControl(ttl: PromptCacheTtl | undefined): CacheControlEphemeral { + return ttl ? { type: "ephemeral", ttl } : { type: "ephemeral" } +} diff --git a/src/api/transform/caching/vercel-ai-gateway.ts b/src/api/transform/caching/vercel-ai-gateway.ts index 0d3834bbe5..7272de34d7 100644 --- a/src/api/transform/caching/vercel-ai-gateway.ts +++ b/src/api/transform/caching/vercel-ai-gateway.ts @@ -1,12 +1,22 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + // Apply cache_control to system message at the message level messages[0] = { role: "system", content: systemPrompt, // @ts-ignore-next-line - cache_control: { type: "ephemeral" }, + cache_control: control, } // Add cache_control to the last two user messages for conversation context caching @@ -23,7 +33,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. if (lastTextPart && lastTextPart.text && lastTextPart.text.length > 0) { // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } } }) diff --git a/src/api/transform/caching/vertex.ts b/src/api/transform/caching/vertex.ts index 48bf261587..e223bb4d11 100644 --- a/src/api/transform/caching/vertex.ts +++ b/src/api/transform/caching/vertex.ts @@ -1,6 +1,13 @@ import { Anthropic } from "@anthropic-ai/sdk" -export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + messages: Anthropic.Messages.MessageParam[], + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam[] { // Find indices of user messages that we want to cache. // We only cache the last two user messages to stay within the 4-block limit // (1 block for system + 1 block each for last two user messages = 3 total). @@ -12,18 +19,23 @@ export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) return messages.map((message, index) => message.role !== "assistant" && (index === lastIndex || index === secondLastIndex) - ? cachedMessage(message) + ? cachedMessage(message, ttl) : message, ) } -function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Messages.MessageParam { +function cachedMessage( + message: Anthropic.Messages.MessageParam, + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam { + const control = cacheControl(ttl) + // For string content, we convert to array format with optional cache control. if (typeof message.content === "string") { return { ...message, // For string content, we only have one block so it's always the last block. - content: [{ type: "text" as const, text: message.content, cache_control: { type: "ephemeral" } }], + content: [{ type: "text" as const, text: message.content, cache_control: control }], } } @@ -41,7 +53,7 @@ function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Mess ? { ...content, // Check if this is the last text block using our pre-calculated index. - ...(index === lastTextBlockIndex && { cache_control: { type: "ephemeral" } }), + ...(index === lastTextBlockIndex && { cache_control: control }), } : content, ), diff --git a/src/api/transform/stream.ts b/src/api/transform/stream.ts index 40efeb3bb0..a3d4b533e7 100644 --- a/src/api/transform/stream.ts +++ b/src/api/transform/stream.ts @@ -59,7 +59,18 @@ export interface ApiStreamUsageChunk { type: "usage" inputTokens: number outputTokens: number + /** Total cache-write tokens, across every TTL class. */ cacheWriteTokens?: number + /** + * The 1h-TTL subset of `cacheWriteTokens` (Anthropic `ephemeral_1h_input_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite1hTokens?: number + /** + * The 12h-TTL subset of `cacheWriteTokens` (OpenAI `ttl_12h_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite12hTokens?: number cacheReadTokens?: number reasoningTokens?: number totalCost?: number diff --git a/src/core/config/__tests__/importExport.spec.ts b/src/core/config/__tests__/importExport.spec.ts index 90db81d8a1..6c66655325 100644 --- a/src/core/config/__tests__/importExport.spec.ts +++ b/src/core/config/__tests__/importExport.spec.ts @@ -357,6 +357,65 @@ describe("importExport", () => { expect(mockContextProxy.setValues).not.toHaveBeenCalled() }) + it("round-trips a prompt cache TTL through export and import", async () => { + // The knob rides the provider profile, so the export/import schema is the + // only thing that can lose it. Both classes are exercised because a + // schema that only accepted "1h" would still pass a one-sided test. + const profiles = { + currentApiConfigName: "cache", + apiConfigs: { + cache: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + id: "cache-id", + promptCacheTtl: "1h" as const, + }, + explicit: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + id: "explicit-id", + promptCacheTtl: "5m" as const, + }, + }, + } + + ;(vscode.window.showSaveDialog as Mock).mockResolvedValue({ fsPath: "/mock/path/export.json" }) + mockProviderSettingsManager.export.mockResolvedValue(profiles) + mockContextProxy.export.mockResolvedValue({ mode: "code" }) + + await exportSettings({ + providerSettingsManager: mockProviderSettingsManager, + contextProxy: mockContextProxy, + }) + + const exported = (safeWriteJson as Mock).mock.calls.at(-1)?.[1] + expect(exported.providerProfiles.apiConfigs.cache.promptCacheTtl).toBe("1h") + expect(exported.providerProfiles.apiConfigs.explicit.promptCacheTtl).toBe("5m") + + ;(vscode.window.showOpenDialog as Mock).mockResolvedValue([{ fsPath: "/mock/path/export.json" }]) + ;(fs.readFile as Mock).mockResolvedValue(JSON.stringify(exported)) + mockProviderSettingsManager.export.mockResolvedValue({ currentApiConfigName: "default", apiConfigs: {} }) + mockProviderSettingsManager.listConfig.mockResolvedValue([]) + + const result = await importSettings({ + providerSettingsManager: mockProviderSettingsManager, + contextProxy: mockContextProxy, + customModesManager: mockCustomModesManager, + }) + + expect(result.success).toBe(true) + expect(mockProviderSettingsManager.import).toHaveBeenCalledWith( + expect.objectContaining({ + apiConfigs: expect.objectContaining({ + cache: expect.objectContaining({ promptCacheTtl: "1h" }), + explicit: expect.objectContaining({ promptCacheTtl: "5m" }), + }), + }), + ) + }) + it("should import settings successfully when globalSettings key is missing", async () => { ;(vscode.window.showOpenDialog as Mock).mockResolvedValue([{ fsPath: "/mock/path/settings.json" }]) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 3202358c09..8a904e94d6 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -4021,6 +4021,8 @@ export class Task extends EventEmitter implements TaskLike { try { let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 + let cacheWrite12hTokens = 0 let cacheReadTokens = 0 let inputTokens = 0 let outputTokens = 0 @@ -4056,6 +4058,8 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, + cacheWrite12hTokens, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4063,6 +4067,9 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + undefined, + cacheWrite1hTokens, + cacheWrite12hTokens, ) this.clineMessages[lastApiReqIndex].text = JSON.stringify({ @@ -4210,6 +4217,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens += chunk.inputTokens outputTokens += chunk.outputTokens cacheWriteTokens += chunk.cacheWriteTokens ?? 0 + cacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + cacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 cacheReadTokens += chunk.cacheReadTokens ?? 0 totalCost = chunk.totalCost break @@ -4450,6 +4459,8 @@ export class Task extends EventEmitter implements TaskLike { input: inputTokens, output: outputTokens, cacheWrite: cacheWriteTokens, + cacheWrite1h: cacheWrite1hTokens, + cacheWrite12h: cacheWrite12hTokens, cacheRead: cacheReadTokens, total: totalCost, } @@ -4466,6 +4477,8 @@ export class Task extends EventEmitter implements TaskLike { let bgInputTokens = currentTokens.input let bgOutputTokens = currentTokens.output let bgCacheWriteTokens = currentTokens.cacheWrite + let bgCacheWrite1hTokens = currentTokens.cacheWrite1h + let bgCacheWrite12hTokens = currentTokens.cacheWrite12h let bgCacheReadTokens = currentTokens.cacheRead let bgTotalCost = currentTokens.total @@ -4475,6 +4488,8 @@ export class Task extends EventEmitter implements TaskLike { input: number output: number cacheWrite: number + cacheWrite1h: number + cacheWrite12h: number cacheRead: number total?: number }, @@ -4491,6 +4506,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens = tokens.input outputTokens = tokens.output cacheWriteTokens = tokens.cacheWrite + cacheWrite1hTokens = tokens.cacheWrite1h + cacheWrite12hTokens = tokens.cacheWrite12h cacheReadTokens = tokens.cacheRead totalCost = tokens.total @@ -4512,7 +4529,10 @@ export class Task extends EventEmitter implements TaskLike { modelId, ) - // Use the appropriate cost function based on the API protocol + // Use the appropriate cost function based on the API protocol. + // The per-TTL split must be passed here too: without it this + // telemetry recompute bills every write at the short-TTL rate, + // which is exactly the under-report this PR exists to fix. const costResult = apiProtocol === "anthropic" ? calculateApiCostAnthropic( @@ -4521,6 +4541,8 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4528,6 +4550,9 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + undefined, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) TelemetryService.instance.captureLlmCompletion(this.taskId, { @@ -4568,6 +4593,8 @@ export class Task extends EventEmitter implements TaskLike { bgInputTokens += chunk.inputTokens bgOutputTokens += chunk.outputTokens bgCacheWriteTokens += chunk.cacheWriteTokens ?? 0 + bgCacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + bgCacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 bgCacheReadTokens += chunk.cacheReadTokens ?? 0 bgTotalCost = chunk.totalCost } @@ -4586,6 +4613,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, @@ -4611,6 +4640,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index bbf77a85e9..dab89538fa 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -534,6 +534,59 @@ describe("Cline", () => { }) }) + describe("cache-write TTL split in telemetry", () => { + it("bills a 1h cache write at the 1h rate when usage is captured after a cancelled stream", async () => { + // glm P1: the telemetry cost recompute inside captureUsageData must receive + // the per-TTL split. Before the fix it passed only the aggregate write count, + // so a 1h write was billed at the 5m rate (0.375 instead of 0.6 for 100k + // tokens at claude-3-5-sonnet's 3.0 input price). + const task = await createTaskWithManualRetries() + const captureLlmCompletion = vi.spyOn(TelemetryService.instance, "captureLlmCompletion") + + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) + // A stream that reports the split but never yields a final cost chunk, which is + // the shape a cancelled request leaves behind. The second call throws so the + // request loop terminates the way the neighbouring tests do. + vi.spyOn(task, "attemptApiRequest") + .mockImplementationOnce(() => + stream([ + { + type: "usage", + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 100_000, + cacheReadTokens: 0, + }, + { type: "text", text: "partial response" }, + ]), + ) + .mockImplementation(() => { + throw new Error("stop after usage capture") + }) + + await task.recursivelyMakeClineRequests([{ type: "text", text: "original user request" }]) + + // The drain is fire-and-forget and reaches `captureLlmCompletion` after several + // awaits, some of which touch the filesystem. A fixed sleep races that work on a + // loaded worker, so poll for the captured call instead. + await vi.waitFor(() => + expect( + captureLlmCompletion.mock.calls.some(([, payload]) => payload?.cacheWriteTokens === 100_000), + ).toBe(true), + ) + + const call = captureLlmCompletion.mock.calls.find(([, payload]) => payload?.cacheWriteTokens === 100_000) + expect(call).toBeDefined() + + // claude-3-5-sonnet-20241022: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-fix lumped behaviour billed (3.75 / 1_000_000) * 100_000 = 0.375. + expect(call![1].cost).toBeCloseTo(0.6, 9) + expect(call![1].cost).not.toBeCloseTo(0.375, 9) + }) + }) + describe("request continuation", () => { it("sends the no-tool-use reminder in a second request when the first response has no tool use", async () => { const task = await createTaskWithManualRetries() diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 779313d385..73a9ad8964 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -2111,6 +2111,65 @@ describe("ClineProvider", () => { expect(state.allowedReadFiles).toEqual(["notes.md"]) }) + describe("prompt cache TTL round trip", () => { + // The provider-settings route: the knob rides `apiConfiguration`, so the + // round trip is the profile write plus the state post, with no dedicated + // message type and no settings-specific branch in the message handler. + test("a saved 1h survives a fresh getStateToPostToWebview payload", async () => { + await provider.resolveWebviewView(mockWebviewView) + + await provider.upsertProviderProfile("ttl-profile", { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + promptCacheTtl: "1h", + }) + + // A fresh read, not the in-memory value the save path just wrote: this is + // the "saved control reverts visually" case the checklist calls out. + const posted = await provider.getStateToPostToWebview() + + expect(posted.apiConfiguration.promptCacheTtl).toBe("1h") + expect((await provider.getState()).apiConfiguration.promptCacheTtl).toBe("1h") + }) + + test("an unset knob stays absent rather than becoming an explicit 5m", async () => { + await provider.resolveWebviewView(mockWebviewView) + + await provider.upsertProviderProfile("default-ttl-profile", { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-sonnet-4-5", + apiKey: "test-key", + }) + + const posted = await provider.getStateToPostToWebview() + + // Consumers apply the vendor default, so the stored value must be absent + // rather than a literal "5m" that would look like a user choice. + expect(posted.apiConfiguration.promptCacheTtl).toBeUndefined() + expect(posted.apiConfiguration).not.toHaveProperty("promptCacheTtl", "5m") + }) + + test("a hand-edited 1h for a model without the capability is stored, not silently rewritten", async () => { + await provider.resolveWebviewView(mockWebviewView) + + // MiniMax caches prompts but advertises no TTL class. The profile can only + // have been written by hand or imported, and the save path must keep what + // the user stored: capability is enforced where the request is built (the + // emitter fails closed), so a save-time rewrite would be a second, silent + // source of truth for the same rule. + await provider.upsertProviderProfile("hand-edited-profile", { + apiProvider: providerIdentifiers.minimax, + apiKey: "test-key", + promptCacheTtl: "1h", + }) + + const posted = await provider.getStateToPostToWebview() + + expect(posted.apiConfiguration.promptCacheTtl).toBe("1h") + }) + }) + test("getStateToPostToWebview defaults allowed read files to an empty list", async () => { await provider.resolveWebviewView(mockWebviewView) diff --git a/src/core/webview/__tests__/messageEnhancer.test.ts b/src/core/webview/__tests__/messageEnhancer.test.ts index 964c4ebdef..6bf16b66e8 100644 --- a/src/core/webview/__tests__/messageEnhancer.test.ts +++ b/src/core/webview/__tests__/messageEnhancer.test.ts @@ -77,6 +77,7 @@ describe("MessageEnhancer", () => { expect(mockSingleCompletionHandler).toHaveBeenCalledWith( mockApiConfiguration, expect.stringContaining("Write a function to calculate fibonacci"), + undefined, ) }) @@ -99,7 +100,7 @@ describe("MessageEnhancer", () => { apiKey: "enhancement-key", apiModelId: "claude-3", } - expect(mockSingleCompletionHandler).toHaveBeenCalledWith(expectedConfig, expect.any(String)) + expect(mockSingleCompletionHandler).toHaveBeenCalledWith(expectedConfig, expect.any(String), undefined) }) it("should include task history when enabled", async () => { @@ -238,7 +239,40 @@ describe("MessageEnhancer", () => { }) // Should use the default config - expect(mockSingleCompletionHandler).toHaveBeenCalledWith(mockApiConfiguration, expect.any(String)) + expect(mockSingleCompletionHandler).toHaveBeenCalledWith( + mockApiConfiguration, + expect.any(String), + undefined, + ) + }) + + it("passes the task id through to the completion so the cache key stays task-scoped", async () => { + await MessageEnhancer.enhanceMessage({ + text: "Test", + apiConfiguration: mockApiConfiguration, + listApiConfigMeta: mockListApiConfigMeta, + providerSettingsManager: mockProviderSettingsManager, + taskId: "task-123", + }) + + expect(mockSingleCompletionHandler).toHaveBeenCalledWith(mockApiConfiguration, expect.any(String), { + taskId: "task-123", + }) + }) + + it("omits the completion options when no task is open", async () => { + await MessageEnhancer.enhanceMessage({ + text: "Test", + apiConfiguration: mockApiConfiguration, + listApiConfigMeta: mockListApiConfigMeta, + providerSettingsManager: mockProviderSettingsManager, + }) + + expect(mockSingleCompletionHandler).toHaveBeenCalledWith( + mockApiConfiguration, + expect.any(String), + undefined, + ) }) it("should handle empty task history gracefully", async () => { diff --git a/src/core/webview/messageEnhancer.ts b/src/core/webview/messageEnhancer.ts index 90c1f3b37d..6994e57291 100644 --- a/src/core/webview/messageEnhancer.ts +++ b/src/core/webview/messageEnhancer.ts @@ -13,6 +13,12 @@ export interface MessageEnhancerOptions { includeTaskHistoryInEnhance?: boolean currentClineMessages?: ClineMessage[] providerSettingsManager: ProviderSettingsManager + /** + * Task the enhancement belongs to, when one is open. Passed through to the provider so + * repeated enhancements of one task reuse a single prompt-cache bucket instead of + * starting a new one per handler instance. + */ + taskId?: string } export interface MessageEnhancerResult { @@ -41,6 +47,7 @@ export class MessageEnhancer { includeTaskHistoryInEnhance, currentClineMessages, providerSettingsManager, + taskId, } = options // Determine which API configuration to use @@ -76,7 +83,11 @@ export class MessageEnhancer { ) // Call the single completion handler to get the enhanced prompt - const enhancedText = await singleCompletionHandler(configToUse, enhancementPrompt) + const enhancedText = await singleCompletionHandler( + configToUse, + enhancementPrompt, + taskId ? { taskId } : undefined, + ) return { success: true, diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index bd7c589958..5da8019187 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -2252,6 +2252,7 @@ export const webviewMessageHandler = async ( includeTaskHistoryInEnhance, currentClineMessages: currentCline?.clineMessages, providerSettingsManager: provider.providerSettingsManager, + taskId: currentCline?.taskId, }) if (result.success && result.enhancedText) { diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 754a2d84df..bc1f2f64d8 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -1,4 +1,10 @@ -import { OpenAiServiceTier, type ModelInfo, type ServiceTier } from "@roo-code/types" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + OpenAiServiceTier, + type ModelInfo, + type ServiceTier, +} from "@roo-code/types" export interface ApiCostResult { totalInputTokens: number @@ -6,6 +12,24 @@ export interface ApiCostResult { totalCost: number } +/** + * Resolve the per-million rate for a TTL-specific cache-write class. + * + * Precedence is fail-closed: an explicit catalog rate wins, then the vendor + * multiplier applied to the base input price, and only then the aggregate + * `cacheWritesPrice`. The last step is the pre-split behaviour, reachable only + * when a model has neither a 1h/12h rate nor an input price to derive one from. + */ +function resolveCacheWriteRate( + modelInfo: ModelInfo, + explicit: number | undefined, + multiplier: number, +): number | undefined { + if (explicit !== undefined) return explicit + if (modelInfo.inputPrice !== undefined) return modelInfo.inputPrice * multiplier + return modelInfo.cacheWritesPrice +} + function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, serviceTier?: ServiceTier): ModelInfo { const pricing = modelInfo.longContextPricing if (!pricing || totalInputTokens <= pricing.thresholdTokens) { @@ -31,6 +55,16 @@ function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, modelInfo.cacheWritesPrice !== undefined && pricing.cacheWritesPriceMultiplier !== undefined ? modelInfo.cacheWritesPrice * pricing.cacheWritesPriceMultiplier : modelInfo.cacheWritesPrice, + // The TTL-specific write rates scale with the same multiplier as the aggregate + // write rate; leaving them unscaled would under-bill long-context 1h writes. + cacheWritesPrice1h: + modelInfo.cacheWritesPrice1h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice1h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice1h, + cacheWritesPrice12h: + modelInfo.cacheWritesPrice12h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice12h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice12h, cacheReadsPrice: modelInfo.cacheReadsPrice !== undefined && pricing.cacheReadsPriceMultiplier !== undefined ? modelInfo.cacheReadsPrice * pricing.cacheReadsPriceMultiplier @@ -46,8 +80,28 @@ function calculateApiCostInternal( cacheReadInputTokens: number, totalInputTokens: number, totalOutputTokens: number, + cacheCreation1hInputTokens = 0, + cacheCreation12hInputTokens = 0, ): ApiCostResult { - const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens + // The per-TTL counts are a decomposition of `cacheCreationInputTokens`, never + // an addition to it. Deriving the 5m remainder this way keeps the aggregate + // path byte-identical when a vendor reports no split (both terms are 0). + // Each subset also caps at what the aggregate has left, so a malformed split + // (1h + 12h > total) cannot bill more writes than the vendor reported. + const writesTotal = Math.max(0, cacheCreationInputTokens) + const writes1h = Math.min(Math.max(0, cacheCreation1hInputTokens), writesTotal) + const writes12h = Math.min(Math.max(0, cacheCreation12hInputTokens), writesTotal - writes1h) + const writes5m = Math.max(0, writesTotal - writes1h - writes12h) + + const writes5mRate = modelInfo.cacheWritesPrice || 0 + const writes1hRate = resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice1h, CACHE_WRITE_MULTIPLIER_1H) || 0 + const writes12hRate = + resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice12h, CACHE_WRITE_MULTIPLIER_12H) || 0 + + const cacheWritesCost = + (writes5mRate / 1_000_000) * writes5m + + (writes1hRate / 1_000_000) * writes1h + + (writes12hRate / 1_000_000) * writes12h const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens const outputCost = ((modelInfo.outputPrice || 0) / 1_000_000) * outputTokens @@ -68,6 +122,8 @@ export function calculateApiCostAnthropic( outputTokens: number, cacheCreationInputTokens?: number, cacheReadInputTokens?: number, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreation = cacheCreationInputTokens || 0 const cacheRead = cacheReadInputTokens || 0 @@ -84,6 +140,8 @@ export function calculateApiCostAnthropic( cacheRead, totalInputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } @@ -95,6 +153,8 @@ export function calculateApiCostOpenAI( cacheCreationInputTokens?: number, cacheReadInputTokens?: number, serviceTier?: ServiceTier, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreationInputTokensNum = cacheCreationInputTokens || 0 const cacheReadInputTokensNum = cacheReadInputTokens || 0 @@ -111,6 +171,8 @@ export function calculateApiCostOpenAI( cacheReadInputTokensNum, inputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b369a0405b..6782de5e6f 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -303,4 +303,291 @@ describe("Cost Utility", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split accounting", () => { + const anthropicModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, // 1.25x input + cacheWritesPrice1h: 6.0, // 2x input + cacheReadsPrice: 0.3, + } + + const openAiModel: ModelInfo = { + maxTokens: 128_000, + contextWindow: 1_050_000, + supportsPromptCache: true, + inputPrice: 4.0, + outputPrice: 20.0, + cacheWritesPrice: 5.0, // 1.25x input (30m class) + cacheWritesPrice12h: 10.0, // 2.5x input + cacheReadsPrice: 0.4, + } + + it("bills 100k 1h tokens at the 1h rate, not the 5m rate (37.5% tripwire)", () => { + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 100_000) + + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour would have charged the 5m rate: + // (3.75 / 1_000_000) * 100_000 = 0.375, under-reporting by 37.5%. + expect(result.totalCost).toBeCloseTo(0.6, 9) + expect(result.totalCost).not.toBeCloseTo(0.375, 9) + }) + + it("bills a mixed 5m/1h split per class (60/40 split, 0.51)", () => { + // 60k of the 100k writes are 1h, 40k remain at the 5m rate. + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 60_000) + + // 5m writes: (3.75 / 1_000_000) * 40_000 = 0.15 + // 1h writes: (6.0 / 1_000_000) * 60_000 = 0.36 + // Total: 0.15 + 0.36 = 0.51 + expect(result.totalCost).toBeCloseTo(0.51, 9) + }) + + it("derives the 5m remainder as total minus 1h minus 12h", () => { + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 30_000, 20_000) + + // 5m remainder: 100k - 30k - 20k = 50k at 3.75 => 0.1875 + // 1h: 30k at 6.0 => 0.18 + // 12h: 20k at 7.5 => 0.15 + // Total: 0.1875 + 0.18 + 0.15 = 0.5175 + expect(result.totalCost).toBeCloseTo(0.5175, 9) + }) + + it("caps each TTL class at the aggregate write total", () => { + // A malformed response claiming more 1h tokens than the write total must + // not bill the excess. The 5m remainder already clamps at zero, but the 1h + // and 12h classes were billed at their full reported count. + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 150_000) + + // 1h writes cap at the 100k aggregate; the 5m remainder clamps to 0. + // 100k at 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) + expect(result.totalCost).toBeGreaterThan(0) + // The pre-fix behaviour billed all 150k at 6.0 => 0.9. + expect(result.totalCost).not.toBeCloseTo(0.9, 9) + }) + + it("caps the 12h class at the remainder left after the 1h class", () => { + // 1h and 12h are a decomposition of one aggregate, so they cannot both + // claim the whole total. 1h wins the ordering; 12h gets what is left. + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 60_000, 80_000) + + // 1h: 60k at 6.0 => 0.36 + // 12h: capped at 100k - 60k = 40k at 7.5 => 0.3 + // 5m: 100k - 60k - 40k = 0 + // Total: 0.66 (the pre-fix behaviour billed 80k 12h => 0.36 + 0.6 = 0.96) + expect(result.totalCost).toBeCloseTo(0.66, 9) + expect(result.totalCost).not.toBeCloseTo(0.96, 9) + }) + + it("is byte-identical to the pre-split cost when no split is reported", () => { + // Invariant I1: absent split fields must reproduce today's pricing exactly. + const withSplitArgs = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000) + const explicitZeros = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000, 0, 0) + + expect(withSplitArgs.totalCost).toBe(0.0189) + expect(explicitZeros.totalCost).toBe(withSplitArgs.totalCost) + }) + + it("falls back to the 5m rate when a model has no 1h rate and no input price", () => { + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + } + + // Fail-closed: without a derivable 1h rate the write is billed at the + // aggregate rate rather than dropped or invented. + const result = calculateApiCostAnthropic(bareModel, 0, 0, 100_000, 0, 100_000) + expect(result.totalCost).toBeCloseTo(0.375, 9) + }) + + it("derives the 1h rate from inputPrice when the catalog omits it", () => { + const modelWithout1h: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, + } + + const result = calculateApiCostAnthropic(modelWithout1h, 0, 0, 100_000, 0, 100_000) + // Derived 1h rate = 3.0 * 2 = 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) + }) + + it("bills OpenAI 12h writes at the 12h rate, not the 30m rate", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 100_000) + + // 12h writes: (10.0 / 1_000_000) * 100_000 = 1.0 + // The pre-split behaviour would have charged the 30m rate: + // (5.0 / 1_000_000) * 100_000 = 0.5, under-reporting by 50%. + expect(result.totalCost).toBeCloseTo(1.0, 9) + expect(result.totalCost).not.toBeCloseTo(0.5, 9) + }) + + it("bills a mixed OpenAI 30m/12h split per class", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 40_000) + + // 30m remainder: 100k - 40k = 60k at 5.0 => 0.3 + // 12h: 40k at 10.0 => 0.4 + // Total: 0.7 + expect(result.totalCost).toBeCloseTo(0.7, 9) + }) + + it("scales the 1h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + // calculateApiCostOpenAI applies long-context pricing; total input 300k > 200k. + const result = calculateApiCostOpenAI(modelWithLongContext, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input: (3.0 / 1_000_000) * (300_000 - 100_000) = 0.6 + // 1h writes scale 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8. Unscaled 1h would have given 0.6 + 0.6 = 1.2. + expect(result.totalCost).toBeCloseTo(1.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.2, 9) + }) + + it("scales the 12h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...openAiModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithLongContext, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (4.0 / 1_000_000) * (300_000 - 100_000) = 0.8 + // 12h writes scale 10.0 -> 20.0: (20.0 / 1_000_000) * 100_000 = 2.0 + // Total: 0.8 + 2.0 = 2.8. Unscaled 12h would have given 0.8 + 1.0 = 1.8. + expect(result.totalCost).toBeCloseTo(2.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.8, 9) + }) + + it("leaves the 1h rate undefined when neither a catalog rate nor an input price exists", () => { + // Covers the `cacheWritesPrice1h !== undefined` false branch of long-context scaling. + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(bareModel, 300_000, 0, 100_000, 0, undefined, 100_000) + + // No input price => no derivable 1h rate => falls back to the scaled aggregate + // write rate (3.75 * 2 = 7.5): (7.5 / 1_000_000) * 100_000 = 0.75 + expect(result.totalCost).toBeCloseTo(0.75, 9) + }) + + it("does not scale TTL rates when long-context pricing has no write multiplier", () => { + // Covers the `cacheWritesPriceMultiplier !== undefined` false branch for both + // TTL classes: without a multiplier the rates must pass through unchanged. + const modelWithoutMultiplier: ModelInfo = { + ...anthropicModel, + cacheWritesPrice12h: 7.5, + longContextPricing: { + thresholdTokens: 200_000, + inputPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithoutMultiplier, + 300_000, + 0, + 100_000, + 0, + undefined, + 50_000, + 50_000, + ) + + // Base input scales: (3.0 * 2 / 1_000_000) * (300_000 - 100_000) = 1.2 + // 1h unscaled: (6.0 / 1_000_000) * 50_000 = 0.3 + // 12h unscaled: (7.5 / 1_000_000) * 50_000 = 0.375 + // Total: 1.2 + 0.3 + 0.375 = 1.875 + expect(result.totalCost).toBeCloseTo(1.875, 9) + }) + + it("keeps an undefined 12h rate undefined when a write multiplier is present", () => { + // Covers the `cacheWritesPrice12h !== undefined` false branch: a model with a + // long-context write multiplier but no 12h rate must not gain a fabricated one. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(modelWithout12h, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input (no inputPriceMultiplier, so unscaled): (3.0 / 1_000_000) * 200_000 = 0.6 + // 1h scales 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8 + expect(result.totalCost).toBeCloseTo(1.8, 9) + }) + + it("derives the 12h rate from inputPrice when a write multiplier is present but no 12h rate", () => { + // Distinguishes the guarded branch from an unguarded multiply: an unguarded + // `undefined * multiplier` yields NaN, which the cost layer reads as 0 and + // would silently drop the 12h write cost entirely. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithout12h, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (3.0 / 1_000_000) * 200_000 = 0.6 + // No 12h rate in the catalog, so it derives from inputPrice * 2.5 = 7.5 + // (the long-context multiplier does not apply to a rate that is not there): + // (7.5 / 1_000_000) * 100_000 = 0.75 + // Total: 0.6 + 0.75 = 1.35 + expect(result.totalCost).toBeCloseTo(1.35, 9) + expect(result.totalCost).not.toBeCloseTo(0.6, 9) + }) + }) }) diff --git a/webview-ui/playwright/gallery/stories.tsx b/webview-ui/playwright/gallery/stories.tsx index 85d6919db4..563b5f2270 100644 --- a/webview-ui/playwright/gallery/stories.tsx +++ b/webview-ui/playwright/gallery/stories.tsx @@ -415,6 +415,36 @@ export const stories: Record = { ) }, + "anthropic-settings": async () => { + const [{ AppProviders }, { Anthropic }, { providerIdentifiers }] = await Promise.all([ + import("../AppProviders"), + import("@/components/settings/providers/Anthropic"), + import("@roo-code/types"), + ]) + function AnthropicSettings() { + const [config, setConfig] = useState({ + apiProvider: providerIdentifiers.anthropic, + apiKey: "sk-ant-example-key", + }) + return ( +
+
+ + setConfig((previous) => ({ ...previous, [field]: value })) + } + /> +
+
+ ) + } + return ( + + + + ) + }, "ui-settings": async () => { const { UISettingsStory } = await import("@/components/settings/__tests__/UISettings.visual.fixture") return diff --git a/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx b/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx index 0835f255f9..14a3100057 100644 --- a/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx @@ -33,6 +33,12 @@ vi.mock("../ApiOptions", () => ({ data-testid="set-other-field" onClick={() => setApiConfigurationField("openAiBaseUrl", "https://example.test")} /> + {String(apiConfiguration?.promptCacheTtl)} +