diff --git a/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts new file mode 100644 index 0000000000..e62f658016 --- /dev/null +++ b/packages/types/src/__tests__/cache-write-ttl-pricing.test.ts @@ -0,0 +1,84 @@ +// npx vitest run src/__tests__/cache-write-ttl-pricing.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels } from "../providers/anthropic.js" +import { bedrockModels } from "../providers/bedrock.js" +import { openAiNativeModels } from "../providers/openai.js" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + deriveCacheWritesPrice12h, + deriveCacheWritesPrice1h, + type ModelInfo, +} from "../model.js" + +/** + * Catalog invariant for the TTL accounting program (ADR-0006 phase 1). + * + * A model that advertises prompt caching on a provider which can report a + * long-lived cache write must carry the matching per-TTL write rate. Without it + * the cost layer can only fall back to the aggregate write rate, which + * under-reports every long-lived write (37.5% for Anthropic 1h, 50% for OpenAI 12h). + */ +describe("cache-write TTL pricing catalog invariant", () => { + const anthropicFamilyEntries = Object.entries(anthropicModels as Record) + + const bedrockAnthropicEntries = Object.entries(bedrockModels as Record).filter(([id]) => + id.startsWith("anthropic."), + ) + + const openAiLongLivedEntries = Object.entries(openAiNativeModels as Record).filter( + ([id]) => id.startsWith("gpt-5.6") || id.startsWith("gpt-6"), + ) + + it("every Anthropic catalog entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = anthropicFamilyEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every Bedrock Anthropic-family entry with prompt caching defines cacheWritesPrice1h", () => { + const missing = bedrockAnthropicEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("every OpenAI 5.6/6-class entry defines cacheWritesPrice12h", () => { + expect(openAiLongLivedEntries.length).toBeGreaterThan(0) + + const missing = openAiLongLivedEntries + .filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice12h === undefined) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("Anthropic 1h rates match the documented 2x input-price relationship", () => { + for (const [id, info] of anthropicFamilyEntries) { + if (info.cacheWritesPrice1h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice1h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_1H, 9) + } + }) + + it("OpenAI 12h rates match the documented 2.5x input-price relationship", () => { + for (const [id, info] of openAiLongLivedEntries) { + if (info.cacheWritesPrice12h === undefined || info.inputPrice === undefined) continue + expect(info.cacheWritesPrice12h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_12H, 9) + } + }) + + it("deriveCacheWritesPrice1h scales inputPrice by 2x and stays undefined without one", () => { + expect(deriveCacheWritesPrice1h(3.0)).toBe(6.0) + expect(deriveCacheWritesPrice1h(undefined)).toBeUndefined() + }) + + it("deriveCacheWritesPrice12h scales inputPrice by 2.5x and stays undefined without one", () => { + expect(deriveCacheWritesPrice12h(4.0)).toBe(10.0) + expect(deriveCacheWritesPrice12h(undefined)).toBeUndefined() + }) +}) diff --git a/packages/types/src/__tests__/openai-models.test.ts b/packages/types/src/__tests__/openai-models.test.ts index 78a3259f09..e1e0f1e323 100644 --- a/packages/types/src/__tests__/openai-models.test.ts +++ b/packages/types/src/__tests__/openai-models.test.ts @@ -122,6 +122,7 @@ describe("OpenAI native models", () => { inputPrice: 2, outputPrice: 10, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -166,6 +167,7 @@ describe("OpenAI native models", () => { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, diff --git a/packages/types/src/__tests__/prompt-cache-ttl.test.ts b/packages/types/src/__tests__/prompt-cache-ttl.test.ts new file mode 100644 index 0000000000..75424df071 --- /dev/null +++ b/packages/types/src/__tests__/prompt-cache-ttl.test.ts @@ -0,0 +1,102 @@ +// npx vitest run src/__tests__/prompt-cache-ttl.test.ts + +import { describe, it, expect } from "vitest" + +import { anthropicModels, anthropicProviderDefinition } from "../providers/anthropic.js" +import { vertexModels } from "../providers/vertex.js" +import { openRouterDefaultModelInfo } from "../providers/openrouter.js" +import { + vercelAiGatewayDefaultModelId, + VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS, +} from "../providers/vercel-ai-gateway.js" +import { + DEFAULT_PROMPT_CACHE_TTL, + modelInfoSchema, + promptCacheTtlSchema, + promptCacheTtlValues, + type ModelInfo, +} from "../model.js" +import { baseProviderSettingsShape } from "../providers/common.js" +import { providerIdentifiers } from "../provider-identifiers.js" +import { globalSettingsSchema } from "../global-settings.js" + +/** + * Catalog invariant for the request-side TTL knob (ADR-0006 phase 2, PR-C5b). + * + * Emission is gated by `ModelInfo.promptCacheTtlSupport`, so a model that can + * report a 1h cache write but carries no capability array would silently never + * emit the TTL the user asked for. These tests pin both directions: capable + * catalogs advertise it, and catalogs that cannot know stay absent (fail closed). + */ +describe("prompt cache TTL schema", () => { + it("accepts exactly the 5m and 1h classes", () => { + expect(promptCacheTtlValues).toEqual(["5m", "1h"]) + expect(promptCacheTtlSchema.parse("5m")).toBe("5m") + expect(promptCacheTtlSchema.parse("1h")).toBe("1h") + expect(promptCacheTtlSchema.safeParse("30m").success).toBe(false) + expect(promptCacheTtlSchema.safeParse("12h").success).toBe(false) + }) + + it("defaults to undefined so an unset knob emits no ttl key", () => { + expect(DEFAULT_PROMPT_CACHE_TTL).toBeUndefined() + }) + + it("is optional on ModelInfo", () => { + const parsed = modelInfoSchema.parse({ contextWindow: 200_000, supportsPromptCache: true }) + expect(parsed.promptCacheTtlSupport).toBeUndefined() + + const withCapability = modelInfoSchema.parse({ + contextWindow: 200_000, + supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], + }) + expect(withCapability.promptCacheTtlSupport).toEqual(["5m", "1h"]) + }) +}) + +describe("prompt cache TTL knob location", () => { + it("lives in provider settings so handlers can see it", () => { + // ApiHandlerOptions = Omit, so only a + // provider-settings key reaches a handler (ADR-0006 decision item 2). + expect(baseProviderSettingsShape.promptCacheTtl).toBeDefined() + + const parsed = anthropicProviderDefinition.schema.safeParse({ + apiProvider: providerIdentifiers.anthropic, + promptCacheTtl: "1h", + }) + expect(parsed.success).toBe(true) + expect(parsed.success && parsed.data.promptCacheTtl).toBe("1h") + }) + + it("is not a global setting", () => { + const shape = globalSettingsSchema.shape as Record + expect(shape.promptCacheTtl).toBeUndefined() + }) +}) + +describe("prompt cache TTL capability catalog", () => { + it("every Anthropic entry with prompt caching advertises both TTL classes", () => { + const missing = Object.entries(anthropicModels as Record) + .filter(([, info]) => info.supportsPromptCache && !info.promptCacheTtlSupport?.includes("1h")) + .map(([id]) => id) + + expect(missing).toEqual([]) + }) + + it("leaves the Vertex catalog without capability so it fails closed", () => { + // PR-C5b scopes the catalog capability to the Anthropic entries; the Vertex + // catalog is populated separately, so its models stay 5m-only for now. + const withCapability = Object.entries(vertexModels as Record) + .filter(([, info]) => info.promptCacheTtlSupport !== undefined) + .map(([id]) => id) + + expect(withCapability).toEqual([]) + }) + + it("leaves dynamic-catalog defaults without capability so they fail closed", () => { + // OpenRouter and the gateway catalogs are fetched at runtime and carry no TTL + // capability data, which ADR-0006 maps to "5m only" (invariant I2). + expect(openRouterDefaultModelInfo.promptCacheTtlSupport).toBeUndefined() + expect(VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(vercelAiGatewayDefaultModelId)).toBe(true) + }) +}) diff --git a/packages/types/src/model.ts b/packages/types/src/model.ts index 9ddc4ea182..b3c2b8151d 100644 --- a/packages/types/src/model.ts +++ b/packages/types/src/model.ts @@ -95,6 +95,20 @@ export type ModelParameter = z.infer export const isModelParameter = (value: string): value is ModelParameter => modelParameters.includes(value as ModelParameter) +/** + * Prompt cache TTL classes a request-side cache marker can carry. `"5m"` is the + * vendor default and is never emitted explicitly, so only `"1h"` changes the wire + * shape and an unset knob stays byte-identical to a pre-knob build (ADR-0006 I3). + */ +export const promptCacheTtlValues = ["5m", "1h"] as const + +export const promptCacheTtlSchema = z.enum(promptCacheTtlValues) + +export type PromptCacheTtl = z.infer + +/** Deliberately `undefined`, not `"5m"`: emitting `ttl: "5m"` would change every request's bytes. */ +export const DEFAULT_PROMPT_CACHE_TTL: PromptCacheTtl | undefined = undefined + /** * ModelInfo */ @@ -112,6 +126,8 @@ export const modelInfoSchema = z.object({ // When set to "24h", extended prompt caching will be requested; when omitted // or set to "in_memory", the default in‑memory cache is used. promptCacheRetention: z.enum(["in_memory", "24h"]).optional(), + // Cache TTLs this model accepts. Absent means "5m only" (ADR-0006 I2, fail closed). + promptCacheTtlSupport: z.array(promptCacheTtlSchema).optional(), // Capability flag to indicate whether the model supports an output verbosity parameter supportsVerbosity: z.boolean().optional(), // Capability flag to indicate whether the model exposes a user-configurable max output @@ -136,6 +152,13 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL cache-write rates. Vendors bill a longer cache lifetime at a higher + // write rate than the default 5m/30m class, so a single `cacheWritesPrice` + // under-reports whenever the response reports a long-lived write. Omitted + // means "no long-lived writes are expected on this model"; the cost layer + // derives a fail-closed rate from `inputPrice` if one still arrives. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), longContextPricing: z .object({ @@ -180,6 +203,11 @@ export const modelInfoSchema = z.object({ inputPrice: z.number().optional(), outputPrice: z.number().optional(), cacheWritesPrice: z.number().optional(), + // Per-TTL write rates for this tier. A tier that overrides inputPrice + // must also override these, otherwise a long-context or flex/priority + // write bills at the base tier's TTL rate. + cacheWritesPrice1h: z.number().optional(), + cacheWritesPrice12h: z.number().optional(), cacheReadsPrice: z.number().optional(), }), ) @@ -188,6 +216,34 @@ export const modelInfoSchema = z.object({ export type ModelInfo = z.infer +/** + * Multipliers vendors apply to the base input price for a cache write, by TTL class. + * Anthropic: 5m = 1.25x, 1h = 2x. OpenAI: 30m = 1.25x, 12h = 2.5x. + */ +export const CACHE_WRITE_MULTIPLIER_5M = 1.25 +export const CACHE_WRITE_MULTIPLIER_1H = 2.0 +export const CACHE_WRITE_MULTIPLIER_12H = 2.5 + +/** + * Derive the 1h cache-write rate from the base input price. + * + * Stated once here so every Anthropic-family catalog entry and every runtime + * fallback agrees on the 2x relationship. Returns `undefined` when the model + * has no input price to scale, which keeps the cost layer fail-closed: it will + * fall back to `cacheWritesPrice` rather than invent a rate. + */ +export function deriveCacheWritesPrice1h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_1H +} + +/** + * Derive the 12h cache-write rate from the base input price (OpenAI 12h class). + * Same fail-closed contract as {@link deriveCacheWritesPrice1h}. + */ +export function deriveCacheWritesPrice12h(inputPrice: number | undefined): number | undefined { + return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_12H +} + export type ModelRecord = Record export type RouterModels = Record diff --git a/packages/types/src/providers/anthropic.ts b/packages/types/src/providers/anthropic.ts index ba3c2f820e..0861156ffc 100644 --- a/packages/types/src/providers/anthropic.ts +++ b/packages/types/src/providers/anthropic.ts @@ -12,6 +12,9 @@ import { // https://docs.anthropic.com/en/docs/about-claude/models // https://platform.claude.com/docs/en/about-claude/pricing +// Every entry here is an Anthropic-hosted Claude model, and 1h cache writes are GA +// on all active Claude models, so each one advertises both TTL classes. The 5m class +// is the vendor default and is never emitted explicitly (ADR-0006 I3). export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-sonnet-4-5" export const ANTHROPIC_API_PROTOCOL = "anthropic" @@ -22,9 +25,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -34,6 +39,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -43,9 +49,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) // Sonnet 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Fable 5 on the direct Anthropic provider path. Manual @@ -62,9 +70,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -74,6 +84,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -83,9 +94,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag 'context-1m-2025-08-07') @@ -95,6 +108,7 @@ export const anthropicModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -104,9 +118,11 @@ export const anthropicModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, // Tiered pricing for extended context (requires beta flag) @@ -116,6 +132,7 @@ export const anthropicModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -125,9 +142,11 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Keep the hybrid-reasoning capability so Anthropic token-cap handling and // stored max-token overrides behave the same as before. @@ -142,9 +161,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required, same as 4.7) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens (regular tier) outputPrice: 25.0, // $25 per million output tokens (regular tier) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // 4.8 inherits the adaptive-thinking model introduced in 4.7 — no breaking // API changes. supportsReasoningBudget is kept true so the existing token-cap @@ -160,9 +181,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens // Opus 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ and Sonnet 5 on the direct Anthropic provider path. Manual @@ -178,9 +201,11 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // Opus 5.5 always uses adaptive thinking and rejects manual budget_tokens. supportsReasoningBinary: true, @@ -193,6 +218,7 @@ export const anthropicModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], // Standard pricing (prompts of 100,000 tokens or fewer) from Anthropic's // pricing page. Prompts over 100,000 tokens bill at tiered rates // ($0.50 in / $2.50 out / $0.625 5m cache write / $0.05 cache read per @@ -201,6 +227,7 @@ export const anthropicModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // Haiku 5.5 uses the same adaptive-thinking / binary-toggle convention as // Sonnet 5 and Opus 5 on the direct Anthropic provider path. Manual @@ -217,9 +244,11 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, // $0.25 per million tokens (2.5% of base input price) // Fable 5.1 only accepts adaptive thinking (or an omitted thinking field). // Manual budget_tokens and non-default sampling parameters return a 400. @@ -234,9 +263,11 @@ export const anthropicModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 10.0, // $10 per million input tokens outputPrice: 50.0, // $50 per million output tokens cacheWritesPrice: 12.5, // $12.50 per million tokens + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens // Fable 5 uses the same adaptive-thinking / binary-toggle convention as // Opus 4.7+ on the direct Anthropic provider path. @@ -251,9 +282,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens supportsReasoningBudget: true, }, @@ -262,9 +295,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -273,9 +308,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, // $15 per million input tokens outputPrice: 75.0, // $75 per million output tokens cacheWritesPrice: 18.75, // $18.75 per million tokens + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, // $1.50 per million tokens supportsReasoningBudget: true, }, @@ -284,9 +321,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens supportsReasoningBudget: true, requiredReasoningBudget: true, @@ -296,9 +335,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-sonnet-20241022": { @@ -306,9 +347,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 3.0, // $3 per million input tokens outputPrice: 15.0, // $15 per million output tokens cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens }, "claude-3-5-haiku-20241022": { @@ -316,9 +359,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: false, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, }, "claude-3-opus-20240229": { @@ -326,9 +371,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, }, "claude-3-haiku-20240307": { @@ -336,9 +383,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 0.25, outputPrice: 1.25, cacheWritesPrice: 0.3, + cacheWritesPrice1h: 0.5, // $0.50 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.03, }, "claude-haiku-4-5-20251001": { @@ -346,9 +395,11 @@ export const anthropicModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, supportsReasoningBudget: true, description: diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index ca599cb29a..6447f0d73f 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -21,16 +21,26 @@ export const bedrockDefaultPromptRouterModelId: BedrockModelId = "anthropic.clau // https://aws.amazon.com/bedrock/pricing, including older models that are part // of the default prompt routers AWS enabled for GA of the promot router // feature. +// +// `promptCacheTtlSupport` marks the models whose Converse cache points accept an +// explicit `ttl` (ADR-0006 PR-C7). The 1h boundary is not a version rule: Opus 4.1 +// and Opus 4 are 5m-only while Opus 4.6/4.7/4.8 are 1h-capable, so the flag is set +// per exact model id from the AWS "Supported TTL" table and never inferred from a +// pattern. Rows without it are 5m-only, which makes Bedrock reject an explicit ttl +// (ValidationException), so absence must stay the fail-closed default. +// https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html export const bedrockModels = { "anthropic.claude-sonnet-4-5-20250929-v1:0": { maxTokens: 64_000, contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -41,10 +51,12 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 3.0, // $3 per million input tokens (≤200K context) outputPrice: 15.0, // $15 per million output tokens (≤200K context) cacheWritesPrice: 3.75, // $3.75 per million tokens + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, // $0.30 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -56,6 +68,7 @@ export const bedrockModels = { inputPrice: 6.0, // $6 per million input tokens (>200K context) outputPrice: 22.5, // $22.50 per million output tokens (>200K context) cacheWritesPrice: 7.5, // $7.50 per million tokens (>200K context) + cacheWritesPrice1h: 12, // $12.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 0.6, // $0.60 per million tokens (>200K context) }, ], @@ -66,12 +79,14 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026) outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026) cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026) + cacheWritesPrice1h: 4, // $4.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens (introductory pricing through Aug 31, 2026) minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -152,6 +167,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -166,6 +182,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -176,10 +193,12 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, // $5 per million input tokens (≤200K context) outputPrice: 25.0, // $25 per million output tokens (≤200K context) cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -191,6 +210,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -200,10 +220,12 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -215,6 +237,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -224,10 +247,12 @@ export const bedrockModels = { contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07' supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, // $5 per million input tokens (≤200K context) — verify against Bedrock console outputPrice: 25.0, // $25 per million output tokens (≤200K context) — verify against Bedrock console cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -241,6 +266,7 @@ export const bedrockModels = { inputPrice: 10.0, // $10 per million input tokens (>200K context) outputPrice: 37.5, // $37.50 per million output tokens (>200K context) cacheWritesPrice: 12.5, // $12.50 per million tokens (>200K context) + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x tier input price, 1h cache writes) cacheReadsPrice: 1.0, // $1.00 per million tokens (>200K context) }, ], @@ -251,12 +277,14 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, inputPrice: 5.0, // $5 per million input tokens outputPrice: 25.0, // $25 per million output tokens cacheWritesPrice: 6.25, // $6.25 per million tokens + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, // $0.50 per million tokens minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -271,12 +299,14 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, inputPrice: 4.0, // $4 per million input tokens outputPrice: 20.0, // $20 per million output tokens cacheWritesPrice: 5.0, // $5 per million tokens + cacheWritesPrice1h: 8, // $8.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.2, // $0.20 per million tokens // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Opus 5.5 (matching the Fable 5.1 entry below). @@ -295,6 +325,7 @@ export const bedrockModels = { contextWindow: 1_000_000, // 1M context window native (no beta header required) supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, @@ -306,6 +337,7 @@ export const bedrockModels = { inputPrice: 0.1, // $0.10 per million input tokens outputPrice: 0.5, // $0.50 per million output tokens cacheWritesPrice: 0.125, // $0.125 per million tokens (5m cache writes) + cacheWritesPrice1h: 0.2, // $0.20 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.01, // $0.01 per million tokens (cache hits / refreshes) // The same Bedrock model card lists 512 as the minimum tokens per cache // checkpoint for Haiku 5.5 (matching the Opus 5.5 entry above). @@ -320,12 +352,14 @@ export const bedrockModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.25, minTokensPerCachePoint: 512, maxCachePoints: 4, @@ -338,12 +372,14 @@ export const bedrockModels = { contextWindow: 1_000_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, supportsReasoningBinary: true, supportsTemperature: false, inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice1h: 20, // $20.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.0, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -356,10 +392,12 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 5.0, outputPrice: 25.0, cacheWritesPrice: 6.25, + cacheWritesPrice1h: 10, // $10.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -374,6 +412,7 @@ export const bedrockModels = { inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, + cacheWritesPrice1h: 30, // $30.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 1.5, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -388,6 +427,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -401,6 +441,7 @@ export const bedrockModels = { inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, + cacheWritesPrice1h: 6, // $6.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.3, minTokensPerCachePoint: 1024, maxCachePoints: 4, @@ -414,6 +455,7 @@ export const bedrockModels = { inputPrice: 0.8, outputPrice: 4.0, cacheWritesPrice: 1.0, + cacheWritesPrice1h: 1.6, // $1.60 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.08, minTokensPerCachePoint: 2048, maxCachePoints: 4, @@ -424,10 +466,12 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + promptCacheTtlSupport: ["5m", "1h"], supportsReasoningBudget: true, inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, // 5m cache writes + cacheWritesPrice1h: 2, // $2.00 per million tokens (2x input price, 1h cache writes) cacheReadsPrice: 0.1, // cache hits / refreshes minTokensPerCachePoint: 2048, maxCachePoints: 4, diff --git a/packages/types/src/providers/common.ts b/packages/types/src/providers/common.ts index 2f580ab380..59c463adcc 100644 --- a/packages/types/src/providers/common.ts +++ b/packages/types/src/providers/common.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { reasoningEffortSettingSchema, verbosityLevelsSchema } from "../model.js" +import { reasoningEffortSettingSchema, verbosityLevelsSchema, promptCacheTtlSchema } from "../model.js" import { type ProviderIdentifier } from "../provider-identifiers.js" export const API_PROVIDER_FIELD = "apiProvider" @@ -39,6 +39,11 @@ export const baseProviderSettingsShape = { modelMaxTokens: z.number().optional(), modelMaxThinkingTokens: z.number().optional(), verbosity: verbosityLevelsSchema.optional(), + // Opt-in prompt cache TTL for this provider profile. Lives here, not in global + // settings, because `ApiHandlerOptions = Omit` + // (src/shared/api.ts) makes a global key invisible to handlers. Unset emits no + // ttl key at all, not "5m" (ADR-0006 I3). + promptCacheTtl: promptCacheTtlSchema.optional(), } export const apiModelIdProviderModelShape = { diff --git a/packages/types/src/providers/openai.ts b/packages/types/src/providers/openai.ts index cb50a7d40a..7f648841dd 100644 --- a/packages/types/src/providers/openai.ts +++ b/packages/types/src/providers/openai.ts @@ -38,6 +38,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.1, longContextPricing: { thresholdTokens: 272_000, @@ -82,6 +83,7 @@ export const openAiNativeModels = { inputPrice: 10.0, outputPrice: 50.0, cacheWritesPrice: 12.5, + cacheWritesPrice12h: 25, // $25.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 1.0, longContextPricing: { thresholdTokens: 272_000, @@ -124,6 +126,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 10.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -167,6 +170,7 @@ export const openAiNativeModels = { inputPrice: 0.1, outputPrice: 0.5, cacheWritesPrice: 0.125, + cacheWritesPrice12h: 0.25, // $0.25 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.01, longContextPricing: { thresholdTokens: 272_000, @@ -210,6 +214,7 @@ export const openAiNativeModels = { inputPrice: 4.0, outputPrice: 20.0, cacheWritesPrice: 5.0, + cacheWritesPrice12h: 10, // $10.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.4, longContextPricing: { thresholdTokens: 272_000, @@ -253,6 +258,7 @@ export const openAiNativeModels = { inputPrice: 2.0, outputPrice: 12.0, cacheWritesPrice: 2.5, + cacheWritesPrice12h: 5, // $5.00 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.2, longContextPricing: { thresholdTokens: 272_000, @@ -296,6 +302,7 @@ export const openAiNativeModels = { inputPrice: 0.2, outputPrice: 1.2, cacheWritesPrice: 0.25, + cacheWritesPrice12h: 0.5, // $0.50 per million tokens (2.5x input price, 12h cache writes) cacheReadsPrice: 0.02, longContextPricing: { thresholdTokens: 272_000, diff --git a/src/api/index.ts b/src/api/index.ts index d61eea033f..e737ebd4bd 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -61,6 +61,14 @@ import { NativeOllamaHandler } from "./providers/native-ollama" export interface CompletePromptOptions extends Pick { /** Optional timeout override (ms) — falls back to provider default if omitted */ timeoutMs?: number + /** + * Task the completion belongs to, when it belongs to one. + * + * Providers that key cache affinity off a body field use this to route repeated + * completions for one task to the same bucket. Omit it when the completion is not + * scoped to a task; providers then fall back to a per-handler key. + */ + taskId?: string } export interface SingleCompletionHandler { diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 34788df89b..f4860e9914 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -10,6 +10,7 @@ import { AnthropicVertexHandler } from "../anthropic-vertex" import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" +import type { ApiStreamChunk } from "../../../api/transform/stream" vitest.mock("../utils/timeout-config", () => ({ getApiRequestTimeout: vitest.fn().mockReturnValue(300_000), @@ -259,6 +260,108 @@ describe("VertexHandler", () => { ) }) + describe("prompt cache TTL", () => { + const minimalStream = asyncStreamFrom([ + { + type: "message_start", + message: { usage: { input_tokens: 10, output_tokens: 0 } }, + }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // The handler's client is private; reach it once here rather than casting at + // each call site. `messages.create` is replaced with a plain vitest mock. + const stubCreate = (mockCreate: ReturnType) => { + const client = handler["client"] as unknown as { messages: { create: unknown } } + client.messages.create = mockCreate + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on the system prompt and every user marker when the model is capable", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + // The Vertex catalog carries no TTL capability yet, so inject one to + // exercise the emission path this PR adds. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + // System + the user marker the vertex transform emits. + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls.length).toBeGreaterThanOrEqual(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("fails closed when the Vertex catalog advertises no TTL capability", async () => { + handler = new AnthropicVertexHandler({ + apiModelId: "claude-3-5-sonnet-v2@20241022", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + promptCacheTtl: "1h", + }) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(minimalStream) + stubCreate(mockCreate) + + await collectStream(handler.createMessage(systemPrompt, mockMessages)) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should handle multiple content blocks with line breaks for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", @@ -453,6 +556,41 @@ describe("VertexHandler", () => { ) }) + it("should surface the 1h cache-write split from cache_creation", async () => { + const mockStream = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ] + + const mockCreate = vitest.fn().mockResolvedValue(asyncStreamFrom(mockStream)) + // Typed structural cast rather than `as any`: the client is a private field and + // this file already has 31 suppressed `any` uses, a budget this test must not grow. + ;(handler["client"].messages as unknown as { create: typeof mockCreate }).create = mockCreate + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is Extract => + chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + it("should handle cache-related usage metrics for Claude", async () => { handler = new AnthropicVertexHandler({ apiModelId: "claude-3-5-sonnet-v2@20241022", diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 3eedcfc906..4f15564323 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -1,14 +1,13 @@ // npx vitest run src/api/providers/__tests__/anthropic.spec.ts -import { type ApiStreamUsageChunk } from "../../../api/transform/stream" import { AnthropicHandler } from "../anthropic" import { MAX_TIMEOUT_MS } from "../utils/abort-signal" import { ApiHandlerOptions } from "../../../shared/api" -import { calculateApiCostAnthropic } from "../../../shared/cost" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { makeCreateMessageMetadata } from "../../../test-utils/api" -import type { ApiStreamChunk } from "../../../api/transform/stream" +import type { ApiStreamChunk, ApiStreamUsageChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" +import { calculateApiCostAnthropic } from "../../../shared/cost" // Mock TelemetryService vitest.mock("@roo-code/telemetry", () => ({ @@ -241,6 +240,78 @@ describe("AnthropicHandler", () => { expect(usageChunks[0]?.totalCost).not.toBe(calculateApiCostAnthropic(info, 100, 1).totalCost) }) + describe("prompt cache TTL", () => { + const threeTurnMessages = () => [ + { role: "user" as const, content: [{ type: "text" as const, text: "First message" }] }, + { role: "assistant" as const, content: [{ type: "text" as const, text: "Response" }] }, + { role: "user" as const, content: [{ type: "text" as const, text: "Second message" }] }, + ] + + /** Every cache_control object in the serialized request body. */ + const requestCacheControls = () => { + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const serialized = JSON.parse(JSON.stringify(body)) + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(serialized) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + await collectStream(handler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on all three marker sites when the knob is 1h", async () => { + const ttlHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "1h", + }) + + await collectStream(ttlHandler.createMessage(systemPrompt, threeTurnMessages())) + + // System + last two user messages: a single missed site would show up + // here as a mixed-TTL request (ADR-0006 I4). + const controls = requestCacheControls() + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(new Set(controls.map((control) => control.ttl))).toEqual(new Set(["1h"])) + }) + + it("emits no ttl key for an explicit 5m knob", async () => { + const fiveMinuteHandler = new AnthropicHandler({ + ...mockOptions, + promptCacheTtl: "5m", + }) + + await collectStream(fiveMinuteHandler.createMessage(systemPrompt, threeTurnMessages())) + + const body = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(JSON.stringify(body)).not.toContain("ttl") + }) + }) + it("should include 1M context beta header for Claude Sonnet 4.6 when enabled", async () => { const sonnet46Handler = new AnthropicHandler({ apiKey: "test-api-key", @@ -1096,6 +1167,103 @@ describe("AnthropicHandler", () => { "fine-grained-tool-streaming-2025-05-14,prompt-caching-2024-07-31", ) }) + + it("should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(100_000) + expect(usageChunk?.cacheWrite1hTokens).toBe(60_000) + }) + + it("should leave the 1h split undefined when cache_creation is absent", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.cacheWriteTokens !== undefined, + ) + expect(usageChunk?.cacheWriteTokens).toBe(20) + expect(usageChunk?.cacheWrite1hTokens).toBeUndefined() + }) + + it("should price a 1h write at the 1h rate in the final cost chunk", async () => { + // claude-sonnet-4-5: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + const sonnetHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + }) + + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 0, + ephemeral_1h_input_tokens: 100_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = sonnetHandler.createMessage(systemPrompt, [{ role: "user", content: "Hello" }]) + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const costChunk = chunks.find( + (chunk): chunk is ApiStreamUsageChunk => chunk.type === "usage" && chunk.totalCost !== undefined, + ) + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour charged the 5m rate: 0.375 (37.5% low). + expect(costChunk?.totalCost).toBeCloseTo(0.6, 9) + expect(costChunk?.totalCost).not.toBeCloseTo(0.375, 9) + }) }) describe("completePrompt", () => { @@ -1492,6 +1660,38 @@ describe("AnthropicHandler", () => { expect(model.info.outputPrice).toBe(22.5) }) + it("carries the tier's own cacheWritesPrice1h when the 1M context tier applies", () => { + // glm P2: the tier merge overrode inputPrice but not the TTL rate, so the + // entry-level 1h rate (derived from the 200K input price) survived and a + // 1M-context 1h write billed 6.0 instead of the tier's 12.0 (50% under). + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + // Tier input price 6.0 => 1h rate must be 2x that, not the 200K entry's 6.0. + expect(model.info.cacheWritesPrice1h).toBe(12.0) + expect(model.info.cacheWritesPrice1h).not.toBe(6.0) + }) + + it("bills a 1M-context 1h cache write at the tier rate, not the 200K rate", () => { + const handler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-4-5", + anthropicBeta1MContext: true, + }) + const model = handler.getModel() + + const { totalCost } = calculateApiCostAnthropic(model.info, 0, 0, 100_000, 0, 100_000) + + // Tier 1h rate 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // The pre-fix behaviour billed the 200K 1h rate: 0.6 (50% under). + expect(totalCost).toBeCloseTo(1.2, 9) + expect(totalCost).not.toBeCloseTo(0.6, 9) + }) + it("should honor a custom/unrecognized model ID instead of silently falling back to anthropicDefaultModelId", () => { const handler = new AnthropicHandler({ apiKey: "test-api-key", diff --git a/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts new file mode 100644 index 0000000000..45a7a7db46 --- /dev/null +++ b/src/api/providers/__tests__/bedrock-cache-ttl.spec.ts @@ -0,0 +1,577 @@ +// npx vitest run src/api/providers/__tests__/bedrock-cache-ttl.spec.ts + +const mockCaptureException = vi.fn() + +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + instance: { + captureException: (...args: unknown[]) => mockCaptureException(...args), + }, + }, +})) + +vi.mock("@aws-sdk/credential-providers", () => { + const mockFromIni = vi.fn().mockReturnValue({ + accessKeyId: "profile-access-key", + secretAccessKey: "profile-secret-key", + }) + return { fromIni: mockFromIni } +}) + +vi.mock("../../../utils/networkProxy", () => ({ + getSystemProxyUrl: vi.fn().mockReturnValue(undefined), +})) + +vi.mock("@smithy/node-http-handler", () => ({ + NodeHttpHandler: vi.fn(), +})) + +vi.mock("http-proxy-agent", () => ({ + HttpProxyAgent: vi.fn(), +})) + +vi.mock("https-proxy-agent", () => ({ + HttpsProxyAgent: vi.fn(), +})) + +vi.mock("@aws-sdk/client-bedrock-runtime", async () => { + const actual = await vi.importActual( + "@aws-sdk/client-bedrock-runtime", + ) + const mockSend = vi.fn().mockResolvedValue({ stream: [] }) + const mockConverseStreamCommand = vi.fn() + + return { + ...actual, + BedrockRuntimeClient: vi.fn().mockImplementation(function () { + return { send: mockSend } + }), + ConverseStreamCommand: mockConverseStreamCommand, + ConverseCommand: vi.fn(), + } +}) + +import { AwsBedrockHandler } from "../bedrock" +import { ConverseStreamCommand } from "@aws-sdk/client-bedrock-runtime" +import { bedrockModels, BEDROCK_CUSTOM_ARN_OTHER_BASE_MODEL, type ProviderSettings } from "@roo-code/types" + +import type { Anthropic } from "@anthropic-ai/sdk" +import { clearAllMocks } from "../../../test-utils/reset" +import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" + +const mockConverseStreamCommand = vi.mocked(ConverseStreamCommand) + +/** + * Bedrock Converse cache-point TTL emission (ADR-0006 PR-C7). + * + * Every assertion here runs on the serialized `ConverseStreamCommand` input, i.e. + * the bytes that would reach AWS. Bedrock rejects an explicit `ttl` for a model + * that does not support it (`ValidationException`, 400, no fallback), and it also + * rejects a shorter TTL appearing before a longer one across tools -> system -> + * messages, so both "did we emit it everywhere" and "did we emit it nowhere" have + * to be checked by counting cache points rather than by spot-checking one. + */ + +/** Collect every `cachePoint` object reachable in a serialized request. */ +const collectCachePoints = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cachePoint" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +/** True when any `ttl` key exists anywhere in the serialized request. */ +const containsTtlKey = (value: unknown): boolean => { + if (Array.isArray(value)) return value.some(containsTtlKey) + if (value && typeof value === "object") { + return Object.entries(value as Record).some( + ([key, child]) => key === "ttl" || containsTtlKey(child), + ) + } + return false +} + +/** The serialized command input, so assertions run on the bytes that would be sent. */ +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) as Record + +// The system prompt must clear minTokensPerCachePoint (1024 for the Claude rows) so +// the strategy actually places a system cache point, and the message tail must clear +// it too so a message-level cache point is placed. Both emitters then appear in one +// request, which is the case a single-emitter assertion would miss. +const longSystemPrompt = "You are a helpful assistant. ".repeat(200) +const longUserMessage = "Please review this file. ".repeat(200) + +const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: longUserMessage }, + { role: "assistant", content: "Here is the review. ".repeat(200) }, + { role: "user", content: longUserMessage }, +] + +const createHandler = (overrides: Partial = {}) => + new AwsBedrockHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + ...overrides, + }) + +/** Run one createMessage and return the serialized ConverseStreamCommand input. */ +const captureConverseInput = async (handler: AwsBedrockHandler): Promise> => { + mockConverseStreamCommand.mockClear() + const stream = handler.createMessage(longSystemPrompt, messages) + await collectStream(stream) + + expect(mockConverseStreamCommand).toHaveBeenCalledTimes(1) + return serialize(mockConverseStreamCommand.mock.calls[0][0]) +} + +describe("Bedrock cache-point TTL emission", () => { + beforeEach(() => { + clearAllMocks() + mockCaptureException.mockClear() + }) + + describe("1h-capable model with the knob at 1h", () => { + it("carries ttl 1h on every cache point in the request", async () => { + const handler = createHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + const cachePoints = collectCachePoints(input) + + // Both emitters must be present, otherwise the "every cache point" claim is vacuous. + const systemCachePoints = collectCachePoints(input.system) + const messageCachePoints = collectCachePoints(input.messages) + expect(systemCachePoints.length).toBeGreaterThan(0) + expect(messageCachePoints.length).toBeGreaterThan(0) + expect(cachePoints.length).toBe(systemCachePoints.length + messageCachePoints.length) + + for (const cachePoint of cachePoints) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + + it("emits one uniform TTL per request, so the tools -> system -> messages ordering hazard stays unreachable", async () => { + const handler = createHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + const ttls = new Set(collectCachePoints(input).map((cachePoint) => cachePoint.ttl)) + + // A set of size 1 is the whole invariant: with one value there is no + // "shorter before longer" pair to reject, regardless of placement order. + expect(ttls).toEqual(new Set(["1h"])) + }) + + it("keeps the cache-point placement identical to the unset-knob request", async () => { + const capable = { apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" } as const + + const withoutKnob = createHandler(capable) + withoutKnob["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const baseInput = await captureConverseInput(withoutKnob) + + const withKnob = createHandler({ ...capable, promptCacheTtl: "1h" }) + withKnob["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const ttlInput = await captureConverseInput(withKnob) + + // Strip the ttl the knob adds and the two requests must be byte-identical: + // adding a TTL must not move a single cache point. + const stripTtl = (value: unknown): unknown => + JSON.parse(JSON.stringify(value, (key, child) => (key === "ttl" ? undefined : child))) + + expect(stripTtl(ttlInput)).toEqual(baseInput) + expect(collectCachePoints(baseInput).length).toBe(collectCachePoints(ttlInput).length) + }) + }) + + describe("5m-only models with the knob at 1h (ValidationException tripwire)", () => { + // AWS rejects an explicit ttl for these with a 400 that kills the turn, so the + // assertion is that no `ttl` key exists anywhere in the serialized request. + it.each([ + ["Opus 4.1 (4.x generation, 5m-only)", "anthropic.claude-opus-4-1-20250805-v1:0"], + ["Opus 4 (pre-4.1, 5m-only)", "anthropic.claude-opus-4-20250514-v1:0"], + ["Claude 3.7 Sonnet (3.x, 5m-only)", "anthropic.claude-3-7-sonnet-20250219-v1:0"], + ["Nova Pro (non-Anthropic, 5m-only)", "amazon.nova-pro-v1:0"], + ])("emits no ttl for %s", async (_label, apiModelId) => { + const handler = createHandler({ apiModelId, promptCacheTtl: "1h" }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(containsTtlKey(input)).toBe(false) + // The cache points themselves must still be there: the gate suppresses the + // ttl, not the caching. + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default" }) + } + }) + }) + + describe("knob unset (default parity)", () => { + it("serializes a byte-identical request to a pre-knob build for a capable model", async () => { + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(containsTtlKey(input)).toBe(false) + expect(collectCachePoints(input).length).toBeGreaterThan(0) + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default" }) + } + }) + + it("treats an explicit 5m knob as unset so the wire shape does not change", async () => { + const capable = { apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" } as const + + const unset = createHandler(capable) + unset["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const unsetInput = await captureConverseInput(unset) + + const explicit5m = createHandler({ ...capable, promptCacheTtl: "5m" }) + explicit5m["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + const explicit5mInput = await captureConverseInput(explicit5m) + + expect(explicit5mInput).toEqual(unsetInput) + expect(containsTtlKey(explicit5mInput)).toBe(false) + }) + }) + + describe("custom ARN / inference-profile resolution (fail closed)", () => { + it("emits 1h for an inference profile whose resolved base id is exactly listed", async () => { + // `us.` is a cross-region prefix, not part of the catalog id, so this only + // works if the prefix is stripped before the capability lookup. + const handler = createHandler({ + apiModelId: "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(collectCachePoints(input).length).toBeGreaterThan(0) + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + + it("emits no ttl for an inference-profile ARN that names a 5m-only model", async () => { + // The ARN names Opus 4.1, which is 5m-only despite being a 4.x generation. + // Resolution succeeds (so cache points are placed), but the capability + // lookup must still refuse the TTL. + const handler = createHandler({ + awsCustomArn: + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-opus-4-1-20250805-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + // Cache points exist, so the assertion is about the gate rather than about + // caching being off for an unrelated reason. + expect(collectCachePoints(input).length).toBeGreaterThan(0) + expect(containsTtlKey(input)).toBe(false) + }) + + it("places no cache points at all when the ARN resolves to no catalog model", async () => { + // The deepest fail-closed path: an application inference profile the user + // marked as "other" resolves to no catalog entry, so the guessed ModelInfo + // reports no prompt-cache support and no capability can exist. + const handler = createHandler({ + awsCustomArn: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + awsCustomArnBaseModelId: BEDROCK_CUSTOM_ARN_OTHER_BASE_MODEL, + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(collectCachePoints(input)).toEqual([]) + expect(containsTtlKey(input)).toBe(false) + }) + + it("emits no ttl for an unlisted model that the heuristic fallback guesses", async () => { + // The fallback spreads the default row (which is 1h-capable) and then overlays + // the guess, so the capability is inherited unless it is cleared. An unlisted + // id is exactly the case where we cannot prove the model accepts 1h, and Bedrock + // answers an unsupported ttl with a ValidationException that kills the turn. + const handler = createHandler({ + awsCustomArn: + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + // The guess still reports caching support, so cache points exist and the + // assertion is about the capability gate rather than caching being off. + expect(collectCachePoints(input).length).toBeGreaterThan(0) + expect(containsTtlKey(input)).toBe(false) + }) + + it("emits 1h for a foundation-model ARN that names a listed model", async () => { + const handler = createHandler({ + awsCustomArn: "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ stream: asyncStreamFrom([]) }) + + const input = await captureConverseInput(handler) + + expect(collectCachePoints(input).length).toBeGreaterThan(0) + for (const cachePoint of collectCachePoints(input)) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + }) + + describe("usage cacheDetails split", () => { + it("reports the 1h share of the cache writes when Converse returns a split", async () => { + const handler = createHandler({ + apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", + promptCacheTtl: "1h", + }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 3000, + cacheDetails: [ + { ttl: "1h", inputTokens: 2000 }, + { ttl: "5m", inputTokens: 1000 }, + ], + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ + type: "usage", + cacheWriteTokens: 3000, + cacheWrite1hTokens: 2000, + }) + }) + + it("omits the 1h share when the response carries no cacheDetails", async () => { + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { metadata: { usage: { inputTokens: 10, outputTokens: 5, cacheWriteInputTokens: 3000 } } }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 3000 }) + expect((usage as { cacheWrite1hTokens?: number }).cacheWrite1hTokens).toBeUndefined() + }) + + it.each([ + ["a 5m entry and no 1h entry", [{ ttl: "5m", inputTokens: 1000 }]], + ["a negative 1h count", [{ ttl: "1h", inputTokens: -5 }]], + ["a zero 1h count", [{ ttl: "1h", inputTokens: 0 }]], + ])("omits the 1h share when the split carries %s", async (_label, cacheDetails) => { + // Each row is a distinct malformed shape the reader must absorb rather than + // bill: an absent 1h entry (optional chain), a negative count (the `<= 0` + // guard, which `!oneHour` alone does not cover), and a zero count. + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 3000, + cacheDetails, + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 3000 }) + expect((usage as { cacheWrite1hTokens?: number }).cacheWrite1hTokens).toBeUndefined() + }) + + it("reads the 1h entry wherever it sits in the split, not just the first entry", async () => { + // Converse orders the split longest-TTL-first today, but the reader must match + // on the ttl value rather than trusting position: a 5m entry ahead of the 1h + // entry would otherwise report the 5m count as the 1h share. + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 3000, + cacheDetails: [ + { ttl: "5m", inputTokens: 1000 }, + { ttl: "1h", inputTokens: 2000 }, + ], + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 3000, cacheWrite1hTokens: 2000 }) + }) + + it("clamps the 1h share to the reported aggregate so a malformed split cannot over-bill", async () => { + const handler = createHandler({ apiModelId: "anthropic.claude-sonnet-4-5-20250929-v1:0" }) + handler["client"].send = vi.fn().mockResolvedValue({ + stream: asyncStreamFrom([ + { + metadata: { + usage: { + inputTokens: 10, + outputTokens: 5, + cacheWriteInputTokens: 500, + cacheDetails: [{ ttl: "1h", inputTokens: 900 }], + }, + }, + }, + ]), + }) + + const chunks = await collectStream(handler.createMessage(longSystemPrompt, messages)) + const usage = chunks.find((chunk) => chunk.type === "usage") + + expect(usage).toMatchObject({ type: "usage", cacheWriteTokens: 500, cacheWrite1hTokens: 500 }) + }) + }) +}) + +describe("Bedrock cache TTL capability catalog", () => { + const oneHourIds = Object.entries(bedrockModels) + .filter(([, info]) => (info as { promptCacheTtlSupport?: string[] }).promptCacheTtlSupport?.includes("1h")) + .map(([id]) => id) + .sort() + + it("marks exactly the 1h-capable Claude rows the AWS TTL table lists", () => { + // Pinned by exact id, not by pattern: Opus 4.1 and Opus 4 are 5m-only while + // Opus 4.6/4.7/4.8 are 1h-capable, so a version heuristic mis-classifies the + // boundary in both directions. Adding a row here means adding it to the AWS + // table first. + expect(oneHourIds).toEqual( + [ + "anthropic.claude-fable-5", + "anthropic.claude-fable-5-1", + "anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-haiku-5-5", + "anthropic.claude-opus-4-5-20251101-v1:0", + "anthropic.claude-opus-4-6-v1", + "anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "anthropic.claude-opus-5", + "anthropic.claude-opus-5-5", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-5", + ].sort(), + ) + }) + + it("never marks a Nova row, because Nova rejects an extended TTL outright", () => { + // The observed error for a Nova model is "Extended TTL prompt caching is only + // supported for Anthropic models", and Nova Premier is unresolved in the AWS + // table, so no Nova row may claim 1h. + const novaRows = Object.entries(bedrockModels) + .filter(([id]) => id.startsWith("amazon.nova")) + .filter(([, info]) => (info as { promptCacheTtlSupport?: string[] }).promptCacheTtlSupport !== undefined) + .map(([id]) => id) + + expect(novaRows).toEqual([]) + }) + + it("leaves the 5m-only Claude rows without capability so they fail closed", () => { + const fiveMinuteOnly = [ + "anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic.claude-opus-4-1-20250805-v1:0", + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + ] + + for (const id of fiveMinuteOnly) { + const info = (bedrockModels as Record)[id] + expect(info, id).toBeDefined() + expect(info.promptCacheTtlSupport, id).toBeUndefined() + } + }) + + it("keys every 1h cache-write rate off the same tier's input price", () => { + // The tier-shadow bug class: a tier that overrides inputPrice must override + // cacheWritesPrice1h too, otherwise a long-context 1h write bills at the base + // tier's rate. Checked per entry and per tier, not just on the top-level row. + for (const [id, info] of Object.entries(bedrockModels)) { + const entry = info as { + inputPrice?: number + cacheWritesPrice1h?: number + tiers?: Array<{ inputPrice?: number; cacheWritesPrice1h?: number }> + } + if (entry.cacheWritesPrice1h !== undefined && entry.inputPrice !== undefined) { + expect(entry.cacheWritesPrice1h, id).toBeCloseTo(entry.inputPrice * 2, 9) + } + + for (const [index, tier] of (entry.tiers ?? []).entries()) { + // A tier that reprices input without repricing the 1h write is exactly the + // shadow this test exists to catch, so it must fail rather than be skipped. + // The exemption is only for rows that price no 1h write at all: a 5m-only + // model's tier has no 1h rate to shadow. + if (tier.cacheWritesPrice1h === undefined) { + expect( + tier.inputPrice === undefined || entry.cacheWritesPrice1h === undefined, + `${id} tier ${index} reprices input but not the 1h write`, + ).toBe(true) + continue + } + expect(tier.inputPrice, `${id} tier ${index}`).toBeDefined() + expect(tier.cacheWritesPrice1h, `${id} tier ${index}`).toBeCloseTo((tier.inputPrice as number) * 2, 9) + } + } + }) +}) diff --git a/src/api/providers/__tests__/minimax.spec.ts b/src/api/providers/__tests__/minimax.spec.ts index 278168ff54..29edebdbfa 100644 --- a/src/api/providers/__tests__/minimax.spec.ts +++ b/src/api/providers/__tests__/minimax.spec.ts @@ -375,6 +375,38 @@ describe("MiniMaxHandler", () => { expect(firstChunk.value).toEqual({ type: "usage", inputTokens: 10, outputTokens: 20 }) }) + it("createMessage should surface the 1h cache-write split from cache_creation", async () => { + mockCreate.mockResolvedValueOnce( + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 10, + output_tokens: 0, + cache_creation_input_tokens: 100_000, + cache_creation: { + ephemeral_5m_input_tokens: 40_000, + ephemeral_1h_input_tokens: 60_000, + }, + cache_read_input_tokens: 0, + }, + }, + }, + ]), + ) + + const stream = handler.createMessage("system prompt", []) + const firstChunk = await stream.next() + + expect(firstChunk.done).toBe(false) + expect(firstChunk.value).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 60_000, + }) + }) + it("createMessage should adopt cumulative message_delta output tokens for the final totalCost", async () => { mockCreate.mockResolvedValueOnce( asyncStreamFrom([ diff --git a/src/api/providers/__tests__/openai-native-usage.spec.ts b/src/api/providers/__tests__/openai-native-usage.spec.ts index 8922c2cf9c..b79145efdd 100644 --- a/src/api/providers/__tests__/openai-native-usage.spec.ts +++ b/src/api/providers/__tests__/openai-native-usage.spec.ts @@ -1,6 +1,8 @@ import { describe, it, expect, beforeEach } from "vitest" +import { createHash } from "crypto" import { OpenAiNativeHandler } from "../openai-native" -import { OpenAiServiceTier, openAiNativeModels } from "@roo-code/types" +import { OpenAiServiceTier, openAiNativeModels, type ModelInfo } from "@roo-code/types" +import type { ApiStreamUsageChunk } from "../../transform/stream" describe("OpenAiNativeHandler - normalizeUsage", () => { let handler: OpenAiNativeHandler @@ -440,6 +442,72 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { }) }) + describe("OpenAiNativeHandler - prompt cache key", () => { + let handler: OpenAiNativeHandler + + beforeEach(() => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + }) + }) + + const buildRequestBody = (modelId: string, taskId?: string) => { + handler["options"].apiModelId = modelId + const model = handler.getModel() + const metadata = taskId === undefined ? undefined : { taskId } + return handler["buildRequestBody"](model, [], "", model.verbosity, undefined, metadata) + } + + it("should send the task id as the body prompt_cache_key", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + }) + + it("should keep the key stable across turns of the same task", () => { + const firstTurn = buildRequestBody("gpt-5.1", "task-stable") + const secondTurn = buildRequestBody("gpt-5.1", "task-stable") + expect(firstTurn.prompt_cache_key).toBe("task-stable") + expect(secondTurn.prompt_cache_key).toBe(firstTurn.prompt_cache_key) + }) + + it("should give distinct keys to distinct tasks", () => { + const first = buildRequestBody("gpt-5.1", "task-one") + const second = buildRequestBody("gpt-5.1", "task-two") + expect(first.prompt_cache_key).toBe("task-one") + expect(second.prompt_cache_key).toBe("task-two") + expect(first.prompt_cache_key).not.toBe(second.prompt_cache_key) + }) + + it("should fall back to the handler session id when no task id is supplied", () => { + const body = buildRequestBody("gpt-5.1") + expect(body.prompt_cache_key).toBe(handler["sessionId"]) + expect(body.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the key when the model does not support prompt caching", () => { + const modelId = "codex-mini-latest" + expect(openAiNativeModels[modelId as keyof typeof openAiNativeModels].supportsPromptCache).toBe(false) + + const body = buildRequestBody(modelId, "task-abc") + expect(body.prompt_cache_key).toBeUndefined() + }) + + it("should clamp an over-length task id to OpenAI's 64-character limit", () => { + const longTaskId = "t".repeat(80) + const body = buildRequestBody("gpt-5.1", longTaskId) + expect(body.prompt_cache_key).toHaveLength(64) + // A hash, not a slice: truncation would collapse ids that share a prefix. + expect(body.prompt_cache_key).toBe(createHash("sha256").update(longTaskId).digest("hex")) + expect(body.prompt_cache_key).not.toBe(longTaskId.slice(0, 64)) + }) + + it("should keep prompt_cache_retention alongside the key for 24h models", () => { + const body = buildRequestBody("gpt-5.1", "task-abc") + expect(body.prompt_cache_key).toBe("task-abc") + expect(body.prompt_cache_retention).toBe("24h") + }) + }) + describe("cost calculation", () => { it.each([ { tier: OpenAiServiceTier.Default, expectedCost: 4.525 }, @@ -463,6 +531,35 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(expectedCost, 6) }) + it.each([ + // gpt-6-astra: default input 10.0 / 12h 25.0; flex input 5.0; priority input 20.0. + // The 12h rate must follow the tier's own input price (2.5x), not the + // standard-tier 12h rate, or a flex/priority 12h write is mis-billed. + { tier: OpenAiServiceTier.Default, expectedCost: 2.5 }, + { tier: OpenAiServiceTier.Flex, expectedCost: 1.25 }, + { tier: OpenAiServiceTier.Priority, expectedCost: 5.0 }, + ])("derives the 12h rate from the $tier tier's own input price", ({ tier, expectedCost }) => { + handler = new OpenAiNativeHandler({ + openAiNativeApiKey: "test-key", + openAiNativeServiceTier: tier, + }) + const result = handler["normalizeUsage"]( + { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { ttl_12h_tokens: 100_000 }, + }, + }, + getGpt6AstraModel(), + ) + + if (!result) throw new Error("Expected usage") + // All input is consumed by the write, so the whole cost is the 12h class. + expect(result.totalCost).toBeCloseTo(expectedCost, 6) + }) + it("should pass total input tokens to calculateApiCostOpenAI", () => { const usage = { input_tokens: 100, @@ -551,4 +648,146 @@ describe("OpenAiNativeHandler - normalizeUsage", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split (Responses API)", () => { + // `normalizeUsage` is private. A narrow structural cast keeps the call typed + // without widening the handler to `any`; the return is asserted non-null because + // every call site below passes a usage object that yields a chunk. + const normalizeUsage = ( + h: OpenAiNativeHandler, + usage: unknown, + model: { id: string; info: ModelInfo }, + ): ApiStreamUsageChunk => { + const result = ( + h as unknown as { + normalizeUsage(u: unknown, m: { id: string; info: ModelInfo }): ApiStreamUsageChunk | undefined + } + ).normalizeUsage(usage, model) + expect(result).toBeDefined() + return result as ApiStreamUsageChunk + } + + it("should read ttl_12h_tokens and bill it at the 12h rate", () => { + // gpt-5.6-sol: input 4.0, cacheWritesPrice 5.0 (30m class), cacheWritesPrice12h 10.0. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 0, + ttl_12h_tokens: 100_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + type: "usage", + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 100_000, + }) + + // gpt-6-astra: input 10.0, 12h rate 25.0. + // Base input: (10.0 / 1_000_000) * (100_000 - 100_000) = 0 + // 12h writes: (25.0 / 1_000_000) * 100_000 = 2.5 + // The pre-split behaviour charged the 30m rate: 12.5 * 0.1 = 1.25 (50% low). + expect(result.totalCost).toBeCloseTo(2.5, 6) + expect(result.totalCost).not.toBeCloseTo(1.25, 6) + }) + + it("should split a mixed 30m/12h write total without double counting", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_30m_tokens: 60_000, + ttl_12h_tokens: 40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result).toMatchObject({ + cacheWriteTokens: 100_000, + cacheWrite12hTokens: 40_000, + }) + + // 30m remainder: 60_000 at 12.5 => 0.75 + // 12h: 40_000 at 25.0 => 1.0 + // Total: 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + + it("should leave the split undefined when the response omits it", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + }, + } + + const result = normalizeUsage(handler, usage, model) + + expect(result.cacheWrite12hTokens).toBeUndefined() + // All writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) + + it("should clamp a 12h split that exceeds the write total", () => { + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 50_000, + cache_write_tokens_details: { + ttl_12h_tokens: 80_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // Clamped to the write total so no more 12h tokens are billed than written. + expect(result.cacheWrite12hTokens).toBe(50_000) + // Base input: (10.0 / 1_000_000) * (100_000 - 50_000) = 0.5 + // 12h writes: (25.0 / 1_000_000) * 50_000 = 1.25 + // Total: 0.5 + 1.25 = 1.75 + expect(result.totalCost).toBeCloseTo(1.75, 6) + }) + + it("should clamp a negative 12h split at zero instead of emitting a negative count", () => { + // A misbehaving gateway can report a negative ttl_12h_tokens. Math.min alone + // leaves it negative, so the negative count reaches the Task accumulator and + // the webview. Shared costing clamps, but the emitted chunk must not lie. + const model = getGpt6AstraModel() + const usage = { + input_tokens: 100_000, + output_tokens: 0, + input_tokens_details: { + cache_write_tokens: 100_000, + cache_write_tokens_details: { + ttl_12h_tokens: -40_000, + }, + }, + } + + const result = normalizeUsage(handler, usage, model) + + // A zero split is indistinguishable from "no split", so the field stays unset. + expect(result.cacheWrite12hTokens).toBeUndefined() + expect(result.cacheWrite12hTokens ?? 0).toBeGreaterThanOrEqual(0) + // All 100k writes bill at the 30m rate: (12.5 / 1_000_000) * 100_000 = 1.25 + expect(result.totalCost).toBeCloseTo(1.25, 6) + }) + }) }) diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 8a086b48db..c7e3663efb 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -534,6 +534,26 @@ describe("OpenAiNativeHandler", () => { expect(result).toBe("") }) + + it("should scope the prompt_cache_key to the handler session", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request.prompt_cache_key).toBe(handler["sessionId"]) + expect(request.prompt_cache_key).toEqual(expect.any(String)) + }) + + it("should omit the prompt_cache_key when the model does not support prompt caching", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + handler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "codex-mini-latest" }) + + await handler.completePrompt("Test prompt") + + const [request] = mockResponsesCreate.mock.calls[0] + expect(request).not.toHaveProperty("prompt_cache_key") + }) }) describe("getModel", () => { @@ -1446,6 +1466,130 @@ describe("OpenAiNativeHandler", () => { expect(callBody.previous_response_id).toBeUndefined() }) + it("sends a task-scoped prompt_cache_key in the body and no session_id header", async () => { + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + // Force the hand-built SSE path so the wire body and headers are inspectable. + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["task-cache-one", "task-cache-one", "task-cache-two"]) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(3) + expect(bodies[0].prompt_cache_key).toBe("task-cache-one") + // Same task, second turn: the key must not drift, or the growing prefix loses affinity. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + // A different task must not share the bucket. + expect(bodies[2].prompt_cache_key).toBe("task-cache-two") + expect(bodies[2].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + + // The retention control is untouched by this change. + expect(bodies[0].prompt_cache_retention).toBe("24h") + + // The affinity header is gone: OpenAI ignores it for caching and hardened + // gateways reject underscore header names outright. + for (const [, init] of mockFetch.mock.calls) { + expect(init.headers).not.toHaveProperty("session_id") + } + }) + + it("keeps two over-long task ids distinct instead of truncating them to one bucket", async () => { + // Task ids are uuidv7 today (36 chars), so this guards the wire contract for + // any caller that sends a longer id: a 64-char slice drops the distinguishing + // tail, so two ids sharing a 64-char prefix would land in one routing bucket. + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as typeof fetch + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + // 70 characters: identical for the first 64, different in the tail. + const sharedPrefix = "a".repeat(64) + const longTaskIds = [`${sharedPrefix}-one`, `${sharedPrefix}-two`] + + for (const taskId of longTaskIds) { + for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) { + // consume + } + } + + const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body)) + expect(bodies).toHaveLength(2) + // OpenAI rejects keys over 64 characters, so the key must still fit. + expect(bodies[0].prompt_cache_key.length).toBeLessThanOrEqual(64) + expect(bodies[1].prompt_cache_key.length).toBeLessThanOrEqual(64) + // ...but the two tasks must not collide on that shortened form. + expect(bodies[1].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key) + }) + + it("sends the task-scoped prompt_cache_key through the SDK path", async () => { + mockResponsesCreate.mockResolvedValue(asyncStreamFrom([])) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + for (const taskId of ["sdk-task-one", "sdk-task-one", "sdk-task-two"]) { + await collectStream(cacheHandler.createMessage(systemPrompt, messages, { taskId })) + } + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies[0].prompt_cache_key).toBe("sdk-task-one") + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + expect(bodies[2].prompt_cache_key).toBe("sdk-task-two") + expect(bodies[0].prompt_cache_retention).toBe("24h") + expect(mockResponsesCreate.mock.calls[0][1].headers).not.toHaveProperty("session_id") + }) + + it("keys completePrompt off the task id when the caller supplies one", async () => { + // Prompt enhancement builds a fresh handler per request, so a session-scoped key + // would start a new cache bucket every time. The task id keeps repeated + // enhancements of one task on the same bucket. + mockResponsesCreate.mockResolvedValue({ output: [] }) + + const first = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + const second = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + await first.completePrompt("enhance me", { taskId: "task-enhance" }) + await second.completePrompt("enhance me", { taskId: "task-enhance" }) + + const bodies = mockResponsesCreate.mock.calls.map(([body]) => body) + expect(bodies).toHaveLength(2) + expect(bodies[0].prompt_cache_key).toBe("task-enhance") + // Different handler instances, same task: the key must not drift. + expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key) + }) + + it("falls back to the handler session id when completePrompt gets no task id", async () => { + mockResponsesCreate.mockResolvedValue({ output: [] }) + + const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" }) + + await cacheHandler.completePrompt("enhance me") + + const [body] = mockResponsesCreate.mock.calls[0] + expect(body.prompt_cache_key).toBe(cacheHandler["sessionId"]) + }) + it("should provide helpful error messages for different error codes", async () => { const testCases = [ { status: 400, expectedMessage: "Invalid request to Responses API" }, diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index 1bf6669324..fec3e4a2e2 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -485,6 +485,115 @@ describe("OpenRouterHandler", () => { ) }) + describe("prompt cache TTL", () => { + const mockStreamWithUsage = () => + asyncStreamFrom([ + { id: "test-id", choices: [{ delta: { content: "test response" } }] }, + { id: "test-id", choices: [{ delta: {} }], usage: { prompt_tokens: 10, completion_tokens: 20 } }, + ]) + + const requestBody = (mockCreate: ReturnType) => + mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + + // `openai` is mocked for this file, so the prototype is a plain object here. + // Reaching it once keeps the casts out of each test body. + const stubChatCreate = (mockCreate: ReturnType) => { + const proto = OpenAI.prototype as unknown as { chat: { completions: { create: unknown } } } + proto.chat = { completions: { create: mockCreate } } + } + + const cacheControlsIn = (body: unknown) => { + const found: Array> = [] + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + walk(JSON.parse(JSON.stringify(body))) + return found + } + + it("emits no ttl key when the knob is unset", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ ...mockOptions, openRouterModelId: "anthropic/claude-sonnet-4" }), + ) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("fails closed when the resolved model advertises no TTL capability", async () => { + // OpenRouter models come from a dynamic catalog that carries no TTL + // capability data, so a 1h knob must not reach the wire (ADR-0006 I2, + // and the ADR's stated risk for dynamic catalogs). + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + expect(handler.getModel().info.promptCacheTtlSupport).toBeUndefined() + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const body = requestBody(mockCreate) + expect(JSON.stringify(body)).not.toContain("ttl") + expect(cacheControlsIn(body).every((control) => control.ttl === undefined)).toBe(true) + }) + + it("carries ttl 1h on every marker when the resolved model advertises it", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + promptCacheTtl: "1h", + }), + ) + + // Simulate a catalog that does carry capability data. + const model = handler.getModel() + vitest.spyOn(handler, "getModel").mockReturnValue({ + ...model, + info: { ...model.info, promptCacheTtlSupport: ["5m", "1h"] }, + }) + + const mockCreate = vitest.fn().mockResolvedValue(mockStreamWithUsage()) + stubChatCreate(mockCreate) + + await collectStream( + handler.createMessage("test system prompt", [{ role: "user", content: "test message" }]), + ) + + const controls = cacheControlsIn(requestBody(mockCreate)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + }) + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockStream = asyncStreamFrom([{ error: { message: "API Error", code: 500 } }]) diff --git a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts index e69240c1da..0231d34fd1 100644 --- a/src/api/providers/__tests__/vercel-ai-gateway.spec.ts +++ b/src/api/providers/__tests__/vercel-ai-gateway.spec.ts @@ -449,6 +449,29 @@ describe("VercelAiGatewayHandler", () => { expect(addCacheBreakpoints).toHaveBeenCalled() }) + it("does not emit a ttl for a gateway model with no TTL capability", async () => { + // The gateway catalog is fetched at runtime and carries no TTL capability, + // so a 1h knob must fail closed (ADR-0006 I2). + const { addCacheBreakpoints } = await import("../../transform/caching/vercel-ai-gateway") + const handler = new VercelAiGatewayHandler( + makeApiHandlerOptions({ + ...mockOptions, + vercelAiGatewayModelId: "anthropic/claude-3.5-haiku", + promptCacheTtl: "1h", + }), + ) + + await handler.createMessage("You are a helpful assistant.", [{ role: "user", content: "Hello" }]).next() + + // `addCacheBreakpoints` is file-mocked, so the request body never receives + // `cache_control` and `JSON.stringify(mockCreate...)` cannot see the TTL. + // The TTL only exists as the third argument to the mocked call. + expect(addCacheBreakpoints).toHaveBeenCalled() + const lastCall = vitest.mocked(addCacheBreakpoints).mock.calls.at(-1) + expect(lastCall).toBeDefined() + expect(lastCall![2]).toBeUndefined() + }) + it("sets correct max_completion_tokens", async () => { const handler = new VercelAiGatewayHandler(mockOptions) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index c1ffdc6d7f..d200f4c774 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -15,6 +15,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { addCacheBreakpoints } from "../transform/caching/vertex" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" @@ -79,6 +80,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple const { supportsPromptCache } = info + // One TTL for the whole request (ADR-0006 I4). + const cacheTtl = resolveCacheTtl(this.options, info) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -116,9 +120,9 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple thinking, // Cache the system prompt if caching is enabled. system: supportsPromptCache - ? [{ text: systemPrompt, type: "text" as const, cache_control: { type: "ephemeral" } }] + ? [{ text: systemPrompt, type: "text" as const, cache_control: cacheControl(cacheTtl) }] : systemPrompt, - messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages) : sanitizedMessages, + messages: supportsPromptCache ? addCacheBreakpoints(sanitizedMessages, cacheTtl) : sanitizedMessages, stream: true, ...nativeToolParams, } as Anthropic.Messages.MessageCreateParamsStreaming @@ -145,6 +149,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple case "message_start": { const usage = chunk.message.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const cacheWrite1hTokens = usage.cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens; surfaced for per-turn // thinking telemetry (billing already includes it in output_tokens). @@ -153,6 +161,7 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0, cacheWriteTokens: usage.cache_creation_input_tokens || undefined, + cacheWrite1hTokens: cacheWrite1hTokens || undefined, cacheReadTokens: usage.cache_read_input_tokens || undefined, ...(typeof usage.output_tokens_details?.thinking_tokens === "number" ? { reasoningTokens: usage.output_tokens_details.thinking_tokens } @@ -270,6 +279,10 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from + // the 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } @@ -313,11 +326,12 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple try { const { id, - info: { supportsPromptCache }, + info, temperature, maxTokens = ANTHROPIC_DEFAULT_MAX_TOKENS, reasoning: thinking, } = this.getModel() + const { supportsPromptCache } = info const params = { model: id, @@ -328,7 +342,13 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple { role: "user", content: supportsPromptCache - ? [{ type: "text" as const, text: prompt, cache_control: { type: "ephemeral" } }] + ? [ + { + type: "text" as const, + text: prompt, + cache_control: cacheControl(resolveCacheTtl(this.options, info)), + }, + ] : prompt, }, ], diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 4983e0cbaf..71355f878e 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -19,6 +19,7 @@ import { ApiStream } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { filterNonAnthropicBlocks } from "../transform/anthropic-filter" import { getAnthropicProviderReasoning } from "../transform/reasoning" +import { cacheControl, resolveCacheTtl } from "../transform/caching/ttl" import { BaseProvider } from "./base-provider" import { bridgeExternalAbortSignal, hasUsableTimeout, toSdkTimeoutMs } from "./utils/abort-signal" @@ -64,7 +65,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { let stream: AnthropicStream - const cacheControl: CacheControlEphemeral = { type: "ephemeral" } const { id: modelId, betas = ["fine-grained-tool-streaming-2025-05-14"], @@ -79,6 +79,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa settings: this.options, }) + // One TTL for the whole request (ADR-0006 I4): every marker below reuses + // this single object, so a mixed-TTL request is unrepresentable. + const cacheTtl = resolveCacheTtl(this.options, info) + const marker: CacheControlEphemeral = cacheControl(cacheTtl) + // Filter out non-Anthropic blocks (reasoning, thoughtSignature, etc.) before sending to the API const sanitizedMessages = filterNonAnthropicBlocks(messages) @@ -169,17 +174,17 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa temperature, thinking, // Setting cache breakpoint for system prompt so new tasks can reuse it. - system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + system: [{ text: systemPrompt, type: "text", cache_control: marker }], messages: sanitizedMessages.map((message, index) => { if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { return { ...message, content: typeof message.content === "string" - ? [{ type: "text", text: message.content, cache_control: cacheControl }] + ? [{ type: "text", text: message.content, cache_control: marker }] : message.content.map((content, contentIndex) => contentIndex === message.content.length - 1 - ? { ...content, cache_control: cacheControl } + ? { ...content, cache_control: marker } : content, ), } @@ -256,6 +261,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -266,9 +272,14 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // The per-TTL breakdown decomposes `cache_creation_input_tokens`; it is + // absent for providers that do not report it, which leaves the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + // `output_tokens_details.thinking_tokens` decomposes the billed // output into internal reasoning tokens (0 ⇒ the model skipped // thinking this turn). `output_tokens` already includes them for @@ -280,6 +291,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, ...(typeof startThinkingTokens === "number" ? { reasoningTokens: startThinkingTokens } : {}), } @@ -287,6 +299,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -385,6 +398,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { @@ -437,6 +451,11 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa inputPrice: tier.inputPrice, outputPrice: tier.outputPrice, cacheWritesPrice: tier.cacheWritesPrice, + // The tier's own TTL rate must come along: the entry-level rate was + // derived from the 200K input price, so keeping it here would bill a + // 1M-context 1h write at half its tier rate. + cacheWritesPrice1h: tier.cacheWritesPrice1h ?? info.cacheWritesPrice1h, + cacheWritesPrice12h: tier.cacheWritesPrice12h ?? info.cacheWritesPrice12h, cacheReadsPrice: tier.cacheReadsPrice, } } diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index e09cc0b17b..c72dc8f360 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -3,12 +3,13 @@ import { ConverseStreamCommand, ConverseCommand, BedrockRuntimeClientConfig, - ContentBlock, Message, SystemContentBlock, Tool, ToolConfiguration, ToolChoice, + CacheTTL, + type CacheDetail, } from "@aws-sdk/client-bedrock-runtime" import { NodeHttpHandler } from "@smithy/node-http-handler" import OpenAI from "openai" @@ -22,6 +23,7 @@ import { type ProviderSettings, type BedrockModelId, type BedrockServiceTier, + type PromptCacheTtl, bedrockDefaultModelId, bedrockModels, bedrockDefaultPromptRouterModelId, @@ -47,6 +49,7 @@ import { logger } from "../../utils/logging" import { Package } from "../../shared/package" import { MultiPointStrategy } from "../transform/cache-strategy/multi-point-strategy" import { ModelInfo as CacheModelInfo } from "../transform/cache-strategy/types" +import { resolveCacheTtl } from "../transform/caching/ttl" import { convertToBedrockConverseMessages as sharedConverter } from "../transform/bedrock-converse-format" import { getModelParams } from "../transform/model-params" import { shouldUseReasoningBudget } from "../../shared/api" @@ -179,6 +182,9 @@ export interface StreamEvent { cacheWriteInputTokens?: number cacheReadInputTokenCount?: number cacheWriteInputTokenCount?: number + // Per-TTL split of the cache writes. Converse returns one entry per TTL + // class that received writes, longest TTL first. + cacheDetails?: CacheDetail[] } metrics?: { latencyMs: number @@ -210,6 +216,25 @@ export type UsageType = { cacheWriteInputTokens?: number cacheReadInputTokenCount?: number cacheWriteInputTokenCount?: number + cacheDetails?: CacheDetail[] +} + +/** + * Read the 1h share of a Converse `cacheDetails` split. + * + * Best effort by construction: the field is optional in the pinned SDK and absent + * on every pre-1h response, so a missing split means "no long-lived write was + * reported" and the cost layer prices the whole aggregate at the 5m rate. The + * counts are a decomposition of `cacheWriteInputTokens`, never an addition to it, + * so the value is clamped to the aggregate to keep a malformed response from + * billing more writes than were reported. + */ +function readCacheWrite1hTokens(usage: UsageType, cacheWriteTokens: number): number | undefined { + const oneHour = usage.cacheDetails?.find((detail) => detail.ttl === CacheTTL.ONE_HOUR)?.inputTokens + if (!oneHour || oneHour <= 0) { + return undefined + } + return Math.min(oneHour, cacheWriteTokens) } /************************************************************************************ @@ -448,6 +473,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH usePromptCache, modelConfig.info, conversationId, + resolveCacheTtl(this.options, modelConfig.info), ) let additionalModelRequestFields: BedrockAdditionalModelFields | undefined @@ -642,6 +668,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH outputTokens: usage.outputTokens || 0, cacheReadTokens: cacheReadTokens, cacheWriteTokens: cacheWriteTokens, + cacheWrite1hTokens: readCacheWrite1hTokens(usage, cacheWriteTokens), } continue } @@ -974,7 +1001,11 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH } /** - * Convert Anthropic messages to Bedrock Converse format + * Convert Anthropic messages to Bedrock Converse format. + * + * `cacheTtl` is resolved once by the caller and threaded into the strategy, so + * both cache-point emitters in this method (the strategy's system/message + * blocks and the message-level literal below) carry the same value. */ private convertToBedrockConverseMessages( anthropicMessages: Anthropic.Messages.MessageParam[] | { role: string; content: string }[], @@ -982,6 +1013,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH usePromptCache: boolean = false, modelInfo?: any, conversationId?: string, // Optional conversation ID to track cache points across messages + cacheTtl?: PromptCacheTtl, ): { system: SystemContentBlock[]; messages: Message[] } { // First convert messages using shared converter for proper image handling const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[]) @@ -1017,6 +1049,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH messages: anthropicMessages as Anthropic.Messages.MessageParam[], usePromptCache, previousCachePointPlacements: previousPlacements, + cacheTtl, } // Get cache point placements @@ -1028,13 +1061,15 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH this.previousCachePointPlacements[conversationId] = cacheResult.messageCachePointPlacements } - // Apply cache points to the properly converted messages + // Apply cache points to the properly converted messages. The strategy owns the + // cache-point shape, so the message-level emitter asks it rather than repeating + // the literal: that is what keeps both emitters on one TTL. const messagesWithCache = convertedMessages.map((msg, index) => { const placement = cacheResult.messageCachePointPlacements?.find((p) => p.index === index) if (placement) { return { ...msg, - content: [...(msg.content || []), { cachePoint: { type: "default" } } as ContentBlock], + content: [...(msg.content || []), strategy.createCachePoint()], } } return msg @@ -1212,13 +1247,15 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH } else { // Use heuristics for model info, then allow overrides from ProviderSettings const guessed = this.guessModelInfoFromId(modelId) - model = { - id: bedrockDefaultModelId, - info: { - ...JSON.parse(JSON.stringify(bedrockModels[bedrockDefaultModelId])), - ...guessed, - }, + const info: ModelInfo = { + ...JSON.parse(JSON.stringify(bedrockModels[bedrockDefaultModelId])), + ...guessed, } + // The default row is 1h-capable, but an id we cannot resolve is not. Inheriting + // the capability would send `ttl: "1h"` to a model that may reject it with a + // ValidationException that kills the turn, so an unlisted id fails closed (I2). + delete info.promptCacheTtlSupport + model = { id: bedrockDefaultModelId, info } } // Always allow user to override detected/guessed maxTokens and contextWindow @@ -1295,6 +1332,10 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH inputPrice: tier?.inputPrice ?? modelConfig.info.inputPrice, outputPrice: tier?.outputPrice ?? modelConfig.info.outputPrice, cacheWritesPrice: tier?.cacheWritesPrice ?? modelConfig.info.cacheWritesPrice, + // Carry the tier's own TTL rate; the entry-level one was derived from the + // 200K input price and would halve a 1M-context 1h write. + cacheWritesPrice1h: tier?.cacheWritesPrice1h ?? modelConfig.info.cacheWritesPrice1h, + cacheWritesPrice12h: tier?.cacheWritesPrice12h ?? modelConfig.info.cacheWritesPrice12h, cacheReadsPrice: tier?.cacheReadsPrice ?? modelConfig.info.cacheReadsPrice, } } diff --git a/src/api/providers/minimax.ts b/src/api/providers/minimax.ts index 88c8a443a1..2addaf1a39 100644 --- a/src/api/providers/minimax.ts +++ b/src/api/providers/minimax.ts @@ -137,6 +137,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand let inputTokens = 0 let outputTokens = 0 let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 let cacheReadTokens = 0 for await (const chunk of withFinallyCleanup(stream, removeExternalAbortListener)) { @@ -147,20 +148,27 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand input_tokens = 0, output_tokens = 0, cache_creation_input_tokens, + cache_creation, cache_read_input_tokens, } = chunk.message.usage + // Per-TTL breakdown of the write total; absent when the provider + // does not report it, leaving the split at 0. + const startCacheWrite1hTokens = cache_creation?.ephemeral_1h_input_tokens + yield { type: "usage", inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, + cacheWrite1hTokens: startCacheWrite1hTokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, } inputTokens += input_tokens outputTokens += output_tokens cacheWriteTokens += cache_creation_input_tokens || 0 + cacheWrite1hTokens += startCacheWrite1hTokens || 0 cacheReadTokens += cache_read_input_tokens || 0 break @@ -247,6 +255,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, ) yield { diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index 9711156347..d6ab76a0f8 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -1,4 +1,5 @@ import * as os from "os" +import { createHash } from "crypto" import { v7 as uuidv7 } from "uuid" import { Anthropic } from "@anthropic-ai/sdk" import OpenAI from "openai" @@ -16,6 +17,8 @@ import { SERVICE_TIER_KEY, type ServiceTier, ApiProviderError, + CACHE_WRITE_MULTIPLIER_1H, + CACHE_WRITE_MULTIPLIER_12H, } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" @@ -34,6 +37,12 @@ import { sanitizeOpenAiCallId } from "../../utils/tool-id" export type OpenAiNativeModel = ReturnType +/** + * OpenAI caps `prompt_cache_key` at 64 characters; longer values are rejected with a 400. + * Task ids are uuidv7 today (36 chars), so this is a guard rather than a routine truncation. + */ +const PROMPT_CACHE_KEY_MAX_LENGTH = 64 + export class OpenAiNativeHandler extends BaseProvider implements SingleCompletionHandler { protected options: ApiHandlerOptions private client: OpenAI @@ -96,14 +105,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio this.options.enableResponsesReasoningSummary = true } const apiKey = this.options.openAiNativeApiKey ?? NOT_PROVIDED - // Include originator, session_id, and User-Agent headers for API tracking and debugging + // Include originator and User-Agent headers for API tracking and debugging. + // Cache affinity rides on the body `prompt_cache_key`, not on a header: OpenAI documents + // the `session_id` header as ignored on the public Responses API, and underscore header + // names are rejected outright by hardened gateways (Envoy/nginx `underscores_in_headers`). const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` this.client = new OpenAI({ baseURL: this.options.openAiNativeBaseUrl || undefined, apiKey, defaultHeaders: { originator: "zoo-code", - session_id: this.sessionId, "User-Agent": userAgent, }, timeout: this.timeoutMs, @@ -124,6 +135,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const writesFromDetails = typeof inputDetails?.cache_write_tokens === "number" ? inputDetails.cache_write_tokens : 0 + // The Responses API splits cache writes by TTL class under + // `input_tokens_details.cache_write_tokens_details`. The installed SDK types + // (openai@5.23.2) do not declare it yet, so it is read defensively; the + // fields are a decomposition of `cache_write_tokens`, not an addition. + const writeDetails = inputDetails?.cache_write_tokens_details + const writes12hFromDetails = typeof writeDetails?.ttl_12h_tokens === "number" ? writeDetails.ttl_12h_tokens : 0 + // If total input tokens are missing but we have details, derive from them let totalInputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0 if ( @@ -144,6 +162,12 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0 + // Clamp the 12h subset to the write total so a malformed response cannot bill + // more 12h tokens than were written at all. The lower clamp matters too: a + // negative `ttl_12h_tokens` from a misbehaving gateway would otherwise reach the + // Task accumulator and the webview as a negative count. + const cacheWrite12hTokens = Math.max(0, Math.min(writes12hFromDetails, cacheWriteTokens)) + // Resolve effective tier: prefer actual tier from response; otherwise requested tier const effectiveTier = this.lastServiceTier || this.options.openAiNativeServiceTier || undefined const effectiveInfo = this.applyServiceTierPricing(model.info, effectiveTier) @@ -157,6 +181,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio cacheWriteTokens, cacheReadTokens, effectiveTier, + undefined, + cacheWrite12hTokens, ) const reasoningTokens = @@ -170,6 +196,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio inputTokens: totalInputTokens, outputTokens: totalOutputTokens, cacheWriteTokens, + cacheWrite12hTokens: cacheWrite12hTokens || undefined, cacheReadTokens, ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}), totalCost, @@ -329,6 +356,11 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio include?: string[] /** Prompt cache retention policy: "in_memory" (default) or "24h" for extended caching */ prompt_cache_retention?: "in_memory" | "24h" + /** + * Cache routing affinity key. OpenAI documents this body field as the routing lever; + * the `session_id` request header is not part of the prompt-caching API. + */ + prompt_cache_key?: string tools?: Array<{ type: "function" name: string @@ -346,6 +378,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Decide whether to enable extended prompt cache retention for this request const promptCacheRetention = this.getPromptCacheRetention(model) + // Cache routing affinity: a stable, task-scoped key lets OpenAI route every turn of the + // same task to the machine holding its cached prefix. + const promptCacheKey = this.getPromptCacheKey(model, metadata) + const body: ResponsesRequestBody = { model: model.id, input: formattedInput, @@ -378,6 +414,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Enable extended prompt cache retention for models that support it. // This uses the OpenAI Responses API `prompt_cache_retention` parameter. ...(promptCacheRetention ? { prompt_cache_retention: promptCacheRetention } : {}), + // Cache routing affinity key (body field, not the inert `session_id` header). + ...(promptCacheKey ? { prompt_cache_key: promptCacheKey } : {}), tools: (metadata?.tools ?? []) .filter((tool) => tool.type === "function") .map((tool) => { @@ -417,12 +455,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` const requestHeaders: Record = { originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, } @@ -556,7 +592,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio private async *makeResponsesApiRequest( requestBody: any, model: OpenAiNativeModel, - metadata?: ApiHandlerCreateMessageMetadata, + _metadata?: ApiHandlerCreateMessageMetadata, _systemPrompt?: string, _messages?: Anthropic.Messages.MessageParam[], ): ApiStream { @@ -567,8 +603,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio // Create AbortController for cancellation this.abortController = new AbortController() - // Build per-request headers using taskId when available, falling back to sessionId - const taskId = metadata?.taskId + // Build per-request headers for API tracking and debugging const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}` try { @@ -578,7 +613,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, originator: "zoo-code", - session_id: taskId || this.sessionId, "User-Agent": userAgent, }, body: JSON.stringify(requestBody), @@ -1419,6 +1453,33 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio return undefined } + /** + * Returns the prompt cache routing key for this request, if the model caches prompts. + * + * OpenAI keys cache affinity off the `prompt_cache_key` body field; the `session_id` + * request header is not part of the prompt-caching API. The key must stay identical + * across every turn of one task (so its growing prefix keeps landing on the same + * machine) and differ between tasks (so unrelated prefixes do not compete for one + * routing bucket). `metadata.taskId` is the longest-lived identifier the handler + * receives: it is the persisted task id, stable across turns, restarts and + * rehydration, whereas `this.sessionId` is regenerated per handler instance. + * The handler session id is the fallback for callers that send no task id. + */ + private getPromptCacheKey( + model: OpenAiNativeModel, + metadata?: ApiHandlerCreateMessageMetadata, + ): string | undefined { + if (!model.info.supportsPromptCache) return undefined + + const key = metadata?.taskId || this.sessionId + if (!key) return undefined + + // Hash rather than slice: a 64-char prefix drops the distinguishing tail, so two + // distinct task ids that share a long prefix would land in one routing bucket. + // sha256 hex is 64 characters, so the result fits the cap without truncation. + return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? createHash("sha256").update(key).digest("hex") : key + } + /** * Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available. * If no tier or no overrides exist, the original ModelInfo is returned. @@ -1436,6 +1497,17 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio outputPrice: tierInfo.outputPrice ?? info.outputPrice, cacheReadsPrice: tierInfo.cacheReadsPrice ?? info.cacheReadsPrice, cacheWritesPrice: tierInfo.cacheWritesPrice ?? info.cacheWritesPrice, + // A service tier that overrides the write rate must also override the TTL + // rate, otherwise a flex/priority 12h write bills at the standard-tier 12h + // rate. Derived from the tier's own input price so the 2.5x ratio holds. + cacheWritesPrice12h: + tierInfo.cacheWritesPrice12h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_12H : undefined) ?? + info.cacheWritesPrice12h, + cacheWritesPrice1h: + tierInfo.cacheWritesPrice1h ?? + (tierInfo.inputPrice !== undefined ? tierInfo.inputPrice * CACHE_WRITE_MULTIPLIER_1H : undefined) ?? + info.cacheWritesPrice1h, } } @@ -1499,7 +1571,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio getResponseId(): string | undefined { return this.lastResponseId } - async completePrompt(prompt: string, _options?: CompletePromptOptions): Promise { + async completePrompt(prompt: string, options?: CompletePromptOptions): Promise { try { this.abortController = new AbortController() @@ -1530,6 +1602,17 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio requestBody[SERVICE_TIER_KEY] = serviceTier } + // Cache routing affinity key. A caller that knows the task passes it, so repeated + // completions for one task share a bucket across handler instances; otherwise the + // handler session id scopes the key to this instance. + const promptCacheKey = this.getPromptCacheKey( + model, + options?.taskId ? { taskId: options.taskId } : undefined, + ) + if (promptCacheKey) { + requestBody.prompt_cache_key = promptCacheKey + } + // Add reasoning if supported if (reasoningEffort) { requestBody.reasoning = { diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 36e09aa883..6d2057a7a3 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -26,6 +26,7 @@ import { ApiStreamChunk } from "../transform/stream" import { convertToR1Format } from "../transform/r1-format" import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic" import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { OpenRouterReasoningParams } from "../transform/reasoning" import { getModelParams } from "../transform/model-params" @@ -306,9 +307,12 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH // TODO: Add a `promptCacheStratey` field to `ModelInfo`. if (OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId)) { if (modelId.startsWith("google")) { + // Gemini markers keep their default TTL: ADR-0006 scopes the knob to + // Anthropic-shaped markers only, and Gemini has no 1h class. addGeminiCacheBreakpoints(systemPrompt, openAiMessages) } else { - addAnthropicCacheBreakpoints(systemPrompt, openAiMessages) + // Capability comes from the resolved ModelInfo, never from the id. + addAnthropicCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, model.info)) } } diff --git a/src/api/providers/vercel-ai-gateway.ts b/src/api/providers/vercel-ai-gateway.ts index 2f1586b928..135ac8bef6 100644 --- a/src/api/providers/vercel-ai-gateway.ts +++ b/src/api/providers/vercel-ai-gateway.ts @@ -17,6 +17,7 @@ import { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -82,7 +83,7 @@ export class VercelAiGatewayHandler extends RouterProvider implements SingleComp ] if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } const supportsTemperature = info.supportsTemperature !== false && this.supportsTemperature(modelId) diff --git a/src/api/providers/zoo-gateway.ts b/src/api/providers/zoo-gateway.ts index 66ccac5067..5f33ee48fe 100644 --- a/src/api/providers/zoo-gateway.ts +++ b/src/api/providers/zoo-gateway.ts @@ -19,6 +19,7 @@ import { t } from "../../i18n" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" +import { resolveCacheTtl } from "../transform/caching/ttl" import type { SingleCompletionHandler, @@ -208,7 +209,7 @@ export class ZooGatewayHandler extends RouterProvider implements SingleCompletio // Apply prompt caching for models that support it // Zoo Gateway serves the same models as Vercel AI Gateway, so caching support is identical if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { - addCacheBreakpoints(systemPrompt, openAiMessages) + addCacheBreakpoints(systemPrompt, openAiMessages, resolveCacheTtl(this.options, info)) } // Build request headers with enrichment metadata diff --git a/src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts b/src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts new file mode 100644 index 0000000000..e295d9ef6a --- /dev/null +++ b/src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts @@ -0,0 +1,121 @@ +// npx vitest run src/api/transform/cache-strategy/__tests__/cache-strategy-ttl.spec.ts + +import type { ContentBlock, SystemContentBlock } from "@aws-sdk/client-bedrock-runtime" + +import { MultiPointStrategy } from "../multi-point-strategy" +import type { CacheStrategyConfig, ModelInfo } from "../types" + +/** + * TTL threading through the cache strategy (ADR-0006 PR-C7). + * + * The strategy is the single owner of the cache-point shape, so these tests pin the + * two properties the provider relies on: a configured TTL reaches every cache point + * it emits, and an unset TTL leaves the block byte-identical to a pre-knob build. + */ + +const modelInfo: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + maxCachePoints: 4, + minTokensPerCachePoint: 50, + cachableFields: ["system", "messages", "tools"], +} + +// The strategy estimates tokens by counting whitespace-separated words, so the +// fixtures need real words rather than one long run of characters. +const longText = (tokens: number) => "word ".repeat(tokens) + +const createConfig = (overrides: Partial = {}): CacheStrategyConfig => ({ + modelInfo, + systemPrompt: longText(200), + messages: [ + { role: "user", content: longText(200) }, + { role: "assistant", content: longText(200) }, + { role: "user", content: longText(200) }, + ], + usePromptCache: true, + ...overrides, +}) + +const collectCachePoints = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cachePoint" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) + +describe("MultiPointStrategy cache TTL", () => { + it("carries the configured TTL on the system and message cache points", () => { + const strategy = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })) + const result = strategy.determineOptimalCachePoints() + + const cachePoints = collectCachePoints(serialize({ system: result.system, messages: result.messages })) + + expect(collectCachePoints(serialize(result.system)).length).toBeGreaterThan(0) + expect(collectCachePoints(serialize(result.messages)).length).toBeGreaterThan(0) + for (const cachePoint of cachePoints) { + expect(cachePoint).toEqual({ type: "default", ttl: "1h" }) + } + }) + + it("omits the ttl key entirely when no TTL is configured", () => { + const strategy = new MultiPointStrategy(createConfig()) + const result = strategy.determineOptimalCachePoints() + + const cachePoints = collectCachePoints(serialize({ system: result.system, messages: result.messages })) + + expect(cachePoints.length).toBeGreaterThan(0) + for (const cachePoint of cachePoints) { + // Deep-equality against the pre-knob literal, not just "no ttl": a stray + // extra key would change the request bytes just as surely. + expect(cachePoint).toEqual({ type: "default" }) + } + }) + + it("places the same cache points with and without a TTL", () => { + const withoutTtl = new MultiPointStrategy(createConfig()).determineOptimalCachePoints() + const withTtl = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })).determineOptimalCachePoints() + + expect(withTtl.messageCachePointPlacements).toEqual(withoutTtl.messageCachePointPlacements) + expect(collectCachePoints(serialize(withTtl.system)).length).toBe( + collectCachePoints(serialize(withoutTtl.system)).length, + ) + }) + + it("keeps createCachePoint public so the provider's second emitter can reuse it", () => { + // The provider's message-level emitter used to build its own literal, which is + // how two emitters drift apart. Asserting the method is reachable from outside + // the class is the cheap structural guard against it going back to private. + const strategy = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })) + const block = strategy.createCachePoint() as unknown as { cachePoint: Record } + + expect(block.cachePoint).toEqual({ type: "default", ttl: "1h" }) + expect(typeof (block as unknown as ContentBlock)).toBe("object") + }) + + it("returns an untyped-compatible block for the system union as well", () => { + const strategy = new MultiPointStrategy(createConfig({ cacheTtl: "1h" })) + const block = strategy.createCachePoint() as unknown as SystemContentBlock + + expect(collectCachePoints(serialize(block))).toEqual([{ type: "default", ttl: "1h" }]) + }) +}) diff --git a/src/api/transform/cache-strategy/base-strategy.ts b/src/api/transform/cache-strategy/base-strategy.ts index 1bc05cdb84..740940d805 100644 --- a/src/api/transform/cache-strategy/base-strategy.ts +++ b/src/api/transform/cache-strategy/base-strategy.ts @@ -48,10 +48,21 @@ export abstract class CacheStrategy { } /** - * Create a cache point content block + * Create a cache point content block. + * + * `ttl` is omitted, not set to "5m", when no TTL was resolved: AWS documents + * omitting the field as the way to get the default 5-minute behaviour, and an + * explicit "5m" would change the bytes of every existing request. + * + * Public so the provider's message-level emitter can reuse it instead of + * building its own literal: one factory means one TTL per request (I4) and + * makes a mixed-TTL request unrepresentable rather than merely unlikely. */ - protected createCachePoint(): ContentBlock { - return { cachePoint: { type: "default" } } as unknown as ContentBlock + public createCachePoint(): ContentBlock { + const { cacheTtl } = this.config + return { + cachePoint: cacheTtl ? { type: "default", ttl: cacheTtl } : { type: "default" }, + } as ContentBlock } /** diff --git a/src/api/transform/cache-strategy/types.ts b/src/api/transform/cache-strategy/types.ts index 4215295d6a..94a02f9c44 100644 --- a/src/api/transform/cache-strategy/types.ts +++ b/src/api/transform/cache-strategy/types.ts @@ -1,5 +1,6 @@ import { Anthropic } from "@anthropic-ai/sdk" import { SystemContentBlock, Message } from "@aws-sdk/client-bedrock-runtime" +import type { PromptCacheTtl } from "@roo-code/types" /** * Information about a model's capabilities and constraints @@ -57,4 +58,12 @@ export interface CacheStrategyConfig { usePromptCache: boolean /** Previous cache point placements (for maintaining consistency across consecutive messages) */ previousCachePointPlacements?: CachePointPlacement[] + /** + * Cache TTL every cache point in this request carries, already resolved against + * the model's catalog capability. One value per request is the point: Bedrock + * Converse rejects a shorter TTL appearing before a longer one across + * tools -> system -> messages, so a per-point TTL would be a 400 waiting for the + * first mixed placement. Leaving it unset omits `ttl` from the wire entirely. + */ + cacheTtl?: PromptCacheTtl } diff --git a/src/api/transform/caching/__tests__/ttl.spec.ts b/src/api/transform/caching/__tests__/ttl.spec.ts new file mode 100644 index 0000000000..7c4253320c --- /dev/null +++ b/src/api/transform/caching/__tests__/ttl.spec.ts @@ -0,0 +1,185 @@ +// npx vitest run src/api/transform/caching/__tests__/ttl.spec.ts + +import OpenAI from "openai" + +import { addCacheBreakpoints as addAnthropicBreakpoints } from "../anthropic" +import { addCacheBreakpoints as addVertexBreakpoints } from "../vertex" +import { addCacheBreakpoints as addGatewayBreakpoints } from "../vercel-ai-gateway" +import { cacheControl, resolveCacheTtl } from "../ttl" + +/** + * Collect every `cache_control` object reachable in a serialized request. + * + * Counting occurrences rather than spot-checking one marker is the point: a + * transform that forgot one of its marker sites would still pass a single-site + * assertion, which is exactly the failure mode ADR-0006 I4 is meant to exclude. + */ +const collectCacheControls = (value: unknown): Array> => { + const found: Array> = [] + + const walk = (node: unknown) => { + if (Array.isArray(node)) { + node.forEach(walk) + return + } + + if (node && typeof node === "object") { + for (const [key, child] of Object.entries(node as Record)) { + if (key === "cache_control" && child && typeof child === "object") { + found.push(child as Record) + } + walk(child) + } + } + } + + walk(value) + return found +} + +/** The serialized request body, so assertions run on the bytes that would be sent. */ +const serialize = (value: unknown) => JSON.parse(JSON.stringify(value)) + +describe("resolveCacheTtl", () => { + it("returns undefined when the knob is unset, even for a capable model", () => { + expect(resolveCacheTtl({}, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + expect(resolveCacheTtl(undefined, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns undefined for an explicit 5m so the wire shape stays unchanged", () => { + expect(resolveCacheTtl({ promptCacheTtl: "5m" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBeUndefined() + }) + + it("returns 1h when the knob asks for it and the model advertises it", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m", "1h"] })).toBe("1h") + }) + + it("fails closed when the model advertises no capability at all", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, {})).toBeUndefined() + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, undefined)).toBeUndefined() + }) + + it("fails closed when the model advertises 5m only", () => { + expect(resolveCacheTtl({ promptCacheTtl: "1h" }, { promptCacheTtlSupport: ["5m"] })).toBeUndefined() + }) +}) + +describe("cacheControl", () => { + it("is the historical literal when no TTL is resolved", () => { + expect(cacheControl(undefined)).toEqual({ type: "ephemeral" }) + expect(Object.keys(cacheControl(undefined))).toEqual(["type"]) + }) + + it("carries the ttl key only when a TTL is resolved", () => { + expect(cacheControl("1h")).toEqual({ type: "ephemeral", ttl: "1h" }) + }) +}) + +describe("caching transforms under a prompt cache TTL", () => { + const systemPrompt = "You are a helpful assistant." + + const buildOpenAiMessages = (): OpenAI.Chat.ChatCompletionMessageParam[] => [ + { role: "system", content: systemPrompt }, + { role: "user", content: "First message" }, + { role: "assistant", content: "First response" }, + { role: "user", content: "Second message" }, + ] + + const buildAnthropicMessages = () => [ + { role: "user" as const, content: "First message" }, + { role: "assistant" as const, content: "First response" }, + { role: "user" as const, content: "Second message" }, + ] + + describe("default parity (ADR-0006 I3)", () => { + it("emits no ttl key anywhere when the TTL argument is omitted", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages) + + const serialized = serialize(messages) + const rawJson = JSON.stringify(messages) + + expect(rawJson).not.toContain("ttl") + expect(serialized).not.toContain("ttl") + + const controls = collectCacheControls(serialized) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === undefined)).toBe(true) + }) + + it("is byte-identical to the pre-knob literal for every transform", () => { + const anthropicMessages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, anthropicMessages) + + const gatewayMessages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, gatewayMessages) + + const vertexMessages = addVertexBreakpoints(buildAnthropicMessages()) + + // Compare the raw serialized bytes, not parsed objects: `toEqual` ignores key + // insertion order, so a transform that reorders request keys would pass while + // the bytes on the wire changed. The expected strings are literals for the + // same reason: building them from the same helper would re-derive the order. + expect(JSON.stringify(anthropicMessages)).toBe( + '[{"role":"system","content":[{"type":"text","text":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}}]},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', + ) + + expect(JSON.stringify(gatewayMessages)).toBe( + '[{"role":"system","content":"You are a helpful assistant.","cache_control":{"type":"ephemeral"}},{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', + ) + + expect(JSON.stringify(vertexMessages)).toBe( + '[{"role":"user","content":[{"type":"text","text":"First message","cache_control":{"type":"ephemeral"}}]},{"role":"assistant","content":"First response"},{"role":"user","content":[{"type":"text","text":"Second message","cache_control":{"type":"ephemeral"}}]}]', + ) + }) + }) + + describe("opt-in uniformity (ADR-0006 I4)", () => { + it("puts ttl 1h on every anthropic marker, not just the first", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + expect(controls.every((control) => control.type === "ephemeral")).toBe(true) + }) + + it("puts ttl 1h on every gateway marker", () => { + const messages = buildOpenAiMessages() + addGatewayBreakpoints(systemPrompt, messages, "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(3) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("puts ttl 1h on every vertex marker", () => { + const messages = addVertexBreakpoints(buildAnthropicMessages(), "1h") + + const controls = collectCacheControls(serialize(messages)) + expect(controls).toHaveLength(2) + expect(controls.every((control) => control.ttl === "1h")).toBe(true) + }) + + it("never mixes TTLs within one request", () => { + const messages = buildOpenAiMessages() + addAnthropicBreakpoints(systemPrompt, messages, "1h") + + const ttls = new Set(collectCacheControls(serialize(messages)).map((control) => control.ttl)) + expect(ttls).toEqual(new Set(["1h"])) + }) + }) + + describe("signature is additive-optional", () => { + it("keeps the two-argument call compiling and byte-identical", () => { + const withDefault = buildOpenAiMessages() + const withExplicitUndefined = buildOpenAiMessages() + + addAnthropicBreakpoints(systemPrompt, withDefault) + addAnthropicBreakpoints(systemPrompt, withExplicitUndefined, undefined) + + expect(serialize(withDefault)).toEqual(serialize(withExplicitUndefined)) + }) + }) +}) diff --git a/src/api/transform/caching/anthropic.ts b/src/api/transform/caching/anthropic.ts index cff671a56c..3bdcba4ba1 100644 --- a/src/api/transform/caching/anthropic.ts +++ b/src/api/transform/caching/anthropic.ts @@ -1,10 +1,20 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + messages[0] = { role: "system", // @ts-ignore-next-line - content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }], + content: [{ type: "text", text: systemPrompt, cache_control: control }], } // Ensure all user messages have content in array format first @@ -35,7 +45,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. } // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } }) } diff --git a/src/api/transform/caching/ttl.ts b/src/api/transform/caching/ttl.ts new file mode 100644 index 0000000000..4ba2652fe2 --- /dev/null +++ b/src/api/transform/caching/ttl.ts @@ -0,0 +1,32 @@ +import type { ModelInfo, PromptCacheTtl } from "@roo-code/types" +import type { CacheControlEphemeral } from "@anthropic-ai/sdk/resources" + +/** + * Resolve the TTL a request may emit, gated by the provider-profile knob and the + * model's catalog capability. Both gates fail closed: absent capability means + * "5m only" (ADR-0006 I2), and capability is read from the resolved `ModelInfo` + * rather than inferred from the model id. + * + * `"5m"` resolves to `undefined` on purpose. It is the vendor default, so emitting + * it would add bytes without changing behavior; only `"1h"` may change the wire, + * which keeps default parity (I3) checkable by a single grep. + */ +export function resolveCacheTtl( + settings: { promptCacheTtl?: PromptCacheTtl } | undefined, + info: Pick | undefined, +): PromptCacheTtl | undefined { + if (settings?.promptCacheTtl !== "1h") { + return undefined + } + + return info?.promptCacheTtlSupport?.includes("1h") ? "1h" : undefined +} + +/** + * Build the `cache_control` object for one marker. Every marker in a request goes + * through this function, which makes one uniform TTL per request (I4) structural + * instead of a convention each call site has to remember. + */ +export function cacheControl(ttl: PromptCacheTtl | undefined): CacheControlEphemeral { + return ttl ? { type: "ephemeral", ttl } : { type: "ephemeral" } +} diff --git a/src/api/transform/caching/vercel-ai-gateway.ts b/src/api/transform/caching/vercel-ai-gateway.ts index 0d3834bbe5..7272de34d7 100644 --- a/src/api/transform/caching/vercel-ai-gateway.ts +++ b/src/api/transform/caching/vercel-ai-gateway.ts @@ -1,12 +1,22 @@ import OpenAI from "openai" -export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + systemPrompt: string, + messages: OpenAI.Chat.ChatCompletionMessageParam[], + ttl?: PromptCacheTtl, +) { + const control = cacheControl(ttl) + // Apply cache_control to system message at the message level messages[0] = { role: "system", content: systemPrompt, // @ts-ignore-next-line - cache_control: { type: "ephemeral" }, + cache_control: control, } // Add cache_control to the last two user messages for conversation context caching @@ -23,7 +33,7 @@ export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat. if (lastTextPart && lastTextPart.text && lastTextPart.text.length > 0) { // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } + lastTextPart["cache_control"] = control } } }) diff --git a/src/api/transform/caching/vertex.ts b/src/api/transform/caching/vertex.ts index 48bf261587..e223bb4d11 100644 --- a/src/api/transform/caching/vertex.ts +++ b/src/api/transform/caching/vertex.ts @@ -1,6 +1,13 @@ import { Anthropic } from "@anthropic-ai/sdk" -export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) { +import type { PromptCacheTtl } from "@roo-code/types" + +import { cacheControl } from "./ttl" + +export function addCacheBreakpoints( + messages: Anthropic.Messages.MessageParam[], + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam[] { // Find indices of user messages that we want to cache. // We only cache the last two user messages to stay within the 4-block limit // (1 block for system + 1 block each for last two user messages = 3 total). @@ -12,18 +19,23 @@ export function addCacheBreakpoints(messages: Anthropic.Messages.MessageParam[]) return messages.map((message, index) => message.role !== "assistant" && (index === lastIndex || index === secondLastIndex) - ? cachedMessage(message) + ? cachedMessage(message, ttl) : message, ) } -function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Messages.MessageParam { +function cachedMessage( + message: Anthropic.Messages.MessageParam, + ttl?: PromptCacheTtl, +): Anthropic.Messages.MessageParam { + const control = cacheControl(ttl) + // For string content, we convert to array format with optional cache control. if (typeof message.content === "string") { return { ...message, // For string content, we only have one block so it's always the last block. - content: [{ type: "text" as const, text: message.content, cache_control: { type: "ephemeral" } }], + content: [{ type: "text" as const, text: message.content, cache_control: control }], } } @@ -41,7 +53,7 @@ function cachedMessage(message: Anthropic.Messages.MessageParam): Anthropic.Mess ? { ...content, // Check if this is the last text block using our pre-calculated index. - ...(index === lastTextBlockIndex && { cache_control: { type: "ephemeral" } }), + ...(index === lastTextBlockIndex && { cache_control: control }), } : content, ), diff --git a/src/api/transform/stream.ts b/src/api/transform/stream.ts index 40efeb3bb0..a3d4b533e7 100644 --- a/src/api/transform/stream.ts +++ b/src/api/transform/stream.ts @@ -59,7 +59,18 @@ export interface ApiStreamUsageChunk { type: "usage" inputTokens: number outputTokens: number + /** Total cache-write tokens, across every TTL class. */ cacheWriteTokens?: number + /** + * The 1h-TTL subset of `cacheWriteTokens` (Anthropic `ephemeral_1h_input_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite1hTokens?: number + /** + * The 12h-TTL subset of `cacheWriteTokens` (OpenAI `ttl_12h_tokens`). + * This is a decomposition, not an addition: `cacheWriteTokens` already includes it. + */ + cacheWrite12hTokens?: number cacheReadTokens?: number reasoningTokens?: number totalCost?: number diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 3202358c09..8a904e94d6 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -4021,6 +4021,8 @@ export class Task extends EventEmitter implements TaskLike { try { let cacheWriteTokens = 0 + let cacheWrite1hTokens = 0 + let cacheWrite12hTokens = 0 let cacheReadTokens = 0 let inputTokens = 0 let outputTokens = 0 @@ -4056,6 +4058,8 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + cacheWrite1hTokens, + cacheWrite12hTokens, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4063,6 +4067,9 @@ export class Task extends EventEmitter implements TaskLike { outputTokens, cacheWriteTokens, cacheReadTokens, + undefined, + cacheWrite1hTokens, + cacheWrite12hTokens, ) this.clineMessages[lastApiReqIndex].text = JSON.stringify({ @@ -4210,6 +4217,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens += chunk.inputTokens outputTokens += chunk.outputTokens cacheWriteTokens += chunk.cacheWriteTokens ?? 0 + cacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + cacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 cacheReadTokens += chunk.cacheReadTokens ?? 0 totalCost = chunk.totalCost break @@ -4450,6 +4459,8 @@ export class Task extends EventEmitter implements TaskLike { input: inputTokens, output: outputTokens, cacheWrite: cacheWriteTokens, + cacheWrite1h: cacheWrite1hTokens, + cacheWrite12h: cacheWrite12hTokens, cacheRead: cacheReadTokens, total: totalCost, } @@ -4466,6 +4477,8 @@ export class Task extends EventEmitter implements TaskLike { let bgInputTokens = currentTokens.input let bgOutputTokens = currentTokens.output let bgCacheWriteTokens = currentTokens.cacheWrite + let bgCacheWrite1hTokens = currentTokens.cacheWrite1h + let bgCacheWrite12hTokens = currentTokens.cacheWrite12h let bgCacheReadTokens = currentTokens.cacheRead let bgTotalCost = currentTokens.total @@ -4475,6 +4488,8 @@ export class Task extends EventEmitter implements TaskLike { input: number output: number cacheWrite: number + cacheWrite1h: number + cacheWrite12h: number cacheRead: number total?: number }, @@ -4491,6 +4506,8 @@ export class Task extends EventEmitter implements TaskLike { inputTokens = tokens.input outputTokens = tokens.output cacheWriteTokens = tokens.cacheWrite + cacheWrite1hTokens = tokens.cacheWrite1h + cacheWrite12hTokens = tokens.cacheWrite12h cacheReadTokens = tokens.cacheRead totalCost = tokens.total @@ -4512,7 +4529,10 @@ export class Task extends EventEmitter implements TaskLike { modelId, ) - // Use the appropriate cost function based on the API protocol + // Use the appropriate cost function based on the API protocol. + // The per-TTL split must be passed here too: without it this + // telemetry recompute bills every write at the short-TTL rate, + // which is exactly the under-report this PR exists to fix. const costResult = apiProtocol === "anthropic" ? calculateApiCostAnthropic( @@ -4521,6 +4541,8 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) : calculateApiCostOpenAI( streamModelInfo, @@ -4528,6 +4550,9 @@ export class Task extends EventEmitter implements TaskLike { tokens.output, tokens.cacheWrite, tokens.cacheRead, + undefined, + tokens.cacheWrite1h, + tokens.cacheWrite12h, ) TelemetryService.instance.captureLlmCompletion(this.taskId, { @@ -4568,6 +4593,8 @@ export class Task extends EventEmitter implements TaskLike { bgInputTokens += chunk.inputTokens bgOutputTokens += chunk.outputTokens bgCacheWriteTokens += chunk.cacheWriteTokens ?? 0 + bgCacheWrite1hTokens += chunk.cacheWrite1hTokens ?? 0 + bgCacheWrite12hTokens += chunk.cacheWrite12hTokens ?? 0 bgCacheReadTokens += chunk.cacheReadTokens ?? 0 bgTotalCost = chunk.totalCost } @@ -4586,6 +4613,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, @@ -4611,6 +4640,8 @@ export class Task extends EventEmitter implements TaskLike { input: bgInputTokens, output: bgOutputTokens, cacheWrite: bgCacheWriteTokens, + cacheWrite1h: bgCacheWrite1hTokens, + cacheWrite12h: bgCacheWrite12hTokens, cacheRead: bgCacheReadTokens, total: bgTotalCost, }, diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index bbf77a85e9..dab89538fa 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -534,6 +534,59 @@ describe("Cline", () => { }) }) + describe("cache-write TTL split in telemetry", () => { + it("bills a 1h cache write at the 1h rate when usage is captured after a cancelled stream", async () => { + // glm P1: the telemetry cost recompute inside captureUsageData must receive + // the per-TTL split. Before the fix it passed only the aggregate write count, + // so a 1h write was billed at the 5m rate (0.375 instead of 0.6 for 100k + // tokens at claude-3-5-sonnet's 3.0 input price). + const task = await createTaskWithManualRetries() + const captureLlmCompletion = vi.spyOn(TelemetryService.instance, "captureLlmCompletion") + + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) + // A stream that reports the split but never yields a final cost chunk, which is + // the shape a cancelled request leaves behind. The second call throws so the + // request loop terminates the way the neighbouring tests do. + vi.spyOn(task, "attemptApiRequest") + .mockImplementationOnce(() => + stream([ + { + type: "usage", + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 100_000, + cacheWrite1hTokens: 100_000, + cacheReadTokens: 0, + }, + { type: "text", text: "partial response" }, + ]), + ) + .mockImplementation(() => { + throw new Error("stop after usage capture") + }) + + await task.recursivelyMakeClineRequests([{ type: "text", text: "original user request" }]) + + // The drain is fire-and-forget and reaches `captureLlmCompletion` after several + // awaits, some of which touch the filesystem. A fixed sleep races that work on a + // loaded worker, so poll for the captured call instead. + await vi.waitFor(() => + expect( + captureLlmCompletion.mock.calls.some(([, payload]) => payload?.cacheWriteTokens === 100_000), + ).toBe(true), + ) + + const call = captureLlmCompletion.mock.calls.find(([, payload]) => payload?.cacheWriteTokens === 100_000) + expect(call).toBeDefined() + + // claude-3-5-sonnet-20241022: input 3.0, cacheWritesPrice 3.75, cacheWritesPrice1h 6.0. + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-fix lumped behaviour billed (3.75 / 1_000_000) * 100_000 = 0.375. + expect(call![1].cost).toBeCloseTo(0.6, 9) + expect(call![1].cost).not.toBeCloseTo(0.375, 9) + }) + }) + describe("request continuation", () => { it("sends the no-tool-use reminder in a second request when the first response has no tool use", async () => { const task = await createTaskWithManualRetries() diff --git a/src/core/webview/__tests__/messageEnhancer.test.ts b/src/core/webview/__tests__/messageEnhancer.test.ts index 964c4ebdef..6bf16b66e8 100644 --- a/src/core/webview/__tests__/messageEnhancer.test.ts +++ b/src/core/webview/__tests__/messageEnhancer.test.ts @@ -77,6 +77,7 @@ describe("MessageEnhancer", () => { expect(mockSingleCompletionHandler).toHaveBeenCalledWith( mockApiConfiguration, expect.stringContaining("Write a function to calculate fibonacci"), + undefined, ) }) @@ -99,7 +100,7 @@ describe("MessageEnhancer", () => { apiKey: "enhancement-key", apiModelId: "claude-3", } - expect(mockSingleCompletionHandler).toHaveBeenCalledWith(expectedConfig, expect.any(String)) + expect(mockSingleCompletionHandler).toHaveBeenCalledWith(expectedConfig, expect.any(String), undefined) }) it("should include task history when enabled", async () => { @@ -238,7 +239,40 @@ describe("MessageEnhancer", () => { }) // Should use the default config - expect(mockSingleCompletionHandler).toHaveBeenCalledWith(mockApiConfiguration, expect.any(String)) + expect(mockSingleCompletionHandler).toHaveBeenCalledWith( + mockApiConfiguration, + expect.any(String), + undefined, + ) + }) + + it("passes the task id through to the completion so the cache key stays task-scoped", async () => { + await MessageEnhancer.enhanceMessage({ + text: "Test", + apiConfiguration: mockApiConfiguration, + listApiConfigMeta: mockListApiConfigMeta, + providerSettingsManager: mockProviderSettingsManager, + taskId: "task-123", + }) + + expect(mockSingleCompletionHandler).toHaveBeenCalledWith(mockApiConfiguration, expect.any(String), { + taskId: "task-123", + }) + }) + + it("omits the completion options when no task is open", async () => { + await MessageEnhancer.enhanceMessage({ + text: "Test", + apiConfiguration: mockApiConfiguration, + listApiConfigMeta: mockListApiConfigMeta, + providerSettingsManager: mockProviderSettingsManager, + }) + + expect(mockSingleCompletionHandler).toHaveBeenCalledWith( + mockApiConfiguration, + expect.any(String), + undefined, + ) }) it("should handle empty task history gracefully", async () => { diff --git a/src/core/webview/messageEnhancer.ts b/src/core/webview/messageEnhancer.ts index 90c1f3b37d..6994e57291 100644 --- a/src/core/webview/messageEnhancer.ts +++ b/src/core/webview/messageEnhancer.ts @@ -13,6 +13,12 @@ export interface MessageEnhancerOptions { includeTaskHistoryInEnhance?: boolean currentClineMessages?: ClineMessage[] providerSettingsManager: ProviderSettingsManager + /** + * Task the enhancement belongs to, when one is open. Passed through to the provider so + * repeated enhancements of one task reuse a single prompt-cache bucket instead of + * starting a new one per handler instance. + */ + taskId?: string } export interface MessageEnhancerResult { @@ -41,6 +47,7 @@ export class MessageEnhancer { includeTaskHistoryInEnhance, currentClineMessages, providerSettingsManager, + taskId, } = options // Determine which API configuration to use @@ -76,7 +83,11 @@ export class MessageEnhancer { ) // Call the single completion handler to get the enhanced prompt - const enhancedText = await singleCompletionHandler(configToUse, enhancementPrompt) + const enhancedText = await singleCompletionHandler( + configToUse, + enhancementPrompt, + taskId ? { taskId } : undefined, + ) return { success: true, diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index bd7c589958..5da8019187 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -2252,6 +2252,7 @@ export const webviewMessageHandler = async ( includeTaskHistoryInEnhance, currentClineMessages: currentCline?.clineMessages, providerSettingsManager: provider.providerSettingsManager, + taskId: currentCline?.taskId, }) if (result.success && result.enhancedText) { diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 754a2d84df..bc1f2f64d8 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -1,4 +1,10 @@ -import { OpenAiServiceTier, type ModelInfo, type ServiceTier } from "@roo-code/types" +import { + CACHE_WRITE_MULTIPLIER_12H, + CACHE_WRITE_MULTIPLIER_1H, + OpenAiServiceTier, + type ModelInfo, + type ServiceTier, +} from "@roo-code/types" export interface ApiCostResult { totalInputTokens: number @@ -6,6 +12,24 @@ export interface ApiCostResult { totalCost: number } +/** + * Resolve the per-million rate for a TTL-specific cache-write class. + * + * Precedence is fail-closed: an explicit catalog rate wins, then the vendor + * multiplier applied to the base input price, and only then the aggregate + * `cacheWritesPrice`. The last step is the pre-split behaviour, reachable only + * when a model has neither a 1h/12h rate nor an input price to derive one from. + */ +function resolveCacheWriteRate( + modelInfo: ModelInfo, + explicit: number | undefined, + multiplier: number, +): number | undefined { + if (explicit !== undefined) return explicit + if (modelInfo.inputPrice !== undefined) return modelInfo.inputPrice * multiplier + return modelInfo.cacheWritesPrice +} + function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, serviceTier?: ServiceTier): ModelInfo { const pricing = modelInfo.longContextPricing if (!pricing || totalInputTokens <= pricing.thresholdTokens) { @@ -31,6 +55,16 @@ function applyLongContextPricing(modelInfo: ModelInfo, totalInputTokens: number, modelInfo.cacheWritesPrice !== undefined && pricing.cacheWritesPriceMultiplier !== undefined ? modelInfo.cacheWritesPrice * pricing.cacheWritesPriceMultiplier : modelInfo.cacheWritesPrice, + // The TTL-specific write rates scale with the same multiplier as the aggregate + // write rate; leaving them unscaled would under-bill long-context 1h writes. + cacheWritesPrice1h: + modelInfo.cacheWritesPrice1h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice1h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice1h, + cacheWritesPrice12h: + modelInfo.cacheWritesPrice12h !== undefined && pricing.cacheWritesPriceMultiplier !== undefined + ? modelInfo.cacheWritesPrice12h * pricing.cacheWritesPriceMultiplier + : modelInfo.cacheWritesPrice12h, cacheReadsPrice: modelInfo.cacheReadsPrice !== undefined && pricing.cacheReadsPriceMultiplier !== undefined ? modelInfo.cacheReadsPrice * pricing.cacheReadsPriceMultiplier @@ -46,8 +80,28 @@ function calculateApiCostInternal( cacheReadInputTokens: number, totalInputTokens: number, totalOutputTokens: number, + cacheCreation1hInputTokens = 0, + cacheCreation12hInputTokens = 0, ): ApiCostResult { - const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens + // The per-TTL counts are a decomposition of `cacheCreationInputTokens`, never + // an addition to it. Deriving the 5m remainder this way keeps the aggregate + // path byte-identical when a vendor reports no split (both terms are 0). + // Each subset also caps at what the aggregate has left, so a malformed split + // (1h + 12h > total) cannot bill more writes than the vendor reported. + const writesTotal = Math.max(0, cacheCreationInputTokens) + const writes1h = Math.min(Math.max(0, cacheCreation1hInputTokens), writesTotal) + const writes12h = Math.min(Math.max(0, cacheCreation12hInputTokens), writesTotal - writes1h) + const writes5m = Math.max(0, writesTotal - writes1h - writes12h) + + const writes5mRate = modelInfo.cacheWritesPrice || 0 + const writes1hRate = resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice1h, CACHE_WRITE_MULTIPLIER_1H) || 0 + const writes12hRate = + resolveCacheWriteRate(modelInfo, modelInfo.cacheWritesPrice12h, CACHE_WRITE_MULTIPLIER_12H) || 0 + + const cacheWritesCost = + (writes5mRate / 1_000_000) * writes5m + + (writes1hRate / 1_000_000) * writes1h + + (writes12hRate / 1_000_000) * writes12h const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens const outputCost = ((modelInfo.outputPrice || 0) / 1_000_000) * outputTokens @@ -68,6 +122,8 @@ export function calculateApiCostAnthropic( outputTokens: number, cacheCreationInputTokens?: number, cacheReadInputTokens?: number, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreation = cacheCreationInputTokens || 0 const cacheRead = cacheReadInputTokens || 0 @@ -84,6 +140,8 @@ export function calculateApiCostAnthropic( cacheRead, totalInputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } @@ -95,6 +153,8 @@ export function calculateApiCostOpenAI( cacheCreationInputTokens?: number, cacheReadInputTokens?: number, serviceTier?: ServiceTier, + cacheCreation1hInputTokens?: number, + cacheCreation12hInputTokens?: number, ): ApiCostResult { const cacheCreationInputTokensNum = cacheCreationInputTokens || 0 const cacheReadInputTokensNum = cacheReadInputTokens || 0 @@ -111,6 +171,8 @@ export function calculateApiCostOpenAI( cacheReadInputTokensNum, inputTokens, outputTokens, + cacheCreation1hInputTokens, + cacheCreation12hInputTokens, ) } diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b369a0405b..6782de5e6f 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -303,4 +303,291 @@ describe("Cost Utility", () => { expect(result.totalCost).toBeCloseTo(1.08, 6) }) }) + + describe("cache-write TTL split accounting", () => { + const anthropicModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, // 1.25x input + cacheWritesPrice1h: 6.0, // 2x input + cacheReadsPrice: 0.3, + } + + const openAiModel: ModelInfo = { + maxTokens: 128_000, + contextWindow: 1_050_000, + supportsPromptCache: true, + inputPrice: 4.0, + outputPrice: 20.0, + cacheWritesPrice: 5.0, // 1.25x input (30m class) + cacheWritesPrice12h: 10.0, // 2.5x input + cacheReadsPrice: 0.4, + } + + it("bills 100k 1h tokens at the 1h rate, not the 5m rate (37.5% tripwire)", () => { + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 100_000) + + // 1h writes: (6.0 / 1_000_000) * 100_000 = 0.6 + // The pre-split behaviour would have charged the 5m rate: + // (3.75 / 1_000_000) * 100_000 = 0.375, under-reporting by 37.5%. + expect(result.totalCost).toBeCloseTo(0.6, 9) + expect(result.totalCost).not.toBeCloseTo(0.375, 9) + }) + + it("bills a mixed 5m/1h split per class (60/40 split, 0.51)", () => { + // 60k of the 100k writes are 1h, 40k remain at the 5m rate. + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 60_000) + + // 5m writes: (3.75 / 1_000_000) * 40_000 = 0.15 + // 1h writes: (6.0 / 1_000_000) * 60_000 = 0.36 + // Total: 0.15 + 0.36 = 0.51 + expect(result.totalCost).toBeCloseTo(0.51, 9) + }) + + it("derives the 5m remainder as total minus 1h minus 12h", () => { + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 30_000, 20_000) + + // 5m remainder: 100k - 30k - 20k = 50k at 3.75 => 0.1875 + // 1h: 30k at 6.0 => 0.18 + // 12h: 20k at 7.5 => 0.15 + // Total: 0.1875 + 0.18 + 0.15 = 0.5175 + expect(result.totalCost).toBeCloseTo(0.5175, 9) + }) + + it("caps each TTL class at the aggregate write total", () => { + // A malformed response claiming more 1h tokens than the write total must + // not bill the excess. The 5m remainder already clamps at zero, but the 1h + // and 12h classes were billed at their full reported count. + const result = calculateApiCostAnthropic(anthropicModel, 0, 0, 100_000, 0, 150_000) + + // 1h writes cap at the 100k aggregate; the 5m remainder clamps to 0. + // 100k at 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) + expect(result.totalCost).toBeGreaterThan(0) + // The pre-fix behaviour billed all 150k at 6.0 => 0.9. + expect(result.totalCost).not.toBeCloseTo(0.9, 9) + }) + + it("caps the 12h class at the remainder left after the 1h class", () => { + // 1h and 12h are a decomposition of one aggregate, so they cannot both + // claim the whole total. 1h wins the ordering; 12h gets what is left. + const model: ModelInfo = { ...anthropicModel, cacheWritesPrice12h: 7.5 } + const result = calculateApiCostAnthropic(model, 0, 0, 100_000, 0, 60_000, 80_000) + + // 1h: 60k at 6.0 => 0.36 + // 12h: capped at 100k - 60k = 40k at 7.5 => 0.3 + // 5m: 100k - 60k - 40k = 0 + // Total: 0.66 (the pre-fix behaviour billed 80k 12h => 0.36 + 0.6 = 0.96) + expect(result.totalCost).toBeCloseTo(0.66, 9) + expect(result.totalCost).not.toBeCloseTo(0.96, 9) + }) + + it("is byte-identical to the pre-split cost when no split is reported", () => { + // Invariant I1: absent split fields must reproduce today's pricing exactly. + const withSplitArgs = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000) + const explicitZeros = calculateApiCostAnthropic(anthropicModel, 1_000, 500, 2_000, 3_000, 0, 0) + + expect(withSplitArgs.totalCost).toBe(0.0189) + expect(explicitZeros.totalCost).toBe(withSplitArgs.totalCost) + }) + + it("falls back to the 5m rate when a model has no 1h rate and no input price", () => { + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + } + + // Fail-closed: without a derivable 1h rate the write is billed at the + // aggregate rate rather than dropped or invented. + const result = calculateApiCostAnthropic(bareModel, 0, 0, 100_000, 0, 100_000) + expect(result.totalCost).toBeCloseTo(0.375, 9) + }) + + it("derives the 1h rate from inputPrice when the catalog omits it", () => { + const modelWithout1h: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheWritesPrice: 3.75, + } + + const result = calculateApiCostAnthropic(modelWithout1h, 0, 0, 100_000, 0, 100_000) + // Derived 1h rate = 3.0 * 2 = 6.0 => 0.6 + expect(result.totalCost).toBeCloseTo(0.6, 9) + }) + + it("bills OpenAI 12h writes at the 12h rate, not the 30m rate", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 100_000) + + // 12h writes: (10.0 / 1_000_000) * 100_000 = 1.0 + // The pre-split behaviour would have charged the 30m rate: + // (5.0 / 1_000_000) * 100_000 = 0.5, under-reporting by 50%. + expect(result.totalCost).toBeCloseTo(1.0, 9) + expect(result.totalCost).not.toBeCloseTo(0.5, 9) + }) + + it("bills a mixed OpenAI 30m/12h split per class", () => { + const result = calculateApiCostOpenAI(openAiModel, 100_000, 0, 100_000, 0, undefined, undefined, 40_000) + + // 30m remainder: 100k - 40k = 60k at 5.0 => 0.3 + // 12h: 40k at 10.0 => 0.4 + // Total: 0.7 + expect(result.totalCost).toBeCloseTo(0.7, 9) + }) + + it("scales the 1h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + // calculateApiCostOpenAI applies long-context pricing; total input 300k > 200k. + const result = calculateApiCostOpenAI(modelWithLongContext, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input: (3.0 / 1_000_000) * (300_000 - 100_000) = 0.6 + // 1h writes scale 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8. Unscaled 1h would have given 0.6 + 0.6 = 1.2. + expect(result.totalCost).toBeCloseTo(1.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.2, 9) + }) + + it("scales the 12h write rate with the long-context multiplier", () => { + const modelWithLongContext: ModelInfo = { + ...openAiModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithLongContext, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (4.0 / 1_000_000) * (300_000 - 100_000) = 0.8 + // 12h writes scale 10.0 -> 20.0: (20.0 / 1_000_000) * 100_000 = 2.0 + // Total: 0.8 + 2.0 = 2.8. Unscaled 12h would have given 0.8 + 1.0 = 1.8. + expect(result.totalCost).toBeCloseTo(2.8, 9) + expect(result.totalCost).not.toBeCloseTo(1.8, 9) + }) + + it("leaves the 1h rate undefined when neither a catalog rate nor an input price exists", () => { + // Covers the `cacheWritesPrice1h !== undefined` false branch of long-context scaling. + const bareModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + cacheWritesPrice: 3.75, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(bareModel, 300_000, 0, 100_000, 0, undefined, 100_000) + + // No input price => no derivable 1h rate => falls back to the scaled aggregate + // write rate (3.75 * 2 = 7.5): (7.5 / 1_000_000) * 100_000 = 0.75 + expect(result.totalCost).toBeCloseTo(0.75, 9) + }) + + it("does not scale TTL rates when long-context pricing has no write multiplier", () => { + // Covers the `cacheWritesPriceMultiplier !== undefined` false branch for both + // TTL classes: without a multiplier the rates must pass through unchanged. + const modelWithoutMultiplier: ModelInfo = { + ...anthropicModel, + cacheWritesPrice12h: 7.5, + longContextPricing: { + thresholdTokens: 200_000, + inputPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithoutMultiplier, + 300_000, + 0, + 100_000, + 0, + undefined, + 50_000, + 50_000, + ) + + // Base input scales: (3.0 * 2 / 1_000_000) * (300_000 - 100_000) = 1.2 + // 1h unscaled: (6.0 / 1_000_000) * 50_000 = 0.3 + // 12h unscaled: (7.5 / 1_000_000) * 50_000 = 0.375 + // Total: 1.2 + 0.3 + 0.375 = 1.875 + expect(result.totalCost).toBeCloseTo(1.875, 9) + }) + + it("keeps an undefined 12h rate undefined when a write multiplier is present", () => { + // Covers the `cacheWritesPrice12h !== undefined` false branch: a model with a + // long-context write multiplier but no 12h rate must not gain a fabricated one. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI(modelWithout12h, 300_000, 0, 100_000, 0, undefined, 100_000) + + // Base input (no inputPriceMultiplier, so unscaled): (3.0 / 1_000_000) * 200_000 = 0.6 + // 1h scales 6.0 -> 12.0: (12.0 / 1_000_000) * 100_000 = 1.2 + // Total: 0.6 + 1.2 = 1.8 + expect(result.totalCost).toBeCloseTo(1.8, 9) + }) + + it("derives the 12h rate from inputPrice when a write multiplier is present but no 12h rate", () => { + // Distinguishes the guarded branch from an unguarded multiply: an unguarded + // `undefined * multiplier` yields NaN, which the cost layer reads as 0 and + // would silently drop the 12h write cost entirely. + const modelWithout12h: ModelInfo = { + ...anthropicModel, + longContextPricing: { + thresholdTokens: 200_000, + cacheWritesPriceMultiplier: 2, + }, + } + + const result = calculateApiCostOpenAI( + modelWithout12h, + 300_000, + 0, + 100_000, + 0, + undefined, + undefined, + 100_000, + ) + + // Base input: (3.0 / 1_000_000) * 200_000 = 0.6 + // No 12h rate in the catalog, so it derives from inputPrice * 2.5 = 7.5 + // (the long-context multiplier does not apply to a rate that is not there): + // (7.5 / 1_000_000) * 100_000 = 0.75 + // Total: 0.6 + 0.75 = 1.35 + expect(result.totalCost).toBeCloseTo(1.35, 9) + expect(result.totalCost).not.toBeCloseTo(0.6, 9) + }) + }) })