Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions packages/types/src/__tests__/cache-write-ttl-pricing.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// npx vitest run src/__tests__/cache-write-ttl-pricing.test.ts

import { describe, it, expect } from "vitest"

import { anthropicModels } from "../providers/anthropic.js"
import { bedrockModels } from "../providers/bedrock.js"
import { openAiNativeModels } from "../providers/openai.js"
import {
CACHE_WRITE_MULTIPLIER_12H,
CACHE_WRITE_MULTIPLIER_1H,
deriveCacheWritesPrice12h,
deriveCacheWritesPrice1h,
type ModelInfo,
} from "../model.js"

/**
* Catalog invariant for the TTL accounting program (ADR-0006 phase 1).
*
* A model that advertises prompt caching on a provider which can report a
* long-lived cache write must carry the matching per-TTL write rate. Without it
* the cost layer can only fall back to the aggregate write rate, which
* under-reports every long-lived write (37.5% for Anthropic 1h, 50% for OpenAI 12h).
*/
describe("cache-write TTL pricing catalog invariant", () => {
const anthropicFamilyEntries = Object.entries(anthropicModels as Record<string, ModelInfo>)

const bedrockAnthropicEntries = Object.entries(bedrockModels as Record<string, ModelInfo>).filter(([id]) =>
id.startsWith("anthropic."),
)

const openAiLongLivedEntries = Object.entries(openAiNativeModels as Record<string, ModelInfo>).filter(
([id]) => id.startsWith("gpt-5.6") || id.startsWith("gpt-6"),
)

it("every Anthropic catalog entry with prompt caching defines cacheWritesPrice1h", () => {
const missing = anthropicFamilyEntries
.filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined)
.map(([id]) => id)

expect(missing).toEqual([])
})

it("every Bedrock Anthropic-family entry with prompt caching defines cacheWritesPrice1h", () => {
const missing = bedrockAnthropicEntries
.filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice1h === undefined)
.map(([id]) => id)

expect(missing).toEqual([])
})

it("every OpenAI 5.6/6-class entry defines cacheWritesPrice12h", () => {
expect(openAiLongLivedEntries.length).toBeGreaterThan(0)

const missing = openAiLongLivedEntries
.filter(([, info]) => info.supportsPromptCache && info.cacheWritesPrice12h === undefined)
.map(([id]) => id)

expect(missing).toEqual([])
})

it("Anthropic 1h rates match the documented 2x input-price relationship", () => {
for (const [id, info] of anthropicFamilyEntries) {
if (info.cacheWritesPrice1h === undefined || info.inputPrice === undefined) continue
expect(info.cacheWritesPrice1h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_1H, 9)
}
})

it("OpenAI 12h rates match the documented 2.5x input-price relationship", () => {
for (const [id, info] of openAiLongLivedEntries) {
if (info.cacheWritesPrice12h === undefined || info.inputPrice === undefined) continue
expect(info.cacheWritesPrice12h, id).toBeCloseTo(info.inputPrice * CACHE_WRITE_MULTIPLIER_12H, 9)
}
})

it("deriveCacheWritesPrice1h scales inputPrice by 2x and stays undefined without one", () => {
expect(deriveCacheWritesPrice1h(3.0)).toBe(6.0)
expect(deriveCacheWritesPrice1h(undefined)).toBeUndefined()
})

it("deriveCacheWritesPrice12h scales inputPrice by 2.5x and stays undefined without one", () => {
expect(deriveCacheWritesPrice12h(4.0)).toBe(10.0)
expect(deriveCacheWritesPrice12h(undefined)).toBeUndefined()
})
})
2 changes: 2 additions & 0 deletions packages/types/src/__tests__/openai-models.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ describe("OpenAI native models", () => {
inputPrice: 2,
outputPrice: 10,
cacheWritesPrice: 2.5,
cacheWritesPrice12h: 5,
cacheReadsPrice: 0.2,
longContextPricing: {
thresholdTokens: 272_000,
Expand Down Expand Up @@ -166,6 +167,7 @@ describe("OpenAI native models", () => {
inputPrice: 0.1,
outputPrice: 0.5,
cacheWritesPrice: 0.125,
cacheWritesPrice12h: 0.25,
cacheReadsPrice: 0.01,
longContextPricing: {
thresholdTokens: 272_000,
Expand Down
103 changes: 103 additions & 0 deletions packages/types/src/__tests__/prompt-cache-ttl.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// npx vitest run src/__tests__/prompt-cache-ttl.test.ts

import { describe, it, expect } from "vitest"

import { anthropicModels } from "../providers/anthropic.js"
import { vertexModels } from "../providers/vertex.js"
import { openRouterDefaultModelInfo } from "../providers/openrouter.js"
import {
vercelAiGatewayDefaultModelId,
VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS,
} from "../providers/vercel-ai-gateway.js"
import {
DEFAULT_PROMPT_CACHE_TTL,
modelInfoSchema,
promptCacheTtlSchema,
promptCacheTtlValues,
type ModelInfo,
} from "../model.js"
import { anthropicProviderDefinition } from "../provider-settings/anthropic.js"
import { baseProviderSettingsShape } from "../provider-settings/common.js"
import { providerIdentifiers } from "../provider-identifiers.js"
import { globalSettingsSchema } from "../global-settings.js"

/**
* Catalog invariant for the request-side TTL knob (ADR-0006 phase 2, PR-C5b).
*
* Emission is gated by `ModelInfo.promptCacheTtlSupport`, so a model that can
* report a 1h cache write but carries no capability array would silently never
* emit the TTL the user asked for. These tests pin both directions: capable
* catalogs advertise it, and catalogs that cannot know stay absent (fail closed).
*/
describe("prompt cache TTL schema", () => {
it("accepts exactly the 5m and 1h classes", () => {
expect(promptCacheTtlValues).toEqual(["5m", "1h"])
expect(promptCacheTtlSchema.parse("5m")).toBe("5m")
expect(promptCacheTtlSchema.parse("1h")).toBe("1h")
expect(promptCacheTtlSchema.safeParse("30m").success).toBe(false)
expect(promptCacheTtlSchema.safeParse("12h").success).toBe(false)
})

it("defaults to undefined so an unset knob emits no ttl key", () => {
expect(DEFAULT_PROMPT_CACHE_TTL).toBeUndefined()
})

it("is optional on ModelInfo", () => {
const parsed = modelInfoSchema.parse({ contextWindow: 200_000, supportsPromptCache: true })
expect(parsed.promptCacheTtlSupport).toBeUndefined()

const withCapability = modelInfoSchema.parse({
contextWindow: 200_000,
supportsPromptCache: true,
promptCacheTtlSupport: ["5m", "1h"],
})
expect(withCapability.promptCacheTtlSupport).toEqual(["5m", "1h"])
})
})

describe("prompt cache TTL knob location", () => {
it("lives in provider settings so handlers can see it", () => {
// ApiHandlerOptions = Omit<ProviderSettings, "apiProvider">, so only a
// provider-settings key reaches a handler (ADR-0006 decision item 2).
expect(baseProviderSettingsShape.promptCacheTtl).toBeDefined()

const parsed = anthropicProviderDefinition.schema.safeParse({
apiProvider: providerIdentifiers.anthropic,
promptCacheTtl: "1h",
})
expect(parsed.success).toBe(true)
expect(parsed.success && parsed.data.promptCacheTtl).toBe("1h")
})

it("is not a global setting", () => {
const shape = globalSettingsSchema.shape as Record<string, unknown>
expect(shape.promptCacheTtl).toBeUndefined()
})
})

describe("prompt cache TTL capability catalog", () => {
it("every Anthropic entry with prompt caching advertises both TTL classes", () => {
const missing = Object.entries(anthropicModels as Record<string, ModelInfo>)
.filter(([, info]) => info.supportsPromptCache && !info.promptCacheTtlSupport?.includes("1h"))
.map(([id]) => id)

expect(missing).toEqual([])
})

it("leaves the Vertex catalog without capability so it fails closed", () => {
// PR-C5b scopes the catalog capability to the Anthropic entries; the Vertex
// catalog is populated separately, so its models stay 5m-only for now.
const withCapability = Object.entries(vertexModels as Record<string, ModelInfo>)
.filter(([, info]) => info.promptCacheTtlSupport !== undefined)
.map(([id]) => id)

expect(withCapability).toEqual([])
})

it("leaves dynamic-catalog defaults without capability so they fail closed", () => {
// OpenRouter and the gateway catalogs are fetched at runtime and carry no TTL
// capability data, which ADR-0006 maps to "5m only" (invariant I2).
expect(openRouterDefaultModelInfo.promptCacheTtlSupport).toBeUndefined()
expect(VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(vercelAiGatewayDefaultModelId)).toBe(true)
})
})
56 changes: 56 additions & 0 deletions packages/types/src/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,20 @@ export type ModelParameter = z.infer<typeof modelParametersSchema>
export const isModelParameter = (value: string): value is ModelParameter =>
modelParameters.includes(value as ModelParameter)

/**
* Prompt cache TTL classes a request-side cache marker can carry. `"5m"` is the
* vendor default and is never emitted explicitly, so only `"1h"` changes the wire
* shape and an unset knob stays byte-identical to a pre-knob build (ADR-0006 I3).
*/
export const promptCacheTtlValues = ["5m", "1h"] as const

export const promptCacheTtlSchema = z.enum(promptCacheTtlValues)

export type PromptCacheTtl = z.infer<typeof promptCacheTtlSchema>

/** Deliberately `undefined`, not `"5m"`: emitting `ttl: "5m"` would change every request's bytes. */
export const DEFAULT_PROMPT_CACHE_TTL: PromptCacheTtl | undefined = undefined

/**
* ModelInfo
*/
Expand All @@ -112,6 +126,8 @@ export const modelInfoSchema = z.object({
// When set to "24h", extended prompt caching will be requested; when omitted
// or set to "in_memory", the default in‑memory cache is used.
promptCacheRetention: z.enum(["in_memory", "24h"]).optional(),
// Cache TTLs this model accepts. Absent means "5m only" (ADR-0006 I2, fail closed).
promptCacheTtlSupport: z.array(promptCacheTtlSchema).optional(),
// Capability flag to indicate whether the model supports an output verbosity parameter
supportsVerbosity: z.boolean().optional(),
// Capability flag to indicate whether the model exposes a user-configurable max output
Expand All @@ -136,6 +152,13 @@ export const modelInfoSchema = z.object({
inputPrice: z.number().optional(),
outputPrice: z.number().optional(),
cacheWritesPrice: z.number().optional(),
// Per-TTL cache-write rates. Vendors bill a longer cache lifetime at a higher
// write rate than the default 5m/30m class, so a single `cacheWritesPrice`
// under-reports whenever the response reports a long-lived write. Omitted
// means "no long-lived writes are expected on this model"; the cost layer
// derives a fail-closed rate from `inputPrice` if one still arrives.
cacheWritesPrice1h: z.number().optional(),
cacheWritesPrice12h: z.number().optional(),
cacheReadsPrice: z.number().optional(),
longContextPricing: z
.object({
Expand Down Expand Up @@ -180,6 +203,11 @@ export const modelInfoSchema = z.object({
inputPrice: z.number().optional(),
outputPrice: z.number().optional(),
cacheWritesPrice: z.number().optional(),
// Per-TTL write rates for this tier. A tier that overrides inputPrice
// must also override these, otherwise a long-context or flex/priority
// write bills at the base tier's TTL rate.
cacheWritesPrice1h: z.number().optional(),
cacheWritesPrice12h: z.number().optional(),
cacheReadsPrice: z.number().optional(),
}),
)
Expand All @@ -188,6 +216,34 @@ export const modelInfoSchema = z.object({

export type ModelInfo = z.infer<typeof modelInfoSchema>

/**
* Multipliers vendors apply to the base input price for a cache write, by TTL class.
* Anthropic: 5m = 1.25x, 1h = 2x. OpenAI: 30m = 1.25x, 12h = 2.5x.
*/
export const CACHE_WRITE_MULTIPLIER_5M = 1.25
export const CACHE_WRITE_MULTIPLIER_1H = 2.0
export const CACHE_WRITE_MULTIPLIER_12H = 2.5

/**
* Derive the 1h cache-write rate from the base input price.
*
* Stated once here so every Anthropic-family catalog entry and every runtime
* fallback agrees on the 2x relationship. Returns `undefined` when the model
* has no input price to scale, which keeps the cost layer fail-closed: it will
* fall back to `cacheWritesPrice` rather than invent a rate.
*/
export function deriveCacheWritesPrice1h(inputPrice: number | undefined): number | undefined {
return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_1H
}

/**
* Derive the 12h cache-write rate from the base input price (OpenAI 12h class).
* Same fail-closed contract as {@link deriveCacheWritesPrice1h}.
*/
export function deriveCacheWritesPrice12h(inputPrice: number | undefined): number | undefined {
return inputPrice === undefined ? undefined : inputPrice * CACHE_WRITE_MULTIPLIER_12H
}

export type ModelRecord = Record<string, ModelInfo>

export type RouterModels = Record<DynamicProvider | LocalProvider, ModelRecord>
Expand Down
7 changes: 6 additions & 1 deletion packages/types/src/provider-settings/common.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { z } from "zod"

import { reasoningEffortSettingSchema, verbosityLevelsSchema } from "../model.js"
import { reasoningEffortSettingSchema, verbosityLevelsSchema, promptCacheTtlSchema } from "../model.js"
import type { ProviderIdentifier } from "../provider-identifiers.js"

export const API_PROVIDER_FIELD = "apiProvider"
Expand Down Expand Up @@ -33,6 +33,11 @@ export const baseProviderSettingsShape = {
modelMaxTokens: z.number().optional(),
modelMaxThinkingTokens: z.number().optional(),
verbosity: verbosityLevelsSchema.optional(),
// Opt-in prompt cache TTL for this provider profile. Lives here, not in global
// settings, because `ApiHandlerOptions = Omit<ProviderSettings, "apiProvider">`
// (src/shared/api.ts) makes a global key invisible to handlers. Unset emits no
// ttl key at all, not "5m" (ADR-0006 I3).
promptCacheTtl: promptCacheTtlSchema.optional(),
}

export const apiModelIdProviderModelShape = {
Expand Down
Loading
Loading