diff --git a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts index 1ffd63aa12..2e753ba7c5 100644 --- a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts +++ b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts @@ -51,6 +51,32 @@ class TestOpenAiCompatibleProvider extends BaseOpenAiCompatibleProvider<"test-mo } } +// A provider whose catalog row publishes an explicit zero cache-write price, +// the shape friendli and moonshot ship. It exists so the miss-token path can be +// priced end to end rather than only asserted as a token label. +const ZERO_WRITE_PRICE_MODEL_INFO: ModelInfo = { + maxTokens: 4096, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: true, + inputPrice: 1.4, + outputPrice: 4.4, + cacheWritesPrice: 0, + cacheReadsPrice: 0.26, +} + +class ZeroWritePriceProvider extends BaseOpenAiCompatibleProvider<"zero-write"> { + constructor(apiKey: string) { + super({ + providerName: "ZeroWriteProvider", + baseURL: "https://test.example.com/v1", + defaultProviderModelId: "zero-write", + providerModels: { "zero-write": ZERO_WRITE_PRICE_MODEL_INFO }, + apiKey, + }) + } +} + describe("BaseOpenAiCompatibleProvider", () => { let handler: TestOpenAiCompatibleProvider @@ -297,6 +323,159 @@ describe("BaseOpenAiCompatibleProvider", () => { expect(firstChunk.done).toBe(false) expect(firstChunk.value).toMatchObject({ type: "usage", inputTokens: 100, outputTokens: 50 }) }) + + describe("cache accounting", () => { + const usageChunksFor = async (usage: Record) => { + mockCreate.mockImplementationOnce(() => asyncStreamFrom([{ choices: [{ delta: {} }], usage }])) + + const chunks = await collectStream(handler.createMessage("system prompt", [])) + + return chunks.filter((chunk) => chunk.type === "usage") + } + + it("reads the nested shape it already understood", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }), + ]) + }) + + it("reads the top-level cache_creation_input_tokens write shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + cache_creation_input_tokens: 30, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 30, + }), + ]) + }) + + it("reads the top-level cache_read_input_tokens read shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + cache_read_input_tokens: 20, + }), + ).toEqual([ + expect.objectContaining({ type: "usage", inputTokens: 100, outputTokens: 50, cacheReadTokens: 20 }), + ]) + }) + + it("bills a DeepSeek-shaped miss counter as ordinary input, not as a cache write", async () => { + // The base class serves baseten/fireworks/friendli/sambanova/zai, none of + // which price misses through the cache-write field. A miss is input that + // `prompt_tokens` already covers, so it must stay in the input bucket. + const [chunk] = await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 30, + }) + + expect(chunk).toMatchObject({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + }) + expect(chunk.cacheWriteTokens).toBeUndefined() + + // TEST_MODEL_INFO prices input at 0.5 and output at 1.5 with no cache + // prices, so the 80 uncached tokens bill at the input price and the 20 + // reads bill at $0: (80 * 0.5 + 50 * 1.5) / 1e6. + expect(chunk.totalCost).toBeCloseTo(0.000115, 9) + + // Folding the 30 misses onto the unpriced write bucket would subtract + // them from the input pool and bill them nowhere: 0.0001. + expect(chunk.totalCost).not.toBeCloseTo(0.0001, 9) + }) + + it("never sums two shapes that report the same quantity", async () => { + // All shapes below describe the same 20 reads and 30 writes. An + // additive reader would bill 60 reads and 90 writes. + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 30, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 30, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }), + ]) + }) + + it("bills a miss on an explicit-zero cache-write row at the input price", async () => { + // friendli and moonshot publish `cacheWritesPrice: 0`, so the kernel fix + // alone cannot repair this shape: a miss routed to the write bucket is + // still billed at zero. Only keeping misses out of that bucket bills them. + mockCreate.mockImplementationOnce(() => + asyncStreamFrom([ + { + choices: [{ delta: {} }], + usage: { + prompt_tokens: 1000, + completion_tokens: 500, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 400, + }, + }, + ]), + ) + + const chunks = await collectStream(new ZeroWritePriceProvider("test-api-key").createMessage("s", [])) + const [chunk] = chunks.filter((c) => c.type === "usage") + + expect(chunk.cacheWriteTokens).toBeUndefined() + + // The 400 misses stay inside prompt_tokens, so they are priced as input. + // Uncached input: (1.4 / 1_000_000) * 900 = 0.00126 + // Cache reads: (0.26 / 1_000_000) * 100 = 0.000026 + // Output cost: (4.4 / 1_000_000) * 500 = 0.0022 + // Total: 0.00126 + 0.000026 + 0.0022 = 0.003486 + expect(chunk.totalCost).toBeCloseTo(0.003486, 9) + + // The defect moved those 400 tokens onto the row's zero write price and + // out of the input pool, billing 0.002926: 16.1% short. + expect(chunk.totalCost).not.toBeCloseTo(0.002926, 9) + }) + + it("reports no cache counters when the provider reports none", async () => { + const [chunk] = await usageChunksFor({ prompt_tokens: 100, completion_tokens: 50 }) + + expect(chunk).toMatchObject({ type: "usage", inputTokens: 100, outputTokens: 50 }) + expect(chunk.cacheReadTokens).toBeUndefined() + expect(chunk.cacheWriteTokens).toBeUndefined() + }) + }) }) describe("getModel", () => { diff --git a/src/api/providers/__tests__/deepseek.spec.ts b/src/api/providers/__tests__/deepseek.spec.ts index 7878eaa2e1..124a0271a2 100644 --- a/src/api/providers/__tests__/deepseek.spec.ts +++ b/src/api/providers/__tests__/deepseek.spec.ts @@ -27,9 +27,10 @@ vi.mock("openai", () => { completion_tokens: 5, total_tokens: 15, prompt_tokens_details: { - cache_miss_tokens: 8, cached_tokens: 2, }, + prompt_cache_hit_tokens: 2, + prompt_cache_miss_tokens: 8, }, } } @@ -111,9 +112,10 @@ vi.mock("openai", () => { completion_tokens: 5, total_tokens: 15, prompt_tokens_details: { - cache_miss_tokens: 8, cached_tokens: 2, }, + prompt_cache_hit_tokens: 2, + prompt_cache_miss_tokens: 8, }, }) @@ -129,9 +131,16 @@ vi.mock("openai", () => { import OpenAI from "openai" import type { Anthropic } from "@anthropic-ai/sdk" -import { deepSeekDefaultModelId, DEEP_SEEK_DEFAULT_TEMPERATURE, type ModelInfo, DeepSeekModelId } from "@roo-code/types" +import { + deepSeekDefaultModelId, + deepSeekModels, + DEEP_SEEK_DEFAULT_TEMPERATURE, + type ModelInfo, + DeepSeekModelId, +} from "@roo-code/types" import type { ApiHandlerOptions } from "../../../shared/api" +import { calculateApiCostOpenAI } from "../../../shared/cost" import { DeepSeekHandler, normalizeDeepSeekReasoningEffort } from "../deepseek" @@ -517,17 +526,55 @@ describe("DeepSeekHandler", () => { }) describe("processUsageMetrics", () => { - it("should correctly process usage metrics including cache information", () => { - // We need to access the protected method, so we'll create a test subclass - class TestDeepSeekHandler extends DeepSeekHandler { - public testProcessUsageMetrics(usage: any) { - return this.processUsageMetrics(usage) - } + class TestDeepSeekHandler extends DeepSeekHandler { + public testProcessUsageMetrics(usage: any) { + return this.processUsageMetrics(usage) } + } - const testHandler = new TestDeepSeekHandler(mockOptions) + // mockOptions is assigned in beforeEach, so the handler must be built + // per test rather than at describe-collection time. + const processUsage = (usage: any) => new TestDeepSeekHandler(mockOptions).testProcessUsageMetrics(usage) + + // DeepSeek's OpenAPI marks prompt_cache_hit_tokens and + // prompt_cache_miss_tokens as required top-level members, and declares + // only cached_tokens inside prompt_tokens_details. This is the vendor's + // own documented example response. + // https://api-docs.deepseek.com/api/create-chat-completion + it("reads the documented top-level cache counters", () => { + const result = processUsage({ + prompt_tokens: 7, + completion_tokens: 11, + total_tokens: 18, + prompt_tokens_details: { cached_tokens: 0 }, + prompt_cache_hit_tokens: 0, + prompt_cache_miss_tokens: 7, + }) - const usage = { + expect(result.type).toBe("usage") + expect(result.inputTokens).toBe(7) + expect(result.outputTokens).toBe(11) + expect(result.cacheReadTokens).toBe(0) + expect(result.cacheWriteTokens).toBe(7) + }) + + it("reads top-level cache counters when prompt_tokens_details is absent", () => { + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 80, + }) + + expect(result.inputTokens).toBe(100) + expect(result.outputTokens).toBe(50) + expect(result.cacheWriteTokens).toBe(80) + expect(result.cacheReadTokens).toBe(20) + }) + + it("falls back to the nested cache counters for legacy shapes", () => { + const result = processUsage({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150, @@ -535,34 +582,37 @@ describe("DeepSeekHandler", () => { cache_miss_tokens: 80, cached_tokens: 20, }, - } - - const result = testHandler.testProcessUsageMetrics(usage) + }) - expect(result.type).toBe("usage") expect(result.inputTokens).toBe(100) expect(result.outputTokens).toBe(50) expect(result.cacheWriteTokens).toBe(80) expect(result.cacheReadTokens).toBe(20) }) - it("should handle missing cache metrics gracefully", () => { - class TestDeepSeekHandler extends DeepSeekHandler { - public testProcessUsageMetrics(usage: any) { - return this.processUsageMetrics(usage) - } - } + it("does not double-count when both shapes are present", () => { + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + prompt_tokens_details: { cached_tokens: 20, cache_miss_tokens: 80 }, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 80, + }) - const testHandler = new TestDeepSeekHandler(mockOptions) + expect(result.cacheWriteTokens).toBe(80) + expect(result.cacheReadTokens).toBe(20) + }) + it("should handle missing cache metrics gracefully", () => { const usage = { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150, - // No prompt_tokens_details + // No prompt_tokens_details and no top-level cache counters } - const result = testHandler.testProcessUsageMetrics(usage) + const result = processUsage(usage) expect(result.type).toBe("usage") expect(result.inputTokens).toBe(100) @@ -570,6 +620,62 @@ describe("DeepSeekHandler", () => { expect(result.cacheWriteTokens).toBeUndefined() expect(result.cacheReadTokens).toBeUndefined() }) + + it("bills a cache miss as a cache write, because the catalog prices input at zero", () => { + // This is the one handler that opts in to the normalizer's miss counter. + // Every deepSeekModels row sets `inputPrice: 0` on purpose ("the inputs + // are priced as cache read/write"), so a miss left in the input bucket + // would be billed at zero. The money is asserted, not just the label: + // a token-label assertion passes whether or not the price is right. + const usage = { + prompt_tokens: 1000, + completion_tokens: 500, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 400, + } + + const result = processUsage(usage) + + expect(result.cacheWriteTokens).toBe(400) + + const { totalCost } = calculateApiCostOpenAI( + deepSeekModels["deepseek-flash"], + result.inputTokens, + result.outputTokens, + result.cacheWriteTokens, + result.cacheReadTokens, + ) + + // Cache writes: (0.3 / 1_000_000) * 400 = 0.00012 + // Cache reads: (0.006 / 1_000_000) * 100 = 0.0000006 + // Uncached input: (0 / 1_000_000) * 500 = 0 + // Output cost: (1.2 / 1_000_000) * 500 = 0.0006 + expect(totalCost).toBeCloseTo(0.0007206, 9) + }) + + it("preserves a reported zero miss counter instead of treating it as unreported", () => { + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 0, + }) + + expect(result.cacheWriteTokens).toBe(0) + }) + + it("keeps a genuine write counter ahead of the miss counter", () => { + // A response can report both concepts. The write counter is the more + // specific one, so it wins rather than being shadowed by the miss count. + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cache_write_tokens: 30 }, + prompt_cache_miss_tokens: 80, + }) + + expect(result.cacheWriteTokens).toBe(30) + }) }) describe("interleaved thinking mode", () => { diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 0441270498..0d59c81ca5 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -2,7 +2,10 @@ import { Anthropic } from "@anthropic-ai/sdk" +import type { ModelInfo } from "@roo-code/types" + import { ApiHandlerOptions } from "../../../shared/api" +import { calculateApiCostOpenAI } from "../../../shared/cost" import { OpenAiHandler } from "../openai" import { makeApiHandlerOptions } from "../../../test-utils/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" @@ -428,4 +431,125 @@ describe("OpenAiHandler with usage tracking fix", () => { ]) }, ) + + describe.each([true, false])("cache-write shape coverage with streaming=%s", (openAiStreamingEnabled) => { + const usageChunksFor = async (usage: Record) => { + const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled }) + mockCreate.mockResolvedValueOnce( + openAiStreamingEnabled + ? asyncStreamFrom([{ choices: [], usage }]) + : { choices: [{ message: { content: "Response" } }], usage }, + ) + + const chunks = await collectStream(cacheHandler.createMessage("system prompt", [])) + + return chunks.filter((chunk) => chunk.type === "usage") + } + + it("reads the nested cache_write_tokens shape it previously ignored", async () => { + // This is the OpenAiHandler write gap K7 names: the handler read only + // the top-level Anthropic-style counter, so a proxy reporting writes + // under prompt_tokens_details billed them as zero. + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30, cacheReadTokens: 20 }]) + }) + + it("still reads the top-level cache_creation_input_tokens shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + cache_creation_input_tokens: 7, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 7 }]) + }) + + it("does not bill a top-level prompt_cache_miss_tokens counter as a cache write", async () => { + // A miss is a token that was looked up in the cache and not found, so it + // is ordinary input that prompt_tokens already covers. Reporting it as a + // write would move it off the input price line onto the cache-write line, + // where most catalogs charge nothing. openai-native.ts states the same + // rule: "missFromDetails is NOT used as fallback for cache writes". + expect( + await usageChunksFor({ + prompt_tokens: 1000, + completion_tokens: 5, + prompt_cache_miss_tokens: 400, + }), + ).toEqual([{ type: "usage", inputTokens: 1000, outputTokens: 5 }]) + }) + + it("leaves the miss tokens in the input pool that Task.ts prices", async () => { + // This handler emits no totalCost; Task.ts recomputes it from the counters + // in the chunk. The chunk is therefore the contract, and the money is + // asserted against it so a label-only regression cannot pass. + const [chunk] = await usageChunksFor({ + prompt_tokens: 1000, + completion_tokens: 5, + prompt_cache_miss_tokens: 400, + }) + + const customModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + } + + const { totalCost } = calculateApiCostOpenAI( + customModel, + chunk.inputTokens, + chunk.outputTokens, + chunk.cacheWriteTokens, + chunk.cacheReadTokens, + ) + + // All 1000 prompt tokens bill at the input price, including the 400 misses: + // (3.0 / 1_000_000) * 1000 = 0.003 + // Output: (15.0 / 1_000_000) * 5 = 0.000075 + // Total: 0.003075 + expect(totalCost).toBeCloseTo(0.003075, 9) + + // The defect subtracted the 400 misses from the input pool and priced them + // at a cache-write rate the model never declared, billing 0.001875. + expect(totalCost).not.toBeCloseTo(0.001875, 9) + }) + + it("never sums two write shapes that report the same quantity", async () => { + // Every shape below describes the same 30 written tokens. Summing + // would bill 90 and is the additive-usage trap the wave forbids. + const chunks = await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_cache_miss_tokens: 30, + cache_creation_input_tokens: 30, + prompt_tokens_details: { cache_miss_tokens: 30, cache_write_tokens: 30 }, + }) + + expect(chunks).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30 }]) + }) + + it("keeps a genuine write counter ahead of the Anthropic-style one", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cache_write_tokens: 30 }, + cache_creation_input_tokens: 99, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30 }]) + }) + + it("reports no write counter when the provider reports none", async () => { + expect(await usageChunksFor({ prompt_tokens: 100, completion_tokens: 5 })).toEqual([ + { type: "usage", inputTokens: 100, outputTokens: 5 }, + ]) + }) + }) }) diff --git a/src/api/providers/base-openai-compatible-provider.ts b/src/api/providers/base-openai-compatible-provider.ts index b177726f85..6c27f8fe29 100644 --- a/src/api/providers/base-openai-compatible-provider.ts +++ b/src/api/providers/base-openai-compatible-provider.ts @@ -7,6 +7,7 @@ import { type ApiHandlerOptions, getModelMaxOutputTokens } from "../../shared/ap import { TagMatcher } from "../../utils/tag-matcher" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index" import { DEFAULT_HEADERS } from "./constants" @@ -193,10 +194,14 @@ export abstract class BaseOpenAiCompatibleProvider } protected processUsageMetrics(usage: any, modelInfo?: any): ApiStreamUsageChunk { - const inputTokens = usage?.prompt_tokens || 0 - const outputTokens = usage?.completion_tokens || 0 - const cacheWriteTokens = usage?.prompt_tokens_details?.cache_write_tokens || 0 - const cacheReadTokens = usage?.prompt_tokens_details?.cached_tokens || 0 + // The normalizer resolves one value per field across the OpenAI-compatible + // cache shapes. It is the only reader here, so nothing is mapped twice and + // two shapes describing the same quantity are never summed. + const normalized = normalizeChatUsage(usage) + const inputTokens = normalized?.inputTokens ?? 0 + const outputTokens = normalized?.outputTokens ?? 0 + const cacheWriteTokens = normalized?.cacheWriteTokens ?? 0 + const cacheReadTokens = normalized?.cacheReadTokens ?? 0 const { totalCost } = modelInfo ? calculateApiCostOpenAI(modelInfo, inputTokens, outputTokens, cacheWriteTokens, cacheReadTokens) diff --git a/src/api/providers/deepseek.ts b/src/api/providers/deepseek.ts index b65973ac96..6aeb6b9ec2 100644 --- a/src/api/providers/deepseek.ts +++ b/src/api/providers/deepseek.ts @@ -15,6 +15,7 @@ import type { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { convertToR1Format } from "../transform/r1-format" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" import { OpenAiHandler } from "./openai" import { NOT_PROVIDED } from "./constants" @@ -227,13 +228,27 @@ export class DeepSeekHandler extends OpenAiHandler { } // Override to handle DeepSeek's usage metrics, including caching. + // DeepSeek reports cache counters in two shapes that describe the same + // quantities: top-level `prompt_cache_hit_tokens` / `prompt_cache_miss_tokens` + // and nested `prompt_tokens_details.cached_tokens`. The normalizer resolves + // one value per field and never sums the shapes. + // + // This is the only handler that bills cache misses as cache writes, and it + // is the reason the normalizer keeps the two counters apart. Every + // `deepSeekModels` row sets `inputPrice: 0` on purpose ("the inputs are + // priced as cache read/write"), so a miss left in the input bucket would be + // billed at zero. Every other provider prices misses through `inputPrice`, + // which is why they must not opt in here. + // https://api-docs.deepseek.com/api/create-chat-completion protected override processUsageMetrics(usage: any, _modelInfo?: any): ApiStreamUsageChunk { + const normalized = normalizeChatUsage(usage) + return { type: "usage", - inputTokens: usage?.prompt_tokens || 0, - outputTokens: usage?.completion_tokens || 0, - cacheWriteTokens: usage?.prompt_tokens_details?.cache_miss_tokens, - cacheReadTokens: usage?.prompt_tokens_details?.cached_tokens, + inputTokens: normalized?.inputTokens ?? 0, + outputTokens: normalized?.outputTokens ?? 0, + cacheWriteTokens: normalized?.cacheWriteTokens ?? normalized?.cacheMissTokens, + cacheReadTokens: normalized?.cacheReadTokens, } } } diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index 98ea18df8a..4182504cb9 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -21,6 +21,7 @@ import { TagMatcher } from "../../utils/tag-matcher" import { convertToOpenAiMessages } from "../transform/openai-format" import { sanitizeSurrogates } from "../transform/sanitize-surrogates" import { convertToR1Format } from "../transform/r1-format" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" @@ -317,7 +318,10 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl type: "usage", inputTokens, outputTokens: usage?.completion_tokens || 0, - cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, + // Reads keep their provider-specific validation above; writes are + // plain counters, so the shared normalizer resolves them across the + // OpenAI-compatible write shapes instead of reading one field. + cacheWriteTokens: normalizeChatUsage(usage)?.cacheWriteTokens || undefined, cacheReadTokens, } } diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts new file mode 100644 index 0000000000..e9c2586bcf --- /dev/null +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -0,0 +1,462 @@ +// npx vitest run api/transform/__tests__/chat-usage-normalization.spec.ts + +import { CompactLogger } from "../../../utils/logging/CompactLogger" + +import { normalizeChatUsage } from "../chat-usage-normalization" + +// Documented DeepSeek usage block, copied from the vendor's OpenAPI example. +// https://api-docs.deepseek.com/api/create-chat-completion +const documentedDeepSeekUsage = { + prompt_tokens: 7, + completion_tokens: 11, + total_tokens: 18, + prompt_tokens_details: { cached_tokens: 0 }, + prompt_cache_hit_tokens: 0, + prompt_cache_miss_tokens: 7, +} + +describe("normalizeChatUsage", () => { + it("returns undefined when the provider sent no usage object", () => { + expect(normalizeChatUsage(undefined)).toBeUndefined() + expect(normalizeChatUsage(null)).toBeUndefined() + }) + + it("returns undefined for a truthy non-object usage value", () => { + // A malformed proxy can send a scalar where the usage object belongs. + expect(normalizeChatUsage("usage" as unknown as Record)).toBeUndefined() + expect(normalizeChatUsage(0 as unknown as Record)).toBeUndefined() + }) + + it("reads the documented DeepSeek top-level shape", () => { + expect(normalizeChatUsage(documentedDeepSeekUsage)).toEqual({ + inputTokens: 7, + outputTokens: 11, + cacheReadTokens: 0, + cacheMissTokens: 7, + }) + }) + + it("reports a top-level-only miss counter as a miss, not a write", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + completion_tokens: 30, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 20, + }), + ).toEqual({ inputTokens: 120, outputTokens: 30, cacheReadTokens: 100, cacheMissTokens: 20 }) + }) + + it("reads the nested-only shape used by OpenAI and by legacy DeepSeek fixtures", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 20, cache_miss_tokens: 80 }, + }), + ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, cacheMissTokens: 80 }) + }) + + it("does not add the nested and top-level shapes together when both are present", () => { + // Both shapes describe the same 100 reads / 20 misses. Summing would + // report 200 / 40 and double the billed cache tokens. + const normalized = normalizeChatUsage({ + prompt_tokens: 120, + completion_tokens: 30, + prompt_tokens_details: { cached_tokens: 100, cache_miss_tokens: 20 }, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 20, + }) + + expect(normalized?.cacheReadTokens).toBe(100) + expect(normalized?.cacheMissTokens).toBe(20) + }) + + it("prefers the nested cached_tokens over the top-level hit counter for reads", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cached_tokens: 100 }, + prompt_cache_hit_tokens: 90, + })?.cacheReadTokens, + ).toBe(100) + }) + + it("prefers the top-level miss counter over the nested one", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cache_miss_tokens: 20 }, + prompt_cache_miss_tokens: 30, + })?.cacheMissTokens, + ).toBe(30) + }) + + it("falls through a malformed alias to the next one", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_miss_tokens: "30", + prompt_tokens_details: { cache_miss_tokens: 20 }, + })?.cacheMissTokens, + ).toBe(20) + }) + + describe("write and miss counters are separate concepts", () => { + // A write is a token stored for later reuse; a miss is a token that was + // looked up and not found, which is ordinary input. Merging them moves + // miss tokens off the input price line and onto the cache-write line, + // where most catalogs charge nothing. Each alias must land in exactly + // one field, so both lists are pinned here by behavior. + it.each(["prompt_cache_miss_tokens", "prompt_tokens_details.cache_miss_tokens"])( + "reports %s as a miss and never as a write", + (alias) => { + const usage = alias.includes(".") + ? { prompt_tokens: 100, prompt_tokens_details: { cache_miss_tokens: 40 } } + : { prompt_tokens: 100, prompt_cache_miss_tokens: 40 } + + const normalized = normalizeChatUsage(usage) + + expect(normalized?.cacheMissTokens).toBe(40) + expect(normalized).not.toHaveProperty("cacheWriteTokens") + }, + ) + + it.each(["prompt_tokens_details.cache_write_tokens", "cache_creation_input_tokens"])( + "reports %s as a write and never as a miss", + (alias) => { + const usage = alias.includes(".") + ? { prompt_tokens: 100, prompt_tokens_details: { cache_write_tokens: 40 } } + : { prompt_tokens: 100, cache_creation_input_tokens: 40 } + + const normalized = normalizeChatUsage(usage) + + expect(normalized?.cacheWriteTokens).toBe(40) + expect(normalized).not.toHaveProperty("cacheMissTokens") + }, + ) + + it("resolves a write and a miss from the same response without shadowing either", () => { + // 20 tokens were written into the cache and 40 were looked up but not + // found. They are different tokens, so both counters survive. + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_miss_tokens: 40, + cache_creation_input_tokens: 20, + }), + ).toMatchObject({ cacheWriteTokens: 20, cacheMissTokens: 40 }) + }) + }) + + describe("cache_creation_input_tokens (LiteLLM write shape)", () => { + it("reads the LiteLLM top-level write counter when no other write alias is present", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 100, + completion_tokens: 50, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }), + ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 30, cacheWriteTokens: 20 }) + }) + + it("keeps the nested cache_write_tokens alias ahead of it", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cache_write_tokens: 20 }, + cache_creation_input_tokens: 99, + })?.cacheWriteTokens, + ).toBe(20) + }) + + it("does not sum it with a write alias reporting the same quantity", () => { + const normalized = normalizeChatUsage({ + prompt_tokens: 120, + completion_tokens: 30, + prompt_tokens_details: { cache_write_tokens: 20 }, + cache_creation_input_tokens: 20, + }) + + expect(normalized?.cacheWriteTokens).toBe(20) + }) + }) + + describe("malformed counter hardening", () => { + it.each([ + ["a numeric string", "30"], + ["a fractional number", 1.5], + ["a negative number", -1], + ["NaN", NaN], + ["Infinity", Infinity], + ["an object", { tokens: 30 }], + ["a boolean", true], + ])("discards %s and falls through to the next alias", (_name, malformed) => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_miss_tokens: malformed, + prompt_tokens_details: { cache_miss_tokens: 20 }, + })?.cacheMissTokens, + ).toBe(20) + }) + + it("reports a present-but-malformed counter instead of discarding it silently", async () => { + // A fresh module instance keeps this case independent of the dedupe + // state that earlier cases in this file have already populated. + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + + expect(warn).toHaveBeenCalledWith(expect.any(String), { + path: "prompt_cache_miss_tokens", + type: "string", + length: 2, + preview: "30", + }) + } finally { + warn.mockRestore() + } + }) + + it("emits through a sink that is active outside tests", async () => { + // The shared `logger` export is a no-op unless NODE_ENV is "test", so a + // diagnostic routed through it is discarded in every real install. This + // case simulates the production branch and asserts the warning still + // reaches a real logger. + const originalNodeEnv = process.env.NODE_ENV + process.env.NODE_ENV = "production" + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + + expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { + path: "prompt_cache_miss_tokens", + type: "string", + length: 2, + preview: "30", + }) + } finally { + warn.mockRestore() + process.env.NODE_ENV = originalNodeEnv + } + }) + + it("does not abort usage accounting when the sink itself fails", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => { + throw new Error("log file unwritable") + }) + + try { + // The malformed counter is still discarded and the valid alias still + // resolves; a broken log sink must not surface as a usage failure. + expect( + freshNormalize({ + prompt_tokens: 120, + prompt_cache_miss_tokens: "30", + prompt_tokens_details: { cache_miss_tokens: 20 }, + })?.cacheMissTokens, + ).toBe(20) + } finally { + warn.mockRestore() + } + }) + + it("reports a malformed counter once per alias rather than on every chunk", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + + expect(warn).toHaveBeenCalledTimes(1) + } finally { + warn.mockRestore() + } + }) + + it("names the exact field that was discarded", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: "100", completion_tokens: -1 }) + + expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { + path: "prompt_tokens", + type: "string", + length: 3, + preview: "100", + }) + expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { + path: "completion_tokens", + type: "number", + preview: -1, + }) + } finally { + warn.mockRestore() + } + }) + + it("keeps the logged payload bounded for an arbitrarily large provider value", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + const huge = "9".repeat(1_000_000) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: huge }) + + const [, meta] = warn.mock.calls[0] as [string, Record] + + // The raw megabyte must not reach the transport, which stringifies + // and writes it synchronously to stdout and the log file. + expect(meta).not.toHaveProperty("value") + expect(meta.type).toBe("string") + expect(meta.length).toBe(1_000_000) + expect(meta.preview).toBe(`${"9".repeat(64)}…`) + expect(JSON.stringify(meta).length).toBeLessThan(200) + } finally { + warn.mockRestore() + } + }) + + it.each([ + ["exactly at the cap", 64, "9".repeat(64)], + ["one past the cap", 65, `${"9".repeat(64)}…`], + ])("keeps a preview %s without truncating or over-truncating", async (_name, length, expected) => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "9".repeat(length) }) + + const [, meta] = warn.mock.calls[0] as [string, Record] + + // A length-only assertion cannot tell an empty preview from a full + // one, nor 64 characters from 65, so the boundary is pinned exactly. + expect(meta.preview).toBe(expected) + expect(meta.length).toBe(length) + expect(meta).not.toHaveProperty("value") + } finally { + warn.mockRestore() + } + }) + + it.each([ + ["a boolean", true, { type: "boolean" }], + ["an object", { tokens: 30 }, { type: "object" }], + ["an array", [1, 2, 3], { type: "array" }], + ["NaN", NaN, { type: "number", preview: "NaN" }], + ["Infinity", Infinity, { type: "number", preview: "Infinity" }], + ["-Infinity", -Infinity, { type: "number", preview: "-Infinity" }], + ["a fractional number", 1.5, { type: "number", preview: 1.5 }], + ])("describes %s without logging the raw value", async (_name, malformed, expected) => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: malformed }) + + const [, meta] = warn.mock.calls[0] as [string, Record] + + expect(meta).toEqual({ path: "prompt_cache_miss_tokens", ...expected }) + } finally { + warn.mockRestore() + } + }) + + it("keeps non-finite previews distinguishable after JSON serialization", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + // Distinct aliases so the per-path dedupe does not swallow two of the + // three warnings. + const serialized = [ + { prompt_cache_miss_tokens: NaN }, + { prompt_tokens_details: { cache_miss_tokens: Infinity } }, + { cache_creation_input_tokens: -Infinity }, + ].map((malformed) => { + freshNormalize({ prompt_tokens: 120, ...malformed }) + const [, meta] = warn.mock.calls[warn.mock.calls.length - 1] as [string, Record] + return JSON.stringify(meta) + }) + + // The transport persists via JSON.stringify, which turns all three into + // `null`. They must stay tellable apart in the written log line. + expect(serialized).toEqual([ + expect.stringContaining('"preview":"NaN"'), + expect.stringContaining('"preview":"Infinity"'), + expect.stringContaining('"preview":"-Infinity"'), + ]) + expect(new Set(serialized).size).toBe(3) + serialized.forEach((line) => expect(line).not.toContain('"preview":null')) + } finally { + warn.mockRestore() + } + }) + + it("stays silent for counters the provider simply did not report", () => { + const warn = vi.spyOn(CompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + normalizeChatUsage({ prompt_tokens: 100, completion_tokens: 50 }) + normalizeChatUsage({ prompt_tokens: 100, prompt_tokens_details: null }) + normalizeChatUsage({ prompt_tokens: 100, prompt_cache_hit_tokens: null }) + + expect(warn).not.toHaveBeenCalled() + } finally { + warn.mockRestore() + } + }) + }) + + it("leaves cache counters undefined when the provider reports none", () => { + const normalized = normalizeChatUsage({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }) + + expect(normalized).toEqual({ inputTokens: 100, outputTokens: 50 }) + expect(normalized).not.toHaveProperty("cacheReadTokens") + expect(normalized).not.toHaveProperty("cacheWriteTokens") + }) + + it("rejects negative and fractional token counts", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_hit_tokens: -1, + prompt_cache_miss_tokens: 1.5, + }), + ).toEqual({ inputTokens: 120, outputTokens: 0 }) + }) + + it("defaults missing prompt and completion tokens to zero", () => { + expect(normalizeChatUsage({})).toEqual({ inputTokens: 0, outputTokens: 0 }) + }) +}) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts new file mode 100644 index 0000000000..67f6c2252e --- /dev/null +++ b/src/api/transform/chat-usage-normalization.ts @@ -0,0 +1,249 @@ +/** + * Normalization of OpenAI-shaped chat-completions usage objects into the + * cache token fields Zoo Code bills on. + * + * Providers that speak the OpenAI chat-completions wire format disagree on + * where they put cache counters. Some publish them top level, some nest them + * under `prompt_tokens_details`, and some publish both shapes describing the + * same quantities. Every reader here therefore resolves a single field through + * an ordered list of aliases and stops at the first numeric value. The aliases + * are never summed: a response that carries both shapes reports the same + * tokens twice, so adding them would double-count. + */ + +import { CompactLogger } from "../../utils/logging/CompactLogger" + +/** + * A usage object as it arrives off the wire, before any provider-specific + * typing. Fields are deliberately `unknown`: providers type their own usage + * loosely and the normalizer is the single place that validates them. + */ +export interface ChatUsageLike { + prompt_tokens?: unknown + completion_tokens?: unknown + prompt_tokens_details?: unknown + [key: string]: unknown +} + +export interface NormalizedChatUsage { + /** Total prompt tokens, including cache reads and cache writes. */ + inputTokens: number + outputTokens: number + /** + * Tokens written into the provider's prompt cache on this request. + * `undefined` when the provider reported no such counter at all, which is + * distinct from a reported zero. + */ + cacheReadTokens?: number + cacheWriteTokens?: number + /** + * Tokens that were looked up in the prompt cache and not found. These are + * ordinary input tokens for every provider except DeepSeek, so they are + * reported separately and are never folded into `cacheWriteTokens`. + * `undefined` when the provider reported no such counter at all, which is + * distinct from a reported zero. + */ + cacheMissTokens?: number +} + +/** + * Aliases for true cache-write tokens, highest priority first. + * + * A cache write is a token the provider durably stored for reuse on a later + * request. It is billed at the provider's cache-write price, which is often + * zero (a write surcharge is not universal) but is a real, separate counter. + * + * `cache_creation_input_tokens` is the Anthropic-style top-level counter that + * LiteLLM publishes for cached writes. + * + * https://api-docs.deepseek.com/api/create-chat-completion + */ +const CACHE_WRITE_ALIASES = ["prompt_tokens_details.cache_write_tokens", "cache_creation_input_tokens"] as const + +/** + * Aliases for cache-miss tokens, highest priority first. + * + * A cache miss is a token that was looked up in the prompt cache and not + * found. Economically it is the opposite of a write: it is ordinary input + * that the provider already charges for through `prompt_tokens`, and it must + * not be moved onto the cache-write price line, because doing so removes it + * from the input bucket (`cost.ts` subtracts cache writes from the input + * total) and re-prices it at a cache-write rate that most catalogs leave + * unset, billing it at zero. + * + * `openai-native.ts` already states this rule and applies it: "missFromDetails + * is NOT used as fallback for cache writes". This module now agrees with it. + * + * DeepSeek is the one provider whose catalog prices misses through the + * cache-write field: all four `deepSeekModels` rows set `inputPrice: 0` by + * design ("the inputs are priced as cache read/write") and publish + * `cacheWritesPrice`. Only `DeepSeekHandler` therefore opts in to reading + * this list; see `processUsageMetrics` in `providers/deepseek.ts`. + * + * https://api-docs.deepseek.com/api/create-chat-completion + */ +const CACHE_MISS_ALIASES = ["prompt_cache_miss_tokens", "prompt_tokens_details.cache_miss_tokens"] as const + +/** + * Aliases for cache-read (cache-hit) tokens, highest priority first. + * + * `prompt_tokens_details.cached_tokens` is the shape DeepSeek documents and + * also the shape OpenAI documents, so it wins. `prompt_cache_hit_tokens` is + * DeepSeek's required top-level counterpart and covers responses that omit the + * nested object. + * + * https://api-docs.deepseek.com/api/create-chat-completion + */ +const CACHE_READ_ALIASES = [ + "prompt_tokens_details.cached_tokens", + "prompt_cache_hit_tokens", + "cache_read_input_tokens", +] as const + +const readPath = (usage: ChatUsageLike, path: string): unknown => { + let current: unknown = usage + + for (const segment of path.split(".")) { + if (typeof current !== "object" || current === null) return undefined + current = (current as Record)[segment] + } + + return current +} + +/** + * Counters are token counts: only non-negative integers are accepted. Anything + * else (missing, null, a string, a float, a negative sentinel) is treated as + * "not reported" so a malformed field falls through to the next alias instead + * of poisoning the total. + * + * `typeof value === "number"` is redundant for the check itself, because + * `Number.isInteger` already rejects every non-number. It is kept so the type + * predicate narrows to `number` and callers can return the value typed. + */ +const isTokenCount = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0 + +/** + * Diagnostics sink for malformed counters. + * + * The shared `logger` export in `utils/logging` is a no-op outside tests + * (`utils/logging/index.ts:25` selects `CompactLogger` only when + * `NODE_ENV === "test"`), so routing a production diagnostic through it would + * discard the message in every real install. This module owns its own + * `CompactLogger` instead, which is active regardless of `NODE_ENV`. + * + * `CompactLogger`'s constructor only assigns fields, so constructing it at + * module scope has no side effects. Its `write()` can throw while creating the + * log file, and this runs inside live usage accounting, so emission is + * guarded: a logging failure must never break billing or abort a stream. + */ +const usageLogger = new CompactLogger() + +/** + * Longest string preview kept in a diagnostic. A provider can send an + * arbitrarily large value, and the transport stringifies it and writes it + * synchronously to stdout and to the log file, so the payload is capped. + */ +const MAX_PREVIEW_CHARS = 64 + +/** + * Size-safe description of a discarded counter value. + * + * The raw value is never logged: it comes off the wire and is unbounded, while + * `CompactTransport.write` serializes and writes it synchronously. A number is + * at most a couple of dozen characters when stringified, so its value is both + * safe and the most diagnosable thing to keep. A string keeps its length plus a + * capped preview. Everything else keeps only its type. + */ +const describeMalformedCount = (value: unknown): { type: string; preview?: string | number; length?: number } => { + if (typeof value === "number") { + // JSON.stringify writes `null` for NaN and both infinities, which would make + // the three indistinguishable in the persisted log. They are stringified so + // the name survives serialization; finite numbers stay numeric. + return { type: "number", preview: Number.isFinite(value) ? value : String(value) } + } + + if (typeof value === "string") { + return { + type: "string", + length: value.length, + preview: value.length > MAX_PREVIEW_CHARS ? `${value.slice(0, MAX_PREVIEW_CHARS)}…` : value, + } + } + + if (Array.isArray(value)) return { type: "array" } + + return { type: value === null ? "null" : typeof value } +} + +const emitMalformedCountWarning = (path: string, value: unknown): void => { + try { + usageLogger.warn("Discarded a malformed token counter while normalizing chat usage", { + path, + ...describeMalformedCount(value), + }) + } catch { + // Diagnostics are best-effort; never let a sink failure escape into usage accounting. + } +} + +/** + * A counter that is absent or null was simply not reported, which is a normal + * shape and resolves to `undefined` silently. A counter that is present but + * unusable is a provider or proxy defect: it is still discarded, but it is + * reported so a billing regression cannot hide behind an otherwise valid + * response. The warning is emitted once per alias per process, because usage + * objects arrive on every streamed chunk and an unconditional warning would + * flood the log. + */ +const warnMalformedCount = (path: string, value: unknown): void => { + if (warnedMalformedCounters.has(path)) return + warnedMalformedCounters.add(path) + emitMalformedCountWarning(path, value) +} + +const warnedMalformedCounters = new Set() + +const asTokenCount = (value: unknown, path: string): number | undefined => { + if (isTokenCount(value)) return value + if (value !== undefined && value !== null) warnMalformedCount(path, value) + return undefined +} + +const resolveTokenCount = (usage: ChatUsageLike, aliases: readonly string[]): number | undefined => { + for (const alias of aliases) { + const resolved = asTokenCount(readPath(usage, alias), alias) + if (resolved !== undefined) return resolved + } + + return undefined +} + +/** + * Normalizes an OpenAI-shaped chat-completions usage object. + * + * Returns `undefined` for a missing usage object so callers can skip emitting a + * usage chunk entirely. Token counts that the provider did not report stay + * `undefined` rather than defaulting to 0, which keeps "no cache counter" and + * "zero cache hits" distinguishable for cost reporting. + * + * `cacheMissTokens` is reported but not merged into `cacheWriteTokens`: a miss + * is ordinary input, and only DeepSeek's catalog prices it as a write. Callers + * that bill misses as writes must opt in explicitly. + */ +export function normalizeChatUsage(usage: ChatUsageLike | null | undefined): NormalizedChatUsage | undefined { + if (!usage || typeof usage !== "object") return undefined + + const cacheReadTokens = resolveTokenCount(usage, CACHE_READ_ALIASES) + const cacheWriteTokens = resolveTokenCount(usage, CACHE_WRITE_ALIASES) + const cacheMissTokens = resolveTokenCount(usage, CACHE_MISS_ALIASES) + + return { + inputTokens: asTokenCount(usage.prompt_tokens, "prompt_tokens") ?? 0, + outputTokens: asTokenCount(usage.completion_tokens, "completion_tokens") ?? 0, + ...(cacheReadTokens !== undefined ? { cacheReadTokens } : {}), + ...(cacheWriteTokens !== undefined ? { cacheWriteTokens } : {}), + ...(cacheMissTokens !== undefined ? { cacheMissTokens } : {}), + } +} diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 754a2d84df..e7342eb34b 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -47,6 +47,10 @@ function calculateApiCostInternal( totalInputTokens: number, totalOutputTokens: number, ): ApiCostResult { + // A cache price the catalog does not declare bills at $0 by design: an + // undeclared read discount is never re-priced at the input price, and an + // unpriced model must not leak `undefined * 0` as `NaN`. A writes fallback + // needs its own kernel PR with catalog reachability evidence. const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b369a0405b..96724629f0 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -1,6 +1,6 @@ // npx vitest utils/__tests__/cost.spec.ts -import { OpenAiServiceTier, type ModelInfo } from "@roo-code/types" +import { OpenAiServiceTier, deepSeekModels, friendliModels, moonshotModels, type ModelInfo } from "@roo-code/types" import { calculateApiCostAnthropic, calculateApiCostOpenAI } from "../../shared/cost" @@ -95,7 +95,7 @@ describe("Cost Utility", () => { expect(result.totalOutputTokens).toBe(500) }) - it("should handle missing cache prices", () => { + it("bills cache tokens at $0 when the catalog omits cache prices", () => { const modelWithoutCachePrices: ModelInfo = { ...mockModelInfo, cacheWritesPrice: undefined, @@ -104,14 +104,49 @@ describe("Cost Utility", () => { const result = calculateApiCostAnthropic(modelWithoutCachePrices, 1000, 500, 2000, 3000) - // Should only include input and output costs + // A model that publishes no cache price declares no cache rate, so the + // cache buckets bill at $0 rather than borrowing the input price. // Input cost: (3.0 / 1_000_000) * 1000 = 0.003 + // Cache writes: 0 (no price declared) + // Cache reads: 0 (no price declared) // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.003 + 0.0075 = 0.0105 - expect(result.totalCost).toBe(0.0105) + // Total: 0.003 + 0 + 0 + 0.0075 = 0.0105 + expect(result.totalCost).toBeCloseTo(0.0105, 9) expect(result.totalInputTokens).toBe(6000) // 1000 + 2000 + 3000 expect(result.totalOutputTokens).toBe(500) }) + + it("honors an explicit zero cache price instead of billing the input price", () => { + // A provider that publishes `cacheWritesPrice: 0` charges nothing for + // writes. A zero is a price, not an absence, so the kernel must never + // re-price this row at the input price. + const modelWithFreeWrites: ModelInfo = { + ...mockModelInfo, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + } + + const result = calculateApiCostAnthropic(modelWithFreeWrites, 1000, 500, 2000, 3000) + + // Input cost: (3.0 / 1_000_000) * 1000 = 0.003 + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + // Cache writes and reads: 0 by the catalog's own price + expect(result.totalCost).toBeCloseTo(0.0105, 9) + }) + + it("bills nothing when the model declares no prices at all", () => { + // `undefined * 0` is NaN, so the fallback chain has to end in a real 0. + const unpricedModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + } + + const result = calculateApiCostAnthropic(unpricedModel, 1000, 500, 2000, 3000) + + expect(result.totalCost).toBe(0) + expect(Number.isNaN(result.totalCost)).toBe(false) + }) }) describe("calculateApiCostOpenAI", () => { @@ -204,7 +239,7 @@ describe("Cost Utility", () => { expect(result.totalOutputTokens).toBe(500) }) - it("should handle missing cache prices", () => { + it("bills cache tokens at $0 when the catalog omits cache prices", () => { const modelWithoutCachePrices: ModelInfo = { ...mockModelInfo, cacheWritesPrice: undefined, @@ -213,15 +248,143 @@ describe("Cost Utility", () => { const result = calculateApiCostOpenAI(modelWithoutCachePrices, 6000, 500, 2000, 3000) - // Should only include input and output costs - // Input cost: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // OpenAI usage counts cached tokens inside prompt_tokens, so the cache + // buckets are removed from the input pool and re-priced. With no cache + // price declared both buckets bill at $0. + // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // Cache writes: 0 (no price declared) + // Cache reads: 0 (no price declared) // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.003 + 0.0075 = 0.0105 - expect(result.totalCost).toBe(0.0105) + // Total: 0.003 + 0 + 0 + 0.0075 = 0.0105 + expect(result.totalCost).toBeCloseTo(0.0105, 9) expect(result.totalInputTokens).toBe(6000) // Total already includes cache expect(result.totalOutputTokens).toBe(500) }) + it("honors an explicit zero cache price instead of billing the input price", () => { + // friendli's GLM rows and every moonshot row publish `cacheWritesPrice: 0` + // on purpose. A zero is a price, not an absence, so those rows must not be + // re-priced at the input price. + const modelWithFreeWrites: ModelInfo = { + ...mockModelInfo, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + } + + const result = calculateApiCostOpenAI(modelWithFreeWrites, 6000, 500, 2000, 3000) + + // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + expect(result.totalCost).toBeCloseTo(0.0105, 9) + }) + + it("bills nothing when the model declares no prices at all", () => { + const unpricedModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + } + + const result = calculateApiCostOpenAI(unpricedModel, 6000, 500, 2000, 3000) + + expect(result.totalCost).toBe(0) + expect(Number.isNaN(result.totalCost)).toBe(false) + }) + + it("bills cache reads at $0 when cacheReadsPrice is missing", () => { + // The mistral invariant: reporting reads without a declared read price + // must not silently bill them at the input price. A missing read price + // bills $0, so this model's reads contribute nothing while its writes + // bill at their own declared price. + const modelWithoutReadsPrice: ModelInfo = { + ...mockModelInfo, + cacheWritesPrice: 3.75, + cacheReadsPrice: undefined, + } + + const result = calculateApiCostOpenAI(modelWithoutReadsPrice, 6000, 500, 2000, 3000) + + // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // Cache writes: (3.75 / 1_000_000) * 2000 = 0.0075 + // Cache reads: 0 (no price declared) + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + // Total: 0.003 + 0.0075 + 0 + 0.0075 = 0.018 + expect(result.totalCost).toBeCloseTo(0.018, 9) + + // Re-symmetrizing the reads arm to the input price would bill + // (3.0 / 1_000_000) * 3000 = 0.009 for reads and total 0.027. + expect(result.totalCost).not.toBeCloseTo(0.027, 9) + }) + + it("stays byte-identical for the DeepSeek catalog, whose input price is zero by design", () => { + // All four deepSeekModels rows set `inputPrice: 0` and price misses through + // cacheWritesPrice. A fallback that resolved through `inputPrice` would + // zero out their cache-write revenue, so the rows are pinned exactly. + expect(deepSeekModels["deepseek-flash"].inputPrice).toBe(0) + + const result = calculateApiCostOpenAI(deepSeekModels["deepseek-flash"], 1000, 500, 400, 100) + + // Cache writes: (0.3 / 1_000_000) * 400 = 0.00012 + // Cache reads: (0.006 / 1_000_000) * 100 = 0.0000006 + // Uncached input: (0 / 1_000_000) * 500 = 0 + // Output cost: (1.2 / 1_000_000) * 500 = 0.0006 + expect(result.totalCost).toBeCloseTo(0.0007206, 9) + }) + + describe("catalog rows with an explicit zero cache price", () => { + // These rows publish `cacheWritesPrice: 0` deliberately: the provider + // charges nothing for writes. They are the cells that a `||` fallback + // would silently re-price at the input price, so each is pinned to the + // exact total it bills today. + it.each([ + ["friendli GLM-5.2", friendliModels["zai-org/GLM-5.2"]], + ["moonshot kimi-k3", moonshotModels["kimi-k3"]], + ])("leaves %s byte-identical", (_name, modelInfo) => { + expect(modelInfo.cacheWritesPrice).toBe(0) + + const result = calculateApiCostOpenAI(modelInfo, 1000, 500, 400, 100) + + const expected = + ((modelInfo.inputPrice ?? 0) / 1_000_000) * 500 + // uncached input + ((modelInfo.outputPrice ?? 0) / 1_000_000) * 500 + // output + ((modelInfo.cacheReadsPrice ?? 0) / 1_000_000) * 100 // reads only; writes are free + + expect(result.totalCost).toBeCloseTo(expected, 9) + }) + }) + + it("keeps a custom model's miss tokens in the input pool instead of giving them away", () => { + // The defect this repairs: a custom OpenAI-compatible model that prices + // input but publishes no cache price. Miss tokens arrived as cache + // writes, were subtracted from the input pool, and were then priced at + // zero, so the user was billed for neither. The repair keeps a miss in + // the input bucket, where the input price bills it. + const customModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheReadsPrice: 0.3, + } + + // 1000 prompt tokens: 400 missed the cache, 100 hit it, 500 were plain. + // A miss reaches this kernel as ordinary input, never as a cache write. + const result = calculateApiCostOpenAI(customModel, 1000, 500, 0, 100) + + // Uncached input: (3.0 / 1_000_000) * 900 = 0.0027 + // Cache writes: 0 (none reported) + // Cache reads: (0.3 / 1_000_000) * 100 = 0.00003 + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + // Total: 0.0027 + 0 + 0.00003 + 0.0075 = 0.01023 + expect(result.totalCost).toBeCloseTo(0.01023, 9) + + // The under-bill the defect produced, kept as the regression's own number: + // folding the same 400 tokens onto an unpriced write bucket yields + // 0.00903, 11.7% short. + expect(result.totalCost).not.toBeCloseTo(0.00903, 9) + }) + it("should not apply long-context pricing at the threshold", () => { const modelWithLongContextPricing: ModelInfo = { ...mockModelInfo,