From c6773dfcc2e986832e0b4f178785248b458bafcc Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 00:16:10 +0900 Subject: [PATCH 01/12] fix(deepseek): read documented top-level prompt-cache counters What: - Add src/api/transform/chat-usage-normalization.ts, a shared normalizer that resolves OpenAI-shaped chat-completions usage into { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens }. - DeepSeekHandler.processUsageMetrics now uses the normalizer instead of reading usage.prompt_tokens_details.cache_miss_tokens directly. - Rewrite the deepseek spec fixtures to DeepSeek's documented wire shape and add top-level-only, nested-only-legacy, and both-shapes-present cases. Why: DeepSeek's OpenAPI marks prompt_cache_hit_tokens and prompt_cache_miss_tokens as required top-level members and declares only cached_tokens inside prompt_tokens_details. The nested cache_miss_tokens field the provider read does not exist, so cacheWriteTokens was permanently undefined. DeepSeek catalog entries set inputPrice: 0 (inputs are priced as cache read/write only), so calculateApiCostOpenAI priced every cache-miss input token at zero: a permanent, near-total under-report of DeepSeek input cost. The old spec hid this by mocking the non-existent nested shape and asserting cacheWriteTokens from it, so the suite was green because it mocked the bug. Read priority is per-field with no summing, because the two shapes describe the same quantities: - cacheWriteTokens: prompt_cache_miss_tokens, then nested cache_miss_tokens (legacy fallback), then nested cache_write_tokens. - cacheReadTokens: nested cached_tokens, then prompt_cache_hit_tokens, then cache_read_input_tokens. Impact: DeepSeek input cost is now reported instead of collapsing to ~$0 for cache-miss tokens. The helper is generic over OpenAI-shaped usage so the family-wide provider rewiring can adopt it without a second implementation. No behavior change for providers that do not use the helper. Refs: https://api-docs.deepseek.com/api/create-chat-completion https://api-docs.deepseek.com/api/create-completion --- src/api/providers/__tests__/deepseek.spec.ts | 89 +++++++++---- src/api/providers/deepseek.ts | 16 ++- .../chat-usage-normalization.spec.ts | 124 ++++++++++++++++++ src/api/transform/chat-usage-normalization.ts | 121 +++++++++++++++++ 4 files changed, 323 insertions(+), 27 deletions(-) create mode 100644 src/api/transform/__tests__/chat-usage-normalization.spec.ts create mode 100644 src/api/transform/chat-usage-normalization.ts diff --git a/src/api/providers/__tests__/deepseek.spec.ts b/src/api/providers/__tests__/deepseek.spec.ts index 7878eaa2e1..3b6c2496c9 100644 --- a/src/api/providers/__tests__/deepseek.spec.ts +++ b/src/api/providers/__tests__/deepseek.spec.ts @@ -27,9 +27,10 @@ vi.mock("openai", () => { completion_tokens: 5, total_tokens: 15, prompt_tokens_details: { - cache_miss_tokens: 8, cached_tokens: 2, }, + prompt_cache_hit_tokens: 2, + prompt_cache_miss_tokens: 8, }, } } @@ -111,9 +112,10 @@ vi.mock("openai", () => { completion_tokens: 5, total_tokens: 15, prompt_tokens_details: { - cache_miss_tokens: 8, cached_tokens: 2, }, + prompt_cache_hit_tokens: 2, + prompt_cache_miss_tokens: 8, }, }) @@ -517,17 +519,55 @@ describe("DeepSeekHandler", () => { }) describe("processUsageMetrics", () => { - it("should correctly process usage metrics including cache information", () => { - // We need to access the protected method, so we'll create a test subclass - class TestDeepSeekHandler extends DeepSeekHandler { - public testProcessUsageMetrics(usage: any) { - return this.processUsageMetrics(usage) - } + class TestDeepSeekHandler extends DeepSeekHandler { + public testProcessUsageMetrics(usage: any) { + return this.processUsageMetrics(usage) } + } - const testHandler = new TestDeepSeekHandler(mockOptions) + // mockOptions is assigned in beforeEach, so the handler must be built + // per test rather than at describe-collection time. + const processUsage = (usage: any) => new TestDeepSeekHandler(mockOptions).testProcessUsageMetrics(usage) + + // DeepSeek's OpenAPI marks prompt_cache_hit_tokens and + // prompt_cache_miss_tokens as required top-level members, and declares + // only cached_tokens inside prompt_tokens_details. This is the vendor's + // own documented example response. + // https://api-docs.deepseek.com/api/create-chat-completion + it("reads the documented top-level cache counters", () => { + const result = processUsage({ + prompt_tokens: 7, + completion_tokens: 11, + total_tokens: 18, + prompt_tokens_details: { cached_tokens: 0 }, + prompt_cache_hit_tokens: 0, + prompt_cache_miss_tokens: 7, + }) - const usage = { + expect(result.type).toBe("usage") + expect(result.inputTokens).toBe(7) + expect(result.outputTokens).toBe(11) + expect(result.cacheReadTokens).toBe(0) + expect(result.cacheWriteTokens).toBe(7) + }) + + it("reads top-level cache counters when prompt_tokens_details is absent", () => { + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 80, + }) + + expect(result.inputTokens).toBe(100) + expect(result.outputTokens).toBe(50) + expect(result.cacheWriteTokens).toBe(80) + expect(result.cacheReadTokens).toBe(20) + }) + + it("falls back to the nested cache counters for legacy shapes", () => { + const result = processUsage({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150, @@ -535,34 +575,37 @@ describe("DeepSeekHandler", () => { cache_miss_tokens: 80, cached_tokens: 20, }, - } - - const result = testHandler.testProcessUsageMetrics(usage) + }) - expect(result.type).toBe("usage") expect(result.inputTokens).toBe(100) expect(result.outputTokens).toBe(50) expect(result.cacheWriteTokens).toBe(80) expect(result.cacheReadTokens).toBe(20) }) - it("should handle missing cache metrics gracefully", () => { - class TestDeepSeekHandler extends DeepSeekHandler { - public testProcessUsageMetrics(usage: any) { - return this.processUsageMetrics(usage) - } - } + it("does not double-count when both shapes are present", () => { + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + prompt_tokens_details: { cached_tokens: 20, cache_miss_tokens: 80 }, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 80, + }) - const testHandler = new TestDeepSeekHandler(mockOptions) + expect(result.cacheWriteTokens).toBe(80) + expect(result.cacheReadTokens).toBe(20) + }) + it("should handle missing cache metrics gracefully", () => { const usage = { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150, - // No prompt_tokens_details + // No prompt_tokens_details and no top-level cache counters } - const result = testHandler.testProcessUsageMetrics(usage) + const result = processUsage(usage) expect(result.type).toBe("usage") expect(result.inputTokens).toBe(100) diff --git a/src/api/providers/deepseek.ts b/src/api/providers/deepseek.ts index b65973ac96..25e660abec 100644 --- a/src/api/providers/deepseek.ts +++ b/src/api/providers/deepseek.ts @@ -15,6 +15,7 @@ import type { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { convertToR1Format } from "../transform/r1-format" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" import { OpenAiHandler } from "./openai" import { NOT_PROVIDED } from "./constants" @@ -227,13 +228,20 @@ export class DeepSeekHandler extends OpenAiHandler { } // Override to handle DeepSeek's usage metrics, including caching. + // DeepSeek reports cache counters in two shapes that describe the same + // quantities: top-level `prompt_cache_hit_tokens` / `prompt_cache_miss_tokens` + // and nested `prompt_tokens_details.cached_tokens`. The normalizer resolves + // one value per field and never sums the shapes. + // https://api-docs.deepseek.com/api/create-chat-completion protected override processUsageMetrics(usage: any, _modelInfo?: any): ApiStreamUsageChunk { + const normalized = normalizeChatUsage(usage) + return { type: "usage", - inputTokens: usage?.prompt_tokens || 0, - outputTokens: usage?.completion_tokens || 0, - cacheWriteTokens: usage?.prompt_tokens_details?.cache_miss_tokens, - cacheReadTokens: usage?.prompt_tokens_details?.cached_tokens, + inputTokens: normalized?.inputTokens ?? 0, + outputTokens: normalized?.outputTokens ?? 0, + cacheWriteTokens: normalized?.cacheWriteTokens, + cacheReadTokens: normalized?.cacheReadTokens, } } } diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts new file mode 100644 index 0000000000..581e995923 --- /dev/null +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -0,0 +1,124 @@ +// npx vitest run api/transform/__tests__/chat-usage-normalization.spec.ts + +import { normalizeChatUsage } from "../chat-usage-normalization" + +// Documented DeepSeek usage block, copied from the vendor's OpenAPI example. +// https://api-docs.deepseek.com/api/create-chat-completion +const documentedDeepSeekUsage = { + prompt_tokens: 7, + completion_tokens: 11, + total_tokens: 18, + prompt_tokens_details: { cached_tokens: 0 }, + prompt_cache_hit_tokens: 0, + prompt_cache_miss_tokens: 7, +} + +describe("normalizeChatUsage", () => { + it("returns undefined when the provider sent no usage object", () => { + expect(normalizeChatUsage(undefined)).toBeUndefined() + expect(normalizeChatUsage(null)).toBeUndefined() + }) + + it("returns undefined for a truthy non-object usage value", () => { + // A malformed proxy can send a scalar where the usage object belongs. + expect(normalizeChatUsage("usage" as unknown as Record)).toBeUndefined() + expect(normalizeChatUsage(0 as unknown as Record)).toBeUndefined() + }) + + it("reads the documented DeepSeek top-level shape", () => { + expect(normalizeChatUsage(documentedDeepSeekUsage)).toEqual({ + inputTokens: 7, + outputTokens: 11, + cacheReadTokens: 0, + cacheWriteTokens: 7, + }) + }) + + it("reads a top-level-only usage object with no nested details", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + completion_tokens: 30, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 20, + }), + ).toEqual({ inputTokens: 120, outputTokens: 30, cacheReadTokens: 100, cacheWriteTokens: 20 }) + }) + + it("reads the nested-only shape used by OpenAI and by legacy DeepSeek fixtures", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 20, cache_miss_tokens: 80 }, + }), + ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, cacheWriteTokens: 80 }) + }) + + it("does not add the nested and top-level shapes together when both are present", () => { + // Both shapes describe the same 100 reads / 20 writes. Summing would + // report 200 / 40 and double the billed cache tokens. + const normalized = normalizeChatUsage({ + prompt_tokens: 120, + completion_tokens: 30, + prompt_tokens_details: { cached_tokens: 100, cache_miss_tokens: 20 }, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 20, + }) + + expect(normalized?.cacheReadTokens).toBe(100) + expect(normalized?.cacheWriteTokens).toBe(20) + }) + + it("prefers the nested cached_tokens over the top-level hit counter for reads", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cached_tokens: 100 }, + prompt_cache_hit_tokens: 90, + })?.cacheReadTokens, + ).toBe(100) + }) + + it("prefers the top-level miss counter over the nested one for writes", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cache_miss_tokens: 20 }, + prompt_cache_miss_tokens: 30, + })?.cacheWriteTokens, + ).toBe(30) + }) + + it("falls through a malformed alias to the next one", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_miss_tokens: "30", + prompt_tokens_details: { cache_miss_tokens: 20 }, + })?.cacheWriteTokens, + ).toBe(20) + }) + + it("leaves cache counters undefined when the provider reports none", () => { + const normalized = normalizeChatUsage({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }) + + expect(normalized).toEqual({ inputTokens: 100, outputTokens: 50 }) + expect(normalized).not.toHaveProperty("cacheReadTokens") + expect(normalized).not.toHaveProperty("cacheWriteTokens") + }) + + it("rejects negative and fractional token counts", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_hit_tokens: -1, + prompt_cache_miss_tokens: 1.5, + }), + ).toEqual({ inputTokens: 120, outputTokens: 0 }) + }) + + it("defaults missing prompt and completion tokens to zero", () => { + expect(normalizeChatUsage({})).toEqual({ inputTokens: 0, outputTokens: 0 }) + }) +}) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts new file mode 100644 index 0000000000..ef08b9508b --- /dev/null +++ b/src/api/transform/chat-usage-normalization.ts @@ -0,0 +1,121 @@ +/** + * Normalization of OpenAI-shaped chat-completions usage objects into the + * cache token fields Zoo Code bills on. + * + * Providers that speak the OpenAI chat-completions wire format disagree on + * where they put cache counters. Some publish them top level, some nest them + * under `prompt_tokens_details`, and some publish both shapes describing the + * same quantities. Every reader here therefore resolves a single field through + * an ordered list of aliases and stops at the first numeric value. The aliases + * are never summed: a response that carries both shapes reports the same + * tokens twice, so adding them would double-count. + */ + +/** + * A usage object as it arrives off the wire, before any provider-specific + * typing. Fields are deliberately `unknown`: providers type their own usage + * loosely and the normalizer is the single place that validates them. + */ +export interface ChatUsageLike { + prompt_tokens?: unknown + completion_tokens?: unknown + prompt_tokens_details?: unknown + [key: string]: unknown +} + +export interface NormalizedChatUsage { + /** Total prompt tokens, including cache reads and cache writes. */ + inputTokens: number + outputTokens: number + /** + * Tokens written into the provider's prompt cache on this request. + * `undefined` when the provider reported no such counter at all, which is + * distinct from a reported zero. + */ + cacheReadTokens?: number + cacheWriteTokens?: number +} + +/** + * Aliases for cache-write (cache-miss) tokens, highest priority first. + * + * DeepSeek documents `prompt_cache_miss_tokens` at the top level of the usage + * object and marks it required; its `prompt_tokens_details` schema declares + * only `cached_tokens`. `prompt_tokens_details.cache_miss_tokens` is not part + * of that schema and is kept last as a legacy fallback for proxies that + * forwarded a rewritten shape. + * + * https://api-docs.deepseek.com/api/create-chat-completion + */ +const CACHE_WRITE_ALIASES = [ + "prompt_cache_miss_tokens", + "prompt_tokens_details.cache_miss_tokens", + "prompt_tokens_details.cache_write_tokens", +] as const + +/** + * Aliases for cache-read (cache-hit) tokens, highest priority first. + * + * `prompt_tokens_details.cached_tokens` is the shape DeepSeek documents and + * also the shape OpenAI documents, so it wins. `prompt_cache_hit_tokens` is + * DeepSeek's required top-level counterpart and covers responses that omit the + * nested object. + * + * https://api-docs.deepseek.com/api/create-chat-completion + */ +const CACHE_READ_ALIASES = [ + "prompt_tokens_details.cached_tokens", + "prompt_cache_hit_tokens", + "cache_read_input_tokens", +] as const + +const readPath = (usage: ChatUsageLike, path: string): unknown => { + let current: unknown = usage + + for (const segment of path.split(".")) { + if (typeof current !== "object" || current === null) return undefined + current = (current as Record)[segment] + } + + return current +} + +/** + * Counters are token counts: only non-negative integers are accepted. Anything + * else (missing, null, a string, a float, a negative sentinel) is treated as + * "not reported" so a malformed field falls through to the next alias instead + * of poisoning the total. + */ +const asTokenCount = (value: unknown): number | undefined => + typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined + +const resolveTokenCount = (usage: ChatUsageLike, aliases: readonly string[]): number | undefined => { + for (const alias of aliases) { + const resolved = asTokenCount(readPath(usage, alias)) + if (resolved !== undefined) return resolved + } + + return undefined +} + +/** + * Normalizes an OpenAI-shaped chat-completions usage object. + * + * Returns `undefined` for a missing usage object so callers can skip emitting a + * usage chunk entirely. Token counts that the provider did not report stay + * `undefined` rather than defaulting to 0, which keeps "no cache counter" and + * "zero cache hits" distinguishable for cost reporting. + */ +export function normalizeChatUsage(usage: ChatUsageLike | null | undefined): NormalizedChatUsage | undefined { + if (!usage || typeof usage !== "object") return undefined + + const cacheReadTokens = resolveTokenCount(usage, CACHE_READ_ALIASES) + const cacheWriteTokens = resolveTokenCount(usage, CACHE_WRITE_ALIASES) + + return { + inputTokens: asTokenCount(usage.prompt_tokens) ?? 0, + outputTokens: asTokenCount(usage.completion_tokens) ?? 0, + ...(cacheReadTokens !== undefined ? { cacheReadTokens } : {}), + ...(cacheWriteTokens !== undefined ? { cacheWriteTokens } : {}), + } +} From aed82bbcd5a8a5ffcb6f8cbc2497687b54cb1da2 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 00:49:54 +0900 Subject: [PATCH 02/12] fix(usage): add LiteLLM write alias and surface malformed counters What: add `cache_creation_input_tokens` to the cache-write alias list at the lowest priority, behind every DeepSeek alias, and replace the silent `asTokenCount` fall-through with an explicit guard that warns once per alias when a present counter is unusable. Why: two P2 findings from the #2252 review pair. glm F-1: the helper's write list omitted `cache_creation_input_tokens`, which lite-llm.ts:390-395 reads today, so adopting the helper verbatim would zero LiteLLM write accounting. mlarge P2: a present-but-malformed counter was discarded silently, which is the same silent under-billing class this wave fixes, reachable through a proxy quirk. Impact: DeepSeek resolution is unchanged (documented top-level miss counter still wins, aliases are never summed). Providers that publish only the Anthropic-style write counter now bill cache writes instead of reporting none. Malformed counters still fall through, but log once per alias. Refs: docs/261031_0001_session_caching-diet-wave/code-reviewer-report-pr-c1-glm.md F-1, code-reviewer-report-pr-c1-mlarge.md [P2] --- .../chat-usage-normalization.spec.ts | 128 ++++++++++++++++++ src/api/transform/chat-usage-normalization.ts | 41 +++++- 2 files changed, 164 insertions(+), 5 deletions(-) diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index 581e995923..99fa188d2e 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -1,5 +1,7 @@ // npx vitest run api/transform/__tests__/chat-usage-normalization.spec.ts +import { logger } from "../../../utils/logging" + import { normalizeChatUsage } from "../chat-usage-normalization" // Documented DeepSeek usage block, copied from the vendor's OpenAPI example. @@ -100,6 +102,132 @@ describe("normalizeChatUsage", () => { ).toBe(20) }) + describe("cache_creation_input_tokens (LiteLLM write shape)", () => { + it("reads the LiteLLM top-level write counter when no DeepSeek write alias is present", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 100, + completion_tokens: 50, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }), + ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 30, cacheWriteTokens: 20 }) + }) + + it("keeps the documented DeepSeek top-level miss counter ahead of it", () => { + // Both describe the same writes. DeepSeek's documented shape wins and + // the two are never added together. + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_miss_tokens: 20, + cache_creation_input_tokens: 20, + })?.cacheWriteTokens, + ).toBe(20) + }) + + it("keeps every DeepSeek write alias ahead of it", () => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cache_miss_tokens: 20 }, + cache_creation_input_tokens: 99, + })?.cacheWriteTokens, + ).toBe(20) + + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_tokens_details: { cache_write_tokens: 20 }, + cache_creation_input_tokens: 99, + })?.cacheWriteTokens, + ).toBe(20) + }) + + it("does not sum it with a DeepSeek write alias reporting the same quantity", () => { + const normalized = normalizeChatUsage({ + prompt_tokens: 120, + completion_tokens: 30, + prompt_cache_miss_tokens: 20, + prompt_tokens_details: { cache_miss_tokens: 20, cache_write_tokens: 20 }, + cache_creation_input_tokens: 20, + }) + + expect(normalized?.cacheWriteTokens).toBe(20) + }) + }) + + describe("malformed counter hardening", () => { + it.each([ + ["a numeric string", "30"], + ["a fractional number", 1.5], + ["a negative number", -1], + ["NaN", NaN], + ["Infinity", Infinity], + ["an object", { tokens: 30 }], + ["a boolean", true], + ])("discards %s and falls through to the next alias", (_name, malformed) => { + expect( + normalizeChatUsage({ + prompt_tokens: 120, + prompt_cache_miss_tokens: malformed, + prompt_tokens_details: { cache_miss_tokens: 20 }, + })?.cacheWriteTokens, + ).toBe(20) + }) + + it("reports a present-but-malformed counter instead of discarding it silently", async () => { + // A fresh module instance keeps this case independent of the dedupe + // state that earlier cases in this file have already populated. + vi.resetModules() + const { logger: freshLogger } = await import("../../../utils/logging") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(freshLogger, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + + expect(warn).toHaveBeenCalledWith(expect.any(String), { + path: "prompt_cache_miss_tokens", + value: "30", + }) + } finally { + warn.mockRestore() + } + }) + + it("reports a malformed counter once per alias rather than on every chunk", async () => { + vi.resetModules() + const { logger: freshLogger } = await import("../../../utils/logging") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(freshLogger, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + + expect(warn).toHaveBeenCalledTimes(1) + } finally { + warn.mockRestore() + } + }) + + it("stays silent for counters the provider simply did not report", () => { + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}) + + try { + normalizeChatUsage({ prompt_tokens: 100, completion_tokens: 50 }) + normalizeChatUsage({ prompt_tokens: 100, prompt_tokens_details: null }) + normalizeChatUsage({ prompt_tokens: 100, prompt_cache_hit_tokens: null }) + + expect(warn).not.toHaveBeenCalled() + } finally { + warn.mockRestore() + } + }) + }) + it("leaves cache counters undefined when the provider reports none", () => { const normalized = normalizeChatUsage({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts index ef08b9508b..1ba50363c6 100644 --- a/src/api/transform/chat-usage-normalization.ts +++ b/src/api/transform/chat-usage-normalization.ts @@ -11,6 +11,8 @@ * tokens twice, so adding them would double-count. */ +import { logger } from "../../utils/logging" + /** * A usage object as it arrives off the wire, before any provider-specific * typing. Fields are deliberately `unknown`: providers type their own usage @@ -45,12 +47,18 @@ export interface NormalizedChatUsage { * of that schema and is kept last as a legacy fallback for proxies that * forwarded a rewritten shape. * + * `cache_creation_input_tokens` is the Anthropic-style top-level counter that + * LiteLLM publishes for cached writes. It sits below every DeepSeek alias so + * the documented DeepSeek shape keeps priority, and it is reached only by + * providers that publish no DeepSeek counter at all. + * * https://api-docs.deepseek.com/api/create-chat-completion */ const CACHE_WRITE_ALIASES = [ "prompt_cache_miss_tokens", "prompt_tokens_details.cache_miss_tokens", "prompt_tokens_details.cache_write_tokens", + "cache_creation_input_tokens", ] as const /** @@ -86,12 +94,35 @@ const readPath = (usage: ChatUsageLike, path: string): unknown => { * "not reported" so a malformed field falls through to the next alias instead * of poisoning the total. */ -const asTokenCount = (value: unknown): number | undefined => - typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined +const isTokenCount = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0 + +/** + * A counter that is absent or null was simply not reported, which is a normal + * shape and resolves to `undefined` silently. A counter that is present but + * unusable is a provider or proxy defect: it is still discarded, but it is + * reported so a billing regression cannot hide behind an otherwise valid + * response. The warning is emitted once per alias per process, because usage + * objects arrive on every streamed chunk and an unconditional warning would + * flood the log. + */ +const warnMalformedCount = (path: string, value: unknown): void => { + if (warnedMalformedCounters.has(path)) return + warnedMalformedCounters.add(path) + logger.warn("Discarded a malformed token counter while normalizing chat usage", { path, value }) +} + +const warnedMalformedCounters = new Set() + +const asTokenCount = (value: unknown, path?: string): number | undefined => { + if (isTokenCount(value)) return value + if (value !== undefined && value !== null) warnMalformedCount(path ?? "unknown", value) + return undefined +} const resolveTokenCount = (usage: ChatUsageLike, aliases: readonly string[]): number | undefined => { for (const alias of aliases) { - const resolved = asTokenCount(readPath(usage, alias)) + const resolved = asTokenCount(readPath(usage, alias), alias) if (resolved !== undefined) return resolved } @@ -113,8 +144,8 @@ export function normalizeChatUsage(usage: ChatUsageLike | null | undefined): Nor const cacheWriteTokens = resolveTokenCount(usage, CACHE_WRITE_ALIASES) return { - inputTokens: asTokenCount(usage.prompt_tokens) ?? 0, - outputTokens: asTokenCount(usage.completion_tokens) ?? 0, + inputTokens: asTokenCount(usage.prompt_tokens, "prompt_tokens") ?? 0, + outputTokens: asTokenCount(usage.completion_tokens, "completion_tokens") ?? 0, ...(cacheReadTokens !== undefined ? { cacheReadTokens } : {}), ...(cacheWriteTokens !== undefined ? { cacheWriteTokens } : {}), } From 65f6249b7aa51a0ed32cea2ae4b87e2ed379200f Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 00:53:20 +0900 Subject: [PATCH 03/12] fix(providers): adopt chat usage normalization across the OpenAI-compatible family What: route cache accounting in OpenAiHandler and BaseOpenAiCompatibleProvider through normalizeChatUsage. Why: FINAL-PLAN K7. OpenAiHandler read only the top-level cache_creation_input_tokens for writes, so a proxy reporting writes under prompt_tokens_details.cache_write_tokens or prompt_cache_miss_tokens billed them as zero. BaseOpenAiCompatibleProvider read only the nested prompt_tokens_details shape, so its cache reads and writes silently missed every top-level counter. Impact: both handlers now resolve one value per field across the documented OpenAI-compatible shapes and never sum two shapes that describe the same quantity. OpenAiHandler reads keep the #1662 validated subset semantics unchanged; only the write resolution moved to the shared helper. BaseOpenAiCompatibleProvider is the only reader of its usage object, so nothing is mapped twice. DeepSeek keeps its own override and is untouched. lite-llm.ts is deliberately not adopted: its local fallback is not behavior-identical to the helper (see report). Refs: docs/261030_0005_session_diet-caching-validation/FINAL-PLAN.md K7 --- .../base-openai-compatible-provider.spec.ts | 108 ++++++++++++++++++ .../__tests__/openai-usage-tracking.spec.ts | 79 +++++++++++++ .../base-openai-compatible-provider.ts | 13 ++- src/api/providers/openai.ts | 6 +- 4 files changed, 201 insertions(+), 5 deletions(-) diff --git a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts index 1ffd63aa12..7bb152db92 100644 --- a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts +++ b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts @@ -297,6 +297,114 @@ describe("BaseOpenAiCompatibleProvider", () => { expect(firstChunk.done).toBe(false) expect(firstChunk.value).toMatchObject({ type: "usage", inputTokens: 100, outputTokens: 50 }) }) + + describe("cache accounting", () => { + const usageChunksFor = async (usage: Record) => { + mockCreate.mockImplementationOnce(() => asyncStreamFrom([{ choices: [{ delta: {} }], usage }])) + + const chunks = await collectStream(handler.createMessage("system prompt", [])) + + return chunks.filter((chunk) => chunk.type === "usage") + } + + it("reads the nested shape it already understood", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }), + ]) + }) + + it("reads the top-level cache_creation_input_tokens write shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + cache_creation_input_tokens: 30, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 30, + }), + ]) + }) + + it("reads the top-level cache_read_input_tokens read shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + cache_read_input_tokens: 20, + }), + ).toEqual([ + expect.objectContaining({ type: "usage", inputTokens: 100, outputTokens: 50, cacheReadTokens: 20 }), + ]) + }) + + it("reads the DeepSeek top-level hit and miss counters", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 30, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }), + ]) + }) + + it("never sums two shapes that report the same quantity", async () => { + // All shapes below describe the same 20 reads and 30 writes. An + // additive reader would bill 60 reads and 90 writes. + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 30, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 30, + }), + ).toEqual([ + expect.objectContaining({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheWriteTokens: 30, + }), + ]) + }) + + it("reports no cache counters when the provider reports none", async () => { + const [chunk] = await usageChunksFor({ prompt_tokens: 100, completion_tokens: 50 }) + + expect(chunk).toMatchObject({ type: "usage", inputTokens: 100, outputTokens: 50 }) + expect(chunk.cacheReadTokens).toBeUndefined() + expect(chunk.cacheWriteTokens).toBeUndefined() + }) + }) }) describe("getModel", () => { diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 0441270498..60eeec02aa 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -428,4 +428,83 @@ describe("OpenAiHandler with usage tracking fix", () => { ]) }, ) + + describe.each([true, false])("cache-write shape coverage with streaming=%s", (openAiStreamingEnabled) => { + const usageChunksFor = async (usage: Record) => { + const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled }) + mockCreate.mockResolvedValueOnce( + openAiStreamingEnabled + ? asyncStreamFrom([{ choices: [], usage }]) + : { choices: [{ message: { content: "Response" } }], usage }, + ) + + const chunks = await collectStream(cacheHandler.createMessage("system prompt", [])) + + return chunks.filter((chunk) => chunk.type === "usage") + } + + it("reads the nested cache_write_tokens shape it previously ignored", async () => { + // This is the OpenAiHandler write gap K7 names: the handler read only + // the top-level Anthropic-style counter, so a proxy reporting writes + // under prompt_tokens_details billed them as zero. + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30, cacheReadTokens: 20 }]) + }) + + it("still reads the top-level cache_creation_input_tokens shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + cache_creation_input_tokens: 7, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 7 }]) + }) + + it("reads the top-level prompt_cache_miss_tokens shape", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_cache_miss_tokens: 12, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 12 }]) + }) + + it("never sums two write shapes that report the same quantity", async () => { + // Every shape below describes the same 30 written tokens. Summing + // would bill 90 and is the additive-usage trap the wave forbids. + const chunks = await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_cache_miss_tokens: 30, + cache_creation_input_tokens: 30, + prompt_tokens_details: { cache_miss_tokens: 30, cache_write_tokens: 30 }, + }) + + expect(chunks).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30 }]) + }) + + it("keeps the documented DeepSeek write counter ahead of the Anthropic-style one", async () => { + expect( + await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_cache_miss_tokens: 30, + cache_creation_input_tokens: 99, + }), + ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30 }]) + }) + + it("reports no write counter when the provider reports none", async () => { + expect(await usageChunksFor({ prompt_tokens: 100, completion_tokens: 5 })).toEqual([ + { type: "usage", inputTokens: 100, outputTokens: 5 }, + ]) + }) + }) }) diff --git a/src/api/providers/base-openai-compatible-provider.ts b/src/api/providers/base-openai-compatible-provider.ts index b177726f85..6c27f8fe29 100644 --- a/src/api/providers/base-openai-compatible-provider.ts +++ b/src/api/providers/base-openai-compatible-provider.ts @@ -7,6 +7,7 @@ import { type ApiHandlerOptions, getModelMaxOutputTokens } from "../../shared/ap import { TagMatcher } from "../../utils/tag-matcher" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index" import { DEFAULT_HEADERS } from "./constants" @@ -193,10 +194,14 @@ export abstract class BaseOpenAiCompatibleProvider } protected processUsageMetrics(usage: any, modelInfo?: any): ApiStreamUsageChunk { - const inputTokens = usage?.prompt_tokens || 0 - const outputTokens = usage?.completion_tokens || 0 - const cacheWriteTokens = usage?.prompt_tokens_details?.cache_write_tokens || 0 - const cacheReadTokens = usage?.prompt_tokens_details?.cached_tokens || 0 + // The normalizer resolves one value per field across the OpenAI-compatible + // cache shapes. It is the only reader here, so nothing is mapped twice and + // two shapes describing the same quantity are never summed. + const normalized = normalizeChatUsage(usage) + const inputTokens = normalized?.inputTokens ?? 0 + const outputTokens = normalized?.outputTokens ?? 0 + const cacheWriteTokens = normalized?.cacheWriteTokens ?? 0 + const cacheReadTokens = normalized?.cacheReadTokens ?? 0 const { totalCost } = modelInfo ? calculateApiCostOpenAI(modelInfo, inputTokens, outputTokens, cacheWriteTokens, cacheReadTokens) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index 98ea18df8a..4182504cb9 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -21,6 +21,7 @@ import { TagMatcher } from "../../utils/tag-matcher" import { convertToOpenAiMessages } from "../transform/openai-format" import { sanitizeSurrogates } from "../transform/sanitize-surrogates" import { convertToR1Format } from "../transform/r1-format" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" @@ -317,7 +318,10 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl type: "usage", inputTokens, outputTokens: usage?.completion_tokens || 0, - cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, + // Reads keep their provider-specific validation above; writes are + // plain counters, so the shared normalizer resolves them across the + // OpenAI-compatible write shapes instead of reading one field. + cacheWriteTokens: normalizeChatUsage(usage)?.cacheWriteTokens || undefined, cacheReadTokens, } } From b0567c5cb8b41e110d4582d1c48b61e1da0a514e Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 01:05:43 +0900 Subject: [PATCH 04/12] test(usage): pin the malformed-counter warning and drop a dead fallback What: make the alias path a required argument of asTokenCount, add a spec case that pins the warning message and the exact field names it reports, and document why the typeof guard stays in isTokenCount. Why: mutation testing on the helper left four surviving StringLiteral mutants and one NoCoverage mutant on the `path ?? "unknown"` fallback. The fallback was unreachable because every caller passes an alias, and the warning text and field names were unpinned, so a future edit could silently rename the diagnostic that makes a billing regression visible. Impact: no behavior change for valid or malformed counters. The diagnostic is now covered by a test. Mutation score on the helper: 65 killed, 1 survived (provably equivalent typeof guard), 0 no-coverage. --- .../chat-usage-normalization.spec.ts | 22 +++++++++++++++++++ src/api/transform/chat-usage-normalization.ts | 8 +++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index 99fa188d2e..c38d80da90 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -213,6 +213,28 @@ describe("normalizeChatUsage", () => { } }) + it("names the exact field that was discarded", async () => { + vi.resetModules() + const { logger: freshLogger } = await import("../../../utils/logging") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(freshLogger, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: "100", completion_tokens: -1 }) + + expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { + path: "prompt_tokens", + value: "100", + }) + expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { + path: "completion_tokens", + value: -1, + }) + } finally { + warn.mockRestore() + } + }) + it("stays silent for counters the provider simply did not report", () => { const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts index 1ba50363c6..a8fd5a4dc0 100644 --- a/src/api/transform/chat-usage-normalization.ts +++ b/src/api/transform/chat-usage-normalization.ts @@ -93,6 +93,10 @@ const readPath = (usage: ChatUsageLike, path: string): unknown => { * else (missing, null, a string, a float, a negative sentinel) is treated as * "not reported" so a malformed field falls through to the next alias instead * of poisoning the total. + * + * `typeof value === "number"` is redundant for the check itself, because + * `Number.isInteger` already rejects every non-number. It is kept so the type + * predicate narrows to `number` and callers can return the value typed. */ const isTokenCount = (value: unknown): value is number => typeof value === "number" && Number.isInteger(value) && value >= 0 @@ -114,9 +118,9 @@ const warnMalformedCount = (path: string, value: unknown): void => { const warnedMalformedCounters = new Set() -const asTokenCount = (value: unknown, path?: string): number | undefined => { +const asTokenCount = (value: unknown, path: string): number | undefined => { if (isTokenCount(value)) return value - if (value !== undefined && value !== null) warnMalformedCount(path ?? "unknown", value) + if (value !== undefined && value !== null) warnMalformedCount(path, value) return undefined } From 41d6ab0de507c2823da610f584e3d857018575c3 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 02:23:52 +0900 Subject: [PATCH 05/12] fix(usage): route malformed-counter diagnostics to a production-active sink What: replace the `logger` import in chat-usage-normalization.ts with a module-local `CompactLogger` instance, and guard the emit so a sink failure cannot escape into usage accounting. Rewrite the spec cases to spy on `CompactLogger.prototype.warn`, and add two cases: one that simulates the production branch (NODE_ENV=production) and asserts the warning still reaches a real logger, and one that asserts a throwing sink does not abort accounting. Why: CodeRabbit thread r4426014105 (and reviewer F-01). The exported `logger` is `process.env.NODE_ENV === "test" ? new CompactLogger() : noopLogger`, so every `logger.warn` in production is discarded. The hardening added for the mlarge P2 finding therefore promised a diagnostic that no real install would ever emit. The sink is now active regardless of NODE_ENV. Impact: malformed-counter warnings are actually emitted in production, once per alias, with the same dedupe semantics as before. Valid-shape behavior is unchanged: all pre-existing helper cases pass unmodified. A broken log sink degrades to silence rather than surfacing as a usage failure. Disclosed separately, NOT fixed here: `src/utils/logging/index.ts:25` selects CompactLogger only when NODE_ENV === "test", which is the inverse of its own docstring ("Uses CompactLogger for normal operation, switches to noop logger in Jest test environment"). Correcting that ternary would change logging for every extension call site, so it is reported as a pre-existing repo-wide finding rather than widened into this PR. Refs: https://github.com/Zoo-Code-Org/Zoo-Code/pull/2254#discussion_r4426014105 --- .../chat-usage-normalization.spec.ts | 65 ++++++++++++++++--- src/api/transform/chat-usage-normalization.ts | 28 +++++++- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index c38d80da90..422a5d5303 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -1,6 +1,6 @@ // npx vitest run api/transform/__tests__/chat-usage-normalization.spec.ts -import { logger } from "../../../utils/logging" +import { CompactLogger } from "../../../utils/logging/CompactLogger" import { normalizeChatUsage } from "../chat-usage-normalization" @@ -180,9 +180,9 @@ describe("normalizeChatUsage", () => { // A fresh module instance keeps this case independent of the dedupe // state that earlier cases in this file have already populated. vi.resetModules() - const { logger: freshLogger } = await import("../../../utils/logging") + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") - const warn = vi.spyOn(freshLogger, "warn").mockImplementation(() => {}) + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) try { freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) @@ -196,11 +196,60 @@ describe("normalizeChatUsage", () => { } }) + it("emits through a sink that is active outside tests", async () => { + // The shared `logger` export is a no-op unless NODE_ENV is "test", so a + // diagnostic routed through it is discarded in every real install. This + // case simulates the production branch and asserts the warning still + // reaches a real logger. + const originalNodeEnv = process.env.NODE_ENV + process.env.NODE_ENV = "production" + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) + + expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { + path: "prompt_cache_miss_tokens", + value: "30", + }) + } finally { + warn.mockRestore() + process.env.NODE_ENV = originalNodeEnv + } + }) + + it("does not abort usage accounting when the sink itself fails", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => { + throw new Error("log file unwritable") + }) + + try { + // The malformed counter is still discarded and the valid alias still + // resolves; a broken log sink must not surface as a usage failure. + expect( + freshNormalize({ + prompt_tokens: 120, + prompt_cache_miss_tokens: "30", + prompt_tokens_details: { cache_miss_tokens: 20 }, + })?.cacheWriteTokens, + ).toBe(20) + } finally { + warn.mockRestore() + } + }) + it("reports a malformed counter once per alias rather than on every chunk", async () => { vi.resetModules() - const { logger: freshLogger } = await import("../../../utils/logging") + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") - const warn = vi.spyOn(freshLogger, "warn").mockImplementation(() => {}) + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) try { freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "30" }) @@ -215,9 +264,9 @@ describe("normalizeChatUsage", () => { it("names the exact field that was discarded", async () => { vi.resetModules() - const { logger: freshLogger } = await import("../../../utils/logging") + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") - const warn = vi.spyOn(freshLogger, "warn").mockImplementation(() => {}) + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) try { freshNormalize({ prompt_tokens: "100", completion_tokens: -1 }) @@ -236,7 +285,7 @@ describe("normalizeChatUsage", () => { }) it("stays silent for counters the provider simply did not report", () => { - const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}) + const warn = vi.spyOn(CompactLogger.prototype, "warn").mockImplementation(() => {}) try { normalizeChatUsage({ prompt_tokens: 100, completion_tokens: 50 }) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts index a8fd5a4dc0..99abbc4dca 100644 --- a/src/api/transform/chat-usage-normalization.ts +++ b/src/api/transform/chat-usage-normalization.ts @@ -11,7 +11,7 @@ * tokens twice, so adding them would double-count. */ -import { logger } from "../../utils/logging" +import { CompactLogger } from "../../utils/logging/CompactLogger" /** * A usage object as it arrives off the wire, before any provider-specific @@ -101,6 +101,30 @@ const readPath = (usage: ChatUsageLike, path: string): unknown => { const isTokenCount = (value: unknown): value is number => typeof value === "number" && Number.isInteger(value) && value >= 0 +/** + * Diagnostics sink for malformed counters. + * + * The shared `logger` export in `utils/logging` is a no-op outside tests + * (`utils/logging/index.ts:25` selects `CompactLogger` only when + * `NODE_ENV === "test"`), so routing a production diagnostic through it would + * discard the message in every real install. This module owns its own + * `CompactLogger` instead, which is active regardless of `NODE_ENV`. + * + * `CompactLogger`'s constructor only assigns fields, so constructing it at + * module scope has no side effects. Its `write()` can throw while creating the + * log file, and this runs inside live usage accounting, so emission is + * guarded: a logging failure must never break billing or abort a stream. + */ +const usageLogger = new CompactLogger() + +const emitMalformedCountWarning = (path: string, value: unknown): void => { + try { + usageLogger.warn("Discarded a malformed token counter while normalizing chat usage", { path, value }) + } catch { + // Diagnostics are best-effort; never let a sink failure escape into usage accounting. + } +} + /** * A counter that is absent or null was simply not reported, which is a normal * shape and resolves to `undefined` silently. A counter that is present but @@ -113,7 +137,7 @@ const isTokenCount = (value: unknown): value is number => const warnMalformedCount = (path: string, value: unknown): void => { if (warnedMalformedCounters.has(path)) return warnedMalformedCounters.add(path) - logger.warn("Discarded a malformed token counter while normalizing chat usage", { path, value }) + emitMalformedCountWarning(path, value) } const warnedMalformedCounters = new Set() From cf240159be2e1b11bc6c19f4f2f7ac74bd509650 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 02:51:59 +0900 Subject: [PATCH 06/12] fix(usage): log a bounded descriptor instead of the raw malformed value What: stop passing the raw provider value to the diagnostics sink. The warning now carries the value's type, plus a size-safe payload: the numeric value for numbers, and `length` plus a 64-character preview for strings. Everything else keeps only its type. Why: CodeRabbit thread r4426662673. `CompactTransport.write` serializes the meta object and writes it synchronously to stdout and to ./logs/app.log, so a large provider value would block the event loop and grow the log file. The value arrives straight off the wire and is unbounded, so it must not be logged verbatim. Impact: diagnostics stay useful (which field, what type, how big, and a short preview) while the emitted payload is bounded. A 1MB string now produces a 241-byte log line. Dedupe semantics are unchanged: one warning per alias per process. Tests: the four existing diagnostic cases now assert the descriptor shape rather than the raw value, and seven cases were added: one with a 1MB string asserting the payload stays bounded, and six covering the per-type descriptors. All ten fail on the previous head 41d6ab0de. Refs: https://github.com/Zoo-Code-Org/Zoo-Code/pull/2254#discussion_r4426662673 --- .../chat-usage-normalization.spec.ts | 64 +++++++++++++++++-- src/api/transform/chat-usage-normalization.ts | 39 ++++++++++- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index 422a5d5303..ad93ca2d06 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -189,7 +189,9 @@ describe("normalizeChatUsage", () => { expect(warn).toHaveBeenCalledWith(expect.any(String), { path: "prompt_cache_miss_tokens", - value: "30", + type: "string", + length: 2, + preview: "30", }) } finally { warn.mockRestore() @@ -214,7 +216,9 @@ describe("normalizeChatUsage", () => { expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { path: "prompt_cache_miss_tokens", - value: "30", + type: "string", + length: 2, + preview: "30", }) } finally { warn.mockRestore() @@ -273,17 +277,69 @@ describe("normalizeChatUsage", () => { expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { path: "prompt_tokens", - value: "100", + type: "string", + length: 3, + preview: "100", }) expect(warn).toHaveBeenCalledWith(expect.stringContaining("malformed token counter"), { path: "completion_tokens", - value: -1, + type: "number", + preview: -1, }) } finally { warn.mockRestore() } }) + it("keeps the logged payload bounded for an arbitrarily large provider value", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + const huge = "9".repeat(1_000_000) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: huge }) + + const [, meta] = warn.mock.calls[0] as [string, Record] + + // The raw megabyte must not reach the transport, which stringifies + // and writes it synchronously to stdout and the log file. + expect(meta).not.toHaveProperty("value") + expect(meta.type).toBe("string") + expect(meta.length).toBe(1_000_000) + expect(typeof meta.preview).toBe("string") + expect((meta.preview as string).length).toBeLessThanOrEqual(65) + expect(JSON.stringify(meta).length).toBeLessThan(200) + } finally { + warn.mockRestore() + } + }) + + it.each([ + ["a boolean", true, { type: "boolean" }], + ["an object", { tokens: 30 }, { type: "object" }], + ["an array", [1, 2, 3], { type: "array" }], + ["NaN", NaN, { type: "number", preview: NaN }], + ["Infinity", Infinity, { type: "number", preview: Infinity }], + ["a fractional number", 1.5, { type: "number", preview: 1.5 }], + ])("describes %s without logging the raw value", async (_name, malformed, expected) => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: malformed }) + + const [, meta] = warn.mock.calls[0] as [string, Record] + + expect(meta).toEqual({ path: "prompt_cache_miss_tokens", ...expected }) + } finally { + warn.mockRestore() + } + }) + it("stays silent for counters the provider simply did not report", () => { const warn = vi.spyOn(CompactLogger.prototype, "warn").mockImplementation(() => {}) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts index 99abbc4dca..3b4e990ca0 100644 --- a/src/api/transform/chat-usage-normalization.ts +++ b/src/api/transform/chat-usage-normalization.ts @@ -117,9 +117,46 @@ const isTokenCount = (value: unknown): value is number => */ const usageLogger = new CompactLogger() +/** + * Longest string preview kept in a diagnostic. A provider can send an + * arbitrarily large value, and the transport stringifies it and writes it + * synchronously to stdout and to the log file, so the payload is capped. + */ +const MAX_PREVIEW_CHARS = 64 + +/** + * Size-safe description of a discarded counter value. + * + * The raw value is never logged: it comes off the wire and is unbounded, while + * `CompactTransport.write` serializes and writes it synchronously. A number is + * at most a couple of dozen characters when stringified, so its value is both + * safe and the most diagnosable thing to keep. A string keeps its length plus a + * capped preview. Everything else keeps only its type. + */ +const describeMalformedCount = (value: unknown): { type: string; preview?: string | number; length?: number } => { + if (typeof value === "number") { + return { type: "number", preview: value } + } + + if (typeof value === "string") { + return { + type: "string", + length: value.length, + preview: value.length > MAX_PREVIEW_CHARS ? `${value.slice(0, MAX_PREVIEW_CHARS)}…` : value, + } + } + + if (Array.isArray(value)) return { type: "array" } + + return { type: value === null ? "null" : typeof value } +} + const emitMalformedCountWarning = (path: string, value: unknown): void => { try { - usageLogger.warn("Discarded a malformed token counter while normalizing chat usage", { path, value }) + usageLogger.warn("Discarded a malformed token counter while normalizing chat usage", { + path, + ...describeMalformedCount(value), + }) } catch { // Diagnostics are best-effort; never let a sink failure escape into usage accounting. } From d9b68e7da6e1492c6540ed815b05fe037e8b1265 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 03:58:00 +0900 Subject: [PATCH 07/12] fix(usage): stringify non-finite previews so they survive serialization What: stringify the numeric preview for non-finite values only, so NaN, Infinity and -Infinity keep their names through JSON serialization. Finite numbers stay numeric. Why: CodeRabbit thread r4427098612. CompactTransport.write persists the meta object with JSON.stringify, which writes `null` for all three non-finite values, so the stored warning showed `preview:null` and the three could not be told apart. Impact: one-line behavior change in describeMalformedCount. Finite numbers and strings serialize without loss and are untouched, so the previously pinned assertions for those shapes stay honest. Dedupe semantics unchanged. Tests: the NaN/Infinity cases now expect string previews, a -Infinity case was added, and one case asserts all three serialize to distinct, non-null previews. All four fail on the previous head cf240159b. Refs: https://github.com/Zoo-Code-Org/Zoo-Code/pull/2254#discussion_r4427098612 --- .../chat-usage-normalization.spec.ts | 38 ++++++++++++++++++- src/api/transform/chat-usage-normalization.ts | 5 ++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index ad93ca2d06..e7b8368305 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -320,8 +320,9 @@ describe("normalizeChatUsage", () => { ["a boolean", true, { type: "boolean" }], ["an object", { tokens: 30 }, { type: "object" }], ["an array", [1, 2, 3], { type: "array" }], - ["NaN", NaN, { type: "number", preview: NaN }], - ["Infinity", Infinity, { type: "number", preview: Infinity }], + ["NaN", NaN, { type: "number", preview: "NaN" }], + ["Infinity", Infinity, { type: "number", preview: "Infinity" }], + ["-Infinity", -Infinity, { type: "number", preview: "-Infinity" }], ["a fractional number", 1.5, { type: "number", preview: 1.5 }], ])("describes %s without logging the raw value", async (_name, malformed, expected) => { vi.resetModules() @@ -340,6 +341,39 @@ describe("normalizeChatUsage", () => { } }) + it("keeps non-finite previews distinguishable after JSON serialization", async () => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + // Distinct aliases so the per-path dedupe does not swallow two of the + // three warnings. + const serialized = [ + { prompt_cache_miss_tokens: NaN }, + { prompt_tokens_details: { cache_miss_tokens: Infinity } }, + { cache_creation_input_tokens: -Infinity }, + ].map((malformed) => { + freshNormalize({ prompt_tokens: 120, ...malformed }) + const [, meta] = warn.mock.calls[warn.mock.calls.length - 1] as [string, Record] + return JSON.stringify(meta) + }) + + // The transport persists via JSON.stringify, which turns all three into + // `null`. They must stay tellable apart in the written log line. + expect(serialized).toEqual([ + expect.stringContaining('"preview":"NaN"'), + expect.stringContaining('"preview":"Infinity"'), + expect.stringContaining('"preview":"-Infinity"'), + ]) + expect(new Set(serialized).size).toBe(3) + serialized.forEach((line) => expect(line).not.toContain('"preview":null')) + } finally { + warn.mockRestore() + } + }) + it("stays silent for counters the provider simply did not report", () => { const warn = vi.spyOn(CompactLogger.prototype, "warn").mockImplementation(() => {}) diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts index 3b4e990ca0..917f09d952 100644 --- a/src/api/transform/chat-usage-normalization.ts +++ b/src/api/transform/chat-usage-normalization.ts @@ -135,7 +135,10 @@ const MAX_PREVIEW_CHARS = 64 */ const describeMalformedCount = (value: unknown): { type: string; preview?: string | number; length?: number } => { if (typeof value === "number") { - return { type: "number", preview: value } + // JSON.stringify writes `null` for NaN and both infinities, which would make + // the three indistinguishable in the persisted log. They are stringified so + // the name survives serialization; finite numbers stay numeric. + return { type: "number", preview: Number.isFinite(value) ? value : String(value) } } if (typeof value === "string") { From 55922d9b46b42c4538a854a468e9051f3366e5aa Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 11:11:26 +0900 Subject: [PATCH 08/12] test(usage): pin the preview truncation boundary exactly The bounded-payload case asserted only that the preview was a string of at most 65 characters, so an empty preview, a 63-character preview, and a 64-character preview with a spurious ellipsis all passed. The truncation boundary is now pinned at both sides: a 64-character counter is kept whole and a 65-character counter keeps its first 64 characters plus an ellipsis. Both cases fail on the previous head when the cap is off by one, and the 64-character case fails when the comparison is tightened to >=. --- .../chat-usage-normalization.spec.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index e7b8368305..9062e92433 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -308,14 +308,37 @@ describe("normalizeChatUsage", () => { expect(meta).not.toHaveProperty("value") expect(meta.type).toBe("string") expect(meta.length).toBe(1_000_000) - expect(typeof meta.preview).toBe("string") - expect((meta.preview as string).length).toBeLessThanOrEqual(65) + expect(meta.preview).toBe(`${"9".repeat(64)}…`) expect(JSON.stringify(meta).length).toBeLessThan(200) } finally { warn.mockRestore() } }) + it.each([ + ["exactly at the cap", 64, "9".repeat(64)], + ["one past the cap", 65, `${"9".repeat(64)}…`], + ])("keeps a preview %s without truncating or over-truncating", async (_name, length, expected) => { + vi.resetModules() + const { CompactLogger: FreshCompactLogger } = await import("../../../utils/logging/CompactLogger") + const { normalizeChatUsage: freshNormalize } = await import("../chat-usage-normalization") + const warn = vi.spyOn(FreshCompactLogger.prototype, "warn").mockImplementation(() => {}) + + try { + freshNormalize({ prompt_tokens: 120, prompt_cache_miss_tokens: "9".repeat(length) }) + + const [, meta] = warn.mock.calls[0] as [string, Record] + + // A length-only assertion cannot tell an empty preview from a full + // one, nor 64 characters from 65, so the boundary is pinned exactly. + expect(meta.preview).toBe(expected) + expect(meta.length).toBe(length) + expect(meta).not.toHaveProperty("value") + } finally { + warn.mockRestore() + } + }) + it.each([ ["a boolean", true, { type: "boolean" }], ["an object", { tokens: 30 }, { type: "object" }], From f740f7a004983cb6dba4fc0f7c0e1131e8e4505a Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sat, 31 Oct 2026 12:23:51 +0900 Subject: [PATCH 09/12] fix(usage): bill cache misses as input, not as cache writes What: - chat-usage-normalization.ts: split the single write-alias list into CACHE_WRITE_ALIASES (prompt_tokens_details.cache_write_tokens, cache_creation_input_tokens) and CACHE_MISS_ALIASES (prompt_cache_miss_tokens, prompt_tokens_details.cache_miss_tokens), and report the miss counter as a new NormalizedChatUsage.cacheMissTokens field instead of folding it into cacheWriteTokens. - deepseek.ts: DeepSeekHandler.processUsageMetrics opts in to the miss counter (cacheWriteTokens ?? cacheMissTokens). It is the only handler that does, because every deepSeekModels row sets inputPrice: 0 by design. - cost.ts: cache prices fall back to inputPrice when the catalog omits them, via (cacheWritesPrice ?? inputPrice) || 0. The inner ?? honors an explicit zero price; the outer || 0 keeps a fully unpriced model at 0 instead of NaN. Why: prompt_cache_miss_tokens was aliased to cacheWriteTokens for every provider. calculateApiCostOpenAI subtracts cache writes from the input pool, so a miss left the input bucket and was re-priced at a cache-write rate most catalogs leave unset, billing it at zero. The rule already existed in-repo: openai-native.ts:154 states "missFromDetails is NOT used as fallback for cache writes". This makes the shared normalizer agree. The alias was introduced inside this stack by c6773dfcc; origin/main has no chat-usage-normalization.ts and its openai.ts reads only cache_creation_input_tokens, so this is a regression this PR created. Impact: - Non-DeepSeek providers (baseten, fireworks, friendli, sambanova, zai, openai, kimi-code) bill misses as regular input again. - DeepSeek billing is unchanged: its miss-to-write mapping is preserved. - Custom models with a priced input and no cache price stop under-billing cache tokens (11.7% on the F-01 fixture). - Catalog rows with an explicit cacheWritesPrice: 0 are byte-identical. - Tests now assert totalCost, not just token labels, so a green suite can no longer hide a pricing regression. --- .../base-openai-compatible-provider.spec.ts | 101 ++++++++++-- src/api/providers/__tests__/deepseek.spec.ts | 65 +++++++- .../__tests__/openai-usage-tracking.spec.ts | 57 ++++++- src/api/providers/deepseek.ts | 9 +- .../chat-usage-normalization.spec.ts | 98 +++++++---- src/api/transform/chat-usage-normalization.ts | 59 +++++-- src/shared/cost.ts | 12 +- src/utils/__tests__/cost.spec.ts | 155 ++++++++++++++++-- 8 files changed, 468 insertions(+), 88 deletions(-) diff --git a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts index 7bb152db92..565288071c 100644 --- a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts +++ b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts @@ -51,6 +51,32 @@ class TestOpenAiCompatibleProvider extends BaseOpenAiCompatibleProvider<"test-mo } } +// A provider whose catalog row publishes an explicit zero cache-write price, +// the shape friendli and moonshot ship. It exists so the miss-token path can be +// priced end to end rather than only asserted as a token label. +const ZERO_WRITE_PRICE_MODEL_INFO: ModelInfo = { + maxTokens: 4096, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: true, + inputPrice: 1.4, + outputPrice: 4.4, + cacheWritesPrice: 0, + cacheReadsPrice: 0.26, +} + +class ZeroWritePriceProvider extends BaseOpenAiCompatibleProvider<"zero-write"> { + constructor(apiKey: string) { + super({ + providerName: "ZeroWriteProvider", + baseURL: "https://test.example.com/v1", + defaultProviderModelId: "zero-write", + providerModels: { "zero-write": ZERO_WRITE_PRICE_MODEL_INFO }, + apiKey, + }) + } +} + describe("BaseOpenAiCompatibleProvider", () => { let handler: TestOpenAiCompatibleProvider @@ -354,23 +380,29 @@ describe("BaseOpenAiCompatibleProvider", () => { ]) }) - it("reads the DeepSeek top-level hit and miss counters", async () => { - expect( - await usageChunksFor({ - prompt_tokens: 100, - completion_tokens: 50, - prompt_cache_hit_tokens: 20, - prompt_cache_miss_tokens: 30, - }), - ).toEqual([ - expect.objectContaining({ - type: "usage", - inputTokens: 100, - outputTokens: 50, - cacheReadTokens: 20, - cacheWriteTokens: 30, - }), - ]) + it("bills a DeepSeek-shaped miss counter as ordinary input, not as a cache write", async () => { + // The base class serves baseten/fireworks/friendli/sambanova/zai, none of + // which price misses through the cache-write field. A miss is input that + // `prompt_tokens` already covers, so it must stay in the input bucket. + const [chunk] = await usageChunksFor({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_hit_tokens: 20, + prompt_cache_miss_tokens: 30, + }) + + expect(chunk).toMatchObject({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + }) + expect(chunk.cacheWriteTokens).toBeUndefined() + + // TEST_MODEL_INFO prices input at 0.5 and output at 1.5 with no cache + // prices, so both the 80 uncached tokens and the 20 reads bill at the + // input price: (100 * 0.5 + 50 * 1.5) / 1e6. + expect(chunk.totalCost).toBeCloseTo(0.000125, 9) }) it("never sums two shapes that report the same quantity", async () => { @@ -397,6 +429,41 @@ describe("BaseOpenAiCompatibleProvider", () => { ]) }) + it("bills a miss on an explicit-zero cache-write row at the input price", async () => { + // friendli and moonshot publish `cacheWritesPrice: 0`, so the kernel fix + // alone cannot repair this shape: a miss routed to the write bucket is + // still billed at zero. Only keeping misses out of that bucket bills them. + mockCreate.mockImplementationOnce(() => + asyncStreamFrom([ + { + choices: [{ delta: {} }], + usage: { + prompt_tokens: 1000, + completion_tokens: 500, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 400, + }, + }, + ]), + ) + + const chunks = await collectStream(new ZeroWritePriceProvider("test-api-key").createMessage("s", [])) + const [chunk] = chunks.filter((c) => c.type === "usage") + + expect(chunk.cacheWriteTokens).toBeUndefined() + + // The 400 misses stay inside prompt_tokens, so they are priced as input. + // Uncached input: (1.4 / 1_000_000) * 900 = 0.00126 + // Cache reads: (0.26 / 1_000_000) * 100 = 0.000026 + // Output cost: (4.4 / 1_000_000) * 500 = 0.0022 + // Total: 0.00126 + 0.000026 + 0.0022 = 0.003486 + expect(chunk.totalCost).toBeCloseTo(0.003486, 9) + + // The defect moved those 400 tokens onto the row's zero write price and + // out of the input pool, billing 0.002926: 16.1% short. + expect(chunk.totalCost).not.toBeCloseTo(0.002926, 9) + }) + it("reports no cache counters when the provider reports none", async () => { const [chunk] = await usageChunksFor({ prompt_tokens: 100, completion_tokens: 50 }) diff --git a/src/api/providers/__tests__/deepseek.spec.ts b/src/api/providers/__tests__/deepseek.spec.ts index 3b6c2496c9..124a0271a2 100644 --- a/src/api/providers/__tests__/deepseek.spec.ts +++ b/src/api/providers/__tests__/deepseek.spec.ts @@ -131,9 +131,16 @@ vi.mock("openai", () => { import OpenAI from "openai" import type { Anthropic } from "@anthropic-ai/sdk" -import { deepSeekDefaultModelId, DEEP_SEEK_DEFAULT_TEMPERATURE, type ModelInfo, DeepSeekModelId } from "@roo-code/types" +import { + deepSeekDefaultModelId, + deepSeekModels, + DEEP_SEEK_DEFAULT_TEMPERATURE, + type ModelInfo, + DeepSeekModelId, +} from "@roo-code/types" import type { ApiHandlerOptions } from "../../../shared/api" +import { calculateApiCostOpenAI } from "../../../shared/cost" import { DeepSeekHandler, normalizeDeepSeekReasoningEffort } from "../deepseek" @@ -613,6 +620,62 @@ describe("DeepSeekHandler", () => { expect(result.cacheWriteTokens).toBeUndefined() expect(result.cacheReadTokens).toBeUndefined() }) + + it("bills a cache miss as a cache write, because the catalog prices input at zero", () => { + // This is the one handler that opts in to the normalizer's miss counter. + // Every deepSeekModels row sets `inputPrice: 0` on purpose ("the inputs + // are priced as cache read/write"), so a miss left in the input bucket + // would be billed at zero. The money is asserted, not just the label: + // a token-label assertion passes whether or not the price is right. + const usage = { + prompt_tokens: 1000, + completion_tokens: 500, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 400, + } + + const result = processUsage(usage) + + expect(result.cacheWriteTokens).toBe(400) + + const { totalCost } = calculateApiCostOpenAI( + deepSeekModels["deepseek-flash"], + result.inputTokens, + result.outputTokens, + result.cacheWriteTokens, + result.cacheReadTokens, + ) + + // Cache writes: (0.3 / 1_000_000) * 400 = 0.00012 + // Cache reads: (0.006 / 1_000_000) * 100 = 0.0000006 + // Uncached input: (0 / 1_000_000) * 500 = 0 + // Output cost: (1.2 / 1_000_000) * 500 = 0.0006 + expect(totalCost).toBeCloseTo(0.0007206, 9) + }) + + it("preserves a reported zero miss counter instead of treating it as unreported", () => { + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_cache_hit_tokens: 100, + prompt_cache_miss_tokens: 0, + }) + + expect(result.cacheWriteTokens).toBe(0) + }) + + it("keeps a genuine write counter ahead of the miss counter", () => { + // A response can report both concepts. The write counter is the more + // specific one, so it wins rather than being shadowed by the miss count. + const result = processUsage({ + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cache_write_tokens: 30 }, + prompt_cache_miss_tokens: 80, + }) + + expect(result.cacheWriteTokens).toBe(30) + }) }) describe("interleaved thinking mode", () => { diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 60eeec02aa..0d59c81ca5 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -2,7 +2,10 @@ import { Anthropic } from "@anthropic-ai/sdk" +import type { ModelInfo } from "@roo-code/types" + import { ApiHandlerOptions } from "../../../shared/api" +import { calculateApiCostOpenAI } from "../../../shared/cost" import { OpenAiHandler } from "../openai" import { makeApiHandlerOptions } from "../../../test-utils/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" @@ -466,14 +469,56 @@ describe("OpenAiHandler with usage tracking fix", () => { ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 7 }]) }) - it("reads the top-level prompt_cache_miss_tokens shape", async () => { + it("does not bill a top-level prompt_cache_miss_tokens counter as a cache write", async () => { + // A miss is a token that was looked up in the cache and not found, so it + // is ordinary input that prompt_tokens already covers. Reporting it as a + // write would move it off the input price line onto the cache-write line, + // where most catalogs charge nothing. openai-native.ts states the same + // rule: "missFromDetails is NOT used as fallback for cache writes". expect( await usageChunksFor({ - prompt_tokens: 100, + prompt_tokens: 1000, completion_tokens: 5, - prompt_cache_miss_tokens: 12, + prompt_cache_miss_tokens: 400, }), - ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 12 }]) + ).toEqual([{ type: "usage", inputTokens: 1000, outputTokens: 5 }]) + }) + + it("leaves the miss tokens in the input pool that Task.ts prices", async () => { + // This handler emits no totalCost; Task.ts recomputes it from the counters + // in the chunk. The chunk is therefore the contract, and the money is + // asserted against it so a label-only regression cannot pass. + const [chunk] = await usageChunksFor({ + prompt_tokens: 1000, + completion_tokens: 5, + prompt_cache_miss_tokens: 400, + }) + + const customModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + } + + const { totalCost } = calculateApiCostOpenAI( + customModel, + chunk.inputTokens, + chunk.outputTokens, + chunk.cacheWriteTokens, + chunk.cacheReadTokens, + ) + + // All 1000 prompt tokens bill at the input price, including the 400 misses: + // (3.0 / 1_000_000) * 1000 = 0.003 + // Output: (15.0 / 1_000_000) * 5 = 0.000075 + // Total: 0.003075 + expect(totalCost).toBeCloseTo(0.003075, 9) + + // The defect subtracted the 400 misses from the input pool and priced them + // at a cache-write rate the model never declared, billing 0.001875. + expect(totalCost).not.toBeCloseTo(0.001875, 9) }) it("never sums two write shapes that report the same quantity", async () => { @@ -490,12 +535,12 @@ describe("OpenAiHandler with usage tracking fix", () => { expect(chunks).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30 }]) }) - it("keeps the documented DeepSeek write counter ahead of the Anthropic-style one", async () => { + it("keeps a genuine write counter ahead of the Anthropic-style one", async () => { expect( await usageChunksFor({ prompt_tokens: 100, completion_tokens: 5, - prompt_cache_miss_tokens: 30, + prompt_tokens_details: { cache_write_tokens: 30 }, cache_creation_input_tokens: 99, }), ).toEqual([{ type: "usage", inputTokens: 100, outputTokens: 5, cacheWriteTokens: 30 }]) diff --git a/src/api/providers/deepseek.ts b/src/api/providers/deepseek.ts index 25e660abec..6aeb6b9ec2 100644 --- a/src/api/providers/deepseek.ts +++ b/src/api/providers/deepseek.ts @@ -232,6 +232,13 @@ export class DeepSeekHandler extends OpenAiHandler { // quantities: top-level `prompt_cache_hit_tokens` / `prompt_cache_miss_tokens` // and nested `prompt_tokens_details.cached_tokens`. The normalizer resolves // one value per field and never sums the shapes. + // + // This is the only handler that bills cache misses as cache writes, and it + // is the reason the normalizer keeps the two counters apart. Every + // `deepSeekModels` row sets `inputPrice: 0` on purpose ("the inputs are + // priced as cache read/write"), so a miss left in the input bucket would be + // billed at zero. Every other provider prices misses through `inputPrice`, + // which is why they must not opt in here. // https://api-docs.deepseek.com/api/create-chat-completion protected override processUsageMetrics(usage: any, _modelInfo?: any): ApiStreamUsageChunk { const normalized = normalizeChatUsage(usage) @@ -240,7 +247,7 @@ export class DeepSeekHandler extends OpenAiHandler { type: "usage", inputTokens: normalized?.inputTokens ?? 0, outputTokens: normalized?.outputTokens ?? 0, - cacheWriteTokens: normalized?.cacheWriteTokens, + cacheWriteTokens: normalized?.cacheWriteTokens ?? normalized?.cacheMissTokens, cacheReadTokens: normalized?.cacheReadTokens, } } diff --git a/src/api/transform/__tests__/chat-usage-normalization.spec.ts b/src/api/transform/__tests__/chat-usage-normalization.spec.ts index 9062e92433..e9c2586bcf 100644 --- a/src/api/transform/__tests__/chat-usage-normalization.spec.ts +++ b/src/api/transform/__tests__/chat-usage-normalization.spec.ts @@ -32,11 +32,11 @@ describe("normalizeChatUsage", () => { inputTokens: 7, outputTokens: 11, cacheReadTokens: 0, - cacheWriteTokens: 7, + cacheMissTokens: 7, }) }) - it("reads a top-level-only usage object with no nested details", () => { + it("reports a top-level-only miss counter as a miss, not a write", () => { expect( normalizeChatUsage({ prompt_tokens: 120, @@ -44,7 +44,7 @@ describe("normalizeChatUsage", () => { prompt_cache_hit_tokens: 100, prompt_cache_miss_tokens: 20, }), - ).toEqual({ inputTokens: 120, outputTokens: 30, cacheReadTokens: 100, cacheWriteTokens: 20 }) + ).toEqual({ inputTokens: 120, outputTokens: 30, cacheReadTokens: 100, cacheMissTokens: 20 }) }) it("reads the nested-only shape used by OpenAI and by legacy DeepSeek fixtures", () => { @@ -54,11 +54,11 @@ describe("normalizeChatUsage", () => { completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20, cache_miss_tokens: 80 }, }), - ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, cacheWriteTokens: 80 }) + ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, cacheMissTokens: 80 }) }) it("does not add the nested and top-level shapes together when both are present", () => { - // Both shapes describe the same 100 reads / 20 writes. Summing would + // Both shapes describe the same 100 reads / 20 misses. Summing would // report 200 / 40 and double the billed cache tokens. const normalized = normalizeChatUsage({ prompt_tokens: 120, @@ -69,7 +69,7 @@ describe("normalizeChatUsage", () => { }) expect(normalized?.cacheReadTokens).toBe(100) - expect(normalized?.cacheWriteTokens).toBe(20) + expect(normalized?.cacheMissTokens).toBe(20) }) it("prefers the nested cached_tokens over the top-level hit counter for reads", () => { @@ -82,13 +82,13 @@ describe("normalizeChatUsage", () => { ).toBe(100) }) - it("prefers the top-level miss counter over the nested one for writes", () => { + it("prefers the top-level miss counter over the nested one", () => { expect( normalizeChatUsage({ prompt_tokens: 120, prompt_tokens_details: { cache_miss_tokens: 20 }, prompt_cache_miss_tokens: 30, - })?.cacheWriteTokens, + })?.cacheMissTokens, ).toBe(30) }) @@ -98,43 +98,70 @@ describe("normalizeChatUsage", () => { prompt_tokens: 120, prompt_cache_miss_tokens: "30", prompt_tokens_details: { cache_miss_tokens: 20 }, - })?.cacheWriteTokens, + })?.cacheMissTokens, ).toBe(20) }) - describe("cache_creation_input_tokens (LiteLLM write shape)", () => { - it("reads the LiteLLM top-level write counter when no DeepSeek write alias is present", () => { + describe("write and miss counters are separate concepts", () => { + // A write is a token stored for later reuse; a miss is a token that was + // looked up and not found, which is ordinary input. Merging them moves + // miss tokens off the input price line and onto the cache-write line, + // where most catalogs charge nothing. Each alias must land in exactly + // one field, so both lists are pinned here by behavior. + it.each(["prompt_cache_miss_tokens", "prompt_tokens_details.cache_miss_tokens"])( + "reports %s as a miss and never as a write", + (alias) => { + const usage = alias.includes(".") + ? { prompt_tokens: 100, prompt_tokens_details: { cache_miss_tokens: 40 } } + : { prompt_tokens: 100, prompt_cache_miss_tokens: 40 } + + const normalized = normalizeChatUsage(usage) + + expect(normalized?.cacheMissTokens).toBe(40) + expect(normalized).not.toHaveProperty("cacheWriteTokens") + }, + ) + + it.each(["prompt_tokens_details.cache_write_tokens", "cache_creation_input_tokens"])( + "reports %s as a write and never as a miss", + (alias) => { + const usage = alias.includes(".") + ? { prompt_tokens: 100, prompt_tokens_details: { cache_write_tokens: 40 } } + : { prompt_tokens: 100, cache_creation_input_tokens: 40 } + + const normalized = normalizeChatUsage(usage) + + expect(normalized?.cacheWriteTokens).toBe(40) + expect(normalized).not.toHaveProperty("cacheMissTokens") + }, + ) + + it("resolves a write and a miss from the same response without shadowing either", () => { + // 20 tokens were written into the cache and 40 were looked up but not + // found. They are different tokens, so both counters survive. expect( normalizeChatUsage({ - prompt_tokens: 100, - completion_tokens: 50, + prompt_tokens: 120, + prompt_cache_miss_tokens: 40, cache_creation_input_tokens: 20, - cache_read_input_tokens: 30, }), - ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 30, cacheWriteTokens: 20 }) + ).toMatchObject({ cacheWriteTokens: 20, cacheMissTokens: 40 }) }) + }) - it("keeps the documented DeepSeek top-level miss counter ahead of it", () => { - // Both describe the same writes. DeepSeek's documented shape wins and - // the two are never added together. + describe("cache_creation_input_tokens (LiteLLM write shape)", () => { + it("reads the LiteLLM top-level write counter when no other write alias is present", () => { expect( normalizeChatUsage({ - prompt_tokens: 120, - prompt_cache_miss_tokens: 20, + prompt_tokens: 100, + completion_tokens: 50, cache_creation_input_tokens: 20, - })?.cacheWriteTokens, - ).toBe(20) + cache_read_input_tokens: 30, + }), + ).toEqual({ inputTokens: 100, outputTokens: 50, cacheReadTokens: 30, cacheWriteTokens: 20 }) }) - it("keeps every DeepSeek write alias ahead of it", () => { - expect( - normalizeChatUsage({ - prompt_tokens: 120, - prompt_tokens_details: { cache_miss_tokens: 20 }, - cache_creation_input_tokens: 99, - })?.cacheWriteTokens, - ).toBe(20) - + it("keeps the nested cache_write_tokens alias ahead of it", () => { expect( normalizeChatUsage({ prompt_tokens: 120, @@ -144,12 +171,11 @@ describe("normalizeChatUsage", () => { ).toBe(20) }) - it("does not sum it with a DeepSeek write alias reporting the same quantity", () => { + it("does not sum it with a write alias reporting the same quantity", () => { const normalized = normalizeChatUsage({ prompt_tokens: 120, completion_tokens: 30, - prompt_cache_miss_tokens: 20, - prompt_tokens_details: { cache_miss_tokens: 20, cache_write_tokens: 20 }, + prompt_tokens_details: { cache_write_tokens: 20 }, cache_creation_input_tokens: 20, }) @@ -172,7 +198,7 @@ describe("normalizeChatUsage", () => { prompt_tokens: 120, prompt_cache_miss_tokens: malformed, prompt_tokens_details: { cache_miss_tokens: 20 }, - })?.cacheWriteTokens, + })?.cacheMissTokens, ).toBe(20) }) @@ -242,7 +268,7 @@ describe("normalizeChatUsage", () => { prompt_tokens: 120, prompt_cache_miss_tokens: "30", prompt_tokens_details: { cache_miss_tokens: 20 }, - })?.cacheWriteTokens, + })?.cacheMissTokens, ).toBe(20) } finally { warn.mockRestore() diff --git a/src/api/transform/chat-usage-normalization.ts b/src/api/transform/chat-usage-normalization.ts index 917f09d952..67f6c2252e 100644 --- a/src/api/transform/chat-usage-normalization.ts +++ b/src/api/transform/chat-usage-normalization.ts @@ -36,30 +36,53 @@ export interface NormalizedChatUsage { */ cacheReadTokens?: number cacheWriteTokens?: number + /** + * Tokens that were looked up in the prompt cache and not found. These are + * ordinary input tokens for every provider except DeepSeek, so they are + * reported separately and are never folded into `cacheWriteTokens`. + * `undefined` when the provider reported no such counter at all, which is + * distinct from a reported zero. + */ + cacheMissTokens?: number } /** - * Aliases for cache-write (cache-miss) tokens, highest priority first. + * Aliases for true cache-write tokens, highest priority first. * - * DeepSeek documents `prompt_cache_miss_tokens` at the top level of the usage - * object and marks it required; its `prompt_tokens_details` schema declares - * only `cached_tokens`. `prompt_tokens_details.cache_miss_tokens` is not part - * of that schema and is kept last as a legacy fallback for proxies that - * forwarded a rewritten shape. + * A cache write is a token the provider durably stored for reuse on a later + * request. It is billed at the provider's cache-write price, which is often + * zero (a write surcharge is not universal) but is a real, separate counter. * * `cache_creation_input_tokens` is the Anthropic-style top-level counter that - * LiteLLM publishes for cached writes. It sits below every DeepSeek alias so - * the documented DeepSeek shape keeps priority, and it is reached only by - * providers that publish no DeepSeek counter at all. + * LiteLLM publishes for cached writes. * * https://api-docs.deepseek.com/api/create-chat-completion */ -const CACHE_WRITE_ALIASES = [ - "prompt_cache_miss_tokens", - "prompt_tokens_details.cache_miss_tokens", - "prompt_tokens_details.cache_write_tokens", - "cache_creation_input_tokens", -] as const +const CACHE_WRITE_ALIASES = ["prompt_tokens_details.cache_write_tokens", "cache_creation_input_tokens"] as const + +/** + * Aliases for cache-miss tokens, highest priority first. + * + * A cache miss is a token that was looked up in the prompt cache and not + * found. Economically it is the opposite of a write: it is ordinary input + * that the provider already charges for through `prompt_tokens`, and it must + * not be moved onto the cache-write price line, because doing so removes it + * from the input bucket (`cost.ts` subtracts cache writes from the input + * total) and re-prices it at a cache-write rate that most catalogs leave + * unset, billing it at zero. + * + * `openai-native.ts` already states this rule and applies it: "missFromDetails + * is NOT used as fallback for cache writes". This module now agrees with it. + * + * DeepSeek is the one provider whose catalog prices misses through the + * cache-write field: all four `deepSeekModels` rows set `inputPrice: 0` by + * design ("the inputs are priced as cache read/write") and publish + * `cacheWritesPrice`. Only `DeepSeekHandler` therefore opts in to reading + * this list; see `processUsageMetrics` in `providers/deepseek.ts`. + * + * https://api-docs.deepseek.com/api/create-chat-completion + */ +const CACHE_MISS_ALIASES = ["prompt_cache_miss_tokens", "prompt_tokens_details.cache_miss_tokens"] as const /** * Aliases for cache-read (cache-hit) tokens, highest priority first. @@ -204,17 +227,23 @@ const resolveTokenCount = (usage: ChatUsageLike, aliases: readonly string[]): nu * usage chunk entirely. Token counts that the provider did not report stay * `undefined` rather than defaulting to 0, which keeps "no cache counter" and * "zero cache hits" distinguishable for cost reporting. + * + * `cacheMissTokens` is reported but not merged into `cacheWriteTokens`: a miss + * is ordinary input, and only DeepSeek's catalog prices it as a write. Callers + * that bill misses as writes must opt in explicitly. */ export function normalizeChatUsage(usage: ChatUsageLike | null | undefined): NormalizedChatUsage | undefined { if (!usage || typeof usage !== "object") return undefined const cacheReadTokens = resolveTokenCount(usage, CACHE_READ_ALIASES) const cacheWriteTokens = resolveTokenCount(usage, CACHE_WRITE_ALIASES) + const cacheMissTokens = resolveTokenCount(usage, CACHE_MISS_ALIASES) return { inputTokens: asTokenCount(usage.prompt_tokens, "prompt_tokens") ?? 0, outputTokens: asTokenCount(usage.completion_tokens, "completion_tokens") ?? 0, ...(cacheReadTokens !== undefined ? { cacheReadTokens } : {}), ...(cacheWriteTokens !== undefined ? { cacheWriteTokens } : {}), + ...(cacheMissTokens !== undefined ? { cacheMissTokens } : {}), } } diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 754a2d84df..4eaa3b2712 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -47,8 +47,16 @@ function calculateApiCostInternal( totalInputTokens: number, totalOutputTokens: number, ): ApiCostResult { - const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens - const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens + // Cache tokens are billed at the model's cache price when it has one, and at + // its input price when the catalog omits the cache price entirely. The + // inner `??` (not `||`) is load-bearing: an explicit `cacheWritesPrice: 0` + // is a real price for providers that charge nothing for writes, and `||` + // would silently re-price those rows at `inputPrice`. The outer `|| 0` + // covers a fully unpriced model, where `undefined * 0` would be `NaN`. + const cacheWritesCost = + (((modelInfo.cacheWritesPrice ?? modelInfo.inputPrice) || 0) / 1_000_000) * cacheCreationInputTokens + const cacheReadsCost = + (((modelInfo.cacheReadsPrice ?? modelInfo.inputPrice) || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens const outputCost = ((modelInfo.outputPrice || 0) / 1_000_000) * outputTokens const totalCost = cacheWritesCost + cacheReadsCost + baseInputCost + outputCost diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index b369a0405b..4555a04b54 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -1,6 +1,6 @@ // npx vitest utils/__tests__/cost.spec.ts -import { OpenAiServiceTier, type ModelInfo } from "@roo-code/types" +import { OpenAiServiceTier, deepSeekModels, friendliModels, moonshotModels, type ModelInfo } from "@roo-code/types" import { calculateApiCostAnthropic, calculateApiCostOpenAI } from "../../shared/cost" @@ -95,7 +95,7 @@ describe("Cost Utility", () => { expect(result.totalOutputTokens).toBe(500) }) - it("should handle missing cache prices", () => { + it("bills cache tokens at the input price when the catalog omits cache prices", () => { const modelWithoutCachePrices: ModelInfo = { ...mockModelInfo, cacheWritesPrice: undefined, @@ -104,14 +104,49 @@ describe("Cost Utility", () => { const result = calculateApiCostAnthropic(modelWithoutCachePrices, 1000, 500, 2000, 3000) - // Should only include input and output costs + // A model that prices input but publishes no cache price still charges for + // the tokens it cached; the input price is the only rate it declares. // Input cost: (3.0 / 1_000_000) * 1000 = 0.003 + // Cache writes: (3.0 / 1_000_000) * 2000 = 0.006 + // Cache reads: (3.0 / 1_000_000) * 3000 = 0.009 // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.003 + 0.0075 = 0.0105 - expect(result.totalCost).toBe(0.0105) + // Total: 0.003 + 0.006 + 0.009 + 0.0075 = 0.0255 + expect(result.totalCost).toBeCloseTo(0.0255, 9) expect(result.totalInputTokens).toBe(6000) // 1000 + 2000 + 3000 expect(result.totalOutputTokens).toBe(500) }) + + it("honors an explicit zero cache price instead of falling back to the input price", () => { + // A provider that publishes `cacheWritesPrice: 0` charges nothing for + // writes. Only a *missing* cache price falls back to the input price, so + // `??` is load-bearing here and `||` would silently re-price this row. + const modelWithFreeWrites: ModelInfo = { + ...mockModelInfo, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + } + + const result = calculateApiCostAnthropic(modelWithFreeWrites, 1000, 500, 2000, 3000) + + // Input cost: (3.0 / 1_000_000) * 1000 = 0.003 + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + // Cache writes and reads: 0 by the catalog's own price + expect(result.totalCost).toBeCloseTo(0.0105, 9) + }) + + it("bills nothing when the model declares no prices at all", () => { + // `undefined * 0` is NaN, so the fallback chain has to end in a real 0. + const unpricedModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + } + + const result = calculateApiCostAnthropic(unpricedModel, 1000, 500, 2000, 3000) + + expect(result.totalCost).toBe(0) + expect(Number.isNaN(result.totalCost)).toBe(false) + }) }) describe("calculateApiCostOpenAI", () => { @@ -204,7 +239,7 @@ describe("Cost Utility", () => { expect(result.totalOutputTokens).toBe(500) }) - it("should handle missing cache prices", () => { + it("bills cache tokens at the input price when the catalog omits cache prices", () => { const modelWithoutCachePrices: ModelInfo = { ...mockModelInfo, cacheWritesPrice: undefined, @@ -213,15 +248,115 @@ describe("Cost Utility", () => { const result = calculateApiCostOpenAI(modelWithoutCachePrices, 6000, 500, 2000, 3000) - // Should only include input and output costs - // Input cost: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // OpenAI usage counts cached tokens inside prompt_tokens, so the cache + // buckets are removed from the input pool and re-priced. With no cache + // price published, the input price is the only rate the model declares. + // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // Cache writes: (3.0 / 1_000_000) * 2000 = 0.006 + // Cache reads: (3.0 / 1_000_000) * 3000 = 0.009 // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.003 + 0.0075 = 0.0105 - expect(result.totalCost).toBe(0.0105) + // Total: 0.003 + 0.006 + 0.009 + 0.0075 = 0.0255 + expect(result.totalCost).toBeCloseTo(0.0255, 9) expect(result.totalInputTokens).toBe(6000) // Total already includes cache expect(result.totalOutputTokens).toBe(500) }) + it("honors an explicit zero cache price instead of falling back to the input price", () => { + // friendli's GLM rows and every moonshot row publish `cacheWritesPrice: 0` + // on purpose. A zero is a price, not an absence, so those rows must not be + // re-priced at the input price. + const modelWithFreeWrites: ModelInfo = { + ...mockModelInfo, + cacheWritesPrice: 0, + cacheReadsPrice: 0, + } + + const result = calculateApiCostOpenAI(modelWithFreeWrites, 6000, 500, 2000, 3000) + + // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + expect(result.totalCost).toBeCloseTo(0.0105, 9) + }) + + it("bills nothing when the model declares no prices at all", () => { + const unpricedModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + } + + const result = calculateApiCostOpenAI(unpricedModel, 6000, 500, 2000, 3000) + + expect(result.totalCost).toBe(0) + expect(Number.isNaN(result.totalCost)).toBe(false) + }) + + it("stays byte-identical for the DeepSeek catalog, whose input price is zero by design", () => { + // All four deepSeekModels rows set `inputPrice: 0` and price misses through + // cacheWritesPrice. A fallback that resolved through `inputPrice` would + // zero out their cache-write revenue, so the rows are pinned exactly. + expect(deepSeekModels["deepseek-flash"].inputPrice).toBe(0) + + const result = calculateApiCostOpenAI(deepSeekModels["deepseek-flash"], 1000, 500, 400, 100) + + // Cache writes: (0.3 / 1_000_000) * 400 = 0.00012 + // Cache reads: (0.006 / 1_000_000) * 100 = 0.0000006 + // Uncached input: (0 / 1_000_000) * 500 = 0 + // Output cost: (1.2 / 1_000_000) * 500 = 0.0006 + expect(result.totalCost).toBeCloseTo(0.0007206, 9) + }) + + describe("catalog rows with an explicit zero cache price", () => { + // These rows publish `cacheWritesPrice: 0` deliberately: the provider + // charges nothing for writes. They are the cells that a `||` fallback + // would silently re-price at the input price, so each is pinned to the + // exact total it bills today. + it.each([ + ["friendli GLM-5.2", friendliModels["zai-org/GLM-5.2"]], + ["moonshot kimi-k3", moonshotModels["kimi-k3"]], + ])("leaves %s byte-identical", (_name, modelInfo) => { + expect(modelInfo.cacheWritesPrice).toBe(0) + + const result = calculateApiCostOpenAI(modelInfo, 1000, 500, 400, 100) + + const expected = + ((modelInfo.inputPrice ?? 0) / 1_000_000) * 500 + // uncached input + ((modelInfo.outputPrice ?? 0) / 1_000_000) * 500 + // output + ((modelInfo.cacheReadsPrice ?? 0) / 1_000_000) * 100 // reads only; writes are free + + expect(result.totalCost).toBeCloseTo(expected, 9) + }) + }) + + it("bills the miss share of a custom priced model instead of giving it away", () => { + // The defect this repairs: a custom OpenAI-compatible model that prices + // input but publishes no cache price. Miss tokens arrived as cache + // writes, were subtracted from the input pool, and were then priced at + // zero, so the user was billed for neither. + const customModel: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheReadsPrice: 0.3, + } + + // 1000 prompt tokens: 400 missed the cache, 100 hit it, 500 were plain. + const result = calculateApiCostOpenAI(customModel, 1000, 500, 400, 100) + + // Uncached input: (3.0 / 1_000_000) * 500 = 0.0015 + // Cache writes: (3.0 / 1_000_000) * 400 = 0.0012 (input price, since none published) + // Cache reads: (0.3 / 1_000_000) * 100 = 0.00003 + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + // Total: 0.0015 + 0.0012 + 0.00003 + 0.0075 = 0.01023 + expect(result.totalCost).toBeCloseTo(0.01023, 9) + + // The under-bill the defect produced, kept as the regression's own number: + // pricing the same 400 tokens at zero yields 0.00903, 11.7% short. + expect(result.totalCost).not.toBeCloseTo(0.00903, 9) + }) + it("should not apply long-context pricing at the threshold", () => { const modelWithLongContextPricing: ModelInfo = { ...mockModelInfo, From 6fe18dcf84b36ae1a798fc10462fd6f19c851d31 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sun, 1 Nov 2026 07:32:37 +0900 Subject: [PATCH 10/12] fix(litellm): normalize cache usage through the shared normalizer What: the LiteLLM handler resolved cache counters through its own alias chains (lite-llm.ts#L388-397), folding DeepSeek-style prompt_cache_miss_tokens into cacheWriteTokens. It now calls normalizeChatUsage and reads cacheWriteTokens/cacheReadTokens from the normalized result. Why: a cache miss is ordinary input that prompt_tokens already counts. Folding it into cacheWriteTokens moved it out of the input bucket (calculateApiCostOpenAI subtracts cache writes) and re-priced it on the cache-write line, which bills at zero when the relayed row prices writes at zero. That is the same under-billing class #2254 fixed for DeepSeek, surviving in the last hand-rolled copy of the pre-Z1 chain. Impact: LiteLLM usage chunks are unchanged for the Anthropic-shaped and nested OpenAI-shaped responses the spec pins. Responses carrying a miss counter no longer report it as a cache write and bill it at input price. The handler also gains the normalizer's malformed-counter hardening. The three (lastUsage as any) casts the old chain needed are gone, so the file's no-explicit-any suppression drops 8 -> 5. No count rises. Depends-on: #2254 --- src/api/providers/__tests__/lite-llm.spec.ts | 151 ++++++++++++++++++- src/api/providers/lite-llm.ts | 26 ++-- src/eslint-suppressions.json | 2 +- 3 files changed, 165 insertions(+), 14 deletions(-) diff --git a/src/api/providers/__tests__/lite-llm.spec.ts b/src/api/providers/__tests__/lite-llm.spec.ts index 721f2ccc43..342f07a0c7 100644 --- a/src/api/providers/__tests__/lite-llm.spec.ts +++ b/src/api/providers/__tests__/lite-llm.spec.ts @@ -4,7 +4,8 @@ import { Anthropic } from "@anthropic-ai/sdk" import { LiteLLMHandler } from "../lite-llm" import { getModels } from "../fetchers/modelCache" import { ApiHandlerOptions } from "../../../shared/api" -import { type ModelRecord, litellmDefaultModelId, litellmDefaultModelInfo } from "@roo-code/types" +import type { ApiStreamUsageChunk } from "../../transform/stream" +import { type ModelInfo, type ModelRecord, litellmDefaultModelId, litellmDefaultModelInfo } from "@roo-code/types" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { clearAllMocks } from "../../../test-utils/reset" import { makeCreateMessageMetadata } from "../../../test-utils/api" @@ -2260,6 +2261,154 @@ describe("LiteLLMHandler", () => { }) }) + describe("cache token accounting", () => { + // The cache counters arrive in the shapes LiteLLM relays from the upstream + // provider: Anthropic-style top-level fields, OpenAI-style nested + // `prompt_tokens_details`, and DeepSeek-style top-level hit/miss counters. + // The handler must resolve each counter once through the shared normalizer + // and must never move a cache miss onto the cache-write line. + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }] + + const streamWithUsage = (usage: Record) => { + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ + data: asyncStreamFrom([{ choices: [{ delta: { content: "ok" } }], usage }]), + }), + }) + } + + const collectUsage = async (): Promise => { + const chunks = await collectStream(handler.createMessage("You are helpful", messages)) + const usageChunk = chunks.find((chunk) => chunk.type === "usage") + expect(usageChunk).toBeDefined() + return usageChunk as ApiStreamUsageChunk + } + + // A catalog row as LiteLLM's `/v1/model/info` relays it for a DeepSeek + // deployment: input and output are priced, reads are cheaper, and the + // upstream reports no cache-creation cost, so writes carry no price. + const relayedDeepSeekRow: ModelInfo = { + maxTokens: 8192, + contextWindow: 200_000, + supportsPromptCache: true, + inputPrice: 3.0, + outputPrice: 15.0, + cacheReadsPrice: 0.3, + } + + beforeEach(() => { + handler = new LiteLLMHandler({ ...mockOptions, litellmModelId: "deepseek-chat" }) + handler["models"] = { "deepseek-chat": relayedDeepSeekRow } + }) + + it("keeps DeepSeek-style cache misses out of the cache-write counter", async () => { + // 100 prompt tokens: 60 hit the cache, 40 missed it. A miss is ordinary + // input, so it must not be reported as a cache write. + streamWithUsage({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_cache_hit_tokens: 60, + prompt_cache_miss_tokens: 40, + }) + + const usage = await collectUsage() + + expect(usage.cacheWriteTokens).toBeUndefined() + expect(usage.cacheReadTokens).toBe(60) + expect(usage.inputTokens).toBe(100) + }) + + it("bills a folded miss at zero when the relayed row prices writes at zero", async () => { + // This is the money case. `cache_creation_input_token_cost: 0` upstream + // reaches the catalog as `cacheWritesPrice: 0`, which the cost kernel + // honours as a real price. A folded miss is then subtracted from the + // input pool and billed at zero, so the user pays for neither the input + // nor the write. When the row publishes no write price at all the fold + // is economically neutral instead, because the kernel falls back to + // `inputPrice`; only an explicit zero write price loses revenue. + handler["models"] = { "deepseek-chat": { ...relayedDeepSeekRow, cacheWritesPrice: 0 } } + streamWithUsage({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_cache_hit_tokens: 60, + prompt_cache_miss_tokens: 40, + }) + + const usage = await collectUsage() + + // Misses stay in the input bucket: 40 * (3.0 / 1_000_000) = 0.00012 + // Cache reads: 60 * (0.3 / 1_000_000) = 0.000018 + // Output: 5 * (15.0 / 1_000_000) = 0.000075 + // Total: 0.000213 + expect(usage.totalCost).toBeCloseTo(0.000213, 9) + + // The fold billed 0.000093 here: the 40 miss tokens were subtracted from + // the input pool and priced at the explicit zero write price, 56% short. + expect(usage.totalCost).not.toBeCloseTo(0.000093, 9) + }) + + it("reports a top-level Anthropic-shaped response unchanged", async () => { + streamWithUsage({ + prompt_tokens: 100, + completion_tokens: 50, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }) + + const usage = await collectUsage() + + expect(usage).toMatchObject({ + type: "usage", + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 20, + cacheReadTokens: 30, + }) + }) + + it("reports a nested OpenAI-shaped response unchanged", async () => { + streamWithUsage({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 20, cache_write_tokens: 30 }, + }) + + const usage = await collectUsage() + + expect(usage).toMatchObject({ + type: "usage", + inputTokens: 100, + outputTokens: 5, + cacheWriteTokens: 30, + cacheReadTokens: 20, + }) + }) + + it("omits both cache counters when the response reports neither", async () => { + streamWithUsage({ prompt_tokens: 100, completion_tokens: 5 }) + + const usage = await collectUsage() + + expect(usage.cacheWriteTokens).toBeUndefined() + expect(usage.cacheReadTokens).toBeUndefined() + }) + + it("falls through a malformed counter to the next alias", async () => { + // A proxy can relay a counter as a string. The normalizer discards it and + // keeps resolving instead of poisoning the total with NaN. + streamWithUsage({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 20 }, + cache_read_input_tokens: "30", + }) + + const usage = await collectUsage() + + expect(usage.cacheReadTokens).toBe(20) + }) + }) + describe("model catalog", () => { it("reports its cache scope", () => { const handler = new LiteLLMHandler(mockOptions) diff --git a/src/api/providers/lite-llm.ts b/src/api/providers/lite-llm.ts index aea27805ea..38229125a6 100644 --- a/src/api/providers/lite-llm.ts +++ b/src/api/providers/lite-llm.ts @@ -12,6 +12,8 @@ import { import { calculateApiCostOpenAI } from "../../shared/cost" +import { normalizeChatUsage } from "../transform/chat-usage-normalization" + import { ApiHandlerOptions, GetModelsOptions } from "../../shared/api" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" @@ -383,18 +385,15 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa throwIfAborted(requestSignal) if (lastUsage) { - // Extract cache-related information if available - // LiteLLM may use different field names for cache tokens - const cacheWriteTokens = - lastUsage.cache_creation_input_tokens || - lastUsage.prompt_tokens_details?.cache_write_tokens || - (lastUsage as any).prompt_cache_miss_tokens || - 0 - const cacheReadTokens = - lastUsage.prompt_tokens_details?.cached_tokens || - (lastUsage as any).cache_read_input_tokens || - (lastUsage as any).prompt_cache_hit_tokens || - 0 + // LiteLLM relays whatever cache counters the upstream provider + // publishes: Anthropic-style top-level fields, OpenAI-style nested + // `prompt_tokens_details`, and DeepSeek-style hit/miss counters. The + // shared normalizer resolves one value per counter across those shapes + // and keeps cache misses out of the cache-write total, because a miss + // is ordinary input that `prompt_tokens` already counts. + const normalized = normalizeChatUsage(lastUsage) + const cacheWriteTokens = normalized?.cacheWriteTokens ?? 0 + const cacheReadTokens = normalized?.cacheReadTokens ?? 0 const { totalCost } = calculateApiCostOpenAI( info, @@ -572,9 +571,12 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa } // LiteLLM usage may include an extra field for Anthropic use cases. +// The index signature keeps this assignable to the normalizer's `ChatUsageLike`, +// which an interface without one never is. interface LiteLLMUsage extends OpenAI.CompletionUsage { cache_creation_input_tokens?: number prompt_tokens_details?: OpenAI.CompletionUsage["prompt_tokens_details"] & { cache_write_tokens?: number } + [key: string]: unknown } diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 8a9e1fa4df..a1fe363b7b 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -366,7 +366,7 @@ }, "api/providers/lite-llm.ts": { "@typescript-eslint/no-explicit-any": { - "count": 8 + "count": 5 } }, "api/providers/mimo.ts": { From 27645f6b5666b706f65eebab2cdbf139f9457518 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sun, 1 Nov 2026 08:05:05 +0900 Subject: [PATCH 11/12] test(litellm): correct two cache-accounting test claims What: reword the money-test comment so it no longer claims the fetcher can produce an explicit zero cache-write price, and move the malformed counter in the alias-fallthrough test onto the highest-priority alias. Why: the fetcher maps a numeric 0 through a truthiness test (fetchers/litellm.ts:65-67), so an upstream `cache_creation_input_token_cost: 0` arrives as `cacheWritesPrice: undefined`, never as an explicit zero; the tested row is hand-built. The fallthrough test put the malformed string on the lowest-priority alias while a valid nested value won first, so it never exercised the discard and also passed with the discard removed. Impact: test-only, no production behaviour change. The fallthrough test now fails with `expected 30 to be 20` when the resolver stops discarding malformed counters. --- src/api/providers/__tests__/lite-llm.spec.ts | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/api/providers/__tests__/lite-llm.spec.ts b/src/api/providers/__tests__/lite-llm.spec.ts index 342f07a0c7..5de6354ecd 100644 --- a/src/api/providers/__tests__/lite-llm.spec.ts +++ b/src/api/providers/__tests__/lite-llm.spec.ts @@ -2319,13 +2319,17 @@ describe("LiteLLMHandler", () => { }) it("bills a folded miss at zero when the relayed row prices writes at zero", async () => { - // This is the money case. `cache_creation_input_token_cost: 0` upstream - // reaches the catalog as `cacheWritesPrice: 0`, which the cost kernel - // honours as a real price. A folded miss is then subtracted from the - // input pool and billed at zero, so the user pays for neither the input - // nor the write. When the row publishes no write price at all the fold - // is economically neutral instead, because the kernel falls back to - // `inputPrice`; only an explicit zero write price loses revenue. + // This is the money case, and the row below is hand-built rather than + // fetched: `fetchers/litellm.ts` maps a numeric 0 through a truthiness + // test, so an upstream `cache_creation_input_token_cost: 0` arrives as + // `cacheWritesPrice: undefined`, never as an explicit zero. Explicit-zero + // rows arise outside that fetcher, from a hand-configured or + // differently-sourced catalog, and this test pins the kernel's behaviour + // for them: a folded miss is subtracted from the input pool and billed at + // the row's zero write price, so the user pays for neither the input nor + // the write. On a fetcher-shaped row the fold is economically neutral + // instead, because the kernel falls back to `inputPrice`; it still + // mis-reports `cacheWriteTokens`, which is the defect this change fixes. handler["models"] = { "deepseek-chat": { ...relayedDeepSeekRow, cacheWritesPrice: 0 } } streamWithUsage({ prompt_tokens: 100, @@ -2395,12 +2399,14 @@ describe("LiteLLMHandler", () => { it("falls through a malformed counter to the next alias", async () => { // A proxy can relay a counter as a string. The normalizer discards it and - // keeps resolving instead of poisoning the total with NaN. + // keeps resolving instead of poisoning the total with NaN. The malformed + // value sits on the highest-priority alias so the discard is what the + // assertion depends on: without it the string would win and be reported. streamWithUsage({ prompt_tokens: 100, completion_tokens: 5, - prompt_tokens_details: { cached_tokens: 20 }, - cache_read_input_tokens: "30", + prompt_tokens_details: { cached_tokens: "30" }, + cache_read_input_tokens: 20, }) const usage = await collectUsage() From 9478c847e9dbfa29d6528809ecda42a7e99e3119 Mon Sep 17 00:00:00 2001 From: myk1yt Date: Sun, 1 Nov 2026 09:18:16 +0900 Subject: [PATCH 12/12] fix(cost): restore main's cache-price kernel so #2257's read invariant holds What: revert calculateApiCostInternal's two cache arms to main's `((price || 0) / 1e6) * tokens` shape, and re-pin the four specs that depended on the symmetric `?? inputPrice` fallback. Why: the symmetric fallback collided with #2257's merged negative control (mistral.spec.ts:657). A catalog row that omits cacheReadsPrice was billed at 1.0x input instead of $0, so the merged tree failed with `expected 0.002206 to be less than 0.00039160000000000003`. #2254's own CI was green only because it predates #2257. Three-seat panel adjudication chose option (c): #2254 drops its kernel hunk, and a writes fallback needs a dedicated kernel PR with catalog reachability evidence. Impact: cache reads without a declared read price bill $0 again, matching gemini's own calculateCost precedent and #2257's mistral catalog contract. The DeepSeek normalizer split and handler opt-in, which are #2254's real value, are untouched. New tripwire test pins the reads-$0 rule (red on the pre-revert kernel at 0.027 vs 0.018). --- .../base-openai-compatible-provider.spec.ts | 10 ++- src/shared/cost.ts | 16 ++-- src/utils/__tests__/cost.spec.ts | 76 +++++++++++++------ 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts index 565288071c..2e753ba7c5 100644 --- a/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts +++ b/src/api/providers/__tests__/base-openai-compatible-provider.spec.ts @@ -400,9 +400,13 @@ describe("BaseOpenAiCompatibleProvider", () => { expect(chunk.cacheWriteTokens).toBeUndefined() // TEST_MODEL_INFO prices input at 0.5 and output at 1.5 with no cache - // prices, so both the 80 uncached tokens and the 20 reads bill at the - // input price: (100 * 0.5 + 50 * 1.5) / 1e6. - expect(chunk.totalCost).toBeCloseTo(0.000125, 9) + // prices, so the 80 uncached tokens bill at the input price and the 20 + // reads bill at $0: (80 * 0.5 + 50 * 1.5) / 1e6. + expect(chunk.totalCost).toBeCloseTo(0.000115, 9) + + // Folding the 30 misses onto the unpriced write bucket would subtract + // them from the input pool and bill them nowhere: 0.0001. + expect(chunk.totalCost).not.toBeCloseTo(0.0001, 9) }) it("never sums two shapes that report the same quantity", async () => { diff --git a/src/shared/cost.ts b/src/shared/cost.ts index 4eaa3b2712..e7342eb34b 100644 --- a/src/shared/cost.ts +++ b/src/shared/cost.ts @@ -47,16 +47,12 @@ function calculateApiCostInternal( totalInputTokens: number, totalOutputTokens: number, ): ApiCostResult { - // Cache tokens are billed at the model's cache price when it has one, and at - // its input price when the catalog omits the cache price entirely. The - // inner `??` (not `||`) is load-bearing: an explicit `cacheWritesPrice: 0` - // is a real price for providers that charge nothing for writes, and `||` - // would silently re-price those rows at `inputPrice`. The outer `|| 0` - // covers a fully unpriced model, where `undefined * 0` would be `NaN`. - const cacheWritesCost = - (((modelInfo.cacheWritesPrice ?? modelInfo.inputPrice) || 0) / 1_000_000) * cacheCreationInputTokens - const cacheReadsCost = - (((modelInfo.cacheReadsPrice ?? modelInfo.inputPrice) || 0) / 1_000_000) * cacheReadInputTokens + // A cache price the catalog does not declare bills at $0 by design: an + // undeclared read discount is never re-priced at the input price, and an + // unpriced model must not leak `undefined * 0` as `NaN`. A writes fallback + // needs its own kernel PR with catalog reachability evidence. + const cacheWritesCost = ((modelInfo.cacheWritesPrice || 0) / 1_000_000) * cacheCreationInputTokens + const cacheReadsCost = ((modelInfo.cacheReadsPrice || 0) / 1_000_000) * cacheReadInputTokens const baseInputCost = ((modelInfo.inputPrice || 0) / 1_000_000) * inputTokens const outputCost = ((modelInfo.outputPrice || 0) / 1_000_000) * outputTokens const totalCost = cacheWritesCost + cacheReadsCost + baseInputCost + outputCost diff --git a/src/utils/__tests__/cost.spec.ts b/src/utils/__tests__/cost.spec.ts index 4555a04b54..96724629f0 100644 --- a/src/utils/__tests__/cost.spec.ts +++ b/src/utils/__tests__/cost.spec.ts @@ -95,7 +95,7 @@ describe("Cost Utility", () => { expect(result.totalOutputTokens).toBe(500) }) - it("bills cache tokens at the input price when the catalog omits cache prices", () => { + it("bills cache tokens at $0 when the catalog omits cache prices", () => { const modelWithoutCachePrices: ModelInfo = { ...mockModelInfo, cacheWritesPrice: undefined, @@ -104,22 +104,22 @@ describe("Cost Utility", () => { const result = calculateApiCostAnthropic(modelWithoutCachePrices, 1000, 500, 2000, 3000) - // A model that prices input but publishes no cache price still charges for - // the tokens it cached; the input price is the only rate it declares. + // A model that publishes no cache price declares no cache rate, so the + // cache buckets bill at $0 rather than borrowing the input price. // Input cost: (3.0 / 1_000_000) * 1000 = 0.003 - // Cache writes: (3.0 / 1_000_000) * 2000 = 0.006 - // Cache reads: (3.0 / 1_000_000) * 3000 = 0.009 + // Cache writes: 0 (no price declared) + // Cache reads: 0 (no price declared) // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.003 + 0.006 + 0.009 + 0.0075 = 0.0255 - expect(result.totalCost).toBeCloseTo(0.0255, 9) + // Total: 0.003 + 0 + 0 + 0.0075 = 0.0105 + expect(result.totalCost).toBeCloseTo(0.0105, 9) expect(result.totalInputTokens).toBe(6000) // 1000 + 2000 + 3000 expect(result.totalOutputTokens).toBe(500) }) - it("honors an explicit zero cache price instead of falling back to the input price", () => { + it("honors an explicit zero cache price instead of billing the input price", () => { // A provider that publishes `cacheWritesPrice: 0` charges nothing for - // writes. Only a *missing* cache price falls back to the input price, so - // `??` is load-bearing here and `||` would silently re-price this row. + // writes. A zero is a price, not an absence, so the kernel must never + // re-price this row at the input price. const modelWithFreeWrites: ModelInfo = { ...mockModelInfo, cacheWritesPrice: 0, @@ -239,7 +239,7 @@ describe("Cost Utility", () => { expect(result.totalOutputTokens).toBe(500) }) - it("bills cache tokens at the input price when the catalog omits cache prices", () => { + it("bills cache tokens at $0 when the catalog omits cache prices", () => { const modelWithoutCachePrices: ModelInfo = { ...mockModelInfo, cacheWritesPrice: undefined, @@ -250,18 +250,18 @@ describe("Cost Utility", () => { // OpenAI usage counts cached tokens inside prompt_tokens, so the cache // buckets are removed from the input pool and re-priced. With no cache - // price published, the input price is the only rate the model declares. + // price declared both buckets bill at $0. // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 - // Cache writes: (3.0 / 1_000_000) * 2000 = 0.006 - // Cache reads: (3.0 / 1_000_000) * 3000 = 0.009 + // Cache writes: 0 (no price declared) + // Cache reads: 0 (no price declared) // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.003 + 0.006 + 0.009 + 0.0075 = 0.0255 - expect(result.totalCost).toBeCloseTo(0.0255, 9) + // Total: 0.003 + 0 + 0 + 0.0075 = 0.0105 + expect(result.totalCost).toBeCloseTo(0.0105, 9) expect(result.totalInputTokens).toBe(6000) // Total already includes cache expect(result.totalOutputTokens).toBe(500) }) - it("honors an explicit zero cache price instead of falling back to the input price", () => { + it("honors an explicit zero cache price instead of billing the input price", () => { // friendli's GLM rows and every moonshot row publish `cacheWritesPrice: 0` // on purpose. A zero is a price, not an absence, so those rows must not be // re-priced at the input price. @@ -291,6 +291,31 @@ describe("Cost Utility", () => { expect(Number.isNaN(result.totalCost)).toBe(false) }) + it("bills cache reads at $0 when cacheReadsPrice is missing", () => { + // The mistral invariant: reporting reads without a declared read price + // must not silently bill them at the input price. A missing read price + // bills $0, so this model's reads contribute nothing while its writes + // bill at their own declared price. + const modelWithoutReadsPrice: ModelInfo = { + ...mockModelInfo, + cacheWritesPrice: 3.75, + cacheReadsPrice: undefined, + } + + const result = calculateApiCostOpenAI(modelWithoutReadsPrice, 6000, 500, 2000, 3000) + + // Uncached input: (3.0 / 1_000_000) * (6000 - 2000 - 3000) = 0.003 + // Cache writes: (3.75 / 1_000_000) * 2000 = 0.0075 + // Cache reads: 0 (no price declared) + // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 + // Total: 0.003 + 0.0075 + 0 + 0.0075 = 0.018 + expect(result.totalCost).toBeCloseTo(0.018, 9) + + // Re-symmetrizing the reads arm to the input price would bill + // (3.0 / 1_000_000) * 3000 = 0.009 for reads and total 0.027. + expect(result.totalCost).not.toBeCloseTo(0.027, 9) + }) + it("stays byte-identical for the DeepSeek catalog, whose input price is zero by design", () => { // All four deepSeekModels rows set `inputPrice: 0` and price misses through // cacheWritesPrice. A fallback that resolved through `inputPrice` would @@ -328,11 +353,12 @@ describe("Cost Utility", () => { }) }) - it("bills the miss share of a custom priced model instead of giving it away", () => { + it("keeps a custom model's miss tokens in the input pool instead of giving them away", () => { // The defect this repairs: a custom OpenAI-compatible model that prices // input but publishes no cache price. Miss tokens arrived as cache // writes, were subtracted from the input pool, and were then priced at - // zero, so the user was billed for neither. + // zero, so the user was billed for neither. The repair keeps a miss in + // the input bucket, where the input price bills it. const customModel: ModelInfo = { maxTokens: 8192, contextWindow: 200_000, @@ -343,17 +369,19 @@ describe("Cost Utility", () => { } // 1000 prompt tokens: 400 missed the cache, 100 hit it, 500 were plain. - const result = calculateApiCostOpenAI(customModel, 1000, 500, 400, 100) + // A miss reaches this kernel as ordinary input, never as a cache write. + const result = calculateApiCostOpenAI(customModel, 1000, 500, 0, 100) - // Uncached input: (3.0 / 1_000_000) * 500 = 0.0015 - // Cache writes: (3.0 / 1_000_000) * 400 = 0.0012 (input price, since none published) + // Uncached input: (3.0 / 1_000_000) * 900 = 0.0027 + // Cache writes: 0 (none reported) // Cache reads: (0.3 / 1_000_000) * 100 = 0.00003 // Output cost: (15.0 / 1_000_000) * 500 = 0.0075 - // Total: 0.0015 + 0.0012 + 0.00003 + 0.0075 = 0.01023 + // Total: 0.0027 + 0 + 0.00003 + 0.0075 = 0.01023 expect(result.totalCost).toBeCloseTo(0.01023, 9) // The under-bill the defect produced, kept as the regression's own number: - // pricing the same 400 tokens at zero yields 0.00903, 11.7% short. + // folding the same 400 tokens onto an unpriced write bucket yields + // 0.00903, 11.7% short. expect(result.totalCost).not.toBeCloseTo(0.00903, 9) })