diff --git a/docs-site/src/content/docs/es/guides/providers.md b/docs-site/src/content/docs/es/guides/providers.md index 7910734110f..c97e3e5ae71 100644 --- a/docs-site/src/content/docs/es/guides/providers.md +++ b/docs-site/src/content/docs/es/guides/providers.md @@ -559,6 +559,8 @@ Un resultado denegado por acceso no prueba por sí mismo un inicio de sesión ve Las solicitudes de chat compatibles con la traducción OpenAI reducen las imágenes en línea cuando sus datos base64 combinados superan los 3,5 MiB. Las imágenes más antiguas pierden detalle primero. Este es un presupuesto de imagen de mejor esfuerzo, por lo que el texto grande, los esquemas o las imágenes que no se pueden procesar aún pueden exceder un límite de solicitud ascendente. Las URL de imágenes remotas no se descargan y las imágenes que no se pueden reducir permanecen adjuntas. El passthrough de Native Chat mantiene sus bytes de imagen originales. +Venice también intenta ampliar las imágenes en línea pequeñas hasta su lado mínimo de 64 píxeles. Esta ampliación es de mejor esfuerzo: las dimensiones pueden ser ilegibles cuando, por ejemplo, la cabecera de trama de un JPEG aparece después de más de unos 48 KiB de metadatos. Antes de redimensionar, el códec comprueba las dimensiones originales frente al límite local de 16 millones de píxeles. Las imágenes con dimensiones ilegibles, no válidas o demasiado grandes permanecen adjuntas sin cambios y el proveedor puede rechazarlas; OpenCodex no las decodifica localmente para ampliarlas. + ### Cambios del catálogo y cuotas diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index 5ba44ed6250..7916a8e07a6 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -967,3 +967,5 @@ An access-denied result does not by itself prove an expired login or an ineligib ## Large inline images on Chat providers Translated OpenAI-compatible Chat requests shrink inline images when their combined base64 data exceeds 3.5 MiB. Older images lose detail first. This is a best-effort image budget, so large text, schemas, or images that cannot be processed may still exceed an upstream request limit. Remote image URLs are not downloaded, and images that cannot be shrunk remain attached. Native Chat passthrough keeps its original image bytes. + +Venice also attempts to enlarge small inline images to its 64-pixel minimum edge. This is best effort: dimensions may be unreadable when, for example, a JPEG frame header follows more than about 48 KiB of metadata. Before resizing, the image codec checks the original dimensions against the 16-million-pixel local processing limit. Images with unreadable, invalid, or oversized dimensions remain attached unchanged, so the provider may reject them; OpenCodex does not decode them locally for enlargement. diff --git a/src/adapters/openai-chat-images.ts b/src/adapters/openai-chat-images.ts index 55fd52ebe14..b4baefc0567 100644 --- a/src/adapters/openai-chat-images.ts +++ b/src/adapters/openai-chat-images.ts @@ -23,7 +23,15 @@ export type UpscaleFn = (input: Buffer, width: number, height: number) => Promis /** Default upscaler: Bun.Image resize, re-encoded as PNG so small text strips stay sharp. */ export const bunImageUpscale: UpscaleFn = async (input, width, height) => { - const out = await new Bun.Image(input).resize(width, height).png().toBuffer(); + const image = new Bun.Image(input); + // Sniffed dimensions choose candidates, but only the codec can bound its decode. + const meta = await image.metadata(); + const w = meta.width; + const h = meta.height; + if (!Number.isSafeInteger(w) || !Number.isSafeInteger(h) || w <= 0 || h <= 0 || w * h > MAX_INPUT_PIXELS) { + throw new Error("image dimensions exceed the safe decode limit"); + } + const out = await image.resize(width, height).png().toBuffer(); return { data: Buffer.from(out).toString("base64"), mediaType: "image/png" }; }; diff --git a/structure/transports/streaming-health.md b/structure/transports/streaming-health.md index e4bedee91b6..6016b97c480 100644 --- a/structure/transports/streaming-health.md +++ b/structure/transports/streaming-health.md @@ -364,7 +364,7 @@ Live sideband admission and its bounded upstream handshake follow the [runtime c ## Translated Chat inline-image budget -`src/adapters/openai-chat-images.ts` reuses the shared image normalization ladder for translated Chat bodies above a 3.5 MiB base64-image budget. This is best effort, not a whole-request ceiling. Remote URLs are not fetched; unprocessable and terminal images remain attached, and retained bytes continue to count during demotion. Under-budget construction stays synchronous; delegating MiMo awaits conditional asynchronous construction. Native decoding is admitted through a process-wide four-operation gate, guarded by a 16-million-pixel ceiling confirmed from codec metadata, and request cancellation stops queued and subsequent normalization work. Native Chat passthrough and Anthropic-only 413 retry policy retain their existing behavior. A provider whose registry row declares `minImageEdge` (Venice: 64) then has every decodable inline image whose short edge is below it upscaled to that edge, keeping the aspect ratio unless the long edge would pass 4096, where it is capped (extreme strips are then stretched); this runs after budget normalization because the 320px terminal tier can floor a wide screenshot under the minimum, and an image that fails to decode keeps its original URL. +`src/adapters/openai-chat-images.ts` reuses the shared image normalization ladder for translated Chat bodies above a 3.5 MiB base64-image budget. This is best effort, not a whole-request ceiling. Remote URLs are not fetched; unprocessable and terminal images remain attached, and retained bytes continue to count during demotion. Under-budget construction stays synchronous; delegating MiMo awaits conditional asynchronous construction. Native decoding is admitted through a process-wide four-operation gate, guarded by a 16-million-pixel ceiling confirmed from codec metadata, and request cancellation stops queued and subsequent normalization work. Native Chat passthrough and Anthropic-only 413 retry policy retain their existing behavior. A provider whose registry row declares `minImageEdge` (Venice: 64) then has every decodable inline image whose short edge is below it upscaled to that edge, keeping the aspect ratio unless the long edge would pass 4096, where it is capped (extreme strips are then stretched); this runs after budget normalization because the 320px terminal tier can floor a wide screenshot under the minimum, and an image that fails to decode keeps its original URL. The minimum-edge upscaler also verifies positive integral native codec dimensions against the same 16-million-pixel ceiling before resize starts. Header sniffing selects candidates but cannot authorize a decode; invalid or oversized codec metadata leaves the original URL attached and releases the shared decode slot. `tests/adapters/openai/openai-chat-image-normalization.test.ts` covers disagreement between sniffed and native dimensions, refusal before resize, retained images, and subsequent admitted work. The [explicit model-capability contract](../config.md#explicit-per-model-capability-declarations) preserves operator declarations through provider storage and catalog capture; it does not infer upstream capability or change this surface's routing behavior. diff --git a/tests/adapters/openai/openai-chat-image-normalization.test.ts b/tests/adapters/openai/openai-chat-image-normalization.test.ts index 1590cf14d97..5f462b18f6e 100644 --- a/tests/adapters/openai/openai-chat-image-normalization.test.ts +++ b/tests/adapters/openai/openai-chat-image-normalization.test.ts @@ -1,8 +1,9 @@ -import { beforeEach, describe, expect, test } from "bun:test"; +import { beforeEach, describe, expect, spyOn, test } from "bun:test"; import { createOpenAIChatAdapter } from "../../../src/adapters/openai-chat"; import { createMimoFreeAdapter, resetMimoJwtCache } from "../../../src/adapters/mimo-free"; import { hasShrinkableOpenAIChatImages, + bunImageUpscale, normalizeOpenAIChatImages, OPENAI_CHAT_IMAGE_BASE64_BUDGET, upscaledImageSize, @@ -120,6 +121,25 @@ function headerOnlyBmpB64(width: number, height: number): string { return bytes.toString("base64"); } +/** A tiny JPEG with oversized native SOF dimensions and a misleading APP1 marker. */ +async function misleadingOversizedJpeg(): Promise { + const jpeg = Buffer.from(await new Bun.Image(Buffer.from(ONE_PX_PNG, "base64")).jpeg().toBuffer()); + let patched = false; + for (let o = 2; o + 9 < jpeg.length; o++) { + if (jpeg[o] === 0xff && (jpeg[o + 1] === 0xc0 || jpeg[o + 1] === 0xc2)) { + jpeg.writeUInt16BE(4_000, o + 5); + jpeg.writeUInt16BE(5_000, o + 7); + patched = true; + break; + } + } + if (!patched) throw new Error("fixture JPEG has no SOF"); + const fakeSof = Buffer.from([0xff, 0xc0, 0, 11, 8, 0, 1, 0, 1, 1, 1, 0x11, 0]); + const prefix = Buffer.from([0xff, 0xd8, 0xff, 0, 0, 8, 0, 0, 0xff, 0xe1, 0, fakeSof.length + 2]); + // No large image is encoded or decoded. Tests intercept resize before native work. + return Buffer.concat([prefix, fakeSof, jpeg.subarray(2)]); +} + /** Wrap raw base64 as a data URL, the only image form this wire normalizes. */ function dataUrl(b64: string, mediaType = "image/png"): string { @@ -502,6 +522,61 @@ describe("openai-chat minimum image edge", () => { authMode: "key", }; + test("upscaling checks native pixel dimensions even when the sniffer sees one pixel", async () => { + const input = await misleadingOversizedJpeg(); + expect(input.length).toBeLessThan(1_024); + expect(sniffImageDimensions(input.toString("base64"))).toEqual({ width: 1, height: 1 }); + expect(await new Bun.Image(input).metadata()).toMatchObject({ width: 5_000, height: 4_000 }); + const resize = spyOn(Bun.Image.prototype, "resize").mockImplementation(() => { + throw new Error("unexpected native resize"); + }); + try { + await expect(bunImageUpscale(input, 64, 64)).rejects.toThrow("image dimensions exceed the safe decode limit"); + expect(resize).not.toHaveBeenCalled(); + } finally { + resize.mockRestore(); + } + }); + + test("Venice preserves oversized originals without resize and releases decode admission", async () => { + const input = await misleadingOversizedJpeg(); + const original = dataUrl(input.toString("base64"), "image/jpeg"); + const resize = spyOn(Bun.Image.prototype, "resize").mockImplementation(() => { + throw new Error("unexpected native resize"); + }); + try { + const built = await Promise.all(Array.from({ length: IMAGE_NORMALIZE_CONCURRENCY + 2 }, () => + createOpenAIChatAdapter(venice).buildRequest(parsedWith([imageMessage([original])])))); + for (const request of built) { + expect(imageParts(wireMessages(request.body as string))[0]?.image_url?.url).toBe(original); + } + expect(resize).not.toHaveBeenCalled(); + } finally { + resize.mockRestore(); + } + const next = await createOpenAIChatAdapter(venice).buildRequest(parsedWith([imageMessage([dataUrl(ONE_PX_PNG)])])); + const sent = imageParts(wireMessages(next.body as string))[0]?.image_url?.url ?? ""; + expect(sniffImageDimensions(sent.split(",")[1] ?? "")).toEqual({ width: 64, height: 64 }); + }); + + test("upscaling refuses invalid native dimensions before resize", async () => { + const input = Buffer.from(ONE_PX_PNG, "base64"); + const metadata = spyOn(Bun.Image.prototype, "metadata"); + const resize = spyOn(Bun.Image.prototype, "resize").mockImplementation(() => { + throw new Error("unexpected native resize"); + }); + try { + for (const width of [0, -1, Number.NaN, Infinity, 0.5]) { + metadata.mockResolvedValue({ width, height: 1, format: "png" }); + await expect(bunImageUpscale(input, 64, 64)).rejects.toThrow("image dimensions exceed the safe decode limit"); + } + expect(resize).not.toHaveBeenCalled(); + } finally { + resize.mockRestore(); + metadata.mockRestore(); + } + }); + test("a Venice image below 64px is upscaled with its aspect ratio kept", async () => { const strip = await realPngB64(320, 25); const built = createOpenAIChatAdapter(venice).buildRequest(parsedWith([imageMessage([dataUrl(strip)])]));