Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions harness-tests/core/model-attachments.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,44 @@ beforeEach(() => {
store.reads = 0
})
describe('model attachment boundary', () => {
it.each(['gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna'])(
'prepares images and PDFs for %s while preserving gateway and custom endpoint limits',
async (model) => {
const image = await file('png bytes', 'image/png')
const pdf = await file('%PDF-1.7 synthetic', 'application/pdf')
const projected = await prepareAttachmentMessages(
[message([image, pdf])],
{ env, scope, connection: connection('openai', model) },
)
expect(
projected[0].parts.filter((part) => part.type !== 'text'),
).toMatchObject([
{ type: 'image', source: { type: 'data', mimeType: 'image/png' } },
{
type: 'document',
source: { type: 'data', mimeType: 'application/pdf' },
},
])
for (const selected of [
connection('openrouter', `openai/${model}`),
connection('vercel', `openai/${model}`),
]) {
expect(modelAttachmentSupport(selected)).toMatchObject({
imageTypes: ['image/png', 'image/jpeg'],
pdf: false,
})
expect(() =>
validateModelAttachments([pdf], { connection: selected }),
).toThrow(/verified support/)
}
expect(modelAttachmentSupport(connection('compatible', model))).toEqual({
model,
imageTypes: [],
pdf: false,
})
},
)

it('reads exact same-bot, other-bot and legacy sources independently of the receiving conversation scope', async () => {
const sameBot = await file('Same bot source')
sameBot.conversationId = 'current:source-thread'
Expand Down
67 changes: 67 additions & 0 deletions harness-tests/core/run-model-picker.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { createElement, type ComponentProps } from 'react'
import { renderToStaticMarkup } from 'react-dom/server'
import { describe, expect, it, vi } from 'vitest'
import { RunModelPicker } from '../../src/chat/components/RunModelPicker'
import type { RunModelCatalog } from '../../src/chat/core/run-model'

// Render the popup inline so assertions cover the picker's actual choices.
vi.mock('@base-ui/react/popover', async () => {
const { createElement } = await import('react')
const children = (props: ComponentProps<'div'>) => props.children
return {
Popover: {
Root: children,
Portal: children,
Positioner: children,
Popup: children,
Title: children,
Trigger: (props: ComponentProps<'button'>) =>
createElement('button', props),
},
}
})

const catalog: RunModelCatalog = {
defaultSelection: { provider: 'openai', model: 'gpt-5.6-terra' },
choices: ['gpt-5-mini', 'o3', 'gpt-5.6-terra', 'gpt-5.1', 'o3-mini'].map(
(model) => ({
selection: { provider: 'openai', model },
label: model,
providerLabel: 'OpenAI',
attachments: [],
reasoning: [{ value: 'default', label: 'Default' }],
retiring: model === 'gpt-5-mini' || model === 'o3',
}),
),
}

describe('retiring model picker choices', () => {
it.each(['gpt-5.6-terra', 'gpt-5-mini'])(
'keeps %s selected while hiding other retiring choices',
(model) => {
const html = renderToStaticMarkup(
createElement(RunModelPicker, {
model: {
selection: { provider: 'openai', model },
catalog,
catalogReady: true,
loading: false,
refreshing: false,
error: '',
refetch: vi.fn(),
select: vi.fn(),
},
}),
)
expect(html).toContain('<strong>gpt-5.6-terra</strong>')
expect(html).toContain('<strong>gpt-5.1</strong>')
expect(html).toContain('<strong>o3-mini</strong>')
expect(html).not.toContain('<strong>o3</strong>')
expect(html.includes('<strong>gpt-5-mini</strong>')).toBe(
model === 'gpt-5-mini',
)
expect(html).toContain(`aria-label="Model: ${model}"`)
expect(html).toContain('aria-pressed="true"')
},
)
})
144 changes: 144 additions & 0 deletions harness-tests/core/run-models.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
type Provider,
} from '../../src/chat/core/types'
import type { RunModelSelection } from '../../src/chat/core/run-model'
import { resolveRunModelChoice } from '../../src/chat/components/useRunModel'

const state = vi.hoisted(() => ({
credentials: null as Credentials | null,
Expand Down Expand Up @@ -428,6 +429,149 @@ describe('model catalog and policy', () => {
})
})

describe('OpenAI December 2026 retirement', () => {
const retiringModels = [
'gpt-5',
'gpt-5-2025-08-07',
'gpt-5-mini',
'gpt-5-mini-2025-08-07',
'gpt-5-nano',
'gpt-5-nano-2025-08-07',
'gpt-5-pro',
'gpt-5-pro-2025-10-06',
'o3',
'o3-2025-04-16',
'o3-pro',
'o3-pro-2025-06-10',
]
const modelProviders: Provider[] = ['openai', 'openrouter', 'vercel']

it.each(modelProviders)(
'marks retiring %s models without marking other GPT-5 or o-series models',
async (provider) => {
const prefix = provider === 'openai' ? '' : 'openai/'
configured(provider, `${prefix}gpt-5.6-sol`)
const catalog = await getRunModelCatalog(env, scope)
const choices = catalog.choices
.filter(
(choice) =>
choice.selection.provider === provider && !choice.retiring,
)
.map((choice) => choice.selection.model)
for (const model of retiringModels)
expect(choices).not.toContain(`${prefix}${model}`)
for (const model of ['gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna'])
expect(choices).toContain(`${prefix}${model}`)
const installed =
provider === 'openai'
? OPENAI_CHAT_MODELS
: provider === 'openrouter'
? OPENROUTER_CHAT_MODELS
: VERCEL_GATEWAY_CHAT_MODELS
for (const model of installed)
if (
!model.startsWith('~') &&
!model.endsWith(':batch') &&
!retiringModels.includes(model.replace(/^openai\//, ''))
)
expect(choices).toContain(model)
},
)

it.each(modelProviders)(
'keeps configured and pinned %s selections without silently migrating them',
async (provider) => {
const prefix = provider === 'openai' ? '' : 'openai/'
for (const model of retiringModels) {
const selection = select(provider, `${prefix}${model}`)
configured(provider, selection.model)
const catalog = await getRunModelCatalog(env, scope)
expect(catalog.defaultSelection).toEqual(selection)
expect(
catalog.choices.find(
(choice) => choice.selection.model === selection.model,
)?.label,
).toBe(`${selection.model} (retires December 11, 2026)`)
if (
provider === 'openai' &&
!OPENAI_CHAT_MODELS.some((candidate) => candidate === model)
)
await expect(resolveRunModel(env, scope)).rejects.toMatchObject({
status: 409,
})
else {
expect((await resolveRunModel(env, scope)).selection).toEqual(
selection,
)
expect(resolveRunModelChoice(selection, catalog).error).toBe('')
}
}
const pinned = select(provider, `${prefix}gpt-5-mini`)
configured(provider, `${prefix}gpt-5.6-terra`)
expect(
(await resolveRunModel(env, { ...scope, selection: pinned })).selection,
).toEqual(pinned)
const catalog = await getRunModelCatalog(env, scope)
expect(resolveRunModelChoice(pinned, catalog)).toMatchObject({
choice: { selection: pinned, retiring: true },
error: '',
})
},
)

it.each(['gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna'])(
'exposes reviewed reasoning and attachments for %s',
async (model) => {
configured('openai', model)
const catalog = await getRunModelCatalog(env, scope)
const choice = catalog.choices.find(
(choice) => choice.selection.model === model,
)
const efforts = ['none', 'low', 'medium', 'high', 'xhigh', 'max']
expect(choice?.reasoning.map((option) => option.value)).toEqual([
'default',
...efforts,
])
expect(choice?.attachments).toEqual(['image', 'pdf'])
expect((await resolveRunModel(env, scope)).modelOptions).toEqual({
max_output_tokens: 2048,
})
for (const effort of efforts)
expect(
(
await resolveRunModel(env, {
...scope,
selection: select('openai', model, effort),
})
).modelOptions,
).toEqual({
max_output_tokens: 2048,
reasoning: { effort },
})
await expect(
resolveRunModel(env, {
...scope,
selection: select('openai', model, 'minimal'),
}),
).rejects.toMatchObject({ status: 400 })
},
)

it('does not infer retirement for a custom compatible endpoint', async () => {
configured('compatible', 'gpt-5', {
baseUrl: 'https://models.example/v1',
})
const catalog = await getRunModelCatalog(env, scope)
const selected = resolveRunModelChoice(catalog.defaultSelection, catalog)
expect(selected.error).toBe('')
expect(selected.choice?.retiring).toBeUndefined()
expect(selected.choice?.label).toBe('gpt-5')
expect((await resolveRunModel(env, scope)).selection).toEqual(
catalog.defaultSelection,
)
})
})

describe('reviewed reasoning options', () => {
it('keeps Kimi default off and exposes a genuine on switch without invented effort levels', async () => {
const choice = (await getRunModelCatalog(env, scope)).choices[0]
Expand Down
6 changes: 6 additions & 0 deletions src/chat/components/RunModelPicker.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ export function RunModelPicker({
NonNullable<RunModelController['catalog']>['choices']
>()
for (const choice of model.catalog?.choices ?? []) {
if (
choice.retiring &&
(choice.selection.provider !== selection?.provider ||
choice.selection.model !== selection.model)
)
continue
if (
!`${choice.label} ${choice.providerLabel} ${choice.selection.model}`
.toLowerCase()
Expand Down
1 change: 1 addition & 0 deletions src/chat/core/run-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export interface RunModelChoice {
providerLabel: string
attachments: Array<'image' | 'pdf'>
reasoning: Array<{ value: string; label: string }>
retiring?: boolean
unavailableReason?: string
}
export interface RunModelCatalog {
Expand Down
3 changes: 3 additions & 0 deletions src/chat/server/model-attachments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ const openaiVision = new Set([
'gpt-5.2',
'gpt-5.4',
'gpt-5.5',
'gpt-5.6-sol',
'gpt-5.6-terra',
'gpt-5.6-luna',
'o3',
'o4-mini',
])
Expand Down
30 changes: 29 additions & 1 deletion src/chat/server/run-models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,29 @@ const maxCatalogModels = 2048
const isNative = (provider: Provider, model: string) =>
nativeModels[provider]?.includes(model) === true

// https://developers.openai.com/api/docs/deprecations#2026-06-11-gpt-5-and-o3-model-deprecations
// Hide these from new picker choices, but preserve configured and pinned selections.
const retiringOpenaiModels = new Set([
'gpt-5',
'gpt-5-2025-08-07',
'gpt-5-mini',
'gpt-5-mini-2025-08-07',
'gpt-5-nano',
'gpt-5-nano-2025-08-07',
'gpt-5-pro',
'gpt-5-pro-2025-10-06',
'o3',
'o3-2025-04-16',
'o3-pro',
'o3-pro-2025-06-10',
])

/** Exact reviewed model IDs only. An adapter update does not enable new controls. */
const openaiEfforts: Record<string, readonly string[]> = {
'gpt-6-astra': ['low', 'medium', 'high'],
'gpt-5.6-sol': ['none', 'low', 'medium', 'high', 'xhigh', 'max'],
'gpt-5.6-terra': ['none', 'low', 'medium', 'high', 'xhigh', 'max'],
'gpt-5.6-luna': ['none', 'low', 'medium', 'high', 'xhigh', 'max'],
'gpt-5.5': ['none', 'low', 'medium', 'high'],
'gpt-5.2': ['none', 'low', 'medium', 'high'],
'gpt-5.1': ['none', 'low', 'medium', 'high'],
Expand Down Expand Up @@ -331,6 +351,11 @@ export async function getRunModelCatalog(
503,
)
for (const model of models) {
const retiring =
(provider === 'openai' && retiringOpenaiModels.has(model)) ||
((provider === 'openrouter' || provider === 'vercel') &&
model.startsWith('openai/') &&
retiringOpenaiModels.has(model.slice('openai/'.length)))
const selected = { ...connection, model }
const support = modelAttachmentSupport(selected, env.INCLUDED_MODEL)
const unavailableReason = availability(scope.policy, selected)?.message
Expand All @@ -339,7 +364,9 @@ export async function getRunModelCatalog(
label:
scope.fixture && provider === 'included'
? 'Local preview'
: (modelLabels[model] ?? model),
: retiring
? `${model} (retires December 11, 2026)`
: (modelLabels[model] ?? model),
providerLabel: providerLabels[provider],
attachments: [
...(support.imageTypes.length ? ['image' as const] : []),
Expand All @@ -348,6 +375,7 @@ export async function getRunModelCatalog(
reasoning: ['default', ...reasoningValues(provider, model)].map(
(value) => ({ value, label: reasoningLabels[value] ?? value }),
),
...(retiring ? { retiring: true } : {}),
...(unavailableReason ? { unavailableReason } : {}),
})
}
Expand Down
Loading