From 628e8868085ff182b1b50326735dc1ee74547dc8 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Mon, 22 Jun 2026 17:08:44 -0500 Subject: [PATCH] fix(provider): default custom models to image input --- packages/core/src/config/plugin/provider.ts | 12 +- packages/core/src/config/provider.ts | 8 +- packages/core/src/v1/config/migrate.ts | 19 +++- packages/core/test/config/config.test.ts | 28 +++++ packages/core/test/config/provider.test.ts | 68 +++++++++++- packages/core/test/model.test.ts | 8 ++ packages/opencode/src/provider/provider.ts | 7 +- .../opencode/test/provider/provider.test.ts | 103 +++++++++++++++++- 8 files changed, 243 insertions(+), 10 deletions(-) diff --git a/packages/core/src/config/plugin/provider.ts b/packages/core/src/config/plugin/provider.ts index 0171fee37bb2..5c7b82c66530 100644 --- a/packages/core/src/config/plugin/provider.ts +++ b/packages/core/src/config/plugin/provider.ts @@ -63,16 +63,22 @@ export const Plugin = define({ const providerPackage = providerApi?.type === "aisdk" ? providerApi.package : undefined for (const [id, config] of Object.entries(item.models ?? {})) { + const existing = catalog.model.get(providerID, id) catalog.model.update(providerID, id, (model) => { if (config.family !== undefined) model.family = config.family if (config.name !== undefined) model.name = config.name if (config.api !== undefined) model.api = { ...model.api, ...config.api } const packageName = model.api.type === "aisdk" ? model.api.package : providerPackage + // TODO: Move these defaults to a dedicated configured-model constructor when one exists. + if (existing === undefined) { + model.capabilities.input = ["text", "image"] + model.capabilities.output = ["text"] + } if (config.capabilities !== undefined) { model.capabilities = { - tools: config.capabilities.tools, - input: [...config.capabilities.input], - output: [...config.capabilities.output], + tools: config.capabilities.tools ?? model.capabilities.tools, + input: [...(config.capabilities.input ?? model.capabilities.input)], + output: [...(config.capabilities.output ?? model.capabilities.output)], } } if (config.request !== undefined) { diff --git a/packages/core/src/config/provider.ts b/packages/core/src/config/provider.ts index 1b547570783a..b627d4aa6c5b 100644 --- a/packages/core/src/config/provider.ts +++ b/packages/core/src/config/provider.ts @@ -44,11 +44,17 @@ const ModelApi = Schema.Union([ }), ]) +class Capabilities extends Schema.Class("ConfigV2.Model.Capabilities")({ + tools: ModelV2.Capabilities.fields.tools.pipe(Schema.optional), + input: ModelV2.Capabilities.fields.input.pipe(Schema.optional), + output: ModelV2.Capabilities.fields.output.pipe(Schema.optional), +}) {} + class Model extends Schema.Class("ConfigV2.Model")({ family: ModelV2.Family.pipe(Schema.optional), name: Schema.String.pipe(Schema.optional), api: ModelApi.pipe(Schema.optional), - capabilities: ModelV2.Capabilities.pipe(Schema.optional), + capabilities: Capabilities.pipe(Schema.optional), request: Schema.Struct({ ...Request.fields, variant: Schema.String.pipe(Schema.optional), diff --git a/packages/core/src/v1/config/migrate.ts b/packages/core/src/v1/config/migrate.ts index c474cac51a75..7ae3656fd651 100644 --- a/packages/core/src/v1/config/migrate.ts +++ b/packages/core/src/v1/config/migrate.ts @@ -215,8 +215,23 @@ function migrateModel(info: typeof ConfigProviderV1.Model.Type, packageName?: st : []), ] const capabilities = - info.tool_call !== undefined || info.modalities?.input !== undefined || info.modalities?.output !== undefined - ? { tools: info.tool_call ?? false, input: info.modalities?.input ?? [], output: info.modalities?.output ?? [] } + info.attachment !== undefined || + info.tool_call !== undefined || + info.modalities?.input !== undefined || + info.modalities?.output !== undefined + ? { + ...(info.tool_call === undefined ? {} : { tools: info.tool_call }), + ...(info.modalities?.input !== undefined + ? { input: info.modalities.input } + : info.attachment !== undefined + ? { input: info.attachment ? ["text", "image"] : ["text"] } + : {}), + ...(info.modalities?.output !== undefined + ? { output: info.modalities.output } + : info.attachment !== undefined + ? { output: ["text"] } + : {}), + } : undefined return { family: info.family, diff --git a/packages/core/test/config/config.test.ts b/packages/core/test/config/config.test.ts index 6275d8fed350..4779d7c50569 100644 --- a/packages/core/test/config/config.test.ts +++ b/packages/core/test/config/config.test.ts @@ -116,6 +116,34 @@ describe("Config", () => { }), ) + it.effect("migrates v1 custom model capability overrides without replacing unspecified fields", () => + Effect.sync(() => { + const migrated = ConfigMigrateV1.migrate({ + provider: { + custom: { + models: { + default: { tool_call: true }, + text: { attachment: false }, + explicit: { modalities: { input: ["audio"], output: ["audio"] } }, + }, + }, + }, + }) + + expect(migrated.providers?.custom?.models?.default?.capabilities).toEqual({ + tools: true, + }) + expect(migrated.providers?.custom?.models?.text?.capabilities).toEqual({ + input: ["text"], + output: ["text"], + }) + expect(migrated.providers?.custom?.models?.explicit?.capabilities).toEqual({ + input: ["audio"], + output: ["audio"], + }) + }), + ) + it.effect("migrates v1 command configuration", () => Effect.sync(() => { expect( diff --git a/packages/core/test/config/provider.test.ts b/packages/core/test/config/provider.test.ts index 19311363edc2..bda08c972048 100644 --- a/packages/core/test/config/provider.test.ts +++ b/packages/core/test/config/provider.test.ts @@ -3,6 +3,7 @@ import { Effect, Schema } from "effect" import { Catalog } from "@opencode-ai/core/catalog" import { Config } from "@opencode-ai/core/config" import { ConfigProviderPlugin } from "@opencode-ai/core/config/plugin/provider" +import { ConfigMigrateV1 } from "@opencode-ai/core/v1/config/migrate" import { Integration } from "@opencode-ai/core/integration" import { ModelV2 } from "@opencode-ai/core/model" import { PluginV2 } from "@opencode-ai/core/plugin" @@ -244,7 +245,9 @@ describe("ConfigProviderPlugin.Plugin", () => { const provider = required(yield* catalog.provider.get(providerID)) const model = required(yield* catalog.model.get(providerID, modelID)) - expect((yield* catalog.model.default())?.id).toBe(ModelV2.ID.make("default")) + const defaultModel = required(yield* catalog.model.default()) + expect(defaultModel.id).toBe(ModelV2.ID.make("default")) + expect(defaultModel.capabilities).toEqual({ tools: false, input: ["text", "image"], output: ["text"] }) expect(provider.name).toBe("Renamed") expect((yield* integrations.get(Integration.ID.make("custom")))?.methods).toContainEqual({ type: "env", @@ -271,4 +274,67 @@ describe("ConfigProviderPlugin.Plugin", () => { }), ), ) + + it.effect("lowers a migrated attachment override into modalities", () => + Effect.gen(function* () { + const catalog = yield* Catalog.Service + const providerID = ProviderV2.ID.make("custom") + const modelID = ModelV2.ID.make("chat") + yield* catalog.transform((catalog) => + catalog.model.update(providerID, modelID, (model) => { + model.capabilities = { tools: true, input: ["text", "image", "audio"], output: ["text", "audio"] } + }), + ) + const config = Config.Service.of({ + entries: () => + Effect.succeed([ + new Config.Document({ + type: "document", + info: decode( + ConfigMigrateV1.migrate({ + provider: { custom: { models: { chat: { attachment: false } } } }, + }), + ), + }), + ]), + }) + + yield* addPlugin(config) + + expect(required(yield* catalog.model.get(providerID, modelID)).capabilities).toEqual({ + tools: true, + input: ["text"], + output: ["text"], + }) + }), + ) + + it.effect("defaults a migrated custom model with attachment disabled to text only", () => + Effect.gen(function* () { + const catalog = yield* Catalog.Service + const providerID = ProviderV2.ID.make("custom") + const modelID = ModelV2.ID.make("chat") + const config = Config.Service.of({ + entries: () => + Effect.succeed([ + new Config.Document({ + type: "document", + info: decode( + ConfigMigrateV1.migrate({ + provider: { custom: { models: { chat: { attachment: false } } } }, + }), + ), + }), + ]), + }) + + yield* addPlugin(config) + + expect(required(yield* catalog.model.get(providerID, modelID)).capabilities).toEqual({ + tools: false, + input: ["text"], + output: ["text"], + }) + }), + ) }) diff --git a/packages/core/test/model.test.ts b/packages/core/test/model.test.ts index fe97acc25aad..20869be2113a 100644 --- a/packages/core/test/model.test.ts +++ b/packages/core/test/model.test.ts @@ -21,3 +21,11 @@ describe("ModelV2.Ref", () => { }) }) }) + +describe("ModelV2.Info", () => { + test("creates an empty model without modalities", () => { + const model = ModelV2.Info.empty(ProviderV2.ID.make("custom"), ModelV2.ID.make("model")) + + expect(model.capabilities).toEqual({ tools: false, input: [], output: [] }) + }) +}) diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index d7c85f2ee6ae..8d757da00c06 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -1413,12 +1413,15 @@ export const layer = Layer.effect( capabilities: { temperature: model.temperature ?? existingModel?.capabilities.temperature ?? false, reasoning: model.reasoning ?? existingModel?.capabilities.reasoning ?? false, - attachment: model.attachment ?? existingModel?.capabilities.attachment ?? false, + attachment: model.attachment ?? existingModel?.capabilities.attachment ?? true, toolcall: model.tool_call ?? existingModel?.capabilities.toolcall ?? true, input: { text: model.modalities?.input?.includes("text") ?? existingModel?.capabilities.input.text ?? true, audio: model.modalities?.input?.includes("audio") ?? existingModel?.capabilities.input.audio ?? false, - image: model.modalities?.input?.includes("image") ?? existingModel?.capabilities.input.image ?? false, + image: + model.modalities?.input?.includes("image") ?? + existingModel?.capabilities.input.image ?? + model.attachment !== false, video: model.modalities?.input?.includes("video") ?? existingModel?.capabilities.input.video ?? false, pdf: model.modalities?.input?.includes("pdf") ?? existingModel?.capabilities.input.pdf ?? false, }, diff --git a/packages/opencode/test/provider/provider.test.ts b/packages/opencode/test/provider/provider.test.ts index 6edfc97ca06e..68ba9e9f78d5 100644 --- a/packages/opencode/test/provider/provider.test.ts +++ b/packages/opencode/test/provider/provider.test.ts @@ -544,6 +544,46 @@ it.instance( }, ) +it.instance( + "inherited modalities take precedence over attachment false", + Effect.gen(function* () { + yield* set("ANTHROPIC_API_KEY", "test-api-key") + const providers = yield* list + const model = providers[ProviderV2.ID.anthropic].models["claude-sonnet-4-20250514"] + expect(model.capabilities.attachment).toBe(false) + expect(model.capabilities.input.text).toBe(true) + expect(model.capabilities.input.image).toBe(true) + expect(model.capabilities.output.text).toBe(true) + }), + { + config: { + provider: { anthropic: { models: { "claude-sonnet-4-20250514": { attachment: false } } } }, + }, + }, +) + +it.instance( + "inherited modalities take precedence over attachment true", + Effect.gen(function* () { + const providers = yield* list + const model = providers[ProviderV2.ID.make("perplexity")].models.sonar + expect(model.capabilities.attachment).toBe(true) + expect(model.capabilities.input.text).toBe(true) + expect(model.capabilities.input.image).toBe(false) + expect(model.capabilities.output.text).toBe(true) + }), + { + config: { + provider: { + perplexity: { + env: [], + models: { sonar: { attachment: true } }, + }, + }, + }, + }, +) + it.instance( "disabled_providers prevents loading even with env var", Effect.gen(function* () { @@ -589,11 +629,13 @@ it.instance( ) it.instance( - "model modalities default correctly", + "custom model media capabilities default correctly", Effect.gen(function* () { const providers = yield* list const model = providers[ProviderV2.ID.make("test-provider")].models["test-model"] + expect(model.capabilities.attachment).toBe(true) expect(model.capabilities.input.text).toBe(true) + expect(model.capabilities.input.image).toBe(true) expect(model.capabilities.output.text).toBe(true) }), { @@ -611,6 +653,65 @@ it.instance( }, ) +it.instance( + "custom model media capabilities can be disabled explicitly", + Effect.gen(function* () { + const providers = yield* list + const model = providers[ProviderV2.ID.make("test-provider")].models["test-model"] + expect(model.capabilities.attachment).toBe(false) + expect(model.capabilities.input.text).toBe(true) + expect(model.capabilities.input.image).toBe(false) + }), + { + config: { + provider: { + "test-provider": { + name: "Test", + npm: "@ai-sdk/openai-compatible", + env: [], + models: { + "test-model": { + name: "Test Model", + attachment: false, + }, + }, + }, + }, + }, + }, +) + +it.instance( + "custom model explicit modalities override media defaults", + Effect.gen(function* () { + const providers = yield* list + const provider = providers[ProviderV2.ID.make("test-provider")] + const audio = provider.models.audio + expect(audio.capabilities.input.text).toBe(false) + expect(audio.capabilities.input.audio).toBe(true) + expect(audio.capabilities.input.image).toBe(false) + expect(audio.capabilities.output.audio).toBe(true) + const empty = provider.models.empty + expect(Object.values(empty.capabilities.input).some(Boolean)).toBe(false) + expect(Object.values(empty.capabilities.output).some(Boolean)).toBe(false) + }), + { + config: { + provider: { + "test-provider": { + name: "Test", + npm: "@ai-sdk/openai-compatible", + env: [], + models: { + audio: { modalities: { input: ["audio"], output: ["audio"] } }, + empty: { modalities: { input: [], output: [] } }, + }, + }, + }, + }, + }, +) + it.instance( "model with custom cost values", Effect.gen(function* () {