Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/openai-chat-image-attachments.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"@open-codesign/core": patch
"@open-codesign/providers": patch
---

Forward image attachments through OpenAI-compatible and Anthropic-style provider paths instead of only marking Codex synthesized models as image-capable.
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ export function AddCustomProviderModal({
if (!editTarget.builtin) {
const previous = editTarget.tlsRejectUnauthorized === true;
if (previous !== tlsRejectUnauthorized) {
update.tlsRejectUnauthorized = tlsRejectUnauthorized ? true : false;
update.tlsRejectUnauthorized = !!tlsRejectUnauthorized;
}
}
await window.codesign.config.updateProvider(update);
Expand Down
2 changes: 1 addition & 1 deletion biome.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"$schema": "https://biomejs.dev/schemas/2.4.14/schema.json",
"$schema": "https://biomejs.dev/schemas/2.4.15/schema.json",
"vcs": {
"enabled": true,
"clientKind": "git",
Expand Down
27 changes: 27 additions & 0 deletions packages/core/src/agent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1156,6 +1156,33 @@ describe('generateViaAgent()', () => {
);
});

it('passes image attachments through openai-compatible agent models', async () => {
scriptedAgent = { assistantText: RESPONSE_WITH_ARTIFACT };
await generateViaAgent(
{
prompt: 'replicate this screenshot',
history: [],
model: { provider: 'custom-openai', modelId: 'local-text-or-vision-model' },
apiKey: 'sk-test',
wire: 'openai-chat',
baseUrl: 'https://gateway.example.test/v1',
attachments: [
{
name: 'shot.png',
path: 'references/shot.png',
mediaType: 'image/png',
imageDataUrl: 'data:image/png;base64,aW1n',
},
],
},
{ fs: makeStubFs({}) },
);

expect(agentCalls[0]?.prompts[0]?.images).toEqual([
{ type: 'image', data: 'aW1n', mimeType: 'image/png' },
]);
});

it('blocks preview and done until set_todos has run for fresh multi-step work', async () => {
scriptedAgent = { assistantText: RESPONSE_WITH_ARTIFACT };
await generateViaAgent(
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,9 @@ function supportsImageInput(wire: WireApi | undefined, modelId: string): boolean
if (wire === 'anthropic' || wire === 'openai-responses' || wire === 'openai-codex-responses') {
return true;
}
if (wire === 'openai-chat') {
return true;
}
const lower = modelId.toLowerCase();
return (
lower.includes('vision') ||
Expand Down
51 changes: 51 additions & 0 deletions packages/providers/src/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,57 @@ describe('complete', () => {
expect(result.content).toBe('ok');
});

it('keeps image inputs for synthesized openai-chat models', async () => {
getModelMock.mockReturnValue(undefined);
completeSimpleMock.mockImplementationOnce(async (model, context) => {
expect(model).toMatchObject({
api: 'openai-completions',
input: ['text', 'image'],
baseUrl: 'https://gateway.example.test/v1',
});
expect(context.messages).toEqual([
{
role: 'user',
content: [
{ type: 'text', text: 'use this screenshot' },
{ type: 'image', data: 'AAAA', mimeType: 'image/png' },
],
timestamp: 1,
},
]);
return {
role: 'assistant',
content: [{ type: 'text', text: 'ok' }],
api: 'openai-completions',
provider: 'custom-openai',
model: 'local-text-or-vision-model',
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: 'stop',
timestamp: Date.now(),
};
});

const result = await complete(
{ provider: 'custom-openai', modelId: 'local-text-or-vision-model' },
[{ role: 'user', content: 'use this screenshot' }],
{
apiKey: 'sk-test',
wire: 'openai-chat',
baseUrl: 'https://gateway.example.test/v1',
userImages: [{ data: 'AAAA', mimeType: 'image/png' }],
},
);

expect(result.content).toBe('ok');
});

it('synthesizes openai-chat PiModel with reasoning=false for Qwen DashScope (#183)', async () => {
getModelMock.mockReturnValue(undefined);
completeSimpleMock.mockImplementationOnce(async (model) => {
Expand Down
6 changes: 5 additions & 1 deletion packages/providers/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,11 @@ function synthesizeWireModel(
wire: GenerateOptions['wire'],
baseUrl: string | undefined,
): PiModel {
const supportsImageInput = wire === 'openai-codex-responses';
const supportsImageInput =
wire === 'anthropic' ||
wire === 'openai-chat' ||
wire === 'openai-responses' ||
wire === 'openai-codex-responses';
const api =
wire === 'anthropic'
? 'anthropic-messages'
Expand Down
Loading