+
+
+ Vision override
+ /
+
+
/
@@ -86,6 +98,16 @@
border-top: 1px solid var(--color-border);
}
+ .model-preset-row-nested {
+ border-top: 0 !important;
+ padding-top: var(--spacing-xs);
+ font-size: 0.76rem;
+ }
+
+ .model-preset-identity-vision {
+ grid-column: 3;
+ }
+
.model-preset-icon,
.model-preset-provider,
.model-preset-separator {
diff --git a/prompts/AGENTS.md b/prompts/AGENTS.md
index a3b9fb85ea..10917ba201 100644
--- a/prompts/AGENTS.md
+++ b/prompts/AGENTS.md
@@ -25,6 +25,7 @@
- Read the rendering path before changing placeholders or filenames.
- Prefer small prompt additions over broad rewrites when fixing a specific behavior.
- Keep document/OCR routing explicit: image files, screenshots, scans, charts, photos, and diagrams should prefer vision tools when available, while `document_query` is for documents, large text-heavy files, and fallback OCR.
+- Keep the single `vision_load` tool prompt route-agnostic and preserve one-call loading for related images; internal image-analysis instructions belong in framework prompts, not Python strings.
- Update tests or snapshots when prompt budget, required sections, or generated system content changes.
## Verification
diff --git a/prompts/fw.vision_load.md b/prompts/fw.vision_load.md
new file mode 100644
index 0000000000..1c7320e2f4
--- /dev/null
+++ b/prompts/fw.vision_load.md
@@ -0,0 +1,4 @@
+Analyze the attached image(s) for the current request. Return only relevant visible details, text, and layout.
+
+Current request:
+{{request}}
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 7fba2ba480..9198aff389 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -36,8 +36,13 @@ class _TestAgentContextType:
class _TestResponse(SimpleNamespace):
- def __init__(self, message="", break_loop=False, **kwargs):
- super().__init__(message=message, break_loop=break_loop, **kwargs)
+ def __init__(self, message="", break_loop=False, additional=None, **kwargs):
+ super().__init__(
+ message=message,
+ break_loop=break_loop,
+ additional=additional,
+ **kwargs,
+ )
class _TestTool:
@@ -58,6 +63,15 @@ def __init__(
self.message = message
self.loop_data = loop_data
+ async def after_execution(self, response, **kwargs):
+ self.agent.hist_add_tool_result(
+ self.name,
+ response.message.strip(),
+ id=self.log.id,
+ **(response.additional or {}),
+ )
+ self.log.update(content=response.message.strip())
+
class _TestWsHandler:
def __init__(self, *args, **kwargs):
@@ -96,6 +110,8 @@ def error(code="", message="", correlation_id=None):
_model_config_stub.get_presets = lambda: []
_model_config_stub.get_preset_by_name = lambda name: None
_model_config_stub.get_chat_model_config = lambda agent=None: {}
+_model_config_stub.get_vision_model_config = lambda agent=None: {}
+_model_config_stub.build_vision_model = lambda agent=None: None
sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)
@@ -4297,11 +4313,8 @@ def fake_normalize_a0_path(path):
monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
- monkeypatch.setattr(
- vision_load_module.plugins,
- "get_plugin_config",
- lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},
- )
+ monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10})
+ monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
tool_results = []
messages = []
@@ -4338,7 +4351,7 @@ def fake_normalize_a0_path(path):
assert stored_ref.startswith("/a0/usr/chats/ctx-vision/screenshots/browser/browser-shot-")
stored_path = tmp_path / stored_ref.removeprefix("/a0/")
assert stored_path.read_bytes() == __import__("base64").b64decode(SMALL_JPEG_10X10)
- assert updates[-1]["result"] == "1 images loaded, 0 skipped"
+ assert updates[-1]["content"] == response.message
@pytest.mark.anyio
diff --git a/tests/test_model_config_api_keys.py b/tests/test_model_config_api_keys.py
index eb779aba12..ee9c1eda5e 100644
--- a/tests/test_model_config_api_keys.py
+++ b/tests/test_model_config_api_keys.py
@@ -89,6 +89,34 @@ def test_chat_model_configured_requires_identity_and_key(monkeypatch):
)
+def test_missing_api_key_checks_only_the_active_vision_model(monkeypatch):
+ from plugins._model_config.helpers import model_config
+
+ config = {
+ "chat_model": {"provider": "ollama", "name": "text-main", "vision": True},
+ "vision_model": {"provider": "openai", "name": "vision"},
+ "utility_model": {"provider": "ollama", "name": "utility"},
+ "embedding_model": {
+ "provider": "huggingface",
+ "name": "sentence-transformers/all-MiniLM-L6-v2",
+ },
+ }
+ monkeypatch.setattr(model_config, "get_effective_config", lambda _agent=None: config)
+ monkeypatch.setattr(
+ model_config,
+ "get_embedding_model_config",
+ lambda _agent=None: config["embedding_model"],
+ )
+ monkeypatch.setattr(model_config, "has_provider_api_key", lambda *args: False)
+
+ assert model_config.get_missing_api_key_providers() == []
+
+ config["chat_model"]["vision"] = False
+ assert model_config.get_missing_api_key_providers() == [
+ {"model_type": "Vision Model", "provider": "openai"}
+ ]
+
+
@pytest.mark.asyncio
async def test_missing_api_key_banner_exposes_only_effective_missing_providers(monkeypatch):
from plugins._model_config.helpers import model_config
@@ -149,7 +177,7 @@ def test_model_config_frontend_tracks_provider_api_key_edits():
assert "/plugins/_model_config/missing_api_key_status" not in model_gate_content
assert '@input="$store.modelConfig.setApiKeyValue(_prov, $el.value)"' in config_content
assert "apiKeyMode: 'none'" not in preset_modal_content
- assert preset_modal_content.count("apiKeyMode: 'store'") == 3
+ assert preset_modal_content.count("apiKeyMode: 'store'") == 4
assert "$store.modelConfig.resetApiKeyDrafts();" in preset_modal_content
assert "await $store.modelConfig.refreshApiKeyStatus();" in preset_modal_content
assert "await store.persistAllDirtyApiKeys();" in store_content
diff --git a/tests/test_model_config_project_presets.py b/tests/test_model_config_project_presets.py
index bb8c670454..1c972b43cd 100644
--- a/tests/test_model_config_project_presets.py
+++ b/tests/test_model_config_project_presets.py
@@ -1092,6 +1092,44 @@ def test_preset_application_inherits_optional_slots(monkeypatch, tmp_path):
assert config["embedding_model"] == base_config["embedding_model"]
+def test_preset_vision_slot_is_optional_and_never_inherited(monkeypatch, tmp_path):
+ _prepare_a0_tree(monkeypatch, tmp_path)
+
+ from plugins._model_config.helpers import model_config
+
+ base_config = {
+ "chat_model": {"provider": "openrouter", "name": "main", "vision": False},
+ "vision_model": {
+ "provider": "openrouter",
+ "name": "default-vision",
+ "max_embeds": 2,
+ },
+ }
+
+ without_sidecar = model_config.build_config_from_preset(
+ {"name": "Text only", "chat": {"provider": "openrouter", "name": "text"}},
+ base_config,
+ )
+ with_sidecar = model_config.build_config_from_preset(
+ {
+ "name": "Visual",
+ "chat": {"provider": "openrouter", "name": "text"},
+ "vision": {
+ "provider": "anthropic",
+ "name": "visual",
+ "max_embeds": 5,
+ },
+ },
+ base_config,
+ )
+
+ assert without_sidecar["vision_model"] == {}
+ assert with_sidecar["vision_model"]["provider"] == "anthropic"
+ assert with_sidecar["vision_model"]["name"] == "visual"
+ assert with_sidecar["vision_model"]["max_embeds"] == 5
+ assert "default-vision" not in str(with_sidecar["vision_model"])
+
+
def test_legacy_utility_preset_defaults_preserve_tuning_but_clear_kwargs(
monkeypatch,
tmp_path,
diff --git a/tests/test_model_config_ui.py b/tests/test_model_config_ui.py
index 1feb610619..5c6d54aa8b 100644
--- a/tests/test_model_config_ui.py
+++ b/tests/test_model_config_ui.py
@@ -85,6 +85,49 @@ def test_preset_editor_uses_standard_modal_footer_buttons() -> None:
assert "preset-editor-footer" not in preset_modal
+def test_preset_editor_nests_one_conditional_vision_selector_in_main() -> None:
+ preset_modal = read("plugins", "_model_config", "webui", "main.html")
+ model_field = read("plugins", "_model_config", "webui", "model-field.html")
+ preset_overview = read("plugins", "_model_config", "webui", "preset-overview.html")
+ preset_store = read("plugins", "_model_config", "webui", "model-config-store.js")
+
+ main_start = preset_modal.index('Main Model
')
+ utility_start = preset_modal.index('Utility Model
')
+ selector_start = preset_modal.index('class="vision-sidecar-selector"')
+ supports_start = model_field.index('Supports Vision
')
+ override_start = model_field.index('Use separate Vision Model
')
+ context_start = model_field.index('Context window size
')
+ advanced_start = model_field.index('')
+
+ assert 'Vision Model
' not in preset_modal
+ assert preset_modal.count('class="vision-sidecar-selector"') == 1
+ assert main_start < selector_start < utility_start
+ assert '!selectedPreset.chat.vision || selectedPreset.vision.override_main' in preset_modal
+ assert "get visionModel() { return selectedPreset.vision; }" in preset_modal
+ assert "modelType: 'vision'" in preset_modal
+ assert preset_modal.count("apiKeyMode: 'store'") == 4
+ assert "margin: 0.75rem 0 0;" in preset_modal
+ assert "padding: 0.25rem 0 0;" in preset_modal
+ assert "border-left: 2px solid var(--color-border);" not in preset_modal
+ assert "Use separate Vision Model" in model_field
+ assert supports_start < override_start < context_start < advanced_start
+ assert "When disabled, vision_load uses this model's native vision." in model_field
+ assert "When enabled, vision_load uses the preset's Vision Model" not in model_field
+ assert 'x-model="visionModel.override_main"' in model_field
+ assert '' in model_field
+ assert "model-preset-row-nested" in preset_overview
+ assert "model.title === 'Vision'" in preset_overview
+ assert preset_overview.count("model.title !== 'Vision'") == 2
+ assert 'Vision override' in preset_overview
+ assert "'model-preset-identity-vision': model.title === 'Vision'" in preset_overview
+ assert "grid-column: 3;" in preset_overview
+ assert "padding-top: var(--spacing-xs);" in preset_overview
+ assert "margin-left: 2rem;" not in preset_overview
+ assert "border-left: 1px solid var(--color-border);" not in preset_overview
+ assert "if (slotKey === 'vision') config[sectionKey] = {};" in preset_store
+ assert "['chat', 'vision', 'utility']" in preset_store
+
+
def test_plugin_settings_reset_is_explicit_and_does_not_capture_toast_early() -> None:
settings_modal = read("webui", "components", "plugins", "plugin-settings.html")
settings_store = read("webui", "components", "plugins", "plugin-settings-store.js")
diff --git a/tests/test_parallel_tool.py b/tests/test_parallel_tool.py
index dc1a911993..1e612e43fb 100644
--- a/tests/test_parallel_tool.py
+++ b/tests/test_parallel_tool.py
@@ -347,6 +347,50 @@ async def test_parallel_remove_context_deletes_persisted_worker_chat(monkeypatch
assert removed == ["missing-worker"]
+@pytest.mark.asyncio
+async def test_direct_parallel_worker_inherits_chat_model_override(monkeypatch) -> None:
+ from agent import AgentConfig, AgentContext
+
+ parent_id = "ctx-parallel-model-override"
+ AgentContext.remove(parent_id)
+ parent = AgentContext(
+ AgentConfig(mcp_servers="", profile="agent0"),
+ id=parent_id,
+ set_current=False,
+ )
+ override = {"preset_name": "Text only"}
+ parent.set_data("chat_model_override", override)
+ current_user_message = object()
+ parent.agent0.last_user_message = current_user_message
+ observed = {}
+
+ async def fake_execute_tool_call(agent, *_args, **_kwargs):
+ observed["override"] = agent.context.get_data("chat_model_override")
+ observed["last_user_message"] = agent.last_user_message
+ return "done"
+
+ async def remove_context(context_id):
+ AgentContext.remove(context_id)
+
+ monkeypatch.setattr(parallel_tools, "execute_tool_call", fake_execute_tool_call)
+ monkeypatch.setattr(parallel_tools, "_remove_context", remove_context)
+ job = parallel_tools.ParallelJob(
+ id="vision-load-override",
+ parent_context_id=parent_id,
+ index=0,
+ tool_name="vision_load",
+ tool_args={"paths": ["/tmp/example.png"]},
+ kind="tool",
+ )
+
+ try:
+ assert await parallel_tools._run_direct_tool_job(parent_id, job) == "done"
+ assert observed["override"] == override
+ assert observed["last_user_message"] is current_user_message
+ finally:
+ AgentContext.remove(parent_id)
+
+
@pytest.mark.asyncio
async def test_parallel_recursion_guard_allows_subordinate_children_but_blocks_tool_workers() -> None:
from extensions.python.tool_execute_before._20_block_parallel_recursion import (
diff --git a/tests/test_tool_policy.py b/tests/test_tool_policy.py
index 2db20350c9..f64526e5b2 100644
--- a/tests/test_tool_policy.py
+++ b/tests/test_tool_policy.py
@@ -528,6 +528,10 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
"plugins._model_config.helpers.model_config.get_chat_model_config",
lambda agent: {"vision": True},
)
+ monkeypatch.setattr(
+ "plugins._model_config.helpers.model_config.get_vision_model_config",
+ lambda agent: {},
+ )
monkeypatch.setattr(
tool_policy,
"get_policy",
@@ -546,6 +550,55 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
assert tool_policy.resolve_tool(agent, "vision_load").source == "runtime-config"
+@pytest.mark.asyncio
+async def test_active_vision_model_uses_canonical_vision_prompt(
+ monkeypatch, tmp_path: Path
+) -> None:
+ _write_prompt(tmp_path, "agent.system.tools.md", "TOOLS\n{{tools}}")
+ _write_prompt(
+ tmp_path,
+ "agent.system.tools_vision.md",
+ "### vision_load\ncanonical vision\nargs: `paths`, `query`",
+ )
+ monkeypatch.setattr(tool_policy.subagents, "get_paths", _prompt_paths(tmp_path))
+ monkeypatch.setattr(
+ "plugins._model_config.helpers.model_config.get_chat_model_config",
+ lambda agent: {"vision": False},
+ )
+ monkeypatch.setattr(
+ "plugins._model_config.helpers.model_config.get_vision_model_config",
+ lambda agent: {"provider": "test", "name": "vision"},
+ )
+ monkeypatch.setattr(responses_tools, "_mcp_tools", lambda agent: [])
+ agent = _Agent(tmp_path)
+
+ prompt = await _11_tools_prompt.build_prompt(agent)
+ schemas, _name_map = responses_tools.build_responses_function_tools(agent)
+
+ assert prompt.count("canonical vision") == 1
+ assert schemas[0]["name"] == "vision_load"
+ assert schemas[0]["description"] == "canonical vision"
+
+
+def test_vision_prompt_stays_route_agnostic_and_batches_paths() -> None:
+ source = (
+ Path(__file__).resolve().parents[1]
+ / "prompts"
+ / "agent.system.tools_vision.md"
+ ).read_text(encoding="utf-8")
+ schema = responses_tools._schema_from_prompt(source)
+
+ assert schema == {
+ "type": "object",
+ "properties": {},
+ "additionalProperties": True,
+ }
+ assert "load all relevant images in one call" in source
+ assert "Input schema for tool_args" not in source
+ assert "Vision Model" not in source
+ assert "query" not in source
+
+
def test_mcp_prompt_and_native_schema_omit_blocked_tool(
monkeypatch, tmp_path: Path
) -> None:
diff --git a/tests/test_vision_load_image_refs.py b/tests/test_vision_load_image_refs.py
index 2842a5e1e2..43540b11b0 100644
--- a/tests/test_vision_load_image_refs.py
+++ b/tests/test_vision_load_image_refs.py
@@ -1,3 +1,4 @@
+import asyncio
import types
from types import SimpleNamespace
import sys
@@ -13,8 +14,13 @@
class _TestResponse(SimpleNamespace):
- def __init__(self, message="", break_loop=False, **kwargs):
- super().__init__(message=message, break_loop=break_loop, **kwargs)
+ def __init__(self, message="", break_loop=False, additional=None, **kwargs):
+ super().__init__(
+ message=message,
+ break_loop=break_loop,
+ additional=additional,
+ **kwargs,
+ )
class _TestTool:
@@ -35,6 +41,15 @@ def __init__(
self.message = message
self.loop_data = loop_data
+ async def after_execution(self, response, **kwargs):
+ self.agent.hist_add_tool_result(
+ self.name,
+ response.message.strip(),
+ id=self.log.id,
+ **(response.additional or {}),
+ )
+ self.log.update(content=response.message.strip())
+
def _install_tool_stub(monkeypatch):
tool_stub = types.ModuleType("helpers.tool")
@@ -74,11 +89,8 @@ def fake_normalize_a0_path(path):
monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
- monkeypatch.setattr(
- vision_load_module.plugins,
- "get_plugin_config",
- lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},
- )
+ monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10})
+ monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
async def direct_call(func, *args, **kwargs):
return func(*args, **kwargs)
@@ -111,7 +123,10 @@ async def direct_call(func, *args, **kwargs):
)
tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
- response = await tool.execute(paths=[str(image_path)])
+ invalid = await tool.execute(paths=None)
+ assert invalid.message == "vision_load error: `paths` must be a string or an array."
+
+ response = await tool.execute(paths=str(image_path))
image_path.unlink()
await tool.after_execution(response)
@@ -120,4 +135,266 @@ async def direct_call(func, *args, **kwargs):
assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-")
stored_path = tmp_path / stored_ref.removeprefix("/a0/")
assert stored_path.read_bytes() == b"png-data"
- assert updates[-1]["result"] == "1 images loaded, 0 skipped"
+ assert updates[-1]["content"] == response.message
+
+
+def test_active_vision_model_route_prefers_main_native_vision(monkeypatch):
+ from plugins._model_config.helpers import model_config
+
+ cases = [
+ ({"vision": False}, {}, False),
+ ({"vision": False}, {"provider": "p"}, False),
+ ({"vision": False}, {"name": "v"}, False),
+ ({"vision": True}, {"provider": "p", "name": "v"}, False),
+ ({"vision": False}, {"provider": "p", "name": "v"}, True),
+ (
+ {"vision": True},
+ {"provider": "p", "name": "v", "override_main": True},
+ True,
+ ),
+ ]
+ for chat, vision, expected in cases:
+ monkeypatch.setattr(
+ model_config,
+ "get_effective_config",
+ lambda _agent=None, chat=chat, vision=vision: {
+ "chat_model": chat,
+ "vision_model": vision,
+ },
+ )
+ assert bool(model_config.get_vision_model_config()) is expected
+
+
+@pytest.mark.anyio
+async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_only(
+ monkeypatch,
+ tmp_path,
+):
+ _install_tool_stub(monkeypatch)
+ import tools.vision_load as vision_load_module
+
+ async def direct_call(func, *args, **kwargs):
+ return func(*args, **kwargs)
+
+ calls = []
+
+ class FakeVisionModel:
+ async def unified_call(self, **kwargs):
+ calls.append(kwargs)
+ return "The second screenshot fixes the red login error.", ""
+
+ monkeypatch.setattr(vision_load_module.runtime, "call_development_function", direct_call)
+ monkeypatch.setattr(vision_load_module, "build_vision_model", lambda _agent: FakeVisionModel())
+ monkeypatch.setattr(
+ vision_load_module,
+ "get_chat_model_config",
+ lambda _agent: {"vision": True, "max_embeds": 1},
+ )
+ monkeypatch.setattr(
+ vision_load_module,
+ "get_vision_model_config",
+ lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 5},
+ )
+
+ image_paths = [tmp_path / "before.png", tmp_path / "after.png"]
+ for path in image_paths:
+ path.write_bytes(b"png-data")
+
+ tool_results = []
+ raw_messages = []
+ agent = SimpleNamespace(
+ context=SimpleNamespace(id=""),
+ agent_name="Agent 0",
+ last_user_message=SimpleNamespace(
+ output_text=lambda: "Compare the login errors."
+ ),
+ read_prompt=lambda _name, request: f"Analyze: {request}",
+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
+ hist_add_message=lambda *args, **kwargs: raw_messages.append((args, kwargs)),
+ )
+ tool = vision_load_module.VisionLoad(
+ agent=agent,
+ name="vision_load",
+ method=None,
+ args={"paths": [str(path) for path in image_paths]},
+ message="",
+ loop_data=None,
+ )
+ tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None)
+
+ response = await tool.execute(paths=[str(path) for path in image_paths])
+ response.additional = {"_responses_output_item": {"output": response.message}}
+ await tool.after_execution(response)
+
+ assert len(calls) == 1
+ content = calls[0]["messages"][0].content
+ assert content[0] == {
+ "type": "text",
+ "text": "Analyze: Compare the login errors.",
+ }
+ assert [item["type"] for item in content].count("image_url") == 2
+ assert "max_tokens" not in calls[0]
+ assert "explicit_caching" not in calls[0]
+ assert "fixes the red login error" in response.message
+ assert response.message != "dummy"
+ assert raw_messages == []
+ assert tool.loaded_paths == [str(path) for path in image_paths]
+ assert tool_results[0][1]["_responses_output_item"]["output"] == response.message
+
+
+@pytest.mark.anyio
+async def test_vision_model_empty_response_is_reported_as_error(monkeypatch):
+ _install_tool_stub(monkeypatch)
+ import tools.vision_load as vision_load_module
+
+ class FakeVisionModel:
+ async def unified_call(self, **kwargs):
+ return "", ""
+
+ monkeypatch.setattr(
+ vision_load_module,
+ "build_vision_model",
+ lambda _agent: FakeVisionModel(),
+ )
+ monkeypatch.setattr(
+ vision_load_module,
+ "get_vision_model_config",
+ lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 10},
+ )
+
+ agent = SimpleNamespace(
+ context=SimpleNamespace(id="", get_data=lambda _key: ""),
+ last_user_message=SimpleNamespace(output_text=lambda: "Inspect the image."),
+ read_prompt=lambda _name, request: request,
+ )
+ tool = vision_load_module.VisionLoad(
+ agent=agent,
+ name="vision_load",
+ method=None,
+ args={"paths": ["data:image/png;base64,AA=="]},
+ message="",
+ loop_data=None,
+ )
+
+ response = await tool.execute(paths=["data:image/png;base64,AA=="])
+
+ assert response.message == (
+ "Image analysis error: Vision Model returned an empty response."
+ )
+
+
+@pytest.mark.anyio
+async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_path):
+ _install_tool_stub(monkeypatch)
+ import tools.vision_load as vision_load_module
+
+ def fake_get_abs_path(*parts):
+ return str(tmp_path.joinpath(*parts))
+
+ def fake_normalize_a0_path(path):
+ return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
+
+ monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
+ monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
+ parent_id = "parent-vision"
+ monkeypatch.setattr(
+ vision_load_module,
+ "get_chat_model_config",
+ lambda _agent: {"vision": True, "max_embeds": 10},
+ )
+ monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
+
+ ref = vision_load_module.ephemeral_images.put_image_bytes(
+ context_id=parent_id,
+ mime="image/png",
+ payload=b"png-data",
+ name="shot.png",
+ )
+ context = SimpleNamespace(
+ id="parallel-worker",
+ get_data=lambda key: parent_id
+ if key == vision_load_module.PARALLEL_WORKER_PARENT_CONTEXT_KEY
+ else None,
+ )
+ agent = SimpleNamespace(context=context, agent_name="Agent 0")
+ tool = vision_load_module.VisionLoad(
+ agent=agent,
+ name="vision_load",
+ method=None,
+ args={"paths": [ref]},
+ message="",
+ loop_data=None,
+ )
+
+ await tool.execute(paths=[ref])
+
+ assert tool._context_id() == parent_id
+ assert tool.loaded_paths == ["shot.png"]
+ assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is None
+ stored_ref = tool.images_dict["shot.png"]
+ assert stored_ref.startswith("/a0/usr/chats/parent-vision/images/vision-load/shot-")
+
+
+@pytest.mark.anyio
+async def test_independent_vision_model_calls_can_run_concurrently(monkeypatch, tmp_path):
+ _install_tool_stub(monkeypatch)
+ import tools.vision_load as vision_load_module
+
+ active = 0
+ max_active = 0
+ call_count = 0
+
+ class FakeVisionModel:
+ async def unified_call(self, **kwargs):
+ nonlocal active, max_active, call_count
+ active += 1
+ call_count += 1
+ max_active = max(max_active, active)
+ await asyncio.sleep(0.02)
+ active -= 1
+ return "done", ""
+
+ async def direct_call(func, *args, **kwargs):
+ return func(*args, **kwargs)
+
+ monkeypatch.setattr(vision_load_module.runtime, "call_development_function", direct_call)
+ monkeypatch.setattr(vision_load_module, "build_vision_model", lambda _agent: FakeVisionModel())
+ monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": False})
+ monkeypatch.setattr(
+ vision_load_module,
+ "get_vision_model_config",
+ lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 10},
+ )
+
+ image_paths = [tmp_path / "one.png", tmp_path / "two.png"]
+ for path in image_paths:
+ path.write_bytes(b"png-data")
+
+ def make_tool(index):
+ agent = SimpleNamespace(
+ context=SimpleNamespace(id=""),
+ agent_name=f"Agent {index}",
+ last_user_message=SimpleNamespace(
+ output_text=lambda: f"inspection {index}"
+ ),
+ read_prompt=lambda _name, request: request,
+ )
+ return vision_load_module.VisionLoad(
+ agent=agent,
+ name="vision_load",
+ method=None,
+ args={"paths": [str(path) for path in image_paths]},
+ message="",
+ loop_data=None,
+ )
+
+ responses = await asyncio.gather(
+ *(
+ make_tool(index).execute(paths=[str(path) for path in image_paths])
+ for index in range(4)
+ )
+ )
+
+ assert call_count == 4
+ assert max_active == 4
+ assert all("done" in response.message for response in responses)
diff --git a/tools/vision_load.py b/tools/vision_load.py
index 7358afbdec..791a4dfcee 100644
--- a/tools/vision_load.py
+++ b/tools/vision_load.py
@@ -1,19 +1,34 @@
-from helpers.print_style import PrintStyle
-from helpers.tool import Tool, Response
-from helpers import runtime, files, plugins, ephemeral_images, images, chat_media
from mimetypes import guess_type
-from helpers import history
+
+from langchain_core.messages import HumanMessage
+
+from helpers import chat_media, ephemeral_images, files, history, images, runtime
+from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY
+from helpers.tool import Response, Tool
+from plugins._model_config.helpers.model_config import (
+ build_vision_model,
+ get_chat_model_config,
+ get_vision_model_config,
+)
# image token estimation for context window
TOKENS_ESTIMATE = 1500
class VisionLoad(Tool):
- async def execute(self, paths: list[str] = [], **kwargs) -> Response:
+ async def execute(self, paths: list[str] | str = [], **kwargs) -> Response:
self.images_dict = {}
self.loaded_paths: list[str] = []
self.skipped_paths: list[str] = []
+ self.vision_config = get_vision_model_config(self.agent)
+ if isinstance(paths, str):
+ paths = [paths]
+ if not isinstance(paths, list):
+ return Response(
+ message="vision_load error: `paths` must be a string or an array.",
+ break_loop=False,
+ )
max_embeds = self._get_max_embeds()
requested = [
@@ -62,16 +77,48 @@ async def execute(self, paths: list[str] = [], **kwargs) -> Response:
except (FileNotFoundError, OSError, ValueError):
continue
- return Response(message="dummy", break_loop=False)
+ message = self._summary() if self.images_dict or self.skipped_paths else "No images processed"
+ if self.vision_config and self.images_dict:
+ try:
+ capsule = await self._call_vision_model(list(self.images_dict.values()))
+ message = (
+ f"Analyzed {len(self.images_dict)} image(s)"
+ f"; {len(self.skipped_paths)} skipped.\n\n{capsule.strip()}"
+ )
+ except Exception as exc:
+ message = f"Image analysis error: {str(exc)[:1000]}"
+ return Response(message=message, break_loop=False)
def _get_max_embeds(self) -> int:
- cfg = plugins.get_plugin_config("_model_config", agent=self.agent) or {}
- chat_cfg = cfg.get("chat_model", {})
- max_embeds = chat_cfg.get("max_embeds", 10)
- return int(max_embeds or 0)
+ cfg = self.vision_config or get_chat_model_config(self.agent)
+ return int(cfg.get("max_embeds", 10) or 0)
def _context_id(self) -> str:
- return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
+ context = getattr(self.agent, "context", None)
+ get_data = getattr(context, "get_data", None)
+ parent_id = get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) if get_data else ""
+ return str(parent_id or getattr(context, "id", "") or "").strip()
+
+ async def _call_vision_model(self, image_paths: list[str]) -> str:
+ user_message = getattr(self.agent, "last_user_message", None)
+ output_text = getattr(user_message, "output_text", None)
+ request = str(output_text() if callable(output_text) else "").strip()
+ content = [
+ {
+ "type": "text",
+ "text": self.agent.read_prompt("fw.vision_load.md", request=request),
+ }
+ ]
+ content.extend(
+ {"type": "image_url", "image_url": {"url": path}}
+ for path in image_paths
+ )
+ response, _ = await build_vision_model(self.agent).unified_call(
+ messages=[HumanMessage(content=content)],
+ )
+ if not str(response or "").strip():
+ raise RuntimeError("Vision Model returned an empty response.")
+ return str(response)
def _store_ephemeral_image(self, image: ephemeral_images.EphemeralImage) -> str:
context_id = self._context_id()
@@ -115,6 +162,14 @@ def _store_local_image(self, path: str, *, preferred_name: str = "") -> str:
preferred_name=preferred_name,
)
+ def _summary(self) -> str:
+ loaded = "\n".join(self.loaded_paths) if self.loaded_paths else "none"
+ skipped = "\n".join(self.skipped_paths) if self.skipped_paths else "none"
+ return (
+ f"Loaded images ({len(self.loaded_paths)}):\n{loaded}\n\n"
+ f"Skipped images ({len(self.skipped_paths)}, max {self._get_max_embeds()}):\n{skipped}"
+ )
+
@staticmethod
def _is_data_image_url(value: str) -> bool:
normalized = str(value or "").strip().lower()
@@ -130,52 +185,17 @@ def _display_input_path(cls, value: str, index: int) -> str:
return value
async def after_execution(self, response: Response, **kwargs):
-
- # build image data messages for LLMs, or error message
- content = []
- loaded_count = len(self.loaded_paths)
- skipped_count = len(self.skipped_paths)
- loaded_summary = "\n".join(self.loaded_paths) if self.loaded_paths else "none"
- skipped_summary = "\n".join(self.skipped_paths) if self.skipped_paths else "none"
- summary = (
- f"Loaded images: {loaded_count}\n"
- f"Loaded images:\n{loaded_summary}\n\n"
- f"Skipped images: {skipped_count}\n"
- f"Skipped images (max {self._get_max_embeds()} loaded at a time according to model configuration):\n{skipped_summary}"
- )
- if self.images_dict:
- self.agent.hist_add_tool_result(self.name, summary, id=self.log.id if self.log else "")
- for path, image_path in self.images_dict.items():
- if image_path:
- content.append(
- {
- "type": "image_url",
- "image_url": {"url": image_path},
- }
- )
- else:
- content.append(
- {
- "type": "text",
- "text": "Error processing image " + path,
- }
- )
- # append as raw message content for LLMs with vision tokens estimate
- msg = history.RawMessage(raw_content=content, preview="")
+ await super().after_execution(response, **kwargs)
+ if self.images_dict and not self.vision_config:
+ content = [
+ {"type": "image_url", "image_url": {"url": image_path}}
+ for image_path in self.images_dict.values()
+ ]
self.agent.hist_add_message(
- False, content=msg, tokens=TOKENS_ESTIMATE * len(content)
+ False,
+ content=history.RawMessage(
+ raw_content=content,
+ preview="",
+ ),
+ tokens=TOKENS_ESTIMATE * len(content),
)
- else:
- self.agent.hist_add_tool_result(self.name, summary if self.skipped_paths else "No images processed", id=self.log.id if self.log else "")
-
- # print and log short version
- message = (
- "No images processed"
- if not self.images_dict and not self.skipped_paths
- else f"{loaded_count} images loaded, {skipped_count} skipped"
- )
- PrintStyle(
- font_color="#1B4F72", background_color="white", padding=True, bold=True
- ).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")
- PrintStyle(font_color="#85C1E9").print(message)
- self.log.update(result=message)
diff --git a/tools/vision_load.py.dox.md b/tools/vision_load.py.dox.md
index da95312429..ef09102e07 100644
--- a/tools/vision_load.py.dox.md
+++ b/tools/vision_load.py.dox.md
@@ -3,7 +3,7 @@
## Purpose
- Own the `vision_load.py` agent tool.
-- This module loads images into model-visible content for vision-capable models.
+- This module routes images either into Main model-visible content or through the preset's optional Vision Model.
- Keep this file-level DOX profile synchronized with `vision_load.py` because this directory is intentionally flat.
## Ownership
@@ -12,22 +12,31 @@
- `vision_load.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.
- Classes:
- `VisionLoad` (`Tool`)
- - `async execute(self, paths: list[str]=..., **kwargs) -> Response`
+ - `async execute(self, paths, **kwargs) -> Response`
- `async after_execution(self, response: Response, **kwargs)`
- Notable constants/configuration names: `TOKENS_ESTIMATE`.
## Runtime Contracts
- Tool modules must define `helpers.tool.Tool` subclasses and return `helpers.tool.Response` from `execute(...)`.
+- One call may contain multiple paths; a bare string is treated as one path. The Vision Model route sends every selected path in one request and returns one textual capsule.
+- Model configuration exposes a Vision Model only when the effective preset selects that route; otherwise this tool follows Main's native vision path.
+- The public tool contract is route-agnostic. A Vision Model receives the current user request through `fw.vision_load.md`; direct parallel workers inherit that request from their parent.
+- Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.
+- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
+- Direct parallel workers inherit the parent's model override generically. This tool uses their recorded parent context only to resolve ephemeral refs and durable chat media.
+- `max_embeds` comes from the model that actually receives the images.
+- Vision Model calls use the selected model's Advanced `kwargs`; this tool does not impose a separate timeout or output-token limit.
+- An empty Vision Model response is reported as an image-analysis error instead of a successful empty capsule.
- Update this file whenever tool arguments, output shape, `break_loop` behavior, intervention handling, prompt instructions, or side effects change.
- `VisionLoad` is a `Tool`.
- `VisionLoad` defines `execute(...)`.
- Observed side-effect areas: filesystem writes, model calls, plugin state, settings/state persistence, secret handling.
-- Imported dependency areas include: `helpers`, `helpers.print_style`, `helpers.tool`, `mimetypes`.
+- Imported dependency areas include: `helpers`, `helpers.tool`, `langchain_core.messages`, `mimetypes`, and `_model_config`.
## Key Concepts
-- Important called helpers/classes observed in the source: `self._get_max_embeds`, `Response`, `str.strip`, `self._context_id`, `chat_media.infer_source`, `chat_media.category_for_source`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `str.strip.lower`, `ephemeral_images.is_ref`, `cls._is_data_image_url`, `self._is_data_image_url`, `plugins.get_plugin_config`, `images.to_data_url`, `normalized.startswith`, `ephemeral_images.display_ref`, `join`, `self.agent.hist_add_tool_result`, `history.RawMessage`.
+- Important called helpers/classes observed in the source: `build_vision_model`, `get_vision_model_config`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `history.RawMessage`, `super().after_execution`, `model.unified_call`.
- Keep request/response, tool, or helper semantics documented here at the same time as source changes.
## Work Guidance