From 65b110e4aabbf3674de1eec63c32dfa31dcb6efe Mon Sep 17 00:00:00 2001 From: GreifMax <142740995+GreifMax@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:43:31 +0200 Subject: [PATCH] Added _vision_sidecar --- plugins/_vision_sidecar/LICENSE | 21 + plugins/_vision_sidecar/README.md | 154 +++++++ plugins/_vision_sidecar/default_config.yaml | 4 + plugins/_vision_sidecar/execute.py | 28 ++ plugins/_vision_sidecar/execute_record.json | 1 + .../_10_vision_sidecar_guidance.py | 21 + .../_vision_sidecar/helpers/vision_model.py | 128 ++++++ plugins/_vision_sidecar/hooks.py | 72 +++ plugins/_vision_sidecar/plugin.yaml | 8 + .../prompts/vision_sidecar.delegated.md | 50 ++ .../scripts/enable_vision_slot.py | 431 ++++++++++++++++++ .../scripts/enable_vision_slot.sh | 25 + plugins/_vision_sidecar/thumbnail.jpg | Bin 0 -> 13910 bytes plugins/_vision_sidecar/tools/vision_load.py | 422 +++++++++++++++++ plugins/_vision_sidecar/webui/config.html | 55 +++ plugins/_vision_sidecar/webui/thumbnail.jpg | Bin 0 -> 13910 bytes 16 files changed, 1420 insertions(+) create mode 100644 plugins/_vision_sidecar/LICENSE create mode 100644 plugins/_vision_sidecar/README.md create mode 100644 plugins/_vision_sidecar/default_config.yaml create mode 100644 plugins/_vision_sidecar/execute.py create mode 100644 plugins/_vision_sidecar/execute_record.json create mode 100644 plugins/_vision_sidecar/extensions/python/system_prompt/_10_vision_sidecar_guidance.py create mode 100644 plugins/_vision_sidecar/helpers/vision_model.py create mode 100644 plugins/_vision_sidecar/hooks.py create mode 100644 plugins/_vision_sidecar/plugin.yaml create mode 100644 plugins/_vision_sidecar/prompts/vision_sidecar.delegated.md create mode 100644 plugins/_vision_sidecar/scripts/enable_vision_slot.py create mode 100644 plugins/_vision_sidecar/scripts/enable_vision_slot.sh create mode 100644 plugins/_vision_sidecar/thumbnail.jpg create mode 100644 plugins/_vision_sidecar/tools/vision_load.py create mode 100644 plugins/_vision_sidecar/webui/config.html create mode 100644 plugins/_vision_sidecar/webui/thumbnail.jpg diff --git a/plugins/_vision_sidecar/LICENSE b/plugins/_vision_sidecar/LICENSE new file mode 100644 index 0000000000..2b7954c544 --- /dev/null +++ b/plugins/_vision_sidecar/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Vision Sidecar contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/_vision_sidecar/README.md b/plugins/_vision_sidecar/README.md new file mode 100644 index 0000000000..a3a1bd6dc6 --- /dev/null +++ b/plugins/_vision_sidecar/README.md @@ -0,0 +1,154 @@ +# Vision Sidecar + +![Vision Sidecar](webui/thumbnail.jpg) + +Tolerant `vision_load` + optional dedicated Vision Model. + +## Why + +Two recurring pain points in Agent Zero: + +1. **Bare-string bug.** Core `vision_load` requires `paths` as a list. A bare string `"/a.png"` is iterated char-by-char, loads 0 images, and wastes a turn with no error. +2. **No vision, no images.** Frontier reasoners (GLM 5.2/5.3, DeepSeek V4 Flash/Pro) are cheap and strong but have no vision. On stock A0 that means no `vision_load` at all — even though a cheap `gpt-4o-mini` or `qwen2-vl` could read the image for pennies. + +Vision Sidecar fixes both in one plugin. + +## What it does + +### 1. Tolerant `vision_load` + +`vision_load` now accepts `paths` as `string` or `list[str]`. + +- `{"paths": "/a.png"}` is treated as `["/a.png"]` +- Handles harness quirks: JSON-encoded array strings (`"[\"/a.png\"]"`), quoted single paths (`"\"/a.png\""`) +- Wrong types return a clear tool error — never a `Message misformat` + +### 2. Delegated Vision Model + +Configure an optional **Vision Model** in **Settings → Model Presets → Vision Model**. + +> Optional dedicated model for vision_load — used when Main has no vision. Leave empty to use Main's vision. + +When set: + +- `vision_load(paths, query?, raw?)` materializes images, calls the Vision Model with `query + images`, and returns a **text capsule** instead of injecting `~1500 tok/image` into the main history. +- Your Main (GLM, DeepSeek) never sees raw pixels — only ~300 tokens of focused text. Saves thousands of tokens per future turn. +- `query` is a focused instruction: `"read the top-right error toast"`, `"locate the login button and give coordinates"`. Empty → generic precise description. +- `raw=true` bypasses delegation and injects images directly into Main. Use for side-by-side comparison when Main must see pixels. +- Large images over ~900 KB are auto-compressed to 1280×960 JPEG before the vision call to avoid `Request Entity Too Large` (4 MB PNG → ~250 KB). + +When empty: legacy path — images are injected as `RawMessage` for `chat_model.vision == true`, appearance identical to stock A0. + +Preset defaults for the Vision slot: **64000 context, 70% for history** (new presets only). Existing presets are untouched. + +## Requirements + +- Agent Zero. If your **Settings → Model Presets → Edit** already shows **Main / Vision / Utility / Embedding** (only if you're updating the plugin), nothing else to do. +- If it only shows **Main / Utility / Embedding** (on any A0 instance), run the one-time Vision-slot patch below — otherwise Vision Sidecar still works, but `vision_load` falls back to tolerant direct injection (no delegation). +- Any LiteLLM-compatible vision model for the Vision slot (tested with `openai/gpt-4o-mini`, `qwen2-vl`). + +## Installation + +### From ZIP + +1. Download `vision_sidecar.zip` from Releases +2. Agent Zero → **Settings → Plugins → Install → From ZIP** → select the ZIP +3. Add the Vision slot via the script (check below) +4. Restart the WebUI (`Ctrl+Shift+R`) + +### From Git + +```bash +git clone https://github.com/GreifMax/a0-vision-sidecar +cp -r a0-vision_sidecar /a0/usr/plugins/vision_sidecar +# restart Agent Zero +``` + +### Add Vision slot + +Since **v0.4.0** the Vision slot is applied **automatically on install** (via the plugin's `install()` hook) — no manual step. If Model Presets still shows only Main / Utility / Embedding (e.g. after an A0 core update overwrote `plugins/_model_config`), re-run it in one click: + +- **Settings → Plugins → Vision Sidecar → Execute** (preferred), or +- the manual script below (same logic, also usable for `--status` / `--restore`) + +The patcher is **self-contained pure Python** — no git or `patch(1)` required, idempotent, and creates `.vision_sidecar.bak` backups of every modified file. + +```bash +# any directory works; it auto-finds the Agent Zero root +bash /a0/usr/plugins/vision_sidecar/scripts/enable_vision_slot.sh +# or: python3 /a0/usr/plugins/vision_sidecar/scripts/enable_vision_slot.py +``` + +Docker (run **inside** the Agent Zero container, not on the host — `plugins/` only exists in the image): + +```bash +docker exec -it bash /a0/usr/plugins/vision_sidecar/scripts/enable_vision_slot.sh +``` + +Options: + +| Command | Effect | +| --- | --- | +| (no args) | Apply patch (skips files already patched) | +| `--status` | Show per-file state without changing anything | +| `--restore` | Restore all original files from `.bak` backups | + +If auto-detection fails, point it at your install: `A0_ROOT=/path/to/agent-zero bash enable_vision_slot.sh`. + +Then restart Agent Zero and hard-refresh the browser (Ctrl+Shift+R). Model Presets will show **Main / Vision / Utility / Embedding**. + +> Note: A0 updates can overwrite `plugins/_model_config`. After updating, rerun the script — it is idempotent and will re-apply cleanly. + +## Configuration + +1. **Settings → Model Presets → Edit** → fill **Vision Model** with your cheap vision helper (provider + name + key). Leave empty to use Main's vision. +2. **Main Model → Supports Vision on** → optional **Overrides Vision Model** switch appears right under it: when on, Main's native vision is always used for that preset and the Vision Model is ignored; when off (default), the dedicated Vision Model handles vision when configured. The chat model switcher hides the Vision row for presets where the override is on, and the Agent Config preset preview shows "Overwritten by Main" in place of the Vision model when the override is active. +3. **Settings → Plugins → Vision Sidecar** → tune the delegated system prompt and timeout if needed. + +New presets automatically get `Vision: 64000 / 0.7`. Current presets keep their values. The override flag is per-preset (never inherited from Default). + +## Usage (By A0) + +```json +{ + "tool_name": "vision_load", + "tool_args": { + "paths": ["/a0/usr/uploads/screenshot.png"], + "query": "read the error message in the top-right" + } +} +``` + +- With Vision Model set -> chat shows thumbnails + `N images sent, M images skipped - Description: "..."` (counts + vision-model capsule in one line). The tool step always includes a Query row (Paths / Tool Name / Query / Result), even when the call omitted `query`. +- The delegated `vision_load` prompt declares an explicit JSON tool schema (`paths`, `query`, `raw`), so models see `query` as a real parameter. When the Main model overrides vision (or there is no Vision Model), the stock prompt is used and `query` is absent from the schema. The schema rejects unknown properties (`additionalProperties: false`), and the tool normalizes prompt-style aliases (`Prompt`, `question`, `instruction`, ...) into `query`, so the vision model always receives the intended focus text even if a model ignores the schema. +- With `raw=true` → forces direct injection even when Vision Model is set. +- Without Vision Model -> fully stock: stock prompt (no `query`/`raw`), `Loaded images: N` with thumbnails for Main vision, and no vision tool at all when Main has no vision. +- With **Overrides Vision Model** on (Main vision-capable presets) -> Main's native vision is used even though a Vision Model is set; delegation (and the switcher's Vision row) is skipped for that preset. + +## Reliability notes + +- `vision_load` is safe inside the `parallel` tool: the job result is the real text capsule (or error), never a placeholder — the tool sets the authoritative response message in every outcome path. +- Image blocks are injected into main history only when the preset's main model declares vision support (delegated and legacy paths alike). With a text-only main, delegated calls return the text capsule only and `raw=true` auto-delegates. +- The `vision` flag is the user's declaration and governs: if it mislabels a text-only provider, the provider may reject image blocks (`400 content.type invalid`) — fix the flag in Model Presets; the text capsule still carries the answer meanwhile. +- Inside `parallel` workers the tool writes the `Result` row of its log item. The parent job aggregator skips the body text write when the Result row already carries the same text, so the step shows the text only once (inside the Result row). Errors and tools without a Result row still get body text. +- With no Vision Model configured and a text-only main, the tool reports loaded/skipped counts plus an explicit note that images were not injected. + +## File layout + +``` +plugin.yaml +default_config.yaml +LICENSE +README.md +thumbnail.jpg ← plugin list image (256×256, ≤20 KB) +helpers/vision_model.py ← preset-aware vision dispatch + compression +tools/vision_load.py ← tolerant paths + delegation +prompts/agent.system.tool.vision_load.md +extensions/python/system_prompt/_10_vision_sidecar_guidance.py +webui/config.html +webui/thumbnail.jpg/png +``` + +## License + +MIT — see [LICENSE](LICENSE). diff --git a/plugins/_vision_sidecar/default_config.yaml b/plugins/_vision_sidecar/default_config.yaml new file mode 100644 index 0000000000..7b55fb2b77 --- /dev/null +++ b/plugins/_vision_sidecar/default_config.yaml @@ -0,0 +1,4 @@ +behaviour: + delegated_system: "You are a precise vision analyst. Answer only what was asked about the image(s). Be concise, factual, mention positions/coordinates when asked to locate." + max_tokens: 2000 + timeout: 300 diff --git a/plugins/_vision_sidecar/execute.py b/plugins/_vision_sidecar/execute.py new file mode 100644 index 0000000000..1210fc0555 --- /dev/null +++ b/plugins/_vision_sidecar/execute.py @@ -0,0 +1,28 @@ +"""Vision Sidecar — manual maintenance script. + +Triggered from Settings → Plugins → Vision Sidecar (Execute). Useful after an +A0 update that overwrote plugins/_model_config: re-applies the Vision-slot +patch idempotently (already-patched files are skipped). The install hook does +this automatically on plugin install, so most users never need this. +""" +import sys +from pathlib import Path + + +def main() -> int: + try: + hooks = Path(__file__).resolve().parent / "hooks.py" + import importlib.util + spec = importlib.util.spec_from_file_location("vision_sidecar_hooks", hooks) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.install() + print("Vision Sidecar maintenance completed successfully.") + return 0 + except Exception as e: + print(f"ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/_vision_sidecar/execute_record.json b/plugins/_vision_sidecar/execute_record.json new file mode 100644 index 0000000000..bec3193643 --- /dev/null +++ b/plugins/_vision_sidecar/execute_record.json @@ -0,0 +1 @@ +{"executed_at": "2026-08-19T18:55:52.671678+02:00", "exit_code": 0} \ No newline at end of file diff --git a/plugins/_vision_sidecar/extensions/python/system_prompt/_10_vision_sidecar_guidance.py b/plugins/_vision_sidecar/extensions/python/system_prompt/_10_vision_sidecar_guidance.py new file mode 100644 index 0000000000..83df6d823b --- /dev/null +++ b/plugins/_vision_sidecar/extensions/python/system_prompt/_10_vision_sidecar_guidance.py @@ -0,0 +1,21 @@ +from helpers.extension import Extension + +_D = "Vision Sidecar active: a dedicated Vision Model IS configured (Model Presets -> Vision Model). vision_load DELEGATES:" +_D += chr(10) + "- send a focused query about the images (e.g. read the top-right error toast, locate the login button and give its position) - you will get a concise text capsule, not pixels." +_D += chr(10) + "- if Main cannot see images, ~1500 tok/image stays out of your context (capsule only); if Main is vision-capable, the images are also attached to history alongside the capsule. Use raw=true only when you need pixels without a capsule (e.g. side-by-side comparison)." +_GUIDANCE_DELEGATED = _D + + +class VisionSidecarGuidance(Extension): + async def execute(self, system_prompt: list[str] | None = None, **kwargs): + # With no dedicated Vision Model the plugin is fully stock: no guidance, + # stock prompt, stock result format. + if system_prompt is None: + return + try: + from usr.plugins.vision_sidecar.helpers.vision_model import has_vision_model + if not has_vision_model(self.agent): + return + except Exception: + return + system_prompt.append(_GUIDANCE_DELEGATED) diff --git a/plugins/_vision_sidecar/helpers/vision_model.py b/plugins/_vision_sidecar/helpers/vision_model.py new file mode 100644 index 0000000000..efc2f3e9c9 --- /dev/null +++ b/plugins/_vision_sidecar/helpers/vision_model.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from typing import Any + +import models + +DEFAULT_DELEGATED_SYSTEM = ( + "You are a precise vision analyst. Answer only what was asked about the image(s). " + "Be concise, factual, mention positions/coordinates when asked to locate." +) + +def get_behaviour(agent: Any = None) -> dict[str, Any]: + from helpers import plugins + cfg = plugins.get_plugin_config("vision_sidecar", agent=agent) or {} + b = cfg.get("behaviour") or {} + if not isinstance(b, dict): + b = {} + return { + "delegated_system": str(b.get("delegated_system") or DEFAULT_DELEGATED_SYSTEM), + "max_tokens": int(b.get("max_tokens") or 2000), + "timeout": float(b.get("timeout") or 300), + } + +def get_vision_model_config(agent: Any = None) -> dict[str, Any]: + """Read Vision Model from Model Presets (vision slot), not from sidecar config. + + Preset storage is in _model_config plugin: PRESET_SLOT_CONFIG_SECTIONS["vision"] = "vision_model". + Uses get_effective_config so per-chat overrides are respected. + """ + try: + from plugins._model_config.helpers.model_config import get_effective_config + cfg = get_effective_config(agent) or {} + vm = cfg.get("vision_model") or {} + if not isinstance(vm, dict): + return {} + provider = str(vm.get("provider") or "").strip() + name = str(vm.get("name") or "").strip() + if not provider and not name: + return {} + return vm + except Exception: + return {} + +def has_vision_model(agent: Any = None) -> bool: + vm = get_vision_model_config(agent) + if not (vm.get("provider") and vm.get("name")): + return False + try: + from plugins._model_config.helpers.model_config import get_chat_model_config + chat_cfg = get_chat_model_config(agent) or {} + if chat_cfg.get("vision", False) and chat_cfg.get("vision_override", False): + return False # Main's native vision overrides the Vision Model + except Exception: + pass + return True + +def build_vision_model(agent: Any = None): + vm = get_vision_model_config(agent) + if not vm: + return None + from plugins._model_config.helpers.model_config import build_model_config + mc = build_model_config(vm, models.ModelType.CHAT) + mc.vision = True + return models.get_chat_model(mc.provider, mc.name, model_config=mc, **mc.build_kwargs()) + +async def call_vision_model( + agent: Any, + images_a0_paths: list[str], + query: str, + delegated_system: str | None = None, + timeout: float = 300, +) -> str: + """Call the dedicated vision preset model with images + query, return text capsule.""" + import asyncio + from langchain_core.messages import HumanMessage, SystemMessage + + model = build_vision_model(agent) + if model is None: + raise RuntimeError("vision_model not configured — set it in Model Presets → Vision Model") + + behaviour = get_behaviour(agent) + system = (delegated_system or behaviour["delegated_system"]).strip() or DEFAULT_DELEGATED_SYSTEM + if query and query.strip(): + user_text = query.strip() + else: + user_text = "Describe the image(s) precisely. Be concise, mention key objects, text, and layout." + + content: list[dict[str, Any]] = [{"type": "text", "text": user_text}] + for pa in images_a0_paths: + url = pa + try: + import base64 + from pathlib import Path as _Path + from helpers import files as _files + from helpers.images import compress_image as _compress + raw = str(pa or "").strip() + cand = None + if raw.startswith("/a0/"): + cand = _Path(_files.fix_dev_path(raw) if hasattr(_files, "fix_dev_path") else raw) + if not cand.exists(): + cand = _Path(raw) + else: + cand = _Path(raw) + if cand and cand.exists() and cand.is_file(): + data = cand.read_bytes() + if len(data) > 900 * 1024: + try: + c = _compress(data, max_pixels=1280*960, quality=80) + b64 = base64.b64encode(c).decode() + url = f"data:image/jpeg;base64,{b64}" + except Exception: + url = pa + else: + url = pa + except Exception: + url = pa + content.append({"type": "image_url", "image_url": {"url": url}}) + + messages = [SystemMessage(content=system), HumanMessage(content=content)] + + async def _call(): + kwargs = {"max_tokens": int(behaviour["max_tokens"])} + resp, _reason = await model.unified_call(messages=messages, explicit_caching=False, **kwargs) + return resp + try: + return await asyncio.wait_for(_call(), timeout=float(timeout or behaviour["timeout"])) + except asyncio.TimeoutError: + raise TimeoutError(f"vision_model timed out after {timeout}s") diff --git a/plugins/_vision_sidecar/hooks.py b/plugins/_vision_sidecar/hooks.py new file mode 100644 index 0000000000..1b9230abe7 --- /dev/null +++ b/plugins/_vision_sidecar/hooks.py @@ -0,0 +1,72 @@ +"""Vision Sidecar lifecycle hooks. + +install() - auto-applies the Vision-slot patch when the plugin is installed + via the Plugins UI ("Install from ZIP/Git"). No manual script + needed anymore; scripts/enable_vision_slot.py remains as a + fallback/debug tool. +pre_update() - re-applies the patch idempotently (A0 updates may overwrite + plugins/_model_config). +uninstall() - restores the original _model_config files from .bak backups, + but only if the current files still contain our edits (never + clobbers a core update that already replaced them). +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +_HOOKS_CONTEXT: dict = {} + + +def _run_patcher(**kwargs) -> tuple[bool, str]: + """Import and run scripts/enable_vision_slot.py::run(). Returns (ok, log).""" + plugin_dir = Path(__file__).resolve().parent + patcher = plugin_dir / "scripts" / "enable_vision_slot.py" + spec = importlib.util.spec_from_file_location("vision_sidecar_patcher", patcher) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + lines: list[str] = [] + import contextlib, io + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = mod.run(**kwargs) # type: ignore[attr-defined] + lines = [l for l in buf.getvalue().splitlines() if l.strip()] + return (rc == 0), "\n".join(lines) + + +def install(hook_context: dict | None = None, **kwargs): + try: + ok, log = _run_patcher() + print("[vision_sidecar] install:") + print(log) + if not ok: + print( + "[vision_sidecar] Vision-slot patch could not be fully applied. " + "vision_load still works (tolerant, direct injection); run " + "scripts/enable_vision_slot.py manually to retry, or report at " + "https://github.com/GreifMax/a0-vision-sidecar/issues" + ) + except Exception as e: # never break the installer + print(f"[vision_sidecar] install hook skipped: {e}") + _HOOKS_CONTEXT["installed"] = True + return None + + +def pre_update(hook_context: dict | None = None, **kwargs): + try: + ok, log = _run_patcher() + print("[vision_sidecar] pre_update:") + print(log) + except Exception as e: + print(f"[vision_sidecar] pre_update hook skipped: {e}") + return None + + +def uninstall(hook_context: dict | None = None, **kwargs): + try: + ok, log = _run_patcher(restore=True) + print("[vision_sidecar] uninstall:") + print(log) + except Exception as e: + print(f"[vision_sidecar] uninstall hook skipped: {e}") + return None diff --git a/plugins/_vision_sidecar/plugin.yaml b/plugins/_vision_sidecar/plugin.yaml new file mode 100644 index 0000000000..4b8b14155b --- /dev/null +++ b/plugins/_vision_sidecar/plugin.yaml @@ -0,0 +1,8 @@ +name: vision_sidecar +title: Vision Sidecar +description: Tolerant vision_load (accepts single string) + optional dedicated vision model that answers a focused query about images without bloating the main context. +version: 0.7.9 +settings_sections: + - agent +per_project_config: true +per_agent_config: true diff --git a/plugins/_vision_sidecar/prompts/vision_sidecar.delegated.md b/plugins/_vision_sidecar/prompts/vision_sidecar.delegated.md new file mode 100644 index 0000000000..2d25c003b9 --- /dev/null +++ b/plugins/_vision_sidecar/prompts/vision_sidecar.delegated.md @@ -0,0 +1,50 @@ +## multimodal vision tools + +### vision_load +load images into the model for visual reasoning via the dedicated vision model +args: `paths` list of absolute image paths or ephemeral image refs, `query` optional string describing what to extract, `raw` optional boolean to force direct image injection +Input schema for tool_args: +```json +{ + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": {"type": "string"}, + "description": "Absolute image paths or ephemeral image refs. A single bare string is also accepted." + }, + "query": { + "type": "string", + "description": "Focused instruction for the vision model, e.g. 'read the top-right error toast' or 'locate the login button and give its position'. If omitted, a generic precise description is returned." + }, + "raw": { + "type": "boolean", + "description": "If true, bypass delegation and inject the images directly into the main model (only when the main model can see images)." + } + }, + "required": ["paths"], + "additionalProperties": false +} +``` +rules: +- the focus argument is named exactly `query` — never `Prompt`, `question`, or any other name +- `paths` as a JSON array even for one image: `{"paths": ["/path/to/image.png"]}` — a bare string is also accepted +- a dedicated Vision Model IS configured: you will NOT see the images; vision_load returns a concise text capsule answering your `query` +- write a focused `query` (e.g. "read the top-right error toast", "locate the login button and give its position"); if omitted, a generic precise description is returned +- `raw=true` (default false) bypasses delegation and injects images directly — only use when Main can see images and truly needs pixels (e.g. side-by-side comparison) +- load all relevant images in one call when comparing screenshots or pages; only bitmaps are supported +- large images are auto-compressed before sending +example: +```json +{ + "thoughts": [ + "I need to inspect the screenshot before answering." + ], + "headline": "Loading screenshot for visual analysis", + "tool_name": "vision_load", + "tool_args": { + "paths": ["/path/to/screenshot.png"], + "query": "read any error message visible in the top right" + } +} +``` diff --git a/plugins/_vision_sidecar/scripts/enable_vision_slot.py b/plugins/_vision_sidecar/scripts/enable_vision_slot.py new file mode 100644 index 0000000000..f06f608af6 --- /dev/null +++ b/plugins/_vision_sidecar/scripts/enable_vision_slot.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Vision Sidecar - add the Vision Model slot to Model Presets (one-time, optional). + +Self-contained: no git, no patch(1) needed. Works from any directory. +Usage: + python3 enable_vision_slot.py apply (idempotent) + python3 enable_vision_slot.py --status show state only + python3 enable_vision_slot.py --restore restore backups (.bak) +Docker: run INSIDE the Agent Zero container: + docker exec -it python3 usr/plugins/vision_sidecar/scripts/enable_vision_slot.py +Override root: A0_ROOT=/path/to/agent-zero python3 enable_vision_slot.py +""" +from __future__ import annotations +import os, shutil, sys +from pathlib import Path + +REPO = "https://github.com/GreifMax/a0-vision-sidecar" + +def find_a0_root(script_path: Path) -> Path | None: + env = os.environ.get("A0_ROOT", "").strip() + cands: list[Path] = [] + # Priority: A0_ROOT env, then CWD walk-up (explicit user intent), then script walk-up + if env: + cands.append(Path(env)) + cur = Path.cwd().resolve() + for _ in range(8): + cands.append(cur) + cur = cur.parent + cur = script_path.resolve() + for _ in range(8): + cands.append(cur) + cur = cur.parent + for c in cands: + try: + c = c.resolve() + except Exception: + continue + if (c / "plugins" / "_model_config" / "helpers" / "model_config.py").is_file(): + return c + return None + +# --- edits --------------------------------------------------------------- +V = "Optional dedicated model for vision_load - used when Main has no vision. Leave empty to use Main's vision." + +PY_EDITS = [ + dict(op="insert_after", anchor=' "chat": "chat_model",', + payload=' "vision": "vision_model",\n', done='"vision": "vision_model"'), + dict(op="insert_after", anchor="IMPLICIT_PRESET_SLOT_DEFAULTS = {", + payload=' "vision": {\n "rl_requests": 0,\n "rl_input": 0,\n "rl_output": 0,\n "kwargs": {},\n },\n', + done='"vision": {\n "rl_requests": 0'), + dict(op="insert_after", anchor=" slot_clean = _strip_ui_fields(slot_config, strip_api_key=True)", + payload=' if slot == "vision" and _slot_has_identity(slot_clean):\n slot_clean["vision"] = True\n if "max_embeds" not in slot_clean:\n slot_clean["max_embeds"] = 10\n', + done='slot_clean["vision"] = True'), + dict(op="replace", anchor='for section_name in ("chat_model", "utility_model", "embedding_model"):'.replace('"','\"'), + new='for section_name in ("chat_model", "vision_model", "utility_model", "embedding_model"):'.replace('"','\"'), + done='"chat_model", "vision_model", "utility_model"'), + dict(op="insert_before", anchor="def is_chat_override_allowed", + payload='def get_vision_model_config(agent=None) -> dict:\n """Vision model config from the vision preset slot (Vision Sidecar)."""\n return get_effective_config(agent).get("vision_model", {})\n\n\n', + done="def get_vision_model_config"), + dict(op="replace", + anchor=""" slot_config = _get_preset_slot_config(preset, slot) + if not _should_apply_preset_slot(slot, slot_config): + continue + config[section] = _merge_model_slot( + slot, + config.get(section, {}), + slot_config, + strip_api_key=strip_api_key, + )""", + new=""" slot_config = _get_preset_slot_config(preset, slot) + if not _should_apply_preset_slot(slot, slot_config): + if slot == "vision": + # Vision Sidecar: vision is strictly per-preset — never inherit + # the Default preset's vision model into other presets. + config[section] = {} + continue + base_slot = config.get(section, {}) + if slot == "vision": + # Vision Sidecar: merge vision over an empty base so a preset's own + # vision model replaces, never accumulates on top of, another one. + base_slot = {} + config[section] = _merge_model_slot( + slot, + base_slot, + slot_config, + strip_api_key=strip_api_key, + )""", + done='if slot == "vision":'), +] + +STORE_EDITS = [ + dict(op="insert_after", anchor=" { key: 'chat_model', title: 'Main Model', desc: 'Primary model for chat, reasoning, and browser tasks.' },", + payload=" { key: 'vision_model', title: 'Vision Model', desc: \"" + V + "'s vision.' },\n".replace("'s vision.' }", "\\'s vision.' }", 0) if False else " { key: 'vision_model', title: 'Vision Model', desc: \"Optional dedicated model for vision_load - used when Main has no vision. Leave empty to use Main\\'s vision.\" },\n", + done="key: 'vision_model'"), + dict(op="insert_after", anchor="const IMPLICIT_PRESET_SLOT_DEFAULTS = {", + payload=" vision: {\n rl_requests: 0,\n rl_input: 0,\n rl_output: 0,\n kwargs: {},\n },\n", + done=" vision: {"), + dict(op="insert_after", anchor=" ['chat', 'chat_model'],", + payload=" ['vision', 'vision_model'],\n", done="['vision', 'vision_model']"), + dict(op="insert_after", anchor=" chat_model: slot(rawDefault.chat),", + payload=" vision_model: slot(rawDefault.vision),\n", done="vision_model: slot(rawDefault.vision)"), + dict(op="insert_after", anchor=" chat: { ...slot(effective.chat_model), _kwargs_text: kwargsToText(effective.chat_model?.kwargs) },", + payload=""" vision: { + ...slot(effective.vision_model), + // Vision Sidecar: display defaults (64000 / 0.7) when unset + ...(hasModelIdentity(effective.vision_model || {}) + ? { ctx_length: Number(effective.vision_model?.ctx_length) || 64000, ctx_history: Number(effective.vision_model?.ctx_history ?? 0.7) } + : { ctx_length: 64000, ctx_history: 0.7 }), + _kwargs_text: kwargsToText(effective.vision_model?.kwargs), + }, +""", + done="display defaults (64000 / 0.7) when unset"), + dict(op="replace", + anchor=""" const slot = preset?.[slotKey]; + if (!slot || typeof slot !== 'object') continue; + if (!hasModelIdentity(slot)) continue; + config[sectionKey] = mergeModelSlot(config[sectionKey] || {}, slot, stripApiKey, slotKey);""", + new=""" const slot = preset?.[slotKey]; + const isVision = slotKey === 'vision'; + if (!slot || typeof slot !== 'object') { + if (isVision) config.vision_model = {}; // Vision Sidecar: never inherit from Default + continue; + } + if (!hasModelIdentity(slot)) { + if (isVision) config.vision_model = {}; // Vision Sidecar: never inherit from Default + continue; + } + // Vision Sidecar: merge vision over an empty base (strictly per-preset) + config[sectionKey] = mergeModelSlot(isVision ? {} : (config[sectionKey] || {}), slot, stripApiKey, slotKey);""", + done="const isVision = slotKey === 'vision';"), + dict(op="replace", anchor=" chat: { provider: '', name: '', api_base: '', kwargs: {}, _kwargs_text: '' },", + new=" chat: { provider: '', name: '', api_base: '', ctx_length: 200000, ctx_history: 0.7, kwargs: {}, _kwargs_text: '' },\n vision: { provider: '', name: '', api_base: '', ctx_length: 64000, ctx_history: 0.7, vision: true, max_embeds: 10, kwargs: {}, _kwargs_text: '' },", + done="vision: { provider: '', name: '', api_base: '', ctx_length: 64000"), + dict(op="replace", anchor=" utility: { provider: '', name: '', api_base: '', kwargs: {}, _kwargs_text: '' },", + new=" utility: { provider: '', name: '', api_base: '', ctx_length: 128000, ctx_input: 0.7, kwargs: {}, _kwargs_text: '' },", + done="utility: { provider: '', name: '', api_base: '', ctx_length: 128000"), + dict(op="insert_before", anchor=" const preset = {", + payload=" // ensure Vision slot defaults for new presets (64000 / 0.7)\n if (!base.vision || typeof base.vision.ctx_length === 'undefined') {\n base.vision = { provider: '', name: '', api_base: '', ctx_length: 64000, ctx_history: 0.7, vision: true, max_embeds: 10, kwargs: {}, _kwargs_text: '', ...(base.vision || {}) };\n }\n", + done="base.vision = { provider: ''"), + dict(op="replace", anchor=" for (const slot of ['chat', 'utility']) {", + new=" for (const slot of ['chat', 'vision', 'utility']) {", + done="['chat', 'vision', 'utility']"), + dict(op="replace", anchor=" if (hasModelIdentity(rest)) c[slot] = rest;", + new=" if (hasModelIdentity(rest)) {\n if (slot === 'vision') { rest.vision = true; if (!('max_embeds' in rest)) rest.max_embeds = 10; }\n c[slot] = rest;\n }", + done="rest.vision = true"), + dict(op="insert_after", anchor=" { icon: 'chat', title: 'Main', cfg: preset?.chat, pList: chatP },", + payload=" { icon: 'eye', title: 'Vision', cfg: preset?.vision, pList: chatP },\n", + done="title: 'Vision'"), + dict(op="replace", + anchor=""" ].map(s => ({ icon: s.icon, title: s.title, provider: label(s.pList, s.cfg?.provider), name: s.cfg?.name || '\\u2014' }));""", + new=""" ].map(s => { + const overridden = s.title === 'Vision' + && (s.cfg?.provider || s.cfg?.name) + && preset?.chat?.vision + && preset?.chat?.vision_override; + return { + icon: s.icon, + title: s.title, + provider: label(s.pList, s.cfg?.provider), + name: s.cfg?.name || '\\u2014', + note: overridden ? 'Overwritten by Main' : '', + }; + });""", + done="note: overridden"), +] + +OVERVIEW_EDITS = [ + dict(op="replace", + anchor=""" + + / + + """, + new=""" + + + """, + done="model-preset-note"), + dict(op="replace", + anchor=""" .model-preset-separator { + margin: 0 0.25rem; + }""", + new=""" .model-preset-separator { + margin: 0 0.25rem; + } + + .model-preset-note { + opacity: 0.65; + font-style: italic; + }""", + done=".model-preset-note {"), +] + +MAIN_SECTION = ( +"
\n" +"
\n" +"
Vision Model
\n" +"
" + V + "
\n" +"
\n" +"
\n" +" \n" +"
\n" +"
\n" +"\n" +) + +MAIN_EDITS = [ + dict(op="insert_before_back", anchor="Utility Model", back='
', + payload=MAIN_SECTION, done="section-title\">Vision Model"), +] + +SWITCHER_BLOCK = ( +" \n" +) +SWITCHER_EDITS = [ + dict(op="replace_optional", anchor='