vision: real camera frames to the model (image_url), auto-detected for multimodal ollama models; --vision/--no-vision flags; verified gemma4:12b sees frames and completed the beacon mission

testbed/README.md

@@ -101,6 +101,20 @@ works: `python -m testbed.chat --base-url https://api.openai.com/v1
 --model gpt-4o-mini --api-key $OPENAI_API_KEY`. `--auto-steps N` controls how
 many tool steps the model may chain per request (0 = one action per turn).
 
+### Real vision
+
+By default the model gets frames as a color-grid digest (works for any
+text-only model). If the model can actually see images (gemma3/gemma4,
+qwen2.5-vl, gpt-4o-mini, ...), pass the frames as real images:
+
+- auto-detected for local ollama (`/api/show` capabilities) — nothing to do
+- otherwise force it: `--vision` (or `--no-vision` to disable)
+
+With vision ON, `vision` tool results attach the actual camera frame as an
+image to the conversation — the model sees the crate, the wall, the glowing
+beacon, and orients itself. Verified locally: gemma4:12b completed a beacon
+mission on its first attempt using look_at/move navigation.
+
 ## Real-time mode
 
 Watch the capsule drive live in your browser:

testbed/chat.py

@@ -62,9 +62,12 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
         model=model,
         system_prompt=CHAT_MISSION,
         log=lambda role, msg: print(f"  [{role}] {msg}"),
+        multimodal=args.vision,
     )
     auto_steps = args.auto_steps
     print(f"[chat] model: {model}  (endpoint {args.base_url})")
+    if controller.multimodal:
+        print("[chat] vision: ON — the model sees the actual camera frames")
     print("[chat] type your commands; /help for the command list; /exit to quit\n")
 
     async def cmd_state() -> None:
@@ -413,6 +416,12 @@ def main() -> int:
         default=3,
         help="how many consecutive nudges a mission may use before giving up (default 3)",
     )
+    parser.add_argument(
+        "--vision",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="pass real camera frames to the model as images (auto-detected for local ollama)",
+    )
     args = parser.parse_args()
     try:
         return asyncio.run(run(args))

testbed/demo.py

@@ -139,6 +139,7 @@ async def run_llm_agent(
     model: str,
     max_steps: int,
     recorder: FrameRecorder | None = None,
+    vision: bool | None = None,
 ) -> dict[str, Any]:
     """Autonomous LLM run: controller + nudge/correct loop (see llm_agent)."""
     from testbed.llm_agent import LLMController, run_llm_agent_loop
@@ -150,6 +151,7 @@ async def run_llm_agent(
         api_key=api_key,
         model=model,
         log=lambda role, msg: print(f"[{role}] {msg}"),
+        multimodal=vision,
     )
     try:
         return await run_llm_agent_loop(
@@ -307,6 +309,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                 model=args.model,
                 max_steps=args.max_steps,
                 recorder=recorder,
+                vision=args.vision,
             )
         elif args.agent == "scripted":
             summary = await run_scripted_agent(
@@ -327,6 +330,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                     model=args.model,
                     max_steps=llm_steps,
                     recorder=recorder,
+                    vision=args.vision,
                 )
                 if summary.get("interacted"):
                     return summary
@@ -388,6 +392,12 @@ def main() -> int:
     )
     parser.add_argument("--api-key", default="ollama", help="API key for the endpoint")
     parser.add_argument("--max-steps", type=int, default=60, help="max agent steps")
+    parser.add_argument(
+        "--vision",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="pass real camera frames to the model as images (auto-detected for local ollama)",
+    )
     parser.add_argument(
         "--frame",
         default="demo_final_frame.png",

testbed/llm_agent.py

@@ -184,6 +184,32 @@ class TurnResult:
     message: str | None = None
 
 
+def detect_multimodal(base_url: str, model: str) -> bool:
+    """Best-effort check whether the model can actually see images.
+
+    For a local ollama we ask /api/show (capabilities include 'vision').
+    For other OpenAI-compatible endpoints we cannot introspect — the caller
+    may force it with --vision.
+    """
+    host = base_url.replace("http://", "").replace("https://", "").split("/")[0]
+    if host not in ("localhost:11434", "127.0.0.1:11434"):
+        return False
+    import urllib.request
+
+    endpoint = base_url.rsplit("/v1", 1)[0] + "/api/show"
+    try:
+        req = urllib.request.Request(
+            endpoint,
+            data=json.dumps({"model": model}).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            caps = json.loads(resp.read()).get("capabilities", [])
+        return "vision" in caps
+    except Exception:  # noqa: BLE001 - detection is best-effort
+        return False
+
+
 class LLMController:
     """Drives a tool-calling LLM over an OpenAI-compatible endpoint.
 
@@ -202,11 +228,15 @@ class LLMController:
         model: str,
         system_prompt: str = MISSION,
         log: LogFn | None = None,
+        multimodal: bool | None = None,
     ):
         from openai import AsyncOpenAI
 
         self.client = client
         self.model = model
+        if multimodal is None:
+            multimodal = detect_multimodal(base_url, model)
+        self.multimodal = multimodal
         self.ac = AsyncOpenAI(base_url=base_url, api_key=api_key)
         self.tools = to_openai_tools(manifest.tools)
         self.messages: list[dict[str, Any]] = [
@@ -324,12 +354,44 @@ class LLMController:
                         self.beacon_pos = (b["x"], b["z"])
                 self.observe(output)
                 hint = self.state_hint()
-                content = json.dumps(outcome.output) + extra
+                frame_b64 = None
-                if hint:
+                if name == "vision" and isinstance(outcome.output.get("png_b64"), str):
-                    content = hint + "\n" + content
+                    frame_b64 = outcome.output["png_b64"]
-                self.messages.append(
+                if self.multimodal and frame_b64 is not None:
-                    {"role": "tool", "tool_call_id": tc.id, "content": content}
+                    # Real vision: hand the model the actual frame as an image
-                )
+                    # (data URI), not as base64 text. Keep the digest too —
+                    # it is a cheap textual anchor for the model.
+                    content = (hint + "\n" if hint else "") + (
+                        f"Camera frame ({output['width']}x{output['height']}, "
+                        f"tick {output.get('tick')}) attached as an image.{extra}"
+                    )
+                    self.messages.append(
+                        {"role": "tool", "tool_call_id": tc.id, "content": content}
+                    )
+                    self.messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": "This is what your camera sees right now. Use it to orient yourself.",
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:image/png;base64,{frame_b64}"
+                                    },
+                                },
+                            ],
+                        }
+                    )
+                else:
+                    content = json.dumps(outcome.output) + extra
+                    if hint:
+                        content = hint + "\n" + content
+                    self.messages.append(
+                        {"role": "tool", "tool_call_id": tc.id, "content": content}
+                    )
                 call_result = ToolCallResult(
                     name=name, args=args, ok=True, output=outcome.output, digest=digest
                 )