vision: images are the primary channel for multimodal models (digest off by default); digest is the text-only fallback; --digest/--no-digest to override

testbed/README.md

@@ -103,17 +103,19 @@ many tool steps the model may chain per request (0 = one action per turn).
 
 ### Real vision
 
-By default the model gets frames as a color-grid digest (works for any
-text-only model). If the model can actually see images (gemma3/gemma4,
-qwen2.5-vl, gpt-4o-mini, ...), pass the frames as real images:
+Vision is the **primary channel**: when the model is multimodal, `vision`
+tool results attach the actual camera frame as an image to the conversation —
+the model sees the crate, the wall, the glowing beacon. The color-grid
+digest is only the **fallback** for text-only models:
 
-- auto-detected for local ollama (`/api/show` capabilities) — nothing to do
-- otherwise force it: `--vision` (or `--no-vision` to disable)
+- multimodal model (auto-detected for local ollama via `/api/show`;
+  otherwise `--vision`): frame as image, digest off
+- text-only model: digest on automatically; `--digest` forces the digest to
+  be included even alongside images, `--no-digest` disables it everywhere
 
-With vision ON, `vision` tool results attach the actual camera frame as an
-image to the conversation — the model sees the crate, the wall, the glowing
-beacon, and orients itself. Verified locally: gemma4:12b completed a beacon
-mission on its first attempt using look_at/move navigation.
+Verified locally: gemma4:12b sees frames and completed a beacon mission on
+its first attempt; gemma4:e2b describes what it sees (crates, beacon) and
+orients itself from the image.
 
 ## Real-time mode
 

testbed/chat.py

@@ -63,11 +63,14 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
         system_prompt=CHAT_MISSION,
         log=lambda role, msg: print(f"  [{role}] {msg}"),
         multimodal=args.vision,
+        digest=args.digest,
     )
     auto_steps = args.auto_steps
     print(f"[chat] model: {model}  (endpoint {args.base_url})")
     if controller.multimodal:
         print("[chat] vision: ON — the model sees the actual camera frames")
+    else:
+        print("[chat] vision: text-only model — frames are sent as a color-grid digest")
     print("[chat] type your commands; /help for the command list; /exit to quit\n")
 
     async def cmd_state() -> None:
@@ -422,6 +425,12 @@ def main() -> int:
         default=None,
         help="pass real camera frames to the model as images (auto-detected for local ollama)",
     )
+    parser.add_argument(
+        "--digest",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="always include the color-grid digest alongside images (off by default for multimodal models)",
+    )
     args = parser.parse_args()
     try:
         return asyncio.run(run(args))

testbed/demo.py

@@ -140,6 +140,7 @@ async def run_llm_agent(
     max_steps: int,
     recorder: FrameRecorder | None = None,
     vision: bool | None = None,
+    digest: bool | None = None,
 ) -> dict[str, Any]:
     """Autonomous LLM run: controller + nudge/correct loop (see llm_agent)."""
     from testbed.llm_agent import LLMController, run_llm_agent_loop
@@ -152,6 +153,7 @@ async def run_llm_agent(
         model=model,
         log=lambda role, msg: print(f"[{role}] {msg}"),
         multimodal=vision,
+        digest=digest,
     )
     try:
         return await run_llm_agent_loop(
@@ -310,6 +312,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                 max_steps=args.max_steps,
                 recorder=recorder,
                 vision=args.vision,
+                digest=args.digest,
             )
         elif args.agent == "scripted":
             summary = await run_scripted_agent(
@@ -331,6 +334,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                     max_steps=llm_steps,
                     recorder=recorder,
                     vision=args.vision,
+                    digest=args.digest,
                 )
                 if summary.get("interacted"):
                     return summary
@@ -398,6 +402,12 @@ def main() -> int:
         default=None,
         help="pass real camera frames to the model as images (auto-detected for local ollama)",
     )
+    parser.add_argument(
+        "--digest",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="always include the color-grid digest alongside images (off by default for multimodal models)",
+    )
     parser.add_argument(
         "--frame",
         default="demo_final_frame.png",

testbed/llm_agent.py

@@ -229,6 +229,7 @@ class LLMController:
         system_prompt: str = MISSION,
         log: LogFn | None = None,
         multimodal: bool | None = None,
+        digest: bool | None = None,
     ):
         from openai import AsyncOpenAI
 
@@ -237,6 +238,9 @@ class LLMController:
         if multimodal is None:
             multimodal = detect_multimodal(base_url, model)
         self.multimodal = multimodal
+        # The color-grid digest is the fallback channel for text-only models.
+        # For multimodal models it is off unless explicitly requested.
+        self.include_digest = (not multimodal) if digest is None else digest
         self.ac = AsyncOpenAI(base_url=base_url, api_key=api_key)
         self.tools = to_openai_tools(manifest.tools)
         self.messages: list[dict[str, Any]] = [
@@ -344,9 +348,10 @@ class LLMController:
                 digest = None
                 extra = ""
                 if name == "vision" and isinstance(output.get("png_b64"), str):
-                    digest = vision_digest(output["png_b64"])
-                    extra = "\n" + digest
                     output = dict(output, png_b64="<binary, decoded for digest>")
+                    if self.include_digest:
+                        digest = vision_digest(outcome.output["png_b64"])
+                        extra = "\n" + digest
                 self.log("bridge", f"ok {json.dumps(output)[:500]}{extra}")
                 if name == "world_query":
                     b = output.get("beacon")
@@ -358,13 +363,14 @@ class LLMController:
                 if name == "vision" and isinstance(outcome.output.get("png_b64"), str):
                     frame_b64 = outcome.output["png_b64"]
                 if self.multimodal and frame_b64 is not None:
-                    # Real vision: hand the model the actual frame as an image
-                    # (data URI), not as base64 text. Keep the digest too —
-                    # it is a cheap textual anchor for the model.
-                    content = (hint + "\n" if hint else "") + (
+                    # Primary channel: hand the model the actual frame as an
+                    # image (data URI). The digest is optional and off by
+                    # default for multimodal models.
+                    frame_note = (
                         f"Camera frame ({output['width']}x{output['height']}, "
-                        f"tick {output.get('tick')}) attached as an image.{extra}"
+                        f"tick {output.get('tick')}) attached as an image."
                     )
+                    content = (hint + "\n" if hint else "") + frame_note + extra
                     self.messages.append(
                         {"role": "tool", "tool_call_id": tc.id, "content": content}
                     )
@@ -386,9 +392,14 @@ class LLMController:
                         }
                     )
                 else:
-                    content = json.dumps(outcome.output) + extra
-                    if hint:
-                        content = hint + "\n" + content
+                    # Fallback channel: the digest (plus frame metadata) for
+                    # text-only models.
+                    content = (
+                        (hint + "\n" if hint else "")
+                        + f"Camera frame ({output['width']}x{output['height']}, "
+                        + f"tick {output.get('tick')})."
+                        + extra
+                    )
                     self.messages.append(
                         {"role": "tool", "tool_call_id": tc.id, "content": content}
                     )