visual: top-down map renderer, demo --frames-dir mode (step frames + GIF + summary), example artifacts committed

testbed/demo.py

@@ -29,12 +29,14 @@ import io
 import json
 import math
 import sys
+from pathlib import Path
 from typing import Any
 
 from aicc.client import AICCClient
 from aicc.errors import ToolError
 from aicc.protocol import SessionInit, ToolSpec
 from aicc.transport.websocket import WebSocketClientTransport
+from PIL import Image
 
 DEFAULT_URL = "ws://127.0.0.1:8765"
 DEFAULT_BASE_URL = "http://localhost:11434/v1"
@@ -86,6 +88,137 @@ def _nearest_color(r: float, g: float, b: float) -> str:
     return best
 
 
+class FrameRecorder:
+    """Visual mode: saves first-person frames and a sensor-built top-down map
+    with the capsule's path, then stitches a GIF and a summary image.
+
+    Only uses data observed through protocol sensors (world_query,
+    proprioception, vision) — the map is reconstructed the way the agent sees
+    the world, not read from bridge internals.
+    """
+
+    MAP_SIZE = 320
+    ROOM = 16.0
+
+    def __init__(self, out_dir: str):
+        self.out_dir = Path(out_dir)
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        self.layout: dict[str, Any] | None = None
+        self.path: list[tuple[float, float]] = []
+        self.map_frames: list[Image.Image] = []
+        self.view_frames: list[Image.Image] = []
+        self.step = 0
+        self._log = lambda msg: print(f"[rec] {msg}")
+
+    async def snap(
+        self, client: AICCClient, pos: tuple[float, float], yaw: float
+    ) -> None:
+        """Capture one visual snapshot (one vision call + sensor layout)."""
+        if self.layout is None:
+            wq = (await client.call_tool("world_query", {})).output
+            self.layout = wq
+            self._log("world layout captured from sensors")
+        self.path.append(pos)
+        view = await self._view_frame(client)
+        self.view_frames.append(view)
+        self.step += 1
+        map_img = self._map_frame(pos, yaw)
+        self.map_frames.append(map_img)
+        view.save(self.out_dir / f"step_{self.step:03d}_view.png")
+        map_img.save(self.out_dir / f"step_{self.step:03d}_map.png")
+
+    async def _view_frame(self, client: AICCClient) -> Image.Image:
+        from PIL import Image
+
+        out = (await client.call_tool("vision", {})).output
+        self._log(f"vision({out['width']}x{out['height']}) frame captured")
+        return Image.open(io.BytesIO(base64.b64decode(out["png_b64"]))).convert("RGB")
+
+    def _map_frame(self, pos: tuple[float, float], yaw: float) -> Image.Image:
+        from PIL import Image, ImageDraw
+
+        size = self.MAP_SIZE
+        scale = size / self.ROOM
+
+        def xy(x: float, z: float) -> tuple[float, float]:
+            return (x * scale, size - z * scale)
+
+        img = Image.new("RGB", (size, size), (52, 52, 58))
+        d = ImageDraw.Draw(img)
+        for i in range(int(self.ROOM) + 1):
+            c = 60 if i % 2 == 0 else 54
+            d.line([xy(i, 0), xy(i, self.ROOM)], fill=(c, c, c + 6), width=1)
+            d.line([xy(0, i), xy(self.ROOM, i)], fill=(c, c, c + 6), width=1)
+        if self.layout:
+            for ob in self.layout.get("obstacles", []):
+                x0, z0 = xy(ob["x"] - ob["width"] / 2, ob["z"] - ob["depth"] / 2)
+                x1, z1 = xy(ob["x"] + ob["width"] / 2, ob["z"] + ob["depth"] / 2)
+                d.rectangle(
+                    [min(x0, x1), min(z0, z1), max(x0, x1), max(z0, z1)],
+                    fill=(150, 90, 60),
+                    outline=(20, 20, 26),
+                    width=2,
+                )
+            b = self.layout.get("beacon", {})
+            bx, bz = xy(b.get("x", 12.5), b.get("z", 12.5))
+            d.ellipse(
+                [bx - 8, bz - 8, bx + 8, bz + 8],
+                fill=(120, 210, 235),
+                outline=(20, 20, 26),
+                width=2,
+            )
+        if len(self.path) > 1:
+            pts = [xy(x, z) for x, z in self.path]
+            d.line(pts, fill=(255, 170, 60), width=3)
+        cx, cz = xy(*pos)
+        d.ellipse(
+            [cx - 7, cz - 7, cx + 7, cz + 7],
+            fill=(240, 240, 250),
+            outline=(20, 20, 26),
+            width=2,
+        )
+        rad = math.radians(yaw)
+        tip = xy(pos[0] + math.sin(rad) * 0.7, pos[1] + math.cos(rad) * 0.7)
+        d.line([(cx, cz), tip], fill=(30, 30, 40), width=3)
+        d.text(
+            (8, 8),
+            f"step {self.step}  pos ({pos[0]:.1f}, {pos[1]:.1f}) yaw {yaw:.0f}",
+            fill=(230, 230, 230),
+        )
+        return img
+
+    def finish(self, interacted: bool) -> Image.Image:
+        """Write demo.gif + demo_summary.png; returns the summary image."""
+        from PIL import Image, ImageDraw
+
+        if self.map_frames:
+            gif = self.out_dir / "demo.gif"
+            self.map_frames[0].save(
+                gif,
+                save_all=True,
+                append_images=self.map_frames[1:],
+                duration=350,
+                loop=0,
+            )
+            self._log(f"animation written: {gif}")
+        summary = Image.new("RGB", (self.MAP_SIZE + 320, 240), (20, 20, 26))
+        if self.map_frames:
+            summary.paste(self.map_frames[-1], (0, 0))
+        if self.view_frames:
+            summary.paste(self.view_frames[-1].resize((320, 240)), (self.MAP_SIZE, 0))
+        d = ImageDraw.Draw(summary)
+        status = "BEACON ACTIVATED" if interacted else "mission not completed"
+        d.text(
+            (self.MAP_SIZE + 8, 244 - 16),
+            status,
+            fill=(255, 220, 120) if interacted else (255, 120, 120),
+        )
+        out = self.out_dir / "demo_summary.png"
+        summary.save(out)
+        self._log(f"summary written: {out}")
+        return summary
+
+
 def vision_digest(png_b64: str, cols: int = 12, rows: int = 9) -> str:
     """Coarse color-grid summary of a vision frame (agent-side preprocessing)."""
     try:
@@ -151,6 +284,7 @@ async def run_llm_agent(
     api_key: str,
     model: str,
     max_steps: int,
+    recorder: FrameRecorder | None = None,
 ) -> dict[str, Any]:
     from openai import AsyncOpenAI
 
@@ -248,6 +382,8 @@ async def run_llm_agent(
 
     for step in range(max_steps):
         summary["steps"] = step + 1
+        if recorder is not None and pos is not None:
+            await recorder.snap(client, pos, yaw)
         try:
             resp = await ac.chat.completions.create(
                 model=model, messages=messages, tools=tools, tool_choice="auto"
@@ -357,7 +493,10 @@ async def run_llm_agent(
 
 
 async def run_scripted_agent(
-    client: AICCClient, manifest: SessionInit, max_steps: int
+    client: AICCClient,
+    manifest: SessionInit,
+    max_steps: int,
+    recorder: FrameRecorder | None = None,
 ) -> dict[str, Any]:
     summary: dict[str, Any] = {
         "steps": 0,
@@ -382,6 +521,10 @@ async def run_scripted_agent(
         prop = (await client.call_tool("proprioception", {})).output
         summary["tool_calls"] += 1
         pos = prop["position"]
+        if recorder is not None:
+            await recorder.snap(
+                client, (pos["x"], pos["z"]), prop["rotation"]["yaw_deg"]
+            )
         dist = math.hypot(bx - pos["x"], bz - pos["z"])
         log(
             "agent",
@@ -474,6 +617,11 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
         print(f"[handshake] session {manifest.session_id} world {manifest.world.name}")
         print(f"[handshake] tools: {[t.id for t in manifest.tools]}\n")
 
+        if args.frames_dir:
+            recorder = FrameRecorder(args.frames_dir)
+        else:
+            recorder = None
+
         if args.agent == "llm":
             summary = await run_llm_agent(
                 client,
@@ -482,9 +630,12 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                 api_key=args.api_key,
                 model=args.model,
                 max_steps=args.max_steps,
+                recorder=recorder,
             )
         elif args.agent == "scripted":
-            summary = await run_scripted_agent(client, manifest, args.max_steps)
+            summary = await run_scripted_agent(
+                client, manifest, args.max_steps, recorder=recorder
+            )
         else:  # auto
             from openai import AsyncOpenAI
 
@@ -499,6 +650,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                     api_key=args.api_key,
                     model=args.model,
                     max_steps=llm_steps,
+                    recorder=recorder,
                 )
                 if summary.get("interacted"):
                     return summary
@@ -506,12 +658,19 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
                     f"\n[auto] LLM did not finish in {llm_steps} steps "
                     f"({summary.get('result')}); handing off to the scripted agent\n"
                 )
-                summary = await run_scripted_agent(client, manifest, args.max_steps)
+                summary = await run_scripted_agent(
+                    client, manifest, args.max_steps, recorder=recorder
+                )
             except Exception as exc:  # noqa: BLE001 - fall back to scripted
                 print(
                     f"\n[auto] LLM endpoint unavailable ({type(exc).__name__}: {exc}); falling back to scripted agent"
                 )
-                summary = await run_scripted_agent(client, manifest, args.max_steps)
+                summary = await run_scripted_agent(
+                    client, manifest, args.max_steps, recorder=recorder
+                )
+
+        if recorder is not None:
+            recorder.finish(summary.get("interacted", False))
 
         frame = (await client.call_tool("vision", {})).output
         if args.frame:
@@ -558,6 +717,12 @@ def main() -> int:
         default="demo_final_frame.png",
         help="where to save the final vision frame",
     )
+    parser.add_argument(
+        "--frames-dir",
+        default=None,
+        help="visual mode: save first-person + top-down map frames each step "
+        "into this directory, plus demo.gif animation and demo_summary.png",
+    )
     args = parser.parse_args()
     summary = asyncio.run(run_demo(args))
     return 0 if summary.get("interacted") else 1