diff --git a/demo.gif b/demo.gif new file mode 100644 index 0000000..e1614be Binary files /dev/null and b/demo.gif differ diff --git a/demo_final_frame.png b/demo_final_frame.png new file mode 100644 index 0000000..3ff8882 Binary files /dev/null and b/demo_final_frame.png differ diff --git a/demo_summary.png b/demo_summary.png new file mode 100644 index 0000000..f048b36 Binary files /dev/null and b/demo_summary.png differ diff --git a/testbed/README.md b/testbed/README.md index 91a2991..3aabb0c 100644 --- a/testbed/README.md +++ b/testbed/README.md @@ -60,6 +60,19 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal. The demo prints a full transcript of tool calls/results to stdout and saves the capsule's final first-person frame to `demo_final_frame.png`. +## Visual mode + +```bash +scripts/run_demo.sh --agent scripted --frames-dir frames +``` + +Saves, for every step, the first-person frame (`step_NNN_view.png`) and a +top-down map of the room with the capsule's path (`step_NNN_map.png`), then +writes a `demo.gif` animation and a `demo_summary.png` (final map + last +view). The map is rebuilt purely from sensor data (`world_query`, +`proprioception`, `vision`) — the same view the agent itself has. Generated +examples are committed at the repo root (`demo.gif`, `demo_summary.png`). + ## Conformance ```bash diff --git a/testbed/demo.py b/testbed/demo.py index 8b4343a..0931495 100644 --- a/testbed/demo.py +++ b/testbed/demo.py @@ -29,12 +29,14 @@ import io import json import math import sys +from pathlib import Path from typing import Any from aicc.client import AICCClient from aicc.errors import ToolError from aicc.protocol import SessionInit, ToolSpec from aicc.transport.websocket import WebSocketClientTransport +from PIL import Image DEFAULT_URL = "ws://127.0.0.1:8765" DEFAULT_BASE_URL = "http://localhost:11434/v1" @@ -86,6 +88,137 @@ def _nearest_color(r: float, g: float, b: float) -> str: return best +class FrameRecorder: + """Visual mode: saves first-person frames and a sensor-built top-down map + with the capsule's path, then stitches a GIF and a summary image. + + Only uses data observed through protocol sensors (world_query, + proprioception, vision) — the map is reconstructed the way the agent sees + the world, not read from bridge internals. + """ + + MAP_SIZE = 320 + ROOM = 16.0 + + def __init__(self, out_dir: str): + self.out_dir = Path(out_dir) + self.out_dir.mkdir(parents=True, exist_ok=True) + self.layout: dict[str, Any] | None = None + self.path: list[tuple[float, float]] = [] + self.map_frames: list[Image.Image] = [] + self.view_frames: list[Image.Image] = [] + self.step = 0 + self._log = lambda msg: print(f"[rec] {msg}") + + async def snap( + self, client: AICCClient, pos: tuple[float, float], yaw: float + ) -> None: + """Capture one visual snapshot (one vision call + sensor layout).""" + if self.layout is None: + wq = (await client.call_tool("world_query", {})).output + self.layout = wq + self._log("world layout captured from sensors") + self.path.append(pos) + view = await self._view_frame(client) + self.view_frames.append(view) + self.step += 1 + map_img = self._map_frame(pos, yaw) + self.map_frames.append(map_img) + view.save(self.out_dir / f"step_{self.step:03d}_view.png") + map_img.save(self.out_dir / f"step_{self.step:03d}_map.png") + + async def _view_frame(self, client: AICCClient) -> Image.Image: + from PIL import Image + + out = (await client.call_tool("vision", {})).output + self._log(f"vision({out['width']}x{out['height']}) frame captured") + return Image.open(io.BytesIO(base64.b64decode(out["png_b64"]))).convert("RGB") + + def _map_frame(self, pos: tuple[float, float], yaw: float) -> Image.Image: + from PIL import Image, ImageDraw + + size = self.MAP_SIZE + scale = size / self.ROOM + + def xy(x: float, z: float) -> tuple[float, float]: + return (x * scale, size - z * scale) + + img = Image.new("RGB", (size, size), (52, 52, 58)) + d = ImageDraw.Draw(img) + for i in range(int(self.ROOM) + 1): + c = 60 if i % 2 == 0 else 54 + d.line([xy(i, 0), xy(i, self.ROOM)], fill=(c, c, c + 6), width=1) + d.line([xy(0, i), xy(self.ROOM, i)], fill=(c, c, c + 6), width=1) + if self.layout: + for ob in self.layout.get("obstacles", []): + x0, z0 = xy(ob["x"] - ob["width"] / 2, ob["z"] - ob["depth"] / 2) + x1, z1 = xy(ob["x"] + ob["width"] / 2, ob["z"] + ob["depth"] / 2) + d.rectangle( + [min(x0, x1), min(z0, z1), max(x0, x1), max(z0, z1)], + fill=(150, 90, 60), + outline=(20, 20, 26), + width=2, + ) + b = self.layout.get("beacon", {}) + bx, bz = xy(b.get("x", 12.5), b.get("z", 12.5)) + d.ellipse( + [bx - 8, bz - 8, bx + 8, bz + 8], + fill=(120, 210, 235), + outline=(20, 20, 26), + width=2, + ) + if len(self.path) > 1: + pts = [xy(x, z) for x, z in self.path] + d.line(pts, fill=(255, 170, 60), width=3) + cx, cz = xy(*pos) + d.ellipse( + [cx - 7, cz - 7, cx + 7, cz + 7], + fill=(240, 240, 250), + outline=(20, 20, 26), + width=2, + ) + rad = math.radians(yaw) + tip = xy(pos[0] + math.sin(rad) * 0.7, pos[1] + math.cos(rad) * 0.7) + d.line([(cx, cz), tip], fill=(30, 30, 40), width=3) + d.text( + (8, 8), + f"step {self.step} pos ({pos[0]:.1f}, {pos[1]:.1f}) yaw {yaw:.0f}", + fill=(230, 230, 230), + ) + return img + + def finish(self, interacted: bool) -> Image.Image: + """Write demo.gif + demo_summary.png; returns the summary image.""" + from PIL import Image, ImageDraw + + if self.map_frames: + gif = self.out_dir / "demo.gif" + self.map_frames[0].save( + gif, + save_all=True, + append_images=self.map_frames[1:], + duration=350, + loop=0, + ) + self._log(f"animation written: {gif}") + summary = Image.new("RGB", (self.MAP_SIZE + 320, 240), (20, 20, 26)) + if self.map_frames: + summary.paste(self.map_frames[-1], (0, 0)) + if self.view_frames: + summary.paste(self.view_frames[-1].resize((320, 240)), (self.MAP_SIZE, 0)) + d = ImageDraw.Draw(summary) + status = "BEACON ACTIVATED" if interacted else "mission not completed" + d.text( + (self.MAP_SIZE + 8, 244 - 16), + status, + fill=(255, 220, 120) if interacted else (255, 120, 120), + ) + out = self.out_dir / "demo_summary.png" + summary.save(out) + self._log(f"summary written: {out}") + return summary + + def vision_digest(png_b64: str, cols: int = 12, rows: int = 9) -> str: """Coarse color-grid summary of a vision frame (agent-side preprocessing).""" try: @@ -151,6 +284,7 @@ async def run_llm_agent( api_key: str, model: str, max_steps: int, + recorder: FrameRecorder | None = None, ) -> dict[str, Any]: from openai import AsyncOpenAI @@ -248,6 +382,8 @@ async def run_llm_agent( for step in range(max_steps): summary["steps"] = step + 1 + if recorder is not None and pos is not None: + await recorder.snap(client, pos, yaw) try: resp = await ac.chat.completions.create( model=model, messages=messages, tools=tools, tool_choice="auto" @@ -357,7 +493,10 @@ async def run_llm_agent( async def run_scripted_agent( - client: AICCClient, manifest: SessionInit, max_steps: int + client: AICCClient, + manifest: SessionInit, + max_steps: int, + recorder: FrameRecorder | None = None, ) -> dict[str, Any]: summary: dict[str, Any] = { "steps": 0, @@ -382,6 +521,10 @@ async def run_scripted_agent( prop = (await client.call_tool("proprioception", {})).output summary["tool_calls"] += 1 pos = prop["position"] + if recorder is not None: + await recorder.snap( + client, (pos["x"], pos["z"]), prop["rotation"]["yaw_deg"] + ) dist = math.hypot(bx - pos["x"], bz - pos["z"]) log( "agent", @@ -474,6 +617,11 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]: print(f"[handshake] session {manifest.session_id} world {manifest.world.name}") print(f"[handshake] tools: {[t.id for t in manifest.tools]}\n") + if args.frames_dir: + recorder = FrameRecorder(args.frames_dir) + else: + recorder = None + if args.agent == "llm": summary = await run_llm_agent( client, @@ -482,9 +630,12 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]: api_key=args.api_key, model=args.model, max_steps=args.max_steps, + recorder=recorder, ) elif args.agent == "scripted": - summary = await run_scripted_agent(client, manifest, args.max_steps) + summary = await run_scripted_agent( + client, manifest, args.max_steps, recorder=recorder + ) else: # auto from openai import AsyncOpenAI @@ -499,6 +650,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]: api_key=args.api_key, model=args.model, max_steps=llm_steps, + recorder=recorder, ) if summary.get("interacted"): return summary @@ -506,12 +658,19 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]: f"\n[auto] LLM did not finish in {llm_steps} steps " f"({summary.get('result')}); handing off to the scripted agent\n" ) - summary = await run_scripted_agent(client, manifest, args.max_steps) + summary = await run_scripted_agent( + client, manifest, args.max_steps, recorder=recorder + ) except Exception as exc: # noqa: BLE001 - fall back to scripted print( f"\n[auto] LLM endpoint unavailable ({type(exc).__name__}: {exc}); falling back to scripted agent" ) - summary = await run_scripted_agent(client, manifest, args.max_steps) + summary = await run_scripted_agent( + client, manifest, args.max_steps, recorder=recorder + ) + + if recorder is not None: + recorder.finish(summary.get("interacted", False)) frame = (await client.call_tool("vision", {})).output if args.frame: @@ -558,6 +717,12 @@ def main() -> int: default="demo_final_frame.png", help="where to save the final vision frame", ) + parser.add_argument( + "--frames-dir", + default=None, + help="visual mode: save first-person + top-down map frames each step " + "into this directory, plus demo.gif animation and demo_summary.png", + ) args = parser.parse_args() summary = asyncio.run(run_demo(args)) return 0 if summary.get("interacted") else 1 diff --git a/testbed/room/render.py b/testbed/room/render.py index 8accca7..c1800d5 100644 --- a/testbed/room/render.py +++ b/testbed/room/render.py @@ -11,7 +11,7 @@ from __future__ import annotations import math -from PIL import Image +from PIL import Image, ImageDraw from testbed.room.world import ROOM_HEIGHT, ROOM_SIZE, Room @@ -355,6 +355,59 @@ class Raycaster: return grid +def render_topdown(room: Room, size: int = 400) -> Image.Image: + """Bird's-eye view of the whole room (debug helper; uses world state). + + Floor grid, crates, beacon (cyan / warm yellow when active) and the + capsule as a circle with a heading arrow. + """ + scale = size / ROOM_SIZE + img = Image.new("RGB", (size, size), (52, 52, 58)) + draw = ImageDraw.Draw(img) + + def xy(x: float, z: float) -> tuple[float, float]: + return (x * scale, size - z * scale) + + for i in range(int(ROOM_SIZE) + 1): + c = 60 if i % 2 == 0 else 54 + draw.line([xy(i, 0), xy(i, ROOM_SIZE)], fill=(c, c, c + 6), width=1) + draw.line([xy(0, i), xy(ROOM_SIZE, i)], fill=(c, c, c + 6), width=1) + + for box in room.boxes: + xa, za = xy(box.x_min(), box.z_min()) + xb, zb = xy(box.x_max(), box.z_max()) + draw.rectangle( + [min(xa, xb), min(za, zb), max(xa, xb), max(za, zb)], + fill=box.color, + outline=(20, 20, 26), + width=2, + ) + + b = room.beacon + bx, bz = xy(b.x, b.z) + color = (255, 200, 90) if b.active else (120, 210, 235) + glow = (255, 240, 180) if b.active else (160, 235, 250) + r = b.radius * scale * 2.2 + draw.ellipse([bx - r * 2.4, bz - r * 2.4, bx + r * 2.4, bz + r * 2.4], fill=glow) + draw.ellipse( + [bx - r, bz - r, bx + r, bz + r], fill=color, outline=(20, 20, 26), width=2 + ) + + c = room.capsule + cx, cz = xy(c.x, c.z) + rad = c.radius * scale + draw.ellipse( + [cx - rad, cz - rad, cx + rad, cz + rad], + fill=(240, 240, 250), + outline=(20, 20, 26), + width=2, + ) + fx, fz = c.forward() + tip = xy(c.x + fx * 0.7, c.z + fz * 0.7) + draw.line([(cx, cz), tip], fill=(30, 30, 40), width=3) + return img + + def _ray_aabb( ox: float, oz: float,