diff --git a/scripts/run_live.sh b/scripts/run_live.sh new file mode 100755 index 0000000..a860c9e --- /dev/null +++ b/scripts/run_live.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Real-time mode: starts the bridge + the live browser viewer, then runs the +# demo while you watch. Open http://127.0.0.1:8000 in a browser. +# Usage: scripts/run_live.sh [demo args...] e.g. --agent scripted +set -euo pipefail +cd "$(dirname "$0")/.." +BRIDGE_PID="" +LIVE_PID="" + +# 1. bridge (reuse an already-running one) +if python3 -c 'import socket,sys; s=socket.socket(); sys.exit(0 if s.connect_ex(("127.0.0.1",8765))==0 else 1)'; then + echo "[live] bridge already running on 8765" +else + testbed/.venv/bin/python -m testbed.bridge --port 8765 >/tmp/aicc-bridge.log 2>&1 & + BRIDGE_PID=$! + sleep 1.5 + echo "[live] bridge started (pid $BRIDGE_PID)" +fi + +# 2. live viewer +testbed/.venv/bin/python -m testbed.live --http-port 8000 --ws-port 8001 >/tmp/aicc-live.log 2>&1 & +LIVE_PID=$! +sleep 1.5 +echo "[live] viewer up: open http://127.0.0.1:8000 in your browser" + +# 3. demo (foreground; pass through agent args) +testbed/.venv/bin/python -m testbed.demo "$@" +EXIT=$? + +kill "$LIVE_PID" 2>/dev/null || true +if [ -n "$BRIDGE_PID" ]; then kill "$BRIDGE_PID" 2>/dev/null || true; fi +exit $EXIT diff --git a/testbed/README.md b/testbed/README.md index 3aabb0c..0803dab 100644 --- a/testbed/README.md +++ b/testbed/README.md @@ -17,9 +17,11 @@ over the protocol, no engine hooks. |-------------------------|--------------------------------------------------------------| | `room/world.py` | World state: 16x16 room, capsule physics, crates, beacon, audio. Single source of truth. | | `room/render.py` | Headless first-person raycaster (Pillow): honest frames from world state. | +| `room/mapview.py` | Top-down map drawn from sensor data (shared by recorder + live viewer). | | `bridge.py` | `RoomBridge(Bridge)`: registers all tools, emits events. | | `server.py` entry | `python -m testbed.bridge` — WebSocket server. | | `conformance.py` | Runs the 9 core conformance scenarios against this bridge. | +| `live.py` | Real-time browser viewer (itself an AICC client). | | `demo.py` | Agent demo: LLM driver (OpenAI-compatible) or scripted. | | `tests/` | pytest suite (world, renderer, bridge, protocol). | @@ -60,6 +62,28 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal. The demo prints a full transcript of tool calls/results to stdout and saves the capsule's final first-person frame to `demo_final_frame.png`. +## Real-time mode + +Watch the capsule drive live in your browser: + +```bash +scripts/run_live.sh --agent scripted +``` + +This starts the bridge, a viewer server, and the demo; open +[http://127.0.0.1:8000](http://127.0.0.1:8000) while the agent acts. The page +shows the first-person frame (`vision`) and a top-down map (`world_query` + +`proprioception`) updating a few times per second, with the capsule's path, +heading, and distance to the beacon. + +The viewer (`testbed/live.py`) is itself a plain AICC client — it sees the +world only through the protocol sensors, so it works against any bridge, not +just this one. You can also run it standalone: + +```bash +testbed/.venv/bin/python -m testbed.live # then run the demo in another terminal +``` + ## Visual mode ```bash diff --git a/testbed/demo.py b/testbed/demo.py index 0931495..f98d094 100644 --- a/testbed/demo.py +++ b/testbed/demo.py @@ -135,57 +135,9 @@ class FrameRecorder: return Image.open(io.BytesIO(base64.b64decode(out["png_b64"]))).convert("RGB") def _map_frame(self, pos: tuple[float, float], yaw: float) -> Image.Image: - from PIL import Image, ImageDraw + from testbed.room.mapview import draw_sensor_map - size = self.MAP_SIZE - scale = size / self.ROOM - - def xy(x: float, z: float) -> tuple[float, float]: - return (x * scale, size - z * scale) - - img = Image.new("RGB", (size, size), (52, 52, 58)) - d = ImageDraw.Draw(img) - for i in range(int(self.ROOM) + 1): - c = 60 if i % 2 == 0 else 54 - d.line([xy(i, 0), xy(i, self.ROOM)], fill=(c, c, c + 6), width=1) - d.line([xy(0, i), xy(self.ROOM, i)], fill=(c, c, c + 6), width=1) - if self.layout: - for ob in self.layout.get("obstacles", []): - x0, z0 = xy(ob["x"] - ob["width"] / 2, ob["z"] - ob["depth"] / 2) - x1, z1 = xy(ob["x"] + ob["width"] / 2, ob["z"] + ob["depth"] / 2) - d.rectangle( - [min(x0, x1), min(z0, z1), max(x0, x1), max(z0, z1)], - fill=(150, 90, 60), - outline=(20, 20, 26), - width=2, - ) - b = self.layout.get("beacon", {}) - bx, bz = xy(b.get("x", 12.5), b.get("z", 12.5)) - d.ellipse( - [bx - 8, bz - 8, bx + 8, bz + 8], - fill=(120, 210, 235), - outline=(20, 20, 26), - width=2, - ) - if len(self.path) > 1: - pts = [xy(x, z) for x, z in self.path] - d.line(pts, fill=(255, 170, 60), width=3) - cx, cz = xy(*pos) - d.ellipse( - [cx - 7, cz - 7, cx + 7, cz + 7], - fill=(240, 240, 250), - outline=(20, 20, 26), - width=2, - ) - rad = math.radians(yaw) - tip = xy(pos[0] + math.sin(rad) * 0.7, pos[1] + math.cos(rad) * 0.7) - d.line([(cx, cz), tip], fill=(30, 30, 40), width=3) - d.text( - (8, 8), - f"step {self.step} pos ({pos[0]:.1f}, {pos[1]:.1f}) yaw {yaw:.0f}", - fill=(230, 230, 230), - ) - return img + return draw_sensor_map(self.layout, pos, yaw, self.path, step=self.step) def finish(self, interacted: bool) -> Image.Image: """Write demo.gif + demo_summary.png; returns the summary image.""" diff --git a/testbed/live.py b/testbed/live.py new file mode 100644 index 0000000..a0a358c --- /dev/null +++ b/testbed/live.py @@ -0,0 +1,255 @@ +"""Real-time viewer: watch the capsule drive, live in your browser. + +The viewer is itself a plain AICC client: it polls the bridge through the +protocol sensors (world_query once, proprioception + vision a few times per +second), renders a top-down map from what it observes, and pushes the frames +to any number of browser tabs over its own WebSocket. + +Run it, then open http://127.0.0.1:8000 and start the demo in another +terminal (scripts/run_demo.sh) — you will see the capsule move in real time. + +Usage: + python -m testbed.live [--bridge ws://127.0.0.1:8765] [--http-port 8000] +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import io +import json +import math +from typing import Any + +from aicc.client import AICCClient +from aicc.transport.websocket import WebSocketClientTransport + +from testbed.room.mapview import draw_sensor_map + +HTTP_PORT = 8000 +WS_PORT = 8001 +POLL_SECONDS = 0.25 # ~4 fps + +HTML = """ + + + +AICC capsule — live view + + + +
+

AICC capsule — real-time view

+ connecting… +
+
+

first person (vision)

vision
+

top-down (sensors)

map
+
+ + + +""" + + +def _png_b64(img) -> str: + buf = io.BytesIO() + img.save(buf, format="PNG") + return base64.b64encode(buf.getvalue()).decode("ascii") + + +class LiveViewer: + """Polls the bridge via AICC sensors and broadcasts frames to browsers.""" + + def __init__( + self, + bridge_url: str, + http_port: int, + ws_port: int, + poll_seconds: float = POLL_SECONDS, + ): + self.bridge_url = bridge_url + self.http_port = http_port + self.ws_port = ws_port + self.poll = poll_seconds + self.clients: set[Any] = set() + self.layout: dict[str, Any] | None = None + self.path: list[tuple[float, float]] = [] + + # -- sensor polling ----------------------------------------------------- + + async def _sense(self, client: AICCClient) -> None: + while True: + try: + if self.layout is None: + self.layout = (await client.call_tool("world_query", {})).output + prop = (await client.call_tool("proprioception", {})).output + vision = (await client.call_tool("vision", {})).output + pos = prop["position"] + yaw = prop["rotation"]["yaw_deg"] % 360.0 + self.path.append((pos["x"], pos["z"])) + if len(self.path) > 4000: + self.path = self.path[-2000:] + beacon = self.layout.get("beacon", {}) if self.layout else {} + bx = beacon.get("x", 12.5) + bz = beacon.get("z", 12.5) + dist = math.hypot(bx - pos["x"], bz - pos["z"]) + await self._broadcast( + { + "type": "view", + "png_b64": vision["png_b64"], + "tick": vision["tick"], + } + ) + map_img = draw_sensor_map( + self.layout, (pos["x"], pos["z"]), yaw, self.path, step=prop["tick"] + ) + await self._broadcast( + { + "type": "map", + "png_b64": _png_b64(map_img), + "tick": prop["tick"], + "pos": [pos["x"], pos["z"]], + "yaw": yaw, + "dist": dist, + } + ) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - keep polling through transient errors + print(f"[live] sense error: {type(exc).__name__}: {exc}") + await asyncio.sleep(self.poll) + + async def _broadcast(self, msg: dict[str, Any]) -> None: + if not self.clients: + return + data = json.dumps(msg) + for ws in list(self.clients): + try: + await ws.send(data) + except Exception: # noqa: BLE001 + self.clients.discard(ws) + + # -- servers ------------------------------------------------------------ + + async def run(self) -> None: + import websockets + + async def handler(ws) -> None: + self.clients.add(ws) + try: + await ws.wait_closed() + finally: + self.clients.discard(ws) + + async def http_page( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + try: + await reader.read(4096) + body = HTML.encode("utf-8") + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\n" + + f"Content-Length: {len(body)}\r\n\r\n".encode() + + body + ) + await writer.drain() + finally: + writer.close() + + async def bridge_loop() -> None: + while True: + try: + transport = WebSocketClientTransport(self.bridge_url) + async with AICCClient(transport) as client: + await client.handshake() + print(f"[live] connected to bridge {self.bridge_url}") + await self._sense(client) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - wait for the bridge + print(f"[live] bridge not reachable: {type(exc).__name__}: {exc}") + await asyncio.sleep(1.0) + + ws_server = await websockets.serve(handler, "127.0.0.1", self.ws_port) + http_server = await asyncio.start_server(http_page, "127.0.0.1", self.http_port) + print( + f"[live] open http://127.0.0.1:{self.http_port} (frames on ws://127.0.0.1:{self.ws_port})" + ) + print( + "[live] start the demo in another terminal: scripts/run_demo.sh --agent scripted" + ) + poll_task = asyncio.create_task(bridge_loop()) + try: + await asyncio.sleep(3600) + finally: + poll_task.cancel() + ws_server.close() + http_server.close() + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Real-time browser viewer for the AICC capsule testbed." + ) + parser.add_argument("--bridge", default="ws://127.0.0.1:8765") + parser.add_argument("--http-port", type=int, default=HTTP_PORT) + parser.add_argument("--ws-port", type=int, default=WS_PORT) + parser.add_argument( + "--poll", + type=float, + default=POLL_SECONDS, + help="sensor poll interval in seconds", + ) + args = parser.parse_args() + viewer = LiveViewer(args.bridge, args.http_port, args.ws_port, args.poll) + try: + asyncio.run(viewer.run()) + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/testbed/room/mapview.py b/testbed/room/mapview.py new file mode 100644 index 0000000..d6fa06d --- /dev/null +++ b/testbed/room/mapview.py @@ -0,0 +1,88 @@ +"""Top-down map drawing from sensor data. + +Shared by the demo's frame recorder and the live viewer. Only uses what an +agent can observe: the world_query layout (room, obstacles, beacon) and +proprioception (position, heading). +""" + +from __future__ import annotations + +from typing import Any + +from PIL import Image, ImageDraw + +ROOM_SIZE = 16.0 +MAP_SIZE = 320 +SCALE = MAP_SIZE / ROOM_SIZE + + +def draw_sensor_map( + layout: dict[str, Any] | None, + pos: tuple[float, float], + yaw: float, + path: list[tuple[float, float]] | None = None, + *, + step: int | None = None, + beacon_active: bool = False, +) -> Image.Image: + """Draw the room from the agent's sensor view: floor grid, crates, the + beacon, the capsule's path and current position/heading.""" + path = path or [] + + def xy(x: float, z: float) -> tuple[float, float]: + return (x * SCALE, MAP_SIZE - z * SCALE) + + img = Image.new("RGB", (MAP_SIZE, MAP_SIZE), (52, 52, 58)) + d = ImageDraw.Draw(img) + for i in range(int(ROOM_SIZE) + 1): + c = 60 if i % 2 == 0 else 54 + d.line([xy(i, 0), xy(i, ROOM_SIZE)], fill=(c, c, c + 6), width=1) + d.line([xy(0, i), xy(ROOM_SIZE, i)], fill=(c, c, c + 6), width=1) + + if layout: + for ob in layout.get("obstacles", []): + x0, z0 = xy(ob["x"] - ob["width"] / 2, ob["z"] - ob["depth"] / 2) + x1, z1 = xy(ob["x"] + ob["width"] / 2, ob["z"] + ob["depth"] / 2) + d.rectangle( + [min(x0, x1), min(z0, z1), max(x0, x1), max(z0, z1)], + fill=(150, 90, 60), + outline=(20, 20, 26), + width=2, + ) + b = layout.get("beacon", {}) + bx, bz = xy(b.get("x", 12.5), b.get("z", 12.5)) + core = (255, 200, 90) if beacon_active else (120, 210, 235) + glow = (255, 240, 180) if beacon_active else (160, 235, 250) + d.ellipse([bx - 14, bz - 14, bx + 14, bz + 14], fill=glow) + d.ellipse( + [bx - 7, bz - 7, bx + 7, bz + 7], fill=core, outline=(20, 20, 26), width=2 + ) + + if len(path) > 1: + pts = [xy(x, z) for x, z in path] + d.line(pts, fill=(255, 170, 60), width=3) + + cx, cz = xy(*pos) + d.ellipse( + [cx - 7, cz - 7, cx + 7, cz + 7], + fill=(240, 240, 250), + outline=(20, 20, 26), + width=2, + ) + fx, fz = _forward(yaw) + tip = xy(pos[0] + fx * 0.7, pos[1] + fz * 0.7) + d.line([(cx, cz), tip], fill=(30, 30, 40), width=3) + if step is not None: + d.text( + (8, 8), + f"step {step} pos ({pos[0]:.1f}, {pos[1]:.1f}) yaw {yaw:.0f}", + fill=(230, 230, 230), + ) + return img + + +def _forward(yaw_deg: float) -> tuple[float, float]: + import math + + rad = math.radians(yaw_deg) + return math.sin(rad), math.cos(rad)