live: real-time browser viewer (AICC-client based), sensor-built top-down map shared module, run_live.sh one-command mode

2026-08-08 12:12:21 +03:00
parent 1ebccb660b
commit 9f877339f8
5 changed files with 401 additions and 50 deletions
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Real-time mode: starts the bridge + the live browser viewer, then runs the
+# demo while you watch. Open http://127.0.0.1:8000 in a browser.
+# Usage: scripts/run_live.sh [demo args...]   e.g. --agent scripted
+set -euo pipefail
+cd "$(dirname "$0")/.."
+BRIDGE_PID=""
+LIVE_PID=""
+
+# 1. bridge (reuse an already-running one)
+if python3 -c 'import socket,sys; s=socket.socket(); sys.exit(0 if s.connect_ex(("127.0.0.1",8765))==0 else 1)'; then
+  echo "[live] bridge already running on 8765"
+else
+  testbed/.venv/bin/python -m testbed.bridge --port 8765 >/tmp/aicc-bridge.log 2>&1 &
+  BRIDGE_PID=$!
+  sleep 1.5
+  echo "[live] bridge started (pid $BRIDGE_PID)"
+fi
+
+# 2. live viewer
+testbed/.venv/bin/python -m testbed.live --http-port 8000 --ws-port 8001 >/tmp/aicc-live.log 2>&1 &
+LIVE_PID=$!
+sleep 1.5
+echo "[live] viewer up: open http://127.0.0.1:8000 in your browser"
+
+# 3. demo (foreground; pass through agent args)
+testbed/.venv/bin/python -m testbed.demo "$@"
+EXIT=$?
+
+kill "$LIVE_PID" 2>/dev/null || true
+if [ -n "$BRIDGE_PID" ]; then kill "$BRIDGE_PID" 2>/dev/null || true; fi
+exit $EXIT
@@ -17,9 +17,11 @@ over the protocol, no engine hooks.
 |-------------------------|--------------------------------------------------------------|
 | `room/world.py`         | World state: 16x16 room, capsule physics, crates, beacon, audio. Single source of truth. |
 | `room/render.py`        | Headless first-person raycaster (Pillow): honest frames from world state. |
+| `room/mapview.py`       | Top-down map drawn from sensor data (shared by recorder + live viewer). |
 | `bridge.py`             | `RoomBridge(Bridge)`: registers all tools, emits events.     |
 | `server.py` entry       | `python -m testbed.bridge` — WebSocket server.               |
 | `conformance.py`        | Runs the 9 core conformance scenarios against this bridge.   |
+| `live.py`               | Real-time browser viewer (itself an AICC client).            |
 | `demo.py`               | Agent demo: LLM driver (OpenAI-compatible) or scripted.      |
 | `tests/`                | pytest suite (world, renderer, bridge, protocol).            |

@@ -60,6 +62,28 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal.
 The demo prints a full transcript of tool calls/results to stdout and saves the
 capsule's final first-person frame to `demo_final_frame.png`.

+## Real-time mode
+
+Watch the capsule drive live in your browser:
+
+```bash
+scripts/run_live.sh --agent scripted
+```
+
+This starts the bridge, a viewer server, and the demo; open
+[http://127.0.0.1:8000](http://127.0.0.1:8000) while the agent acts. The page
+shows the first-person frame (`vision`) and a top-down map (`world_query` +
+`proprioception`) updating a few times per second, with the capsule's path,
+heading, and distance to the beacon.
+
+The viewer (`testbed/live.py`) is itself a plain AICC client — it sees the
+world only through the protocol sensors, so it works against any bridge, not
+just this one. You can also run it standalone:
+
+```bash
+testbed/.venv/bin/python -m testbed.live            # then run the demo in another terminal
+```
+
 ## Visual mode

 ```bash
@@ -135,57 +135,9 @@ class FrameRecorder:
        return Image.open(io.BytesIO(base64.b64decode(out["png_b64"]))).convert("RGB")

    def _map_frame(self, pos: tuple[float, float], yaw: float) -> Image.Image:
-        from PIL import Image, ImageDraw
+        from testbed.room.mapview import draw_sensor_map

-        size = self.MAP_SIZE
-        scale = size / self.ROOM
-
-        def xy(x: float, z: float) -> tuple[float, float]:
-            return (x * scale, size - z * scale)
-
-        img = Image.new("RGB", (size, size), (52, 52, 58))
-        d = ImageDraw.Draw(img)
-        for i in range(int(self.ROOM) + 1):
-            c = 60 if i % 2 == 0 else 54
-            d.line([xy(i, 0), xy(i, self.ROOM)], fill=(c, c, c + 6), width=1)
-            d.line([xy(0, i), xy(self.ROOM, i)], fill=(c, c, c + 6), width=1)
-        if self.layout:
-            for ob in self.layout.get("obstacles", []):
-                x0, z0 = xy(ob["x"] - ob["width"] / 2, ob["z"] - ob["depth"] / 2)
-                x1, z1 = xy(ob["x"] + ob["width"] / 2, ob["z"] + ob["depth"] / 2)
-                d.rectangle(
-                    [min(x0, x1), min(z0, z1), max(x0, x1), max(z0, z1)],
-                    fill=(150, 90, 60),
-                    outline=(20, 20, 26),
-                    width=2,
-                )
-            b = self.layout.get("beacon", {})
-            bx, bz = xy(b.get("x", 12.5), b.get("z", 12.5))
-            d.ellipse(
-                [bx - 8, bz - 8, bx + 8, bz + 8],
-                fill=(120, 210, 235),
-                outline=(20, 20, 26),
-                width=2,
-            )
-        if len(self.path) > 1:
-            pts = [xy(x, z) for x, z in self.path]
-            d.line(pts, fill=(255, 170, 60), width=3)
-        cx, cz = xy(*pos)
-        d.ellipse(
-            [cx - 7, cz - 7, cx + 7, cz + 7],
-            fill=(240, 240, 250),
-            outline=(20, 20, 26),
-            width=2,
-        )
-        rad = math.radians(yaw)
-        tip = xy(pos[0] + math.sin(rad) * 0.7, pos[1] + math.cos(rad) * 0.7)
-        d.line([(cx, cz), tip], fill=(30, 30, 40), width=3)
-        d.text(
-            (8, 8),
-            f"step {self.step}  pos ({pos[0]:.1f}, {pos[1]:.1f}) yaw {yaw:.0f}",
-            fill=(230, 230, 230),
-        )
-        return img
+        return draw_sensor_map(self.layout, pos, yaw, self.path, step=self.step)

    def finish(self, interacted: bool) -> Image.Image:
        """Write demo.gif + demo_summary.png; returns the summary image."""
@@ -0,0 +1,255 @@
+"""Real-time viewer: watch the capsule drive, live in your browser.
+
+The viewer is itself a plain AICC client: it polls the bridge through the
+protocol sensors (world_query once, proprioception + vision a few times per
+second), renders a top-down map from what it observes, and pushes the frames
+to any number of browser tabs over its own WebSocket.
+
+Run it, then open http://127.0.0.1:8000 and start the demo in another
+terminal (scripts/run_demo.sh) — you will see the capsule move in real time.
+
+Usage:
+    python -m testbed.live [--bridge ws://127.0.0.1:8765] [--http-port 8000]
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import io
+import json
+import math
+from typing import Any
+
+from aicc.client import AICCClient
+from aicc.transport.websocket import WebSocketClientTransport
+
+from testbed.room.mapview import draw_sensor_map
+
+HTTP_PORT = 8000
+WS_PORT = 8001
+POLL_SECONDS = 0.25  # ~4 fps
+
+HTML = """<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>AICC capsule — live view</title>
+<style>
+  body { background:#101016; color:#d8d8e0; font-family: ui-monospace, monospace; margin:0; }
+  header { padding:10px 18px; background:#181820; border-bottom:1px solid #2a2a36;
+           display:flex; gap:24px; align-items:baseline; font-size:13px; }
+  header h1 { font-size:15px; margin:0; color:#8ecbff; }
+  header .meta { color:#9aa0b0; }
+  main { display:flex; gap:14px; padding:14px; flex-wrap:wrap; }
+  .panel { background:#181820; border:1px solid #2a2a36; border-radius:8px; padding:10px; }
+  .panel h2 { margin:0 0 8px; font-size:12px; color:#ffb860; font-weight:600;
+              text-transform:uppercase; letter-spacing:.08em; }
+  img { display:block; image-rendering: pixelated; background:#000; }
+  #view { width: 400px; }
+  #map  { width: 400px; }
+  #status { color:#9fe8a0; }
+</style>
+</head>
+<body>
+<header>
+  <h1>AICC capsule — real-time view</h1>
+  <span class="meta" id="status">connecting…</span>
+</header>
+<main>
+  <div class="panel"><h2>first person (vision)</h2><img id="view" alt="vision"></div>
+  <div class="panel"><h2>top-down (sensors)</h2><img id="map" alt="map"></div>
+</main>
+<script>
+const view = document.getElementById('view');
+const map = document.getElementById('map');
+const status = document.getElementById('status');
+let lastTick = 0, tickCount = 0, t0 = performance.now();
+function show(msg, kind) {
+  status.textContent = msg;
+  status.style.color = kind === 'err' ? '#ff9090' : '#9fe8a0';
+}
+function connect() {
+  const ws = new WebSocket(`ws://${location.hostname}:${location.port === '8000' ? '8001' : location.port}`);
+  ws.onopen = () => show('connected — waiting for frames');
+  ws.onclose = () => { show('disconnected — retrying…', 'err'); setTimeout(connect, 1000); };
+  ws.onmessage = (ev) => {
+    const m = JSON.parse(ev.data);
+    if (m.type === 'view') view.src = 'data:image/png;base64,' + m.png_b64;
+    if (m.type === 'map') {
+      map.src = 'data:image/png;base64,' + m.png_b64;
+      tickCount++;
+      const fps = tickCount / ((performance.now() - t0) / 1000);
+      show(`tick ${m.tick}  pos (${m.pos[0].toFixed(2)}, ${m.pos[1].toFixed(2)})  ` +
+           `yaw ${m.yaw.toFixed(1)}°  ${m.dist.toFixed(2)} m to beacon  ·  ${fps.toFixed(1)} fps`);
+    }
+  };
+}
+connect();
+</script>
+</body>
+</html>
+"""
+
+
+def _png_b64(img) -> str:
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+class LiveViewer:
+    """Polls the bridge via AICC sensors and broadcasts frames to browsers."""
+
+    def __init__(
+        self,
+        bridge_url: str,
+        http_port: int,
+        ws_port: int,
+        poll_seconds: float = POLL_SECONDS,
+    ):
+        self.bridge_url = bridge_url
+        self.http_port = http_port
+        self.ws_port = ws_port
+        self.poll = poll_seconds
+        self.clients: set[Any] = set()
+        self.layout: dict[str, Any] | None = None
+        self.path: list[tuple[float, float]] = []
+
+    # -- sensor polling -----------------------------------------------------
+
+    async def _sense(self, client: AICCClient) -> None:
+        while True:
+            try:
+                if self.layout is None:
+                    self.layout = (await client.call_tool("world_query", {})).output
+                prop = (await client.call_tool("proprioception", {})).output
+                vision = (await client.call_tool("vision", {})).output
+                pos = prop["position"]
+                yaw = prop["rotation"]["yaw_deg"] % 360.0
+                self.path.append((pos["x"], pos["z"]))
+                if len(self.path) > 4000:
+                    self.path = self.path[-2000:]
+                beacon = self.layout.get("beacon", {}) if self.layout else {}
+                bx = beacon.get("x", 12.5)
+                bz = beacon.get("z", 12.5)
+                dist = math.hypot(bx - pos["x"], bz - pos["z"])
+                await self._broadcast(
+                    {
+                        "type": "view",
+                        "png_b64": vision["png_b64"],
+                        "tick": vision["tick"],
+                    }
+                )
+                map_img = draw_sensor_map(
+                    self.layout, (pos["x"], pos["z"]), yaw, self.path, step=prop["tick"]
+                )
+                await self._broadcast(
+                    {
+                        "type": "map",
+                        "png_b64": _png_b64(map_img),
+                        "tick": prop["tick"],
+                        "pos": [pos["x"], pos["z"]],
+                        "yaw": yaw,
+                        "dist": dist,
+                    }
+                )
+            except asyncio.CancelledError:
+                raise
+            except Exception as exc:  # noqa: BLE001 - keep polling through transient errors
+                print(f"[live] sense error: {type(exc).__name__}: {exc}")
+            await asyncio.sleep(self.poll)
+
+    async def _broadcast(self, msg: dict[str, Any]) -> None:
+        if not self.clients:
+            return
+        data = json.dumps(msg)
+        for ws in list(self.clients):
+            try:
+                await ws.send(data)
+            except Exception:  # noqa: BLE001
+                self.clients.discard(ws)
+
+    # -- servers ------------------------------------------------------------
+
+    async def run(self) -> None:
+        import websockets
+
+        async def handler(ws) -> None:
+            self.clients.add(ws)
+            try:
+                await ws.wait_closed()
+            finally:
+                self.clients.discard(ws)
+
+        async def http_page(
+            reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+        ) -> None:
+            try:
+                await reader.read(4096)
+                body = HTML.encode("utf-8")
+                writer.write(
+                    b"HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\n"
+                    + f"Content-Length: {len(body)}\r\n\r\n".encode()
+                    + body
+                )
+                await writer.drain()
+            finally:
+                writer.close()
+
+        async def bridge_loop() -> None:
+            while True:
+                try:
+                    transport = WebSocketClientTransport(self.bridge_url)
+                    async with AICCClient(transport) as client:
+                        await client.handshake()
+                        print(f"[live] connected to bridge {self.bridge_url}")
+                        await self._sense(client)
+                except asyncio.CancelledError:
+                    raise
+                except Exception as exc:  # noqa: BLE001 - wait for the bridge
+                    print(f"[live] bridge not reachable: {type(exc).__name__}: {exc}")
+                await asyncio.sleep(1.0)
+
+        ws_server = await websockets.serve(handler, "127.0.0.1", self.ws_port)
+        http_server = await asyncio.start_server(http_page, "127.0.0.1", self.http_port)
+        print(
+            f"[live] open http://127.0.0.1:{self.http_port}  (frames on ws://127.0.0.1:{self.ws_port})"
+        )
+        print(
+            "[live] start the demo in another terminal:  scripts/run_demo.sh --agent scripted"
+        )
+        poll_task = asyncio.create_task(bridge_loop())
+        try:
+            await asyncio.sleep(3600)
+        finally:
+            poll_task.cancel()
+            ws_server.close()
+            http_server.close()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Real-time browser viewer for the AICC capsule testbed."
+    )
+    parser.add_argument("--bridge", default="ws://127.0.0.1:8765")
+    parser.add_argument("--http-port", type=int, default=HTTP_PORT)
+    parser.add_argument("--ws-port", type=int, default=WS_PORT)
+    parser.add_argument(
+        "--poll",
+        type=float,
+        default=POLL_SECONDS,
+        help="sensor poll interval in seconds",
+    )
+    args = parser.parse_args()
+    viewer = LiveViewer(args.bridge, args.http_port, args.ws_port, args.poll)
+    try:
+        asyncio.run(viewer.run())
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,88 @@
+"""Top-down map drawing from sensor data.
+
+Shared by the demo's frame recorder and the live viewer. Only uses what an
+agent can observe: the world_query layout (room, obstacles, beacon) and
+proprioception (position, heading).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from PIL import Image, ImageDraw
+
+ROOM_SIZE = 16.0
+MAP_SIZE = 320
+SCALE = MAP_SIZE / ROOM_SIZE
+
+
+def draw_sensor_map(
+    layout: dict[str, Any] | None,
+    pos: tuple[float, float],
+    yaw: float,
+    path: list[tuple[float, float]] | None = None,
+    *,
+    step: int | None = None,
+    beacon_active: bool = False,
+) -> Image.Image:
+    """Draw the room from the agent's sensor view: floor grid, crates, the
+    beacon, the capsule's path and current position/heading."""
+    path = path or []
+
+    def xy(x: float, z: float) -> tuple[float, float]:
+        return (x * SCALE, MAP_SIZE - z * SCALE)
+
+    img = Image.new("RGB", (MAP_SIZE, MAP_SIZE), (52, 52, 58))
+    d = ImageDraw.Draw(img)
+    for i in range(int(ROOM_SIZE) + 1):
+        c = 60 if i % 2 == 0 else 54
+        d.line([xy(i, 0), xy(i, ROOM_SIZE)], fill=(c, c, c + 6), width=1)
+        d.line([xy(0, i), xy(ROOM_SIZE, i)], fill=(c, c, c + 6), width=1)
+
+    if layout:
+        for ob in layout.get("obstacles", []):
+            x0, z0 = xy(ob["x"] - ob["width"] / 2, ob["z"] - ob["depth"] / 2)
+            x1, z1 = xy(ob["x"] + ob["width"] / 2, ob["z"] + ob["depth"] / 2)
+            d.rectangle(
+                [min(x0, x1), min(z0, z1), max(x0, x1), max(z0, z1)],
+                fill=(150, 90, 60),
+                outline=(20, 20, 26),
+                width=2,
+            )
+        b = layout.get("beacon", {})
+        bx, bz = xy(b.get("x", 12.5), b.get("z", 12.5))
+        core = (255, 200, 90) if beacon_active else (120, 210, 235)
+        glow = (255, 240, 180) if beacon_active else (160, 235, 250)
+        d.ellipse([bx - 14, bz - 14, bx + 14, bz + 14], fill=glow)
+        d.ellipse(
+            [bx - 7, bz - 7, bx + 7, bz + 7], fill=core, outline=(20, 20, 26), width=2
+        )
+
+    if len(path) > 1:
+        pts = [xy(x, z) for x, z in path]
+        d.line(pts, fill=(255, 170, 60), width=3)
+
+    cx, cz = xy(*pos)
+    d.ellipse(
+        [cx - 7, cz - 7, cx + 7, cz + 7],
+        fill=(240, 240, 250),
+        outline=(20, 20, 26),
+        width=2,
+    )
+    fx, fz = _forward(yaw)
+    tip = xy(pos[0] + fx * 0.7, pos[1] + fz * 0.7)
+    d.line([(cx, cz), tip], fill=(30, 30, 40), width=3)
+    if step is not None:
+        d.text(
+            (8, 8),
+            f"step {step}  pos ({pos[0]:.1f}, {pos[1]:.1f}) yaw {yaw:.0f}",
+            fill=(230, 230, 230),
+        )
+    return img
+
+
+def _forward(yaw_deg: float) -> tuple[float, float]:
+    import math
+
+    rad = math.radians(yaw_deg)
+    return math.sin(rad), math.cos(rad)