diff --git a/README.md b/README.md index 4290111..191cf99 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,18 @@ model saw, and its chat/reasoning: Regenerate it with `python -m testbed.record_search --provider polza --out search_mission.gif`. +## Smooth & proactive movement + +The same recording setup with a beacon mission and the cruise mode — the +capsule glides with animated `move`/`turn` calls and keeps moving while the +model thinks (`--cruise`). Frames are sampled at ~2.5 Hz by an observer +client, so the GIF shows the actual motion, not step endpoints: + +![smooth mission](smooth_mission.gif) + +Regenerate with `python -m testbed.record_search --provider polza --goal beacon +--out smooth_mission.gif --capture-hz 2.5 --cruise 1.0`. + ## Layout ``` diff --git a/smooth_mission.gif b/smooth_mission.gif new file mode 100644 index 0000000..f0290bd Binary files /dev/null and b/smooth_mission.gif differ diff --git a/testbed/record_search.py b/testbed/record_search.py index 24682bb..b356663 100644 --- a/testbed/record_search.py +++ b/testbed/record_search.py @@ -139,24 +139,22 @@ async def run(args: argparse.Namespace) -> int: api_key=args.api_key, model=args.model, ) + from testbed.llm_agent import MISSION + bridge = build_bridge() - # Fixed marker so the GIF tells a clear story (crate_blue, east face). - bridge.world.marker = { - "box": "crate_blue", - "face": "x1", - "u": 0.5, - "v": 0.55, - "size": 0.42, - "color": (255, 150, 40), - } - m = bridge.world.marker_world_pos() - print( - f"[record] triangle marker on crate_blue east face at {tuple(round(v, 2) for v in m)}" - ) + if args.goal == "search": + # Fixed marker so the GIF tells a clear story (crate_blue, east face). + bridge.world.marker = { + "box": "crate_blue", "face": "x1", "u": 0.5, "v": 0.55, + "size": 0.42, "color": (255, 150, 40), + } + m = bridge.world.marker_world_pos() + print(f"[record] triangle marker on crate_blue east face at {tuple(round(v, 2) for v in m)}") frames: list[Image.Image] = [] chat = ChatPane() marker_seen = False + samples: list[tuple[float, float]] = [] # capsule positions per captured frame def log(role: str, msg: str) -> None: print(f" [{role}] {msg}") @@ -168,62 +166,96 @@ async def run(args: argparse.Namespace) -> int: async with AICCClient(transport) as client: manifest = await client.handshake() ctl = LLMController( - client, - manifest, - base_url=base_url, - api_key=api_key, - model=model, - system_prompt=SEARCH_MISSION, + client, manifest, + base_url=base_url, api_key=api_key, model=model, + system_prompt=SEARCH_MISSION if args.goal == "search" else MISSION, log=log, ) - ctl.messages = [ - ctl.messages[0], - { - "role": "user", - "content": ( + if args.goal == "search": + ctl.messages = [ + ctl.messages[0], + {"role": "user", "content": ( "MISSION: explore the room, find the orange triangle on the " "back of a crate, and report it. Keep exploring until found." + )}, + ] + chat.add("mission", "MISSION: find the orange triangle") + else: + ctl.messages.append({ + "role": "user", + "content": ( + "MISSION: reach the beacon and activate it. " + "Keep calling tools until it is done. Move smoothly (use " + "duration on move/turn). The capsule cruises while you think." ), - }, - ] - chat.add("mission", "MISSION: find the orange triangle") - orig_frame = ctl._frame_message + }) + chat.add("mission", "MISSION: reach and activate the beacon") - def record_frame(png: str, note: str): - nonlocal marker_seen - view = ( - Image.open(io.BytesIO(base64.b64decode(png))) - .convert("RGB") - .resize((PANEL, PANEL)) - ) - top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen) - frames.append(_compose(top, view, chat.render())) - return orig_frame(png, note) + if args.goal == "search" and not args.capture_hz: + # capture at frame-attach events (one frame per step) + orig_frame = ctl._frame_message - ctl._frame_message = record_frame # type: ignore[method-assign] + def record_frame(png: str, note: str): + nonlocal marker_seen + view = ( + Image.open(io.BytesIO(base64.b64decode(png))) + .convert("RGB") + .resize((PANEL, PANEL)) + ) + top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen) + frames.append(_compose(top, view, chat.render())) + samples.append((bridge.world.capsule.x, bridge.world.capsule.z)) + return orig_frame(png, note) + + ctl._frame_message = record_frame # type: ignore[method-assign] def is_success(turn): nonlocal marker_seen for c in turn.calls: - if ( - c.name == "report" - and c.ok - and c.output - and c.output.get("verified") - ): + if c.name == "report" and c.ok and c.output and c.output.get("verified"): marker_seen = True return True return False - summary = await run_llm_agent_loop( - ctl, - args.steps, - log=log, - look_every=1, - cruise=args.cruise, - autonomous=True, - is_success=is_success, - ) + async def observer(agent_done: asyncio.Event) -> None: + """Sample the world at a fixed rate so the GIF shows the actual + gliding motion (mid-animation frames), not just step endpoints.""" + obs_transport = WebSocketClientTransport("ws://127.0.0.1:8765") + async with AICCClient(obs_transport) as obs: + await obs.handshake() + while not agent_done.is_set(): + try: + out = (await obs.call_tool("vision", {})).output + except Exception: # noqa: BLE001 - session teardown + break + view = ( + Image.open(io.BytesIO(base64.b64decode(out["png_b64"]))) + .convert("RGB") + .resize((PANEL, PANEL)) + ) + samples.append((bridge.world.capsule.x, bridge.world.capsule.z)) + top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen) + frames.append(_compose(top, view, chat.render())) + if len(frames) > 500: # keep the GIF bounded + frames[:100] = [] + await asyncio.sleep(1.0 / args.capture_hz) + + agent_done = asyncio.Event() + obs_task = None + if args.capture_hz: + obs_task = asyncio.create_task(observer(agent_done)) + try: + summary = await run_llm_agent_loop( + ctl, args.steps, log=log, + look_every=2 if args.goal == "beacon" else 1, + cruise=args.cruise, + autonomous=args.goal == "search", + is_success=is_success if args.goal == "search" else None, + ) + finally: + agent_done.set() + if obs_task is not None: + await obs_task # Final frame: end state of the map. if marker_seen: @@ -236,6 +268,19 @@ async def run(args: argparse.Namespace) -> int: ) frames.append(_compose(top, last_view, chat.render())) + # Smoothness check: mean movement between consecutive captured frames. + if len(samples) > 2: + from itertools import pairwise + + deltas = [ + ((b[0] - a[0]) ** 2 + (b[1] - a[1]) ** 2) ** 0.5 + for a, b in pairwise(samples) + ] + moving = [d for d in deltas if d > 0.02] + if moving: + print(f"[record] motion between frames: mean {sum(moving)/len(moving):.2f} m " + f"(max {max(moving):.2f}) over {len(moving)} moving samples") + out = Path(args.out) if frames: frames[0].save( @@ -248,21 +293,34 @@ async def run(args: argparse.Namespace) -> int: frames[-1].save(out.with_suffix(".png")) print(f"[record] wrote {out} ({len(frames)} frames, {args.duration} ms/frame)") print(f"[record] mission result: {summary.get('result')}") - return 0 if marker_seen else 1 + return 0 if (marker_seen or summary.get("interacted")) else 1 def main() -> int: parser = argparse.ArgumentParser( - description="Record a search mission into a three-view GIF." + description="Record a mission into a three-view GIF (map | first person | chat)." ) parser.add_argument("--out", default="search_mission.gif") + parser.add_argument( + "--goal", + choices=["search", "beacon"], + default="search", + help="which mission to record (default search)", + ) parser.add_argument("--provider", default=None) parser.add_argument("--base-url", default=None) parser.add_argument("--api-key", default=None) parser.add_argument("--model", default=None) parser.add_argument("--steps", type=int, default=45) parser.add_argument("--cruise", type=float, default=0.6) - parser.add_argument("--duration", type=int, default=350, help="ms per GIF frame") + parser.add_argument("--duration", type=int, default=150, help="ms per GIF frame") + parser.add_argument( + "--capture-hz", + type=float, + default=0.0, + help="sample frames at this rate (Hz) with an observer client to show " + "gliding motion; 0 = one frame per step (search default)", + ) args = parser.parse_args() try: return asyncio.run(run(args))