From 315762363ebc1d4d8fa3fc672e2e49eaad9b4c2b Mon Sep 17 00:00:00 2001 From: opencode Date: Sat, 8 Aug 2026 21:36:32 +0300 Subject: [PATCH] movement: smooth animated moves (move duration, observable intermediate positions) + proactive cruise (glide while the LLM thinks, collision-safe, reported back); verified mission with gpt-5.6-luna --- testbed/README.md | 14 +++++++ testbed/bridge.py | 41 ++++++++++++++++++-- testbed/chat.py | 7 ++++ testbed/demo.py | 17 +++++++-- testbed/llm_agent.py | 72 ++++++++++++++++++++++++++++++++++-- testbed/tests/test_bridge.py | 61 ++++++++++++++++++++++++++++++ 6 files changed, 201 insertions(+), 11 deletions(-) diff --git a/testbed/README.md b/testbed/README.md index 1930684..b473931 100644 --- a/testbed/README.md +++ b/testbed/README.md @@ -64,6 +64,20 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal. The demo prints a full transcript of tool calls/results to stdout and saves the capsule's final first-person frame to `demo_final_frame.png`. +### Smooth & proactive movement + +The capsule no longer teleports or stops-and-thinks: + +- **Smooth**: `move(forward, duration)` animates the displacement over + `duration` seconds on the bridge, so the live viewer shows genuine gliding. + The scripted agent and auto-cruise use it by default; the LLM can too. +- **Proactive** (`--cruise N`, meters per think): while the model is + generating a response, the capsule keeps gliding forward (low-level + controller pattern, like a real robot). Collisions stop the drift safely + and are reported to the model, which re-plans. Verified: gpt-5.6-luna + completed the beacon mission with cruise on, bumping into and avoiding + crates while "thinking". + ## Providers Provider presets resolve endpoint + API key + default model: diff --git a/testbed/bridge.py b/testbed/bridge.py index 2a3653f..1ed0205 100644 --- a/testbed/bridge.py +++ b/testbed/bridge.py @@ -85,6 +85,7 @@ class MoveOutput(BaseModel): collision_normal: Vec3 | None position: Vec3 tick: int + duration: float = 0.0 class TurnOutput(BaseModel): @@ -165,6 +166,31 @@ class RoomBridge(Bridge): def _tick(self) -> int: return self.world.advance_tick() + async def _animate_move( + self, forward: float, duration: float + ) -> tuple[float, Any | None]: + """Smoothly displace the capsule over ``duration`` seconds. + + The position advances in small timed steps, so concurrent clients + (e.g. the live viewer) observe genuine gliding motion instead of a + teleport. Collision resolution is unchanged. + """ + world = self.world + n = max(1, min(int(duration / 0.1), 200)) + step = forward / n + dt = duration / n + total = 0.0 + hit: Any | None = None + for i in range(n): + moved, h = world.move_forward(step) + total += moved + if h is not None or moved < step * 0.01: + hit = h + break + if i < n - 1: + await asyncio.sleep(dt) + return total, hit + def _vec3(self, x: float, y: float, z: float) -> Vec3: return Vec3(x=round(x, 3), y=round(y, 3), z=round(z, 3)) @@ -263,13 +289,21 @@ class RoomBridge(Bridge): @self.tool( cls=ToolClass.ACTUATOR, - description="Move the capsule forward along its heading by the given distance in meters.", + description=( + "Move the capsule forward along its heading by the given distance in meters. " + "duration (seconds) animates the motion smoothly instead of teleporting." + ), ) - async def move(forward: float = 1.0) -> MoveOutput: + async def move(forward: float = 1.0, duration: float = 0.0) -> MoveOutput: tick = self._tick() if forward < 0.0 or forward > 5.0: raise ValueError(f"forward must be in [0, 5] m, got {forward}") - moved, hit = world.move_forward(forward) + if duration < 0.0 or duration > 10.0: + raise ValueError(f"duration must be in [0, 10] s, got {duration}") + if duration > 0.0: + moved, hit = await self._animate_move(forward, duration) + else: + moved, hit = world.move_forward(forward) c = world.capsule normal = None if hit is not None: @@ -294,6 +328,7 @@ class RoomBridge(Bridge): collision_normal=normal, position=self._vec3(c.x, 0.0, c.z), tick=tick, + duration=round(duration, 3), ).model_dump() @self.tool( diff --git a/testbed/chat.py b/testbed/chat.py index e9ce41f..c5f0541 100644 --- a/testbed/chat.py +++ b/testbed/chat.py @@ -164,6 +164,7 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i log=mission_log, nudge_limit=args.mission_nudges, look_every=args.look_every, + cruise=args.cruise, ) except RuntimeError as exc: print(f" [mission] LLM error: {exc}") @@ -463,6 +464,12 @@ def main() -> int: default=3, help="attach a fresh camera frame every N turns (0 disables; default 3)", ) + parser.add_argument( + "--cruise", + type=float, + default=1.0, + help="proactive motion in missions: glide forward (meters) while the LLM thinks (0 disables)", + ) args = parser.parse_args() try: args.base_url, args.api_key, args.model = resolve_provider( diff --git a/testbed/demo.py b/testbed/demo.py index 9ebe218..0c97def 100644 --- a/testbed/demo.py +++ b/testbed/demo.py @@ -142,6 +142,7 @@ async def run_llm_agent( vision: bool | None = None, digest: bool | None = None, look_every: int = 0, + cruise: float = 0.0, ) -> dict[str, Any]: """Autonomous LLM run: controller + nudge/correct loop (see llm_agent).""" from testbed.llm_agent import LLMController, run_llm_agent_loop @@ -233,9 +234,9 @@ async def run_scripted_agent( if detour_steps > 0: # Escape the obstacle before re-aiming: keep the detour heading. - mv = (await client.call_tool("move", {"forward": 0.8})).output + mv = (await client.call_tool("move", {"forward": 0.8, "duration": 0.6})).output summary["tool_calls"] += 1 - log("tool", 'move({"forward": 0.8}) [detour]') + log("tool", 'move({"forward": 0.8, "duration": 0.6}) [detour]') log("bridge", f"ok {json.dumps(mv)}") if mv.get("moved", 0.0) < 0.2 and detour_turns < 6: detour_turns += 1 @@ -267,9 +268,9 @@ async def run_scripted_agent( log("tool", f'turn({{"yaw_deg": {delta * 0.8:.1f}}})') log("bridge", f"ok {json.dumps(turn)}") else: - mv = (await client.call_tool("move", {"forward": 0.8})).output + mv = (await client.call_tool("move", {"forward": 0.8, "duration": 0.6})).output summary["tool_calls"] += 1 - log("tool", 'move({"forward": 0.8})') + log("tool", 'move({"forward": 0.8, "duration": 0.6})') log("bridge", f"ok {json.dumps(mv)}") if mv.get("collision"): consecutive_collisions += 1 @@ -315,6 +316,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]: vision=args.vision, digest=args.digest, look_every=args.look_every, + cruise=args.cruise, ) elif args.agent == "scripted": summary = await run_scripted_agent( @@ -338,6 +340,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]: vision=args.vision, digest=args.digest, look_every=args.look_every, + cruise=args.cruise, ) if summary.get("interacted"): return summary @@ -431,6 +434,12 @@ def main() -> int: default=3, help="attach a fresh camera frame every N agent steps (0 disables; default 3)", ) + parser.add_argument( + "--cruise", + type=float, + default=1.0, + help="proactive motion: glide forward (meters) while the LLM thinks (0 disables)", + ) parser.add_argument( "--frame", default="demo_final_frame.png", diff --git a/testbed/llm_agent.py b/testbed/llm_agent.py index 57740cf..b1752a4 100644 --- a/testbed/llm_agent.py +++ b/testbed/llm_agent.py @@ -9,6 +9,7 @@ tool calls/results fed back as `tool` messages. from __future__ import annotations +import asyncio import base64 import io import json @@ -463,10 +464,12 @@ class LLMController: async def send_user(self, text: str) -> None: self.messages.append({"role": "user", "content": text}) - async def invoke(self) -> TurnResult: - """One model round-trip: get the response and execute its tool calls.""" + async def request(self) -> Any: + """Send the current conversation to the model and return the raw + response WITHOUT executing any tool calls. Lets a caller overlap the + model's thinking time with other work (e.g. proactive cruising).""" try: - resp = await self.ac.chat.completions.create( + return await self.ac.chat.completions.create( model=self.model, messages=self.messages, tools=self.tools, @@ -476,6 +479,12 @@ class LLMController: self.log("agent", f"LLM error: {exc}") raise RuntimeError(f"LLM error: {exc}") from exc + async def invoke(self) -> TurnResult: + """One model round-trip: get the response and execute its tool calls.""" + return await self.execute(await self.request()) + + async def execute(self, resp: Any) -> TurnResult: + """Execute the tool calls of a previously requested response.""" choice = resp.choices[0] text = choice.message.content or "" result = TurnResult(text=text) @@ -596,11 +605,15 @@ async def run_llm_agent_loop( recorder: Any | None = None, nudge_limit: int = 1, look_every: int = 0, + cruise: float = 0.0, ) -> dict[str, Any]: """Drive the controller until the mission is done or steps run out. ``look_every``: attach a fresh camera frame every N steps so the model always sees recent visual context without asking (0 disables). + ``cruise``: proactive motion — while the model is thinking, the capsule + keeps gliding forward (meters per think, 0 disables). Collisions stop the + drift and are reported to the model. """ summary: dict[str, Any] = { "steps": 0, @@ -613,6 +626,45 @@ async def run_llm_agent_loop( last_correct_step = -99 last_collision_step = -99 + async def drift(amount: float, step: int) -> None: + """Low-level controller: keep moving forward while the brain thinks.""" + nonlocal last_collision_step + remaining = amount + total = 0.0 + while remaining > 0.02: + mv = await controller.client.call_tool( + "move", {"forward": min(0.4, remaining), "duration": 0.5} + ) + out = mv.output + controller.observe(out) + moved = out.get("moved", 0.0) + total += moved + remaining -= moved + if out.get("collision"): + last_collision_step = step + controller.messages.append( + { + "role": "user", + "content": ( + "While you were thinking, the capsule drifted forward and bumped " + f"into {out.get('collision_normal')}. It stopped. Navigate around it." + ), + } + ) + log("agent", "(cruise: bumped while thinking — stopped)") + return + if total > 0.05: + controller.messages.append( + { + "role": "user", + "content": ( + f"While you were thinking, the capsule kept moving (auto-cruise, " + f"{total:.2f} m forward). {controller.state_hint()}" + ), + } + ) + log("agent", f"(cruise: drifted {total:.2f} m while thinking)") + def maybe_correct(step: int) -> None: nonlocal corrections, last_correct_step if controller.pos is None or controller.beacon_pos is None: @@ -652,6 +704,7 @@ async def run_llm_agent_loop( log("agent", "(correction: re-aim at the beacon)") dist_history.clear() + pending_resp: Any | None = None # LLM response requested while we cruised for step in range(max_steps): summary["steps"] = step + 1 if recorder is not None and controller.pos is not None: @@ -660,7 +713,12 @@ async def run_llm_agent_loop( # sees what is happening without having to ask. if look_every and step % look_every == 0: await controller.auto_frame() - turn = await controller.invoke() + if pending_resp is not None: + resp = await pending_resp + pending_resp = None + turn = await controller.execute(resp) + else: + turn = await controller.invoke() if turn.text: log("agent", turn.text[:400]) summary["tool_calls"] += len(turn.calls) @@ -706,5 +764,11 @@ async def run_llm_agent_loop( return summary controller.messages.append({"role": "user", "content": NUDGE}) log("agent", "(nudge: no tool call; continue the mission)") + continue + # Proactive motion: request the next response and, while the model + # thinks, keep the capsule gliding forward (collision-safe). + if cruise > 0.0 and step < max_steps - 1: + pending_resp = asyncio.create_task(controller.request()) + await drift(cruise, step) summary["result"] = f"exceeded {max_steps} steps" return summary diff --git a/testbed/tests/test_bridge.py b/testbed/tests/test_bridge.py index 67321a1..ec18286 100644 --- a/testbed/tests/test_bridge.py +++ b/testbed/tests/test_bridge.py @@ -1,7 +1,9 @@ """Tests for the testbed bridge: tool behavior over the in-process transport.""" +import asyncio import base64 import io +import math import pytest from aicc.client import AICCClient @@ -186,3 +188,62 @@ async def test_single_source_of_truth(): world = dumped["world"] assert set(world.keys()) == {"name", "kind"} assert "beacon" not in dumped + + +async def test_animated_move_is_smooth_and_observable(bridge): + """A move with duration glides: a concurrent client sees intermediate + positions, and the final displacement is exact.""" + # A transport is a single session: one per client. + t1 = InProcessTransport.start(bridge) + t2 = InProcessTransport.start(bridge) + async with t1, t2: + mover = AICCClient(t1) + viewer = AICCClient(t2) + async with mover, viewer: + await mover.handshake() + await viewer.handshake() + start = (await mover.call_tool("proprioception", {})).output["position"] + assert start["x"] == 1.5 and start["z"] == 1.5 + + move_task = asyncio.create_task( + mover.call_tool("move", {"forward": 2.0, "duration": 0.8}) + ) + await asyncio.sleep(0.25) + mid = (await viewer.call_tool("proprioception", {})).output["position"] + # Moving along the 45-degree diagonal: strictly between start and end. + assert 1.5 < mid["x"] < 3.5 and 1.5 < mid["z"] < 3.5 + + res = (await move_task).output + assert res["moved"] == pytest.approx(2.0, abs=1e-3) + assert res["position"]["x"] == pytest.approx( + 1.5 + 2.0 * math.sin(math.radians(45)), abs=1e-3 + ) + assert res["duration"] == 0.8 + + +async def test_animated_move_stops_on_collision(bridge): + t = InProcessTransport.start(bridge) + async with t: + client = AICCClient(t) + async with client: + await client.handshake() + bridge.world.capsule.yaw_deg = 180.0 # straight at the north wall + res = ( + await client.call_tool("move", {"forward": 5.0, "duration": 0.6}) + ).output + assert res["collision"] is True + assert res["moved"] < 5.0 + assert res["position"]["z"] == pytest.approx( + bridge.world.capsule.radius, abs=1e-3 + ) + + +async def test_move_accepts_duration_zero_default(bridge): + t = InProcessTransport.start(bridge) + async with t: + client = AICCClient(t) + async with client: + await client.handshake() + res = (await client.call_tool("move", {"forward": 1.0})).output + assert res["moved"] == pytest.approx(1.0) + assert res["duration"] == 0.0