Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5baf698b9e | ||
|
|
e44977cbe4 | ||
|
|
b6b5547ffc | ||
|
|
d49cf46181 | ||
|
|
a9e45fb427 | ||
|
|
daa0371519 | ||
|
|
315762363e | ||
|
|
1cb3bdc46e | ||
|
|
b3d5c5630a | ||
|
|
0dd2e21683 |
@@ -0,0 +1,6 @@
|
||||
.env
|
||||
.playwright-mcp/
|
||||
chat_map.png
|
||||
chat_frame.png
|
||||
.opencode/
|
||||
frames/
|
||||
@@ -1,26 +0,0 @@
|
||||
# Session Goal
|
||||
|
||||
реализуй полностью. автономно
|
||||
|
||||
## Plan
|
||||
1. Read protocol spec (docs/aicc-core.md) and testbed design (docs/testbed-design.md)
|
||||
2. Inspect aicc-py SDK at ~/Desktop/aicc-py
|
||||
3. Build testbed bridge skeleton (world, capsule, bridge with tools)
|
||||
4. Implement tools: proprioception, vision, move, turn, look_at, interact
|
||||
5. Add headless rendering for vision frames
|
||||
6. Write agent demo loop
|
||||
7. Run conformance suite (9/9 green)
|
||||
8. Verify demo end-to-end, write README, commit per milestone
|
||||
|
||||
## Progress
|
||||
- DONE: world engine (room, capsule physics/collision, beacon, audio) — testbed/room/world.py
|
||||
- DONE: headless raycaster renderer (vision 160x120 + aligned depth grid) — testbed/room/render.py
|
||||
- DONE: RoomBridge with all world tools + echo/boom/bump — testbed/bridge.py
|
||||
- DONE: conformance runner — 9/9 green against testbed bridge (testbed/conformance.py)
|
||||
- DONE: pytest suite — 33 tests green; ruff clean
|
||||
- DONE: agent demo loop — LLM (OpenAI-compatible/ollama) + scripted fallback + auto handoff; E2E verified over WebSocket (scripted: 26 steps, beacon activated; LLM mode completed missions with gemma4:e2b)
|
||||
- DONE: testbed/README.md, scripts/setup.sh deps, scripts/run_bridge.sh, BUILDLOG.md updated
|
||||
- DONE: commits per milestone (skeleton, bridge+tools+conformance, tests+physics fixes, demo loop, docs, lint)
|
||||
- VERIFIED: conformance 9/9, pytest 33/33, ruff clean, demo end-to-end
|
||||
|
||||
COMPLETED
|
||||
@@ -1,26 +0,0 @@
|
||||
[ 28151ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 35175ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 36177ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 37178ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 38180ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 39182ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 40184ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 49096ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 50098ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 51100ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 52101ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 53103ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 54105ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 55108ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 56109ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 57112ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 58115ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 59117ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 60119ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 61123ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 62129ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 63134ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 64140ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 78549ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 88888ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 89892ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
@@ -1,11 +0,0 @@
|
||||
- generic [active] [ref=e1]:
|
||||
- banner [ref=e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=e3]
|
||||
- generic [ref=e4]: connected — waiting for frames
|
||||
- main [ref=e5]:
|
||||
- generic [ref=e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=e7]
|
||||
- img "vision" [ref=e8]
|
||||
- generic [ref=e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=e10]
|
||||
- img "map" [ref=e11]
|
||||
@@ -1,11 +0,0 @@
|
||||
- generic [active] [ref=e1]:
|
||||
- banner [ref=e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=e3]
|
||||
- generic [ref=e4]: connected — waiting for frames
|
||||
- main [ref=e5]:
|
||||
- generic [ref=e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=e7]
|
||||
- img "vision" [ref=e8]
|
||||
- generic [ref=e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=e10]
|
||||
- img "map" [ref=e11]
|
||||
@@ -1,11 +0,0 @@
|
||||
- generic [active] [ref=f1e1]:
|
||||
- banner [ref=f1e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=f1e3]
|
||||
- generic [ref=f1e4]: connected — waiting for frames
|
||||
- main [ref=f1e5]:
|
||||
- generic [ref=f1e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=f1e7]
|
||||
- img "vision" [ref=f1e8]
|
||||
- generic [ref=f1e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=f1e10]
|
||||
- img "map" [ref=f1e11]
|
||||
@@ -1,11 +0,0 @@
|
||||
- generic [active] [ref=f1e1]:
|
||||
- banner [ref=f1e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=f1e3]
|
||||
- generic [ref=f1e4]: tick 158 pos (11.86, 11.22) yaw 26.5° 1.43 m to beacon · 3.0 fps
|
||||
- main [ref=f1e5]:
|
||||
- generic [ref=f1e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=f1e7]
|
||||
- img "vision" [ref=f1e8]
|
||||
- generic [ref=f1e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=f1e10]
|
||||
- img "map" [ref=f1e11]
|
||||
@@ -14,6 +14,31 @@ an agent (LLM or scripted) navigates to the beacon and activates it. See
|
||||
`testbed/README.md` for run instructions; `scripts/BUILDLOG.md` for the build
|
||||
log.
|
||||
|
||||
## Search challenge
|
||||
|
||||
The agent can also be sent on an autonomous search: an orange triangle is
|
||||
hidden on the back side of one of the crates, and the model must explore the
|
||||
room with real vision, spot it, and report it. This recording shows the run
|
||||
from three angles — the top-down map, the actual first-person frames the
|
||||
model saw, and its chat/reasoning (recorded with smooth motion, ~2 Hz):
|
||||
|
||||

|
||||
|
||||
Regenerate it with `python -m testbed.record_search --provider polza --goal search
|
||||
--out search_mission.gif --capture-hz 2.0 --cruise 0.8 --steps 60`.
|
||||
|
||||
## Smooth & proactive movement
|
||||
|
||||
The same recording setup with a beacon mission and the cruise mode — the
|
||||
capsule glides with animated `move`/`turn` calls and keeps moving while the
|
||||
model thinks (`--cruise`). Frames are sampled at ~2.5 Hz by an observer
|
||||
client, so the GIF shows the actual motion, not step endpoints:
|
||||
|
||||

|
||||
|
||||
Regenerate with `python -m testbed.record_search --provider polza --goal beacon
|
||||
--out smooth_mission.gif --capture-hz 2.5 --cruise 1.0`.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
|
||||
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 2.9 KiB |
|
Before Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 32 KiB |
|
Before Width: | Height: | Size: 3.6 KiB |
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 3.8 KiB |
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 2.3 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 1.6 KiB |
|
Before Width: | Height: | Size: 4.3 KiB |
|
Before Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 4.3 KiB |
|
Before Width: | Height: | Size: 1.8 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 2.7 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 4.7 KiB |
|
Before Width: | Height: | Size: 2.9 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 4.8 KiB |
|
Before Width: | Height: | Size: 3.2 KiB |
|
Before Width: | Height: | Size: 4.7 KiB |
|
Before Width: | Height: | Size: 3.5 KiB |
|
Before Width: | Height: | Size: 4.7 KiB |
|
Before Width: | Height: | Size: 3.7 KiB |
|
Before Width: | Height: | Size: 4.9 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 5.1 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 6.2 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 7.6 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 8.3 KiB |
@@ -3,6 +3,14 @@
|
||||
# Usage: scripts/run_bridge.sh [port] (default 8765)
|
||||
# Note: the bridge keeps running until interrupted (Ctrl-C).
|
||||
set -euo pipefail
|
||||
|
||||
# Load API keys from the project .env file (if present)
|
||||
if [ -f .env ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
set +a
|
||||
fi
|
||||
cd "$(dirname "$0")/.."
|
||||
PORT="${1:-8765}"
|
||||
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
# Open http://127.0.0.1:8000 to watch the capsule while you talk to it.
|
||||
# Usage: scripts/run_chat.sh [--model gemma4:e2b]
|
||||
set -euo pipefail
|
||||
|
||||
# Load API keys from the project .env file (if present)
|
||||
if [ -f .env ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
set +a
|
||||
fi
|
||||
cd "$(dirname "$0")/.."
|
||||
BRIDGE_PID=""
|
||||
LIVE_PID=""
|
||||
|
||||
@@ -5,5 +5,13 @@
|
||||
# scripts/run_demo.sh --agent scripted
|
||||
# scripts/run_demo.sh --agent llm --model gemma4:e2b
|
||||
set -euo pipefail
|
||||
|
||||
# Load API keys from the project .env file (if present)
|
||||
if [ -f .env ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
set +a
|
||||
fi
|
||||
cd "$(dirname "$0")/.."
|
||||
exec testbed/.venv/bin/python -m testbed.demo "$@"
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
# demo while you watch. Open http://127.0.0.1:8000 in a browser.
|
||||
# Usage: scripts/run_live.sh [demo args...] e.g. --agent scripted
|
||||
set -euo pipefail
|
||||
|
||||
# Load API keys from the project .env file (if present)
|
||||
if [ -f .env ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
set +a
|
||||
fi
|
||||
cd "$(dirname "$0")/.."
|
||||
BRIDGE_PID=""
|
||||
LIVE_PID=""
|
||||
|
||||
|
After Width: | Height: | Size: 4.4 MiB |
|
After Width: | Height: | Size: 2.9 MiB |
@@ -64,6 +64,39 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal.
|
||||
The demo prints a full transcript of tool calls/results to stdout and saves the
|
||||
capsule's final first-person frame to `demo_final_frame.png`.
|
||||
|
||||
### Search challenge
|
||||
|
||||
A small orange triangle is painted on the **back side** of one of the crates
|
||||
(random per run, always on a face hidden from the spawn corner). The agent must
|
||||
explore the room, look at the crate faces, spot the triangle with `vision`,
|
||||
and `report` it (the bridge verifies the report by distance):
|
||||
|
||||
```bash
|
||||
scripts/run_chat.sh --provider polza --search
|
||||
# autonomous exploration, real vision, smooth turns, cruise gliding;
|
||||
# success = report verified
|
||||
```
|
||||
|
||||
Search missions run in `--free` mode: no re-aim corrections, no collision
|
||||
hints — the model plans its own exploration. `--free` also applies to beacon
|
||||
missions if you want fewer guardrails. `look_at` now accepts crates too
|
||||
(crate_red/crate_blue/crate_olive). Verified: gpt-5.6-luna explored the room
|
||||
with smooth moves/turns and reported the triangle at 0.5 m.
|
||||
|
||||
### Smooth & proactive movement
|
||||
|
||||
The capsule no longer teleports or stops-and-thinks:
|
||||
|
||||
- **Smooth**: `move(forward, duration)` animates the displacement over
|
||||
`duration` seconds on the bridge, so the live viewer shows genuine gliding.
|
||||
The scripted agent and auto-cruise use it by default; the LLM can too.
|
||||
- **Proactive** (`--cruise N`, meters per think): while the model is
|
||||
generating a response, the capsule keeps gliding forward (low-level
|
||||
controller pattern, like a real robot). Collisions stop the drift safely
|
||||
and are reported to the model, which re-plans. Verified: gpt-5.6-luna
|
||||
completed the beacon mission with cruise on, bumping into and avoiding
|
||||
crates while "thinking".
|
||||
|
||||
## Providers
|
||||
|
||||
Provider presets resolve endpoint + API key + default model:
|
||||
|
||||
@@ -85,6 +85,7 @@ class MoveOutput(BaseModel):
|
||||
collision_normal: Vec3 | None
|
||||
position: Vec3
|
||||
tick: int
|
||||
duration: float = 0.0
|
||||
|
||||
|
||||
class TurnOutput(BaseModel):
|
||||
@@ -112,6 +113,15 @@ class WorldQueryOutput(BaseModel):
|
||||
tick: int
|
||||
|
||||
|
||||
class ReportOutput(BaseModel):
|
||||
discovery: str
|
||||
success: bool
|
||||
verified: bool
|
||||
distance: float
|
||||
message: str
|
||||
tick: int
|
||||
|
||||
|
||||
class EchoOutput(BaseModel):
|
||||
value: str
|
||||
|
||||
@@ -165,6 +175,42 @@ class RoomBridge(Bridge):
|
||||
def _tick(self) -> int:
|
||||
return self.world.advance_tick()
|
||||
|
||||
async def _animate_turn(
|
||||
self, yaw_deg: float, pitch_deg: float, duration: float
|
||||
) -> None:
|
||||
"""Rotate smoothly over ``duration`` seconds (observable mid-turn)."""
|
||||
n = max(1, min(int(duration / 0.1), 100))
|
||||
dt = duration / n
|
||||
for i in range(n):
|
||||
self.world.turn(yaw_deg / n, pitch_deg / n)
|
||||
if i < n - 1:
|
||||
await asyncio.sleep(dt)
|
||||
|
||||
async def _animate_move(
|
||||
self, forward: float, duration: float
|
||||
) -> tuple[float, Any | None]:
|
||||
"""Smoothly displace the capsule over ``duration`` seconds.
|
||||
|
||||
The position advances in small timed steps, so concurrent clients
|
||||
(e.g. the live viewer) observe genuine gliding motion instead of a
|
||||
teleport. Collision resolution is unchanged.
|
||||
"""
|
||||
world = self.world
|
||||
n = max(1, min(int(duration / 0.1), 200))
|
||||
step = forward / n
|
||||
dt = duration / n
|
||||
total = 0.0
|
||||
hit: Any | None = None
|
||||
for i in range(n):
|
||||
moved, h = world.move_forward(step)
|
||||
total += moved
|
||||
if h is not None or moved < step * 0.01:
|
||||
hit = h
|
||||
break
|
||||
if i < n - 1:
|
||||
await asyncio.sleep(dt)
|
||||
return total, hit
|
||||
|
||||
def _vec3(self, x: float, y: float, z: float) -> Vec3:
|
||||
return Vec3(x=round(x, 3), y=round(y, 3), z=round(z, 3))
|
||||
|
||||
@@ -263,13 +309,21 @@ class RoomBridge(Bridge):
|
||||
|
||||
@self.tool(
|
||||
cls=ToolClass.ACTUATOR,
|
||||
description="Move the capsule forward along its heading by the given distance in meters.",
|
||||
description=(
|
||||
"Move the capsule forward along its heading by the given distance in meters. "
|
||||
"duration (seconds) animates the motion smoothly instead of teleporting."
|
||||
),
|
||||
)
|
||||
async def move(forward: float = 1.0) -> MoveOutput:
|
||||
async def move(forward: float = 1.0, duration: float = 0.0) -> MoveOutput:
|
||||
tick = self._tick()
|
||||
if forward < 0.0 or forward > 5.0:
|
||||
raise ValueError(f"forward must be in [0, 5] m, got {forward}")
|
||||
moved, hit = world.move_forward(forward)
|
||||
if duration < 0.0 or duration > 10.0:
|
||||
raise ValueError(f"duration must be in [0, 10] s, got {duration}")
|
||||
if duration > 0.0:
|
||||
moved, hit = await self._animate_move(forward, duration)
|
||||
else:
|
||||
moved, hit = world.move_forward(forward)
|
||||
c = world.capsule
|
||||
normal = None
|
||||
if hit is not None:
|
||||
@@ -294,28 +348,53 @@ class RoomBridge(Bridge):
|
||||
collision_normal=normal,
|
||||
position=self._vec3(c.x, 0.0, c.z),
|
||||
tick=tick,
|
||||
duration=round(duration, 3),
|
||||
).model_dump()
|
||||
|
||||
@self.tool(
|
||||
cls=ToolClass.ACTUATOR,
|
||||
description="Rotate the camera: yaw_deg turns left/right, pitch_deg tilts up/down.",
|
||||
description=(
|
||||
"Rotate the camera: yaw_deg turns left/right, pitch_deg tilts up/down. "
|
||||
"duration (seconds) animates the rotation smoothly."
|
||||
),
|
||||
)
|
||||
async def turn(yaw_deg: float = 0.0, pitch_deg: float = 0.0) -> TurnOutput:
|
||||
async def turn(
|
||||
yaw_deg: float = 0.0, pitch_deg: float = 0.0, duration: float = 0.0
|
||||
) -> TurnOutput:
|
||||
tick = self._tick()
|
||||
world.turn(yaw_deg, pitch_deg)
|
||||
if duration < 0.0 or duration > 10.0:
|
||||
raise ValueError(f"duration must be in [0, 10] s, got {duration}")
|
||||
if duration > 0.0:
|
||||
await self._animate_turn(yaw_deg, pitch_deg, duration)
|
||||
else:
|
||||
world.turn(yaw_deg, pitch_deg)
|
||||
return TurnOutput(rotation=self._rotation(), tick=tick).model_dump()
|
||||
|
||||
@self.tool(
|
||||
cls=ToolClass.ACTUATOR,
|
||||
description="Orient the camera toward a named target (e.g. 'beacon').",
|
||||
description=(
|
||||
"Orient the camera toward a named target: 'beacon' or a crate "
|
||||
"(crate_red, crate_blue, crate_olive)."
|
||||
),
|
||||
)
|
||||
async def look_at(target: str) -> LookAtOutput:
|
||||
tick = self._tick()
|
||||
if target != "beacon":
|
||||
box = next((b for b in world.boxes if b.id == target), None)
|
||||
if target == "beacon":
|
||||
yaw, pitch = world.look_at_beacon()
|
||||
elif box is not None:
|
||||
dx = box.cx - world.capsule.x
|
||||
dz = box.cz - world.capsule.z
|
||||
dist = math.hypot(dx, dz) or 1.0
|
||||
yaw = math.degrees(math.atan2(dx, dz))
|
||||
pitch = math.degrees(
|
||||
math.atan2(box.height / 2 - world.capsule.eye_height, dist)
|
||||
)
|
||||
world.face(yaw, pitch)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"unknown target {target!r}; available targets: beacon"
|
||||
f"unknown target {target!r}; available: beacon, crate_red, crate_blue, crate_olive"
|
||||
)
|
||||
yaw, pitch = world.look_at_beacon()
|
||||
return LookAtOutput(
|
||||
target=target,
|
||||
rotation=Rotation(
|
||||
@@ -348,6 +427,33 @@ class RoomBridge(Bridge):
|
||||
target=target, success=success, message=message, tick=tick
|
||||
).model_dump()
|
||||
|
||||
@self.tool(
|
||||
cls=ToolClass.ACTUATOR,
|
||||
description=(
|
||||
"Report a discovery to the operator (used in search missions): "
|
||||
"say what you found. The report is verified against your position."
|
||||
),
|
||||
)
|
||||
async def report(discovery: str = "triangle") -> ReportOutput:
|
||||
tick = self._tick()
|
||||
dist = world.distance_to_marker()
|
||||
verified = world.marker is not None and dist <= 6.0
|
||||
if verified:
|
||||
message = f"report verified: {discovery} sighted {dist:.1f} m away"
|
||||
else:
|
||||
message = (
|
||||
f"not verified: nothing matching {discovery!r} within 6 m "
|
||||
f"(closest check {dist:.1f} m) — keep exploring"
|
||||
)
|
||||
return ReportOutput(
|
||||
discovery=discovery,
|
||||
success=True,
|
||||
verified=verified,
|
||||
distance=round(dist, 2),
|
||||
message=message,
|
||||
tick=tick,
|
||||
).model_dump()
|
||||
|
||||
# -- conformance tools (design doc: register alongside world tools) --
|
||||
|
||||
@self.tool(
|
||||
|
||||
@@ -141,16 +141,30 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
||||
async def mission_runner(goal_text: str) -> None:
|
||||
from testbed.llm_agent import run_llm_agent_loop
|
||||
|
||||
controller.messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"MISSION (set by the user): {goal_text}. "
|
||||
"Keep calling tools and do not stop until the goal is achieved. "
|
||||
"Report only when done."
|
||||
),
|
||||
}
|
||||
)
|
||||
if args.search:
|
||||
from testbed.llm_agent import SEARCH_MISSION
|
||||
|
||||
controller.messages = [
|
||||
{"role": "system", "content": SEARCH_MISSION},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"MISSION: {goal_text}. "
|
||||
"Keep exploring with tools until the triangle is found and reported."
|
||||
),
|
||||
},
|
||||
]
|
||||
else:
|
||||
controller.messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"MISSION (set by the user): {goal_text}. "
|
||||
"Keep calling tools and do not stop until the goal is achieved. "
|
||||
"Report only when done."
|
||||
),
|
||||
}
|
||||
)
|
||||
saver = asyncio.create_task(map_saver())
|
||||
try:
|
||||
for attempt in range(1, args.mission_retries + 2):
|
||||
@@ -164,12 +178,28 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
||||
log=mission_log,
|
||||
nudge_limit=args.mission_nudges,
|
||||
look_every=args.look_every,
|
||||
cruise=args.cruise,
|
||||
autonomous=args.free or args.search,
|
||||
is_success=(
|
||||
(
|
||||
lambda turn: any(
|
||||
c.name == "report"
|
||||
and c.ok
|
||||
and isinstance(c.output, dict)
|
||||
and c.output.get("verified")
|
||||
for c in turn.calls
|
||||
)
|
||||
)
|
||||
if args.search
|
||||
else None
|
||||
),
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
print(f" [mission] LLM error: {exc}")
|
||||
break
|
||||
save_map()
|
||||
if summary["interacted"]:
|
||||
done = summary.get("success") if args.search else summary["interacted"]
|
||||
if done:
|
||||
print(f"\n[mission] DONE: {summary['result']}")
|
||||
return
|
||||
print(
|
||||
@@ -225,7 +255,12 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
||||
print(" (no mission running)")
|
||||
await cmd_state()
|
||||
|
||||
if args.mission:
|
||||
if args.mission or args.search:
|
||||
if args.search:
|
||||
mission_goal = (
|
||||
"explore the room, find the orange triangle painted on the back "
|
||||
"of one of the crates, and report it"
|
||||
)
|
||||
start_mission()
|
||||
|
||||
while True:
|
||||
@@ -307,6 +342,8 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
||||
steps_used += 1
|
||||
if turn.text:
|
||||
print(f" model> {turn.text}")
|
||||
if turn.thinking:
|
||||
print(f" [thinking] {turn.thinking[:300]}")
|
||||
if turn.interacted:
|
||||
print(f"\n[chat] BEACON ACTIVATED: {turn.message}")
|
||||
save_map()
|
||||
@@ -427,6 +464,17 @@ def main() -> int:
|
||||
action="store_true",
|
||||
help="start an autonomous mission on connect: reach and activate the beacon",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--search",
|
||||
action="store_true",
|
||||
help="search challenge: explore the room and find the orange triangle "
|
||||
"on the back of a crate, then report it (autonomous mode, fewer guardrails)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--free",
|
||||
action="store_true",
|
||||
help="fewer restrictions: no re-aim corrections or collision hints; the model plans freely",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mission-steps",
|
||||
type=int,
|
||||
@@ -463,6 +511,12 @@ def main() -> int:
|
||||
default=3,
|
||||
help="attach a fresh camera frame every N turns (0 disables; default 3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cruise",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="proactive motion in missions: glide forward (meters) while the LLM thinks (0 disables)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
args.base_url, args.api_key, args.model = resolve_provider(
|
||||
@@ -472,7 +526,9 @@ def main() -> int:
|
||||
model=args.model,
|
||||
)
|
||||
if args.provider:
|
||||
print(f"[chat] provider: {args.provider} -> {args.base_url} model: {args.model}")
|
||||
print(
|
||||
f"[chat] provider: {args.provider} -> {args.base_url} model: {args.model}"
|
||||
)
|
||||
except ValueError as exc:
|
||||
print(f"[chat] {exc}")
|
||||
return 1
|
||||
|
||||
@@ -142,6 +142,8 @@ async def run_llm_agent(
|
||||
vision: bool | None = None,
|
||||
digest: bool | None = None,
|
||||
look_every: int = 0,
|
||||
cruise: float = 0.0,
|
||||
autonomous: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Autonomous LLM run: controller + nudge/correct loop (see llm_agent)."""
|
||||
from testbed.llm_agent import LLMController, run_llm_agent_loop
|
||||
@@ -233,9 +235,9 @@ async def run_scripted_agent(
|
||||
|
||||
if detour_steps > 0:
|
||||
# Escape the obstacle before re-aiming: keep the detour heading.
|
||||
mv = (await client.call_tool("move", {"forward": 0.8})).output
|
||||
mv = (await client.call_tool("move", {"forward": 0.8, "duration": 0.6})).output
|
||||
summary["tool_calls"] += 1
|
||||
log("tool", 'move({"forward": 0.8}) [detour]')
|
||||
log("tool", 'move({"forward": 0.8, "duration": 0.6}) [detour]')
|
||||
log("bridge", f"ok {json.dumps(mv)}")
|
||||
if mv.get("moved", 0.0) < 0.2 and detour_turns < 6:
|
||||
detour_turns += 1
|
||||
@@ -267,9 +269,9 @@ async def run_scripted_agent(
|
||||
log("tool", f'turn({{"yaw_deg": {delta * 0.8:.1f}}})')
|
||||
log("bridge", f"ok {json.dumps(turn)}")
|
||||
else:
|
||||
mv = (await client.call_tool("move", {"forward": 0.8})).output
|
||||
mv = (await client.call_tool("move", {"forward": 0.8, "duration": 0.6})).output
|
||||
summary["tool_calls"] += 1
|
||||
log("tool", 'move({"forward": 0.8})')
|
||||
log("tool", 'move({"forward": 0.8, "duration": 0.6})')
|
||||
log("bridge", f"ok {json.dumps(mv)}")
|
||||
if mv.get("collision"):
|
||||
consecutive_collisions += 1
|
||||
@@ -315,6 +317,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
|
||||
vision=args.vision,
|
||||
digest=args.digest,
|
||||
look_every=args.look_every,
|
||||
cruise=args.cruise,
|
||||
autonomous=args.free,
|
||||
)
|
||||
elif args.agent == "scripted":
|
||||
summary = await run_scripted_agent(
|
||||
@@ -338,6 +342,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
|
||||
vision=args.vision,
|
||||
digest=args.digest,
|
||||
look_every=args.look_every,
|
||||
cruise=args.cruise,
|
||||
autonomous=args.free,
|
||||
)
|
||||
if summary.get("interacted"):
|
||||
return summary
|
||||
@@ -431,6 +437,17 @@ def main() -> int:
|
||||
default=3,
|
||||
help="attach a fresh camera frame every N agent steps (0 disables; default 3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cruise",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="proactive motion: glide forward (meters) while the LLM thinks (0 disables)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--free",
|
||||
action="store_true",
|
||||
help="fewer restrictions: no re-aim corrections or collision hints; the model plans freely",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--frame",
|
||||
default="demo_final_frame.png",
|
||||
|
||||
@@ -9,6 +9,7 @@ tool calls/results fed back as `tool` messages.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
@@ -67,6 +68,19 @@ NUDGE = (
|
||||
"then call a tool (turn or move, or interact if within 1.6 m)."
|
||||
)
|
||||
|
||||
SEARCH_MISSION = (
|
||||
"You are an explorer in a 16x16 m room. Three wooden crates stand in the room, "
|
||||
"and a small ORANGE TRIANGLE is painted on the BACK side of ONE of them — the "
|
||||
"side that faces away from the room's entrance, so it is only visible once you "
|
||||
"walk around the crates. The glowing beacon in the corner is irrelevant: ignore it. "
|
||||
"Your mission: explore the room until you see the orange triangle with your own "
|
||||
'eyes (vision), then walk near it and call report(discovery="triangle"). '
|
||||
"Strategy: walk a loop around the crates; every few steps look at the crate faces "
|
||||
"with vision; check faces on all sides. The CURRENT STATE note in tool results "
|
||||
"tells you your position and heading. Do not stop until you have seen and "
|
||||
"reported the triangle."
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider presets (OpenAI-compatible endpoints)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -104,6 +118,31 @@ _MULTIMODAL_HINTS = (
|
||||
)
|
||||
|
||||
|
||||
def load_dotenv() -> None:
|
||||
"""Load KEY=VALUE pairs from a .env file (project root or cwd) into
|
||||
os.environ without overriding already-set variables."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
candidates = [
|
||||
Path.cwd() / ".env",
|
||||
Path(__file__).resolve().parent.parent / ".env",
|
||||
]
|
||||
for path in candidates:
|
||||
if not path.is_file():
|
||||
continue
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
return
|
||||
|
||||
|
||||
def resolve_provider(
|
||||
*,
|
||||
provider: str | None,
|
||||
@@ -113,11 +152,12 @@ def resolve_provider(
|
||||
) -> tuple[str, str, str]:
|
||||
"""Resolve endpoint + key + model from a provider preset (or explicit args).
|
||||
|
||||
Keys come from the environment (POLZA_API_KEY / OPENAI_API_KEY) so nothing
|
||||
Keys come from the environment or the project's .env file, so nothing
|
||||
secret lives in the repo or on the command line.
|
||||
"""
|
||||
import os
|
||||
|
||||
load_dotenv()
|
||||
if provider:
|
||||
preset = PROVIDERS.get(provider)
|
||||
if preset is None:
|
||||
@@ -161,6 +201,7 @@ def looks_multimodal(model: str) -> bool:
|
||||
COLOR_NAMES = [
|
||||
("beacon_cyan", (120, 210, 235)),
|
||||
("beacon_yellow", (255, 190, 90)),
|
||||
("marker_orange", (255, 150, 40)),
|
||||
("crate_red", (178, 64, 54)),
|
||||
("crate_blue", (64, 96, 178)),
|
||||
("crate_olive", (128, 128, 60)),
|
||||
@@ -273,6 +314,7 @@ class TurnResult:
|
||||
calls: list[ToolCallResult] = field(default_factory=list)
|
||||
interacted: bool = False
|
||||
message: str | None = None
|
||||
thinking: str = ""
|
||||
|
||||
|
||||
def detect_multimodal(base_url: str, model: str) -> bool:
|
||||
@@ -437,10 +479,12 @@ class LLMController:
|
||||
async def send_user(self, text: str) -> None:
|
||||
self.messages.append({"role": "user", "content": text})
|
||||
|
||||
async def invoke(self) -> TurnResult:
|
||||
"""One model round-trip: get the response and execute its tool calls."""
|
||||
async def request(self) -> Any:
|
||||
"""Send the current conversation to the model and return the raw
|
||||
response WITHOUT executing any tool calls. Lets a caller overlap the
|
||||
model's thinking time with other work (e.g. proactive cruising)."""
|
||||
try:
|
||||
resp = await self.ac.chat.completions.create(
|
||||
return await self.ac.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=self.messages,
|
||||
tools=self.tools,
|
||||
@@ -450,9 +494,24 @@ class LLMController:
|
||||
self.log("agent", f"LLM error: {exc}")
|
||||
raise RuntimeError(f"LLM error: {exc}") from exc
|
||||
|
||||
async def invoke(self) -> TurnResult:
|
||||
"""One model round-trip: get the response and execute its tool calls."""
|
||||
return await self.execute(await self.request())
|
||||
|
||||
async def execute(self, resp: Any) -> TurnResult:
|
||||
"""Execute the tool calls of a previously requested response."""
|
||||
choice = resp.choices[0]
|
||||
text = choice.message.content or ""
|
||||
result = TurnResult(text=text)
|
||||
# Some providers return reasoning tokens separately (e.g. deepseek-r1
|
||||
# style models); surface them when present.
|
||||
msg = choice.message
|
||||
thinking = (
|
||||
getattr(msg, "reasoning_content", None)
|
||||
or getattr(msg, "reasoning", None)
|
||||
or ""
|
||||
)
|
||||
result.thinking = thinking if isinstance(thinking, str) else ""
|
||||
calls = choice.message.tool_calls
|
||||
if not calls:
|
||||
self.messages.append({"role": "assistant", "content": text})
|
||||
@@ -570,11 +629,21 @@ async def run_llm_agent_loop(
|
||||
recorder: Any | None = None,
|
||||
nudge_limit: int = 1,
|
||||
look_every: int = 0,
|
||||
cruise: float = 0.0,
|
||||
autonomous: bool = False,
|
||||
is_success: Callable[[TurnResult], bool] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Drive the controller until the mission is done or steps run out.
|
||||
"""Drive the controller until the goal is done or steps run out.
|
||||
|
||||
``look_every``: attach a fresh camera frame every N steps so the model
|
||||
always sees recent visual context without asking (0 disables).
|
||||
``cruise``: proactive motion — while the model is thinking, the capsule
|
||||
keeps gliding forward (meters per think, 0 disables). Collisions stop the
|
||||
drift and are reported to the model.
|
||||
``autonomous``: fewer guardrails — no re-aim corrections and no collision
|
||||
hints; the model plans its own exploration (used for search missions).
|
||||
``is_success``: custom goal predicate over a model turn; by default the
|
||||
mission ends when interact succeeds.
|
||||
"""
|
||||
summary: dict[str, Any] = {
|
||||
"steps": 0,
|
||||
@@ -587,6 +656,45 @@ async def run_llm_agent_loop(
|
||||
last_correct_step = -99
|
||||
last_collision_step = -99
|
||||
|
||||
async def drift(amount: float, step: int) -> None:
|
||||
"""Low-level controller: keep moving forward while the brain thinks."""
|
||||
nonlocal last_collision_step
|
||||
remaining = amount
|
||||
total = 0.0
|
||||
while remaining > 0.02:
|
||||
mv = await controller.client.call_tool(
|
||||
"move", {"forward": min(0.4, remaining), "duration": 0.5}
|
||||
)
|
||||
out = mv.output
|
||||
controller.observe(out)
|
||||
moved = out.get("moved", 0.0)
|
||||
total += moved
|
||||
remaining -= moved
|
||||
if out.get("collision"):
|
||||
last_collision_step = step
|
||||
controller.messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"While you were thinking, the capsule drifted forward and bumped "
|
||||
f"into {out.get('collision_normal')}. It stopped. Navigate around it."
|
||||
),
|
||||
}
|
||||
)
|
||||
log("agent", "(cruise: bumped while thinking — stopped)")
|
||||
return
|
||||
if total > 0.05:
|
||||
controller.messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"While you were thinking, the capsule kept moving (auto-cruise, "
|
||||
f"{total:.2f} m forward). {controller.state_hint()}"
|
||||
),
|
||||
}
|
||||
)
|
||||
log("agent", f"(cruise: drifted {total:.2f} m while thinking)")
|
||||
|
||||
def maybe_correct(step: int) -> None:
|
||||
nonlocal corrections, last_correct_step
|
||||
if controller.pos is None or controller.beacon_pos is None:
|
||||
@@ -626,6 +734,7 @@ async def run_llm_agent_loop(
|
||||
log("agent", "(correction: re-aim at the beacon)")
|
||||
dist_history.clear()
|
||||
|
||||
pending_resp: Any | None = None # LLM response requested while we cruised
|
||||
for step in range(max_steps):
|
||||
summary["steps"] = step + 1
|
||||
if recorder is not None and controller.pos is not None:
|
||||
@@ -634,9 +743,16 @@ async def run_llm_agent_loop(
|
||||
# sees what is happening without having to ask.
|
||||
if look_every and step % look_every == 0:
|
||||
await controller.auto_frame()
|
||||
turn = await controller.invoke()
|
||||
if pending_resp is not None:
|
||||
resp = await pending_resp
|
||||
pending_resp = None
|
||||
turn = await controller.execute(resp)
|
||||
else:
|
||||
turn = await controller.invoke()
|
||||
if turn.text:
|
||||
log("agent", turn.text[:400])
|
||||
if turn.thinking:
|
||||
log("think", turn.thinking[:400])
|
||||
summary["tool_calls"] += len(turn.calls)
|
||||
for c in turn.calls:
|
||||
if (
|
||||
@@ -646,23 +762,37 @@ async def run_llm_agent_loop(
|
||||
and c.output.get("collision")
|
||||
):
|
||||
last_collision_step = step
|
||||
obj = c.output.get("collision_normal")
|
||||
hint_msg = (
|
||||
f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
|
||||
"forward twice to get around it, then follow the CURRENT STATE note to re-aim."
|
||||
)
|
||||
if not (
|
||||
controller.messages
|
||||
and controller.messages[-1].get("content") == hint_msg
|
||||
):
|
||||
controller.messages.append({"role": "user", "content": hint_msg})
|
||||
log("agent", "(collision: go around)")
|
||||
if not autonomous:
|
||||
obj = c.output.get("collision_normal")
|
||||
hint_msg = (
|
||||
f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
|
||||
"forward twice to get around it, then follow the CURRENT STATE note to re-aim."
|
||||
)
|
||||
if not (
|
||||
controller.messages
|
||||
and controller.messages[-1].get("content") == hint_msg
|
||||
):
|
||||
controller.messages.append(
|
||||
{"role": "user", "content": hint_msg}
|
||||
)
|
||||
log("agent", "(collision: go around)")
|
||||
# Show the model what it just bumped into.
|
||||
await controller.auto_frame(
|
||||
"You just bumped into something. Look at what is in front of you."
|
||||
)
|
||||
maybe_correct(step)
|
||||
if turn.interacted:
|
||||
if not autonomous:
|
||||
maybe_correct(step)
|
||||
if is_success is not None:
|
||||
if is_success(turn):
|
||||
summary["success"] = True
|
||||
for c in turn.calls:
|
||||
if c.name == "report" and c.ok and isinstance(c.output, dict):
|
||||
summary["result"] = c.output.get("message", "reported")
|
||||
break
|
||||
else:
|
||||
summary["result"] = "goal achieved"
|
||||
return summary
|
||||
elif turn.interacted:
|
||||
summary["interacted"] = True
|
||||
summary["result"] = turn.message
|
||||
return summary
|
||||
@@ -680,5 +810,11 @@ async def run_llm_agent_loop(
|
||||
return summary
|
||||
controller.messages.append({"role": "user", "content": NUDGE})
|
||||
log("agent", "(nudge: no tool call; continue the mission)")
|
||||
continue
|
||||
# Proactive motion: request the next response and, while the model
|
||||
# thinks, keep the capsule gliding forward (collision-safe).
|
||||
if cruise > 0.0 and step < max_steps - 1:
|
||||
pending_resp = asyncio.create_task(controller.request())
|
||||
await drift(cruise, step)
|
||||
summary["result"] = f"exceeded {max_steps} steps"
|
||||
return summary
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
"""Record a search mission and render a three-view GIF for the README.
|
||||
|
||||
Panes (left -> right):
|
||||
top-down debug map (world state: crates, marker, path, capsule)
|
||||
first person (the actual frames the model received)
|
||||
chat (the model's messages, reasoning, and tool calls)
|
||||
|
||||
Usage:
|
||||
python -m testbed.record_search --out search_mission.gif
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from aicc.client import AICCClient
|
||||
from aicc.transport.websocket import WebSocketClientTransport, WebSocketServer
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from testbed.bridge import build_bridge
|
||||
from testbed.llm_agent import (
|
||||
SEARCH_MISSION,
|
||||
LLMController,
|
||||
resolve_provider,
|
||||
run_llm_agent_loop,
|
||||
)
|
||||
|
||||
PANEL = 400
|
||||
ROOM = 16.0
|
||||
MAX_CHAT_LINES = 21
|
||||
CHAT_LINE_MAX = 46 # chars per chat line
|
||||
|
||||
ROLE_COLORS = {
|
||||
"agent": (220, 220, 230),
|
||||
"think": (150, 165, 200),
|
||||
"tool": (255, 184, 96),
|
||||
"bridge": (150, 210, 150),
|
||||
"mission": (255, 220, 120),
|
||||
"vision": (130, 210, 250),
|
||||
}
|
||||
|
||||
|
||||
def _font(size: int = 13) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
|
||||
try:
|
||||
return ImageFont.load_default(size=size)
|
||||
except TypeError: # older Pillow
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def _wrap(text: str, width: int = CHAT_LINE_MAX) -> list[str]:
|
||||
words = text.split()
|
||||
lines: list[str] = []
|
||||
cur = ""
|
||||
for w in words:
|
||||
if len(cur) + len(w) + 1 > width:
|
||||
if cur:
|
||||
lines.append(cur)
|
||||
cur = w
|
||||
else:
|
||||
cur = (cur + " " + w).strip()
|
||||
if cur:
|
||||
lines.append(cur)
|
||||
return lines[:3] # keep the pane readable
|
||||
|
||||
|
||||
class ChatPane:
|
||||
"""Accumulates the mission chat and renders the last lines."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lines: list[tuple[str, str]] = []
|
||||
|
||||
def add(self, role: str, msg: str) -> None:
|
||||
for piece in _wrap(msg):
|
||||
self.lines.append((role, piece))
|
||||
if len(self.lines) > MAX_CHAT_LINES * 2:
|
||||
self.lines = self.lines[-MAX_CHAT_LINES:]
|
||||
|
||||
def render(self) -> Image.Image:
|
||||
img = Image.new("RGB", (PANEL, PANEL), (12, 12, 18))
|
||||
draw = ImageDraw.Draw(img)
|
||||
font = _font(13)
|
||||
draw.text((8, 6), "chat — what the model says", fill=(160, 170, 190), font=font)
|
||||
y = 26
|
||||
for role, piece in self.lines[-MAX_CHAT_LINES:]:
|
||||
color = ROLE_COLORS.get(role, (200, 200, 200))
|
||||
draw.text((8, y), piece[:CHAT_LINE_MAX], fill=color, font=font)
|
||||
y += 17
|
||||
return img
|
||||
|
||||
|
||||
def _topdown_frame(
|
||||
bridge, path: list[tuple[float, float]], step: int, marker_found: bool
|
||||
) -> Image.Image:
|
||||
from testbed.room.render import render_topdown
|
||||
|
||||
img = render_topdown(bridge.world, PANEL)
|
||||
draw = ImageDraw.Draw(img)
|
||||
scale = PANEL / ROOM
|
||||
if len(path) > 1:
|
||||
pts = [(x * scale, PANEL - z * scale) for x, z in path]
|
||||
draw.line(pts, fill=(255, 170, 60), width=3)
|
||||
m = bridge.world.marker_world_pos()
|
||||
if m is not None:
|
||||
mx, mz = m[0] * scale, PANEL - m[2] * scale
|
||||
r = 9
|
||||
draw.polygon(
|
||||
[(mx, mz - r), (mx - r, mz + r * 0.8), (mx + r, mz + r * 0.8)],
|
||||
outline=(255, 255, 255),
|
||||
width=2,
|
||||
)
|
||||
status = f"step {step} {'FOUND' if marker_found else 'searching...'}"
|
||||
draw.text((10, PANEL - 24), status, fill=(255, 220, 120), font=_font(13))
|
||||
return img
|
||||
|
||||
|
||||
def _compose(map_img, view_img, chat_img) -> Image.Image:
|
||||
canvas = Image.new("RGB", (PANEL * 3, PANEL), (18, 18, 26))
|
||||
canvas.paste(map_img, (0, 0))
|
||||
canvas.paste(view_img, (PANEL, 0))
|
||||
canvas.paste(chat_img, (PANEL * 2, 0))
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
draw.text((8, 6), "top-down", fill=(160, 170, 190), font=_font(12))
|
||||
draw.text((PANEL + 8, 6), "first person", fill=(160, 170, 190), font=_font(12))
|
||||
draw.text((PANEL * 2 + 8, 6), "chat", fill=(160, 170, 190), font=_font(12))
|
||||
for x in (PANEL, PANEL * 2):
|
||||
draw.line([(x, 0), (x, PANEL)], fill=(60, 60, 80))
|
||||
return canvas
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
base_url, api_key, model = resolve_provider(
|
||||
provider=args.provider,
|
||||
base_url=args.base_url,
|
||||
api_key=args.api_key,
|
||||
model=args.model,
|
||||
)
|
||||
from testbed.llm_agent import MISSION
|
||||
|
||||
bridge = build_bridge()
|
||||
if args.goal == "search":
|
||||
# Fixed marker so the GIF tells a clear story (crate_blue, east face).
|
||||
bridge.world.marker = {
|
||||
"box": "crate_blue", "face": "x1", "u": 0.5, "v": 0.55,
|
||||
"size": 0.42, "color": (255, 150, 40),
|
||||
}
|
||||
m = bridge.world.marker_world_pos()
|
||||
print(f"[record] triangle marker on crate_blue east face at {tuple(round(v, 2) for v in m)}")
|
||||
|
||||
frames: list[Image.Image] = []
|
||||
chat = ChatPane()
|
||||
marker_seen = False
|
||||
samples: list[tuple[float, float]] = [] # capsule positions per captured frame
|
||||
|
||||
def log(role: str, msg: str) -> None:
|
||||
print(f" [{role}] {msg}")
|
||||
if role in ("agent", "think", "tool", "mission", "vision"):
|
||||
chat.add(role, msg)
|
||||
|
||||
async with WebSocketServer(bridge, port=8765):
|
||||
transport = WebSocketClientTransport("ws://127.0.0.1:8765")
|
||||
async with AICCClient(transport) as client:
|
||||
manifest = await client.handshake()
|
||||
ctl = LLMController(
|
||||
client, manifest,
|
||||
base_url=base_url, api_key=api_key, model=model,
|
||||
system_prompt=SEARCH_MISSION if args.goal == "search" else MISSION,
|
||||
log=log,
|
||||
)
|
||||
if args.goal == "search":
|
||||
ctl.messages = [
|
||||
ctl.messages[0],
|
||||
{"role": "user", "content": (
|
||||
"MISSION: explore the room, find the orange triangle on the "
|
||||
"back of a crate, and report it. Keep exploring until found."
|
||||
)},
|
||||
]
|
||||
chat.add("mission", "MISSION: find the orange triangle")
|
||||
else:
|
||||
ctl.messages.append({
|
||||
"role": "user",
|
||||
"content": (
|
||||
"MISSION: reach the beacon and activate it. "
|
||||
"Keep calling tools until it is done. Move smoothly (use "
|
||||
"duration on move/turn). The capsule cruises while you think."
|
||||
),
|
||||
})
|
||||
chat.add("mission", "MISSION: reach and activate the beacon")
|
||||
|
||||
if args.goal == "search" and not args.capture_hz:
|
||||
# capture at frame-attach events (one frame per step)
|
||||
orig_frame = ctl._frame_message
|
||||
|
||||
def record_frame(png: str, note: str):
|
||||
nonlocal marker_seen
|
||||
view = (
|
||||
Image.open(io.BytesIO(base64.b64decode(png)))
|
||||
.convert("RGB")
|
||||
.resize((PANEL, PANEL))
|
||||
)
|
||||
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||
frames.append(_compose(top, view, chat.render()))
|
||||
samples.append((bridge.world.capsule.x, bridge.world.capsule.z))
|
||||
return orig_frame(png, note)
|
||||
|
||||
ctl._frame_message = record_frame # type: ignore[method-assign]
|
||||
|
||||
def is_success(turn):
|
||||
nonlocal marker_seen
|
||||
for c in turn.calls:
|
||||
if c.name == "report" and c.ok and c.output and c.output.get("verified"):
|
||||
marker_seen = True
|
||||
return True
|
||||
return False
|
||||
|
||||
async def observer(agent_done: asyncio.Event) -> None:
|
||||
"""Sample the world at a fixed rate so the GIF shows the actual
|
||||
gliding motion (mid-animation frames), not just step endpoints."""
|
||||
obs_transport = WebSocketClientTransport("ws://127.0.0.1:8765")
|
||||
async with AICCClient(obs_transport) as obs:
|
||||
await obs.handshake()
|
||||
while not agent_done.is_set():
|
||||
try:
|
||||
out = (await obs.call_tool("vision", {})).output
|
||||
except Exception: # noqa: BLE001 - session teardown
|
||||
break
|
||||
view = (
|
||||
Image.open(io.BytesIO(base64.b64decode(out["png_b64"])))
|
||||
.convert("RGB")
|
||||
.resize((PANEL, PANEL))
|
||||
)
|
||||
samples.append((bridge.world.capsule.x, bridge.world.capsule.z))
|
||||
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||
frames.append(_compose(top, view, chat.render()))
|
||||
if len(frames) > 350: # keep the GIF bounded
|
||||
frames[:150] = []
|
||||
await asyncio.sleep(1.0 / args.capture_hz)
|
||||
|
||||
agent_done = asyncio.Event()
|
||||
obs_task = None
|
||||
if args.capture_hz:
|
||||
obs_task = asyncio.create_task(observer(agent_done))
|
||||
try:
|
||||
summary = await run_llm_agent_loop(
|
||||
ctl, args.steps, log=log,
|
||||
look_every=2 if args.goal == "beacon" else 1,
|
||||
cruise=args.cruise,
|
||||
autonomous=args.goal == "search",
|
||||
is_success=is_success if args.goal == "search" else None,
|
||||
)
|
||||
finally:
|
||||
agent_done.set()
|
||||
if obs_task is not None:
|
||||
await obs_task
|
||||
|
||||
# Final frame: end state of the map.
|
||||
if marker_seen:
|
||||
chat.add("mission", f"DONE: {summary.get('result', 'triangle found')}")
|
||||
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||
last_view = (
|
||||
frames[-1].crop((PANEL, 0, PANEL * 2, PANEL))
|
||||
if frames
|
||||
else Image.new("RGB", (PANEL, PANEL), (0, 0, 0))
|
||||
)
|
||||
frames.append(_compose(top, last_view, chat.render()))
|
||||
|
||||
# Smoothness check: mean movement between consecutive captured frames.
|
||||
if len(samples) > 2:
|
||||
from itertools import pairwise
|
||||
|
||||
deltas = [
|
||||
((b[0] - a[0]) ** 2 + (b[1] - a[1]) ** 2) ** 0.5
|
||||
for a, b in pairwise(samples)
|
||||
]
|
||||
moving = [d for d in deltas if d > 0.02]
|
||||
if moving:
|
||||
print(f"[record] motion between frames: mean {sum(moving)/len(moving):.2f} m "
|
||||
f"(max {max(moving):.2f}) over {len(moving)} moving samples")
|
||||
|
||||
out = Path(args.out)
|
||||
if frames:
|
||||
frames[0].save(
|
||||
out,
|
||||
save_all=True,
|
||||
append_images=frames[1:],
|
||||
duration=args.duration,
|
||||
loop=0,
|
||||
)
|
||||
frames[-1].save(out.with_suffix(".png"))
|
||||
print(f"[record] wrote {out} ({len(frames)} frames, {args.duration} ms/frame)")
|
||||
print(f"[record] mission result: {summary.get('result')}")
|
||||
return 0 if (marker_seen or summary.get("interacted")) else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Record a mission into a three-view GIF (map | first person | chat)."
|
||||
)
|
||||
parser.add_argument("--out", default="search_mission.gif")
|
||||
parser.add_argument(
|
||||
"--goal",
|
||||
choices=["search", "beacon"],
|
||||
default="search",
|
||||
help="which mission to record (default search)",
|
||||
)
|
||||
parser.add_argument("--provider", default=None)
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--api-key", default=None)
|
||||
parser.add_argument("--model", default=None)
|
||||
parser.add_argument("--steps", type=int, default=45)
|
||||
parser.add_argument("--cruise", type=float, default=0.6)
|
||||
parser.add_argument("--duration", type=int, default=150, help="ms per GIF frame")
|
||||
parser.add_argument(
|
||||
"--capture-hz",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="sample frames at this rate (Hz) with an observer client to show "
|
||||
"gliding motion; 0 = one frame per step (search default)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except KeyboardInterrupt:
|
||||
print("\n[record] interrupted — the partial GIF was NOT written")
|
||||
return 130
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -114,6 +114,7 @@ class Raycaster:
|
||||
self._draw_beacon(room, pixels, depth, fx, fz, rx, rz, eye, tan_pitch)
|
||||
|
||||
img = Image.frombuffer("RGB", (w, h), bytes(pixels), "raw", "RGB", 0, 1)
|
||||
self._draw_marker(room, img, depth, fx, fz, rx, rz, eye, tan_pitch)
|
||||
grid = self._depth_grid(depth, max_depth)
|
||||
return img, grid
|
||||
|
||||
@@ -260,6 +261,65 @@ class Raycaster:
|
||||
col = FLOOR_LINE
|
||||
return _fog(col, t)
|
||||
|
||||
def _draw_marker(
|
||||
self,
|
||||
room: Room,
|
||||
img: Image.Image,
|
||||
depth: list[list[float]],
|
||||
fx: float,
|
||||
fz: float,
|
||||
rx: float,
|
||||
rz: float,
|
||||
eye: float,
|
||||
tan_pitch: float,
|
||||
) -> None:
|
||||
"""Draw the search-challenge triangle (a sprite on the crate's back
|
||||
face), only when the anchor pixel is actually that face."""
|
||||
m = room.marker
|
||||
if m is None:
|
||||
return
|
||||
pos = room.marker_world_pos()
|
||||
if pos is None:
|
||||
return
|
||||
w, h = self.width, self.height
|
||||
cam = room.capsule
|
||||
rel_x = pos[0] - cam.x
|
||||
rel_z = pos[2] - cam.z
|
||||
along = rel_x * fx + rel_z * fz
|
||||
if along < 0.3:
|
||||
return
|
||||
right = rel_x * rx + rel_z * rz
|
||||
col_c = (right / along / self.tan_fx + 1.0) / 2.0 * w
|
||||
vv = (pos[1] - eye) / along - tan_pitch
|
||||
row_c = (1.0 - vv / self.tan_fy) / 2.0 * h
|
||||
r0, c0 = round(row_c), round(col_c)
|
||||
if not (0 <= r0 < h and 0 <= c0 < w):
|
||||
return
|
||||
if abs(depth[r0][c0] - along) > 0.25:
|
||||
return # the anchor pixel is not the marker's face (occluded)
|
||||
half = max(2.5, m["size"] / along / self.tan_fx * w / 2.0)
|
||||
draw = ImageDraw.Draw(img)
|
||||
color = tuple(m["color"])
|
||||
glow = tuple(min(255, c + 90) for c in color)
|
||||
draw.ellipse(
|
||||
[
|
||||
col_c - half * 2.2,
|
||||
row_c - half * 2.2,
|
||||
col_c + half * 2.2,
|
||||
row_c + half * 2.2,
|
||||
],
|
||||
fill=glow,
|
||||
)
|
||||
draw.polygon(
|
||||
[
|
||||
(col_c, row_c - half),
|
||||
(col_c - half * 0.9, row_c + half * 0.7),
|
||||
(col_c + half * 0.9, row_c + half * 0.7),
|
||||
],
|
||||
fill=color,
|
||||
outline=(255, 255, 255),
|
||||
)
|
||||
|
||||
def _draw_beacon(
|
||||
self,
|
||||
room: Room,
|
||||
@@ -405,6 +465,22 @@ def render_topdown(room: Room, size: int = 400) -> Image.Image:
|
||||
fx, fz = c.forward()
|
||||
tip = xy(c.x + fx * 0.7, c.z + fz * 0.7)
|
||||
draw.line([(cx, cz), tip], fill=(30, 30, 40), width=3)
|
||||
|
||||
m = room.marker
|
||||
if m is not None:
|
||||
pos = room.marker_world_pos()
|
||||
if pos is not None:
|
||||
mx, mz = xy(pos[0], pos[2])
|
||||
r = m["size"] * scale
|
||||
draw.polygon(
|
||||
[
|
||||
(mx, mz - r * 1.4),
|
||||
(mx - r, mz + r),
|
||||
(mx + r, mz + r),
|
||||
],
|
||||
fill=tuple(m["color"]),
|
||||
outline=(20, 20, 26),
|
||||
)
|
||||
return img
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
ROOM_SIZE = 16.0
|
||||
ROOM_HEIGHT = 3.0
|
||||
@@ -124,6 +125,62 @@ class Room:
|
||||
]
|
||||
# Pending audio events, drained by the hear sensor.
|
||||
self._audio: list[dict] = []
|
||||
# Search challenge: a small triangle painted on the BACK side of one
|
||||
# of the crates (the face pointing away from the spawn corner). The
|
||||
# agent must find it by exploring and looking with vision.
|
||||
self.marker: dict[str, Any] = self._place_marker()
|
||||
|
||||
# ---------- search marker ----------
|
||||
|
||||
def _place_marker(self) -> dict[str, Any]:
|
||||
"""Put the triangle on a random crate, on the face most hidden from
|
||||
the spawn corner (1.5, 1.5)."""
|
||||
import random
|
||||
|
||||
box = random.choice(self.boxes)
|
||||
to_spawn = (1.5 - box.cx, 1.5 - box.cz)
|
||||
normals = {
|
||||
"x0": (-1.0, 0.0),
|
||||
"x1": (1.0, 0.0),
|
||||
"z0": (0.0, -1.0),
|
||||
"z1": (0.0, 1.0),
|
||||
}
|
||||
faces = sorted(
|
||||
normals,
|
||||
key=lambda f: normals[f][0] * to_spawn[0] + normals[f][1] * to_spawn[1],
|
||||
)[:2] # the two faces pointing away from spawn
|
||||
return {
|
||||
"box": box.id,
|
||||
"face": random.choice(faces),
|
||||
"u": 0.5,
|
||||
"v": 0.55,
|
||||
"size": 0.42,
|
||||
"color": (255, 150, 40),
|
||||
}
|
||||
|
||||
def marker_world_pos(self) -> tuple[float, float, float] | None:
|
||||
"""World coordinates (x, y, z) of the marker center, or None."""
|
||||
m = self.marker
|
||||
if m is None:
|
||||
return None
|
||||
box = next((b for b in self.boxes if b.id == m["box"]), None)
|
||||
if box is None:
|
||||
return None
|
||||
if m["face"] == "x0":
|
||||
x, z = box.x_min(), box.cz - box.half_d + m["u"] * 2 * box.half_d
|
||||
elif m["face"] == "x1":
|
||||
x, z = box.x_max(), box.cz - box.half_d + m["u"] * 2 * box.half_d
|
||||
elif m["face"] == "z0":
|
||||
x, z = box.cx - box.half_w + m["u"] * 2 * box.half_w, box.z_min()
|
||||
else:
|
||||
x, z = box.cx - box.half_w + m["u"] * 2 * box.half_w, box.z_max()
|
||||
return (x, m["v"] * box.height, z)
|
||||
|
||||
def distance_to_marker(self) -> float:
|
||||
pos = self.marker_world_pos()
|
||||
if pos is None:
|
||||
return float("inf")
|
||||
return math.hypot(pos[0] - self.capsule.x, pos[2] - self.capsule.z)
|
||||
|
||||
# ---------- ticks ----------
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for the testbed bridge: tool behavior over the in-process transport."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from aicc.client import AICCClient
|
||||
@@ -186,3 +188,112 @@ async def test_single_source_of_truth():
|
||||
world = dumped["world"]
|
||||
assert set(world.keys()) == {"name", "kind"}
|
||||
assert "beacon" not in dumped
|
||||
|
||||
|
||||
async def test_animated_move_is_smooth_and_observable(bridge):
|
||||
"""A move with duration glides: a concurrent client sees intermediate
|
||||
positions, and the final displacement is exact."""
|
||||
# A transport is a single session: one per client.
|
||||
t1 = InProcessTransport.start(bridge)
|
||||
t2 = InProcessTransport.start(bridge)
|
||||
async with t1, t2:
|
||||
mover = AICCClient(t1)
|
||||
viewer = AICCClient(t2)
|
||||
async with mover, viewer:
|
||||
await mover.handshake()
|
||||
await viewer.handshake()
|
||||
start = (await mover.call_tool("proprioception", {})).output["position"]
|
||||
assert start["x"] == 1.5 and start["z"] == 1.5
|
||||
|
||||
move_task = asyncio.create_task(
|
||||
mover.call_tool("move", {"forward": 2.0, "duration": 0.8})
|
||||
)
|
||||
await asyncio.sleep(0.25)
|
||||
mid = (await viewer.call_tool("proprioception", {})).output["position"]
|
||||
# Moving along the 45-degree diagonal: strictly between start and end.
|
||||
assert 1.5 < mid["x"] < 3.5 and 1.5 < mid["z"] < 3.5
|
||||
|
||||
res = (await move_task).output
|
||||
assert res["moved"] == pytest.approx(2.0, abs=1e-3)
|
||||
assert res["position"]["x"] == pytest.approx(
|
||||
1.5 + 2.0 * math.sin(math.radians(45)), abs=1e-3
|
||||
)
|
||||
assert res["duration"] == 0.8
|
||||
|
||||
|
||||
async def test_animated_move_stops_on_collision(bridge):
|
||||
t = InProcessTransport.start(bridge)
|
||||
async with t:
|
||||
client = AICCClient(t)
|
||||
async with client:
|
||||
await client.handshake()
|
||||
bridge.world.capsule.yaw_deg = 180.0 # straight at the north wall
|
||||
res = (
|
||||
await client.call_tool("move", {"forward": 5.0, "duration": 0.6})
|
||||
).output
|
||||
assert res["collision"] is True
|
||||
assert res["moved"] < 5.0
|
||||
assert res["position"]["z"] == pytest.approx(
|
||||
bridge.world.capsule.radius, abs=1e-3
|
||||
)
|
||||
|
||||
|
||||
async def test_move_accepts_duration_zero_default(bridge):
|
||||
t = InProcessTransport.start(bridge)
|
||||
async with t:
|
||||
client = AICCClient(t)
|
||||
async with client:
|
||||
await client.handshake()
|
||||
res = (await client.call_tool("move", {"forward": 1.0})).output
|
||||
assert res["moved"] == pytest.approx(1.0)
|
||||
assert res["duration"] == 0.0
|
||||
|
||||
|
||||
async def test_report_verifies_near_marker(bridge):
|
||||
t = InProcessTransport.start(bridge)
|
||||
async with t:
|
||||
client = AICCClient(t)
|
||||
async with client:
|
||||
await client.handshake()
|
||||
out = (await client.call_tool("report", {"discovery": "triangle"})).output
|
||||
assert out["success"] is True
|
||||
assert out["verified"] is False # far away from spawn
|
||||
pos = bridge.world.marker_world_pos()
|
||||
bridge.world.capsule.x = pos[0]
|
||||
bridge.world.capsule.z = pos[2]
|
||||
out2 = (await client.call_tool("report", {})).output
|
||||
assert out2["verified"] is True
|
||||
assert "verified" in out2["message"]
|
||||
|
||||
|
||||
async def test_turn_with_duration_is_smooth(bridge):
|
||||
t1 = InProcessTransport.start(bridge)
|
||||
t2 = InProcessTransport.start(bridge)
|
||||
async with t1, t2:
|
||||
mover = AICCClient(t1)
|
||||
viewer = AICCClient(t2)
|
||||
async with mover, viewer:
|
||||
await mover.handshake()
|
||||
await viewer.handshake()
|
||||
task = asyncio.create_task(
|
||||
mover.call_tool("turn", {"yaw_deg": 90.0, "duration": 0.8})
|
||||
)
|
||||
await asyncio.sleep(0.3)
|
||||
mid = (await viewer.call_tool("proprioception", {})).output["rotation"]["yaw_deg"]
|
||||
assert 0.0 < mid < 90.0, f"expected intermediate rotation, got {mid}"
|
||||
res = (await task).output
|
||||
assert res["rotation"]["yaw_deg"] == pytest.approx(135.0, abs=0.5)
|
||||
|
||||
|
||||
async def test_look_at_crate(bridge):
|
||||
t = InProcessTransport.start(bridge)
|
||||
async with t:
|
||||
client = AICCClient(t)
|
||||
async with client:
|
||||
await client.handshake()
|
||||
out = (await client.call_tool("look_at", {"target": "crate_red"})).output
|
||||
dx = bridge.world.boxes[0].cx - bridge.world.capsule.x
|
||||
dz = bridge.world.boxes[0].cz - bridge.world.capsule.z
|
||||
assert out["rotation"]["yaw_deg"] == pytest.approx(
|
||||
math.degrees(math.atan2(dx, dz)) % 360.0, abs=0.5
|
||||
)
|
||||
|
||||
@@ -131,3 +131,51 @@ def test_describe_layout():
|
||||
assert d["room"]["width"] == ROOM_SIZE
|
||||
assert len(d["obstacles"]) == 3
|
||||
assert d["beacon"]["id"] == "beacon"
|
||||
|
||||
|
||||
def test_marker_is_on_a_back_face():
|
||||
"""The triangle must sit on a face pointing away from the spawn corner."""
|
||||
for _ in range(20):
|
||||
room = Room()
|
||||
pos = room.marker_world_pos()
|
||||
assert pos is not None
|
||||
box = next(b for b in room.boxes if b.id == room.marker["box"])
|
||||
face = room.marker["face"]
|
||||
# spawn is at (1.5, 1.5); the marker face normal must point away from it
|
||||
to_spawn = (1.5 - box.cx, 1.5 - box.cz)
|
||||
normal = {
|
||||
"x0": (-1, 0), "x1": (1, 0), "z0": (0, -1), "z1": (0, 1),
|
||||
}[face]
|
||||
assert normal[0] * to_spawn[0] + normal[1] * to_spawn[1] < 0
|
||||
assert pos[0] == pytest.approx(box.x_max()) if face == "x1" else True
|
||||
|
||||
|
||||
def test_marker_not_visible_from_spawn():
|
||||
"""Looking toward the beacon from spawn must not show the triangle."""
|
||||
from testbed.room.render import Raycaster
|
||||
|
||||
room = Room()
|
||||
# Spawn looks at the beacon; the marker is on a back face, so the anchor
|
||||
# pixel of the marker must be occluded (different surface) or off-frame.
|
||||
pos = room.marker_world_pos()
|
||||
assert pos is not None
|
||||
# Stand on the marker's side of the crate, facing it.
|
||||
face = room.marker["face"]
|
||||
offsets = {
|
||||
"x0": (-1.0, 0.0, 90.0), # west face: stand west, face +x
|
||||
"x1": (1.0, 0.0, 270.0), # east face: stand east, face -x
|
||||
"z0": (0.0, -1.0, 0.0), # south face: stand south, face +z
|
||||
"z1": (0.0, 1.0, 180.0), # north face: stand north, face -z
|
||||
}[face]
|
||||
room.capsule.x = pos[0] + offsets[0]
|
||||
room.capsule.z = pos[2] + offsets[1]
|
||||
room.capsule.yaw_deg = offsets[2]
|
||||
img, _ = Raycaster().render(room)
|
||||
px = img.load()
|
||||
orange = 0
|
||||
for y in range(0, 120, 2):
|
||||
for x in range(0, 160, 2):
|
||||
r, g, b = px[x, y]
|
||||
if r > 200 and 100 < g < 190 and b < 90:
|
||||
orange += 1
|
||||
assert orange > 3, "triangle must be visible when looking at the marker face"
|
||||
|
||||