Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b64c4d4639 | ||
|
|
c07544eaa8 | ||
|
|
5baf698b9e | ||
|
|
e44977cbe4 | ||
|
|
b6b5547ffc | ||
|
|
d49cf46181 | ||
|
|
a9e45fb427 | ||
|
|
daa0371519 | ||
|
|
315762363e | ||
|
|
1cb3bdc46e | ||
|
|
b3d5c5630a | ||
|
|
0dd2e21683 |
@@ -0,0 +1,6 @@
|
|||||||
|
.env
|
||||||
|
.playwright-mcp/
|
||||||
|
chat_map.png
|
||||||
|
chat_frame.png
|
||||||
|
.opencode/
|
||||||
|
frames/
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
# Session Goal
|
|
||||||
|
|
||||||
реализуй полностью. автономно
|
|
||||||
|
|
||||||
## Plan
|
|
||||||
1. Read protocol spec (docs/aicc-core.md) and testbed design (docs/testbed-design.md)
|
|
||||||
2. Inspect aicc-py SDK at ~/Desktop/aicc-py
|
|
||||||
3. Build testbed bridge skeleton (world, capsule, bridge with tools)
|
|
||||||
4. Implement tools: proprioception, vision, move, turn, look_at, interact
|
|
||||||
5. Add headless rendering for vision frames
|
|
||||||
6. Write agent demo loop
|
|
||||||
7. Run conformance suite (9/9 green)
|
|
||||||
8. Verify demo end-to-end, write README, commit per milestone
|
|
||||||
|
|
||||||
## Progress
|
|
||||||
- DONE: world engine (room, capsule physics/collision, beacon, audio) — testbed/room/world.py
|
|
||||||
- DONE: headless raycaster renderer (vision 160x120 + aligned depth grid) — testbed/room/render.py
|
|
||||||
- DONE: RoomBridge with all world tools + echo/boom/bump — testbed/bridge.py
|
|
||||||
- DONE: conformance runner — 9/9 green against testbed bridge (testbed/conformance.py)
|
|
||||||
- DONE: pytest suite — 33 tests green; ruff clean
|
|
||||||
- DONE: agent demo loop — LLM (OpenAI-compatible/ollama) + scripted fallback + auto handoff; E2E verified over WebSocket (scripted: 26 steps, beacon activated; LLM mode completed missions with gemma4:e2b)
|
|
||||||
- DONE: testbed/README.md, scripts/setup.sh deps, scripts/run_bridge.sh, BUILDLOG.md updated
|
|
||||||
- DONE: commits per milestone (skeleton, bridge+tools+conformance, tests+physics fixes, demo loop, docs, lint)
|
|
||||||
- VERIFIED: conformance 9/9, pytest 33/33, ruff clean, demo end-to-end
|
|
||||||
|
|
||||||
COMPLETED
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
[ 28151ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 35175ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 36177ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 37178ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 38180ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 39182ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 40184ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 49096ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 50098ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 51100ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 52101ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 53103ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 54105ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 55108ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 56109ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 57112ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 58115ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 59117ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 60119ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 61123ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 62129ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 63134ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 64140ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 78549ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 88888ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
[ 89892ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
- generic [active] [ref=e1]:
|
|
||||||
- banner [ref=e2]:
|
|
||||||
- heading "AICC capsule — real-time view" [level=1] [ref=e3]
|
|
||||||
- generic [ref=e4]: connected — waiting for frames
|
|
||||||
- main [ref=e5]:
|
|
||||||
- generic [ref=e6]:
|
|
||||||
- heading "first person (vision)" [level=2] [ref=e7]
|
|
||||||
- img "vision" [ref=e8]
|
|
||||||
- generic [ref=e9]:
|
|
||||||
- heading "top-down (sensors)" [level=2] [ref=e10]
|
|
||||||
- img "map" [ref=e11]
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
- generic [active] [ref=e1]:
|
|
||||||
- banner [ref=e2]:
|
|
||||||
- heading "AICC capsule — real-time view" [level=1] [ref=e3]
|
|
||||||
- generic [ref=e4]: connected — waiting for frames
|
|
||||||
- main [ref=e5]:
|
|
||||||
- generic [ref=e6]:
|
|
||||||
- heading "first person (vision)" [level=2] [ref=e7]
|
|
||||||
- img "vision" [ref=e8]
|
|
||||||
- generic [ref=e9]:
|
|
||||||
- heading "top-down (sensors)" [level=2] [ref=e10]
|
|
||||||
- img "map" [ref=e11]
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
- generic [active] [ref=f1e1]:
|
|
||||||
- banner [ref=f1e2]:
|
|
||||||
- heading "AICC capsule — real-time view" [level=1] [ref=f1e3]
|
|
||||||
- generic [ref=f1e4]: connected — waiting for frames
|
|
||||||
- main [ref=f1e5]:
|
|
||||||
- generic [ref=f1e6]:
|
|
||||||
- heading "first person (vision)" [level=2] [ref=f1e7]
|
|
||||||
- img "vision" [ref=f1e8]
|
|
||||||
- generic [ref=f1e9]:
|
|
||||||
- heading "top-down (sensors)" [level=2] [ref=f1e10]
|
|
||||||
- img "map" [ref=f1e11]
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
- generic [active] [ref=f1e1]:
|
|
||||||
- banner [ref=f1e2]:
|
|
||||||
- heading "AICC capsule — real-time view" [level=1] [ref=f1e3]
|
|
||||||
- generic [ref=f1e4]: tick 158 pos (11.86, 11.22) yaw 26.5° 1.43 m to beacon · 3.0 fps
|
|
||||||
- main [ref=f1e5]:
|
|
||||||
- generic [ref=f1e6]:
|
|
||||||
- heading "first person (vision)" [level=2] [ref=f1e7]
|
|
||||||
- img "vision" [ref=f1e8]
|
|
||||||
- generic [ref=f1e9]:
|
|
||||||
- heading "top-down (sensors)" [level=2] [ref=f1e10]
|
|
||||||
- img "map" [ref=f1e11]
|
|
||||||
|
Before Width: | Height: | Size: 40 KiB After Width: | Height: | Size: 40 KiB |
@@ -14,6 +14,31 @@ an agent (LLM or scripted) navigates to the beacon and activates it. See
|
|||||||
`testbed/README.md` for run instructions; `scripts/BUILDLOG.md` for the build
|
`testbed/README.md` for run instructions; `scripts/BUILDLOG.md` for the build
|
||||||
log.
|
log.
|
||||||
|
|
||||||
|
## Search challenge
|
||||||
|
|
||||||
|
The agent can also be sent on an autonomous search: an orange triangle is
|
||||||
|
hidden on the back side of one of the crates, and the model must explore the
|
||||||
|
room with real vision, spot it, and report it. This recording shows the run
|
||||||
|
from three angles — the top-down map, the actual first-person frames the
|
||||||
|
model saw, and its chat/reasoning (recorded with smooth motion, ~2 Hz):
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
Regenerate it with `python -m testbed.record_search --provider polza --goal search
|
||||||
|
--out search_mission.gif --capture-hz 2.0 --cruise 0.8 --steps 60`.
|
||||||
|
|
||||||
|
## Smooth & proactive movement
|
||||||
|
|
||||||
|
The same recording setup with a beacon mission and the cruise mode — the
|
||||||
|
capsule glides with animated `move`/`turn` calls and keeps moving while the
|
||||||
|
model thinks (`--cruise`). Frames are sampled at ~2.5 Hz by an observer
|
||||||
|
client, so the GIF shows the actual motion, not step endpoints:
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
Regenerate with `python -m testbed.record_search --provider polza --goal beacon
|
||||||
|
--out smooth_mission.gif --capture-hz 2.5 --cruise 1.0`.
|
||||||
|
|
||||||
## Layout
|
## Layout
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 2.9 KiB |
|
Before Width: | Height: | Size: 40 KiB After Width: | Height: | Size: 2.9 MiB |
|
Before Width: | Height: | Size: 32 KiB |
|
Before Width: | Height: | Size: 3.6 KiB |
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 3.8 KiB |
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 2.3 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 1.6 KiB |
|
Before Width: | Height: | Size: 4.3 KiB |
|
Before Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 4.3 KiB |
|
Before Width: | Height: | Size: 1.8 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 2.7 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 4.7 KiB |
|
Before Width: | Height: | Size: 2.9 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 4.8 KiB |
|
Before Width: | Height: | Size: 3.2 KiB |
|
Before Width: | Height: | Size: 4.7 KiB |
|
Before Width: | Height: | Size: 3.5 KiB |
|
Before Width: | Height: | Size: 4.7 KiB |
|
Before Width: | Height: | Size: 3.7 KiB |
|
Before Width: | Height: | Size: 4.9 KiB |
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 5.1 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 6.2 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 7.6 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 8.3 KiB |
@@ -3,6 +3,14 @@
|
|||||||
# Usage: scripts/run_bridge.sh [port] (default 8765)
|
# Usage: scripts/run_bridge.sh [port] (default 8765)
|
||||||
# Note: the bridge keeps running until interrupted (Ctrl-C).
|
# Note: the bridge keeps running until interrupted (Ctrl-C).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Load API keys from the project .env file (if present)
|
||||||
|
if [ -f .env ]; then
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
PORT="${1:-8765}"
|
PORT="${1:-8765}"
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,14 @@
|
|||||||
# Open http://127.0.0.1:8000 to watch the capsule while you talk to it.
|
# Open http://127.0.0.1:8000 to watch the capsule while you talk to it.
|
||||||
# Usage: scripts/run_chat.sh [--model gemma4:e2b]
|
# Usage: scripts/run_chat.sh [--model gemma4:e2b]
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Load API keys from the project .env file (if present)
|
||||||
|
if [ -f .env ]; then
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
BRIDGE_PID=""
|
BRIDGE_PID=""
|
||||||
LIVE_PID=""
|
LIVE_PID=""
|
||||||
|
|||||||
@@ -5,5 +5,13 @@
|
|||||||
# scripts/run_demo.sh --agent scripted
|
# scripts/run_demo.sh --agent scripted
|
||||||
# scripts/run_demo.sh --agent llm --model gemma4:e2b
|
# scripts/run_demo.sh --agent llm --model gemma4:e2b
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Load API keys from the project .env file (if present)
|
||||||
|
if [ -f .env ]; then
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
exec testbed/.venv/bin/python -m testbed.demo "$@"
|
exec testbed/.venv/bin/python -m testbed.demo "$@"
|
||||||
|
|||||||
@@ -3,6 +3,14 @@
|
|||||||
# demo while you watch. Open http://127.0.0.1:8000 in a browser.
|
# demo while you watch. Open http://127.0.0.1:8000 in a browser.
|
||||||
# Usage: scripts/run_live.sh [demo args...] e.g. --agent scripted
|
# Usage: scripts/run_live.sh [demo args...] e.g. --agent scripted
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Load API keys from the project .env file (if present)
|
||||||
|
if [ -f .env ]; then
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
BRIDGE_PID=""
|
BRIDGE_PID=""
|
||||||
LIVE_PID=""
|
LIVE_PID=""
|
||||||
|
|||||||
|
After Width: | Height: | Size: 4.4 MiB |
@@ -64,6 +64,39 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal.
|
|||||||
The demo prints a full transcript of tool calls/results to stdout and saves the
|
The demo prints a full transcript of tool calls/results to stdout and saves the
|
||||||
capsule's final first-person frame to `demo_final_frame.png`.
|
capsule's final first-person frame to `demo_final_frame.png`.
|
||||||
|
|
||||||
|
### Search challenge
|
||||||
|
|
||||||
|
A small orange triangle is painted on the **back side** of one of the crates
|
||||||
|
(random per run, always on a face hidden from the spawn corner). The agent must
|
||||||
|
explore the room, look at the crate faces, spot the triangle with `vision`,
|
||||||
|
and `report` it (the bridge verifies the report by distance):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/run_chat.sh --provider polza --search
|
||||||
|
# autonomous exploration, real vision, smooth turns, cruise gliding;
|
||||||
|
# success = report verified
|
||||||
|
```
|
||||||
|
|
||||||
|
Search missions run in `--free` mode: no re-aim corrections, no collision
|
||||||
|
hints — the model plans its own exploration. `--free` also applies to beacon
|
||||||
|
missions if you want fewer guardrails. `look_at` now accepts crates too
|
||||||
|
(crate_red/crate_blue/crate_olive). Verified: gpt-5.6-luna explored the room
|
||||||
|
with smooth moves/turns and reported the triangle at 0.5 m.
|
||||||
|
|
||||||
|
### Smooth & proactive movement
|
||||||
|
|
||||||
|
The capsule no longer teleports or stops-and-thinks:
|
||||||
|
|
||||||
|
- **Smooth**: `move(forward, duration)` animates the displacement over
|
||||||
|
`duration` seconds on the bridge, so the live viewer shows genuine gliding.
|
||||||
|
The scripted agent and auto-cruise use it by default; the LLM can too.
|
||||||
|
- **Proactive** (`--cruise N`, meters per think): while the model is
|
||||||
|
generating a response, the capsule keeps gliding forward (low-level
|
||||||
|
controller pattern, like a real robot). Collisions stop the drift safely
|
||||||
|
and are reported to the model, which re-plans. Verified: gpt-5.6-luna
|
||||||
|
completed the beacon mission with cruise on, bumping into and avoiding
|
||||||
|
crates while "thinking".
|
||||||
|
|
||||||
## Providers
|
## Providers
|
||||||
|
|
||||||
Provider presets resolve endpoint + API key + default model:
|
Provider presets resolve endpoint + API key + default model:
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ class MoveOutput(BaseModel):
|
|||||||
collision_normal: Vec3 | None
|
collision_normal: Vec3 | None
|
||||||
position: Vec3
|
position: Vec3
|
||||||
tick: int
|
tick: int
|
||||||
|
duration: float = 0.0
|
||||||
|
|
||||||
|
|
||||||
class TurnOutput(BaseModel):
|
class TurnOutput(BaseModel):
|
||||||
@@ -112,6 +113,15 @@ class WorldQueryOutput(BaseModel):
|
|||||||
tick: int
|
tick: int
|
||||||
|
|
||||||
|
|
||||||
|
class ReportOutput(BaseModel):
|
||||||
|
discovery: str
|
||||||
|
success: bool
|
||||||
|
verified: bool
|
||||||
|
distance: float
|
||||||
|
message: str
|
||||||
|
tick: int
|
||||||
|
|
||||||
|
|
||||||
class EchoOutput(BaseModel):
|
class EchoOutput(BaseModel):
|
||||||
value: str
|
value: str
|
||||||
|
|
||||||
@@ -165,6 +175,42 @@ class RoomBridge(Bridge):
|
|||||||
def _tick(self) -> int:
|
def _tick(self) -> int:
|
||||||
return self.world.advance_tick()
|
return self.world.advance_tick()
|
||||||
|
|
||||||
|
async def _animate_turn(
|
||||||
|
self, yaw_deg: float, pitch_deg: float, duration: float
|
||||||
|
) -> None:
|
||||||
|
"""Rotate smoothly over ``duration`` seconds (observable mid-turn)."""
|
||||||
|
n = max(1, min(int(duration / 0.1), 100))
|
||||||
|
dt = duration / n
|
||||||
|
for i in range(n):
|
||||||
|
self.world.turn(yaw_deg / n, pitch_deg / n)
|
||||||
|
if i < n - 1:
|
||||||
|
await asyncio.sleep(dt)
|
||||||
|
|
||||||
|
async def _animate_move(
|
||||||
|
self, forward: float, duration: float
|
||||||
|
) -> tuple[float, Any | None]:
|
||||||
|
"""Smoothly displace the capsule over ``duration`` seconds.
|
||||||
|
|
||||||
|
The position advances in small timed steps, so concurrent clients
|
||||||
|
(e.g. the live viewer) observe genuine gliding motion instead of a
|
||||||
|
teleport. Collision resolution is unchanged.
|
||||||
|
"""
|
||||||
|
world = self.world
|
||||||
|
n = max(1, min(int(duration / 0.1), 200))
|
||||||
|
step = forward / n
|
||||||
|
dt = duration / n
|
||||||
|
total = 0.0
|
||||||
|
hit: Any | None = None
|
||||||
|
for i in range(n):
|
||||||
|
moved, h = world.move_forward(step)
|
||||||
|
total += moved
|
||||||
|
if h is not None or moved < step * 0.01:
|
||||||
|
hit = h
|
||||||
|
break
|
||||||
|
if i < n - 1:
|
||||||
|
await asyncio.sleep(dt)
|
||||||
|
return total, hit
|
||||||
|
|
||||||
def _vec3(self, x: float, y: float, z: float) -> Vec3:
|
def _vec3(self, x: float, y: float, z: float) -> Vec3:
|
||||||
return Vec3(x=round(x, 3), y=round(y, 3), z=round(z, 3))
|
return Vec3(x=round(x, 3), y=round(y, 3), z=round(z, 3))
|
||||||
|
|
||||||
@@ -263,13 +309,21 @@ class RoomBridge(Bridge):
|
|||||||
|
|
||||||
@self.tool(
|
@self.tool(
|
||||||
cls=ToolClass.ACTUATOR,
|
cls=ToolClass.ACTUATOR,
|
||||||
description="Move the capsule forward along its heading by the given distance in meters.",
|
description=(
|
||||||
|
"Move the capsule forward along its heading by the given distance in meters. "
|
||||||
|
"duration (seconds) animates the motion smoothly instead of teleporting."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
async def move(forward: float = 1.0) -> MoveOutput:
|
async def move(forward: float = 1.0, duration: float = 0.0) -> MoveOutput:
|
||||||
tick = self._tick()
|
tick = self._tick()
|
||||||
if forward < 0.0 or forward > 5.0:
|
if forward < 0.0 or forward > 5.0:
|
||||||
raise ValueError(f"forward must be in [0, 5] m, got {forward}")
|
raise ValueError(f"forward must be in [0, 5] m, got {forward}")
|
||||||
moved, hit = world.move_forward(forward)
|
if duration < 0.0 or duration > 10.0:
|
||||||
|
raise ValueError(f"duration must be in [0, 10] s, got {duration}")
|
||||||
|
if duration > 0.0:
|
||||||
|
moved, hit = await self._animate_move(forward, duration)
|
||||||
|
else:
|
||||||
|
moved, hit = world.move_forward(forward)
|
||||||
c = world.capsule
|
c = world.capsule
|
||||||
normal = None
|
normal = None
|
||||||
if hit is not None:
|
if hit is not None:
|
||||||
@@ -294,28 +348,53 @@ class RoomBridge(Bridge):
|
|||||||
collision_normal=normal,
|
collision_normal=normal,
|
||||||
position=self._vec3(c.x, 0.0, c.z),
|
position=self._vec3(c.x, 0.0, c.z),
|
||||||
tick=tick,
|
tick=tick,
|
||||||
|
duration=round(duration, 3),
|
||||||
).model_dump()
|
).model_dump()
|
||||||
|
|
||||||
@self.tool(
|
@self.tool(
|
||||||
cls=ToolClass.ACTUATOR,
|
cls=ToolClass.ACTUATOR,
|
||||||
description="Rotate the camera: yaw_deg turns left/right, pitch_deg tilts up/down.",
|
description=(
|
||||||
|
"Rotate the camera: yaw_deg turns left/right, pitch_deg tilts up/down. "
|
||||||
|
"duration (seconds) animates the rotation smoothly."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
async def turn(yaw_deg: float = 0.0, pitch_deg: float = 0.0) -> TurnOutput:
|
async def turn(
|
||||||
|
yaw_deg: float = 0.0, pitch_deg: float = 0.0, duration: float = 0.0
|
||||||
|
) -> TurnOutput:
|
||||||
tick = self._tick()
|
tick = self._tick()
|
||||||
world.turn(yaw_deg, pitch_deg)
|
if duration < 0.0 or duration > 10.0:
|
||||||
|
raise ValueError(f"duration must be in [0, 10] s, got {duration}")
|
||||||
|
if duration > 0.0:
|
||||||
|
await self._animate_turn(yaw_deg, pitch_deg, duration)
|
||||||
|
else:
|
||||||
|
world.turn(yaw_deg, pitch_deg)
|
||||||
return TurnOutput(rotation=self._rotation(), tick=tick).model_dump()
|
return TurnOutput(rotation=self._rotation(), tick=tick).model_dump()
|
||||||
|
|
||||||
@self.tool(
|
@self.tool(
|
||||||
cls=ToolClass.ACTUATOR,
|
cls=ToolClass.ACTUATOR,
|
||||||
description="Orient the camera toward a named target (e.g. 'beacon').",
|
description=(
|
||||||
|
"Orient the camera toward a named target: 'beacon' or a crate "
|
||||||
|
"(crate_red, crate_blue, crate_olive)."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
async def look_at(target: str) -> LookAtOutput:
|
async def look_at(target: str) -> LookAtOutput:
|
||||||
tick = self._tick()
|
tick = self._tick()
|
||||||
if target != "beacon":
|
box = next((b for b in world.boxes if b.id == target), None)
|
||||||
|
if target == "beacon":
|
||||||
|
yaw, pitch = world.look_at_beacon()
|
||||||
|
elif box is not None:
|
||||||
|
dx = box.cx - world.capsule.x
|
||||||
|
dz = box.cz - world.capsule.z
|
||||||
|
dist = math.hypot(dx, dz) or 1.0
|
||||||
|
yaw = math.degrees(math.atan2(dx, dz))
|
||||||
|
pitch = math.degrees(
|
||||||
|
math.atan2(box.height / 2 - world.capsule.eye_height, dist)
|
||||||
|
)
|
||||||
|
world.face(yaw, pitch)
|
||||||
|
else:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"unknown target {target!r}; available targets: beacon"
|
f"unknown target {target!r}; available: beacon, crate_red, crate_blue, crate_olive"
|
||||||
)
|
)
|
||||||
yaw, pitch = world.look_at_beacon()
|
|
||||||
return LookAtOutput(
|
return LookAtOutput(
|
||||||
target=target,
|
target=target,
|
||||||
rotation=Rotation(
|
rotation=Rotation(
|
||||||
@@ -348,6 +427,33 @@ class RoomBridge(Bridge):
|
|||||||
target=target, success=success, message=message, tick=tick
|
target=target, success=success, message=message, tick=tick
|
||||||
).model_dump()
|
).model_dump()
|
||||||
|
|
||||||
|
@self.tool(
|
||||||
|
cls=ToolClass.ACTUATOR,
|
||||||
|
description=(
|
||||||
|
"Report a discovery to the operator (used in search missions): "
|
||||||
|
"say what you found. The report is verified against your position."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
async def report(discovery: str = "triangle") -> ReportOutput:
|
||||||
|
tick = self._tick()
|
||||||
|
dist = world.distance_to_marker()
|
||||||
|
verified = world.marker is not None and dist <= 6.0
|
||||||
|
if verified:
|
||||||
|
message = f"report verified: {discovery} sighted {dist:.1f} m away"
|
||||||
|
else:
|
||||||
|
message = (
|
||||||
|
f"not verified: nothing matching {discovery!r} within 6 m "
|
||||||
|
f"(closest check {dist:.1f} m) — keep exploring"
|
||||||
|
)
|
||||||
|
return ReportOutput(
|
||||||
|
discovery=discovery,
|
||||||
|
success=True,
|
||||||
|
verified=verified,
|
||||||
|
distance=round(dist, 2),
|
||||||
|
message=message,
|
||||||
|
tick=tick,
|
||||||
|
).model_dump()
|
||||||
|
|
||||||
# -- conformance tools (design doc: register alongside world tools) --
|
# -- conformance tools (design doc: register alongside world tools) --
|
||||||
|
|
||||||
@self.tool(
|
@self.tool(
|
||||||
|
|||||||
@@ -141,16 +141,30 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
|||||||
async def mission_runner(goal_text: str) -> None:
|
async def mission_runner(goal_text: str) -> None:
|
||||||
from testbed.llm_agent import run_llm_agent_loop
|
from testbed.llm_agent import run_llm_agent_loop
|
||||||
|
|
||||||
controller.messages.append(
|
if args.search:
|
||||||
{
|
from testbed.llm_agent import SEARCH_MISSION
|
||||||
"role": "user",
|
|
||||||
"content": (
|
controller.messages = [
|
||||||
f"MISSION (set by the user): {goal_text}. "
|
{"role": "system", "content": SEARCH_MISSION},
|
||||||
"Keep calling tools and do not stop until the goal is achieved. "
|
{
|
||||||
"Report only when done."
|
"role": "user",
|
||||||
),
|
"content": (
|
||||||
}
|
f"MISSION: {goal_text}. "
|
||||||
)
|
"Keep exploring with tools until the triangle is found and reported."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
controller.messages.append(
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
f"MISSION (set by the user): {goal_text}. "
|
||||||
|
"Keep calling tools and do not stop until the goal is achieved. "
|
||||||
|
"Report only when done."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
saver = asyncio.create_task(map_saver())
|
saver = asyncio.create_task(map_saver())
|
||||||
try:
|
try:
|
||||||
for attempt in range(1, args.mission_retries + 2):
|
for attempt in range(1, args.mission_retries + 2):
|
||||||
@@ -164,12 +178,28 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
|||||||
log=mission_log,
|
log=mission_log,
|
||||||
nudge_limit=args.mission_nudges,
|
nudge_limit=args.mission_nudges,
|
||||||
look_every=args.look_every,
|
look_every=args.look_every,
|
||||||
|
cruise=args.cruise,
|
||||||
|
autonomous=args.free or args.search,
|
||||||
|
is_success=(
|
||||||
|
(
|
||||||
|
lambda turn: any(
|
||||||
|
c.name == "report"
|
||||||
|
and c.ok
|
||||||
|
and isinstance(c.output, dict)
|
||||||
|
and c.output.get("verified")
|
||||||
|
for c in turn.calls
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if args.search
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
except RuntimeError as exc:
|
except RuntimeError as exc:
|
||||||
print(f" [mission] LLM error: {exc}")
|
print(f" [mission] LLM error: {exc}")
|
||||||
break
|
break
|
||||||
save_map()
|
save_map()
|
||||||
if summary["interacted"]:
|
done = summary.get("success") if args.search else summary["interacted"]
|
||||||
|
if done:
|
||||||
print(f"\n[mission] DONE: {summary['result']}")
|
print(f"\n[mission] DONE: {summary['result']}")
|
||||||
return
|
return
|
||||||
print(
|
print(
|
||||||
@@ -225,7 +255,12 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
|||||||
print(" (no mission running)")
|
print(" (no mission running)")
|
||||||
await cmd_state()
|
await cmd_state()
|
||||||
|
|
||||||
if args.mission:
|
if args.mission or args.search:
|
||||||
|
if args.search:
|
||||||
|
mission_goal = (
|
||||||
|
"explore the room, find the orange triangle painted on the back "
|
||||||
|
"of one of the crates, and report it"
|
||||||
|
)
|
||||||
start_mission()
|
start_mission()
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
@@ -307,6 +342,8 @@ async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> i
|
|||||||
steps_used += 1
|
steps_used += 1
|
||||||
if turn.text:
|
if turn.text:
|
||||||
print(f" model> {turn.text}")
|
print(f" model> {turn.text}")
|
||||||
|
if turn.thinking:
|
||||||
|
print(f" [thinking] {turn.thinking[:300]}")
|
||||||
if turn.interacted:
|
if turn.interacted:
|
||||||
print(f"\n[chat] BEACON ACTIVATED: {turn.message}")
|
print(f"\n[chat] BEACON ACTIVATED: {turn.message}")
|
||||||
save_map()
|
save_map()
|
||||||
@@ -427,6 +464,17 @@ def main() -> int:
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="start an autonomous mission on connect: reach and activate the beacon",
|
help="start an autonomous mission on connect: reach and activate the beacon",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--search",
|
||||||
|
action="store_true",
|
||||||
|
help="search challenge: explore the room and find the orange triangle "
|
||||||
|
"on the back of a crate, then report it (autonomous mode, fewer guardrails)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--free",
|
||||||
|
action="store_true",
|
||||||
|
help="fewer restrictions: no re-aim corrections or collision hints; the model plans freely",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--mission-steps",
|
"--mission-steps",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -463,6 +511,12 @@ def main() -> int:
|
|||||||
default=3,
|
default=3,
|
||||||
help="attach a fresh camera frame every N turns (0 disables; default 3)",
|
help="attach a fresh camera frame every N turns (0 disables; default 3)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--cruise",
|
||||||
|
type=float,
|
||||||
|
default=1.0,
|
||||||
|
help="proactive motion in missions: glide forward (meters) while the LLM thinks (0 disables)",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
try:
|
try:
|
||||||
args.base_url, args.api_key, args.model = resolve_provider(
|
args.base_url, args.api_key, args.model = resolve_provider(
|
||||||
@@ -472,7 +526,9 @@ def main() -> int:
|
|||||||
model=args.model,
|
model=args.model,
|
||||||
)
|
)
|
||||||
if args.provider:
|
if args.provider:
|
||||||
print(f"[chat] provider: {args.provider} -> {args.base_url} model: {args.model}")
|
print(
|
||||||
|
f"[chat] provider: {args.provider} -> {args.base_url} model: {args.model}"
|
||||||
|
)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
print(f"[chat] {exc}")
|
print(f"[chat] {exc}")
|
||||||
return 1
|
return 1
|
||||||
|
|||||||
@@ -142,6 +142,8 @@ async def run_llm_agent(
|
|||||||
vision: bool | None = None,
|
vision: bool | None = None,
|
||||||
digest: bool | None = None,
|
digest: bool | None = None,
|
||||||
look_every: int = 0,
|
look_every: int = 0,
|
||||||
|
cruise: float = 0.0,
|
||||||
|
autonomous: bool = False,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Autonomous LLM run: controller + nudge/correct loop (see llm_agent)."""
|
"""Autonomous LLM run: controller + nudge/correct loop (see llm_agent)."""
|
||||||
from testbed.llm_agent import LLMController, run_llm_agent_loop
|
from testbed.llm_agent import LLMController, run_llm_agent_loop
|
||||||
@@ -233,9 +235,9 @@ async def run_scripted_agent(
|
|||||||
|
|
||||||
if detour_steps > 0:
|
if detour_steps > 0:
|
||||||
# Escape the obstacle before re-aiming: keep the detour heading.
|
# Escape the obstacle before re-aiming: keep the detour heading.
|
||||||
mv = (await client.call_tool("move", {"forward": 0.8})).output
|
mv = (await client.call_tool("move", {"forward": 0.8, "duration": 0.6})).output
|
||||||
summary["tool_calls"] += 1
|
summary["tool_calls"] += 1
|
||||||
log("tool", 'move({"forward": 0.8}) [detour]')
|
log("tool", 'move({"forward": 0.8, "duration": 0.6}) [detour]')
|
||||||
log("bridge", f"ok {json.dumps(mv)}")
|
log("bridge", f"ok {json.dumps(mv)}")
|
||||||
if mv.get("moved", 0.0) < 0.2 and detour_turns < 6:
|
if mv.get("moved", 0.0) < 0.2 and detour_turns < 6:
|
||||||
detour_turns += 1
|
detour_turns += 1
|
||||||
@@ -267,9 +269,9 @@ async def run_scripted_agent(
|
|||||||
log("tool", f'turn({{"yaw_deg": {delta * 0.8:.1f}}})')
|
log("tool", f'turn({{"yaw_deg": {delta * 0.8:.1f}}})')
|
||||||
log("bridge", f"ok {json.dumps(turn)}")
|
log("bridge", f"ok {json.dumps(turn)}")
|
||||||
else:
|
else:
|
||||||
mv = (await client.call_tool("move", {"forward": 0.8})).output
|
mv = (await client.call_tool("move", {"forward": 0.8, "duration": 0.6})).output
|
||||||
summary["tool_calls"] += 1
|
summary["tool_calls"] += 1
|
||||||
log("tool", 'move({"forward": 0.8})')
|
log("tool", 'move({"forward": 0.8, "duration": 0.6})')
|
||||||
log("bridge", f"ok {json.dumps(mv)}")
|
log("bridge", f"ok {json.dumps(mv)}")
|
||||||
if mv.get("collision"):
|
if mv.get("collision"):
|
||||||
consecutive_collisions += 1
|
consecutive_collisions += 1
|
||||||
@@ -315,6 +317,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
vision=args.vision,
|
vision=args.vision,
|
||||||
digest=args.digest,
|
digest=args.digest,
|
||||||
look_every=args.look_every,
|
look_every=args.look_every,
|
||||||
|
cruise=args.cruise,
|
||||||
|
autonomous=args.free,
|
||||||
)
|
)
|
||||||
elif args.agent == "scripted":
|
elif args.agent == "scripted":
|
||||||
summary = await run_scripted_agent(
|
summary = await run_scripted_agent(
|
||||||
@@ -338,6 +342,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
vision=args.vision,
|
vision=args.vision,
|
||||||
digest=args.digest,
|
digest=args.digest,
|
||||||
look_every=args.look_every,
|
look_every=args.look_every,
|
||||||
|
cruise=args.cruise,
|
||||||
|
autonomous=args.free,
|
||||||
)
|
)
|
||||||
if summary.get("interacted"):
|
if summary.get("interacted"):
|
||||||
return summary
|
return summary
|
||||||
@@ -431,6 +437,17 @@ def main() -> int:
|
|||||||
default=3,
|
default=3,
|
||||||
help="attach a fresh camera frame every N agent steps (0 disables; default 3)",
|
help="attach a fresh camera frame every N agent steps (0 disables; default 3)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--cruise",
|
||||||
|
type=float,
|
||||||
|
default=1.0,
|
||||||
|
help="proactive motion: glide forward (meters) while the LLM thinks (0 disables)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--free",
|
||||||
|
action="store_true",
|
||||||
|
help="fewer restrictions: no re-aim corrections or collision hints; the model plans freely",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--frame",
|
"--frame",
|
||||||
default="demo_final_frame.png",
|
default="demo_final_frame.png",
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ tool calls/results fed back as `tool` messages.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
@@ -67,6 +68,19 @@ NUDGE = (
|
|||||||
"then call a tool (turn or move, or interact if within 1.6 m)."
|
"then call a tool (turn or move, or interact if within 1.6 m)."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
SEARCH_MISSION = (
|
||||||
|
"You are an explorer in a 16x16 m room. Three wooden crates stand in the room, "
|
||||||
|
"and a small ORANGE TRIANGLE is painted on the BACK side of ONE of them — the "
|
||||||
|
"side that faces away from the room's entrance, so it is only visible once you "
|
||||||
|
"walk around the crates. The glowing beacon in the corner is irrelevant: ignore it. "
|
||||||
|
"Your mission: explore the room until you see the orange triangle with your own "
|
||||||
|
'eyes (vision), then walk near it and call report(discovery="triangle"). '
|
||||||
|
"Strategy: walk a loop around the crates; every few steps look at the crate faces "
|
||||||
|
"with vision; check faces on all sides. The CURRENT STATE note in tool results "
|
||||||
|
"tells you your position and heading. Do not stop until you have seen and "
|
||||||
|
"reported the triangle."
|
||||||
|
)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Provider presets (OpenAI-compatible endpoints)
|
# Provider presets (OpenAI-compatible endpoints)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -104,6 +118,31 @@ _MULTIMODAL_HINTS = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_dotenv() -> None:
|
||||||
|
"""Load KEY=VALUE pairs from a .env file (project root or cwd) into
|
||||||
|
os.environ without overriding already-set variables."""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
candidates = [
|
||||||
|
Path.cwd() / ".env",
|
||||||
|
Path(__file__).resolve().parent.parent / ".env",
|
||||||
|
]
|
||||||
|
for path in candidates:
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
for line in path.read_text().splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#") or "=" not in line:
|
||||||
|
continue
|
||||||
|
key, _, value = line.partition("=")
|
||||||
|
key = key.strip()
|
||||||
|
value = value.strip().strip('"').strip("'")
|
||||||
|
if key and key not in os.environ:
|
||||||
|
os.environ[key] = value
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
def resolve_provider(
|
def resolve_provider(
|
||||||
*,
|
*,
|
||||||
provider: str | None,
|
provider: str | None,
|
||||||
@@ -113,11 +152,12 @@ def resolve_provider(
|
|||||||
) -> tuple[str, str, str]:
|
) -> tuple[str, str, str]:
|
||||||
"""Resolve endpoint + key + model from a provider preset (or explicit args).
|
"""Resolve endpoint + key + model from a provider preset (or explicit args).
|
||||||
|
|
||||||
Keys come from the environment (POLZA_API_KEY / OPENAI_API_KEY) so nothing
|
Keys come from the environment or the project's .env file, so nothing
|
||||||
secret lives in the repo or on the command line.
|
secret lives in the repo or on the command line.
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
load_dotenv()
|
||||||
if provider:
|
if provider:
|
||||||
preset = PROVIDERS.get(provider)
|
preset = PROVIDERS.get(provider)
|
||||||
if preset is None:
|
if preset is None:
|
||||||
@@ -161,6 +201,7 @@ def looks_multimodal(model: str) -> bool:
|
|||||||
COLOR_NAMES = [
|
COLOR_NAMES = [
|
||||||
("beacon_cyan", (120, 210, 235)),
|
("beacon_cyan", (120, 210, 235)),
|
||||||
("beacon_yellow", (255, 190, 90)),
|
("beacon_yellow", (255, 190, 90)),
|
||||||
|
("marker_orange", (255, 150, 40)),
|
||||||
("crate_red", (178, 64, 54)),
|
("crate_red", (178, 64, 54)),
|
||||||
("crate_blue", (64, 96, 178)),
|
("crate_blue", (64, 96, 178)),
|
||||||
("crate_olive", (128, 128, 60)),
|
("crate_olive", (128, 128, 60)),
|
||||||
@@ -273,6 +314,7 @@ class TurnResult:
|
|||||||
calls: list[ToolCallResult] = field(default_factory=list)
|
calls: list[ToolCallResult] = field(default_factory=list)
|
||||||
interacted: bool = False
|
interacted: bool = False
|
||||||
message: str | None = None
|
message: str | None = None
|
||||||
|
thinking: str = ""
|
||||||
|
|
||||||
|
|
||||||
def detect_multimodal(base_url: str, model: str) -> bool:
|
def detect_multimodal(base_url: str, model: str) -> bool:
|
||||||
@@ -437,10 +479,12 @@ class LLMController:
|
|||||||
async def send_user(self, text: str) -> None:
|
async def send_user(self, text: str) -> None:
|
||||||
self.messages.append({"role": "user", "content": text})
|
self.messages.append({"role": "user", "content": text})
|
||||||
|
|
||||||
async def invoke(self) -> TurnResult:
|
async def request(self) -> Any:
|
||||||
"""One model round-trip: get the response and execute its tool calls."""
|
"""Send the current conversation to the model and return the raw
|
||||||
|
response WITHOUT executing any tool calls. Lets a caller overlap the
|
||||||
|
model's thinking time with other work (e.g. proactive cruising)."""
|
||||||
try:
|
try:
|
||||||
resp = await self.ac.chat.completions.create(
|
return await self.ac.chat.completions.create(
|
||||||
model=self.model,
|
model=self.model,
|
||||||
messages=self.messages,
|
messages=self.messages,
|
||||||
tools=self.tools,
|
tools=self.tools,
|
||||||
@@ -450,9 +494,24 @@ class LLMController:
|
|||||||
self.log("agent", f"LLM error: {exc}")
|
self.log("agent", f"LLM error: {exc}")
|
||||||
raise RuntimeError(f"LLM error: {exc}") from exc
|
raise RuntimeError(f"LLM error: {exc}") from exc
|
||||||
|
|
||||||
|
async def invoke(self) -> TurnResult:
|
||||||
|
"""One model round-trip: get the response and execute its tool calls."""
|
||||||
|
return await self.execute(await self.request())
|
||||||
|
|
||||||
|
async def execute(self, resp: Any) -> TurnResult:
|
||||||
|
"""Execute the tool calls of a previously requested response."""
|
||||||
choice = resp.choices[0]
|
choice = resp.choices[0]
|
||||||
text = choice.message.content or ""
|
text = choice.message.content or ""
|
||||||
result = TurnResult(text=text)
|
result = TurnResult(text=text)
|
||||||
|
# Some providers return reasoning tokens separately (e.g. deepseek-r1
|
||||||
|
# style models); surface them when present.
|
||||||
|
msg = choice.message
|
||||||
|
thinking = (
|
||||||
|
getattr(msg, "reasoning_content", None)
|
||||||
|
or getattr(msg, "reasoning", None)
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
result.thinking = thinking if isinstance(thinking, str) else ""
|
||||||
calls = choice.message.tool_calls
|
calls = choice.message.tool_calls
|
||||||
if not calls:
|
if not calls:
|
||||||
self.messages.append({"role": "assistant", "content": text})
|
self.messages.append({"role": "assistant", "content": text})
|
||||||
@@ -570,11 +629,21 @@ async def run_llm_agent_loop(
|
|||||||
recorder: Any | None = None,
|
recorder: Any | None = None,
|
||||||
nudge_limit: int = 1,
|
nudge_limit: int = 1,
|
||||||
look_every: int = 0,
|
look_every: int = 0,
|
||||||
|
cruise: float = 0.0,
|
||||||
|
autonomous: bool = False,
|
||||||
|
is_success: Callable[[TurnResult], bool] | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Drive the controller until the mission is done or steps run out.
|
"""Drive the controller until the goal is done or steps run out.
|
||||||
|
|
||||||
``look_every``: attach a fresh camera frame every N steps so the model
|
``look_every``: attach a fresh camera frame every N steps so the model
|
||||||
always sees recent visual context without asking (0 disables).
|
always sees recent visual context without asking (0 disables).
|
||||||
|
``cruise``: proactive motion — while the model is thinking, the capsule
|
||||||
|
keeps gliding forward (meters per think, 0 disables). Collisions stop the
|
||||||
|
drift and are reported to the model.
|
||||||
|
``autonomous``: fewer guardrails — no re-aim corrections and no collision
|
||||||
|
hints; the model plans its own exploration (used for search missions).
|
||||||
|
``is_success``: custom goal predicate over a model turn; by default the
|
||||||
|
mission ends when interact succeeds.
|
||||||
"""
|
"""
|
||||||
summary: dict[str, Any] = {
|
summary: dict[str, Any] = {
|
||||||
"steps": 0,
|
"steps": 0,
|
||||||
@@ -587,6 +656,45 @@ async def run_llm_agent_loop(
|
|||||||
last_correct_step = -99
|
last_correct_step = -99
|
||||||
last_collision_step = -99
|
last_collision_step = -99
|
||||||
|
|
||||||
|
async def drift(amount: float, step: int) -> None:
|
||||||
|
"""Low-level controller: keep moving forward while the brain thinks."""
|
||||||
|
nonlocal last_collision_step
|
||||||
|
remaining = amount
|
||||||
|
total = 0.0
|
||||||
|
while remaining > 0.02:
|
||||||
|
mv = await controller.client.call_tool(
|
||||||
|
"move", {"forward": min(0.4, remaining), "duration": 0.5}
|
||||||
|
)
|
||||||
|
out = mv.output
|
||||||
|
controller.observe(out)
|
||||||
|
moved = out.get("moved", 0.0)
|
||||||
|
total += moved
|
||||||
|
remaining -= moved
|
||||||
|
if out.get("collision"):
|
||||||
|
last_collision_step = step
|
||||||
|
controller.messages.append(
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
"While you were thinking, the capsule drifted forward and bumped "
|
||||||
|
f"into {out.get('collision_normal')}. It stopped. Navigate around it."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
log("agent", "(cruise: bumped while thinking — stopped)")
|
||||||
|
return
|
||||||
|
if total > 0.05:
|
||||||
|
controller.messages.append(
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
f"While you were thinking, the capsule kept moving (auto-cruise, "
|
||||||
|
f"{total:.2f} m forward). {controller.state_hint()}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
log("agent", f"(cruise: drifted {total:.2f} m while thinking)")
|
||||||
|
|
||||||
def maybe_correct(step: int) -> None:
|
def maybe_correct(step: int) -> None:
|
||||||
nonlocal corrections, last_correct_step
|
nonlocal corrections, last_correct_step
|
||||||
if controller.pos is None or controller.beacon_pos is None:
|
if controller.pos is None or controller.beacon_pos is None:
|
||||||
@@ -626,6 +734,7 @@ async def run_llm_agent_loop(
|
|||||||
log("agent", "(correction: re-aim at the beacon)")
|
log("agent", "(correction: re-aim at the beacon)")
|
||||||
dist_history.clear()
|
dist_history.clear()
|
||||||
|
|
||||||
|
pending_resp: Any | None = None # LLM response requested while we cruised
|
||||||
for step in range(max_steps):
|
for step in range(max_steps):
|
||||||
summary["steps"] = step + 1
|
summary["steps"] = step + 1
|
||||||
if recorder is not None and controller.pos is not None:
|
if recorder is not None and controller.pos is not None:
|
||||||
@@ -634,9 +743,16 @@ async def run_llm_agent_loop(
|
|||||||
# sees what is happening without having to ask.
|
# sees what is happening without having to ask.
|
||||||
if look_every and step % look_every == 0:
|
if look_every and step % look_every == 0:
|
||||||
await controller.auto_frame()
|
await controller.auto_frame()
|
||||||
turn = await controller.invoke()
|
if pending_resp is not None:
|
||||||
|
resp = await pending_resp
|
||||||
|
pending_resp = None
|
||||||
|
turn = await controller.execute(resp)
|
||||||
|
else:
|
||||||
|
turn = await controller.invoke()
|
||||||
if turn.text:
|
if turn.text:
|
||||||
log("agent", turn.text[:400])
|
log("agent", turn.text[:400])
|
||||||
|
if turn.thinking:
|
||||||
|
log("think", turn.thinking[:400])
|
||||||
summary["tool_calls"] += len(turn.calls)
|
summary["tool_calls"] += len(turn.calls)
|
||||||
for c in turn.calls:
|
for c in turn.calls:
|
||||||
if (
|
if (
|
||||||
@@ -646,23 +762,37 @@ async def run_llm_agent_loop(
|
|||||||
and c.output.get("collision")
|
and c.output.get("collision")
|
||||||
):
|
):
|
||||||
last_collision_step = step
|
last_collision_step = step
|
||||||
obj = c.output.get("collision_normal")
|
if not autonomous:
|
||||||
hint_msg = (
|
obj = c.output.get("collision_normal")
|
||||||
f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
|
hint_msg = (
|
||||||
"forward twice to get around it, then follow the CURRENT STATE note to re-aim."
|
f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
|
||||||
)
|
"forward twice to get around it, then follow the CURRENT STATE note to re-aim."
|
||||||
if not (
|
)
|
||||||
controller.messages
|
if not (
|
||||||
and controller.messages[-1].get("content") == hint_msg
|
controller.messages
|
||||||
):
|
and controller.messages[-1].get("content") == hint_msg
|
||||||
controller.messages.append({"role": "user", "content": hint_msg})
|
):
|
||||||
log("agent", "(collision: go around)")
|
controller.messages.append(
|
||||||
|
{"role": "user", "content": hint_msg}
|
||||||
|
)
|
||||||
|
log("agent", "(collision: go around)")
|
||||||
# Show the model what it just bumped into.
|
# Show the model what it just bumped into.
|
||||||
await controller.auto_frame(
|
await controller.auto_frame(
|
||||||
"You just bumped into something. Look at what is in front of you."
|
"You just bumped into something. Look at what is in front of you."
|
||||||
)
|
)
|
||||||
maybe_correct(step)
|
if not autonomous:
|
||||||
if turn.interacted:
|
maybe_correct(step)
|
||||||
|
if is_success is not None:
|
||||||
|
if is_success(turn):
|
||||||
|
summary["success"] = True
|
||||||
|
for c in turn.calls:
|
||||||
|
if c.name == "report" and c.ok and isinstance(c.output, dict):
|
||||||
|
summary["result"] = c.output.get("message", "reported")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
summary["result"] = "goal achieved"
|
||||||
|
return summary
|
||||||
|
elif turn.interacted:
|
||||||
summary["interacted"] = True
|
summary["interacted"] = True
|
||||||
summary["result"] = turn.message
|
summary["result"] = turn.message
|
||||||
return summary
|
return summary
|
||||||
@@ -680,5 +810,11 @@ async def run_llm_agent_loop(
|
|||||||
return summary
|
return summary
|
||||||
controller.messages.append({"role": "user", "content": NUDGE})
|
controller.messages.append({"role": "user", "content": NUDGE})
|
||||||
log("agent", "(nudge: no tool call; continue the mission)")
|
log("agent", "(nudge: no tool call; continue the mission)")
|
||||||
|
continue
|
||||||
|
# Proactive motion: request the next response and, while the model
|
||||||
|
# thinks, keep the capsule gliding forward (collision-safe).
|
||||||
|
if cruise > 0.0 and step < max_steps - 1:
|
||||||
|
pending_resp = asyncio.create_task(controller.request())
|
||||||
|
await drift(cruise, step)
|
||||||
summary["result"] = f"exceeded {max_steps} steps"
|
summary["result"] = f"exceeded {max_steps} steps"
|
||||||
return summary
|
return summary
|
||||||
|
|||||||
@@ -0,0 +1,333 @@
|
|||||||
|
"""Record a search mission and render a three-view GIF for the README.
|
||||||
|
|
||||||
|
Panes (left -> right):
|
||||||
|
top-down debug map (world state: crates, marker, path, capsule)
|
||||||
|
first person (the actual frames the model received)
|
||||||
|
chat (the model's messages, reasoning, and tool calls)
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python -m testbed.record_search --out search_mission.gif
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import base64
|
||||||
|
import io
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from aicc.client import AICCClient
|
||||||
|
from aicc.transport.websocket import WebSocketClientTransport, WebSocketServer
|
||||||
|
from PIL import Image, ImageDraw, ImageFont
|
||||||
|
|
||||||
|
from testbed.bridge import build_bridge
|
||||||
|
from testbed.llm_agent import (
|
||||||
|
SEARCH_MISSION,
|
||||||
|
LLMController,
|
||||||
|
resolve_provider,
|
||||||
|
run_llm_agent_loop,
|
||||||
|
)
|
||||||
|
|
||||||
|
PANEL = 400
|
||||||
|
ROOM = 16.0
|
||||||
|
MAX_CHAT_LINES = 21
|
||||||
|
CHAT_LINE_MAX = 46 # chars per chat line
|
||||||
|
|
||||||
|
ROLE_COLORS = {
|
||||||
|
"agent": (220, 220, 230),
|
||||||
|
"think": (150, 165, 200),
|
||||||
|
"tool": (255, 184, 96),
|
||||||
|
"bridge": (150, 210, 150),
|
||||||
|
"mission": (255, 220, 120),
|
||||||
|
"vision": (130, 210, 250),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _font(size: int = 13) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
|
||||||
|
try:
|
||||||
|
return ImageFont.load_default(size=size)
|
||||||
|
except TypeError: # older Pillow
|
||||||
|
return ImageFont.load_default()
|
||||||
|
|
||||||
|
|
||||||
|
def _wrap(text: str, width: int = CHAT_LINE_MAX) -> list[str]:
|
||||||
|
words = text.split()
|
||||||
|
lines: list[str] = []
|
||||||
|
cur = ""
|
||||||
|
for w in words:
|
||||||
|
if len(cur) + len(w) + 1 > width:
|
||||||
|
if cur:
|
||||||
|
lines.append(cur)
|
||||||
|
cur = w
|
||||||
|
else:
|
||||||
|
cur = (cur + " " + w).strip()
|
||||||
|
if cur:
|
||||||
|
lines.append(cur)
|
||||||
|
return lines[:3] # keep the pane readable
|
||||||
|
|
||||||
|
|
||||||
|
class ChatPane:
|
||||||
|
"""Accumulates the mission chat and renders the last lines."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.lines: list[tuple[str, str]] = []
|
||||||
|
|
||||||
|
def add(self, role: str, msg: str) -> None:
|
||||||
|
for piece in _wrap(msg):
|
||||||
|
self.lines.append((role, piece))
|
||||||
|
if len(self.lines) > MAX_CHAT_LINES * 2:
|
||||||
|
self.lines = self.lines[-MAX_CHAT_LINES:]
|
||||||
|
|
||||||
|
def render(self) -> Image.Image:
|
||||||
|
img = Image.new("RGB", (PANEL, PANEL), (12, 12, 18))
|
||||||
|
draw = ImageDraw.Draw(img)
|
||||||
|
font = _font(13)
|
||||||
|
draw.text((8, 6), "chat — what the model says", fill=(160, 170, 190), font=font)
|
||||||
|
y = 26
|
||||||
|
for role, piece in self.lines[-MAX_CHAT_LINES:]:
|
||||||
|
color = ROLE_COLORS.get(role, (200, 200, 200))
|
||||||
|
draw.text((8, y), piece[:CHAT_LINE_MAX], fill=color, font=font)
|
||||||
|
y += 17
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def _topdown_frame(
|
||||||
|
bridge, path: list[tuple[float, float]], step: int, marker_found: bool
|
||||||
|
) -> Image.Image:
|
||||||
|
from testbed.room.render import render_topdown
|
||||||
|
|
||||||
|
img = render_topdown(bridge.world, PANEL)
|
||||||
|
draw = ImageDraw.Draw(img)
|
||||||
|
scale = PANEL / ROOM
|
||||||
|
if len(path) > 1:
|
||||||
|
pts = [(x * scale, PANEL - z * scale) for x, z in path]
|
||||||
|
draw.line(pts, fill=(255, 170, 60), width=3)
|
||||||
|
m = bridge.world.marker_world_pos()
|
||||||
|
if m is not None:
|
||||||
|
mx, mz = m[0] * scale, PANEL - m[2] * scale
|
||||||
|
r = 9
|
||||||
|
draw.polygon(
|
||||||
|
[(mx, mz - r), (mx - r, mz + r * 0.8), (mx + r, mz + r * 0.8)],
|
||||||
|
outline=(255, 255, 255),
|
||||||
|
width=2,
|
||||||
|
)
|
||||||
|
status = f"step {step} {'FOUND' if marker_found else 'searching...'}"
|
||||||
|
draw.text((10, PANEL - 24), status, fill=(255, 220, 120), font=_font(13))
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def _compose(map_img, view_img, chat_img) -> Image.Image:
|
||||||
|
canvas = Image.new("RGB", (PANEL * 3, PANEL), (18, 18, 26))
|
||||||
|
canvas.paste(map_img, (0, 0))
|
||||||
|
canvas.paste(view_img, (PANEL, 0))
|
||||||
|
canvas.paste(chat_img, (PANEL * 2, 0))
|
||||||
|
draw = ImageDraw.Draw(canvas)
|
||||||
|
draw.text((8, 6), "top-down", fill=(160, 170, 190), font=_font(12))
|
||||||
|
draw.text((PANEL + 8, 6), "first person", fill=(160, 170, 190), font=_font(12))
|
||||||
|
draw.text((PANEL * 2 + 8, 6), "chat", fill=(160, 170, 190), font=_font(12))
|
||||||
|
for x in (PANEL, PANEL * 2):
|
||||||
|
draw.line([(x, 0), (x, PANEL)], fill=(60, 60, 80))
|
||||||
|
return canvas
|
||||||
|
|
||||||
|
|
||||||
|
async def run(args: argparse.Namespace) -> int:
|
||||||
|
base_url, api_key, model = resolve_provider(
|
||||||
|
provider=args.provider,
|
||||||
|
base_url=args.base_url,
|
||||||
|
api_key=args.api_key,
|
||||||
|
model=args.model,
|
||||||
|
)
|
||||||
|
from testbed.llm_agent import MISSION
|
||||||
|
|
||||||
|
bridge = build_bridge()
|
||||||
|
if args.goal == "search":
|
||||||
|
# Fixed marker so the GIF tells a clear story (crate_blue, east face).
|
||||||
|
bridge.world.marker = {
|
||||||
|
"box": "crate_blue", "face": "x1", "u": 0.5, "v": 0.55,
|
||||||
|
"size": 0.42, "color": (255, 150, 40),
|
||||||
|
}
|
||||||
|
m = bridge.world.marker_world_pos()
|
||||||
|
print(f"[record] triangle marker on crate_blue east face at {tuple(round(v, 2) for v in m)}")
|
||||||
|
|
||||||
|
frames: list[Image.Image] = []
|
||||||
|
chat = ChatPane()
|
||||||
|
marker_seen = False
|
||||||
|
samples: list[tuple[float, float]] = [] # capsule positions per captured frame
|
||||||
|
|
||||||
|
def log(role: str, msg: str) -> None:
|
||||||
|
print(f" [{role}] {msg}")
|
||||||
|
if role in ("agent", "think", "tool", "mission", "vision"):
|
||||||
|
chat.add(role, msg)
|
||||||
|
|
||||||
|
async with WebSocketServer(bridge, port=8765):
|
||||||
|
transport = WebSocketClientTransport("ws://127.0.0.1:8765")
|
||||||
|
async with AICCClient(transport) as client:
|
||||||
|
manifest = await client.handshake()
|
||||||
|
ctl = LLMController(
|
||||||
|
client, manifest,
|
||||||
|
base_url=base_url, api_key=api_key, model=model,
|
||||||
|
system_prompt=SEARCH_MISSION if args.goal == "search" else MISSION,
|
||||||
|
log=log,
|
||||||
|
)
|
||||||
|
if args.goal == "search":
|
||||||
|
ctl.messages = [
|
||||||
|
ctl.messages[0],
|
||||||
|
{"role": "user", "content": (
|
||||||
|
"MISSION: explore the room, find the orange triangle on the "
|
||||||
|
"back of a crate, and report it. Keep exploring until found."
|
||||||
|
)},
|
||||||
|
]
|
||||||
|
chat.add("mission", "MISSION: find the orange triangle")
|
||||||
|
else:
|
||||||
|
ctl.messages.append({
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
"MISSION: reach the beacon and activate it. "
|
||||||
|
"Keep calling tools until it is done. Move smoothly (use "
|
||||||
|
"duration on move/turn). The capsule cruises while you think."
|
||||||
|
),
|
||||||
|
})
|
||||||
|
chat.add("mission", "MISSION: reach and activate the beacon")
|
||||||
|
|
||||||
|
if args.goal == "search" and not args.capture_hz:
|
||||||
|
# capture at frame-attach events (one frame per step)
|
||||||
|
orig_frame = ctl._frame_message
|
||||||
|
|
||||||
|
def record_frame(png: str, note: str):
|
||||||
|
nonlocal marker_seen
|
||||||
|
view = (
|
||||||
|
Image.open(io.BytesIO(base64.b64decode(png)))
|
||||||
|
.convert("RGB")
|
||||||
|
.resize((PANEL, PANEL))
|
||||||
|
)
|
||||||
|
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||||
|
frames.append(_compose(top, view, chat.render()))
|
||||||
|
samples.append((bridge.world.capsule.x, bridge.world.capsule.z))
|
||||||
|
return orig_frame(png, note)
|
||||||
|
|
||||||
|
ctl._frame_message = record_frame # type: ignore[method-assign]
|
||||||
|
|
||||||
|
def is_success(turn):
|
||||||
|
nonlocal marker_seen
|
||||||
|
for c in turn.calls:
|
||||||
|
if c.name == "report" and c.ok and c.output and c.output.get("verified"):
|
||||||
|
marker_seen = True
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def observer(agent_done: asyncio.Event) -> None:
|
||||||
|
"""Sample the world at a fixed rate so the GIF shows the actual
|
||||||
|
gliding motion (mid-animation frames), not just step endpoints."""
|
||||||
|
obs_transport = WebSocketClientTransport("ws://127.0.0.1:8765")
|
||||||
|
async with AICCClient(obs_transport) as obs:
|
||||||
|
await obs.handshake()
|
||||||
|
while not agent_done.is_set():
|
||||||
|
try:
|
||||||
|
out = (await obs.call_tool("vision", {})).output
|
||||||
|
except Exception: # noqa: BLE001 - session teardown
|
||||||
|
break
|
||||||
|
view = (
|
||||||
|
Image.open(io.BytesIO(base64.b64decode(out["png_b64"])))
|
||||||
|
.convert("RGB")
|
||||||
|
.resize((PANEL, PANEL))
|
||||||
|
)
|
||||||
|
samples.append((bridge.world.capsule.x, bridge.world.capsule.z))
|
||||||
|
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||||
|
frames.append(_compose(top, view, chat.render()))
|
||||||
|
if len(frames) > 350: # keep the GIF bounded
|
||||||
|
frames[:150] = []
|
||||||
|
await asyncio.sleep(1.0 / args.capture_hz)
|
||||||
|
|
||||||
|
agent_done = asyncio.Event()
|
||||||
|
obs_task = None
|
||||||
|
if args.capture_hz:
|
||||||
|
obs_task = asyncio.create_task(observer(agent_done))
|
||||||
|
try:
|
||||||
|
summary = await run_llm_agent_loop(
|
||||||
|
ctl, args.steps, log=log,
|
||||||
|
look_every=2 if args.goal == "beacon" else 1,
|
||||||
|
cruise=args.cruise,
|
||||||
|
autonomous=args.goal == "search",
|
||||||
|
is_success=is_success if args.goal == "search" else None,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
agent_done.set()
|
||||||
|
if obs_task is not None:
|
||||||
|
await obs_task
|
||||||
|
|
||||||
|
# Final frame: end state of the map.
|
||||||
|
if marker_seen:
|
||||||
|
chat.add("mission", f"DONE: {summary.get('result', 'triangle found')}")
|
||||||
|
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||||
|
last_view = (
|
||||||
|
frames[-1].crop((PANEL, 0, PANEL * 2, PANEL))
|
||||||
|
if frames
|
||||||
|
else Image.new("RGB", (PANEL, PANEL), (0, 0, 0))
|
||||||
|
)
|
||||||
|
frames.append(_compose(top, last_view, chat.render()))
|
||||||
|
|
||||||
|
# Smoothness check: mean movement between consecutive captured frames.
|
||||||
|
if len(samples) > 2:
|
||||||
|
from itertools import pairwise
|
||||||
|
|
||||||
|
deltas = [
|
||||||
|
((b[0] - a[0]) ** 2 + (b[1] - a[1]) ** 2) ** 0.5
|
||||||
|
for a, b in pairwise(samples)
|
||||||
|
]
|
||||||
|
moving = [d for d in deltas if d > 0.02]
|
||||||
|
if moving:
|
||||||
|
print(f"[record] motion between frames: mean {sum(moving)/len(moving):.2f} m "
|
||||||
|
f"(max {max(moving):.2f}) over {len(moving)} moving samples")
|
||||||
|
|
||||||
|
out = Path(args.out)
|
||||||
|
if frames:
|
||||||
|
frames[0].save(
|
||||||
|
out,
|
||||||
|
save_all=True,
|
||||||
|
append_images=frames[1:],
|
||||||
|
duration=args.duration,
|
||||||
|
loop=0,
|
||||||
|
)
|
||||||
|
frames[-1].save(out.with_suffix(".png"))
|
||||||
|
print(f"[record] wrote {out} ({len(frames)} frames, {args.duration} ms/frame)")
|
||||||
|
print(f"[record] mission result: {summary.get('result')}")
|
||||||
|
return 0 if (marker_seen or summary.get("interacted")) else 1
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Record a mission into a three-view GIF (map | first person | chat)."
|
||||||
|
)
|
||||||
|
parser.add_argument("--out", default="search_mission.gif")
|
||||||
|
parser.add_argument(
|
||||||
|
"--goal",
|
||||||
|
choices=["search", "beacon"],
|
||||||
|
default="search",
|
||||||
|
help="which mission to record (default search)",
|
||||||
|
)
|
||||||
|
parser.add_argument("--provider", default=None)
|
||||||
|
parser.add_argument("--base-url", default=None)
|
||||||
|
parser.add_argument("--api-key", default=None)
|
||||||
|
parser.add_argument("--model", default=None)
|
||||||
|
parser.add_argument("--steps", type=int, default=45)
|
||||||
|
parser.add_argument("--cruise", type=float, default=0.6)
|
||||||
|
parser.add_argument("--duration", type=int, default=150, help="ms per GIF frame")
|
||||||
|
parser.add_argument(
|
||||||
|
"--capture-hz",
|
||||||
|
type=float,
|
||||||
|
default=0.0,
|
||||||
|
help="sample frames at this rate (Hz) with an observer client to show "
|
||||||
|
"gliding motion; 0 = one frame per step (search default)",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
return asyncio.run(run(args))
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\n[record] interrupted — the partial GIF was NOT written")
|
||||||
|
return 130
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -114,6 +114,7 @@ class Raycaster:
|
|||||||
self._draw_beacon(room, pixels, depth, fx, fz, rx, rz, eye, tan_pitch)
|
self._draw_beacon(room, pixels, depth, fx, fz, rx, rz, eye, tan_pitch)
|
||||||
|
|
||||||
img = Image.frombuffer("RGB", (w, h), bytes(pixels), "raw", "RGB", 0, 1)
|
img = Image.frombuffer("RGB", (w, h), bytes(pixels), "raw", "RGB", 0, 1)
|
||||||
|
self._draw_marker(room, img, depth, fx, fz, rx, rz, eye, tan_pitch)
|
||||||
grid = self._depth_grid(depth, max_depth)
|
grid = self._depth_grid(depth, max_depth)
|
||||||
return img, grid
|
return img, grid
|
||||||
|
|
||||||
@@ -260,6 +261,65 @@ class Raycaster:
|
|||||||
col = FLOOR_LINE
|
col = FLOOR_LINE
|
||||||
return _fog(col, t)
|
return _fog(col, t)
|
||||||
|
|
||||||
|
def _draw_marker(
|
||||||
|
self,
|
||||||
|
room: Room,
|
||||||
|
img: Image.Image,
|
||||||
|
depth: list[list[float]],
|
||||||
|
fx: float,
|
||||||
|
fz: float,
|
||||||
|
rx: float,
|
||||||
|
rz: float,
|
||||||
|
eye: float,
|
||||||
|
tan_pitch: float,
|
||||||
|
) -> None:
|
||||||
|
"""Draw the search-challenge triangle (a sprite on the crate's back
|
||||||
|
face), only when the anchor pixel is actually that face."""
|
||||||
|
m = room.marker
|
||||||
|
if m is None:
|
||||||
|
return
|
||||||
|
pos = room.marker_world_pos()
|
||||||
|
if pos is None:
|
||||||
|
return
|
||||||
|
w, h = self.width, self.height
|
||||||
|
cam = room.capsule
|
||||||
|
rel_x = pos[0] - cam.x
|
||||||
|
rel_z = pos[2] - cam.z
|
||||||
|
along = rel_x * fx + rel_z * fz
|
||||||
|
if along < 0.3:
|
||||||
|
return
|
||||||
|
right = rel_x * rx + rel_z * rz
|
||||||
|
col_c = (right / along / self.tan_fx + 1.0) / 2.0 * w
|
||||||
|
vv = (pos[1] - eye) / along - tan_pitch
|
||||||
|
row_c = (1.0 - vv / self.tan_fy) / 2.0 * h
|
||||||
|
r0, c0 = round(row_c), round(col_c)
|
||||||
|
if not (0 <= r0 < h and 0 <= c0 < w):
|
||||||
|
return
|
||||||
|
if abs(depth[r0][c0] - along) > 0.25:
|
||||||
|
return # the anchor pixel is not the marker's face (occluded)
|
||||||
|
half = max(2.5, m["size"] / along / self.tan_fx * w / 2.0)
|
||||||
|
draw = ImageDraw.Draw(img)
|
||||||
|
color = tuple(m["color"])
|
||||||
|
glow = tuple(min(255, c + 90) for c in color)
|
||||||
|
draw.ellipse(
|
||||||
|
[
|
||||||
|
col_c - half * 2.2,
|
||||||
|
row_c - half * 2.2,
|
||||||
|
col_c + half * 2.2,
|
||||||
|
row_c + half * 2.2,
|
||||||
|
],
|
||||||
|
fill=glow,
|
||||||
|
)
|
||||||
|
draw.polygon(
|
||||||
|
[
|
||||||
|
(col_c, row_c - half),
|
||||||
|
(col_c - half * 0.9, row_c + half * 0.7),
|
||||||
|
(col_c + half * 0.9, row_c + half * 0.7),
|
||||||
|
],
|
||||||
|
fill=color,
|
||||||
|
outline=(255, 255, 255),
|
||||||
|
)
|
||||||
|
|
||||||
def _draw_beacon(
|
def _draw_beacon(
|
||||||
self,
|
self,
|
||||||
room: Room,
|
room: Room,
|
||||||
@@ -405,6 +465,22 @@ def render_topdown(room: Room, size: int = 400) -> Image.Image:
|
|||||||
fx, fz = c.forward()
|
fx, fz = c.forward()
|
||||||
tip = xy(c.x + fx * 0.7, c.z + fz * 0.7)
|
tip = xy(c.x + fx * 0.7, c.z + fz * 0.7)
|
||||||
draw.line([(cx, cz), tip], fill=(30, 30, 40), width=3)
|
draw.line([(cx, cz), tip], fill=(30, 30, 40), width=3)
|
||||||
|
|
||||||
|
m = room.marker
|
||||||
|
if m is not None:
|
||||||
|
pos = room.marker_world_pos()
|
||||||
|
if pos is not None:
|
||||||
|
mx, mz = xy(pos[0], pos[2])
|
||||||
|
r = m["size"] * scale
|
||||||
|
draw.polygon(
|
||||||
|
[
|
||||||
|
(mx, mz - r * 1.4),
|
||||||
|
(mx - r, mz + r),
|
||||||
|
(mx + r, mz + r),
|
||||||
|
],
|
||||||
|
fill=tuple(m["color"]),
|
||||||
|
outline=(20, 20, 26),
|
||||||
|
)
|
||||||
return img
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
ROOM_SIZE = 16.0
|
ROOM_SIZE = 16.0
|
||||||
ROOM_HEIGHT = 3.0
|
ROOM_HEIGHT = 3.0
|
||||||
@@ -124,6 +125,62 @@ class Room:
|
|||||||
]
|
]
|
||||||
# Pending audio events, drained by the hear sensor.
|
# Pending audio events, drained by the hear sensor.
|
||||||
self._audio: list[dict] = []
|
self._audio: list[dict] = []
|
||||||
|
# Search challenge: a small triangle painted on the BACK side of one
|
||||||
|
# of the crates (the face pointing away from the spawn corner). The
|
||||||
|
# agent must find it by exploring and looking with vision.
|
||||||
|
self.marker: dict[str, Any] = self._place_marker()
|
||||||
|
|
||||||
|
# ---------- search marker ----------
|
||||||
|
|
||||||
|
def _place_marker(self) -> dict[str, Any]:
|
||||||
|
"""Put the triangle on a random crate, on the face most hidden from
|
||||||
|
the spawn corner (1.5, 1.5)."""
|
||||||
|
import random
|
||||||
|
|
||||||
|
box = random.choice(self.boxes)
|
||||||
|
to_spawn = (1.5 - box.cx, 1.5 - box.cz)
|
||||||
|
normals = {
|
||||||
|
"x0": (-1.0, 0.0),
|
||||||
|
"x1": (1.0, 0.0),
|
||||||
|
"z0": (0.0, -1.0),
|
||||||
|
"z1": (0.0, 1.0),
|
||||||
|
}
|
||||||
|
faces = sorted(
|
||||||
|
normals,
|
||||||
|
key=lambda f: normals[f][0] * to_spawn[0] + normals[f][1] * to_spawn[1],
|
||||||
|
)[:2] # the two faces pointing away from spawn
|
||||||
|
return {
|
||||||
|
"box": box.id,
|
||||||
|
"face": random.choice(faces),
|
||||||
|
"u": 0.5,
|
||||||
|
"v": 0.55,
|
||||||
|
"size": 0.42,
|
||||||
|
"color": (255, 150, 40),
|
||||||
|
}
|
||||||
|
|
||||||
|
def marker_world_pos(self) -> tuple[float, float, float] | None:
|
||||||
|
"""World coordinates (x, y, z) of the marker center, or None."""
|
||||||
|
m = self.marker
|
||||||
|
if m is None:
|
||||||
|
return None
|
||||||
|
box = next((b for b in self.boxes if b.id == m["box"]), None)
|
||||||
|
if box is None:
|
||||||
|
return None
|
||||||
|
if m["face"] == "x0":
|
||||||
|
x, z = box.x_min(), box.cz - box.half_d + m["u"] * 2 * box.half_d
|
||||||
|
elif m["face"] == "x1":
|
||||||
|
x, z = box.x_max(), box.cz - box.half_d + m["u"] * 2 * box.half_d
|
||||||
|
elif m["face"] == "z0":
|
||||||
|
x, z = box.cx - box.half_w + m["u"] * 2 * box.half_w, box.z_min()
|
||||||
|
else:
|
||||||
|
x, z = box.cx - box.half_w + m["u"] * 2 * box.half_w, box.z_max()
|
||||||
|
return (x, m["v"] * box.height, z)
|
||||||
|
|
||||||
|
def distance_to_marker(self) -> float:
|
||||||
|
pos = self.marker_world_pos()
|
||||||
|
if pos is None:
|
||||||
|
return float("inf")
|
||||||
|
return math.hypot(pos[0] - self.capsule.x, pos[2] - self.capsule.z)
|
||||||
|
|
||||||
# ---------- ticks ----------
|
# ---------- ticks ----------
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
"""Tests for the testbed bridge: tool behavior over the in-process transport."""
|
"""Tests for the testbed bridge: tool behavior over the in-process transport."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import io
|
import io
|
||||||
|
import math
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from aicc.client import AICCClient
|
from aicc.client import AICCClient
|
||||||
@@ -186,3 +188,112 @@ async def test_single_source_of_truth():
|
|||||||
world = dumped["world"]
|
world = dumped["world"]
|
||||||
assert set(world.keys()) == {"name", "kind"}
|
assert set(world.keys()) == {"name", "kind"}
|
||||||
assert "beacon" not in dumped
|
assert "beacon" not in dumped
|
||||||
|
|
||||||
|
|
||||||
|
async def test_animated_move_is_smooth_and_observable(bridge):
|
||||||
|
"""A move with duration glides: a concurrent client sees intermediate
|
||||||
|
positions, and the final displacement is exact."""
|
||||||
|
# A transport is a single session: one per client.
|
||||||
|
t1 = InProcessTransport.start(bridge)
|
||||||
|
t2 = InProcessTransport.start(bridge)
|
||||||
|
async with t1, t2:
|
||||||
|
mover = AICCClient(t1)
|
||||||
|
viewer = AICCClient(t2)
|
||||||
|
async with mover, viewer:
|
||||||
|
await mover.handshake()
|
||||||
|
await viewer.handshake()
|
||||||
|
start = (await mover.call_tool("proprioception", {})).output["position"]
|
||||||
|
assert start["x"] == 1.5 and start["z"] == 1.5
|
||||||
|
|
||||||
|
move_task = asyncio.create_task(
|
||||||
|
mover.call_tool("move", {"forward": 2.0, "duration": 0.8})
|
||||||
|
)
|
||||||
|
await asyncio.sleep(0.25)
|
||||||
|
mid = (await viewer.call_tool("proprioception", {})).output["position"]
|
||||||
|
# Moving along the 45-degree diagonal: strictly between start and end.
|
||||||
|
assert 1.5 < mid["x"] < 3.5 and 1.5 < mid["z"] < 3.5
|
||||||
|
|
||||||
|
res = (await move_task).output
|
||||||
|
assert res["moved"] == pytest.approx(2.0, abs=1e-3)
|
||||||
|
assert res["position"]["x"] == pytest.approx(
|
||||||
|
1.5 + 2.0 * math.sin(math.radians(45)), abs=1e-3
|
||||||
|
)
|
||||||
|
assert res["duration"] == 0.8
|
||||||
|
|
||||||
|
|
||||||
|
async def test_animated_move_stops_on_collision(bridge):
|
||||||
|
t = InProcessTransport.start(bridge)
|
||||||
|
async with t:
|
||||||
|
client = AICCClient(t)
|
||||||
|
async with client:
|
||||||
|
await client.handshake()
|
||||||
|
bridge.world.capsule.yaw_deg = 180.0 # straight at the north wall
|
||||||
|
res = (
|
||||||
|
await client.call_tool("move", {"forward": 5.0, "duration": 0.6})
|
||||||
|
).output
|
||||||
|
assert res["collision"] is True
|
||||||
|
assert res["moved"] < 5.0
|
||||||
|
assert res["position"]["z"] == pytest.approx(
|
||||||
|
bridge.world.capsule.radius, abs=1e-3
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_move_accepts_duration_zero_default(bridge):
|
||||||
|
t = InProcessTransport.start(bridge)
|
||||||
|
async with t:
|
||||||
|
client = AICCClient(t)
|
||||||
|
async with client:
|
||||||
|
await client.handshake()
|
||||||
|
res = (await client.call_tool("move", {"forward": 1.0})).output
|
||||||
|
assert res["moved"] == pytest.approx(1.0)
|
||||||
|
assert res["duration"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
async def test_report_verifies_near_marker(bridge):
|
||||||
|
t = InProcessTransport.start(bridge)
|
||||||
|
async with t:
|
||||||
|
client = AICCClient(t)
|
||||||
|
async with client:
|
||||||
|
await client.handshake()
|
||||||
|
out = (await client.call_tool("report", {"discovery": "triangle"})).output
|
||||||
|
assert out["success"] is True
|
||||||
|
assert out["verified"] is False # far away from spawn
|
||||||
|
pos = bridge.world.marker_world_pos()
|
||||||
|
bridge.world.capsule.x = pos[0]
|
||||||
|
bridge.world.capsule.z = pos[2]
|
||||||
|
out2 = (await client.call_tool("report", {})).output
|
||||||
|
assert out2["verified"] is True
|
||||||
|
assert "verified" in out2["message"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_turn_with_duration_is_smooth(bridge):
|
||||||
|
t1 = InProcessTransport.start(bridge)
|
||||||
|
t2 = InProcessTransport.start(bridge)
|
||||||
|
async with t1, t2:
|
||||||
|
mover = AICCClient(t1)
|
||||||
|
viewer = AICCClient(t2)
|
||||||
|
async with mover, viewer:
|
||||||
|
await mover.handshake()
|
||||||
|
await viewer.handshake()
|
||||||
|
task = asyncio.create_task(
|
||||||
|
mover.call_tool("turn", {"yaw_deg": 90.0, "duration": 0.8})
|
||||||
|
)
|
||||||
|
await asyncio.sleep(0.3)
|
||||||
|
mid = (await viewer.call_tool("proprioception", {})).output["rotation"]["yaw_deg"]
|
||||||
|
assert 0.0 < mid < 90.0, f"expected intermediate rotation, got {mid}"
|
||||||
|
res = (await task).output
|
||||||
|
assert res["rotation"]["yaw_deg"] == pytest.approx(135.0, abs=0.5)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_look_at_crate(bridge):
|
||||||
|
t = InProcessTransport.start(bridge)
|
||||||
|
async with t:
|
||||||
|
client = AICCClient(t)
|
||||||
|
async with client:
|
||||||
|
await client.handshake()
|
||||||
|
out = (await client.call_tool("look_at", {"target": "crate_red"})).output
|
||||||
|
dx = bridge.world.boxes[0].cx - bridge.world.capsule.x
|
||||||
|
dz = bridge.world.boxes[0].cz - bridge.world.capsule.z
|
||||||
|
assert out["rotation"]["yaw_deg"] == pytest.approx(
|
||||||
|
math.degrees(math.atan2(dx, dz)) % 360.0, abs=0.5
|
||||||
|
)
|
||||||
|
|||||||
@@ -131,3 +131,51 @@ def test_describe_layout():
|
|||||||
assert d["room"]["width"] == ROOM_SIZE
|
assert d["room"]["width"] == ROOM_SIZE
|
||||||
assert len(d["obstacles"]) == 3
|
assert len(d["obstacles"]) == 3
|
||||||
assert d["beacon"]["id"] == "beacon"
|
assert d["beacon"]["id"] == "beacon"
|
||||||
|
|
||||||
|
|
||||||
|
def test_marker_is_on_a_back_face():
|
||||||
|
"""The triangle must sit on a face pointing away from the spawn corner."""
|
||||||
|
for _ in range(20):
|
||||||
|
room = Room()
|
||||||
|
pos = room.marker_world_pos()
|
||||||
|
assert pos is not None
|
||||||
|
box = next(b for b in room.boxes if b.id == room.marker["box"])
|
||||||
|
face = room.marker["face"]
|
||||||
|
# spawn is at (1.5, 1.5); the marker face normal must point away from it
|
||||||
|
to_spawn = (1.5 - box.cx, 1.5 - box.cz)
|
||||||
|
normal = {
|
||||||
|
"x0": (-1, 0), "x1": (1, 0), "z0": (0, -1), "z1": (0, 1),
|
||||||
|
}[face]
|
||||||
|
assert normal[0] * to_spawn[0] + normal[1] * to_spawn[1] < 0
|
||||||
|
assert pos[0] == pytest.approx(box.x_max()) if face == "x1" else True
|
||||||
|
|
||||||
|
|
||||||
|
def test_marker_not_visible_from_spawn():
|
||||||
|
"""Looking toward the beacon from spawn must not show the triangle."""
|
||||||
|
from testbed.room.render import Raycaster
|
||||||
|
|
||||||
|
room = Room()
|
||||||
|
# Spawn looks at the beacon; the marker is on a back face, so the anchor
|
||||||
|
# pixel of the marker must be occluded (different surface) or off-frame.
|
||||||
|
pos = room.marker_world_pos()
|
||||||
|
assert pos is not None
|
||||||
|
# Stand on the marker's side of the crate, facing it.
|
||||||
|
face = room.marker["face"]
|
||||||
|
offsets = {
|
||||||
|
"x0": (-1.0, 0.0, 90.0), # west face: stand west, face +x
|
||||||
|
"x1": (1.0, 0.0, 270.0), # east face: stand east, face -x
|
||||||
|
"z0": (0.0, -1.0, 0.0), # south face: stand south, face +z
|
||||||
|
"z1": (0.0, 1.0, 180.0), # north face: stand north, face -z
|
||||||
|
}[face]
|
||||||
|
room.capsule.x = pos[0] + offsets[0]
|
||||||
|
room.capsule.z = pos[2] + offsets[1]
|
||||||
|
room.capsule.yaw_deg = offsets[2]
|
||||||
|
img, _ = Raycaster().render(room)
|
||||||
|
px = img.load()
|
||||||
|
orange = 0
|
||||||
|
for y in range(0, 120, 2):
|
||||||
|
for x in range(0, 160, 2):
|
||||||
|
r, g, b = px[x, y]
|
||||||
|
if r > 200 and 100 < g < 190 and b < 90:
|
||||||
|
orange += 1
|
||||||
|
assert orange > 3, "triangle must be visible when looking at the marker face"
|
||||||
|
|||||||