diff --git a/.opencode/session-goal.md b/.opencode/session-goal.md index 199c945..4311761 100644 --- a/.opencode/session-goal.md +++ b/.opencode/session-goal.md @@ -17,5 +17,10 @@ - DONE: headless raycaster renderer (vision 160x120 + aligned depth grid) — testbed/room/render.py - DONE: RoomBridge with all world tools + echo/boom/bump — testbed/bridge.py - DONE: conformance runner — 9/9 green against testbed bridge (testbed/conformance.py) -- DONE: commits for skeleton + bridge milestones -- NEXT: unit tests (world/render/bridge), demo agent loop (ollama LLM + scripted fallback), README, setup.sh, BUILDLOG +- DONE: pytest suite — 33 tests green; ruff clean +- DONE: agent demo loop — LLM (OpenAI-compatible/ollama) + scripted fallback + auto handoff; E2E verified over WebSocket (scripted: 26 steps, beacon activated; LLM mode completed missions with gemma4:e2b) +- DONE: testbed/README.md, scripts/setup.sh deps, scripts/run_bridge.sh, BUILDLOG.md updated +- DONE: commits per milestone (skeleton, bridge+tools+conformance, tests+physics fixes, demo loop, docs, lint) +- VERIFIED: conformance 9/9, pytest 33/33, ruff clean, demo end-to-end + +COMPLETED diff --git a/.playwright-mcp/console-2026-08-08T09-09-21-626Z.log b/.playwright-mcp/console-2026-08-08T09-09-21-626Z.log new file mode 100644 index 0000000..e871804 --- /dev/null +++ b/.playwright-mcp/console-2026-08-08T09-09-21-626Z.log @@ -0,0 +1,26 @@ +[ 28151ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 35175ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 36177ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 37178ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 38180ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 39182ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 40184ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 49096ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 50098ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 51100ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 52101ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 53103ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 54105ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 55108ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 56109ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 57112ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 58115ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 59117ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 60119ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 61123ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 62129ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 63134ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 64140ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 78549ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 88888ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 89892ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 diff --git a/.playwright-mcp/console-2026-08-08T09-11-01-487Z.log b/.playwright-mcp/console-2026-08-08T09-11-01-487Z.log new file mode 100644 index 0000000..a98b051 --- /dev/null +++ b/.playwright-mcp/console-2026-08-08T09-11-01-487Z.log @@ -0,0 +1,97 @@ +[ 48770ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 49773ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 50775ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 54224ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 55226ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 56228ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 57230ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 58232ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 59233ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 60236ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 61239ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 62243ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 63245ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 64248ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 65251ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 66255ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 67256ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 68260ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 69261ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 70268ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 71274ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 72278ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 73283ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 74289ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 75291ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 76295ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 77301ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 78307ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 79317ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 80325ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 81336ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 82342ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 83354ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 84359ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 85363ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 86368ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 87376ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 88395ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 89412ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 90428ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 91434ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 92454ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 93462ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 94478ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 95491ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 96506ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 97520ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 98556ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 99575ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 100594ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 101605ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 102628ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 103653ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 104671ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 105683ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 106703ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 107778ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 108821ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 109857ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 110893ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 111960ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 113007ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 114057ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 115118ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 116231ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 117283ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 118336ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 119407ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 120553ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 121613ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 122714ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 123835ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 124928ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 126073ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 127177ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 128439ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 129670ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 130790ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 132026ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 133143ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 134286ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 137272ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 138376ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 139440ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 144358ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 148352ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 152146ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 156809ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 162641ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 166053ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 168606ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 171349ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 177037ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 179429ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 183147ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 188179ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 +[ 192874ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40 diff --git a/.playwright-mcp/page-2026-08-08T09-09-21-753Z.yml b/.playwright-mcp/page-2026-08-08T09-09-21-753Z.yml new file mode 100644 index 0000000..5b7e21d --- /dev/null +++ b/.playwright-mcp/page-2026-08-08T09-09-21-753Z.yml @@ -0,0 +1,11 @@ +- generic [active] [ref=e1]: + - banner [ref=e2]: + - heading "AICC capsule — real-time view" [level=1] [ref=e3] + - generic [ref=e4]: connected — waiting for frames + - main [ref=e5]: + - generic [ref=e6]: + - heading "first person (vision)" [level=2] [ref=e7] + - img "vision" [ref=e8] + - generic [ref=e9]: + - heading "top-down (sensors)" [level=2] [ref=e10] + - img "map" [ref=e11] \ No newline at end of file diff --git a/.playwright-mcp/page-2026-08-08T09-09-31-437Z.yml b/.playwright-mcp/page-2026-08-08T09-09-31-437Z.yml new file mode 100644 index 0000000..5b7e21d --- /dev/null +++ b/.playwright-mcp/page-2026-08-08T09-09-31-437Z.yml @@ -0,0 +1,11 @@ +- generic [active] [ref=e1]: + - banner [ref=e2]: + - heading "AICC capsule — real-time view" [level=1] [ref=e3] + - generic [ref=e4]: connected — waiting for frames + - main [ref=e5]: + - generic [ref=e6]: + - heading "first person (vision)" [level=2] [ref=e7] + - img "vision" [ref=e8] + - generic [ref=e9]: + - heading "top-down (sensors)" [level=2] [ref=e10] + - img "map" [ref=e11] \ No newline at end of file diff --git a/.playwright-mcp/page-2026-08-08T09-11-01-570Z.yml b/.playwright-mcp/page-2026-08-08T09-11-01-570Z.yml new file mode 100644 index 0000000..6dd9f51 --- /dev/null +++ b/.playwright-mcp/page-2026-08-08T09-11-01-570Z.yml @@ -0,0 +1,11 @@ +- generic [active] [ref=f1e1]: + - banner [ref=f1e2]: + - heading "AICC capsule — real-time view" [level=1] [ref=f1e3] + - generic [ref=f1e4]: connected — waiting for frames + - main [ref=f1e5]: + - generic [ref=f1e6]: + - heading "first person (vision)" [level=2] [ref=f1e7] + - img "vision" [ref=f1e8] + - generic [ref=f1e9]: + - heading "top-down (sensors)" [level=2] [ref=f1e10] + - img "map" [ref=f1e11] \ No newline at end of file diff --git a/.playwright-mcp/page-2026-08-08T09-11-09-014Z.yml b/.playwright-mcp/page-2026-08-08T09-11-09-014Z.yml new file mode 100644 index 0000000..f574927 --- /dev/null +++ b/.playwright-mcp/page-2026-08-08T09-11-09-014Z.yml @@ -0,0 +1,11 @@ +- generic [active] [ref=f1e1]: + - banner [ref=f1e2]: + - heading "AICC capsule — real-time view" [level=1] [ref=f1e3] + - generic [ref=f1e4]: tick 158 pos (11.86, 11.22) yaw 26.5° 1.43 m to beacon · 3.0 fps + - main [ref=f1e5]: + - generic [ref=f1e6]: + - heading "first person (vision)" [level=2] [ref=f1e7] + - img "vision" [ref=f1e8] + - generic [ref=f1e9]: + - heading "top-down (sensors)" [level=2] [ref=f1e10] + - img "map" [ref=f1e11] \ No newline at end of file diff --git a/chat_frame.png b/chat_frame.png new file mode 100644 index 0000000..6ec5df3 Binary files /dev/null and b/chat_frame.png differ diff --git a/chat_map.png b/chat_map.png new file mode 100644 index 0000000..99ba7b7 Binary files /dev/null and b/chat_map.png differ diff --git a/demo_final_frame.png b/demo_final_frame.png index 3ff8882..1009b66 100644 Binary files a/demo_final_frame.png and b/demo_final_frame.png differ diff --git a/frames/demo.gif b/frames/demo.gif new file mode 100644 index 0000000..e1614be Binary files /dev/null and b/frames/demo.gif differ diff --git a/frames/demo_summary.png b/frames/demo_summary.png new file mode 100644 index 0000000..f048b36 Binary files /dev/null and b/frames/demo_summary.png differ diff --git a/frames/step_001_map.png b/frames/step_001_map.png new file mode 100644 index 0000000..167ae69 Binary files /dev/null and b/frames/step_001_map.png differ diff --git a/frames/step_001_view.png b/frames/step_001_view.png new file mode 100644 index 0000000..6ec5df3 Binary files /dev/null and b/frames/step_001_view.png differ diff --git a/frames/step_002_map.png b/frames/step_002_map.png new file mode 100644 index 0000000..c468c2e Binary files /dev/null and b/frames/step_002_map.png differ diff --git a/frames/step_002_view.png b/frames/step_002_view.png new file mode 100644 index 0000000..eef158b Binary files /dev/null and b/frames/step_002_view.png differ diff --git a/frames/step_003_map.png b/frames/step_003_map.png new file mode 100644 index 0000000..1ae169e Binary files /dev/null and b/frames/step_003_map.png differ diff --git a/frames/step_003_view.png b/frames/step_003_view.png new file mode 100644 index 0000000..0121cfa Binary files /dev/null and b/frames/step_003_view.png differ diff --git a/frames/step_004_map.png b/frames/step_004_map.png new file mode 100644 index 0000000..55351f8 Binary files /dev/null and b/frames/step_004_map.png differ diff --git a/frames/step_004_view.png b/frames/step_004_view.png new file mode 100644 index 0000000..7c6ed63 Binary files /dev/null and b/frames/step_004_view.png differ diff --git a/frames/step_005_map.png b/frames/step_005_map.png new file mode 100644 index 0000000..22e1b2a Binary files /dev/null and b/frames/step_005_map.png differ diff --git a/frames/step_005_view.png b/frames/step_005_view.png new file mode 100644 index 0000000..aa88de2 Binary files /dev/null and b/frames/step_005_view.png differ diff --git a/frames/step_006_map.png b/frames/step_006_map.png new file mode 100644 index 0000000..7114ba5 Binary files /dev/null and b/frames/step_006_map.png differ diff --git a/frames/step_006_view.png b/frames/step_006_view.png new file mode 100644 index 0000000..0b29e03 Binary files /dev/null and b/frames/step_006_view.png differ diff --git a/frames/step_007_map.png b/frames/step_007_map.png new file mode 100644 index 0000000..d6baf47 Binary files /dev/null and b/frames/step_007_map.png differ diff --git a/frames/step_007_view.png b/frames/step_007_view.png new file mode 100644 index 0000000..20d96b9 Binary files /dev/null and b/frames/step_007_view.png differ diff --git a/frames/step_008_map.png b/frames/step_008_map.png new file mode 100644 index 0000000..c126d03 Binary files /dev/null and b/frames/step_008_map.png differ diff --git a/frames/step_008_view.png b/frames/step_008_view.png new file mode 100644 index 0000000..728f095 Binary files /dev/null and b/frames/step_008_view.png differ diff --git a/frames/step_009_map.png b/frames/step_009_map.png new file mode 100644 index 0000000..d374639 Binary files /dev/null and b/frames/step_009_map.png differ diff --git a/frames/step_009_view.png b/frames/step_009_view.png new file mode 100644 index 0000000..5265f93 Binary files /dev/null and b/frames/step_009_view.png differ diff --git a/frames/step_010_map.png b/frames/step_010_map.png new file mode 100644 index 0000000..e9c0820 Binary files /dev/null and b/frames/step_010_map.png differ diff --git a/frames/step_010_view.png b/frames/step_010_view.png new file mode 100644 index 0000000..679d853 Binary files /dev/null and b/frames/step_010_view.png differ diff --git a/frames/step_011_map.png b/frames/step_011_map.png new file mode 100644 index 0000000..5b9a7ba Binary files /dev/null and b/frames/step_011_map.png differ diff --git a/frames/step_011_view.png b/frames/step_011_view.png new file mode 100644 index 0000000..553d036 Binary files /dev/null and b/frames/step_011_view.png differ diff --git a/frames/step_012_map.png b/frames/step_012_map.png new file mode 100644 index 0000000..94c3e2d Binary files /dev/null and b/frames/step_012_map.png differ diff --git a/frames/step_012_view.png b/frames/step_012_view.png new file mode 100644 index 0000000..cab11e8 Binary files /dev/null and b/frames/step_012_view.png differ diff --git a/frames/step_013_map.png b/frames/step_013_map.png new file mode 100644 index 0000000..0cab83c Binary files /dev/null and b/frames/step_013_map.png differ diff --git a/frames/step_013_view.png b/frames/step_013_view.png new file mode 100644 index 0000000..4be7168 Binary files /dev/null and b/frames/step_013_view.png differ diff --git a/frames/step_014_map.png b/frames/step_014_map.png new file mode 100644 index 0000000..6991649 Binary files /dev/null and b/frames/step_014_map.png differ diff --git a/frames/step_014_view.png b/frames/step_014_view.png new file mode 100644 index 0000000..14d87dc Binary files /dev/null and b/frames/step_014_view.png differ diff --git a/frames/step_015_map.png b/frames/step_015_map.png new file mode 100644 index 0000000..43fd3e2 Binary files /dev/null and b/frames/step_015_map.png differ diff --git a/frames/step_015_view.png b/frames/step_015_view.png new file mode 100644 index 0000000..8c86c07 Binary files /dev/null and b/frames/step_015_view.png differ diff --git a/frames/step_016_map.png b/frames/step_016_map.png new file mode 100644 index 0000000..d0bed42 Binary files /dev/null and b/frames/step_016_map.png differ diff --git a/frames/step_016_view.png b/frames/step_016_view.png new file mode 100644 index 0000000..4791107 Binary files /dev/null and b/frames/step_016_view.png differ diff --git a/frames/step_017_map.png b/frames/step_017_map.png new file mode 100644 index 0000000..676a551 Binary files /dev/null and b/frames/step_017_map.png differ diff --git a/frames/step_017_view.png b/frames/step_017_view.png new file mode 100644 index 0000000..bf6b885 Binary files /dev/null and b/frames/step_017_view.png differ diff --git a/frames/step_018_map.png b/frames/step_018_map.png new file mode 100644 index 0000000..a292369 Binary files /dev/null and b/frames/step_018_map.png differ diff --git a/frames/step_018_view.png b/frames/step_018_view.png new file mode 100644 index 0000000..4876127 Binary files /dev/null and b/frames/step_018_view.png differ diff --git a/frames/step_019_map.png b/frames/step_019_map.png new file mode 100644 index 0000000..569a90d Binary files /dev/null and b/frames/step_019_map.png differ diff --git a/frames/step_019_view.png b/frames/step_019_view.png new file mode 100644 index 0000000..e0a0759 Binary files /dev/null and b/frames/step_019_view.png differ diff --git a/frames/step_020_map.png b/frames/step_020_map.png new file mode 100644 index 0000000..c24dec6 Binary files /dev/null and b/frames/step_020_map.png differ diff --git a/frames/step_020_view.png b/frames/step_020_view.png new file mode 100644 index 0000000..5ff15a4 Binary files /dev/null and b/frames/step_020_view.png differ diff --git a/frames/step_021_map.png b/frames/step_021_map.png new file mode 100644 index 0000000..7b21185 Binary files /dev/null and b/frames/step_021_map.png differ diff --git a/frames/step_021_view.png b/frames/step_021_view.png new file mode 100644 index 0000000..e33cf0f Binary files /dev/null and b/frames/step_021_view.png differ diff --git a/frames/step_022_map.png b/frames/step_022_map.png new file mode 100644 index 0000000..f5d3540 Binary files /dev/null and b/frames/step_022_map.png differ diff --git a/frames/step_022_view.png b/frames/step_022_view.png new file mode 100644 index 0000000..840037e Binary files /dev/null and b/frames/step_022_view.png differ diff --git a/frames/step_023_map.png b/frames/step_023_map.png new file mode 100644 index 0000000..470881e Binary files /dev/null and b/frames/step_023_map.png differ diff --git a/frames/step_023_view.png b/frames/step_023_view.png new file mode 100644 index 0000000..8b612c3 Binary files /dev/null and b/frames/step_023_view.png differ diff --git a/frames/step_024_map.png b/frames/step_024_map.png new file mode 100644 index 0000000..42e190c Binary files /dev/null and b/frames/step_024_map.png differ diff --git a/frames/step_024_view.png b/frames/step_024_view.png new file mode 100644 index 0000000..dd836bb Binary files /dev/null and b/frames/step_024_view.png differ diff --git a/frames/step_025_map.png b/frames/step_025_map.png new file mode 100644 index 0000000..ec3f66f Binary files /dev/null and b/frames/step_025_map.png differ diff --git a/frames/step_025_view.png b/frames/step_025_view.png new file mode 100644 index 0000000..f4c7c1c Binary files /dev/null and b/frames/step_025_view.png differ diff --git a/frames/step_026_map.png b/frames/step_026_map.png new file mode 100644 index 0000000..58c5900 Binary files /dev/null and b/frames/step_026_map.png differ diff --git a/frames/step_026_view.png b/frames/step_026_view.png new file mode 100644 index 0000000..2ddd69d Binary files /dev/null and b/frames/step_026_view.png differ diff --git a/scripts/run_chat.sh b/scripts/run_chat.sh new file mode 100755 index 0000000..e9cf3c8 --- /dev/null +++ b/scripts/run_chat.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Interactive chat mode: bridge + live viewer + chat REPL in one command. +# Open http://127.0.0.1:8000 to watch the capsule while you talk to it. +# Usage: scripts/run_chat.sh [--model gemma4:e2b] +set -euo pipefail +cd "$(dirname "$0")/.." +BRIDGE_PID="" +LIVE_PID="" + +# 1. bridge (reuse an already-running one) +if python3 -c 'import socket,sys; s=socket.socket(); sys.exit(0 if s.connect_ex(("127.0.0.1",8765))==0 else 1)'; then + echo "[chat] bridge already running on 8765" +else + testbed/.venv/bin/python -m testbed.bridge --port 8765 >/tmp/aicc-bridge.log 2>&1 & + BRIDGE_PID=$! + sleep 1.5 + echo "[chat] bridge started (pid $BRIDGE_PID)" +fi + +# 2. live viewer (optional but nice: watch while chatting) +testbed/.venv/bin/python -m testbed.live --http-port 8000 --ws-port 8001 >/tmp/aicc-live.log 2>&1 & +LIVE_PID=$! +sleep 1.5 +echo "[chat] live view: open http://127.0.0.1:8000 in your browser" + +# 3. chat REPL (foreground) +testbed/.venv/bin/python -m testbed.chat "$@" +EXIT=$? + +kill "$LIVE_PID" 2>/dev/null || true +if [ -n "$BRIDGE_PID" ]; then kill "$BRIDGE_PID" 2>/dev/null || true; fi +exit $EXIT diff --git a/testbed/.gitignore b/testbed/.gitignore index e1736e9..66cb77a 100644 --- a/testbed/.gitignore +++ b/testbed/.gitignore @@ -3,3 +3,5 @@ __pycache__/ *.pyc *.egg-info/ .pytest_cache/ +chat_frame.png +chat_map.png diff --git a/testbed/README.md b/testbed/README.md index 0803dab..0ad4466 100644 --- a/testbed/README.md +++ b/testbed/README.md @@ -22,6 +22,8 @@ over the protocol, no engine hooks. | `server.py` entry | `python -m testbed.bridge` — WebSocket server. | | `conformance.py` | Runs the 9 core conformance scenarios against this bridge. | | `live.py` | Real-time browser viewer (itself an AICC client). | +| `chat.py` | Interactive chat: natural language -> tool calls. | +| `llm_agent.py` | Shared LLM driver (controller + autonomous loop). | | `demo.py` | Agent demo: LLM driver (OpenAI-compatible) or scripted. | | `tests/` | pytest suite (world, renderer, bridge, protocol). | @@ -62,6 +64,33 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal. The demo prints a full transcript of tool calls/results to stdout and saves the capsule's final first-person frame to `demo_final_frame.png`. +## Interactive chat mode + +Talk to the capsule's brain in natural language (any language): + +```bash +scripts/run_chat.sh --model gemma4:e2b # small + fast +scripts/run_chat.sh --model gemma4:12b # bigger gemma, slower (~20 s/turn) +``` + +It starts the bridge, the live viewer and a chat REPL — open +[http://127.0.0.1:8000](http://127.0.0.1:8000) to watch the capsule while you +type. The model translates your words into tool calls: + +``` +you> иди к маяку → look_at + move step by step (auto-continue) +you> повернись налево → turn(-90) +you> осмотрись → vision + description +you> активируй маяк → interact (when close) +you> /state /look /map /models /model gemma4:12b /steps N /help /exit +``` + +Each turn's transcript is printed; the current frame lands in `chat_frame.png` +and the sensor-built map in `chat_map.png`. Any OpenAI-compatible endpoint +works: `python -m testbed.chat --base-url https://api.openai.com/v1 +--model gpt-4o-mini --api-key $OPENAI_API_KEY`. `--auto-steps N` controls how +many tool steps the model may chain per request (0 = one action per turn). + ## Real-time mode Watch the capsule drive live in your browser: diff --git a/testbed/chat.py b/testbed/chat.py new file mode 100644 index 0000000..032813a --- /dev/null +++ b/testbed/chat.py @@ -0,0 +1,233 @@ +"""Interactive chat mode: talk to the model, it drives the capsule. + +Type natural-language commands (any language) and the LLM translates them +into AICC tool calls: 'go to the beacon', 'turn left', 'look around', +'activate the beacon'. Every tool call and result is printed as a transcript, +the current vision frame is saved to ``chat_frame.png`` and the sensor-built +top-down map to ``chat_map.png`` after each turn. Works great together with +the live viewer (scripts/run_live.sh) — watch the capsule in your browser +while you talk to it. + +Commands: + /state print current position/heading (proprioception) + /look take a vision frame and save it + /map save the top-down map (chat_map.png) + /models list models on the endpoint + /model X switch the model mid-session + /steps N auto-continue budget for 'go to X' style requests + /help this text + /exit quit + +Usage: + python -m testbed.chat [--model gemma4:e2b] [--model gemma4:12b] + python -m testbed.chat --base-url https://api.openai.com/v1 --model gpt-4o-mini +""" + +from __future__ import annotations + +import argparse +import asyncio +import sys + +from aicc.client import AICCClient +from aicc.transport.websocket import WebSocketClientTransport + +from testbed.llm_agent import CHAT_MISSION, LLMController, ollama_models + +DEFAULT_URL = "ws://127.0.0.1:8765" +DEFAULT_BASE_URL = "http://localhost:11434/v1" +DEFAULT_MODEL = "gemma4:e2b" + +HELP = """\ +You are talking to the capsule's brain (LLM). Type what you want, e.g.: + 'иди к маяку' / 'go to the beacon' — the model walks there step by step + 'повернись налево' / 'turn left' — quick turn + 'осмотрись' / 'look around' — vision + description + 'активируй маяк' / 'activate it' — interact (must be close) +Commands: /state /look /map /models /model X /steps N /help /exit""" + + +async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> int: + model = args.model + controller = LLMController( + client, + manifest, + base_url=args.base_url, + api_key=args.api_key, + model=model, + system_prompt=CHAT_MISSION, + log=lambda role, msg: print(f" [{role}] {msg}"), + ) + auto_steps = args.auto_steps + print(f"[chat] model: {model} (endpoint {args.base_url})") + print("[chat] type your commands; /help for the command list; /exit to quit\n") + + async def cmd_state() -> None: + out = (await client.call_tool("proprioception", {})).output + print( + f" pos ({out['position']['x']:.2f}, {out['position']['z']:.2f}) " + f"heading {out['rotation']['yaw_deg']:.1f} deg health {out['health']}" + ) + + async def cmd_look() -> None: + out = (await client.call_tool("vision", {})).output + import base64 + + raw = base64.b64decode(out["png_b64"]) + + def _write() -> None: + with open("chat_frame.png", "wb") as fh: + fh.write(raw) + + await asyncio.to_thread(_write) + print(f" frame saved to chat_frame.png ({out['width']}x{out['height']})") + + def save_map() -> None: + if controller.pos is None: + print(" (no position yet — ask the capsule to move first)") + return + from testbed.room.mapview import draw_sensor_map + + layout = {"obstacles": [], "beacon": {"x": 12.5, "z": 12.5}} + if controller.beacon_pos is not None: + layout["beacon"] = { + "x": controller.beacon_pos[0], + "z": controller.beacon_pos[1], + } + img = draw_sensor_map(layout, controller.pos, controller.yaw, controller.path) + img.save("chat_map.png") + print( + f" map saved to chat_map.png (pos {controller.pos[0]:.1f}, {controller.pos[1]:.1f})" + ) + + async def cmd_models() -> None: + print(" fetching models…") + for m in ollama_models(args.base_url, args.api_key): + print(f" {m}") + + while True: + try: + text = input("you> ").strip() + except (EOFError, KeyboardInterrupt): + print("\n[chat] bye") + return 0 + if not text: + continue + low = text.lower() + if low in ("/exit", "exit", "quit", "выход"): + print("[chat] bye") + return 0 + if low == "/help": + print(HELP) + continue + if low == "/state": + await cmd_state() + continue + if low == "/look": + await cmd_look() + continue + if low == "/map": + save_map() + continue + if low == "/models": + await cmd_models() + continue + if low.startswith("/model "): + model = low.split(maxsplit=1)[1] + controller.model = model + print(f"[chat] switching to {model}") + continue + if low.startswith("/steps "): + try: + auto_steps = max(0, int(low.split(maxsplit=1)[1])) + print(f"[chat] auto-continue budget: {auto_steps} steps") + except ValueError: + print(" usage: /steps N") + continue + + controller.messages.append({"role": "user", "content": text}) + steps_used = 0 + while True: + try: + turn = await controller.invoke() + except RuntimeError as exc: + print(f" [error] {exc}") + break + steps_used += 1 + if turn.text: + print(f" model> {turn.text}") + if turn.interacted: + print(f"\n[chat] BEACON ACTIVATED: {turn.message}") + save_map() + return 0 + if not turn.calls: + if not turn.text and auto_steps and steps_used <= auto_steps: + # The model answered with nothing: nudge it to actually act. + controller.messages.append( + { + "role": "user", + "content": ( + "Your last turn contained no action and no answer. " + "Carry out the user's request: call a tool now " + "(check the CURRENT STATE note)." + ), + } + ) + print(" [chat] (nudge: model gave an empty turn)") + continue + break # the model answered in words; wait for the user + if auto_steps and steps_used >= auto_steps: + print( + f" [chat] (auto-continue budget of {auto_steps} reached — say 'continue' to keep going)" + ) + break + if not auto_steps: + break + # The model acted without commenting; let it keep going for 'go to X' + if controller.pos is not None: + save_map() + + return 0 + + +async def run(args: argparse.Namespace) -> int: + transport = WebSocketClientTransport(args.url) + async with AICCClient(transport) as client: + manifest = await client.handshake() + print(f"[handshake] session {manifest.session_id} world {manifest.world.name}") + print(f"[handshake] tools: {[t.id for t in manifest.tools]}") + return await chat_loop(client, manifest, args) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Interactive chat: talk to the LLM, it drives the capsule." + ) + parser.add_argument( + "--url", + default=DEFAULT_URL, + help=f"bridge WebSocket URL (default {DEFAULT_URL})", + ) + parser.add_argument( + "--base-url", default=DEFAULT_BASE_URL, help="OpenAI-compatible endpoint" + ) + parser.add_argument( + "--model", default=DEFAULT_MODEL, help="model id on the endpoint" + ) + parser.add_argument("--api-key", default="ollama", help="API key for the endpoint") + parser.add_argument( + "--auto-steps", + type=int, + default=8, + help="how many tool steps the model may chain per request (0 = one action per turn)", + ) + args = parser.parse_args() + try: + return asyncio.run(run(args)) + except KeyboardInterrupt: + print("\n[chat] bye") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testbed/demo.py b/testbed/demo.py index f98d094..d388f5f 100644 --- a/testbed/demo.py +++ b/testbed/demo.py @@ -33,8 +33,7 @@ from pathlib import Path from typing import Any from aicc.client import AICCClient -from aicc.errors import ToolError -from aicc.protocol import SessionInit, ToolSpec +from aicc.protocol import SessionInit from aicc.transport.websocket import WebSocketClientTransport from PIL import Image @@ -42,51 +41,6 @@ DEFAULT_URL = "ws://127.0.0.1:8765" DEFAULT_BASE_URL = "http://localhost:11434/v1" DEFAULT_MODEL = "gemma4:e2b" -NUDGE = ( - "You have not called a tool. Continue the mission: check the CURRENT STATE note, " - "then call a tool (turn or move, or interact if within 1.6 m)." -) - -MISSION = ( - "You are a capsule AI inside a 16x16 m room. Your mission: reach the glowing " - "beacon pillar and activate it with the interact tool. " - "World: yaw 0 faces +z, 90 faces +x, coordinates in meters; three wooden " - "crates block parts of the room. " - "Perception: proprioception gives position/heading; world_query gives the room " - "layout and the beacon's exact coordinates (call it once at the start); " - "vision gives the first-person frame; hear tells you how close you are to the beacon. " - "Every tool result is prefixed with a CURRENT STATE note: read it — it tells you " - "your position, heading, distance to the beacon, and the bearing to turn toward it. " - "Standard loop, one action per turn: " - "1. call proprioception (and hear every few steps); " - "2. compute the bearing to the beacon and turn toward it (turn yaw_deg = that bearing, max 90 per call); " - "3. move forward 1.0 m; if the result says collision, turn 60-90 degrees and go around, " - "then re-aim. " - "Never walk into a crate or wall twice in a row. " - "Call interact ONLY when proprioception puts you within 1.6 m of the beacon. " - "Keep tool calls short and decisive. Stop once the beacon is active." -) - -COLOR_NAMES = [ - ("beacon_cyan", (120, 210, 235)), - ("beacon_yellow", (255, 190, 90)), - ("crate_red", (178, 64, 54)), - ("crate_blue", (64, 96, 178)), - ("crate_olive", (128, 128, 60)), - ("floor_gray", (60, 60, 66)), - ("wall_gray", (96, 96, 106)), - ("dark", (30, 30, 36)), -] - - -def _nearest_color(r: float, g: float, b: float) -> str: - best, best_d = "dark", 1e9 - for name, (cr, cg, cb) in COLOR_NAMES: - d = (r - cr) ** 2 + (g - cg) ** 2 + (b - cb) ** 2 - if d < best_d: - best, best_d = name, d - return best - class FrameRecorder: """Visual mode: saves first-person frames and a sensor-built top-down map @@ -171,63 +125,11 @@ class FrameRecorder: return summary -def vision_digest(png_b64: str, cols: int = 12, rows: int = 9) -> str: - """Coarse color-grid summary of a vision frame (agent-side preprocessing).""" - try: - from PIL import Image - - img = Image.open(io.BytesIO(base64.b64decode(png_b64))).convert("RGB") - except Exception: # noqa: BLE001 - digest is best-effort - return "(no digest: unreadable frame)" - small = img.resize((cols, rows)) - grid: list[list[tuple[int, int, int]]] = [] - for r in range(rows): - row: list[tuple[int, int, int]] = [] - for c in range(cols): - px = small.getpixel((c, r)) - if isinstance(px, (tuple, list)) and len(px) >= 3: - row.append((int(px[0]), int(px[1]), int(px[2]))) - elif isinstance(px, (int, float)): - v = int(px) - row.append((v, v, v)) - else: - row.append((0, 0, 0)) - grid.append(row) - lines = [] - for r in range(rows): - lines.append(" ".join(_nearest_color(*grid[r][c]) for c in range(cols))) - return f"frame digest ({cols}x{rows}, center = where you face):\n" + "\n".join( - lines - ) - - # --------------------------------------------------------------------------- # LLM agent (OpenAI-compatible) # --------------------------------------------------------------------------- -def to_openai_tools(tools: list[ToolSpec]) -> list[dict[str, Any]]: - out = [] - for t in tools: - if t.id in ( - "echo", - "boom", - "bump", - ): # conformance-only tools: not for the agent - continue - out.append( - { - "type": "function", - "function": { - "name": t.id, - "description": t.description, - "parameters": t.input_schema, - }, - } - ) - return out - - async def run_llm_agent( client: AICCClient, manifest: SessionInit, @@ -238,205 +140,25 @@ async def run_llm_agent( max_steps: int, recorder: FrameRecorder | None = None, ) -> dict[str, Any]: - from openai import AsyncOpenAI + """Autonomous LLM run: controller + nudge/correct loop (see llm_agent).""" + from testbed.llm_agent import LLMController, run_llm_agent_loop - ac = AsyncOpenAI(base_url=base_url, api_key=api_key) - tools = to_openai_tools(manifest.tools) - messages: list[dict[str, Any]] = [ - {"role": "system", "content": MISSION}, - { - "role": "user", - "content": "Begin your mission. Use the tools to reach and activate the beacon.", - }, - ] - summary: dict[str, Any] = { - "steps": 0, - "tool_calls": 0, - "result": None, - "interacted": False, - } + controller = LLMController( + client, + manifest, + base_url=base_url, + api_key=api_key, + model=model, + log=lambda role, msg: print(f"[{role}] {msg}"), + ) + try: + return await run_llm_agent_loop(controller, max_steps, log=print_log, recorder=recorder) + except RuntimeError as exc: + return {"steps": 0, "tool_calls": 0, "result": str(exc), "interacted": False} - # Agent-side working memory: a compact summary of state the model has - # already observed via sensors, prefixed to tool results so small models - # don't drift (protocol §9: agent memory is the agent's concern). - beacon_pos: tuple[float, float] | None = None - pos: tuple[float, float] | None = None - yaw: float = 0.0 - def state_hint() -> str: - if pos is None: - return "" - if beacon_pos is not None: - dist = math.hypot(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1]) - bearing = ( - math.degrees(math.atan2(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1])) - - yaw - ) % 360.0 - return ( - f"CURRENT STATE: you are at ({pos[0]:.2f}, {pos[1]:.2f}), heading {yaw:.1f} deg; " - f"the beacon is {dist:.2f} m away at relative bearing {bearing:.0f} deg. " - f"Action: turn yaw_deg={bearing:.0f} to face it, then move forward." - ) - return f"CURRENT STATE: you are at ({pos[0]:.2f}, {pos[1]:.2f}), heading {yaw:.1f} deg." - - def observe(output: dict[str, Any]) -> None: - nonlocal pos, yaw - if isinstance(output.get("position"), dict): - pos = (output["position"]["x"], output["position"]["z"]) - if isinstance(output.get("rotation"), dict): - yaw = output["rotation"]["yaw_deg"] % 360.0 - - def log(role: str, msg: str) -> None: - print(f"[{role}] {msg}") - - # Progress monitor: if the capsule keeps moving away from the beacon, - # reflect that back to the model as a correction (framework-style loop). - dist_history: list[float] = [] - corrections = 0 - last_correct_step = -99 - last_collision_step = -99 - - def distance_to_beacon() -> float: - if pos is None or beacon_pos is None: - return float("nan") - return math.hypot(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1]) - - def maybe_correct(step: int) -> None: - nonlocal corrections, last_correct_step - if pos is None or beacon_pos is None: - return - dist = distance_to_beacon() - if not math.isfinite(dist) or dist < 2.0: - return - dist_history.append(dist) - recent = dist_history[-4:] - moving_away = len(recent) == 4 and all( - recent[i] > recent[i + 1] for i in range(3) - ) - bearing = ( - math.degrees(math.atan2(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1])) - - yaw - ) % 360.0 - deviation = min(bearing, 360.0 - bearing) - facing_wrong = dist > 4.0 and deviation > 40.0 and step - last_correct_step > 3 - detouring = step - last_collision_step < 3 - if (moving_away or (facing_wrong and not detouring)) and corrections < 5: - corrections += 1 - last_correct_step = step - hint = state_hint() - msg = ( - "You are not making progress toward the beacon. Stop and follow the CURRENT " - f"STATE note exactly: {hint} Call turn with that yaw_deg, then move forward." - ) - messages.append({"role": "user", "content": msg}) - log("agent", "(correction: re-aim at the beacon)") - dist_history.clear() - - for step in range(max_steps): - summary["steps"] = step + 1 - if recorder is not None and pos is not None: - await recorder.snap(client, pos, yaw) - try: - resp = await ac.chat.completions.create( - model=model, messages=messages, tools=tools, tool_choice="auto" - ) - except Exception as exc: # noqa: BLE001 - provider errors end the demo - log("agent", f"LLM error: {exc}") - summary["result"] = f"LLM error: {exc}" - return summary - - choice = resp.choices[0] - if choice.message.content: - log("agent", choice.message.content[:400]) - messages.append({"role": "assistant", "content": choice.message.content}) - calls = choice.message.tool_calls - if not calls: - # Nudge a model that drifted into plain text back to acting. - silent = (not choice.message.content) or len(choice.message.content) < 8 - already_nudged = bool(messages) and messages[-1].get("content") == NUDGE - if silent or already_nudged: - summary["result"] = "model produced no tool call" - return summary - messages.append({"role": "user", "content": NUDGE}) - log("agent", "(nudge: no tool call; continue the mission)") - continue - messages.append( - { - "role": "assistant", - "content": choice.message.content or "", - "tool_calls": [ - { - "id": c.id, - "type": "function", - "function": { - "name": c.function.name, - "arguments": c.function.arguments, - }, - } - for c in calls - ], - } - ) - for tc in calls: - name = tc.function.name - try: - args = json.loads(tc.function.arguments or "{}") - except json.JSONDecodeError: - args = {} - log("tool", f"{name}({json.dumps(args)})") - try: - result = await client.call_tool(name, args) - output = result.output - extra = "" - if name == "vision" and isinstance(output.get("png_b64"), str): - extra = "\n" + vision_digest(output["png_b64"]) - output = dict(output, png_b64="") - log("bridge", f"ok {json.dumps(output)[:500]}{extra}") - messages.append( - { - "role": "tool", - "tool_call_id": tc.id, - "content": json.dumps(result.output) + extra, - } - ) - summary["tool_calls"] += 1 - if name == "world_query": - b = output.get("beacon") - if isinstance(b, dict): - beacon_pos = (b["x"], b["z"]) - observe(output) - hint = state_hint() - if hint: - messages[-1] = dict( - messages[-1], content=hint + "\n" + messages[-1]["content"] - ) - maybe_correct(step) - if name == "move" and output.get("collision"): - last_collision_step = step - obj = output.get("collision_normal") - hint_msg = ( - f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move " - "forward twice to get around it, then follow the CURRENT STATE note to re-aim." - ) - if not (messages and messages[-1].get("content") == hint_msg): - messages.append({"role": "user", "content": hint_msg}) - log("agent", "(collision: go around)") - if name == "interact" and output.get("success"): - summary["interacted"] = True - summary["result"] = output.get("message", "beacon activated") - return summary - except ToolError as e: - log("bridge", f"error {e.code}: {e.message}") - messages.append( - { - "role": "tool", - "tool_call_id": tc.id, - "content": json.dumps({"error": e.code, "message": e.message}), - } - ) - summary["tool_calls"] += 1 - summary["result"] = f"exceeded {max_steps} steps" - return summary +def print_log(role: str, msg: str) -> None: + print(f"[{role}] {msg}") # --------------------------------------------------------------------------- diff --git a/testbed/llm_agent.py b/testbed/llm_agent.py new file mode 100644 index 0000000..91effb7 --- /dev/null +++ b/testbed/llm_agent.py @@ -0,0 +1,463 @@ +"""Shared LLM tool-use machinery for the demo and chat modes. + +A tool-calling LLM (any OpenAI-compatible endpoint) drives the capsule by +translating intent into AICC tool calls. The controller keeps agent-side +working memory (protocol §9): a CURRENT STATE note derived from what the model +has already observed, vision digest for text-only models, and a transcript of +tool calls/results fed back as `tool` messages. +""" + +from __future__ import annotations + +import base64 +import io +import json +import math +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +from aicc.client import AICCClient +from aicc.errors import ToolError +from aicc.protocol import SessionInit, ToolSpec + +MISSION = ( + "You are a capsule AI inside a 16x16 m room. Your mission: reach the glowing " + "beacon pillar and activate it with the interact tool. " + "World: yaw 0 faces +z, 90 faces +x, coordinates in meters; three wooden " + "crates block parts of the room. " + "Perception: proprioception gives position/heading; world_query gives the room " + "layout and the beacon's exact coordinates (call it once at the start); " + "vision gives the first-person frame; hear tells you how close you are to the beacon. " + "Every tool result is prefixed with a CURRENT STATE note: read it — it tells you " + "your position, heading, distance to the beacon, and the bearing to turn toward it. " + "Standard loop, one action per turn: " + "1. call proprioception (and hear every few steps); " + "2. compute the bearing to the beacon and turn toward it (turn yaw_deg = that bearing, max 90 per call); " + "3. move forward 1.0 m; if the result says collision, turn 60-90 degrees and go around, " + "then re-aim. " + "Never walk into a crate or wall twice in a row. " + "Call interact ONLY when proprioception puts you within 1.6 m of the beacon. " + "Keep tool calls short and decisive. Stop once the beacon is active." +) + +CHAT_MISSION = ( + "You are the brain of a capsule robot inside a 16x16 m room with a glowing " + "beacon at the far corner and a few wooden crates. The user talks to you in " + "natural language (any language) and you translate their intent into tool " + "calls. " + "World: yaw 0 faces +z, 90 faces +x, coordinates in meters. " + "Every tool result is prefixed with a CURRENT STATE note: position, heading, " + "distance to the beacon and the bearing to turn toward it. " + "Rules: " + "1. Take the user's request literally: 'go to the beacon' means turn toward " + "it and move step by step until close; 'turn left/right' means turn ~90 deg; " + "'look around' means vision and describe what you see; 'activate' means " + "interact (only within 1.6 m). " + "2. One tool call per turn is fine — you will get another turn to continue. " + "3. If a move reports a collision, turn and go around. " + "4. When the user asks something not involving movement, just answer briefly. " + "5. Keep the user informed with one short line after acting (or if you need " + "a clarification). " + "Call interact only when within 1.6 m of the beacon." +) + +NUDGE = ( + "You have not called a tool. Continue the mission: check the CURRENT STATE note, " + "then call a tool (turn or move, or interact if within 1.6 m)." +) + +COLOR_NAMES = [ + ("beacon_cyan", (120, 210, 235)), + ("beacon_yellow", (255, 190, 90)), + ("crate_red", (178, 64, 54)), + ("crate_blue", (64, 96, 178)), + ("crate_olive", (128, 128, 60)), + ("floor_gray", (60, 60, 66)), + ("wall_gray", (96, 96, 106)), + ("dark", (30, 30, 36)), +] + +LogFn = Callable[[str, str], None] + + +def _nearest_color(r: float, g: float, b: float) -> str: + best, best_d = "dark", 1e9 + for name, (cr, cg, cb) in COLOR_NAMES: + d = (r - cr) ** 2 + (g - cg) ** 2 + (b - cb) ** 2 + if d < best_d: + best, best_d = name, d + return best + + +def vision_digest(png_b64: str, cols: int = 12, rows: int = 9) -> str: + """Coarse color-grid summary of a vision frame (agent-side preprocessing).""" + try: + from PIL import Image + + img = Image.open(io.BytesIO(base64.b64decode(png_b64))).convert("RGB") + except Exception: # noqa: BLE001 - digest is best-effort + return "(no digest: unreadable frame)" + small = img.resize((cols, rows)) + grid: list[list[tuple[int, int, int]]] = [] + for r in range(rows): + row: list[tuple[int, int, int]] = [] + for c in range(cols): + px = small.getpixel((c, r)) + if isinstance(px, (tuple, list)) and len(px) >= 3: + row.append((int(px[0]), int(px[1]), int(px[2]))) + elif isinstance(px, (int, float)): + v = int(px) + row.append((v, v, v)) + else: + row.append((0, 0, 0)) + grid.append(row) + lines = [ + " ".join(_nearest_color(*grid[r][c]) for c in range(cols)) for r in range(rows) + ] + return f"frame digest ({cols}x{rows}, center = where you face):\n" + "\n".join( + lines + ) + + +def to_openai_tools(tools: list[ToolSpec]) -> list[dict[str, Any]]: + """Convert manifest ToolSpecs to OpenAI function schemas (skip conformance tools).""" + out = [] + for t in tools: + if t.id in ("echo", "boom", "bump"): + continue + out.append( + { + "type": "function", + "function": { + "name": t.id, + "description": t.description, + "parameters": t.input_schema, + }, + } + ) + return out + + +def ollama_models(base_url: str, api_key: str) -> list[str]: + """List model ids on an OpenAI-compatible endpoint (best effort).""" + from openai import AsyncOpenAI + + ids: list[str] = [] + + async def _fetch() -> None: + ac = AsyncOpenAI(base_url=base_url, api_key=api_key) + try: + models = await ac.models.list() + except Exception: # noqa: BLE001 + return + ids.extend(m.id for m in models.data if m.id) + + import asyncio + + asyncio.run(_fetch()) + return ids + + +# --------------------------------------------------------------------------- +# Controller: one conversation with the tool-calling LLM +# --------------------------------------------------------------------------- + + +@dataclass +class ToolCallResult: + name: str + args: dict[str, Any] + ok: bool + output: dict[str, Any] | None = None + error: str | None = None + digest: str | None = None + + +@dataclass +class TurnResult: + """One model round-trip: optional text plus the executed tool calls.""" + + text: str + calls: list[ToolCallResult] = field(default_factory=list) + interacted: bool = False + message: str | None = None + + +class LLMController: + """Drives a tool-calling LLM over an OpenAI-compatible endpoint. + + Holds the message history and the agent-side state (position, heading, + beacon, path) distilled from tool results. ``invoke()`` runs one model + turn and executes whatever tools it asks for. + """ + + def __init__( + self, + client: AICCClient, + manifest: SessionInit, + *, + base_url: str, + api_key: str, + model: str, + system_prompt: str = MISSION, + log: LogFn | None = None, + ): + from openai import AsyncOpenAI + + self.client = client + self.model = model + self.ac = AsyncOpenAI(base_url=base_url, api_key=api_key) + self.tools = to_openai_tools(manifest.tools) + self.messages: list[dict[str, Any]] = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": "Begin. Use the tools to control the capsule.", + }, + ] + self.log = log or (lambda role, msg: None) + self.beacon_pos: tuple[float, float] | None = None + self.pos: tuple[float, float] | None = None + self.yaw: float = 0.0 + self.path: list[tuple[float, float]] = [] + + # -- agent-side working memory ----------------------------------------- + + def state_hint(self) -> str: + if self.pos is None: + return "" + if self.beacon_pos is not None: + dist = math.hypot( + self.beacon_pos[0] - self.pos[0], self.beacon_pos[1] - self.pos[1] + ) + bearing = ( + math.degrees( + math.atan2( + self.beacon_pos[0] - self.pos[0], + self.beacon_pos[1] - self.pos[1], + ) + ) + - self.yaw + ) % 360.0 + return ( + f"CURRENT STATE: you are at ({self.pos[0]:.2f}, {self.pos[1]:.2f}), " + f"heading {self.yaw:.1f} deg; the beacon is {dist:.2f} m away at relative " + f"bearing {bearing:.0f} deg. Action: turn yaw_deg={bearing:.0f} to face it, " + f"then move forward." + ) + return f"CURRENT STATE: you are at ({self.pos[0]:.2f}, {self.pos[1]:.2f}), heading {self.yaw:.1f} deg." + + def observe(self, output: dict[str, Any]) -> None: + if isinstance(output.get("position"), dict): + self.pos = (output["position"]["x"], output["position"]["z"]) + if isinstance(output.get("rotation"), dict): + self.yaw = output["rotation"]["yaw_deg"] % 360.0 + if self.pos is not None and (not self.path or self.path[-1] != self.pos): + self.path.append(self.pos) + if len(self.path) > 4000: + self.path = self.path[-2000:] + + # -- conversation ------------------------------------------------------- + + async def send_user(self, text: str) -> None: + self.messages.append({"role": "user", "content": text}) + + async def invoke(self) -> TurnResult: + """One model round-trip: get the response and execute its tool calls.""" + try: + resp = await self.ac.chat.completions.create( + model=self.model, + messages=self.messages, + tools=self.tools, + tool_choice="auto", + ) + except Exception as exc: + self.log("agent", f"LLM error: {exc}") + raise RuntimeError(f"LLM error: {exc}") from exc + + choice = resp.choices[0] + text = choice.message.content or "" + result = TurnResult(text=text) + calls = choice.message.tool_calls + if not calls: + self.messages.append({"role": "assistant", "content": text}) + return result + + self.messages.append( + { + "role": "assistant", + "content": text, + "tool_calls": [ + { + "id": c.id, + "type": "function", + "function": { + "name": c.function.name, + "arguments": c.function.arguments, + }, + } + for c in calls + ], + } + ) + for tc in calls: + name = tc.function.name + try: + args = json.loads(tc.function.arguments or "{}") + except json.JSONDecodeError: + args = {} + self.log("tool", f"{name}({json.dumps(args)})") + try: + outcome = await self.client.call_tool(name, args) + output = dict(outcome.output) + digest = None + extra = "" + if name == "vision" and isinstance(output.get("png_b64"), str): + digest = vision_digest(output["png_b64"]) + extra = "\n" + digest + output = dict(output, png_b64="") + self.log("bridge", f"ok {json.dumps(output)[:500]}{extra}") + if name == "world_query": + b = output.get("beacon") + if isinstance(b, dict): + self.beacon_pos = (b["x"], b["z"]) + self.observe(output) + hint = self.state_hint() + content = json.dumps(outcome.output) + extra + if hint: + content = hint + "\n" + content + self.messages.append( + {"role": "tool", "tool_call_id": tc.id, "content": content} + ) + call_result = ToolCallResult( + name=name, args=args, ok=True, output=outcome.output, digest=digest + ) + result.calls.append(call_result) + if name == "interact" and output.get("success"): + result.interacted = True + result.message = output.get("message") + except ToolError as e: + self.log("bridge", f"error {e.code}: {e.message}") + self.messages.append( + { + "role": "tool", + "tool_call_id": tc.id, + "content": json.dumps({"error": e.code, "message": e.message}), + } + ) + result.calls.append( + ToolCallResult(name=name, args=args, ok=False, error=e.message) + ) + return result + + +# --------------------------------------------------------------------------- +# Autonomous loop (used by the demo; nudges + corrections for weak models) +# --------------------------------------------------------------------------- + + +async def run_llm_agent_loop( + controller: LLMController, + max_steps: int, + *, + log: LogFn, + recorder: Any | None = None, +) -> dict[str, Any]: + """Drive the controller until the mission is done or steps run out.""" + summary: dict[str, Any] = { + "steps": 0, + "tool_calls": 0, + "result": None, + "interacted": False, + } + dist_history: list[float] = [] + corrections = 0 + last_correct_step = -99 + last_collision_step = -99 + + def maybe_correct(step: int) -> None: + nonlocal corrections, last_correct_step + if controller.pos is None or controller.beacon_pos is None: + return + dist = math.hypot( + controller.beacon_pos[0] - controller.pos[0], + controller.beacon_pos[1] - controller.pos[1], + ) + if not math.isfinite(dist) or dist < 2.0: + return + dist_history.append(dist) + recent = dist_history[-4:] + moving_away = len(recent) == 4 and all( + recent[i] > recent[i + 1] for i in range(3) + ) + bearing = ( + math.degrees( + math.atan2( + controller.beacon_pos[0] - controller.pos[0], + controller.beacon_pos[1] - controller.pos[1], + ) + ) + - controller.yaw + ) % 360.0 + deviation = min(bearing, 360.0 - bearing) + facing_wrong = dist > 4.0 and deviation > 40.0 and step - last_correct_step > 3 + detouring = step - last_collision_step < 3 + if (moving_away or (facing_wrong and not detouring)) and corrections < 5: + corrections += 1 + last_correct_step = step + hint = controller.state_hint() + msg = ( + "You are not making progress toward the beacon. Stop and follow the CURRENT " + f"STATE note exactly: {hint} Call turn with that yaw_deg, then move forward." + ) + controller.messages.append({"role": "user", "content": msg}) + log("agent", "(correction: re-aim at the beacon)") + dist_history.clear() + + for step in range(max_steps): + summary["steps"] = step + 1 + if recorder is not None and controller.pos is not None: + await recorder.snap(controller.client, controller.pos, controller.yaw) + turn = await controller.invoke() + if turn.text: + log("agent", turn.text[:400]) + summary["tool_calls"] += len(turn.calls) + for c in turn.calls: + if ( + c.name == "move" + and c.ok + and isinstance(c.output, dict) + and c.output.get("collision") + ): + last_collision_step = step + obj = c.output.get("collision_normal") + hint_msg = ( + f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move " + "forward twice to get around it, then follow the CURRENT STATE note to re-aim." + ) + if not ( + controller.messages + and controller.messages[-1].get("content") == hint_msg + ): + controller.messages.append({"role": "user", "content": hint_msg}) + log("agent", "(collision: go around)") + maybe_correct(step) + if turn.interacted: + summary["interacted"] = True + summary["result"] = turn.message + return summary + if not turn.calls: + # Nudge a model that drifted into plain text back to acting. + silent = (not turn.text) or len(turn.text) < 8 + already_nudged = ( + bool(controller.messages) + and controller.messages[-1].get("content") == NUDGE + ) + if silent or already_nudged: + summary["result"] = "model produced no tool call" + return summary + controller.messages.append({"role": "user", "content": NUDGE}) + log("agent", "(nudge: no tool call; continue the mission)") + summary["result"] = f"exceeded {max_steps} steps" + return summary