chat: interactive mode (natural language -> tool calls), shared LLM driver refactor, run_chat.sh; verified with gemma4:e2b and gemma4:12b
This commit is contained in:
opencode
@@ -17,5 +17,10 @@
|
||||
- DONE: headless raycaster renderer (vision 160x120 + aligned depth grid) — testbed/room/render.py
|
||||
- DONE: RoomBridge with all world tools + echo/boom/bump — testbed/bridge.py
|
||||
- DONE: conformance runner — 9/9 green against testbed bridge (testbed/conformance.py)
|
||||
- DONE: commits for skeleton + bridge milestones
|
||||
- NEXT: unit tests (world/render/bridge), demo agent loop (ollama LLM + scripted fallback), README, setup.sh, BUILDLOG
|
||||
- DONE: pytest suite — 33 tests green; ruff clean
|
||||
- DONE: agent demo loop — LLM (OpenAI-compatible/ollama) + scripted fallback + auto handoff; E2E verified over WebSocket (scripted: 26 steps, beacon activated; LLM mode completed missions with gemma4:e2b)
|
||||
- DONE: testbed/README.md, scripts/setup.sh deps, scripts/run_bridge.sh, BUILDLOG.md updated
|
||||
- DONE: commits per milestone (skeleton, bridge+tools+conformance, tests+physics fixes, demo loop, docs, lint)
|
||||
- VERIFIED: conformance 9/9, pytest 33/33, ruff clean, demo end-to-end
|
||||
|
||||
COMPLETED
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[ 28151ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 35175ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 36177ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 37178ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 38180ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 39182ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 40184ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 49096ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 50098ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 51100ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 52101ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 53103ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 54105ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 55108ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 56109ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 57112ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 58115ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 59117ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 60119ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 61123ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 62129ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 63134ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 64140ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 78549ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 88888ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 89892ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
@@ -0,0 +1,97 @@
|
||||
[ 48770ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 49773ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 50775ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 54224ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 55226ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 56228ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 57230ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 58232ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 59233ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 60236ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 61239ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 62243ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 63245ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 64248ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 65251ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 66255ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 67256ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 68260ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 69261ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 70268ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 71274ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 72278ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 73283ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 74289ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 75291ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 76295ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 77301ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 78307ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 79317ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 80325ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 81336ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 82342ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 83354ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 84359ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 85363ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 86368ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 87376ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 88395ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 89412ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 90428ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 91434ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 92454ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 93462ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 94478ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 95491ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 96506ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 97520ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 98556ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 99575ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 100594ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 101605ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 102628ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 103653ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 104671ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 105683ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 106703ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 107778ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 108821ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 109857ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 110893ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 111960ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 113007ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 114057ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 115118ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 116231ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 117283ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 118336ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 119407ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 120553ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 121613ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 122714ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 123835ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 124928ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 126073ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 127177ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 128439ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 129670ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 130790ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 132026ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 133143ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 134286ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 137272ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 138376ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 139440ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 144358ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 148352ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 152146ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 156809ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 162641ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 166053ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 168606ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 171349ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 177037ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 179429ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 183147ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 188179ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
[ 192874ms] [ERROR] WebSocket connection to 'ws://127.0.0.1:8001/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED @ http://127.0.0.1:8000/:40
|
||||
@@ -0,0 +1,11 @@
|
||||
- generic [active] [ref=e1]:
|
||||
- banner [ref=e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=e3]
|
||||
- generic [ref=e4]: connected — waiting for frames
|
||||
- main [ref=e5]:
|
||||
- generic [ref=e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=e7]
|
||||
- img "vision" [ref=e8]
|
||||
- generic [ref=e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=e10]
|
||||
- img "map" [ref=e11]
|
||||
@@ -0,0 +1,11 @@
|
||||
- generic [active] [ref=e1]:
|
||||
- banner [ref=e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=e3]
|
||||
- generic [ref=e4]: connected — waiting for frames
|
||||
- main [ref=e5]:
|
||||
- generic [ref=e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=e7]
|
||||
- img "vision" [ref=e8]
|
||||
- generic [ref=e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=e10]
|
||||
- img "map" [ref=e11]
|
||||
@@ -0,0 +1,11 @@
|
||||
- generic [active] [ref=f1e1]:
|
||||
- banner [ref=f1e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=f1e3]
|
||||
- generic [ref=f1e4]: connected — waiting for frames
|
||||
- main [ref=f1e5]:
|
||||
- generic [ref=f1e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=f1e7]
|
||||
- img "vision" [ref=f1e8]
|
||||
- generic [ref=f1e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=f1e10]
|
||||
- img "map" [ref=f1e11]
|
||||
@@ -0,0 +1,11 @@
|
||||
- generic [active] [ref=f1e1]:
|
||||
- banner [ref=f1e2]:
|
||||
- heading "AICC capsule — real-time view" [level=1] [ref=f1e3]
|
||||
- generic [ref=f1e4]: tick 158 pos (11.86, 11.22) yaw 26.5° 1.43 m to beacon · 3.0 fps
|
||||
- main [ref=f1e5]:
|
||||
- generic [ref=f1e6]:
|
||||
- heading "first person (vision)" [level=2] [ref=f1e7]
|
||||
- img "vision" [ref=f1e8]
|
||||
- generic [ref=f1e9]:
|
||||
- heading "top-down (sensors)" [level=2] [ref=f1e10]
|
||||
- img "map" [ref=f1e11]
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.3 KiB |
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Interactive chat mode: bridge + live viewer + chat REPL in one command.
|
||||
# Open http://127.0.0.1:8000 to watch the capsule while you talk to it.
|
||||
# Usage: scripts/run_chat.sh [--model gemma4:e2b]
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
BRIDGE_PID=""
|
||||
LIVE_PID=""
|
||||
|
||||
# 1. bridge (reuse an already-running one)
|
||||
if python3 -c 'import socket,sys; s=socket.socket(); sys.exit(0 if s.connect_ex(("127.0.0.1",8765))==0 else 1)'; then
|
||||
echo "[chat] bridge already running on 8765"
|
||||
else
|
||||
testbed/.venv/bin/python -m testbed.bridge --port 8765 >/tmp/aicc-bridge.log 2>&1 &
|
||||
BRIDGE_PID=$!
|
||||
sleep 1.5
|
||||
echo "[chat] bridge started (pid $BRIDGE_PID)"
|
||||
fi
|
||||
|
||||
# 2. live viewer (optional but nice: watch while chatting)
|
||||
testbed/.venv/bin/python -m testbed.live --http-port 8000 --ws-port 8001 >/tmp/aicc-live.log 2>&1 &
|
||||
LIVE_PID=$!
|
||||
sleep 1.5
|
||||
echo "[chat] live view: open http://127.0.0.1:8000 in your browser"
|
||||
|
||||
# 3. chat REPL (foreground)
|
||||
testbed/.venv/bin/python -m testbed.chat "$@"
|
||||
EXIT=$?
|
||||
|
||||
kill "$LIVE_PID" 2>/dev/null || true
|
||||
if [ -n "$BRIDGE_PID" ]; then kill "$BRIDGE_PID" 2>/dev/null || true; fi
|
||||
exit $EXIT
|
||||
@@ -3,3 +3,5 @@ __pycache__/
|
||||
*.pyc
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
chat_frame.png
|
||||
chat_map.png
|
||||
|
||||
@@ -22,6 +22,8 @@ over the protocol, no engine hooks.
|
||||
| `server.py` entry | `python -m testbed.bridge` — WebSocket server. |
|
||||
| `conformance.py` | Runs the 9 core conformance scenarios against this bridge. |
|
||||
| `live.py` | Real-time browser viewer (itself an AICC client). |
|
||||
| `chat.py` | Interactive chat: natural language -> tool calls. |
|
||||
| `llm_agent.py` | Shared LLM driver (controller + autonomous loop). |
|
||||
| `demo.py` | Agent demo: LLM driver (OpenAI-compatible) or scripted. |
|
||||
| `tests/` | pytest suite (world, renderer, bridge, protocol). |
|
||||
|
||||
@@ -62,6 +64,33 @@ taken); stop the old one with `fuser -k 8765/tcp` or Ctrl-C in its terminal.
|
||||
The demo prints a full transcript of tool calls/results to stdout and saves the
|
||||
capsule's final first-person frame to `demo_final_frame.png`.
|
||||
|
||||
## Interactive chat mode
|
||||
|
||||
Talk to the capsule's brain in natural language (any language):
|
||||
|
||||
```bash
|
||||
scripts/run_chat.sh --model gemma4:e2b # small + fast
|
||||
scripts/run_chat.sh --model gemma4:12b # bigger gemma, slower (~20 s/turn)
|
||||
```
|
||||
|
||||
It starts the bridge, the live viewer and a chat REPL — open
|
||||
[http://127.0.0.1:8000](http://127.0.0.1:8000) to watch the capsule while you
|
||||
type. The model translates your words into tool calls:
|
||||
|
||||
```
|
||||
you> иди к маяку → look_at + move step by step (auto-continue)
|
||||
you> повернись налево → turn(-90)
|
||||
you> осмотрись → vision + description
|
||||
you> активируй маяк → interact (when close)
|
||||
you> /state /look /map /models /model gemma4:12b /steps N /help /exit
|
||||
```
|
||||
|
||||
Each turn's transcript is printed; the current frame lands in `chat_frame.png`
|
||||
and the sensor-built map in `chat_map.png`. Any OpenAI-compatible endpoint
|
||||
works: `python -m testbed.chat --base-url https://api.openai.com/v1
|
||||
--model gpt-4o-mini --api-key $OPENAI_API_KEY`. `--auto-steps N` controls how
|
||||
many tool steps the model may chain per request (0 = one action per turn).
|
||||
|
||||
## Real-time mode
|
||||
|
||||
Watch the capsule drive live in your browser:
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
"""Interactive chat mode: talk to the model, it drives the capsule.
|
||||
|
||||
Type natural-language commands (any language) and the LLM translates them
|
||||
into AICC tool calls: 'go to the beacon', 'turn left', 'look around',
|
||||
'activate the beacon'. Every tool call and result is printed as a transcript,
|
||||
the current vision frame is saved to ``chat_frame.png`` and the sensor-built
|
||||
top-down map to ``chat_map.png`` after each turn. Works great together with
|
||||
the live viewer (scripts/run_live.sh) — watch the capsule in your browser
|
||||
while you talk to it.
|
||||
|
||||
Commands:
|
||||
/state print current position/heading (proprioception)
|
||||
/look take a vision frame and save it
|
||||
/map save the top-down map (chat_map.png)
|
||||
/models list models on the endpoint
|
||||
/model X switch the model mid-session
|
||||
/steps N auto-continue budget for 'go to X' style requests
|
||||
/help this text
|
||||
/exit quit
|
||||
|
||||
Usage:
|
||||
python -m testbed.chat [--model gemma4:e2b] [--model gemma4:12b]
|
||||
python -m testbed.chat --base-url https://api.openai.com/v1 --model gpt-4o-mini
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from aicc.client import AICCClient
|
||||
from aicc.transport.websocket import WebSocketClientTransport
|
||||
|
||||
from testbed.llm_agent import CHAT_MISSION, LLMController, ollama_models
|
||||
|
||||
DEFAULT_URL = "ws://127.0.0.1:8765"
|
||||
DEFAULT_BASE_URL = "http://localhost:11434/v1"
|
||||
DEFAULT_MODEL = "gemma4:e2b"
|
||||
|
||||
HELP = """\
|
||||
You are talking to the capsule's brain (LLM). Type what you want, e.g.:
|
||||
'иди к маяку' / 'go to the beacon' — the model walks there step by step
|
||||
'повернись налево' / 'turn left' — quick turn
|
||||
'осмотрись' / 'look around' — vision + description
|
||||
'активируй маяк' / 'activate it' — interact (must be close)
|
||||
Commands: /state /look /map /models /model X /steps N /help /exit"""
|
||||
|
||||
|
||||
async def chat_loop(client: AICCClient, manifest, args: argparse.Namespace) -> int:
|
||||
model = args.model
|
||||
controller = LLMController(
|
||||
client,
|
||||
manifest,
|
||||
base_url=args.base_url,
|
||||
api_key=args.api_key,
|
||||
model=model,
|
||||
system_prompt=CHAT_MISSION,
|
||||
log=lambda role, msg: print(f" [{role}] {msg}"),
|
||||
)
|
||||
auto_steps = args.auto_steps
|
||||
print(f"[chat] model: {model} (endpoint {args.base_url})")
|
||||
print("[chat] type your commands; /help for the command list; /exit to quit\n")
|
||||
|
||||
async def cmd_state() -> None:
|
||||
out = (await client.call_tool("proprioception", {})).output
|
||||
print(
|
||||
f" pos ({out['position']['x']:.2f}, {out['position']['z']:.2f}) "
|
||||
f"heading {out['rotation']['yaw_deg']:.1f} deg health {out['health']}"
|
||||
)
|
||||
|
||||
async def cmd_look() -> None:
|
||||
out = (await client.call_tool("vision", {})).output
|
||||
import base64
|
||||
|
||||
raw = base64.b64decode(out["png_b64"])
|
||||
|
||||
def _write() -> None:
|
||||
with open("chat_frame.png", "wb") as fh:
|
||||
fh.write(raw)
|
||||
|
||||
await asyncio.to_thread(_write)
|
||||
print(f" frame saved to chat_frame.png ({out['width']}x{out['height']})")
|
||||
|
||||
def save_map() -> None:
|
||||
if controller.pos is None:
|
||||
print(" (no position yet — ask the capsule to move first)")
|
||||
return
|
||||
from testbed.room.mapview import draw_sensor_map
|
||||
|
||||
layout = {"obstacles": [], "beacon": {"x": 12.5, "z": 12.5}}
|
||||
if controller.beacon_pos is not None:
|
||||
layout["beacon"] = {
|
||||
"x": controller.beacon_pos[0],
|
||||
"z": controller.beacon_pos[1],
|
||||
}
|
||||
img = draw_sensor_map(layout, controller.pos, controller.yaw, controller.path)
|
||||
img.save("chat_map.png")
|
||||
print(
|
||||
f" map saved to chat_map.png (pos {controller.pos[0]:.1f}, {controller.pos[1]:.1f})"
|
||||
)
|
||||
|
||||
async def cmd_models() -> None:
|
||||
print(" fetching models…")
|
||||
for m in ollama_models(args.base_url, args.api_key):
|
||||
print(f" {m}")
|
||||
|
||||
while True:
|
||||
try:
|
||||
text = input("you> ").strip()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
print("\n[chat] bye")
|
||||
return 0
|
||||
if not text:
|
||||
continue
|
||||
low = text.lower()
|
||||
if low in ("/exit", "exit", "quit", "выход"):
|
||||
print("[chat] bye")
|
||||
return 0
|
||||
if low == "/help":
|
||||
print(HELP)
|
||||
continue
|
||||
if low == "/state":
|
||||
await cmd_state()
|
||||
continue
|
||||
if low == "/look":
|
||||
await cmd_look()
|
||||
continue
|
||||
if low == "/map":
|
||||
save_map()
|
||||
continue
|
||||
if low == "/models":
|
||||
await cmd_models()
|
||||
continue
|
||||
if low.startswith("/model "):
|
||||
model = low.split(maxsplit=1)[1]
|
||||
controller.model = model
|
||||
print(f"[chat] switching to {model}")
|
||||
continue
|
||||
if low.startswith("/steps "):
|
||||
try:
|
||||
auto_steps = max(0, int(low.split(maxsplit=1)[1]))
|
||||
print(f"[chat] auto-continue budget: {auto_steps} steps")
|
||||
except ValueError:
|
||||
print(" usage: /steps N")
|
||||
continue
|
||||
|
||||
controller.messages.append({"role": "user", "content": text})
|
||||
steps_used = 0
|
||||
while True:
|
||||
try:
|
||||
turn = await controller.invoke()
|
||||
except RuntimeError as exc:
|
||||
print(f" [error] {exc}")
|
||||
break
|
||||
steps_used += 1
|
||||
if turn.text:
|
||||
print(f" model> {turn.text}")
|
||||
if turn.interacted:
|
||||
print(f"\n[chat] BEACON ACTIVATED: {turn.message}")
|
||||
save_map()
|
||||
return 0
|
||||
if not turn.calls:
|
||||
if not turn.text and auto_steps and steps_used <= auto_steps:
|
||||
# The model answered with nothing: nudge it to actually act.
|
||||
controller.messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Your last turn contained no action and no answer. "
|
||||
"Carry out the user's request: call a tool now "
|
||||
"(check the CURRENT STATE note)."
|
||||
),
|
||||
}
|
||||
)
|
||||
print(" [chat] (nudge: model gave an empty turn)")
|
||||
continue
|
||||
break # the model answered in words; wait for the user
|
||||
if auto_steps and steps_used >= auto_steps:
|
||||
print(
|
||||
f" [chat] (auto-continue budget of {auto_steps} reached — say 'continue' to keep going)"
|
||||
)
|
||||
break
|
||||
if not auto_steps:
|
||||
break
|
||||
# The model acted without commenting; let it keep going for 'go to X'
|
||||
if controller.pos is not None:
|
||||
save_map()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
transport = WebSocketClientTransport(args.url)
|
||||
async with AICCClient(transport) as client:
|
||||
manifest = await client.handshake()
|
||||
print(f"[handshake] session {manifest.session_id} world {manifest.world.name}")
|
||||
print(f"[handshake] tools: {[t.id for t in manifest.tools]}")
|
||||
return await chat_loop(client, manifest, args)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Interactive chat: talk to the LLM, it drives the capsule."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
default=DEFAULT_URL,
|
||||
help=f"bridge WebSocket URL (default {DEFAULT_URL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url", default=DEFAULT_BASE_URL, help="OpenAI-compatible endpoint"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model", default=DEFAULT_MODEL, help="model id on the endpoint"
|
||||
)
|
||||
parser.add_argument("--api-key", default="ollama", help="API key for the endpoint")
|
||||
parser.add_argument(
|
||||
"--auto-steps",
|
||||
type=int,
|
||||
default=8,
|
||||
help="how many tool steps the model may chain per request (0 = one action per turn)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except KeyboardInterrupt:
|
||||
print("\n[chat] bye")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+17
-295
@@ -33,8 +33,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from aicc.client import AICCClient
|
||||
from aicc.errors import ToolError
|
||||
from aicc.protocol import SessionInit, ToolSpec
|
||||
from aicc.protocol import SessionInit
|
||||
from aicc.transport.websocket import WebSocketClientTransport
|
||||
from PIL import Image
|
||||
|
||||
@@ -42,51 +41,6 @@ DEFAULT_URL = "ws://127.0.0.1:8765"
|
||||
DEFAULT_BASE_URL = "http://localhost:11434/v1"
|
||||
DEFAULT_MODEL = "gemma4:e2b"
|
||||
|
||||
NUDGE = (
|
||||
"You have not called a tool. Continue the mission: check the CURRENT STATE note, "
|
||||
"then call a tool (turn or move, or interact if within 1.6 m)."
|
||||
)
|
||||
|
||||
MISSION = (
|
||||
"You are a capsule AI inside a 16x16 m room. Your mission: reach the glowing "
|
||||
"beacon pillar and activate it with the interact tool. "
|
||||
"World: yaw 0 faces +z, 90 faces +x, coordinates in meters; three wooden "
|
||||
"crates block parts of the room. "
|
||||
"Perception: proprioception gives position/heading; world_query gives the room "
|
||||
"layout and the beacon's exact coordinates (call it once at the start); "
|
||||
"vision gives the first-person frame; hear tells you how close you are to the beacon. "
|
||||
"Every tool result is prefixed with a CURRENT STATE note: read it — it tells you "
|
||||
"your position, heading, distance to the beacon, and the bearing to turn toward it. "
|
||||
"Standard loop, one action per turn: "
|
||||
"1. call proprioception (and hear every few steps); "
|
||||
"2. compute the bearing to the beacon and turn toward it (turn yaw_deg = that bearing, max 90 per call); "
|
||||
"3. move forward 1.0 m; if the result says collision, turn 60-90 degrees and go around, "
|
||||
"then re-aim. "
|
||||
"Never walk into a crate or wall twice in a row. "
|
||||
"Call interact ONLY when proprioception puts you within 1.6 m of the beacon. "
|
||||
"Keep tool calls short and decisive. Stop once the beacon is active."
|
||||
)
|
||||
|
||||
COLOR_NAMES = [
|
||||
("beacon_cyan", (120, 210, 235)),
|
||||
("beacon_yellow", (255, 190, 90)),
|
||||
("crate_red", (178, 64, 54)),
|
||||
("crate_blue", (64, 96, 178)),
|
||||
("crate_olive", (128, 128, 60)),
|
||||
("floor_gray", (60, 60, 66)),
|
||||
("wall_gray", (96, 96, 106)),
|
||||
("dark", (30, 30, 36)),
|
||||
]
|
||||
|
||||
|
||||
def _nearest_color(r: float, g: float, b: float) -> str:
|
||||
best, best_d = "dark", 1e9
|
||||
for name, (cr, cg, cb) in COLOR_NAMES:
|
||||
d = (r - cr) ** 2 + (g - cg) ** 2 + (b - cb) ** 2
|
||||
if d < best_d:
|
||||
best, best_d = name, d
|
||||
return best
|
||||
|
||||
|
||||
class FrameRecorder:
|
||||
"""Visual mode: saves first-person frames and a sensor-built top-down map
|
||||
@@ -171,63 +125,11 @@ class FrameRecorder:
|
||||
return summary
|
||||
|
||||
|
||||
def vision_digest(png_b64: str, cols: int = 12, rows: int = 9) -> str:
|
||||
"""Coarse color-grid summary of a vision frame (agent-side preprocessing)."""
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
img = Image.open(io.BytesIO(base64.b64decode(png_b64))).convert("RGB")
|
||||
except Exception: # noqa: BLE001 - digest is best-effort
|
||||
return "(no digest: unreadable frame)"
|
||||
small = img.resize((cols, rows))
|
||||
grid: list[list[tuple[int, int, int]]] = []
|
||||
for r in range(rows):
|
||||
row: list[tuple[int, int, int]] = []
|
||||
for c in range(cols):
|
||||
px = small.getpixel((c, r))
|
||||
if isinstance(px, (tuple, list)) and len(px) >= 3:
|
||||
row.append((int(px[0]), int(px[1]), int(px[2])))
|
||||
elif isinstance(px, (int, float)):
|
||||
v = int(px)
|
||||
row.append((v, v, v))
|
||||
else:
|
||||
row.append((0, 0, 0))
|
||||
grid.append(row)
|
||||
lines = []
|
||||
for r in range(rows):
|
||||
lines.append(" ".join(_nearest_color(*grid[r][c]) for c in range(cols)))
|
||||
return f"frame digest ({cols}x{rows}, center = where you face):\n" + "\n".join(
|
||||
lines
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM agent (OpenAI-compatible)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def to_openai_tools(tools: list[ToolSpec]) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for t in tools:
|
||||
if t.id in (
|
||||
"echo",
|
||||
"boom",
|
||||
"bump",
|
||||
): # conformance-only tools: not for the agent
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": t.id,
|
||||
"description": t.description,
|
||||
"parameters": t.input_schema,
|
||||
},
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def run_llm_agent(
|
||||
client: AICCClient,
|
||||
manifest: SessionInit,
|
||||
@@ -238,205 +140,25 @@ async def run_llm_agent(
|
||||
max_steps: int,
|
||||
recorder: FrameRecorder | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from openai import AsyncOpenAI
|
||||
"""Autonomous LLM run: controller + nudge/correct loop (see llm_agent)."""
|
||||
from testbed.llm_agent import LLMController, run_llm_agent_loop
|
||||
|
||||
ac = AsyncOpenAI(base_url=base_url, api_key=api_key)
|
||||
tools = to_openai_tools(manifest.tools)
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": MISSION},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Begin your mission. Use the tools to reach and activate the beacon.",
|
||||
},
|
||||
]
|
||||
summary: dict[str, Any] = {
|
||||
"steps": 0,
|
||||
"tool_calls": 0,
|
||||
"result": None,
|
||||
"interacted": False,
|
||||
}
|
||||
controller = LLMController(
|
||||
client,
|
||||
manifest,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
log=lambda role, msg: print(f"[{role}] {msg}"),
|
||||
)
|
||||
try:
|
||||
return await run_llm_agent_loop(controller, max_steps, log=print_log, recorder=recorder)
|
||||
except RuntimeError as exc:
|
||||
return {"steps": 0, "tool_calls": 0, "result": str(exc), "interacted": False}
|
||||
|
||||
# Agent-side working memory: a compact summary of state the model has
|
||||
# already observed via sensors, prefixed to tool results so small models
|
||||
# don't drift (protocol §9: agent memory is the agent's concern).
|
||||
beacon_pos: tuple[float, float] | None = None
|
||||
pos: tuple[float, float] | None = None
|
||||
yaw: float = 0.0
|
||||
|
||||
def state_hint() -> str:
|
||||
if pos is None:
|
||||
return ""
|
||||
if beacon_pos is not None:
|
||||
dist = math.hypot(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1])
|
||||
bearing = (
|
||||
math.degrees(math.atan2(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1]))
|
||||
- yaw
|
||||
) % 360.0
|
||||
return (
|
||||
f"CURRENT STATE: you are at ({pos[0]:.2f}, {pos[1]:.2f}), heading {yaw:.1f} deg; "
|
||||
f"the beacon is {dist:.2f} m away at relative bearing {bearing:.0f} deg. "
|
||||
f"Action: turn yaw_deg={bearing:.0f} to face it, then move forward."
|
||||
)
|
||||
return f"CURRENT STATE: you are at ({pos[0]:.2f}, {pos[1]:.2f}), heading {yaw:.1f} deg."
|
||||
|
||||
def observe(output: dict[str, Any]) -> None:
|
||||
nonlocal pos, yaw
|
||||
if isinstance(output.get("position"), dict):
|
||||
pos = (output["position"]["x"], output["position"]["z"])
|
||||
if isinstance(output.get("rotation"), dict):
|
||||
yaw = output["rotation"]["yaw_deg"] % 360.0
|
||||
|
||||
def log(role: str, msg: str) -> None:
|
||||
print(f"[{role}] {msg}")
|
||||
|
||||
# Progress monitor: if the capsule keeps moving away from the beacon,
|
||||
# reflect that back to the model as a correction (framework-style loop).
|
||||
dist_history: list[float] = []
|
||||
corrections = 0
|
||||
last_correct_step = -99
|
||||
last_collision_step = -99
|
||||
|
||||
def distance_to_beacon() -> float:
|
||||
if pos is None or beacon_pos is None:
|
||||
return float("nan")
|
||||
return math.hypot(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1])
|
||||
|
||||
def maybe_correct(step: int) -> None:
|
||||
nonlocal corrections, last_correct_step
|
||||
if pos is None or beacon_pos is None:
|
||||
return
|
||||
dist = distance_to_beacon()
|
||||
if not math.isfinite(dist) or dist < 2.0:
|
||||
return
|
||||
dist_history.append(dist)
|
||||
recent = dist_history[-4:]
|
||||
moving_away = len(recent) == 4 and all(
|
||||
recent[i] > recent[i + 1] for i in range(3)
|
||||
)
|
||||
bearing = (
|
||||
math.degrees(math.atan2(beacon_pos[0] - pos[0], beacon_pos[1] - pos[1]))
|
||||
- yaw
|
||||
) % 360.0
|
||||
deviation = min(bearing, 360.0 - bearing)
|
||||
facing_wrong = dist > 4.0 and deviation > 40.0 and step - last_correct_step > 3
|
||||
detouring = step - last_collision_step < 3
|
||||
if (moving_away or (facing_wrong and not detouring)) and corrections < 5:
|
||||
corrections += 1
|
||||
last_correct_step = step
|
||||
hint = state_hint()
|
||||
msg = (
|
||||
"You are not making progress toward the beacon. Stop and follow the CURRENT "
|
||||
f"STATE note exactly: {hint} Call turn with that yaw_deg, then move forward."
|
||||
)
|
||||
messages.append({"role": "user", "content": msg})
|
||||
log("agent", "(correction: re-aim at the beacon)")
|
||||
dist_history.clear()
|
||||
|
||||
for step in range(max_steps):
|
||||
summary["steps"] = step + 1
|
||||
if recorder is not None and pos is not None:
|
||||
await recorder.snap(client, pos, yaw)
|
||||
try:
|
||||
resp = await ac.chat.completions.create(
|
||||
model=model, messages=messages, tools=tools, tool_choice="auto"
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - provider errors end the demo
|
||||
log("agent", f"LLM error: {exc}")
|
||||
summary["result"] = f"LLM error: {exc}"
|
||||
return summary
|
||||
|
||||
choice = resp.choices[0]
|
||||
if choice.message.content:
|
||||
log("agent", choice.message.content[:400])
|
||||
messages.append({"role": "assistant", "content": choice.message.content})
|
||||
calls = choice.message.tool_calls
|
||||
if not calls:
|
||||
# Nudge a model that drifted into plain text back to acting.
|
||||
silent = (not choice.message.content) or len(choice.message.content) < 8
|
||||
already_nudged = bool(messages) and messages[-1].get("content") == NUDGE
|
||||
if silent or already_nudged:
|
||||
summary["result"] = "model produced no tool call"
|
||||
return summary
|
||||
messages.append({"role": "user", "content": NUDGE})
|
||||
log("agent", "(nudge: no tool call; continue the mission)")
|
||||
continue
|
||||
messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": choice.message.content or "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": c.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": c.function.name,
|
||||
"arguments": c.function.arguments,
|
||||
},
|
||||
}
|
||||
for c in calls
|
||||
],
|
||||
}
|
||||
)
|
||||
for tc in calls:
|
||||
name = tc.function.name
|
||||
try:
|
||||
args = json.loads(tc.function.arguments or "{}")
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
log("tool", f"{name}({json.dumps(args)})")
|
||||
try:
|
||||
result = await client.call_tool(name, args)
|
||||
output = result.output
|
||||
extra = ""
|
||||
if name == "vision" and isinstance(output.get("png_b64"), str):
|
||||
extra = "\n" + vision_digest(output["png_b64"])
|
||||
output = dict(output, png_b64="<binary, decoded for digest>")
|
||||
log("bridge", f"ok {json.dumps(output)[:500]}{extra}")
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": json.dumps(result.output) + extra,
|
||||
}
|
||||
)
|
||||
summary["tool_calls"] += 1
|
||||
if name == "world_query":
|
||||
b = output.get("beacon")
|
||||
if isinstance(b, dict):
|
||||
beacon_pos = (b["x"], b["z"])
|
||||
observe(output)
|
||||
hint = state_hint()
|
||||
if hint:
|
||||
messages[-1] = dict(
|
||||
messages[-1], content=hint + "\n" + messages[-1]["content"]
|
||||
)
|
||||
maybe_correct(step)
|
||||
if name == "move" and output.get("collision"):
|
||||
last_collision_step = step
|
||||
obj = output.get("collision_normal")
|
||||
hint_msg = (
|
||||
f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
|
||||
"forward twice to get around it, then follow the CURRENT STATE note to re-aim."
|
||||
)
|
||||
if not (messages and messages[-1].get("content") == hint_msg):
|
||||
messages.append({"role": "user", "content": hint_msg})
|
||||
log("agent", "(collision: go around)")
|
||||
if name == "interact" and output.get("success"):
|
||||
summary["interacted"] = True
|
||||
summary["result"] = output.get("message", "beacon activated")
|
||||
return summary
|
||||
except ToolError as e:
|
||||
log("bridge", f"error {e.code}: {e.message}")
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": json.dumps({"error": e.code, "message": e.message}),
|
||||
}
|
||||
)
|
||||
summary["tool_calls"] += 1
|
||||
summary["result"] = f"exceeded {max_steps} steps"
|
||||
return summary
|
||||
def print_log(role: str, msg: str) -> None:
|
||||
print(f"[{role}] {msg}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
"""Shared LLM tool-use machinery for the demo and chat modes.
|
||||
|
||||
A tool-calling LLM (any OpenAI-compatible endpoint) drives the capsule by
|
||||
translating intent into AICC tool calls. The controller keeps agent-side
|
||||
working memory (protocol §9): a CURRENT STATE note derived from what the model
|
||||
has already observed, vision digest for text-only models, and a transcript of
|
||||
tool calls/results fed back as `tool` messages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from aicc.client import AICCClient
|
||||
from aicc.errors import ToolError
|
||||
from aicc.protocol import SessionInit, ToolSpec
|
||||
|
||||
MISSION = (
|
||||
"You are a capsule AI inside a 16x16 m room. Your mission: reach the glowing "
|
||||
"beacon pillar and activate it with the interact tool. "
|
||||
"World: yaw 0 faces +z, 90 faces +x, coordinates in meters; three wooden "
|
||||
"crates block parts of the room. "
|
||||
"Perception: proprioception gives position/heading; world_query gives the room "
|
||||
"layout and the beacon's exact coordinates (call it once at the start); "
|
||||
"vision gives the first-person frame; hear tells you how close you are to the beacon. "
|
||||
"Every tool result is prefixed with a CURRENT STATE note: read it — it tells you "
|
||||
"your position, heading, distance to the beacon, and the bearing to turn toward it. "
|
||||
"Standard loop, one action per turn: "
|
||||
"1. call proprioception (and hear every few steps); "
|
||||
"2. compute the bearing to the beacon and turn toward it (turn yaw_deg = that bearing, max 90 per call); "
|
||||
"3. move forward 1.0 m; if the result says collision, turn 60-90 degrees and go around, "
|
||||
"then re-aim. "
|
||||
"Never walk into a crate or wall twice in a row. "
|
||||
"Call interact ONLY when proprioception puts you within 1.6 m of the beacon. "
|
||||
"Keep tool calls short and decisive. Stop once the beacon is active."
|
||||
)
|
||||
|
||||
CHAT_MISSION = (
|
||||
"You are the brain of a capsule robot inside a 16x16 m room with a glowing "
|
||||
"beacon at the far corner and a few wooden crates. The user talks to you in "
|
||||
"natural language (any language) and you translate their intent into tool "
|
||||
"calls. "
|
||||
"World: yaw 0 faces +z, 90 faces +x, coordinates in meters. "
|
||||
"Every tool result is prefixed with a CURRENT STATE note: position, heading, "
|
||||
"distance to the beacon and the bearing to turn toward it. "
|
||||
"Rules: "
|
||||
"1. Take the user's request literally: 'go to the beacon' means turn toward "
|
||||
"it and move step by step until close; 'turn left/right' means turn ~90 deg; "
|
||||
"'look around' means vision and describe what you see; 'activate' means "
|
||||
"interact (only within 1.6 m). "
|
||||
"2. One tool call per turn is fine — you will get another turn to continue. "
|
||||
"3. If a move reports a collision, turn and go around. "
|
||||
"4. When the user asks something not involving movement, just answer briefly. "
|
||||
"5. Keep the user informed with one short line after acting (or if you need "
|
||||
"a clarification). "
|
||||
"Call interact only when within 1.6 m of the beacon."
|
||||
)
|
||||
|
||||
NUDGE = (
|
||||
"You have not called a tool. Continue the mission: check the CURRENT STATE note, "
|
||||
"then call a tool (turn or move, or interact if within 1.6 m)."
|
||||
)
|
||||
|
||||
COLOR_NAMES = [
|
||||
("beacon_cyan", (120, 210, 235)),
|
||||
("beacon_yellow", (255, 190, 90)),
|
||||
("crate_red", (178, 64, 54)),
|
||||
("crate_blue", (64, 96, 178)),
|
||||
("crate_olive", (128, 128, 60)),
|
||||
("floor_gray", (60, 60, 66)),
|
||||
("wall_gray", (96, 96, 106)),
|
||||
("dark", (30, 30, 36)),
|
||||
]
|
||||
|
||||
LogFn = Callable[[str, str], None]
|
||||
|
||||
|
||||
def _nearest_color(r: float, g: float, b: float) -> str:
|
||||
best, best_d = "dark", 1e9
|
||||
for name, (cr, cg, cb) in COLOR_NAMES:
|
||||
d = (r - cr) ** 2 + (g - cg) ** 2 + (b - cb) ** 2
|
||||
if d < best_d:
|
||||
best, best_d = name, d
|
||||
return best
|
||||
|
||||
|
||||
def vision_digest(png_b64: str, cols: int = 12, rows: int = 9) -> str:
|
||||
"""Coarse color-grid summary of a vision frame (agent-side preprocessing)."""
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
img = Image.open(io.BytesIO(base64.b64decode(png_b64))).convert("RGB")
|
||||
except Exception: # noqa: BLE001 - digest is best-effort
|
||||
return "(no digest: unreadable frame)"
|
||||
small = img.resize((cols, rows))
|
||||
grid: list[list[tuple[int, int, int]]] = []
|
||||
for r in range(rows):
|
||||
row: list[tuple[int, int, int]] = []
|
||||
for c in range(cols):
|
||||
px = small.getpixel((c, r))
|
||||
if isinstance(px, (tuple, list)) and len(px) >= 3:
|
||||
row.append((int(px[0]), int(px[1]), int(px[2])))
|
||||
elif isinstance(px, (int, float)):
|
||||
v = int(px)
|
||||
row.append((v, v, v))
|
||||
else:
|
||||
row.append((0, 0, 0))
|
||||
grid.append(row)
|
||||
lines = [
|
||||
" ".join(_nearest_color(*grid[r][c]) for c in range(cols)) for r in range(rows)
|
||||
]
|
||||
return f"frame digest ({cols}x{rows}, center = where you face):\n" + "\n".join(
|
||||
lines
|
||||
)
|
||||
|
||||
|
||||
def to_openai_tools(tools: list[ToolSpec]) -> list[dict[str, Any]]:
|
||||
"""Convert manifest ToolSpecs to OpenAI function schemas (skip conformance tools)."""
|
||||
out = []
|
||||
for t in tools:
|
||||
if t.id in ("echo", "boom", "bump"):
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": t.id,
|
||||
"description": t.description,
|
||||
"parameters": t.input_schema,
|
||||
},
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def ollama_models(base_url: str, api_key: str) -> list[str]:
|
||||
"""List model ids on an OpenAI-compatible endpoint (best effort)."""
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
ids: list[str] = []
|
||||
|
||||
async def _fetch() -> None:
|
||||
ac = AsyncOpenAI(base_url=base_url, api_key=api_key)
|
||||
try:
|
||||
models = await ac.models.list()
|
||||
except Exception: # noqa: BLE001
|
||||
return
|
||||
ids.extend(m.id for m in models.data if m.id)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(_fetch())
|
||||
return ids
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Controller: one conversation with the tool-calling LLM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallResult:
|
||||
name: str
|
||||
args: dict[str, Any]
|
||||
ok: bool
|
||||
output: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
digest: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurnResult:
|
||||
"""One model round-trip: optional text plus the executed tool calls."""
|
||||
|
||||
text: str
|
||||
calls: list[ToolCallResult] = field(default_factory=list)
|
||||
interacted: bool = False
|
||||
message: str | None = None
|
||||
|
||||
|
||||
class LLMController:
|
||||
"""Drives a tool-calling LLM over an OpenAI-compatible endpoint.
|
||||
|
||||
Holds the message history and the agent-side state (position, heading,
|
||||
beacon, path) distilled from tool results. ``invoke()`` runs one model
|
||||
turn and executes whatever tools it asks for.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: AICCClient,
|
||||
manifest: SessionInit,
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
system_prompt: str = MISSION,
|
||||
log: LogFn | None = None,
|
||||
):
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
self.client = client
|
||||
self.model = model
|
||||
self.ac = AsyncOpenAI(base_url=base_url, api_key=api_key)
|
||||
self.tools = to_openai_tools(manifest.tools)
|
||||
self.messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Begin. Use the tools to control the capsule.",
|
||||
},
|
||||
]
|
||||
self.log = log or (lambda role, msg: None)
|
||||
self.beacon_pos: tuple[float, float] | None = None
|
||||
self.pos: tuple[float, float] | None = None
|
||||
self.yaw: float = 0.0
|
||||
self.path: list[tuple[float, float]] = []
|
||||
|
||||
# -- agent-side working memory -----------------------------------------
|
||||
|
||||
def state_hint(self) -> str:
|
||||
if self.pos is None:
|
||||
return ""
|
||||
if self.beacon_pos is not None:
|
||||
dist = math.hypot(
|
||||
self.beacon_pos[0] - self.pos[0], self.beacon_pos[1] - self.pos[1]
|
||||
)
|
||||
bearing = (
|
||||
math.degrees(
|
||||
math.atan2(
|
||||
self.beacon_pos[0] - self.pos[0],
|
||||
self.beacon_pos[1] - self.pos[1],
|
||||
)
|
||||
)
|
||||
- self.yaw
|
||||
) % 360.0
|
||||
return (
|
||||
f"CURRENT STATE: you are at ({self.pos[0]:.2f}, {self.pos[1]:.2f}), "
|
||||
f"heading {self.yaw:.1f} deg; the beacon is {dist:.2f} m away at relative "
|
||||
f"bearing {bearing:.0f} deg. Action: turn yaw_deg={bearing:.0f} to face it, "
|
||||
f"then move forward."
|
||||
)
|
||||
return f"CURRENT STATE: you are at ({self.pos[0]:.2f}, {self.pos[1]:.2f}), heading {self.yaw:.1f} deg."
|
||||
|
||||
def observe(self, output: dict[str, Any]) -> None:
|
||||
if isinstance(output.get("position"), dict):
|
||||
self.pos = (output["position"]["x"], output["position"]["z"])
|
||||
if isinstance(output.get("rotation"), dict):
|
||||
self.yaw = output["rotation"]["yaw_deg"] % 360.0
|
||||
if self.pos is not None and (not self.path or self.path[-1] != self.pos):
|
||||
self.path.append(self.pos)
|
||||
if len(self.path) > 4000:
|
||||
self.path = self.path[-2000:]
|
||||
|
||||
# -- conversation -------------------------------------------------------
|
||||
|
||||
async def send_user(self, text: str) -> None:
|
||||
self.messages.append({"role": "user", "content": text})
|
||||
|
||||
async def invoke(self) -> TurnResult:
|
||||
"""One model round-trip: get the response and execute its tool calls."""
|
||||
try:
|
||||
resp = await self.ac.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=self.messages,
|
||||
tools=self.tools,
|
||||
tool_choice="auto",
|
||||
)
|
||||
except Exception as exc:
|
||||
self.log("agent", f"LLM error: {exc}")
|
||||
raise RuntimeError(f"LLM error: {exc}") from exc
|
||||
|
||||
choice = resp.choices[0]
|
||||
text = choice.message.content or ""
|
||||
result = TurnResult(text=text)
|
||||
calls = choice.message.tool_calls
|
||||
if not calls:
|
||||
self.messages.append({"role": "assistant", "content": text})
|
||||
return result
|
||||
|
||||
self.messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": text,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": c.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": c.function.name,
|
||||
"arguments": c.function.arguments,
|
||||
},
|
||||
}
|
||||
for c in calls
|
||||
],
|
||||
}
|
||||
)
|
||||
for tc in calls:
|
||||
name = tc.function.name
|
||||
try:
|
||||
args = json.loads(tc.function.arguments or "{}")
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
self.log("tool", f"{name}({json.dumps(args)})")
|
||||
try:
|
||||
outcome = await self.client.call_tool(name, args)
|
||||
output = dict(outcome.output)
|
||||
digest = None
|
||||
extra = ""
|
||||
if name == "vision" and isinstance(output.get("png_b64"), str):
|
||||
digest = vision_digest(output["png_b64"])
|
||||
extra = "\n" + digest
|
||||
output = dict(output, png_b64="<binary, decoded for digest>")
|
||||
self.log("bridge", f"ok {json.dumps(output)[:500]}{extra}")
|
||||
if name == "world_query":
|
||||
b = output.get("beacon")
|
||||
if isinstance(b, dict):
|
||||
self.beacon_pos = (b["x"], b["z"])
|
||||
self.observe(output)
|
||||
hint = self.state_hint()
|
||||
content = json.dumps(outcome.output) + extra
|
||||
if hint:
|
||||
content = hint + "\n" + content
|
||||
self.messages.append(
|
||||
{"role": "tool", "tool_call_id": tc.id, "content": content}
|
||||
)
|
||||
call_result = ToolCallResult(
|
||||
name=name, args=args, ok=True, output=outcome.output, digest=digest
|
||||
)
|
||||
result.calls.append(call_result)
|
||||
if name == "interact" and output.get("success"):
|
||||
result.interacted = True
|
||||
result.message = output.get("message")
|
||||
except ToolError as e:
|
||||
self.log("bridge", f"error {e.code}: {e.message}")
|
||||
self.messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": json.dumps({"error": e.code, "message": e.message}),
|
||||
}
|
||||
)
|
||||
result.calls.append(
|
||||
ToolCallResult(name=name, args=args, ok=False, error=e.message)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Autonomous loop (used by the demo; nudges + corrections for weak models)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_llm_agent_loop(
|
||||
controller: LLMController,
|
||||
max_steps: int,
|
||||
*,
|
||||
log: LogFn,
|
||||
recorder: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Drive the controller until the mission is done or steps run out."""
|
||||
summary: dict[str, Any] = {
|
||||
"steps": 0,
|
||||
"tool_calls": 0,
|
||||
"result": None,
|
||||
"interacted": False,
|
||||
}
|
||||
dist_history: list[float] = []
|
||||
corrections = 0
|
||||
last_correct_step = -99
|
||||
last_collision_step = -99
|
||||
|
||||
def maybe_correct(step: int) -> None:
|
||||
nonlocal corrections, last_correct_step
|
||||
if controller.pos is None or controller.beacon_pos is None:
|
||||
return
|
||||
dist = math.hypot(
|
||||
controller.beacon_pos[0] - controller.pos[0],
|
||||
controller.beacon_pos[1] - controller.pos[1],
|
||||
)
|
||||
if not math.isfinite(dist) or dist < 2.0:
|
||||
return
|
||||
dist_history.append(dist)
|
||||
recent = dist_history[-4:]
|
||||
moving_away = len(recent) == 4 and all(
|
||||
recent[i] > recent[i + 1] for i in range(3)
|
||||
)
|
||||
bearing = (
|
||||
math.degrees(
|
||||
math.atan2(
|
||||
controller.beacon_pos[0] - controller.pos[0],
|
||||
controller.beacon_pos[1] - controller.pos[1],
|
||||
)
|
||||
)
|
||||
- controller.yaw
|
||||
) % 360.0
|
||||
deviation = min(bearing, 360.0 - bearing)
|
||||
facing_wrong = dist > 4.0 and deviation > 40.0 and step - last_correct_step > 3
|
||||
detouring = step - last_collision_step < 3
|
||||
if (moving_away or (facing_wrong and not detouring)) and corrections < 5:
|
||||
corrections += 1
|
||||
last_correct_step = step
|
||||
hint = controller.state_hint()
|
||||
msg = (
|
||||
"You are not making progress toward the beacon. Stop and follow the CURRENT "
|
||||
f"STATE note exactly: {hint} Call turn with that yaw_deg, then move forward."
|
||||
)
|
||||
controller.messages.append({"role": "user", "content": msg})
|
||||
log("agent", "(correction: re-aim at the beacon)")
|
||||
dist_history.clear()
|
||||
|
||||
for step in range(max_steps):
|
||||
summary["steps"] = step + 1
|
||||
if recorder is not None and controller.pos is not None:
|
||||
await recorder.snap(controller.client, controller.pos, controller.yaw)
|
||||
turn = await controller.invoke()
|
||||
if turn.text:
|
||||
log("agent", turn.text[:400])
|
||||
summary["tool_calls"] += len(turn.calls)
|
||||
for c in turn.calls:
|
||||
if (
|
||||
c.name == "move"
|
||||
and c.ok
|
||||
and isinstance(c.output, dict)
|
||||
and c.output.get("collision")
|
||||
):
|
||||
last_collision_step = step
|
||||
obj = c.output.get("collision_normal")
|
||||
hint_msg = (
|
||||
f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
|
||||
"forward twice to get around it, then follow the CURRENT STATE note to re-aim."
|
||||
)
|
||||
if not (
|
||||
controller.messages
|
||||
and controller.messages[-1].get("content") == hint_msg
|
||||
):
|
||||
controller.messages.append({"role": "user", "content": hint_msg})
|
||||
log("agent", "(collision: go around)")
|
||||
maybe_correct(step)
|
||||
if turn.interacted:
|
||||
summary["interacted"] = True
|
||||
summary["result"] = turn.message
|
||||
return summary
|
||||
if not turn.calls:
|
||||
# Nudge a model that drifted into plain text back to acting.
|
||||
silent = (not turn.text) or len(turn.text) < 8
|
||||
already_nudged = (
|
||||
bool(controller.messages)
|
||||
and controller.messages[-1].get("content") == NUDGE
|
||||
)
|
||||
if silent or already_nudged:
|
||||
summary["result"] = "model produced no tool call"
|
||||
return summary
|
||||
controller.messages.append({"role": "user", "content": NUDGE})
|
||||
log("agent", "(nudge: no tool call; continue the mission)")
|
||||
summary["result"] = f"exceeded {max_steps} steps"
|
||||
return summary
|
||||
Reference in New Issue
Block a user