record: observer-based capture (--capture-hz) + --goal beacon; smooth_mission.gif artifact (136 frames, gliding motion verified); README section
This commit is contained in:
opencode
+115
-57
@@ -139,24 +139,22 @@ async def run(args: argparse.Namespace) -> int:
|
||||
api_key=args.api_key,
|
||||
model=args.model,
|
||||
)
|
||||
from testbed.llm_agent import MISSION
|
||||
|
||||
bridge = build_bridge()
|
||||
# Fixed marker so the GIF tells a clear story (crate_blue, east face).
|
||||
bridge.world.marker = {
|
||||
"box": "crate_blue",
|
||||
"face": "x1",
|
||||
"u": 0.5,
|
||||
"v": 0.55,
|
||||
"size": 0.42,
|
||||
"color": (255, 150, 40),
|
||||
}
|
||||
m = bridge.world.marker_world_pos()
|
||||
print(
|
||||
f"[record] triangle marker on crate_blue east face at {tuple(round(v, 2) for v in m)}"
|
||||
)
|
||||
if args.goal == "search":
|
||||
# Fixed marker so the GIF tells a clear story (crate_blue, east face).
|
||||
bridge.world.marker = {
|
||||
"box": "crate_blue", "face": "x1", "u": 0.5, "v": 0.55,
|
||||
"size": 0.42, "color": (255, 150, 40),
|
||||
}
|
||||
m = bridge.world.marker_world_pos()
|
||||
print(f"[record] triangle marker on crate_blue east face at {tuple(round(v, 2) for v in m)}")
|
||||
|
||||
frames: list[Image.Image] = []
|
||||
chat = ChatPane()
|
||||
marker_seen = False
|
||||
samples: list[tuple[float, float]] = [] # capsule positions per captured frame
|
||||
|
||||
def log(role: str, msg: str) -> None:
|
||||
print(f" [{role}] {msg}")
|
||||
@@ -168,62 +166,96 @@ async def run(args: argparse.Namespace) -> int:
|
||||
async with AICCClient(transport) as client:
|
||||
manifest = await client.handshake()
|
||||
ctl = LLMController(
|
||||
client,
|
||||
manifest,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
system_prompt=SEARCH_MISSION,
|
||||
client, manifest,
|
||||
base_url=base_url, api_key=api_key, model=model,
|
||||
system_prompt=SEARCH_MISSION if args.goal == "search" else MISSION,
|
||||
log=log,
|
||||
)
|
||||
ctl.messages = [
|
||||
ctl.messages[0],
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
if args.goal == "search":
|
||||
ctl.messages = [
|
||||
ctl.messages[0],
|
||||
{"role": "user", "content": (
|
||||
"MISSION: explore the room, find the orange triangle on the "
|
||||
"back of a crate, and report it. Keep exploring until found."
|
||||
)},
|
||||
]
|
||||
chat.add("mission", "MISSION: find the orange triangle")
|
||||
else:
|
||||
ctl.messages.append({
|
||||
"role": "user",
|
||||
"content": (
|
||||
"MISSION: reach the beacon and activate it. "
|
||||
"Keep calling tools until it is done. Move smoothly (use "
|
||||
"duration on move/turn). The capsule cruises while you think."
|
||||
),
|
||||
},
|
||||
]
|
||||
chat.add("mission", "MISSION: find the orange triangle")
|
||||
orig_frame = ctl._frame_message
|
||||
})
|
||||
chat.add("mission", "MISSION: reach and activate the beacon")
|
||||
|
||||
def record_frame(png: str, note: str):
|
||||
nonlocal marker_seen
|
||||
view = (
|
||||
Image.open(io.BytesIO(base64.b64decode(png)))
|
||||
.convert("RGB")
|
||||
.resize((PANEL, PANEL))
|
||||
)
|
||||
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||
frames.append(_compose(top, view, chat.render()))
|
||||
return orig_frame(png, note)
|
||||
if args.goal == "search" and not args.capture_hz:
|
||||
# capture at frame-attach events (one frame per step)
|
||||
orig_frame = ctl._frame_message
|
||||
|
||||
ctl._frame_message = record_frame # type: ignore[method-assign]
|
||||
def record_frame(png: str, note: str):
|
||||
nonlocal marker_seen
|
||||
view = (
|
||||
Image.open(io.BytesIO(base64.b64decode(png)))
|
||||
.convert("RGB")
|
||||
.resize((PANEL, PANEL))
|
||||
)
|
||||
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||
frames.append(_compose(top, view, chat.render()))
|
||||
samples.append((bridge.world.capsule.x, bridge.world.capsule.z))
|
||||
return orig_frame(png, note)
|
||||
|
||||
ctl._frame_message = record_frame # type: ignore[method-assign]
|
||||
|
||||
def is_success(turn):
|
||||
nonlocal marker_seen
|
||||
for c in turn.calls:
|
||||
if (
|
||||
c.name == "report"
|
||||
and c.ok
|
||||
and c.output
|
||||
and c.output.get("verified")
|
||||
):
|
||||
if c.name == "report" and c.ok and c.output and c.output.get("verified"):
|
||||
marker_seen = True
|
||||
return True
|
||||
return False
|
||||
|
||||
summary = await run_llm_agent_loop(
|
||||
ctl,
|
||||
args.steps,
|
||||
log=log,
|
||||
look_every=1,
|
||||
cruise=args.cruise,
|
||||
autonomous=True,
|
||||
is_success=is_success,
|
||||
)
|
||||
async def observer(agent_done: asyncio.Event) -> None:
|
||||
"""Sample the world at a fixed rate so the GIF shows the actual
|
||||
gliding motion (mid-animation frames), not just step endpoints."""
|
||||
obs_transport = WebSocketClientTransport("ws://127.0.0.1:8765")
|
||||
async with AICCClient(obs_transport) as obs:
|
||||
await obs.handshake()
|
||||
while not agent_done.is_set():
|
||||
try:
|
||||
out = (await obs.call_tool("vision", {})).output
|
||||
except Exception: # noqa: BLE001 - session teardown
|
||||
break
|
||||
view = (
|
||||
Image.open(io.BytesIO(base64.b64decode(out["png_b64"])))
|
||||
.convert("RGB")
|
||||
.resize((PANEL, PANEL))
|
||||
)
|
||||
samples.append((bridge.world.capsule.x, bridge.world.capsule.z))
|
||||
top = _topdown_frame(bridge, ctl.path, len(frames), marker_seen)
|
||||
frames.append(_compose(top, view, chat.render()))
|
||||
if len(frames) > 500: # keep the GIF bounded
|
||||
frames[:100] = []
|
||||
await asyncio.sleep(1.0 / args.capture_hz)
|
||||
|
||||
agent_done = asyncio.Event()
|
||||
obs_task = None
|
||||
if args.capture_hz:
|
||||
obs_task = asyncio.create_task(observer(agent_done))
|
||||
try:
|
||||
summary = await run_llm_agent_loop(
|
||||
ctl, args.steps, log=log,
|
||||
look_every=2 if args.goal == "beacon" else 1,
|
||||
cruise=args.cruise,
|
||||
autonomous=args.goal == "search",
|
||||
is_success=is_success if args.goal == "search" else None,
|
||||
)
|
||||
finally:
|
||||
agent_done.set()
|
||||
if obs_task is not None:
|
||||
await obs_task
|
||||
|
||||
# Final frame: end state of the map.
|
||||
if marker_seen:
|
||||
@@ -236,6 +268,19 @@ async def run(args: argparse.Namespace) -> int:
|
||||
)
|
||||
frames.append(_compose(top, last_view, chat.render()))
|
||||
|
||||
# Smoothness check: mean movement between consecutive captured frames.
|
||||
if len(samples) > 2:
|
||||
from itertools import pairwise
|
||||
|
||||
deltas = [
|
||||
((b[0] - a[0]) ** 2 + (b[1] - a[1]) ** 2) ** 0.5
|
||||
for a, b in pairwise(samples)
|
||||
]
|
||||
moving = [d for d in deltas if d > 0.02]
|
||||
if moving:
|
||||
print(f"[record] motion between frames: mean {sum(moving)/len(moving):.2f} m "
|
||||
f"(max {max(moving):.2f}) over {len(moving)} moving samples")
|
||||
|
||||
out = Path(args.out)
|
||||
if frames:
|
||||
frames[0].save(
|
||||
@@ -248,21 +293,34 @@ async def run(args: argparse.Namespace) -> int:
|
||||
frames[-1].save(out.with_suffix(".png"))
|
||||
print(f"[record] wrote {out} ({len(frames)} frames, {args.duration} ms/frame)")
|
||||
print(f"[record] mission result: {summary.get('result')}")
|
||||
return 0 if marker_seen else 1
|
||||
return 0 if (marker_seen or summary.get("interacted")) else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Record a search mission into a three-view GIF."
|
||||
description="Record a mission into a three-view GIF (map | first person | chat)."
|
||||
)
|
||||
parser.add_argument("--out", default="search_mission.gif")
|
||||
parser.add_argument(
|
||||
"--goal",
|
||||
choices=["search", "beacon"],
|
||||
default="search",
|
||||
help="which mission to record (default search)",
|
||||
)
|
||||
parser.add_argument("--provider", default=None)
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--api-key", default=None)
|
||||
parser.add_argument("--model", default=None)
|
||||
parser.add_argument("--steps", type=int, default=45)
|
||||
parser.add_argument("--cruise", type=float, default=0.6)
|
||||
parser.add_argument("--duration", type=int, default=350, help="ms per GIF frame")
|
||||
parser.add_argument("--duration", type=int, default=150, help="ms per GIF frame")
|
||||
parser.add_argument(
|
||||
"--capture-hz",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="sample frames at this rate (Hz) with an observer client to show "
|
||||
"gliding motion; 0 = one frame per step (search default)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
|
||||
Reference in New Issue
Block a user