search challenge: triangle marker on a crate's back face (rendered in vision, in digest), report tool with distance verification, smooth turn(duration), --free/--search autonomous modes, look_at crates; verified with gpt-5.6-luna

testbed/llm_agent.py

@@ -68,6 +68,19 @@ NUDGE = (
     "then call a tool (turn or move, or interact if within 1.6 m)."
 )
 
+SEARCH_MISSION = (
+    "You are an explorer in a 16x16 m room. Three wooden crates stand in the room, "
+    "and a small ORANGE TRIANGLE is painted on the BACK side of ONE of them — the "
+    "side that faces away from the room's entrance, so it is only visible once you "
+    "walk around the crates. The glowing beacon in the corner is irrelevant: ignore it. "
+    "Your mission: explore the room until you see the orange triangle with your own "
+    'eyes (vision), then walk near it and call report(discovery="triangle"). '
+    "Strategy: walk a loop around the crates; every few steps look at the crate faces "
+    "with vision; check faces on all sides. The CURRENT STATE note in tool results "
+    "tells you your position and heading. Do not stop until you have seen and "
+    "reported the triangle."
+)
+
 # ---------------------------------------------------------------------------
 # Provider presets (OpenAI-compatible endpoints)
 # ---------------------------------------------------------------------------
@@ -188,6 +201,7 @@ def looks_multimodal(model: str) -> bool:
 COLOR_NAMES = [
     ("beacon_cyan", (120, 210, 235)),
     ("beacon_yellow", (255, 190, 90)),
+    ("marker_orange", (255, 150, 40)),
     ("crate_red", (178, 64, 54)),
     ("crate_blue", (64, 96, 178)),
     ("crate_olive", (128, 128, 60)),
@@ -606,14 +620,20 @@ async def run_llm_agent_loop(
     nudge_limit: int = 1,
     look_every: int = 0,
     cruise: float = 0.0,
+    autonomous: bool = False,
+    is_success: Callable[[TurnResult], bool] | None = None,
 ) -> dict[str, Any]:
-    """Drive the controller until the mission is done or steps run out.
+    """Drive the controller until the goal is done or steps run out.
 
     ``look_every``: attach a fresh camera frame every N steps so the model
     always sees recent visual context without asking (0 disables).
     ``cruise``: proactive motion — while the model is thinking, the capsule
     keeps gliding forward (meters per think, 0 disables). Collisions stop the
     drift and are reported to the model.
+    ``autonomous``: fewer guardrails — no re-aim corrections and no collision
+    hints; the model plans its own exploration (used for search missions).
+    ``is_success``: custom goal predicate over a model turn; by default the
+    mission ends when interact succeeds.
     """
     summary: dict[str, Any] = {
         "steps": 0,
@@ -730,23 +750,37 @@ async def run_llm_agent_loop(
                 and c.output.get("collision")
             ):
                 last_collision_step = step
-                obj = c.output.get("collision_normal")
-                hint_msg = (
-                    f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
-                    "forward twice to get around it, then follow the CURRENT STATE note to re-aim."
-                )
-                if not (
-                    controller.messages
-                    and controller.messages[-1].get("content") == hint_msg
-                ):
-                    controller.messages.append({"role": "user", "content": hint_msg})
-                    log("agent", "(collision: go around)")
+                if not autonomous:
+                    obj = c.output.get("collision_normal")
+                    hint_msg = (
+                        f"You collided with an obstacle (normal {obj}). Turn yaw_deg=90 and move "
+                        "forward twice to get around it, then follow the CURRENT STATE note to re-aim."
+                    )
+                    if not (
+                        controller.messages
+                        and controller.messages[-1].get("content") == hint_msg
+                    ):
+                        controller.messages.append(
+                            {"role": "user", "content": hint_msg}
+                        )
+                        log("agent", "(collision: go around)")
                 # Show the model what it just bumped into.
                 await controller.auto_frame(
                     "You just bumped into something. Look at what is in front of you."
                 )
-            maybe_correct(step)
-        if turn.interacted:
+            if not autonomous:
+                maybe_correct(step)
+        if is_success is not None:
+            if is_success(turn):
+                summary["success"] = True
+                for c in turn.calls:
+                    if c.name == "report" and c.ok and isinstance(c.output, dict):
+                        summary["result"] = c.output.get("message", "reported")
+                        break
+                else:
+                    summary["result"] = "goal achieved"
+                return summary
+        elif turn.interacted:
             summary["interacted"] = True
             summary["result"] = turn.message
             return summary