feat: conformance scenarios (core-01..core-09)

Declarative conformance scenarios for aicc/0.1 covering: handshake, manifest_request, tool call ok/unknown/execution-failed, protocol mismatch, heartbeat, session close, event delivery. Plus runner documentation (conformance/README.md).
2026-08-08 03:27:20 +03:00
parent e75ba873d8
commit fa5ef35bff
10 changed files with 305 additions and 0 deletions
@@ -0,0 +1,21 @@
+{
+  "id": "core-01-handshake",
+  "title": "Handshake: bridge sends a valid session_init",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "On connect, the bridge must push a session_init message with a valid envelope and manifest structure.",
+  "steps": [
+    {
+      "expect": {
+        "type": "session_init",
+        "protocol": "aicc/0.1",
+        "session_id": {"$kind": "uuid"},
+        "tick_rate_hz": {"$gt": 0},
+        "tick_mode": {"$enum": ["fixed", "event", "hybrid"]},
+        "world": {"$required": ["name", "kind"]},
+        "capabilities": {"$required": ["sensors", "actuators", "generators"]},
+        "tools": {"$kind": "array"}
+      }
+    }
+  ]
+}
@@ -0,0 +1,20 @@
+{
+  "id": "core-02-manifest-request",
+  "title": "manifest_request returns the same session_init",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "A manifest_request must be answered with the session manifest for the current session.",
+  "steps": [
+    {
+      "send": {
+        "type": "manifest_request"
+      }
+    },
+    {
+      "expect": {
+        "type": "session_init",
+        "session_id_echo": true
+      }
+    }
+  ]
+}
@@ -0,0 +1,24 @@
+{
+  "id": "core-03-tool-call-ok",
+  "title": "Valid tool call returns ok:true with echo call_id",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "A tool registered in the manifest must execute and return ok:true, echoing the caller's call_id.",
+  "steps": [
+    {
+      "send": {
+        "type": "tool_call",
+        "tool": "echo",
+        "input": {"value": "hello"}
+      }
+    },
+    {
+      "expect": {
+        "type": "tool_result",
+        "ok": true,
+        "call_id_echo": true,
+        "output": {"value": "hello"}
+      }
+    }
+  ]
+}
@@ -0,0 +1,27 @@
+{
+  "id": "core-04-tool-unknown",
+  "title": "Unknown tool returns tool_unknown",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "Calling a tool that is not in the manifest must fail with error code tool_unknown and retryable:false.",
+  "steps": [
+    {
+      "send": {
+        "type": "tool_call",
+        "tool": "definitely_not_a_tool",
+        "input": {}
+      }
+    },
+    {
+      "expect": {
+        "type": "tool_result",
+        "ok": false,
+        "call_id_echo": true,
+        "error": {
+          "code": "tool_unknown",
+          "retryable": false
+        }
+      }
+    }
+  ]
+}
@@ -0,0 +1,27 @@
+{
+  "id": "core-05-tool-execution-failed",
+  "title": "Tool that raises returns execution_failed",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "A tool implementation that throws must be caught by the bridge and surfaced as error code execution_failed, not crash the session.",
+  "steps": [
+    {
+      "send": {
+        "type": "tool_call",
+        "tool": "boom",
+        "input": {}
+      }
+    },
+    {
+      "expect": {
+        "type": "tool_result",
+        "ok": false,
+        "call_id_echo": true,
+        "error": {
+          "code": "execution_failed",
+          "retryable": false
+        }
+      }
+    }
+  ]
+}
@@ -0,0 +1,26 @@
+{
+  "id": "core-06-protocol-mismatch",
+  "title": "Protocol mismatch is rejected",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "A message with a wrong protocol version must be rejected with protocol_mismatch.",
+  "steps": [
+    {
+      "send": {
+        "protocol": "aicc/0.2",
+        "type": "tool_call",
+        "tool": "echo",
+        "input": {}
+      }
+    },
+    {
+      "expect": {
+        "type": "error",
+        "error": {
+          "code": "protocol_mismatch",
+          "retryable": false
+        }
+      }
+    }
+  ]
+}
@@ -0,0 +1,19 @@
+{
+  "id": "core-07-heartbeat",
+  "title": "Heartbeat is answered with heartbeat",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "A heartbeat must be acknowledged with a heartbeat, keeping the session alive.",
+  "steps": [
+    {
+      "send": {
+        "type": "heartbeat"
+      }
+    },
+    {
+      "expect": {
+        "type": "heartbeat"
+      }
+    }
+  ]
+}
@@ -0,0 +1,36 @@
+{
+  "id": "core-08-session-close",
+  "title": "session_close ends the session",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "After session_close, further tool calls must fail with session_expired.",
+  "steps": [
+    {
+      "send": {
+        "type": "session_close",
+        "reason": "conformance"
+      }
+    },
+    {
+      "expect": {
+        "type": "session_close"
+      }
+    },
+    {
+      "send": {
+        "type": "tool_call",
+        "tool": "echo",
+        "input": {}
+      }
+    },
+    {
+      "expect": {
+        "type": "error",
+        "error": {
+          "code": "session_expired",
+          "retryable": false
+        }
+      }
+    }
+  ]
+}
@@ -0,0 +1,31 @@
+{
+  "id": "core-09-event-delivery",
+  "title": "Environment events are delivered to the agent",
+  "category": "core",
+  "version": "aicc/0.1",
+  "description": "Events pushed by the bridge (e.g. collision) arrive as event messages with a tick in meta.",
+  "steps": [
+    {
+      "send": {
+        "type": "tool_call",
+        "tool": "bump",
+        "input": {}
+      }
+    },
+    {
+      "expect": {
+        "type": "tool_result",
+        "ok": true
+      }
+    },
+    {
+      "expect": {
+        "type": "event",
+        "topic": "collision",
+        "meta": {
+          "tick": {"$kind": "integer"}
+        }
+      }
+    }
+  ]
+}