diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..0157554 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,74 @@ +# AICC Conformance Scenarios + +This directory contains declarative conformance scenarios for the AICC Protocol (aicc/0.1). + +Each JSON file describes a sequence of messages exchanged with a bridge and +the expected responses. Any bridge that passes all scenarios in this +directory satisfies the conformance criteria in ยง13 of `../core.md`. + +## Scenario format + +```json +{ + "id": "core-01-handshake", + "title": "Human readable title", + "category": "core", + "version": "aicc/0.1", + "steps": [ + { "send": { "type": "tool_call", "tool": "echo", "input": {} } }, + { "expect": { "type": "tool_result", "ok": true, "call_id_echo": true } } + ] +} +``` + +### Steps + +| Step kind | Semantics | +|-----------|-----------| +| `send` | Send a message to the bridge. `session_id` and `message_id` are filled in by the runner if absent. | +| `expect` | Wait for the next message and assert it matches the expectation. | + +### Expectation matching + +Expectations are matched **subset-wise**: every field present in the +expectation must match the received message; extra fields in the received +message are allowed. + +Special matchers: + +| Field | Meaning | +|-------|---------| +| `call_id_echo: true` | Received `call_id` must equal the `call_id` of the last `send`. | +| `session_id_echo: true` | Received `session_id` must equal the session's id. | +| `{"$kind": "uuid"}` | Field must look like a UUID. | +| `{"$kind": "integer"}` / `{"$kind": "array"}` | Field must be of that JSON type. | +| `{"$gt": N}` | Field must be numerically greater than N. | +| `{"$enum": [...]}` | Field must be one of the listed values. | +| `{"$required": [...]}` | Object field must contain these keys. | + +## Running + +Reference runner: `aicc-py` (see https://github.com/emil28092005/aicc-py). +Run `python -m aicc.conformance ` or the pytest suite. + +## Scenario inventory + +| ID | Checks | +|----|--------| +| core-01-handshake | Valid session_init on connect | +| core-02-manifest-request | manifest_request returns manifest | +| core-03-tool-call-ok | Valid tool call succeeds, call_id echoed | +| core-04-tool-unknown | Unknown tool โ†’ tool_unknown | +| core-05-tool-execution-failed | Throwing tool โ†’ execution_failed | +| core-06-protocol-mismatch | Wrong protocol โ†’ protocol_mismatch | +| core-07-heartbeat | Heartbeat acknowledged | +| core-08-session-close | Close ends session, then session_expired | +| core-09-event-delivery | Environment events delivered | + +## Adding scenarios + +1. Create `N-.json` following the format above. +2. Add a row to the inventory table. +3. Run the runner against the reference bridge in aicc-py. +4. Scenarios requiring tools not in the reference bridge must document + the requirement in `description`. diff --git a/conformance/scenarios/core-01-handshake.json b/conformance/scenarios/core-01-handshake.json new file mode 100644 index 0000000..5a08fad --- /dev/null +++ b/conformance/scenarios/core-01-handshake.json @@ -0,0 +1,21 @@ +{ + "id": "core-01-handshake", + "title": "Handshake: bridge sends a valid session_init", + "category": "core", + "version": "aicc/0.1", + "description": "On connect, the bridge must push a session_init message with a valid envelope and manifest structure.", + "steps": [ + { + "expect": { + "type": "session_init", + "protocol": "aicc/0.1", + "session_id": {"$kind": "uuid"}, + "tick_rate_hz": {"$gt": 0}, + "tick_mode": {"$enum": ["fixed", "event", "hybrid"]}, + "world": {"$required": ["name", "kind"]}, + "capabilities": {"$required": ["sensors", "actuators", "generators"]}, + "tools": {"$kind": "array"} + } + } + ] +} diff --git a/conformance/scenarios/core-02-manifest-request.json b/conformance/scenarios/core-02-manifest-request.json new file mode 100644 index 0000000..f039cd7 --- /dev/null +++ b/conformance/scenarios/core-02-manifest-request.json @@ -0,0 +1,20 @@ +{ + "id": "core-02-manifest-request", + "title": "manifest_request returns the same session_init", + "category": "core", + "version": "aicc/0.1", + "description": "A manifest_request must be answered with the session manifest for the current session.", + "steps": [ + { + "send": { + "type": "manifest_request" + } + }, + { + "expect": { + "type": "session_init", + "session_id_echo": true + } + } + ] +} diff --git a/conformance/scenarios/core-03-tool-call-ok.json b/conformance/scenarios/core-03-tool-call-ok.json new file mode 100644 index 0000000..c644652 --- /dev/null +++ b/conformance/scenarios/core-03-tool-call-ok.json @@ -0,0 +1,24 @@ +{ + "id": "core-03-tool-call-ok", + "title": "Valid tool call returns ok:true with echo call_id", + "category": "core", + "version": "aicc/0.1", + "description": "A tool registered in the manifest must execute and return ok:true, echoing the caller's call_id.", + "steps": [ + { + "send": { + "type": "tool_call", + "tool": "echo", + "input": {"value": "hello"} + } + }, + { + "expect": { + "type": "tool_result", + "ok": true, + "call_id_echo": true, + "output": {"value": "hello"} + } + } + ] +} diff --git a/conformance/scenarios/core-04-tool-unknown.json b/conformance/scenarios/core-04-tool-unknown.json new file mode 100644 index 0000000..3663f4a --- /dev/null +++ b/conformance/scenarios/core-04-tool-unknown.json @@ -0,0 +1,27 @@ +{ + "id": "core-04-tool-unknown", + "title": "Unknown tool returns tool_unknown", + "category": "core", + "version": "aicc/0.1", + "description": "Calling a tool that is not in the manifest must fail with error code tool_unknown and retryable:false.", + "steps": [ + { + "send": { + "type": "tool_call", + "tool": "definitely_not_a_tool", + "input": {} + } + }, + { + "expect": { + "type": "tool_result", + "ok": false, + "call_id_echo": true, + "error": { + "code": "tool_unknown", + "retryable": false + } + } + } + ] +} diff --git a/conformance/scenarios/core-05-tool-execution-failed.json b/conformance/scenarios/core-05-tool-execution-failed.json new file mode 100644 index 0000000..7da9d2c --- /dev/null +++ b/conformance/scenarios/core-05-tool-execution-failed.json @@ -0,0 +1,27 @@ +{ + "id": "core-05-tool-execution-failed", + "title": "Tool that raises returns execution_failed", + "category": "core", + "version": "aicc/0.1", + "description": "A tool implementation that throws must be caught by the bridge and surfaced as error code execution_failed, not crash the session.", + "steps": [ + { + "send": { + "type": "tool_call", + "tool": "boom", + "input": {} + } + }, + { + "expect": { + "type": "tool_result", + "ok": false, + "call_id_echo": true, + "error": { + "code": "execution_failed", + "retryable": false + } + } + } + ] +} diff --git a/conformance/scenarios/core-06-protocol-mismatch.json b/conformance/scenarios/core-06-protocol-mismatch.json new file mode 100644 index 0000000..c51abf8 --- /dev/null +++ b/conformance/scenarios/core-06-protocol-mismatch.json @@ -0,0 +1,26 @@ +{ + "id": "core-06-protocol-mismatch", + "title": "Protocol mismatch is rejected", + "category": "core", + "version": "aicc/0.1", + "description": "A message with a wrong protocol version must be rejected with protocol_mismatch.", + "steps": [ + { + "send": { + "protocol": "aicc/0.2", + "type": "tool_call", + "tool": "echo", + "input": {} + } + }, + { + "expect": { + "type": "error", + "error": { + "code": "protocol_mismatch", + "retryable": false + } + } + } + ] +} diff --git a/conformance/scenarios/core-07-heartbeat.json b/conformance/scenarios/core-07-heartbeat.json new file mode 100644 index 0000000..8c8c662 --- /dev/null +++ b/conformance/scenarios/core-07-heartbeat.json @@ -0,0 +1,19 @@ +{ + "id": "core-07-heartbeat", + "title": "Heartbeat is answered with heartbeat", + "category": "core", + "version": "aicc/0.1", + "description": "A heartbeat must be acknowledged with a heartbeat, keeping the session alive.", + "steps": [ + { + "send": { + "type": "heartbeat" + } + }, + { + "expect": { + "type": "heartbeat" + } + } + ] +} diff --git a/conformance/scenarios/core-08-session-close.json b/conformance/scenarios/core-08-session-close.json new file mode 100644 index 0000000..d7a45b1 --- /dev/null +++ b/conformance/scenarios/core-08-session-close.json @@ -0,0 +1,36 @@ +{ + "id": "core-08-session-close", + "title": "session_close ends the session", + "category": "core", + "version": "aicc/0.1", + "description": "After session_close, further tool calls must fail with session_expired.", + "steps": [ + { + "send": { + "type": "session_close", + "reason": "conformance" + } + }, + { + "expect": { + "type": "session_close" + } + }, + { + "send": { + "type": "tool_call", + "tool": "echo", + "input": {} + } + }, + { + "expect": { + "type": "error", + "error": { + "code": "session_expired", + "retryable": false + } + } + } + ] +} diff --git a/conformance/scenarios/core-09-event-delivery.json b/conformance/scenarios/core-09-event-delivery.json new file mode 100644 index 0000000..24c4ab0 --- /dev/null +++ b/conformance/scenarios/core-09-event-delivery.json @@ -0,0 +1,31 @@ +{ + "id": "core-09-event-delivery", + "title": "Environment events are delivered to the agent", + "category": "core", + "version": "aicc/0.1", + "description": "Events pushed by the bridge (e.g. collision) arrive as event messages with a tick in meta.", + "steps": [ + { + "send": { + "type": "tool_call", + "tool": "bump", + "input": {} + } + }, + { + "expect": { + "type": "tool_result", + "ok": true + } + }, + { + "expect": { + "type": "event", + "topic": "collision", + "meta": { + "tick": {"$kind": "integer"} + } + } + } + ] +}