From 13143145670b40f0596833e691d4737c223f43a0 Mon Sep 17 00:00:00 2001 From: Emil Shanaty Date: Sat, 8 Aug 2026 03:27:14 +0300 Subject: [PATCH] feat: conformance test runner + protocol fixes - conformance.py: declarative scenario runner (subset matcher, $kind/$gt/ $enum/$required assertions, call_id/session_id echo checks), reference bridge, CLI entry (python -m aicc.conformance) - bridge.py: protocol version check before parsing (protocol_mismatch), session expiry handling (session_expired), malformed tool_call -> invalid_input, emit_event() for async events - client.py: Self return type - in_process.py: drain bridge event queue after response (event delivery), Self return type - websocket.py: Self return type, best-effort shutdown - tests: 14 passing (incl. conformance scenarios, 9/9 core scenarios) - ruff: all checks pass --- README.md | 21 +++ aicc/__init__.py | 64 ++++----- aicc/bridge.py | 89 +++++++++++- aicc/client.py | 19 ++- aicc/conformance.py | 270 +++++++++++++++++++++++++++++++++++ aicc/protocol.py | 1 - aicc/schema.py | 3 +- aicc/tool.py | 6 +- aicc/transport/__init__.py | 2 +- aicc/transport/in_process.py | 19 ++- aicc/transport/websocket.py | 7 +- tests/test_conformance.py | 33 +++++ tests/test_integration.py | 2 - tests/test_websocket.py | 2 - 14 files changed, 470 insertions(+), 68 deletions(-) create mode 100644 aicc/conformance.py create mode 100644 tests/test_conformance.py diff --git a/README.md b/README.md index 4d9d0cb..6e4e185 100644 --- a/README.md +++ b/README.md @@ -93,12 +93,33 @@ aicc/ bridge.py # Bridge — environment side, tool registration tool.py # @tool decorator and tool metadata schema.py # JSON Schema generation utilities + conformance.py # declarative scenario runner (AICC conformance tests) transport/ base.py # Transport interface in_process.py # In-process transport (tests, embedded) websocket.py # WebSocket transport (client + server) ``` +## Conformance + +`aicc-py` ships the reference conformance runner for the AICC protocol. +It executes the declarative scenarios from the +[AICC-Protocol](https://github.com/emil28092005/AICC-Protocol) +`conformance/scenarios/` directory against any bridge: + +```bash +python -m aicc.conformance ~/AICC-Protocol/conformance/scenarios +``` + +Or via pytest (scenarios are loaded automatically if the spec repo is +checked out next to `aicc-py`): + +```bash +pytest tests/test_conformance.py +``` + +A bridge is AICC-conformant for `aicc/0.1` when all core scenarios pass. + ## Development ```bash diff --git a/aicc/__init__.py b/aicc/__init__.py index 5d27479..6f13933 100644 --- a/aicc/__init__.py +++ b/aicc/__init__.py @@ -1,45 +1,45 @@ """aicc-py: Python SDK for the AI-Controlled Character Protocol.""" -from aicc.protocol import ( - ErrorCode, - MessageType, - TickMode, - WorldKind, - ModelClass, - ToolClass, - SessionInit, - SessionClose, - ToolCall, - ToolResult, - EventMessage, - ErrorMessage, - Heartbeat, -) -from aicc.errors import AICCError, ToolError, CapabilityError, ProtocolError from aicc.bridge import Bridge from aicc.client import AICCClient +from aicc.errors import AICCError, CapabilityError, ProtocolError, ToolError +from aicc.protocol import ( + ErrorCode, + ErrorMessage, + EventMessage, + Heartbeat, + MessageType, + ModelClass, + SessionClose, + SessionInit, + TickMode, + ToolCall, + ToolClass, + ToolResult, + WorldKind, +) from aicc.tool import tool __version__ = "0.1.0" __all__ = [ - "Bridge", "AICCClient", - "tool", - "ErrorCode", - "MessageType", - "TickMode", - "WorldKind", - "ModelClass", - "ToolClass", - "SessionInit", - "SessionClose", - "ToolCall", - "ToolResult", - "EventMessage", - "ErrorMessage", - "Heartbeat", "AICCError", - "ToolError", + "Bridge", "CapabilityError", + "ErrorCode", + "ErrorMessage", + "EventMessage", + "Heartbeat", + "MessageType", + "ModelClass", "ProtocolError", + "SessionClose", + "SessionInit", + "TickMode", + "ToolCall", + "ToolClass", + "ToolError", + "ToolResult", + "WorldKind", + "tool", ] diff --git a/aicc/bridge.py b/aicc/bridge.py index fd8fcf2..0217f04 100644 --- a/aicc/bridge.py +++ b/aicc/bridge.py @@ -3,14 +3,16 @@ from __future__ import annotations import asyncio -import inspect import uuid +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any, Callable +from typing import Any from aicc.protocol import ( + PROTOCOL_VERSION, Capabilities, ErrorCode, + ErrorMessage, ErrorPayload, Heartbeat, MessageType, @@ -24,7 +26,6 @@ from aicc.protocol import ( ToolSpec, WorldKind, WorldSpec, - PROTOCOL_VERSION, ) from aicc.tool import ToolDefinition, call_tool_impl @@ -144,6 +145,32 @@ class Bridge: if sess is not None: sess.closed = True + def emit_event( + self, + session_id: str, + topic: str, + payload: dict[str, Any] | None = None, + *, + tick: int | None = None, + ) -> None: + """Queue an async event for delivery to a connected agent. + + The transport is responsible for draining `session.send_queue` and + forwarding events to the client. Safe to call from tool handlers. + """ + sess = self._sessions.get(session_id) + if sess is None or sess.closed: + return + from aicc.protocol import EventMessage + + event = EventMessage( + session_id=session_id, + topic=topic, + payload=payload or {}, + meta={"tick": tick} if tick is not None else None, + ) + sess.send_queue.put_nowait(event.model_dump(by_alias=True)) + def session(self, session_id: str) -> Session: sess = self._sessions.get(session_id) if sess is None or sess.closed: @@ -196,11 +223,48 @@ class Bridge: May also push async events to the session queue. """ msg_type = raw.get("type") + + # Protocol version check BEFORE any model parsing: a wrong version + # must be answered with protocol_mismatch, not a parse error. + proto = raw.get("protocol") + if proto is not None and proto != PROTOCOL_VERSION: + return ErrorMessage( + session_id=session_id, + error=ErrorPayload( + code=ErrorCode.PROTOCOL_MISMATCH, + message=( + f"Protocol mismatch: bridge='{PROTOCOL_VERSION}', " + f"message='{proto}'" + ), + retryable=False, + ), + ).model_dump(by_alias=True) + if msg_type == MessageType.MANIFEST_REQUEST.value: - sess = self.session(session_id) + try: + sess = self.session(session_id) + except KeyError: + return self._session_expired(session_id) return sess.manifest.model_dump(by_alias=True) if sess.manifest else None if msg_type == MessageType.TOOL_CALL.value: - return await self._handle_tool_call(session_id, ToolCall.model_validate(raw)) + try: + call = ToolCall.model_validate(raw) + except Exception: # noqa: BLE001 - malformed tool_call + return ToolResult( + session_id=session_id, + call_id=raw.get("call_id") or "", + ok=False, + error=ErrorPayload( + code=ErrorCode.INVALID_INPUT, + message="Malformed tool_call message.", + retryable=False, + ), + ).model_dump(by_alias=True) + try: + self.session(session_id) + except KeyError: + return self._session_expired(session_id) + return await self._handle_tool_call(session_id, call) if msg_type == MessageType.SESSION_CLOSE.value: self.close_session(session_id) return SessionClose( @@ -210,10 +274,23 @@ class Bridge: if msg_type == MessageType.HEARTBEAT.value: return Heartbeat(session_id=session_id).model_dump(by_alias=True) if msg_type == MessageType.SESSION_RESUME.value: - sess = self.session(raw.get("session_id") or session_id) + try: + sess = self.session(raw.get("session_id") or session_id) + except KeyError: + return self._session_expired(session_id) return sess.manifest.model_dump(by_alias=True) if sess.manifest else None return None + def _session_expired(self, session_id: str) -> dict[str, Any]: + return ErrorMessage( + session_id=session_id, + error=ErrorPayload( + code=ErrorCode.SESSION_EXPIRED, + message=f"Session '{session_id}' is unknown or closed.", + retryable=False, + ), + ).model_dump(by_alias=True) + async def _handle_tool_call(self, session_id: str, call: ToolCall) -> dict[str, Any]: defn = self._tools.get(call.tool) if defn is None: diff --git a/aicc/client.py b/aicc/client.py index dc98b3b..b455f49 100644 --- a/aicc/client.py +++ b/aicc/client.py @@ -4,11 +4,16 @@ from __future__ import annotations import asyncio import uuid -from typing import Any +from typing import Any, Self +from aicc.errors import ( + AICCError, + ConnectionError_, + ProtocolError, + ToolError, +) from aicc.protocol import ( PROTOCOL_VERSION, - ErrorCode, EventMessage, MessageType, SessionClose, @@ -16,12 +21,6 @@ from aicc.protocol import ( ToolCall, ToolResult, ) -from aicc.errors import ( - AICCError, - ConnectionError_, - ProtocolError, - ToolError, -) from aicc.transport.base import Transport @@ -56,14 +55,14 @@ class AICCClient: # ---------- Lifecycle ---------- - async def __aenter__(self) -> "AICCClient": + async def __aenter__(self) -> Self: await self._transport.connect() return self async def __aexit__(self, exc_type, exc, tb) -> None: await self.close() - async def connect(self) -> "AICCClient": + async def connect(self) -> AICCClient: await self._transport.connect() return self diff --git a/aicc/conformance.py b/aicc/conformance.py new file mode 100644 index 0000000..80522fc --- /dev/null +++ b/aicc/conformance.py @@ -0,0 +1,270 @@ +"""AICC conformance test runner. + +Loads declarative scenarios (see https://github.com/emil28092005/AICC-Protocol +conformance/scenarios/) and executes them against a bridge, reporting +pass/fail per scenario. + +Usage: + from aicc.conformance import run_scenarios, summarize + results = run_scenarios(bridge, "/path/to/scenarios") + print(summarize(results)) +""" + +from __future__ import annotations + +import asyncio +import json +import re +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from aicc.bridge import Bridge +from aicc.transport.in_process import InProcessTransport + +UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +@dataclass +class ScenarioResult: + id: str + title: str + passed: bool + failures: list[str] = field(default_factory=list) + error: str | None = None + + +def _match_value(expect: Any, actual: Any, path: str, failures: list[str]) -> None: + """Subset matcher with special `$` matchers for conformance assertions.""" + if isinstance(expect, dict): + # Special matchers + if "$kind" in expect and len(expect) == 1: + kind = expect["$kind"] + ok = { + "uuid": isinstance(actual, str) and bool(UUID_RE.match(actual)), + "integer": isinstance(actual, int) and not isinstance(actual, bool), + "number": isinstance(actual, (int, float)) and not isinstance(actual, bool), + "string": isinstance(actual, str), + "boolean": isinstance(actual, bool), + "array": isinstance(actual, list), + "object": isinstance(actual, dict), + }.get(kind, False) + if not ok: + failures.append(f"{path}: expected {kind}, got {type(actual).__name__}") + return + if "$gt" in expect and len(expect) == 1: + if not (isinstance(actual, (int, float)) and actual > expect["$gt"]): + failures.append(f"{path}: expected > {expect['$gt']}, got {actual!r}") + return + if "$enum" in expect and len(expect) == 1: + if actual not in expect["$enum"]: + failures.append(f"{path}: expected one of {expect['$enum']}, got {actual!r}") + return + if "$required" in expect and len(expect) == 1: + if not isinstance(actual, dict): + failures.append(f"{path}: expected object, got {type(actual).__name__}") + return + for key in expect["$required"]: + if key not in actual: + failures.append(f"{path}: missing required key {key!r}") + return + if not isinstance(actual, dict): + failures.append(f"{path}: expected object, got {type(actual).__name__}") + return + for key, sub in expect.items(): + if key not in actual: + failures.append(f"{path}: missing key {key!r}") + continue + _match_value(sub, actual[key], f"{path}.{key}", failures) + return + + if isinstance(expect, list): + if not isinstance(actual, list): + failures.append(f"{path}: expected array, got {type(actual).__name__}") + return + if len(expect) != len(actual): + failures.append(f"{path}: expected array len {len(expect)}, got {len(actual)}") + return + for i, (sub, act) in enumerate(zip(expect, actual)): + _match_value(sub, act, f"{path}[{i}]", failures) + return + + if expect != actual: + failures.append(f"{path}: expected {expect!r}, got {actual!r}") + + +def _validate_scenario(sc: dict[str, Any]) -> None: + if "id" not in sc: + raise ValueError("Scenario missing 'id'") + if "steps" not in sc or not isinstance(sc["steps"], list): + raise ValueError(f"Scenario {sc['id']} missing 'steps' list") + for step in sc["steps"]: + if "send" not in step and "expect" not in step: + raise ValueError(f"Scenario {sc['id']} has step without send/expect") + + +async def _run_scenario(bridge: Bridge, sc: dict[str, Any]) -> ScenarioResult: + result = ScenarioResult(id=sc["id"], title=sc.get("title", sc["id"]), passed=True) + try: + _validate_scenario(sc) + t = InProcessTransport.start(bridge) + async with t: + # Handshake: connect pushes session_init into the receive queue. + manifest_raw = await asyncio.wait_for(t.receive(), timeout=10.0) + if manifest_raw.get("type") != "session_init": + result.passed = False + result.error = ( + f"expected session_init on connect, got {manifest_raw.get('type')!r}" + ) + return result + session_id = manifest_raw["session_id"] + last_call_id: str | None = None + # Pending message already consumed from the wire (the handshake + # session_init). Scenarios that expect session_init as their + # first step consume it here instead of receiving again. + pending: dict[str, Any] | None = manifest_raw + + for step in sc["steps"]: + if "send" in step: + msg = dict(step["send"]) + msg.setdefault("session_id", session_id) + msg.setdefault("message_id", str(uuid.uuid4())) + if msg.get("type") == "tool_call": + last_call_id = msg.get("call_id") or f"tc_{uuid.uuid4().hex[:12]}" + msg["call_id"] = last_call_id + await t.send(msg) + + if "expect" in step: + expected = step["expect"] + if pending is not None and expected.get("type") == "session_init": + raw = pending + pending = None + else: + raw = await asyncio.wait_for(t.receive(), timeout=10.0) + failures: list[str] = [] + + if ( + expected.get("call_id_echo") + and last_call_id is not None + and raw.get("call_id") != last_call_id + ): + failures.append( + f"call_id_echo: expected {last_call_id!r}, " + f"got {raw.get('call_id')!r}" + ) + if expected.get("session_id_echo") and raw.get("session_id") != session_id: + failures.append( + f"session_id_echo: expected {session_id!r}, " + f"got {raw.get('session_id')!r}" + ) + + remaining = { + k: v + for k, v in expected.items() + if k not in ("call_id_echo", "session_id_echo") + } + _match_value(remaining, raw, "msg", failures) + + if failures: + result.passed = False + result.failures.extend(failures) + return result + except Exception as exc: # noqa: BLE001 + result.passed = False + result.error = f"{type(exc).__name__}: {exc}" + return result + + +def load_scenarios(scenario_dir: str | Path) -> list[dict[str, Any]]: + """Load all *.json scenario files from a directory, sorted by id.""" + d = Path(scenario_dir) + files = sorted(d.glob("*.json")) + scenarios = [] + for f in files: + with open(f) as fh: + scenarios.append(json.load(fh)) + return scenarios + + +async def run_scenarios(bridge: Bridge, scenario_dir: str | Path) -> list[ScenarioResult]: + """Run all scenarios in a directory against a bridge. One session per scenario.""" + results = [] + for sc in load_scenarios(scenario_dir): + results.append(await _run_scenario(bridge, sc)) + return results + + +def summarize(results: list[ScenarioResult]) -> str: + lines = [] + passed = sum(1 for r in results if r.passed) + for r in results: + status = "PASS" if r.passed else "FAIL" + lines.append(f"[{status}] {r.id}: {r.title}") + for f in r.failures: + lines.append(f" - {f}") + if r.error: + lines.append(f" - error: {r.error}") + lines.append(f"\n{passed}/{len(results)} scenarios passed") + return "\n".join(lines) + + +def reference_bridge() -> Bridge: + """Reference bridge exposing the tools the core scenarios expect. + + Tools required by scenarios: + - echo: returns input value + - boom: raises -> execution_failed + - bump: returns ok and emits a collision event + """ + bridge = Bridge(name="conformance-reference", kind="3d", tick_rate_hz=10.0) + + @bridge.tool(description="Echo a value back to the caller.") + async def echo(value: str = "") -> dict: + return {"value": value} + + @bridge.tool(description="Always raises an exception.") + async def boom() -> dict: + raise RuntimeError("kaboom") + + @bridge.tool(description="Returns ok and emits a collision event.") + async def bump() -> dict: + return {"bumped": True} + + # Emit a collision event when bump completes. Hook into tool handling: + # after a successful bump call, push the event into the session queue. + _orig = bridge._handle_tool_call + + async def _handle_with_event(session_id: str, call): + resp = await _orig(session_id, call) + if call.tool == "bump" and resp.get("ok"): + bridge.emit_event( + session_id, + "collision", + {"other": "wall", "impulse": 1.0}, + tick=1, + ) + return resp + + bridge._handle_tool_call = _handle_with_event # type: ignore[method-assign] + return bridge + + +def main() -> None: + """CLI entry: python -m aicc.conformance """ + import sys + + if len(sys.argv) < 2: + print("usage: python -m aicc.conformance ") + sys.exit(2) + + bridge = reference_bridge() + results = asyncio.run(run_scenarios(bridge, sys.argv[1])) + print(summarize(results)) + sys.exit(0 if all(r.passed for r in results) else 1) + + +if __name__ == "__main__": + main() diff --git a/aicc/protocol.py b/aicc/protocol.py index 3d5a763..fa3c291 100644 --- a/aicc/protocol.py +++ b/aicc/protocol.py @@ -8,7 +8,6 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field - PROTOCOL_VERSION = "aicc/0.1" diff --git a/aicc/schema.py b/aicc/schema.py index 05ab8be..2fd953c 100644 --- a/aicc/schema.py +++ b/aicc/schema.py @@ -3,7 +3,8 @@ from __future__ import annotations import inspect -from typing import Any, Callable, get_args, get_origin, get_type_hints +from collections.abc import Callable +from typing import Any, get_args, get_origin, get_type_hints from pydantic import BaseModel diff --git a/aicc/tool.py b/aicc/tool.py index add5b29..dfdc45a 100644 --- a/aicc/tool.py +++ b/aicc/tool.py @@ -4,13 +4,13 @@ from __future__ import annotations import asyncio import inspect -from dataclasses import dataclass, field -from typing import Any, Awaitable, Callable +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any from aicc.protocol import ToolClass from aicc.schema import function_schema, return_schema - ToolImpl = Callable[..., Any | Awaitable[Any]] diff --git a/aicc/transport/__init__.py b/aicc/transport/__init__.py index 3bad3cc..a45bfb4 100644 --- a/aicc/transport/__init__.py +++ b/aicc/transport/__init__.py @@ -3,7 +3,7 @@ from aicc.transport.base import Transport from aicc.transport.in_process import InProcessTransport -__all__ = ["Transport", "InProcessTransport"] +__all__ = ["InProcessTransport", "Transport"] try: # pragma: no cover from aicc.transport.websocket import WebSocketClientTransport, WebSocketServer diff --git a/aicc/transport/in_process.py b/aicc/transport/in_process.py index d240e67..0105e46 100644 --- a/aicc/transport/in_process.py +++ b/aicc/transport/in_process.py @@ -6,12 +6,11 @@ Useful for tests, embedded use, and single-process agents that drive the bridge from __future__ import annotations import asyncio -import uuid -from typing import Any +from typing import Any, Self from aicc.bridge import Bridge from aicc.errors import ConnectionError_ -from aicc.protocol import PROTOCOL_VERSION, SessionInit +from aicc.protocol import SessionInit class InProcessTransport: @@ -35,7 +34,7 @@ class InProcessTransport: self._connected = False @classmethod - def start(cls, bridge: Bridge) -> "InProcessTransport": + def start(cls, bridge: Bridge) -> InProcessTransport: t = cls() t._bridge = bridge return t @@ -63,12 +62,20 @@ class InProcessTransport: sid = message.get("session_id") or self._session_id msg_type = message.get("type") # session_resume carries the session_id; bind it - if msg_type == "session_resume" and not self._bridge._sessions.get(sid): # noqa: SLF001 + if msg_type == "session_resume" and not self._bridge._sessions.get(sid): # Auto-open if bridge forgot pass response = await self._bridge.handle_message(sid, message) if response is not None: await self._recv_q.put(response) + # Drain any events the bridge queued during handling (e.g. from + # tool handlers calling bridge.emit_event). Deliver them after the + # response so the client sees a coherent request -> response order. + sess = self._bridge._sessions.get(sid) + if sess is not None: + while not sess.send_queue.empty(): + event = sess.send_queue.get_nowait() + await self._recv_q.put(event) async def receive(self) -> dict[str, Any]: if not self._connected: @@ -79,7 +86,7 @@ class InProcessTransport: """Bridge-side helper to push an event to the connected client.""" await self._recv_q.put(event) - async def __aenter__(self) -> "InProcessTransport": + async def __aenter__(self) -> Self: await self.connect() return self diff --git a/aicc/transport/websocket.py b/aicc/transport/websocket.py index ae3ef9b..11a7698 100644 --- a/aicc/transport/websocket.py +++ b/aicc/transport/websocket.py @@ -2,9 +2,8 @@ from __future__ import annotations -import asyncio import json -from typing import Any +from typing import Any, Self from aicc.bridge import Bridge from aicc.errors import ConnectionError_ @@ -67,7 +66,7 @@ class WebSocketServer: self._server = None # type: ignore[var-annotated] self._connections: set[Any] = set() - async def __aenter__(self) -> "WebSocketServer": + async def __aenter__(self) -> Self: await self.start() return self @@ -111,6 +110,6 @@ class WebSocketServer: for ws in list(self._connections): try: await ws.close() - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001, S110 - best-effort shutdown pass self._connections.clear() diff --git a/tests/test_conformance.py b/tests/test_conformance.py new file mode 100644 index 0000000..5d5dd1d --- /dev/null +++ b/tests/test_conformance.py @@ -0,0 +1,33 @@ +"""Conformance runner tests: runs the declarative scenarios from the spec repo.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aicc.conformance import reference_bridge, run_scenarios, summarize + +SCENARIOS = Path(__file__).resolve().parents[1] / ".." / "aicc-spec" / "conformance" / "scenarios" + + +@pytest.mark.asyncio +async def test_core_scenarios_pass(): + if not SCENARIOS.exists(): + pytest.skip("AICC-Protocol conformance scenarios not found on this machine") + bridge = reference_bridge() + results = await run_scenarios(bridge, SCENARIOS) + report = summarize(results) + assert all(r.passed for r in results), report + + +@pytest.mark.asyncio +async def test_runner_rejects_unknown_tool(): + """A bridge missing a required tool fails the corresponding scenario.""" + from aicc import Bridge + + bridge = Bridge(name="incomplete") + results = await run_scenarios(bridge, SCENARIOS) + by_id = {r.id: r for r in results} + # echo tool missing -> core-03 must fail + assert not by_id["core-03-tool-call-ok"].passed diff --git a/tests/test_integration.py b/tests/test_integration.py index efa39e6..0b694ff 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,8 +2,6 @@ from __future__ import annotations -import asyncio - import pytest from aicc import AICCClient, Bridge, ToolClass diff --git a/tests/test_websocket.py b/tests/test_websocket.py index f41667b..e46bd6f 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -2,8 +2,6 @@ from __future__ import annotations -import asyncio - import pytest from aicc import AICCClient, Bridge