feat: conformance test runner + protocol fixes

- conformance.py: declarative scenario runner (subset matcher, $kind/$gt/
  $enum/$required assertions, call_id/session_id echo checks), reference
  bridge, CLI entry (python -m aicc.conformance)
- bridge.py: protocol version check before parsing (protocol_mismatch),
  session expiry handling (session_expired), malformed tool_call ->
  invalid_input, emit_event() for async events
- client.py: Self return type
- in_process.py: drain bridge event queue after response (event delivery),
  Self return type
- websocket.py: Self return type, best-effort shutdown
- tests: 14 passing (incl. conformance scenarios, 9/9 core scenarios)
- ruff: all checks pass
This commit is contained in:
Emil Shanaty
2026-08-08 03:27:14 +03:00
parent 0cfb4277b5
commit 1314314567
14 changed files with 470 additions and 68 deletions
+21
View File
@@ -93,12 +93,33 @@ aicc/
bridge.py # Bridge — environment side, tool registration
tool.py # @tool decorator and tool metadata
schema.py # JSON Schema generation utilities
conformance.py # declarative scenario runner (AICC conformance tests)
transport/
base.py # Transport interface
in_process.py # In-process transport (tests, embedded)
websocket.py # WebSocket transport (client + server)
```
## Conformance
`aicc-py` ships the reference conformance runner for the AICC protocol.
It executes the declarative scenarios from the
[AICC-Protocol](https://github.com/emil28092005/AICC-Protocol)
`conformance/scenarios/` directory against any bridge:
```bash
python -m aicc.conformance ~/AICC-Protocol/conformance/scenarios
```
Or via pytest (scenarios are loaded automatically if the spec repo is
checked out next to `aicc-py`):
```bash
pytest tests/test_conformance.py
```
A bridge is AICC-conformant for `aicc/0.1` when all core scenarios pass.
## Development
```bash
+32 -32
View File
@@ -1,45 +1,45 @@
"""aicc-py: Python SDK for the AI-Controlled Character Protocol."""
from aicc.protocol import (
ErrorCode,
MessageType,
TickMode,
WorldKind,
ModelClass,
ToolClass,
SessionInit,
SessionClose,
ToolCall,
ToolResult,
EventMessage,
ErrorMessage,
Heartbeat,
)
from aicc.errors import AICCError, ToolError, CapabilityError, ProtocolError
from aicc.bridge import Bridge
from aicc.client import AICCClient
from aicc.errors import AICCError, CapabilityError, ProtocolError, ToolError
from aicc.protocol import (
ErrorCode,
ErrorMessage,
EventMessage,
Heartbeat,
MessageType,
ModelClass,
SessionClose,
SessionInit,
TickMode,
ToolCall,
ToolClass,
ToolResult,
WorldKind,
)
from aicc.tool import tool
__version__ = "0.1.0"
__all__ = [
"Bridge",
"AICCClient",
"tool",
"ErrorCode",
"MessageType",
"TickMode",
"WorldKind",
"ModelClass",
"ToolClass",
"SessionInit",
"SessionClose",
"ToolCall",
"ToolResult",
"EventMessage",
"ErrorMessage",
"Heartbeat",
"AICCError",
"ToolError",
"Bridge",
"CapabilityError",
"ErrorCode",
"ErrorMessage",
"EventMessage",
"Heartbeat",
"MessageType",
"ModelClass",
"ProtocolError",
"SessionClose",
"SessionInit",
"TickMode",
"ToolCall",
"ToolClass",
"ToolError",
"ToolResult",
"WorldKind",
"tool",
]
+81 -4
View File
@@ -3,14 +3,16 @@
from __future__ import annotations
import asyncio
import inspect
import uuid
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any, Callable
from typing import Any
from aicc.protocol import (
PROTOCOL_VERSION,
Capabilities,
ErrorCode,
ErrorMessage,
ErrorPayload,
Heartbeat,
MessageType,
@@ -24,7 +26,6 @@ from aicc.protocol import (
ToolSpec,
WorldKind,
WorldSpec,
PROTOCOL_VERSION,
)
from aicc.tool import ToolDefinition, call_tool_impl
@@ -144,6 +145,32 @@ class Bridge:
if sess is not None:
sess.closed = True
def emit_event(
self,
session_id: str,
topic: str,
payload: dict[str, Any] | None = None,
*,
tick: int | None = None,
) -> None:
"""Queue an async event for delivery to a connected agent.
The transport is responsible for draining `session.send_queue` and
forwarding events to the client. Safe to call from tool handlers.
"""
sess = self._sessions.get(session_id)
if sess is None or sess.closed:
return
from aicc.protocol import EventMessage
event = EventMessage(
session_id=session_id,
topic=topic,
payload=payload or {},
meta={"tick": tick} if tick is not None else None,
)
sess.send_queue.put_nowait(event.model_dump(by_alias=True))
def session(self, session_id: str) -> Session:
sess = self._sessions.get(session_id)
if sess is None or sess.closed:
@@ -196,11 +223,48 @@ class Bridge:
May also push async events to the session queue.
"""
msg_type = raw.get("type")
# Protocol version check BEFORE any model parsing: a wrong version
# must be answered with protocol_mismatch, not a parse error.
proto = raw.get("protocol")
if proto is not None and proto != PROTOCOL_VERSION:
return ErrorMessage(
session_id=session_id,
error=ErrorPayload(
code=ErrorCode.PROTOCOL_MISMATCH,
message=(
f"Protocol mismatch: bridge='{PROTOCOL_VERSION}', "
f"message='{proto}'"
),
retryable=False,
),
).model_dump(by_alias=True)
if msg_type == MessageType.MANIFEST_REQUEST.value:
try:
sess = self.session(session_id)
except KeyError:
return self._session_expired(session_id)
return sess.manifest.model_dump(by_alias=True) if sess.manifest else None
if msg_type == MessageType.TOOL_CALL.value:
return await self._handle_tool_call(session_id, ToolCall.model_validate(raw))
try:
call = ToolCall.model_validate(raw)
except Exception: # noqa: BLE001 - malformed tool_call
return ToolResult(
session_id=session_id,
call_id=raw.get("call_id") or "",
ok=False,
error=ErrorPayload(
code=ErrorCode.INVALID_INPUT,
message="Malformed tool_call message.",
retryable=False,
),
).model_dump(by_alias=True)
try:
self.session(session_id)
except KeyError:
return self._session_expired(session_id)
return await self._handle_tool_call(session_id, call)
if msg_type == MessageType.SESSION_CLOSE.value:
self.close_session(session_id)
return SessionClose(
@@ -210,10 +274,23 @@ class Bridge:
if msg_type == MessageType.HEARTBEAT.value:
return Heartbeat(session_id=session_id).model_dump(by_alias=True)
if msg_type == MessageType.SESSION_RESUME.value:
try:
sess = self.session(raw.get("session_id") or session_id)
except KeyError:
return self._session_expired(session_id)
return sess.manifest.model_dump(by_alias=True) if sess.manifest else None
return None
def _session_expired(self, session_id: str) -> dict[str, Any]:
return ErrorMessage(
session_id=session_id,
error=ErrorPayload(
code=ErrorCode.SESSION_EXPIRED,
message=f"Session '{session_id}' is unknown or closed.",
retryable=False,
),
).model_dump(by_alias=True)
async def _handle_tool_call(self, session_id: str, call: ToolCall) -> dict[str, Any]:
defn = self._tools.get(call.tool)
if defn is None:
+9 -10
View File
@@ -4,11 +4,16 @@ from __future__ import annotations
import asyncio
import uuid
from typing import Any
from typing import Any, Self
from aicc.errors import (
AICCError,
ConnectionError_,
ProtocolError,
ToolError,
)
from aicc.protocol import (
PROTOCOL_VERSION,
ErrorCode,
EventMessage,
MessageType,
SessionClose,
@@ -16,12 +21,6 @@ from aicc.protocol import (
ToolCall,
ToolResult,
)
from aicc.errors import (
AICCError,
ConnectionError_,
ProtocolError,
ToolError,
)
from aicc.transport.base import Transport
@@ -56,14 +55,14 @@ class AICCClient:
# ---------- Lifecycle ----------
async def __aenter__(self) -> "AICCClient":
async def __aenter__(self) -> Self:
await self._transport.connect()
return self
async def __aexit__(self, exc_type, exc, tb) -> None:
await self.close()
async def connect(self) -> "AICCClient":
async def connect(self) -> AICCClient:
await self._transport.connect()
return self
+270
View File
@@ -0,0 +1,270 @@
"""AICC conformance test runner.
Loads declarative scenarios (see https://github.com/emil28092005/AICC-Protocol
conformance/scenarios/) and executes them against a bridge, reporting
pass/fail per scenario.
Usage:
from aicc.conformance import run_scenarios, summarize
results = run_scenarios(bridge, "/path/to/scenarios")
print(summarize(results))
"""
from __future__ import annotations
import asyncio
import json
import re
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from aicc.bridge import Bridge
from aicc.transport.in_process import InProcessTransport
UUID_RE = re.compile(
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
)
@dataclass
class ScenarioResult:
id: str
title: str
passed: bool
failures: list[str] = field(default_factory=list)
error: str | None = None
def _match_value(expect: Any, actual: Any, path: str, failures: list[str]) -> None:
"""Subset matcher with special `$` matchers for conformance assertions."""
if isinstance(expect, dict):
# Special matchers
if "$kind" in expect and len(expect) == 1:
kind = expect["$kind"]
ok = {
"uuid": isinstance(actual, str) and bool(UUID_RE.match(actual)),
"integer": isinstance(actual, int) and not isinstance(actual, bool),
"number": isinstance(actual, (int, float)) and not isinstance(actual, bool),
"string": isinstance(actual, str),
"boolean": isinstance(actual, bool),
"array": isinstance(actual, list),
"object": isinstance(actual, dict),
}.get(kind, False)
if not ok:
failures.append(f"{path}: expected {kind}, got {type(actual).__name__}")
return
if "$gt" in expect and len(expect) == 1:
if not (isinstance(actual, (int, float)) and actual > expect["$gt"]):
failures.append(f"{path}: expected > {expect['$gt']}, got {actual!r}")
return
if "$enum" in expect and len(expect) == 1:
if actual not in expect["$enum"]:
failures.append(f"{path}: expected one of {expect['$enum']}, got {actual!r}")
return
if "$required" in expect and len(expect) == 1:
if not isinstance(actual, dict):
failures.append(f"{path}: expected object, got {type(actual).__name__}")
return
for key in expect["$required"]:
if key not in actual:
failures.append(f"{path}: missing required key {key!r}")
return
if not isinstance(actual, dict):
failures.append(f"{path}: expected object, got {type(actual).__name__}")
return
for key, sub in expect.items():
if key not in actual:
failures.append(f"{path}: missing key {key!r}")
continue
_match_value(sub, actual[key], f"{path}.{key}", failures)
return
if isinstance(expect, list):
if not isinstance(actual, list):
failures.append(f"{path}: expected array, got {type(actual).__name__}")
return
if len(expect) != len(actual):
failures.append(f"{path}: expected array len {len(expect)}, got {len(actual)}")
return
for i, (sub, act) in enumerate(zip(expect, actual)):
_match_value(sub, act, f"{path}[{i}]", failures)
return
if expect != actual:
failures.append(f"{path}: expected {expect!r}, got {actual!r}")
def _validate_scenario(sc: dict[str, Any]) -> None:
if "id" not in sc:
raise ValueError("Scenario missing 'id'")
if "steps" not in sc or not isinstance(sc["steps"], list):
raise ValueError(f"Scenario {sc['id']} missing 'steps' list")
for step in sc["steps"]:
if "send" not in step and "expect" not in step:
raise ValueError(f"Scenario {sc['id']} has step without send/expect")
async def _run_scenario(bridge: Bridge, sc: dict[str, Any]) -> ScenarioResult:
result = ScenarioResult(id=sc["id"], title=sc.get("title", sc["id"]), passed=True)
try:
_validate_scenario(sc)
t = InProcessTransport.start(bridge)
async with t:
# Handshake: connect pushes session_init into the receive queue.
manifest_raw = await asyncio.wait_for(t.receive(), timeout=10.0)
if manifest_raw.get("type") != "session_init":
result.passed = False
result.error = (
f"expected session_init on connect, got {manifest_raw.get('type')!r}"
)
return result
session_id = manifest_raw["session_id"]
last_call_id: str | None = None
# Pending message already consumed from the wire (the handshake
# session_init). Scenarios that expect session_init as their
# first step consume it here instead of receiving again.
pending: dict[str, Any] | None = manifest_raw
for step in sc["steps"]:
if "send" in step:
msg = dict(step["send"])
msg.setdefault("session_id", session_id)
msg.setdefault("message_id", str(uuid.uuid4()))
if msg.get("type") == "tool_call":
last_call_id = msg.get("call_id") or f"tc_{uuid.uuid4().hex[:12]}"
msg["call_id"] = last_call_id
await t.send(msg)
if "expect" in step:
expected = step["expect"]
if pending is not None and expected.get("type") == "session_init":
raw = pending
pending = None
else:
raw = await asyncio.wait_for(t.receive(), timeout=10.0)
failures: list[str] = []
if (
expected.get("call_id_echo")
and last_call_id is not None
and raw.get("call_id") != last_call_id
):
failures.append(
f"call_id_echo: expected {last_call_id!r}, "
f"got {raw.get('call_id')!r}"
)
if expected.get("session_id_echo") and raw.get("session_id") != session_id:
failures.append(
f"session_id_echo: expected {session_id!r}, "
f"got {raw.get('session_id')!r}"
)
remaining = {
k: v
for k, v in expected.items()
if k not in ("call_id_echo", "session_id_echo")
}
_match_value(remaining, raw, "msg", failures)
if failures:
result.passed = False
result.failures.extend(failures)
return result
except Exception as exc: # noqa: BLE001
result.passed = False
result.error = f"{type(exc).__name__}: {exc}"
return result
def load_scenarios(scenario_dir: str | Path) -> list[dict[str, Any]]:
"""Load all *.json scenario files from a directory, sorted by id."""
d = Path(scenario_dir)
files = sorted(d.glob("*.json"))
scenarios = []
for f in files:
with open(f) as fh:
scenarios.append(json.load(fh))
return scenarios
async def run_scenarios(bridge: Bridge, scenario_dir: str | Path) -> list[ScenarioResult]:
"""Run all scenarios in a directory against a bridge. One session per scenario."""
results = []
for sc in load_scenarios(scenario_dir):
results.append(await _run_scenario(bridge, sc))
return results
def summarize(results: list[ScenarioResult]) -> str:
lines = []
passed = sum(1 for r in results if r.passed)
for r in results:
status = "PASS" if r.passed else "FAIL"
lines.append(f"[{status}] {r.id}: {r.title}")
for f in r.failures:
lines.append(f" - {f}")
if r.error:
lines.append(f" - error: {r.error}")
lines.append(f"\n{passed}/{len(results)} scenarios passed")
return "\n".join(lines)
def reference_bridge() -> Bridge:
"""Reference bridge exposing the tools the core scenarios expect.
Tools required by scenarios:
- echo: returns input value
- boom: raises -> execution_failed
- bump: returns ok and emits a collision event
"""
bridge = Bridge(name="conformance-reference", kind="3d", tick_rate_hz=10.0)
@bridge.tool(description="Echo a value back to the caller.")
async def echo(value: str = "") -> dict:
return {"value": value}
@bridge.tool(description="Always raises an exception.")
async def boom() -> dict:
raise RuntimeError("kaboom")
@bridge.tool(description="Returns ok and emits a collision event.")
async def bump() -> dict:
return {"bumped": True}
# Emit a collision event when bump completes. Hook into tool handling:
# after a successful bump call, push the event into the session queue.
_orig = bridge._handle_tool_call
async def _handle_with_event(session_id: str, call):
resp = await _orig(session_id, call)
if call.tool == "bump" and resp.get("ok"):
bridge.emit_event(
session_id,
"collision",
{"other": "wall", "impulse": 1.0},
tick=1,
)
return resp
bridge._handle_tool_call = _handle_with_event # type: ignore[method-assign]
return bridge
def main() -> None:
"""CLI entry: python -m aicc.conformance <scenarios-dir>"""
import sys
if len(sys.argv) < 2:
print("usage: python -m aicc.conformance <scenarios-dir>")
sys.exit(2)
bridge = reference_bridge()
results = asyncio.run(run_scenarios(bridge, sys.argv[1]))
print(summarize(results))
sys.exit(0 if all(r.passed for r in results) else 1)
if __name__ == "__main__":
main()
-1
View File
@@ -8,7 +8,6 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
PROTOCOL_VERSION = "aicc/0.1"
+2 -1
View File
@@ -3,7 +3,8 @@
from __future__ import annotations
import inspect
from typing import Any, Callable, get_args, get_origin, get_type_hints
from collections.abc import Callable
from typing import Any, get_args, get_origin, get_type_hints
from pydantic import BaseModel
+3 -3
View File
@@ -4,13 +4,13 @@ from __future__ import annotations
import asyncio
import inspect
from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from typing import Any
from aicc.protocol import ToolClass
from aicc.schema import function_schema, return_schema
ToolImpl = Callable[..., Any | Awaitable[Any]]
+1 -1
View File
@@ -3,7 +3,7 @@
from aicc.transport.base import Transport
from aicc.transport.in_process import InProcessTransport
__all__ = ["Transport", "InProcessTransport"]
__all__ = ["InProcessTransport", "Transport"]
try: # pragma: no cover
from aicc.transport.websocket import WebSocketClientTransport, WebSocketServer
+13 -6
View File
@@ -6,12 +6,11 @@ Useful for tests, embedded use, and single-process agents that drive the bridge
from __future__ import annotations
import asyncio
import uuid
from typing import Any
from typing import Any, Self
from aicc.bridge import Bridge
from aicc.errors import ConnectionError_
from aicc.protocol import PROTOCOL_VERSION, SessionInit
from aicc.protocol import SessionInit
class InProcessTransport:
@@ -35,7 +34,7 @@ class InProcessTransport:
self._connected = False
@classmethod
def start(cls, bridge: Bridge) -> "InProcessTransport":
def start(cls, bridge: Bridge) -> InProcessTransport:
t = cls()
t._bridge = bridge
return t
@@ -63,12 +62,20 @@ class InProcessTransport:
sid = message.get("session_id") or self._session_id
msg_type = message.get("type")
# session_resume carries the session_id; bind it
if msg_type == "session_resume" and not self._bridge._sessions.get(sid): # noqa: SLF001
if msg_type == "session_resume" and not self._bridge._sessions.get(sid):
# Auto-open if bridge forgot
pass
response = await self._bridge.handle_message(sid, message)
if response is not None:
await self._recv_q.put(response)
# Drain any events the bridge queued during handling (e.g. from
# tool handlers calling bridge.emit_event). Deliver them after the
# response so the client sees a coherent request -> response order.
sess = self._bridge._sessions.get(sid)
if sess is not None:
while not sess.send_queue.empty():
event = sess.send_queue.get_nowait()
await self._recv_q.put(event)
async def receive(self) -> dict[str, Any]:
if not self._connected:
@@ -79,7 +86,7 @@ class InProcessTransport:
"""Bridge-side helper to push an event to the connected client."""
await self._recv_q.put(event)
async def __aenter__(self) -> "InProcessTransport":
async def __aenter__(self) -> Self:
await self.connect()
return self
+3 -4
View File
@@ -2,9 +2,8 @@
from __future__ import annotations
import asyncio
import json
from typing import Any
from typing import Any, Self
from aicc.bridge import Bridge
from aicc.errors import ConnectionError_
@@ -67,7 +66,7 @@ class WebSocketServer:
self._server = None # type: ignore[var-annotated]
self._connections: set[Any] = set()
async def __aenter__(self) -> "WebSocketServer":
async def __aenter__(self) -> Self:
await self.start()
return self
@@ -111,6 +110,6 @@ class WebSocketServer:
for ws in list(self._connections):
try:
await ws.close()
except Exception: # noqa: BLE001
except Exception: # noqa: BLE001, S110 - best-effort shutdown
pass
self._connections.clear()
+33
View File
@@ -0,0 +1,33 @@
"""Conformance runner tests: runs the declarative scenarios from the spec repo."""
from __future__ import annotations
from pathlib import Path
import pytest
from aicc.conformance import reference_bridge, run_scenarios, summarize
SCENARIOS = Path(__file__).resolve().parents[1] / ".." / "aicc-spec" / "conformance" / "scenarios"
@pytest.mark.asyncio
async def test_core_scenarios_pass():
if not SCENARIOS.exists():
pytest.skip("AICC-Protocol conformance scenarios not found on this machine")
bridge = reference_bridge()
results = await run_scenarios(bridge, SCENARIOS)
report = summarize(results)
assert all(r.passed for r in results), report
@pytest.mark.asyncio
async def test_runner_rejects_unknown_tool():
"""A bridge missing a required tool fails the corresponding scenario."""
from aicc import Bridge
bridge = Bridge(name="incomplete")
results = await run_scenarios(bridge, SCENARIOS)
by_id = {r.id: r for r in results}
# echo tool missing -> core-03 must fail
assert not by_id["core-03-tool-call-ok"].passed
-2
View File
@@ -2,8 +2,6 @@
from __future__ import annotations
import asyncio
import pytest
from aicc import AICCClient, Bridge, ToolClass
-2
View File
@@ -2,8 +2,6 @@
from __future__ import annotations
import asyncio
import pytest
from aicc import AICCClient, Bridge