feat: conformance test runner + protocol fixes
- conformance.py: declarative scenario runner (subset matcher, $kind/$gt/ $enum/$required assertions, call_id/session_id echo checks), reference bridge, CLI entry (python -m aicc.conformance) - bridge.py: protocol version check before parsing (protocol_mismatch), session expiry handling (session_expired), malformed tool_call -> invalid_input, emit_event() for async events - client.py: Self return type - in_process.py: drain bridge event queue after response (event delivery), Self return type - websocket.py: Self return type, best-effort shutdown - tests: 14 passing (incl. conformance scenarios, 9/9 core scenarios) - ruff: all checks pass
This commit is contained in:
@@ -93,12 +93,33 @@ aicc/
|
|||||||
bridge.py # Bridge — environment side, tool registration
|
bridge.py # Bridge — environment side, tool registration
|
||||||
tool.py # @tool decorator and tool metadata
|
tool.py # @tool decorator and tool metadata
|
||||||
schema.py # JSON Schema generation utilities
|
schema.py # JSON Schema generation utilities
|
||||||
|
conformance.py # declarative scenario runner (AICC conformance tests)
|
||||||
transport/
|
transport/
|
||||||
base.py # Transport interface
|
base.py # Transport interface
|
||||||
in_process.py # In-process transport (tests, embedded)
|
in_process.py # In-process transport (tests, embedded)
|
||||||
websocket.py # WebSocket transport (client + server)
|
websocket.py # WebSocket transport (client + server)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Conformance
|
||||||
|
|
||||||
|
`aicc-py` ships the reference conformance runner for the AICC protocol.
|
||||||
|
It executes the declarative scenarios from the
|
||||||
|
[AICC-Protocol](https://github.com/emil28092005/AICC-Protocol)
|
||||||
|
`conformance/scenarios/` directory against any bridge:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m aicc.conformance ~/AICC-Protocol/conformance/scenarios
|
||||||
|
```
|
||||||
|
|
||||||
|
Or via pytest (scenarios are loaded automatically if the spec repo is
|
||||||
|
checked out next to `aicc-py`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pytest tests/test_conformance.py
|
||||||
|
```
|
||||||
|
|
||||||
|
A bridge is AICC-conformant for `aicc/0.1` when all core scenarios pass.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
+32
-32
@@ -1,45 +1,45 @@
|
|||||||
"""aicc-py: Python SDK for the AI-Controlled Character Protocol."""
|
"""aicc-py: Python SDK for the AI-Controlled Character Protocol."""
|
||||||
|
|
||||||
from aicc.protocol import (
|
|
||||||
ErrorCode,
|
|
||||||
MessageType,
|
|
||||||
TickMode,
|
|
||||||
WorldKind,
|
|
||||||
ModelClass,
|
|
||||||
ToolClass,
|
|
||||||
SessionInit,
|
|
||||||
SessionClose,
|
|
||||||
ToolCall,
|
|
||||||
ToolResult,
|
|
||||||
EventMessage,
|
|
||||||
ErrorMessage,
|
|
||||||
Heartbeat,
|
|
||||||
)
|
|
||||||
from aicc.errors import AICCError, ToolError, CapabilityError, ProtocolError
|
|
||||||
from aicc.bridge import Bridge
|
from aicc.bridge import Bridge
|
||||||
from aicc.client import AICCClient
|
from aicc.client import AICCClient
|
||||||
|
from aicc.errors import AICCError, CapabilityError, ProtocolError, ToolError
|
||||||
|
from aicc.protocol import (
|
||||||
|
ErrorCode,
|
||||||
|
ErrorMessage,
|
||||||
|
EventMessage,
|
||||||
|
Heartbeat,
|
||||||
|
MessageType,
|
||||||
|
ModelClass,
|
||||||
|
SessionClose,
|
||||||
|
SessionInit,
|
||||||
|
TickMode,
|
||||||
|
ToolCall,
|
||||||
|
ToolClass,
|
||||||
|
ToolResult,
|
||||||
|
WorldKind,
|
||||||
|
)
|
||||||
from aicc.tool import tool
|
from aicc.tool import tool
|
||||||
|
|
||||||
__version__ = "0.1.0"
|
__version__ = "0.1.0"
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Bridge",
|
|
||||||
"AICCClient",
|
"AICCClient",
|
||||||
"tool",
|
|
||||||
"ErrorCode",
|
|
||||||
"MessageType",
|
|
||||||
"TickMode",
|
|
||||||
"WorldKind",
|
|
||||||
"ModelClass",
|
|
||||||
"ToolClass",
|
|
||||||
"SessionInit",
|
|
||||||
"SessionClose",
|
|
||||||
"ToolCall",
|
|
||||||
"ToolResult",
|
|
||||||
"EventMessage",
|
|
||||||
"ErrorMessage",
|
|
||||||
"Heartbeat",
|
|
||||||
"AICCError",
|
"AICCError",
|
||||||
"ToolError",
|
"Bridge",
|
||||||
"CapabilityError",
|
"CapabilityError",
|
||||||
|
"ErrorCode",
|
||||||
|
"ErrorMessage",
|
||||||
|
"EventMessage",
|
||||||
|
"Heartbeat",
|
||||||
|
"MessageType",
|
||||||
|
"ModelClass",
|
||||||
"ProtocolError",
|
"ProtocolError",
|
||||||
|
"SessionClose",
|
||||||
|
"SessionInit",
|
||||||
|
"TickMode",
|
||||||
|
"ToolCall",
|
||||||
|
"ToolClass",
|
||||||
|
"ToolError",
|
||||||
|
"ToolResult",
|
||||||
|
"WorldKind",
|
||||||
|
"tool",
|
||||||
]
|
]
|
||||||
|
|||||||
+83
-6
@@ -3,14 +3,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import inspect
|
|
||||||
import uuid
|
import uuid
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, Callable
|
from typing import Any
|
||||||
|
|
||||||
from aicc.protocol import (
|
from aicc.protocol import (
|
||||||
|
PROTOCOL_VERSION,
|
||||||
Capabilities,
|
Capabilities,
|
||||||
ErrorCode,
|
ErrorCode,
|
||||||
|
ErrorMessage,
|
||||||
ErrorPayload,
|
ErrorPayload,
|
||||||
Heartbeat,
|
Heartbeat,
|
||||||
MessageType,
|
MessageType,
|
||||||
@@ -24,7 +26,6 @@ from aicc.protocol import (
|
|||||||
ToolSpec,
|
ToolSpec,
|
||||||
WorldKind,
|
WorldKind,
|
||||||
WorldSpec,
|
WorldSpec,
|
||||||
PROTOCOL_VERSION,
|
|
||||||
)
|
)
|
||||||
from aicc.tool import ToolDefinition, call_tool_impl
|
from aicc.tool import ToolDefinition, call_tool_impl
|
||||||
|
|
||||||
@@ -144,6 +145,32 @@ class Bridge:
|
|||||||
if sess is not None:
|
if sess is not None:
|
||||||
sess.closed = True
|
sess.closed = True
|
||||||
|
|
||||||
|
def emit_event(
|
||||||
|
self,
|
||||||
|
session_id: str,
|
||||||
|
topic: str,
|
||||||
|
payload: dict[str, Any] | None = None,
|
||||||
|
*,
|
||||||
|
tick: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Queue an async event for delivery to a connected agent.
|
||||||
|
|
||||||
|
The transport is responsible for draining `session.send_queue` and
|
||||||
|
forwarding events to the client. Safe to call from tool handlers.
|
||||||
|
"""
|
||||||
|
sess = self._sessions.get(session_id)
|
||||||
|
if sess is None or sess.closed:
|
||||||
|
return
|
||||||
|
from aicc.protocol import EventMessage
|
||||||
|
|
||||||
|
event = EventMessage(
|
||||||
|
session_id=session_id,
|
||||||
|
topic=topic,
|
||||||
|
payload=payload or {},
|
||||||
|
meta={"tick": tick} if tick is not None else None,
|
||||||
|
)
|
||||||
|
sess.send_queue.put_nowait(event.model_dump(by_alias=True))
|
||||||
|
|
||||||
def session(self, session_id: str) -> Session:
|
def session(self, session_id: str) -> Session:
|
||||||
sess = self._sessions.get(session_id)
|
sess = self._sessions.get(session_id)
|
||||||
if sess is None or sess.closed:
|
if sess is None or sess.closed:
|
||||||
@@ -196,11 +223,48 @@ class Bridge:
|
|||||||
May also push async events to the session queue.
|
May also push async events to the session queue.
|
||||||
"""
|
"""
|
||||||
msg_type = raw.get("type")
|
msg_type = raw.get("type")
|
||||||
|
|
||||||
|
# Protocol version check BEFORE any model parsing: a wrong version
|
||||||
|
# must be answered with protocol_mismatch, not a parse error.
|
||||||
|
proto = raw.get("protocol")
|
||||||
|
if proto is not None and proto != PROTOCOL_VERSION:
|
||||||
|
return ErrorMessage(
|
||||||
|
session_id=session_id,
|
||||||
|
error=ErrorPayload(
|
||||||
|
code=ErrorCode.PROTOCOL_MISMATCH,
|
||||||
|
message=(
|
||||||
|
f"Protocol mismatch: bridge='{PROTOCOL_VERSION}', "
|
||||||
|
f"message='{proto}'"
|
||||||
|
),
|
||||||
|
retryable=False,
|
||||||
|
),
|
||||||
|
).model_dump(by_alias=True)
|
||||||
|
|
||||||
if msg_type == MessageType.MANIFEST_REQUEST.value:
|
if msg_type == MessageType.MANIFEST_REQUEST.value:
|
||||||
sess = self.session(session_id)
|
try:
|
||||||
|
sess = self.session(session_id)
|
||||||
|
except KeyError:
|
||||||
|
return self._session_expired(session_id)
|
||||||
return sess.manifest.model_dump(by_alias=True) if sess.manifest else None
|
return sess.manifest.model_dump(by_alias=True) if sess.manifest else None
|
||||||
if msg_type == MessageType.TOOL_CALL.value:
|
if msg_type == MessageType.TOOL_CALL.value:
|
||||||
return await self._handle_tool_call(session_id, ToolCall.model_validate(raw))
|
try:
|
||||||
|
call = ToolCall.model_validate(raw)
|
||||||
|
except Exception: # noqa: BLE001 - malformed tool_call
|
||||||
|
return ToolResult(
|
||||||
|
session_id=session_id,
|
||||||
|
call_id=raw.get("call_id") or "",
|
||||||
|
ok=False,
|
||||||
|
error=ErrorPayload(
|
||||||
|
code=ErrorCode.INVALID_INPUT,
|
||||||
|
message="Malformed tool_call message.",
|
||||||
|
retryable=False,
|
||||||
|
),
|
||||||
|
).model_dump(by_alias=True)
|
||||||
|
try:
|
||||||
|
self.session(session_id)
|
||||||
|
except KeyError:
|
||||||
|
return self._session_expired(session_id)
|
||||||
|
return await self._handle_tool_call(session_id, call)
|
||||||
if msg_type == MessageType.SESSION_CLOSE.value:
|
if msg_type == MessageType.SESSION_CLOSE.value:
|
||||||
self.close_session(session_id)
|
self.close_session(session_id)
|
||||||
return SessionClose(
|
return SessionClose(
|
||||||
@@ -210,10 +274,23 @@ class Bridge:
|
|||||||
if msg_type == MessageType.HEARTBEAT.value:
|
if msg_type == MessageType.HEARTBEAT.value:
|
||||||
return Heartbeat(session_id=session_id).model_dump(by_alias=True)
|
return Heartbeat(session_id=session_id).model_dump(by_alias=True)
|
||||||
if msg_type == MessageType.SESSION_RESUME.value:
|
if msg_type == MessageType.SESSION_RESUME.value:
|
||||||
sess = self.session(raw.get("session_id") or session_id)
|
try:
|
||||||
|
sess = self.session(raw.get("session_id") or session_id)
|
||||||
|
except KeyError:
|
||||||
|
return self._session_expired(session_id)
|
||||||
return sess.manifest.model_dump(by_alias=True) if sess.manifest else None
|
return sess.manifest.model_dump(by_alias=True) if sess.manifest else None
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _session_expired(self, session_id: str) -> dict[str, Any]:
|
||||||
|
return ErrorMessage(
|
||||||
|
session_id=session_id,
|
||||||
|
error=ErrorPayload(
|
||||||
|
code=ErrorCode.SESSION_EXPIRED,
|
||||||
|
message=f"Session '{session_id}' is unknown or closed.",
|
||||||
|
retryable=False,
|
||||||
|
),
|
||||||
|
).model_dump(by_alias=True)
|
||||||
|
|
||||||
async def _handle_tool_call(self, session_id: str, call: ToolCall) -> dict[str, Any]:
|
async def _handle_tool_call(self, session_id: str, call: ToolCall) -> dict[str, Any]:
|
||||||
defn = self._tools.get(call.tool)
|
defn = self._tools.get(call.tool)
|
||||||
if defn is None:
|
if defn is None:
|
||||||
|
|||||||
+9
-10
@@ -4,11 +4,16 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Any
|
from typing import Any, Self
|
||||||
|
|
||||||
|
from aicc.errors import (
|
||||||
|
AICCError,
|
||||||
|
ConnectionError_,
|
||||||
|
ProtocolError,
|
||||||
|
ToolError,
|
||||||
|
)
|
||||||
from aicc.protocol import (
|
from aicc.protocol import (
|
||||||
PROTOCOL_VERSION,
|
PROTOCOL_VERSION,
|
||||||
ErrorCode,
|
|
||||||
EventMessage,
|
EventMessage,
|
||||||
MessageType,
|
MessageType,
|
||||||
SessionClose,
|
SessionClose,
|
||||||
@@ -16,12 +21,6 @@ from aicc.protocol import (
|
|||||||
ToolCall,
|
ToolCall,
|
||||||
ToolResult,
|
ToolResult,
|
||||||
)
|
)
|
||||||
from aicc.errors import (
|
|
||||||
AICCError,
|
|
||||||
ConnectionError_,
|
|
||||||
ProtocolError,
|
|
||||||
ToolError,
|
|
||||||
)
|
|
||||||
from aicc.transport.base import Transport
|
from aicc.transport.base import Transport
|
||||||
|
|
||||||
|
|
||||||
@@ -56,14 +55,14 @@ class AICCClient:
|
|||||||
|
|
||||||
# ---------- Lifecycle ----------
|
# ---------- Lifecycle ----------
|
||||||
|
|
||||||
async def __aenter__(self) -> "AICCClient":
|
async def __aenter__(self) -> Self:
|
||||||
await self._transport.connect()
|
await self._transport.connect()
|
||||||
return self
|
return self
|
||||||
|
|
||||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||||
await self.close()
|
await self.close()
|
||||||
|
|
||||||
async def connect(self) -> "AICCClient":
|
async def connect(self) -> AICCClient:
|
||||||
await self._transport.connect()
|
await self._transport.connect()
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,270 @@
|
|||||||
|
"""AICC conformance test runner.
|
||||||
|
|
||||||
|
Loads declarative scenarios (see https://github.com/emil28092005/AICC-Protocol
|
||||||
|
conformance/scenarios/) and executes them against a bridge, reporting
|
||||||
|
pass/fail per scenario.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
from aicc.conformance import run_scenarios, summarize
|
||||||
|
results = run_scenarios(bridge, "/path/to/scenarios")
|
||||||
|
print(summarize(results))
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from aicc.bridge import Bridge
|
||||||
|
from aicc.transport.in_process import InProcessTransport
|
||||||
|
|
||||||
|
UUID_RE = re.compile(
|
||||||
|
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ScenarioResult:
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
passed: bool
|
||||||
|
failures: list[str] = field(default_factory=list)
|
||||||
|
error: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _match_value(expect: Any, actual: Any, path: str, failures: list[str]) -> None:
|
||||||
|
"""Subset matcher with special `$` matchers for conformance assertions."""
|
||||||
|
if isinstance(expect, dict):
|
||||||
|
# Special matchers
|
||||||
|
if "$kind" in expect and len(expect) == 1:
|
||||||
|
kind = expect["$kind"]
|
||||||
|
ok = {
|
||||||
|
"uuid": isinstance(actual, str) and bool(UUID_RE.match(actual)),
|
||||||
|
"integer": isinstance(actual, int) and not isinstance(actual, bool),
|
||||||
|
"number": isinstance(actual, (int, float)) and not isinstance(actual, bool),
|
||||||
|
"string": isinstance(actual, str),
|
||||||
|
"boolean": isinstance(actual, bool),
|
||||||
|
"array": isinstance(actual, list),
|
||||||
|
"object": isinstance(actual, dict),
|
||||||
|
}.get(kind, False)
|
||||||
|
if not ok:
|
||||||
|
failures.append(f"{path}: expected {kind}, got {type(actual).__name__}")
|
||||||
|
return
|
||||||
|
if "$gt" in expect and len(expect) == 1:
|
||||||
|
if not (isinstance(actual, (int, float)) and actual > expect["$gt"]):
|
||||||
|
failures.append(f"{path}: expected > {expect['$gt']}, got {actual!r}")
|
||||||
|
return
|
||||||
|
if "$enum" in expect and len(expect) == 1:
|
||||||
|
if actual not in expect["$enum"]:
|
||||||
|
failures.append(f"{path}: expected one of {expect['$enum']}, got {actual!r}")
|
||||||
|
return
|
||||||
|
if "$required" in expect and len(expect) == 1:
|
||||||
|
if not isinstance(actual, dict):
|
||||||
|
failures.append(f"{path}: expected object, got {type(actual).__name__}")
|
||||||
|
return
|
||||||
|
for key in expect["$required"]:
|
||||||
|
if key not in actual:
|
||||||
|
failures.append(f"{path}: missing required key {key!r}")
|
||||||
|
return
|
||||||
|
if not isinstance(actual, dict):
|
||||||
|
failures.append(f"{path}: expected object, got {type(actual).__name__}")
|
||||||
|
return
|
||||||
|
for key, sub in expect.items():
|
||||||
|
if key not in actual:
|
||||||
|
failures.append(f"{path}: missing key {key!r}")
|
||||||
|
continue
|
||||||
|
_match_value(sub, actual[key], f"{path}.{key}", failures)
|
||||||
|
return
|
||||||
|
|
||||||
|
if isinstance(expect, list):
|
||||||
|
if not isinstance(actual, list):
|
||||||
|
failures.append(f"{path}: expected array, got {type(actual).__name__}")
|
||||||
|
return
|
||||||
|
if len(expect) != len(actual):
|
||||||
|
failures.append(f"{path}: expected array len {len(expect)}, got {len(actual)}")
|
||||||
|
return
|
||||||
|
for i, (sub, act) in enumerate(zip(expect, actual)):
|
||||||
|
_match_value(sub, act, f"{path}[{i}]", failures)
|
||||||
|
return
|
||||||
|
|
||||||
|
if expect != actual:
|
||||||
|
failures.append(f"{path}: expected {expect!r}, got {actual!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_scenario(sc: dict[str, Any]) -> None:
|
||||||
|
if "id" not in sc:
|
||||||
|
raise ValueError("Scenario missing 'id'")
|
||||||
|
if "steps" not in sc or not isinstance(sc["steps"], list):
|
||||||
|
raise ValueError(f"Scenario {sc['id']} missing 'steps' list")
|
||||||
|
for step in sc["steps"]:
|
||||||
|
if "send" not in step and "expect" not in step:
|
||||||
|
raise ValueError(f"Scenario {sc['id']} has step without send/expect")
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_scenario(bridge: Bridge, sc: dict[str, Any]) -> ScenarioResult:
|
||||||
|
result = ScenarioResult(id=sc["id"], title=sc.get("title", sc["id"]), passed=True)
|
||||||
|
try:
|
||||||
|
_validate_scenario(sc)
|
||||||
|
t = InProcessTransport.start(bridge)
|
||||||
|
async with t:
|
||||||
|
# Handshake: connect pushes session_init into the receive queue.
|
||||||
|
manifest_raw = await asyncio.wait_for(t.receive(), timeout=10.0)
|
||||||
|
if manifest_raw.get("type") != "session_init":
|
||||||
|
result.passed = False
|
||||||
|
result.error = (
|
||||||
|
f"expected session_init on connect, got {manifest_raw.get('type')!r}"
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
session_id = manifest_raw["session_id"]
|
||||||
|
last_call_id: str | None = None
|
||||||
|
# Pending message already consumed from the wire (the handshake
|
||||||
|
# session_init). Scenarios that expect session_init as their
|
||||||
|
# first step consume it here instead of receiving again.
|
||||||
|
pending: dict[str, Any] | None = manifest_raw
|
||||||
|
|
||||||
|
for step in sc["steps"]:
|
||||||
|
if "send" in step:
|
||||||
|
msg = dict(step["send"])
|
||||||
|
msg.setdefault("session_id", session_id)
|
||||||
|
msg.setdefault("message_id", str(uuid.uuid4()))
|
||||||
|
if msg.get("type") == "tool_call":
|
||||||
|
last_call_id = msg.get("call_id") or f"tc_{uuid.uuid4().hex[:12]}"
|
||||||
|
msg["call_id"] = last_call_id
|
||||||
|
await t.send(msg)
|
||||||
|
|
||||||
|
if "expect" in step:
|
||||||
|
expected = step["expect"]
|
||||||
|
if pending is not None and expected.get("type") == "session_init":
|
||||||
|
raw = pending
|
||||||
|
pending = None
|
||||||
|
else:
|
||||||
|
raw = await asyncio.wait_for(t.receive(), timeout=10.0)
|
||||||
|
failures: list[str] = []
|
||||||
|
|
||||||
|
if (
|
||||||
|
expected.get("call_id_echo")
|
||||||
|
and last_call_id is not None
|
||||||
|
and raw.get("call_id") != last_call_id
|
||||||
|
):
|
||||||
|
failures.append(
|
||||||
|
f"call_id_echo: expected {last_call_id!r}, "
|
||||||
|
f"got {raw.get('call_id')!r}"
|
||||||
|
)
|
||||||
|
if expected.get("session_id_echo") and raw.get("session_id") != session_id:
|
||||||
|
failures.append(
|
||||||
|
f"session_id_echo: expected {session_id!r}, "
|
||||||
|
f"got {raw.get('session_id')!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
remaining = {
|
||||||
|
k: v
|
||||||
|
for k, v in expected.items()
|
||||||
|
if k not in ("call_id_echo", "session_id_echo")
|
||||||
|
}
|
||||||
|
_match_value(remaining, raw, "msg", failures)
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
result.passed = False
|
||||||
|
result.failures.extend(failures)
|
||||||
|
return result
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
result.passed = False
|
||||||
|
result.error = f"{type(exc).__name__}: {exc}"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def load_scenarios(scenario_dir: str | Path) -> list[dict[str, Any]]:
|
||||||
|
"""Load all *.json scenario files from a directory, sorted by id."""
|
||||||
|
d = Path(scenario_dir)
|
||||||
|
files = sorted(d.glob("*.json"))
|
||||||
|
scenarios = []
|
||||||
|
for f in files:
|
||||||
|
with open(f) as fh:
|
||||||
|
scenarios.append(json.load(fh))
|
||||||
|
return scenarios
|
||||||
|
|
||||||
|
|
||||||
|
async def run_scenarios(bridge: Bridge, scenario_dir: str | Path) -> list[ScenarioResult]:
|
||||||
|
"""Run all scenarios in a directory against a bridge. One session per scenario."""
|
||||||
|
results = []
|
||||||
|
for sc in load_scenarios(scenario_dir):
|
||||||
|
results.append(await _run_scenario(bridge, sc))
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def summarize(results: list[ScenarioResult]) -> str:
|
||||||
|
lines = []
|
||||||
|
passed = sum(1 for r in results if r.passed)
|
||||||
|
for r in results:
|
||||||
|
status = "PASS" if r.passed else "FAIL"
|
||||||
|
lines.append(f"[{status}] {r.id}: {r.title}")
|
||||||
|
for f in r.failures:
|
||||||
|
lines.append(f" - {f}")
|
||||||
|
if r.error:
|
||||||
|
lines.append(f" - error: {r.error}")
|
||||||
|
lines.append(f"\n{passed}/{len(results)} scenarios passed")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def reference_bridge() -> Bridge:
|
||||||
|
"""Reference bridge exposing the tools the core scenarios expect.
|
||||||
|
|
||||||
|
Tools required by scenarios:
|
||||||
|
- echo: returns input value
|
||||||
|
- boom: raises -> execution_failed
|
||||||
|
- bump: returns ok and emits a collision event
|
||||||
|
"""
|
||||||
|
bridge = Bridge(name="conformance-reference", kind="3d", tick_rate_hz=10.0)
|
||||||
|
|
||||||
|
@bridge.tool(description="Echo a value back to the caller.")
|
||||||
|
async def echo(value: str = "") -> dict:
|
||||||
|
return {"value": value}
|
||||||
|
|
||||||
|
@bridge.tool(description="Always raises an exception.")
|
||||||
|
async def boom() -> dict:
|
||||||
|
raise RuntimeError("kaboom")
|
||||||
|
|
||||||
|
@bridge.tool(description="Returns ok and emits a collision event.")
|
||||||
|
async def bump() -> dict:
|
||||||
|
return {"bumped": True}
|
||||||
|
|
||||||
|
# Emit a collision event when bump completes. Hook into tool handling:
|
||||||
|
# after a successful bump call, push the event into the session queue.
|
||||||
|
_orig = bridge._handle_tool_call
|
||||||
|
|
||||||
|
async def _handle_with_event(session_id: str, call):
|
||||||
|
resp = await _orig(session_id, call)
|
||||||
|
if call.tool == "bump" and resp.get("ok"):
|
||||||
|
bridge.emit_event(
|
||||||
|
session_id,
|
||||||
|
"collision",
|
||||||
|
{"other": "wall", "impulse": 1.0},
|
||||||
|
tick=1,
|
||||||
|
)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
bridge._handle_tool_call = _handle_with_event # type: ignore[method-assign]
|
||||||
|
return bridge
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""CLI entry: python -m aicc.conformance <scenarios-dir>"""
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("usage: python -m aicc.conformance <scenarios-dir>")
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
bridge = reference_bridge()
|
||||||
|
results = asyncio.run(run_scenarios(bridge, sys.argv[1]))
|
||||||
|
print(summarize(results))
|
||||||
|
sys.exit(0 if all(r.passed for r in results) else 1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -8,7 +8,6 @@ from typing import Any, Literal
|
|||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
|
||||||
PROTOCOL_VERSION = "aicc/0.1"
|
PROTOCOL_VERSION = "aicc/0.1"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+2
-1
@@ -3,7 +3,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import inspect
|
import inspect
|
||||||
from typing import Any, Callable, get_args, get_origin, get_type_hints
|
from collections.abc import Callable
|
||||||
|
from typing import Any, get_args, get_origin, get_type_hints
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -4,13 +4,13 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import inspect
|
import inspect
|
||||||
from dataclasses import dataclass, field
|
from collections.abc import Awaitable, Callable
|
||||||
from typing import Any, Awaitable, Callable
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from aicc.protocol import ToolClass
|
from aicc.protocol import ToolClass
|
||||||
from aicc.schema import function_schema, return_schema
|
from aicc.schema import function_schema, return_schema
|
||||||
|
|
||||||
|
|
||||||
ToolImpl = Callable[..., Any | Awaitable[Any]]
|
ToolImpl = Callable[..., Any | Awaitable[Any]]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
from aicc.transport.base import Transport
|
from aicc.transport.base import Transport
|
||||||
from aicc.transport.in_process import InProcessTransport
|
from aicc.transport.in_process import InProcessTransport
|
||||||
|
|
||||||
__all__ = ["Transport", "InProcessTransport"]
|
__all__ = ["InProcessTransport", "Transport"]
|
||||||
|
|
||||||
try: # pragma: no cover
|
try: # pragma: no cover
|
||||||
from aicc.transport.websocket import WebSocketClientTransport, WebSocketServer
|
from aicc.transport.websocket import WebSocketClientTransport, WebSocketServer
|
||||||
|
|||||||
@@ -6,12 +6,11 @@ Useful for tests, embedded use, and single-process agents that drive the bridge
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import uuid
|
from typing import Any, Self
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from aicc.bridge import Bridge
|
from aicc.bridge import Bridge
|
||||||
from aicc.errors import ConnectionError_
|
from aicc.errors import ConnectionError_
|
||||||
from aicc.protocol import PROTOCOL_VERSION, SessionInit
|
from aicc.protocol import SessionInit
|
||||||
|
|
||||||
|
|
||||||
class InProcessTransport:
|
class InProcessTransport:
|
||||||
@@ -35,7 +34,7 @@ class InProcessTransport:
|
|||||||
self._connected = False
|
self._connected = False
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def start(cls, bridge: Bridge) -> "InProcessTransport":
|
def start(cls, bridge: Bridge) -> InProcessTransport:
|
||||||
t = cls()
|
t = cls()
|
||||||
t._bridge = bridge
|
t._bridge = bridge
|
||||||
return t
|
return t
|
||||||
@@ -63,12 +62,20 @@ class InProcessTransport:
|
|||||||
sid = message.get("session_id") or self._session_id
|
sid = message.get("session_id") or self._session_id
|
||||||
msg_type = message.get("type")
|
msg_type = message.get("type")
|
||||||
# session_resume carries the session_id; bind it
|
# session_resume carries the session_id; bind it
|
||||||
if msg_type == "session_resume" and not self._bridge._sessions.get(sid): # noqa: SLF001
|
if msg_type == "session_resume" and not self._bridge._sessions.get(sid):
|
||||||
# Auto-open if bridge forgot
|
# Auto-open if bridge forgot
|
||||||
pass
|
pass
|
||||||
response = await self._bridge.handle_message(sid, message)
|
response = await self._bridge.handle_message(sid, message)
|
||||||
if response is not None:
|
if response is not None:
|
||||||
await self._recv_q.put(response)
|
await self._recv_q.put(response)
|
||||||
|
# Drain any events the bridge queued during handling (e.g. from
|
||||||
|
# tool handlers calling bridge.emit_event). Deliver them after the
|
||||||
|
# response so the client sees a coherent request -> response order.
|
||||||
|
sess = self._bridge._sessions.get(sid)
|
||||||
|
if sess is not None:
|
||||||
|
while not sess.send_queue.empty():
|
||||||
|
event = sess.send_queue.get_nowait()
|
||||||
|
await self._recv_q.put(event)
|
||||||
|
|
||||||
async def receive(self) -> dict[str, Any]:
|
async def receive(self) -> dict[str, Any]:
|
||||||
if not self._connected:
|
if not self._connected:
|
||||||
@@ -79,7 +86,7 @@ class InProcessTransport:
|
|||||||
"""Bridge-side helper to push an event to the connected client."""
|
"""Bridge-side helper to push an event to the connected client."""
|
||||||
await self._recv_q.put(event)
|
await self._recv_q.put(event)
|
||||||
|
|
||||||
async def __aenter__(self) -> "InProcessTransport":
|
async def __aenter__(self) -> Self:
|
||||||
await self.connect()
|
await self.connect()
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,8 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import json
|
import json
|
||||||
from typing import Any
|
from typing import Any, Self
|
||||||
|
|
||||||
from aicc.bridge import Bridge
|
from aicc.bridge import Bridge
|
||||||
from aicc.errors import ConnectionError_
|
from aicc.errors import ConnectionError_
|
||||||
@@ -67,7 +66,7 @@ class WebSocketServer:
|
|||||||
self._server = None # type: ignore[var-annotated]
|
self._server = None # type: ignore[var-annotated]
|
||||||
self._connections: set[Any] = set()
|
self._connections: set[Any] = set()
|
||||||
|
|
||||||
async def __aenter__(self) -> "WebSocketServer":
|
async def __aenter__(self) -> Self:
|
||||||
await self.start()
|
await self.start()
|
||||||
return self
|
return self
|
||||||
|
|
||||||
@@ -111,6 +110,6 @@ class WebSocketServer:
|
|||||||
for ws in list(self._connections):
|
for ws in list(self._connections):
|
||||||
try:
|
try:
|
||||||
await ws.close()
|
await ws.close()
|
||||||
except Exception: # noqa: BLE001
|
except Exception: # noqa: BLE001, S110 - best-effort shutdown
|
||||||
pass
|
pass
|
||||||
self._connections.clear()
|
self._connections.clear()
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Conformance runner tests: runs the declarative scenarios from the spec repo."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from aicc.conformance import reference_bridge, run_scenarios, summarize
|
||||||
|
|
||||||
|
SCENARIOS = Path(__file__).resolve().parents[1] / ".." / "aicc-spec" / "conformance" / "scenarios"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_core_scenarios_pass():
|
||||||
|
if not SCENARIOS.exists():
|
||||||
|
pytest.skip("AICC-Protocol conformance scenarios not found on this machine")
|
||||||
|
bridge = reference_bridge()
|
||||||
|
results = await run_scenarios(bridge, SCENARIOS)
|
||||||
|
report = summarize(results)
|
||||||
|
assert all(r.passed for r in results), report
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runner_rejects_unknown_tool():
|
||||||
|
"""A bridge missing a required tool fails the corresponding scenario."""
|
||||||
|
from aicc import Bridge
|
||||||
|
|
||||||
|
bridge = Bridge(name="incomplete")
|
||||||
|
results = await run_scenarios(bridge, SCENARIOS)
|
||||||
|
by_id = {r.id: r for r in results}
|
||||||
|
# echo tool missing -> core-03 must fail
|
||||||
|
assert not by_id["core-03-tool-call-ok"].passed
|
||||||
@@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from aicc import AICCClient, Bridge, ToolClass
|
from aicc import AICCClient, Bridge, ToolClass
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from aicc import AICCClient, Bridge
|
from aicc import AICCClient, Bridge
|
||||||
|
|||||||
Reference in New Issue
Block a user