Add workload SDK foundation

This commit is contained in:
Emil
2026-08-01 23:22:20 +03:00
parent 11e9333033
commit c43af32495
32 changed files with 12256 additions and 20 deletions
+664
View File
@@ -0,0 +1,664 @@
"""Compatibility tests for the built-in SDK bridge and scientific reference."""
from __future__ import annotations
import csv
from dataclasses import replace
from pathlib import Path
from uuid import uuid4
import pytest
from scimesh.chemistry.dataset import find_molecule_by_id
from scimesh.sdk import (
ArtifactCollection,
ArtifactSchema,
CheckpointPolicy,
CompatibilityError,
ComponentRef,
DeterminismProfile,
FeatureRequirement,
GangSpec,
JobRequest,
LocalArtifactStore,
LocalCoreBatchExecutor,
LocalPlanningContext,
NetworkPolicy,
PortRef,
ProcessModel,
RetryPolicy,
SchemaRef,
StageKind,
TrustMode,
VerificationDecision,
VerificationStatus,
VersionRange,
WorkloadDefinition,
WorkloadRegistry,
assert_manifest_round_trip,
default_sdk_registry,
default_sdk_runtime,
similarity_search_sdk_adapter,
)
from scimesh.workloads.similarity_search import search_similar, write_search_results
def _write_tiny_dataset(path: Path) -> None:
path.write_text(
"chembl_id\tcanonical_smiles\textra\n"
"QUERY\tCCO\tquery\n"
"ALCOHOL\tCCCO\talcohol\n"
"ALKANE\tCCCC\talkane\n"
"BROKEN\tnot-a-smiles\tinvalid\n"
"DUPLICATE\tCCO\tduplicate\n"
"AMINE\tCCN\tamine\n",
encoding="utf-8",
)
def _registered_similarity_search(shard_rows: int = 2):
registry = default_sdk_registry(shard_rows=shard_rows)
runtime = default_sdk_runtime()
descriptions = registry.descriptions()
assert len(descriptions) == 1
description = descriptions[0]
definition, negotiated = registry.require(
description.workload.name,
description.workload.version,
description.package_digest,
runtime=runtime,
)
return registry, runtime, description, definition, negotiated
def _request_for(
dataset: Path,
artifact_store: LocalArtifactStore,
definition: WorkloadDefinition,
) -> JobRequest:
input_port = definition.manifest.inputs["input"]
dataset_artifact = artifact_store.import_file(
dataset,
declaration=input_port.schema,
)
return JobRequest(
workload=definition.manifest.workload,
parameters={"query_id": "QUERY", "top_k": 3, "progress_every": 0},
inputs={"input": ArtifactCollection.single(dataset_artifact)},
)
def test_builtin_similarity_search_manifest_is_registered_and_negotiable() -> None:
_, _, description, definition, negotiated = _registered_similarity_search()
manifest = definition.manifest
assert description.enabled is True
assert manifest.workload.name == "similarity-search"
assert manifest.workload.version == "1.0.0"
assert manifest.determinism is DeterminismProfile.BYTE_EXACT
assert manifest.conformance_profiles == ("core-batch-v1",)
assert [stage.kind for stage in manifest.workflow.stages] == [
StageKind.MAP,
StageKind.REDUCE,
]
assert set(definition.runners) == {manifest.workflow.stages[0].entry_point}
assert set(definition.reducers) == {manifest.workflow.stages[1].entry_point}
assert negotiated is not None
assert negotiated.manifest == manifest
assert_manifest_round_trip(manifest)
def test_local_sdk_executor_matches_similarity_search_reference(tmp_path: Path) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
registry, runtime, description, definition, _ = _registered_similarity_search()
artifact_store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, artifact_store, definition)
result = LocalCoreBatchExecutor(
registry,
runtime,
artifact_store,
tmp_path / "sdk-work",
).execute(request, description.package_digest)
result_artifact = result.outputs["result"].items[0].artifact
reference_path = tmp_path / "reference.csv"
query = find_molecule_by_id(dataset, "QUERY")
reference = search_similar(dataset, query, top_k=3, progress_every=0)
write_search_results(reference_path, reference.matches)
assert artifact_store.materialize(result_artifact).read_bytes() == reference_path.read_bytes()
assert result.task_key == "reduce/final"
assert result.metrics == {"matches_emitted": 3, "partial_count": 3}
def test_legacy_adapter_planning_is_deterministic_ordered_and_path_free(
tmp_path: Path,
) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
registry, runtime, description, definition, _ = _registered_similarity_search()
artifact_store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, artifact_store, definition)
input_artifact = request.inputs["input"].items[0].artifact
first = registry.plan(
request,
description.package_digest,
runtime,
LocalPlanningContext(
artifact_store,
artifact_store,
tmp_path / "first-plan",
allowed_artifacts=(input_artifact,),
),
)
second = registry.plan(
request,
description.package_digest,
runtime,
LocalPlanningContext(
artifact_store,
artifact_store,
tmp_path / "second-plan",
allowed_artifacts=(input_artifact,),
),
)
assert first.to_json() == second.to_json()
assert first.digest == second.digest
assert first.package_digest == definition.manifest.package.digest
assert first.manifest_digest == definition.manifest.digest
assert first.trust_mode is request.trust_mode
assert JobRequest.from_json(request.to_json()) == request
assert [task.task_key for task in first.tasks] == [
"map/00000000",
"map/00000001",
"map/00000002",
]
assert all(task.stage_id == "map" for task in first.tasks)
assert all(task.package_digest == first.package_digest for task in first.tasks)
assert all(task.manifest_digest == first.manifest_digest for task in first.tasks)
assert all(task.trust_mode is first.trust_mode for task in first.tasks)
assert all("query_id" not in task.parameters for task in first.tasks)
assert all(task.parameters["query_smiles"] == "CCO" for task in first.tasks)
shard_ids: list[list[str]] = []
for task in first.tasks:
artifact = task.inputs["input"].items[0].artifact
with artifact_store.materialize(artifact).open(encoding="utf-8", newline="") as source:
shard_ids.append(
[row["chembl_id"] for row in csv.DictReader(source, delimiter="\t")]
)
assert set(artifact.to_dict()) == {
"artifact_id",
"sha256",
"schema",
"media_type",
"size_bytes",
"records",
"dimensions",
}
assert shard_ids == [
["QUERY", "ALCOHOL"],
["ALKANE", "BROKEN"],
["DUPLICATE", "AMINE"],
]
wire_payload = first.to_json()
assert str(tmp_path) not in wire_payload
assert "file://" not in wire_payload
assert "worker://" not in wire_payload
assert "workspace" not in wire_payload
def test_local_context_sink_cannot_seal_files_outside_the_attempt(
tmp_path: Path,
) -> None:
_, _, _, definition, _ = _registered_similarity_search()
artifact_store = LocalArtifactStore(tmp_path / "artifacts")
workspace = tmp_path / "attempt"
context = LocalPlanningContext(artifact_store, artifact_store, workspace)
outside = tmp_path / "private.txt"
outside.write_text("private", encoding="utf-8")
schema = definition.manifest.inputs["input"].schema
with pytest.raises(ValueError, match="inside its workspace"):
context.sink.seal(outside, declaration=schema)
workspace.mkdir(parents=True, exist_ok=True)
link = workspace / "result"
link.symlink_to(outside)
with pytest.raises(ValueError, match="real workspace directories"):
context.sink.seal(link, declaration=schema)
def test_local_store_rejects_malformed_content_before_publishing(
tmp_path: Path,
) -> None:
malformed = tmp_path / "malformed.json"
malformed.write_text('{"unfinished":', encoding="utf-8")
declaration = ArtifactSchema(
SchemaRef("json-result", 1),
"application/json",
"utf-8",
max_bytes=1_024,
validator=ComponentRef("json-document", 1),
)
store = LocalArtifactStore(tmp_path / "artifacts")
with pytest.raises(ValueError, match="not a valid bounded document"):
store.import_file(malformed, declaration=declaration)
assert tuple(path for path in store.root.iterdir() if not path.name.startswith(".seal-")) == ()
def test_delimited_validator_rejects_headerless_data_and_enforces_record_limit(
tmp_path: Path,
) -> None:
declaration = ArtifactSchema(
SchemaRef("bounded-table", 1),
"text/csv",
"utf-8",
max_bytes=1_024,
validator=ComponentRef("delimited-table", 1),
validator_configuration={"columns": ["value"]},
max_records=1,
)
store = LocalArtifactStore(tmp_path / "artifacts")
headerless = tmp_path / "headerless.csv"
headerless.write_text("1\n2\n", encoding="utf-8")
oversized = tmp_path / "oversized.csv"
oversized.write_text("value\n1\n2\n", encoding="utf-8")
with pytest.raises(ValueError, match="header does not match"):
store.import_file(headerless, declaration=declaration)
with pytest.raises(ValueError, match="record limit"):
store.import_file(oversized, declaration=declaration)
def test_custom_artifact_inspector_is_bound_to_schema_and_validator_identity(
tmp_path: Path,
) -> None:
schema_ref = SchemaRef("matrix-result", 1)
validator = ComponentRef("matrix-inspector", 1)
declaration = ArtifactSchema(
schema_ref,
"application/x-matrix",
None,
max_bytes=1_024,
validator=validator,
validator_configuration={"layout": "row-major"},
max_records=1,
max_dimensions=(2, 2),
)
source = tmp_path / "matrix.bin"
source.write_bytes(b"matrix")
wrong = LocalArtifactStore(
tmp_path / "wrong-store",
inspectors={
schema_ref.canonical: (
ComponentRef("other-inspector", 1),
lambda _path, _configuration: (1, (2, 2)),
)
},
)
with pytest.raises(ValueError, match="no matching registered validator"):
wrong.import_file(source, declaration=declaration)
def inspect(_path: Path, configuration):
assert dict(configuration) == {"layout": "row-major"}
return 1, (2, 2)
store = LocalArtifactStore(
tmp_path / "store",
inspectors={schema_ref.canonical: (validator, inspect)},
)
artifact = store.import_file(source, declaration=declaration)
assert artifact.records == 1
assert artifact.dimensions == (2, 2)
@pytest.mark.parametrize(
("forgery", "message"),
(
("artifact", "artifacts sealed by its attempt"),
("provenance", "provenance does not match"),
),
)
def test_local_executor_rejects_handler_forged_outputs(
tmp_path: Path,
forgery: str,
message: str,
) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
_, runtime, description, original, _ = _registered_similarity_search()
map_stage = next(stage for stage in original.manifest.workflow.stages if stage.kind is StageKind.MAP)
inner = original.runners[map_stage.entry_point]
class ForgingRunner:
def run(self, context):
result = inner.run(context)
if forgery == "provenance":
forged = replace(
result.provenance,
worker_runtime={"kind": "forged-runtime"},
)
return replace(result, provenance=forged)
original_ref = result.outputs["partial"].items[0].artifact
forged_ref = replace(original_ref, artifact_id=str(uuid4()))
return replace(
result,
outputs={"partial": ArtifactCollection.single(forged_ref)},
)
definition = WorkloadDefinition(
original.manifest,
original.planner,
{map_stage.entry_point: ForgingRunner()},
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, store, definition)
with pytest.raises(ValueError, match=message):
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
description.package_digest,
)
def test_local_executor_rejects_profiles_that_claim_network_isolation(tmp_path: Path) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
_, runtime, description, original, _ = _registered_similarity_search()
stages = tuple(
replace(stage, execution=replace(stage.execution, network=NetworkPolicy.NONE))
for stage in original.manifest.workflow.stages
)
workflow = replace(original.manifest.workflow, stages=stages)
manifest = replace(original.manifest, workflow=workflow)
definition = WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, store, definition)
with pytest.raises(CompatibilityError) as raised:
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
description.package_digest,
)
assert raised.value.code == "feature-undeclared"
def test_local_executor_rejects_aliased_terminal_outputs_before_planning(
tmp_path: Path,
) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
_, runtime, description, original, _ = _registered_similarity_search()
reducer = next(
stage for stage in original.manifest.workflow.stages if stage.kind is StageKind.REDUCE
)
internal_name = next(iter(reducer.outputs))
workflow = replace(
original.manifest.workflow,
outputs={"aliased": PortRef(internal_name, reducer.stage_id)},
)
manifest = replace(
original.manifest,
workflow=workflow,
outputs={"aliased": reducer.outputs[internal_name]},
)
definition = WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, store, definition)
with pytest.raises(ValueError, match="identity-mapped reducer outputs"):
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
description.package_digest,
)
def test_local_executor_rejects_non_trusted_trust_modes(tmp_path: Path) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
_, runtime, _, original, _ = _registered_similarity_search()
stages = tuple(
replace(stage, trust_modes=("trusted", "verified"))
for stage in original.manifest.workflow.stages
)
manifest = replace(
original.manifest,
workflow=replace(original.manifest.workflow, stages=stages),
trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED),
)
definition = WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = replace(
_request_for(dataset, store, definition),
trust_mode=TrustMode.VERIFIED,
)
runtime = replace(runtime, trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED))
with pytest.raises(ValueError, match="supports only trusted workloads"):
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
manifest.package.digest,
)
def _advanced_execution_manifest(
original: WorkloadDefinition,
case: str,
) -> tuple[tuple[str, ...], object]:
"""Declare one negotiable advanced profile the local executor cannot enforce."""
stages = original.manifest.workflow.stages
if case == "process-pool":
features = ("process-pools", "multi-process")
changed = tuple(
replace(
stage,
resources=replace(stage.resources, cpu_cores=2),
execution=replace(
stage.execution,
process_model=ProcessModel.PROCESS_POOL,
max_processes=2,
),
)
for stage in stages
)
elif case == "checkpoints":
features = ("checkpoints",)
changed = tuple(
replace(
stage,
execution=replace(
stage.execution,
checkpoint=CheckpointPolicy(
enabled=True,
schema=SchemaRef("task-state", 1),
compatibility_version=1,
),
),
)
for stage in stages
)
elif case == "retries":
features = ("retries",)
changed = tuple(
replace(stage, retry=RetryPolicy(max_attempts=2))
for stage in stages
)
elif case == "secrets":
features = ("secret-injection",)
changed = tuple(
replace(
stage,
execution=replace(stage.execution, secret_handles=("db-credential",)),
)
for stage in stages
)
elif case == "gang":
features = ("gang-leases",)
changed = tuple(
replace(
stage,
gang=GangSpec(replicas=2, per_replica_resources=stage.resources),
)
for stage in stages
)
elif case == "network-isolation":
features = ("network-isolation",)
changed = tuple(
replace(
stage,
execution=replace(stage.execution, network=NetworkPolicy.NONE),
)
for stage in stages
)
else:
assert case == "service-stage"
features = ("services",)
changed = (replace(stages[0], kind=StageKind.SERVICE),) + stages[1:]
manifest = replace(
original.manifest,
workflow=replace(original.manifest.workflow, stages=changed),
required_features=original.manifest.required_features
+ tuple(
FeatureRequirement(feature, VersionRange(">=1,<2")) for feature in features
),
)
return features, manifest
@pytest.mark.parametrize(
("case", "message"),
(
("process-pool", "one non-nested host thread"),
("checkpoints", "cannot enforce this stage profile"),
("retries", "does not implement retries"),
("secrets", "cannot enforce this stage profile"),
("gang", "cannot enforce this stage profile"),
("network-isolation", "cannot enforce a restricted network policy"),
("service-stage", "does not implement advanced stages"),
),
)
def test_local_executor_rejects_profiles_it_cannot_enforce(
tmp_path: Path,
case: str,
message: str,
) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
_, runtime, _, original, _ = _registered_similarity_search()
features, manifest = _advanced_execution_manifest(original, case)
definition = WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, store, definition)
runtime = replace(
runtime,
features={**runtime.features, **{feature: "1.0.0" for feature in features}},
)
with pytest.raises(ValueError, match=message):
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
manifest.package.digest,
)
def test_local_executor_fails_when_the_declared_verifier_rejects(tmp_path: Path) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
_, runtime, _, original, _ = _registered_similarity_search()
class RejectingVerifier:
identity = ComponentRef("exact-artifact", 1)
def verify(self, context, candidates):
return VerificationDecision(
VerificationStatus.REJECTED,
self.identity,
"forced-rejection",
{},
)
definition = WorkloadDefinition(
original.manifest,
original.planner,
original.runners,
original.reducers,
{ComponentRef("exact-artifact", 1).canonical: RejectingVerifier()},
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, store, definition)
with pytest.raises(ValueError, match="did not pass its declared verifier"):
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
original.manifest.package.digest,
)
def test_local_executor_enforces_the_declared_output_byte_budget(tmp_path: Path) -> None:
dataset = tmp_path / "molecules.tsv"
_write_tiny_dataset(dataset)
runtime = default_sdk_runtime()
adapter = similarity_search_sdk_adapter(shard_rows=2)
# The planner pins its own manifest into every task, so the budget cut must
# be applied to the adapter's manifest for plan and definition to agree.
adapter.manifest = replace(
adapter.manifest,
workflow=replace(adapter.manifest.workflow, max_output_bytes=256),
limits=replace(adapter.manifest.limits, max_output_bytes=256),
)
definition = adapter.definition()
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
store = LocalArtifactStore(tmp_path / "artifacts")
request = _request_for(dataset, store, definition)
with pytest.raises(ValueError, match="bytes exceed their sink limit"):
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
request,
definition.manifest.package.digest,
)
File diff suppressed because it is too large Load Diff
+609
View File
@@ -0,0 +1,609 @@
"""Security and version-pinning tests for the installed SDK registry."""
from __future__ import annotations
from dataclasses import replace
from importlib import metadata
from pathlib import Path
import py_compile
from types import SimpleNamespace
import pytest
from scimesh.sdk import (
AllowedPackage,
ArtifactCollection,
ArtifactRef,
CompatibilityError,
FeatureRequirement,
JobRequest,
LocalArtifactStore,
LocalPlanningContext,
PackageSpec,
TrustMode,
VersionRange,
WorkloadDefinition,
WorkloadId,
WorkloadRegistry,
current_scimesh_package_digest,
default_sdk_runtime,
installed_distribution_digest,
similarity_search_sdk_adapter,
)
from scimesh.sdk.schema import (
ParameterValidationError,
validate_parameter_instance,
validate_schema_definition,
)
def _definition(*, version: str = "1.0.0", digest_character: str = "a") -> WorkloadDefinition:
original = similarity_search_sdk_adapter(shard_rows=2).definition()
manifest = replace(
original.manifest,
workload=WorkloadId("similarity-search", version),
package=PackageSpec("scimesh", "sha256:" + digest_character * 64),
)
return WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
def test_registry_requires_an_explicit_enabled_version_and_digest() -> None:
first = _definition(version="1.0.0", digest_character="a")
second = _definition(version="2.0.0", digest_character="b")
registry = WorkloadRegistry()
registry.register(first, enabled=True)
registry.register(second)
resolved, _ = registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
assert resolved is first
with pytest.raises(ValueError, match="unknown workload version"):
registry.require("similarity-search", "3.0.0", "sha256:" + "a" * 64)
with pytest.raises(ValueError, match="not enabled"):
registry.require("similarity-search", "2.0.0", "sha256:" + "b" * 64)
with pytest.raises(ValueError, match="not enabled"):
registry.require("similarity-search", "1.0.0", "sha256:" + "c" * 64)
with pytest.raises(ValueError, match="already registered"):
registry.register(first)
registry.enable("similarity-search", "2.0.0", "sha256:" + "b" * 64)
assert [item.workload.version for item in registry.descriptions()] == ["1.0.0", "2.0.0"]
def test_compatibility_failure_occurs_before_planner_invocation(
tmp_path: Path,
) -> None:
original = similarity_search_sdk_adapter(shard_rows=2).definition()
class CountingPlanner:
calls = 0
def validate(self, request):
self.calls += 1
return original.planner.validate(request)
def plan(self, job, context):
self.calls += 1
return original.planner.plan(job, context)
planner = CountingPlanner()
definition = WorkloadDefinition(
original.manifest,
planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
input_port = definition.manifest.inputs["input"]
artifact = ArtifactRef(
"11111111-1111-4111-8111-111111111111",
"a" * 64,
input_port.schema.ref,
input_port.schema.media_type,
1,
)
request = JobRequest(
definition.manifest.workload,
{"query_smiles": "CCO"},
{"input": ArtifactCollection.single(artifact)},
)
incompatible = replace(default_sdk_runtime(), protocol_version="2.0.0")
store = LocalArtifactStore(tmp_path / "artifacts")
with pytest.raises(CompatibilityError) as raised:
registry.plan(
request,
definition.manifest.package.digest,
incompatible,
LocalPlanningContext(store, store, tmp_path / "plan"),
)
assert raised.value.code == "protocol-mismatch"
assert planner.calls == 0
@pytest.mark.parametrize(
("request_changes", "error_code"),
(
({"required_features": ("undeclared-feature",)}, "feature-undeclared"),
({"trust_mode": TrustMode.VERIFIED}, "trust-mode-undeclared"),
),
)
def test_job_selected_features_and_trust_mode_fail_closed_before_planning(
tmp_path: Path,
request_changes: dict[str, object],
error_code: str,
) -> None:
definition = similarity_search_sdk_adapter(shard_rows=2).definition()
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
input_port = definition.manifest.inputs["input"]
artifact = ArtifactRef(
"11111111-1111-4111-8111-111111111111",
"a" * 64,
input_port.schema.ref,
input_port.schema.media_type,
1,
records=1,
)
values: dict[str, object] = {
"workload": definition.manifest.workload,
"parameters": {"query_smiles": "CCO"},
"inputs": {"input": ArtifactCollection.single(artifact)},
}
values.update(request_changes)
request = JobRequest(**values) # type: ignore[arg-type]
store = LocalArtifactStore(tmp_path / "artifacts")
with pytest.raises(CompatibilityError) as raised:
registry.plan(
request,
definition.manifest.package.digest,
default_sdk_runtime(),
LocalPlanningContext(store, store, tmp_path / "plan"),
)
assert raised.value.code == error_code
class _EntryPoints(tuple):
def select(self, *, group: str):
assert group == WorkloadRegistry.ENTRY_POINT_GROUP
return self
def test_discovery_imports_only_an_exact_allowlisted_installed_entry_point(
monkeypatch: pytest.MonkeyPatch,
) -> None:
definition = similarity_search_sdk_adapter().definition()
loaded: list[str] = []
class EntryPoint:
def __init__(self, name: str, distribution: str) -> None:
self.name = name
self.dist = (
metadata.distribution("scimesh")
if distribution == "scimesh"
else SimpleNamespace(name=distribution)
)
self.value = "scimesh.sdk.builtins:similarity_search_sdk_adapter"
@property
def module(self) -> str:
return self.value.partition(":")[0]
def load(self):
loaded.append(self.name)
return lambda: definition
monkeypatch.setattr(
"scimesh.sdk.registry.metadata.entry_points",
lambda: _EntryPoints(
(
EntryPoint("evil-workload@1.0.0", "unapproved"),
EntryPoint("similarity-search@1.0.0", "scimesh"),
)
),
)
monkeypatch.setattr(
"scimesh.sdk.registry.installed_distribution_digest",
lambda _distribution: definition.manifest.package.digest,
)
registry = WorkloadRegistry()
registry.discover_installed(
(
AllowedPackage(
"scimesh",
definition.manifest.workload,
definition.manifest.package.digest,
),
)
)
assert loaded == ["similarity-search@1.0.0"]
assert registry.descriptions()[0].enabled
def test_discovery_measures_package_before_importing_entry_point(
monkeypatch: pytest.MonkeyPatch,
) -> None:
definition = similarity_search_sdk_adapter().definition()
loaded = False
class EntryPoint:
name = "similarity-search@1.0.0"
dist = metadata.distribution("scimesh")
value = "scimesh.sdk.builtins:similarity_search_sdk_adapter"
module = "scimesh.sdk.builtins"
def load(self):
nonlocal loaded
loaded = True
return lambda: definition
monkeypatch.setattr(
"scimesh.sdk.registry.metadata.entry_points",
lambda: _EntryPoints((EntryPoint(),)),
)
monkeypatch.setattr(
"scimesh.sdk.registry.installed_distribution_digest",
lambda _distribution: "sha256:" + "f" * 64,
)
with pytest.raises(ValueError, match="content does not match"):
WorkloadRegistry().discover_installed(
(
AllowedPackage(
"scimesh",
definition.manifest.workload,
definition.manifest.package.digest,
),
)
)
assert loaded is False
def test_installed_digest_is_stable_when_python_generates_a_pycache(
tmp_path: Path,
) -> None:
package = tmp_path / "fixture_pkg"
package.mkdir()
source = package / "__init__.py"
source.write_text("VALUE = 1\n", encoding="utf-8")
class FixtureDistribution:
name = "fixture-dist"
files = (Path("fixture_pkg/__init__.py"),)
entry_points = ()
@staticmethod
def read_text(name: str) -> str | None:
return "fixture_pkg\n" if name == "top_level.txt" else None
@staticmethod
def locate_file(value: object) -> Path:
return tmp_path / str(value)
distribution = FixtureDistribution()
before = installed_distribution_digest(distribution) # type: ignore[arg-type]
py_compile.compile(str(source), doraise=True)
assert installed_distribution_digest(distribution) == before # type: ignore[arg-type]
def test_discovery_rejects_entry_point_module_owned_by_another_distribution(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
owned = tmp_path / "owned_pkg"
owned.mkdir()
(owned / "__init__.py").write_text("", encoding="utf-8")
loaded = False
class Distribution:
name = "allowed-dist"
files = (Path("owned_pkg/__init__.py"),)
entry_points = ()
@staticmethod
def read_text(name: str) -> str | None:
return "owned_pkg\n" if name == "top_level.txt" else None
@staticmethod
def locate_file(value: object) -> Path:
return tmp_path / str(value)
class EntryPoint:
name = "similarity-search@1.0.0"
dist = Distribution()
value = "foreign_pkg.workload:factory"
module = "foreign_pkg.workload"
def load(self):
nonlocal loaded
loaded = True
raise AssertionError("foreign entry point must not load")
monkeypatch.setattr(
"scimesh.sdk.registry.metadata.entry_points",
lambda: _EntryPoints((EntryPoint(),)),
)
definition = similarity_search_sdk_adapter().definition()
with pytest.raises(ValueError, match="outside its distribution"):
WorkloadRegistry().discover_installed(
(
AllowedPackage(
"allowed-dist",
definition.manifest.workload,
"sha256:" + "a" * 64,
),
)
)
assert loaded is False
def test_missing_allowlisted_entry_point_fails_without_loading_or_registering(
monkeypatch: pytest.MonkeyPatch,
) -> None:
loaded: list[str] = []
class EntryPoint:
name = "job-selected-module@1.0.0"
dist = SimpleNamespace(name="unapproved")
def load(self):
loaded.append(self.name)
raise AssertionError("unapproved entry point must not load")
monkeypatch.setattr(
"scimesh.sdk.registry.metadata.entry_points",
lambda: _EntryPoints((EntryPoint(),)),
)
registry = WorkloadRegistry()
with pytest.raises(ValueError, match="were not installed"):
registry.discover_installed(
(
AllowedPackage(
"scimesh",
WorkloadId("similarity-search", "1.0.0"),
current_scimesh_package_digest(),
),
)
)
assert loaded == []
assert registry.descriptions() == ()
def test_parameter_schema_accepts_finite_big_integer_bounds() -> None:
bound = 10**400
schema = {"type": "integer", "minimum": -bound, "maximum": bound}
validate_schema_definition(schema)
validate_parameter_instance(bound, schema)
with pytest.raises(ParameterValidationError, match="violates maximum"):
validate_parameter_instance(bound + 1, schema)
def test_job_parameters_reject_unbounded_json_integers_early() -> None:
with pytest.raises(ValueError, match="4096-bit JSON bound"):
JobRequest(
WorkloadId("similarity-search", "1.0.0"),
{"value": 10**2_000},
{},
)
@pytest.mark.parametrize(
("value", "multiple", "accepted"),
[
(3 * 10**400, 3, True),
(10**400, 3, False),
(10**400, 0.1, True),
(0.3, 0.1, True),
(0.31, 0.1, False),
],
)
def test_parameter_schema_multiple_of_is_exact_without_float_overflow(
value: int | float,
multiple: int | float,
accepted: bool,
) -> None:
schema = {"type": "number", "multipleOf": multiple}
validate_schema_definition(schema)
if accepted:
validate_parameter_instance(value, schema)
else:
with pytest.raises(ParameterValidationError, match="violates multipleOf"):
validate_parameter_instance(value, schema)
def test_parameter_schema_equality_uses_json_types() -> None:
validate_schema_definition({"enum": [True, 1]})
with pytest.raises(ValueError, match="enum values must be unique"):
validate_schema_definition({"enum": [1, 1.0]})
validate_parameter_instance(True, {"enum": [True]})
with pytest.raises(ParameterValidationError, match="outside enum"):
validate_parameter_instance(1, {"enum": [True]})
validate_parameter_instance(1.0, {"enum": [1]})
validate_parameter_instance({"enabled": True}, {"const": {"enabled": True}})
with pytest.raises(ParameterValidationError, match="does not match const"):
validate_parameter_instance({"enabled": 1}, {"const": {"enabled": True}})
unique = {"type": "array", "uniqueItems": True}
validate_parameter_instance([True, 1, {"enabled": True}, {"enabled": 1}], unique)
with pytest.raises(ParameterValidationError, match="items must be unique"):
validate_parameter_instance([1, 1.0], unique)
def test_disabled_workload_is_not_resolvable_until_re_enabled() -> None:
definition = _definition()
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
resolved, _ = registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
assert resolved is definition
registry.disable("similarity-search", "1.0.0", "sha256:" + "a" * 64)
with pytest.raises(ValueError, match="not enabled"):
registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
registry.enable("similarity-search", "1.0.0", "sha256:" + "a" * 64)
resolved, _ = registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
assert resolved is definition
def test_discovery_rechecks_the_package_digest_after_loading(
monkeypatch: pytest.MonkeyPatch,
) -> None:
definition = similarity_search_sdk_adapter().definition()
digests = iter((definition.manifest.package.digest, "sha256:" + "e" * 64))
class EntryPoint:
name = "similarity-search@1.0.0"
dist = metadata.distribution("scimesh")
value = "scimesh.sdk.builtins:similarity_search_sdk_adapter"
module = "scimesh.sdk.builtins"
def load(self):
return lambda: definition
monkeypatch.setattr(
"scimesh.sdk.registry.metadata.entry_points",
lambda: _EntryPoints((EntryPoint(),)),
)
monkeypatch.setattr(
"scimesh.sdk.registry.installed_distribution_digest",
lambda _distribution: next(digests),
)
registry = WorkloadRegistry()
with pytest.raises(ValueError, match="changed while loading"):
registry.discover_installed(
(
AllowedPackage(
"scimesh",
definition.manifest.workload,
definition.manifest.package.digest,
),
)
)
assert registry.descriptions() == ()
def test_request_trust_mode_must_be_enforceable_by_runtime_and_stages(tmp_path: Path) -> None:
original = similarity_search_sdk_adapter(shard_rows=2).definition()
manifest = replace(
original.manifest,
trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED),
)
definition = WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
input_port = definition.manifest.inputs["input"]
artifact = ArtifactRef(
"11111111-1111-4111-8111-111111111111",
"a" * 64,
input_port.schema.ref,
input_port.schema.media_type,
1,
records=1,
)
request = JobRequest(
definition.manifest.workload,
{"query_smiles": "CCO"},
{"input": ArtifactCollection.single(artifact)},
trust_mode=TrustMode.VERIFIED,
)
store = LocalArtifactStore(tmp_path / "artifacts")
with pytest.raises(CompatibilityError) as raised:
registry.plan(
request,
manifest.package.digest,
default_sdk_runtime(),
LocalPlanningContext(store, store, tmp_path / "runtime-plan"),
)
assert raised.value.code == "trust-mode-unavailable"
runtime = replace(
default_sdk_runtime(),
trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED),
)
with pytest.raises(CompatibilityError) as raised:
registry.plan(
request,
manifest.package.digest,
runtime,
LocalPlanningContext(store, store, tmp_path / "stage-plan"),
)
assert raised.value.code == "stage-trust-unavailable"
def test_job_cannot_require_a_feature_outside_the_runtime(tmp_path: Path) -> None:
original = similarity_search_sdk_adapter(shard_rows=2).definition()
manifest = replace(
original.manifest,
optional_features=(
FeatureRequirement("gpu-fastpath", VersionRange(">=1,<2"), "cpu-fallback"),
),
)
definition = WorkloadDefinition(
manifest,
original.planner,
original.runners,
original.reducers,
original.verifiers,
)
registry = WorkloadRegistry()
registry.register(definition, enabled=True)
input_port = definition.manifest.inputs["input"]
artifact = ArtifactRef(
"11111111-1111-4111-8111-111111111111",
"a" * 64,
input_port.schema.ref,
input_port.schema.media_type,
1,
records=1,
)
request = JobRequest(
definition.manifest.workload,
{"query_smiles": "CCO"},
{"input": ArtifactCollection.single(artifact)},
required_features=("gpu-fastpath",),
)
negotiated = registry.require(
"similarity-search",
"1.0.0",
manifest.package.digest,
runtime=default_sdk_runtime(),
)[1]
assert negotiated is not None
assert negotiated.optional_fallbacks == {"gpu-fastpath": "cpu-fallback"}
with pytest.raises(CompatibilityError) as raised:
registry.plan(
request,
manifest.package.digest,
default_sdk_runtime(),
LocalPlanningContext(
LocalArtifactStore(tmp_path / "artifacts"),
LocalArtifactStore(tmp_path / "artifacts"),
tmp_path / "plan",
),
)
assert raised.value.code == "feature-unavailable"
+227
View File
@@ -0,0 +1,227 @@
"""Resource inventory and atomic local allocation tests for the SDK Agent layer."""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from threading import Barrier
import pytest
from scimesh.sdk import (
AcceleratorDevice,
AcceleratorMode,
ResourceInventory,
ResourcePool,
ResourceRequirements,
ResourceUnavailableError,
)
ENVIRONMENT_DIGEST = "sha256:" + "d" * 64
def gpu(device_id: str, *, topology_group: str = "socket-0") -> AcceleratorDevice:
return AcceleratorDevice(
kind="gpu",
vendor="nvidia",
device_id=device_id,
model="Test GPU",
memory_mb=16_384,
modes=(AcceleratorMode.EXCLUSIVE_DEVICE,),
capabilities={"compute": "9.0", "driver": "test"},
topology_group=topology_group,
)
def cpu_requirements(*, cpu_cores: int = 1, memory_mb: int = 256) -> ResourceRequirements:
return ResourceRequirements(
profile="cpu-v1",
cpu_cores=cpu_cores,
memory_mb=memory_mb,
scratch_mb=128,
architecture="x86-64",
environment_digest=ENVIRONMENT_DIGEST,
max_duration_seconds=120,
)
def gpu_requirements(*, accelerator_count: int) -> ResourceRequirements:
return ResourceRequirements(
profile="gpu-v1",
cpu_cores=1,
memory_mb=512,
scratch_mb=128,
accelerator_count=accelerator_count,
accelerator_kind="gpu",
accelerator_memory_mb=8_192,
accelerator_mode=AcceleratorMode.EXCLUSIVE_DEVICE,
architecture="x86-64",
environment_digest=ENVIRONMENT_DIGEST,
max_duration_seconds=120,
)
@pytest.mark.parametrize("device_id", ("GPU-0,GPU-1", "file:/dev/gpu0", "/dev/gpu0"))
def test_accelerator_ids_are_opaque_visibility_tokens(device_id: str) -> None:
with pytest.raises(ValueError, match="opaque"):
gpu(device_id)
def test_resource_inventory_and_requirements_round_trip_without_mutable_aliases() -> None:
capabilities = {"compute": "9.0"}
device = AcceleratorDevice(
kind="gpu",
vendor="nvidia",
device_id="gpu-0",
model="Test GPU",
memory_mb=16_384,
modes=(AcceleratorMode.EXCLUSIVE_DEVICE,),
capabilities=capabilities,
topology_group="socket-0",
)
inventory = ResourceInventory(
cpu_cores=8,
memory_mb=32_768,
scratch_mb=8_192,
architecture="x86-64",
accelerators=(device,),
environment_digests=(ENVIRONMENT_DIGEST,),
)
requirements = gpu_requirements(accelerator_count=1)
capabilities["compute"] = "mutated"
assert device.capabilities["compute"] == "9.0"
with pytest.raises(TypeError):
device.capabilities["compute"] = "mutated"
assert ResourceInventory.from_dict(inventory.to_dict()) == inventory
assert ResourceRequirements.from_dict(requirements.to_dict()) == requirements
assert requirements.eligibility_errors(inventory) == ()
incompatible = ResourceRequirements.from_dict(
{**requirements.to_dict(), "architecture": "arm64"}
)
assert incompatible.eligibility_errors(inventory) == ("architecture-mismatch",)
def test_failed_multi_accelerator_reservation_is_atomic_and_releases_nothing_partial() -> None:
inventory = ResourceInventory(
cpu_cores=4,
memory_mb=4_096,
scratch_mb=2_048,
architecture="x86-64",
accelerators=(gpu("gpu-0"), gpu("gpu-1")),
environment_digests=(ENVIRONMENT_DIGEST,),
)
pool = ResourcePool(inventory, max_concurrency=3)
first = pool.reserve("task/first", gpu_requirements(accelerator_count=1))
with pytest.raises(ResourceUnavailableError, match="accelerator-unavailable"):
pool.reserve("task/gang", gpu_requirements(accelerator_count=2))
assert pool.active_allocations() == (first,)
assert pool.release(first.allocation_id)
gang = pool.reserve("task/gang", gpu_requirements(accelerator_count=2))
assert gang.accelerator_ids == ("gpu-0", "gpu-1")
assert pool.active_allocations() == (gang,)
def test_resource_pool_enforces_aggregate_limits_under_concurrent_reservations() -> None:
inventory = ResourceInventory(
cpu_cores=4,
memory_mb=1_024,
scratch_mb=512,
architecture="x86-64",
environment_digests=(ENVIRONMENT_DIGEST,),
)
pool = ResourcePool(inventory, max_concurrency=8)
barrier = Barrier(8)
def attempt(index: int):
barrier.wait()
try:
return pool.reserve(f"task/{index}", cpu_requirements())
except ResourceUnavailableError:
return None
with ThreadPoolExecutor(max_workers=8) as executor:
results = tuple(executor.map(attempt, range(8)))
successful = tuple(result for result in results if result is not None)
assert len(successful) == 4
assert sum(item.cpu_cores for item in successful) == inventory.cpu_cores
assert sum(item.memory_mb for item in successful) <= inventory.memory_mb
assert sum(item.scratch_mb for item in successful) <= inventory.scratch_mb
assert pool.active_allocations() == tuple(sorted(successful, key=lambda item: item.task_key))
def test_resource_pool_slot_and_task_identity_limits_do_not_leak_capacity() -> None:
inventory = ResourceInventory(
cpu_cores=4,
memory_mb=2_048,
scratch_mb=1_024,
architecture="x86-64",
environment_digests=(ENVIRONMENT_DIGEST,),
)
pool = ResourcePool(inventory, max_concurrency=1)
allocation = pool.reserve("task/one", cpu_requirements())
with pytest.raises(ValueError, match="already has"):
pool.reserve("task/one", cpu_requirements())
with pytest.raises(ResourceUnavailableError, match="execution-slot-unavailable"):
pool.reserve("task/two", cpu_requirements())
assert pool.active_allocations() == (allocation,)
assert pool.release(allocation.allocation_id)
assert not pool.release(allocation.allocation_id)
replacement = pool.reserve("task/two", cpu_requirements())
assert replacement.task_key == "task/two"
def test_exclusive_gpu_and_its_partitions_share_one_conflict_domain() -> None:
full = AcceleratorDevice(
kind="gpu",
vendor="nvidia",
device_id="gpu-0",
model="Test GPU",
memory_mb=16_384,
modes=(AcceleratorMode.EXCLUSIVE_DEVICE, AcceleratorMode.PARTITION),
capabilities={},
)
partitions = tuple(
AcceleratorDevice(
kind="gpu",
vendor="nvidia",
device_id="gpu-0",
partition_id=f"mig-{index}",
model="Test MIG",
memory_mb=8_192,
modes=(AcceleratorMode.PARTITION,),
capabilities={},
)
for index in range(2)
)
inventory = ResourceInventory(
cpu_cores=4,
memory_mb=4_096,
scratch_mb=2_048,
architecture="x86-64",
accelerators=(full, *partitions),
environment_digests=(ENVIRONMENT_DIGEST,),
)
pool = ResourcePool(inventory, max_concurrency=3)
exclusive = pool.reserve("task/exclusive", gpu_requirements(accelerator_count=1))
partition_request = ResourceRequirements(
**{
**gpu_requirements(accelerator_count=1).to_dict(),
"accelerator_mode": AcceleratorMode.PARTITION,
}
)
with pytest.raises(ResourceUnavailableError, match="accelerator-unavailable"):
pool.reserve("task/partition", partition_request)
pool.release(exclusive.allocation_id)
first = pool.reserve("task/partition-0", partition_request)
second = pool.reserve("task/partition-1", partition_request)
assert set(first.accelerator_ids + second.accelerator_ids) == {"mig-0", "mig-1"}
with pytest.raises(ResourceUnavailableError, match="accelerator-unavailable"):
pool.reserve("task/full", gpu_requirements(accelerator_count=1))
+744
View File
@@ -0,0 +1,744 @@
"""Contract tests for SDK verifier decisions and built-in verifiers."""
from __future__ import annotations
import hashlib
import json
import math
from collections.abc import Callable
from dataclasses import replace
from uuid import NAMESPACE_URL, uuid5
import pytest
from scimesh.sdk import (
ArtifactCollection,
ArtifactRef,
ArtifactSchema,
CandidateOutput,
CandidateOutputs,
CanonicalRecordVerifier,
ComponentRef,
ExactArtifactVerifier,
NumericTolerance,
NumericToleranceVerifier,
OutputManifest,
PortSpec,
Provenance,
SchemaRef,
TrustMode,
VerificationDecision,
VerificationBinding,
VerificationStatus,
VerifyContext,
WorkloadId,
)
def _sha256(seed: str) -> str:
return hashlib.sha256(seed.encode("utf-8")).hexdigest()
OUTPUT_SCHEMA = ArtifactSchema(
ref=SchemaRef("verification-result", 1),
media_type="application/json",
encoding="utf-8",
max_bytes=1_024,
validator=ComponentRef("json-document", 1),
)
OUTPUT_PORT = PortSpec(OUTPUT_SCHEMA)
AUTHENTICATION_KEY = b"sdk-verification-test-key-000001"
JOB_ID = str(uuid5(NAMESPACE_URL, "verification-job"))
TASK_ID = str(uuid5(NAMESPACE_URL, "verification-task"))
EXECUTION_CONTRACT_DIGEST = _sha256("execution-contract")
def _artifact(seed: str, *, size_bytes: int = 16) -> ArtifactRef:
return ArtifactRef(
artifact_id=str(uuid5(NAMESPACE_URL, f"artifact:{seed}")),
sha256=_sha256(seed),
schema=OUTPUT_SCHEMA.ref,
media_type=OUTPUT_SCHEMA.media_type,
size_bytes=size_bytes,
)
def _provenance(attempt: str) -> Provenance:
return Provenance(
workload=WorkloadId("verification-fixture", "1.0.0"),
sdk_api_version="1.0.0",
protocol_version="1.0",
manifest_schema_version=1,
workflow_schema_version=1,
verifier=ComponentRef("exact-artifact", 1),
artifact_schemas=(OUTPUT_SCHEMA.ref,),
package_digest=f"sha256:{_sha256('package')}",
manifest_digest=_sha256("manifest"),
environment_digest=f"sha256:{_sha256('environment')}",
worker_runtime={"attempt": attempt},
allocated_resource_ids=(f"cpu-{attempt}",),
parameters_digest=_sha256("parameters"),
input_collection_digest=_sha256("inputs"),
execution_contract_digest=EXECUTION_CONTRACT_DIGEST,
selected_features={"exact-verifier": "1.0.0"},
optional_fallbacks={},
job_id=JOB_ID,
task_id=TASK_ID,
started_at="2026-08-01T10:00:00Z",
finished_at="2026-08-01T10:00:01Z",
)
def _manifest(
output_seed: str,
attempt: str,
*,
port_name: str = "result",
size_bytes: int = 16,
) -> OutputManifest:
return OutputManifest(
task_key="verify/0",
outputs={port_name: ArtifactCollection.single(_artifact(output_seed, size_bytes=size_bytes))},
metrics={"elapsed_seconds": 1.0},
provenance=_provenance(attempt),
)
def _candidate(
output_seed: str,
attempt: str,
*,
owner: str | None,
candidate_id: str | None = None,
port_name: str = "result",
size_bytes: int = 16,
authenticated: bool = True,
) -> CandidateOutput:
resolved_candidate_id = candidate_id or str(
uuid5(NAMESPACE_URL, f"candidate:{attempt}")
)
resolved_owner_id = None if owner is None else str(uuid5(NAMESPACE_URL, f"owner:{owner}"))
manifest = _manifest(
output_seed,
attempt,
port_name=port_name,
size_bytes=size_bytes,
)
if resolved_owner_id is not None and authenticated:
return CandidateOutput.from_coordinator_record(
resolved_candidate_id,
resolved_owner_id,
manifest,
AUTHENTICATION_KEY,
)
return CandidateOutput(resolved_candidate_id, resolved_owner_id, manifest)
def _context(
*,
minimum_matches: int = 1,
reference: OutputManifest | None = None,
require_distinct_owners: bool = False,
trust_mode: TrustMode = TrustMode.TRUSTED,
) -> VerifyContext:
provenance = _provenance("binding")
return VerifyContext(
expected_outputs={"result": OUTPUT_PORT},
max_output_bytes=1_024,
minimum_matches=minimum_matches,
reference=reference,
require_distinct_owners=require_distinct_owners,
binding=VerificationBinding(
workload=provenance.workload,
task_key="verify/0",
package_digest=provenance.package_digest,
manifest_digest=provenance.manifest_digest,
environment_digest=provenance.environment_digest,
parameters_digest=provenance.parameters_digest,
input_collection_digest=provenance.input_collection_digest,
execution_contract_digest=provenance.execution_contract_digest,
selected_features=provenance.selected_features,
optional_fallbacks=provenance.optional_fallbacks,
job_id=provenance.job_id,
task_id=provenance.task_id,
verifier=provenance.verifier,
sdk_api_version=provenance.sdk_api_version,
protocol_version=provenance.protocol_version,
manifest_schema_version=provenance.manifest_schema_version,
workflow_schema_version=provenance.workflow_schema_version,
artifact_schemas=provenance.artifact_schemas,
trust_mode=trust_mode,
),
trust_mode=trust_mode,
)
def test_verification_decision_is_strict_immutable_and_round_trips() -> None:
source = {"summary": {"counts": [1, 2]}}
decision = VerificationDecision(
VerificationStatus.REJECTED,
ComponentRef("test-verifier", 1),
"comparison-failed",
source,
)
source["summary"]["counts"].append(3) # type: ignore[index, union-attr]
assert decision.evidence["summary"]["counts"] == (1, 2)
assert VerificationDecision.from_dict(decision.to_dict()) == decision
with pytest.raises(TypeError):
decision.evidence["new"] = True # type: ignore[index]
with pytest.raises(TypeError):
decision.evidence["summary"]["new"] = True # type: ignore[index]
@pytest.mark.parametrize(
("status", "accepted_digest"),
[
(VerificationStatus.ACCEPTED, None),
(VerificationStatus.REJECTED, "a" * 64),
(VerificationStatus.INCONCLUSIVE, "a" * 64),
],
)
def test_verification_decision_enforces_digest_status_invariant(
status: VerificationStatus,
accepted_digest: str | None,
) -> None:
with pytest.raises(ValueError, match="accepted_digest|accepted verification"):
VerificationDecision(
status,
ComponentRef("test-verifier", 1),
"test-result",
{},
accepted_digest,
)
@pytest.mark.parametrize(
"unsafe",
[
"/home/worker/private.log",
"https://worker.invalid/evidence",
"run-123/tasks/map/result.csv",
"path=attempts/job-123/private.txt",
"https%253A%252F%252Fworker.invalid%252Fevidence%253Ftoken%253Dsecret",
],
)
def test_verification_decision_rejects_private_locations(unsafe: str) -> None:
with pytest.raises(ValueError, match="URI or local path"):
VerificationDecision(
VerificationStatus.REJECTED,
ComponentRef("test-verifier", 1),
"unsafe-evidence",
{"detail": unsafe},
)
def test_verification_decision_bounds_evidence_and_rejects_unknown_fields() -> None:
with pytest.raises(ValueError, match="exceeds 16 KiB"):
VerificationDecision(
VerificationStatus.REJECTED,
ComponentRef("test-verifier", 1),
"oversized-evidence",
{"detail": "x" * 17_000},
)
payload = VerificationDecision(
VerificationStatus.REJECTED,
ComponentRef("test-verifier", 1),
"test-result",
{},
).to_dict()
payload["unexpected"] = True
with pytest.raises(ValueError, match="unknown unexpected"):
VerificationDecision.from_dict(payload)
def test_candidate_output_envelope_is_strict_and_round_trips() -> None:
candidate = _candidate("output", "attempt-one", owner="owner-one")
decoded_candidate = CandidateOutput.from_dict(candidate.to_dict())
assert decoded_candidate.to_dict() == candidate.to_dict()
assert not decoded_candidate.coordinator_authenticated
candidates = CandidateOutputs(candidates=(candidate,))
decoded = CandidateOutputs.from_dict(candidates.to_dict())
assert not decoded.candidates[0].coordinator_authenticated
assert CandidateOutputs.from_authenticated_dict(
candidates.to_dict(), AUTHENTICATION_KEY
) == candidates
with pytest.raises(ValueError, match="authentication failed"):
CandidateOutputs.from_authenticated_dict(candidates.to_dict(), b"x" * 32)
assert candidates.manifests == (candidate.manifest,)
with pytest.raises(ValueError, match="opaque coordinator identity"):
CandidateOutput("../worker-path", candidate.owner_id, candidate.manifest)
def test_trusted_single_manifest_compatibility_uses_an_anonymous_envelope() -> None:
manifest = _manifest("output", "trusted")
candidates = CandidateOutputs((manifest,))
assert candidates.manifests == (manifest,)
assert candidates.candidates[0].owner_id is None
decision = ExactArtifactVerifier().verify(_context(), candidates)
assert decision.status is VerificationStatus.ACCEPTED
def test_raw_manifests_cannot_form_a_quorum() -> None:
with pytest.raises(ValueError, match="one trusted candidate"):
CandidateOutputs(
(
_manifest("output", "trusted-one"),
_manifest("output", "trusted-two"),
)
)
def test_multi_vote_context_automatically_requires_distinct_owners() -> None:
assert _context(minimum_matches=2).require_distinct_owners
assert _context(require_distinct_owners=True).require_distinct_owners
with pytest.raises(ValueError, match="boolean"):
_context(require_distinct_owners=1) # type: ignore[arg-type]
with pytest.raises(ValueError, match="coordinator binding"):
VerifyContext(
expected_outputs={"result": OUTPUT_PORT},
max_output_bytes=1_024,
minimum_matches=2,
)
with pytest.raises(ValueError, match="at least two"):
_context(trust_mode=TrustMode.UNTRUSTED_QUORUM)
def test_exact_verifier_reports_no_candidates_as_inconclusive() -> None:
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs(()))
assert decision.status is VerificationStatus.INCONCLUSIVE
assert decision.reason_code == "no-candidates"
assert decision.evidence == {"candidate_count": 0, "invalid_count": 0}
assert decision.accepted_digest is None
def test_exact_verifier_rejects_candidates_that_violate_output_contract() -> None:
invalid = _manifest("same-output", "invalid", port_name="undeclared")
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((invalid,)))
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "no-valid-candidates"
assert decision.evidence == {"candidate_count": 1, "invalid_count": 1}
def test_exact_verifier_rejects_candidate_from_another_scientific_binding() -> None:
candidate = _candidate("same-output", "other-job", owner="owner-one")
forged_provenance = replace(
candidate.manifest.provenance,
parameters_digest=_sha256("different-parameters"),
)
forged = replace(
candidate,
manifest=replace(candidate.manifest, provenance=forged_provenance),
)
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((forged,)))
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "no-valid-candidates"
def test_exact_verifier_accepts_unique_quorum_and_ignores_invalid_candidates() -> None:
first = _candidate("same-output", "one", owner="owner-one")
second = _candidate("same-output", "two", owner="owner-two")
minority = _candidate("different-output", "three", owner="owner-three")
invalid = _candidate(
"same-output",
"four",
owner="owner-four",
port_name="undeclared",
)
decision = ExactArtifactVerifier().verify(
_context(minimum_matches=2),
CandidateOutputs((first, second, minority, invalid)),
)
assert first.manifest.digest == second.manifest.digest
assert first.manifest.manifest_digest != second.manifest.manifest_digest
assert decision.status is VerificationStatus.ACCEPTED
assert decision.reason_code == "quorum-match"
assert decision.accepted_digest == first.manifest.digest
assert decision.evidence == {
"matched": 2,
"required": 2,
"distinct_digests": 2,
"invalid_count": 1,
}
def test_exact_verifier_rejects_conflicting_quorums() -> None:
candidates = CandidateOutputs(
(
_candidate("group-a", "a-one", owner="a-one"),
_candidate("group-a", "a-two", owner="a-two"),
_candidate("group-b", "b-one", owner="b-one"),
_candidate("group-b", "b-two", owner="b-two"),
)
)
decision = ExactArtifactVerifier().verify(_context(minimum_matches=2), candidates)
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "conflicting-quorums"
assert decision.accepted_digest is None
assert decision.evidence["largest_group"] == 2
assert decision.evidence["distinct_digests"] == 2
def test_exact_verifier_distinguishes_insufficient_evidence_from_reference_mismatch() -> None:
reference = _manifest("reference", "reference")
one_mismatch = CandidateOutputs((_candidate("other", "one", owner="owner-one"),))
two_mismatches = CandidateOutputs(
(
_candidate("other-a", "two", owner="owner-two"),
_candidate("other-b", "three", owner="owner-three"),
)
)
insufficient = ExactArtifactVerifier().verify(
_context(minimum_matches=2, reference=reference),
one_mismatch,
)
rejected = ExactArtifactVerifier().verify(
_context(minimum_matches=2, reference=reference),
two_mismatches,
)
assert (insufficient.status, insufficient.reason_code) == (
VerificationStatus.INCONCLUSIVE,
"insufficient-evidence",
)
assert (rejected.status, rejected.reason_code) == (
VerificationStatus.REJECTED,
"reference-mismatch",
)
def test_exact_verifier_accepts_declared_reference_quorum() -> None:
reference = _manifest("reference", "reference")
candidates = CandidateOutputs(
(
_candidate("reference", "worker-one", owner="owner-one"),
_candidate("reference", "worker-two", owner="owner-two"),
_candidate("other", "worker-three", owner="owner-three"),
)
)
decision = ExactArtifactVerifier().verify(
_context(minimum_matches=2, reference=reference),
candidates,
)
assert decision.status is VerificationStatus.ACCEPTED
assert decision.reason_code == "reference-match"
assert decision.accepted_digest == reference.digest
assert decision.evidence == {"matched": 2, "required": 2, "invalid_count": 0}
def test_candidate_outputs_rejects_duplicate_candidate_ids() -> None:
candidate = _candidate("output", "one", owner="owner-one")
replay = _candidate(
"different-output",
"two",
owner="owner-two",
candidate_id=candidate.candidate_id,
)
with pytest.raises(ValueError, match="candidate_id values must be unique"):
CandidateOutputs((candidate, replay))
def test_exact_quorum_rejects_candidates_without_authenticated_owners() -> None:
candidates = CandidateOutputs(
(
_candidate("output", "one", owner=None),
_candidate("output", "two", owner="owner-two"),
)
)
decision = ExactArtifactVerifier().verify(
_context(minimum_matches=2),
candidates,
)
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "coordinator-authentication-required"
assert decision.evidence == {"candidate_count": 2, "unauthenticated_count": 1}
def test_exact_verifier_counts_at_most_one_vote_per_owner() -> None:
candidates = CandidateOutputs(
(
_candidate("output", "one", owner="same-owner"),
_candidate("output", "two", owner="same-owner"),
)
)
decision = ExactArtifactVerifier().verify(
_context(minimum_matches=2),
candidates,
)
assert decision.status is VerificationStatus.INCONCLUSIVE
assert decision.reason_code == "insufficient-evidence"
assert decision.evidence == {
"largest_group": 1,
"required": 2,
"distinct_digests": 1,
"invalid_count": 0,
"duplicate_owner_candidates": 1,
}
def test_exact_verifier_rejects_owner_equivocation_without_leaking_identity() -> None:
owner = "equivocating-owner"
candidates = CandidateOutputs(
(
_candidate("output-a", "one", owner=owner),
_candidate("output-b", "two", owner=owner),
_candidate("output-a", "three", owner="honest-owner"),
)
)
decision = ExactArtifactVerifier().verify(
_context(minimum_matches=2),
candidates,
)
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "owner-equivocation"
assert decision.accepted_digest is None
assert decision.evidence == {
"candidate_count": 3,
"equivocating_owner_count": 1,
}
assert candidates.candidates[0].owner_id not in json.dumps(decision.to_dict())
def test_numeric_verifier_accepts_nested_values_with_absolute_and_relative_tolerance() -> None:
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001, relative=0.01))
decision = verifier.verify_values(
{"energies": [10.0, 0.05], "converged": True},
{"energies": [10.05, 0.0505], "converged": True},
)
assert decision.status is VerificationStatus.ACCEPTED
assert decision.reason_code == "within-tolerance"
assert decision.accepted_digest is not None
assert decision.evidence == {
"absolute": 0.001,
"relative": 0.01,
"max_ulps": 0,
}
def test_numeric_verifier_supports_ulp_tolerance() -> None:
adjacent = math.nextafter(1.0, 2.0)
verifier = NumericToleranceVerifier(NumericTolerance(max_ulps=1))
decision = verifier.verify_values(1.0, adjacent)
assert decision.status is VerificationStatus.ACCEPTED
def test_numeric_verifier_reports_bounded_location_and_error_evidence() -> None:
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.1))
decision = verifier.verify_values(
{"matrix": [[1.0, 2.0]]},
{"matrix": [[1.0, 2.5]]},
)
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "numeric-mismatch"
assert decision.evidence["location"] == "$.matrix[0][1]"
assert decision.evidence["absolute_error"] == pytest.approx(0.5)
assert decision.evidence["allowed_error"] == pytest.approx(0.1)
assert isinstance(decision.evidence["ulp_distance"], int)
def test_numeric_verifier_rejects_shape_changes_before_value_comparison() -> None:
verifier = NumericToleranceVerifier(NumericTolerance())
decision = verifier.verify_values(
{"energy": 1.0, "iterations": 4},
{"energy": 1.0, "converged": True},
)
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "shape-mismatch"
assert decision.evidence == {
"location": "$",
"missing_keys": ("iterations",),
"extra_keys": ("converged",),
}
def test_numeric_verifier_applies_declared_nan_policy() -> None:
reject = NumericToleranceVerifier(NumericTolerance(nan_policy="reject"))
equal = NumericToleranceVerifier(NumericTolerance(nan_policy="equal"))
rejected = reject.verify_values(float("nan"), float("nan"))
accepted = equal.verify_values(float("nan"), float("nan"))
assert (rejected.status, rejected.reason_code) == (
VerificationStatus.REJECTED,
"nan-policy",
)
assert accepted.status is VerificationStatus.ACCEPTED
@pytest.mark.parametrize(
"factory",
[
lambda: NumericTolerance(absolute=-0.1),
lambda: NumericTolerance(relative=float("inf")),
lambda: NumericTolerance(max_ulps=True),
lambda: NumericTolerance(nan_policy="propagate"),
lambda: NumericTolerance(max_elements=0),
],
)
def test_numeric_tolerance_rejects_ambiguous_or_non_finite_policy(
factory: Callable[[], NumericTolerance],
) -> None:
with pytest.raises(ValueError, match="numeric tolerance"):
factory()
def test_numeric_verifier_rejects_unrepresentable_integer_without_raising() -> None:
verifier = NumericToleranceVerifier(NumericTolerance())
decision = verifier.verify_values(10**10_000, 10**10_000)
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "numeric-range"
assert decision.evidence["location"] == "$"
def test_numeric_verifier_rejects_shapes_above_its_manifest_bound() -> None:
verifier = NumericToleranceVerifier(NumericTolerance(max_elements=3))
decision = verifier.verify_values([1, 2, 3], [1, 2, 3])
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "element-limit"
assert decision.evidence == {"max_elements": 3}
assert verifier.configuration["max_elements"] == 3
def test_numeric_verifier_does_not_round_mixed_integer_and_float_values() -> None:
verifier = NumericToleranceVerifier(NumericTolerance())
decision = verifier.verify_values(2**53 + 1, float(2**53))
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "numeric-mismatch"
def test_numeric_verifier_rejects_non_json_mapping_keys() -> None:
verifier = NumericToleranceVerifier(NumericTolerance())
decision = verifier.verify_values({1: 2.0}, {1: 2.0})
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "type-mismatch"
def test_numeric_verifier_implements_manifest_verifier_protocol_with_loader() -> None:
reference = _manifest("reference", "reference")
candidate = _manifest("candidate", "candidate")
values = {
reference.outputs["result"].items[0].artifact.sha256: {"energy": 1.0},
candidate.outputs["result"].items[0].artifact.sha256: {"energy": 1.0001},
}
def load(manifest: OutputManifest) -> object:
return values[manifest.outputs["result"].items[0].artifact.sha256]
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001), load)
decision = verifier.verify(
_context(reference=reference),
CandidateOutputs((candidate,)),
)
assert decision.status is VerificationStatus.ACCEPTED
assert decision.accepted_digest == reference.digest
def test_manifest_verifiers_fail_closed_without_artifact_loaders() -> None:
candidate = CandidateOutputs((_manifest("candidate", "candidate"),))
numeric = NumericToleranceVerifier(NumericTolerance()).verify(_context(), candidate)
canonical = CanonicalRecordVerifier(_canonical_json_record).verify(_context(), candidate)
assert numeric.reason_code == "loader-unavailable"
assert canonical.reason_code == "loader-unavailable"
def _canonical_json_record(record: object) -> bytes:
return json.dumps(record, sort_keys=True, separators=(",", ":")).encode("utf-8")
def test_canonical_record_verifier_accepts_normalized_records() -> None:
verifier = CanonicalRecordVerifier(_canonical_json_record)
expected = [{"name": "molecule", "score": 0.75}, {"id": 2}]
actual = [{"score": 0.75, "name": "molecule"}, {"id": 2}]
decision = verifier.verify_records(expected, actual)
assert decision.status is VerificationStatus.ACCEPTED
assert decision.reason_code == "canonical-match"
assert decision.evidence == {"records": 2}
assert decision.accepted_digest is not None
def test_canonical_record_verifier_rejects_content_or_count_mismatch() -> None:
verifier = CanonicalRecordVerifier(_canonical_json_record)
decision = verifier.verify_records([{"id": 1}, {"id": 2}], [{"id": 1}])
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "canonical-mismatch"
assert decision.evidence == {"expected_records": 2, "actual_records": 1}
def test_canonical_record_verifier_enforces_record_limit() -> None:
verifier = CanonicalRecordVerifier(_canonical_json_record, max_records=2)
decision = verifier.verify_records([1, 2, 3], [1, 2, 3])
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "record-limit"
assert decision.evidence == {"max_records": 2}
@pytest.mark.parametrize(
"canonicalizer",
[
lambda _record: "not-bytes",
lambda _record: (_ for _ in ()).throw(ValueError("/private/worker/path")),
],
)
def test_canonical_record_verifier_sanitizes_canonicalization_failures(
canonicalizer: Callable[[object], object],
) -> None:
verifier = CanonicalRecordVerifier(canonicalizer) # type: ignore[arg-type]
decision = verifier.verify_records([{"id": 1}], [{"id": 1}])
assert decision.status is VerificationStatus.REJECTED
assert decision.reason_code == "canonicalization-failed"
assert decision.evidence == {}
assert decision.accepted_digest is None