Add workload SDK foundation
This commit is contained in:
@@ -0,0 +1,664 @@
|
||||
"""Compatibility tests for the built-in SDK bridge and scientific reference."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from scimesh.chemistry.dataset import find_molecule_by_id
|
||||
from scimesh.sdk import (
|
||||
ArtifactCollection,
|
||||
ArtifactSchema,
|
||||
CheckpointPolicy,
|
||||
CompatibilityError,
|
||||
ComponentRef,
|
||||
DeterminismProfile,
|
||||
FeatureRequirement,
|
||||
GangSpec,
|
||||
JobRequest,
|
||||
LocalArtifactStore,
|
||||
LocalCoreBatchExecutor,
|
||||
LocalPlanningContext,
|
||||
NetworkPolicy,
|
||||
PortRef,
|
||||
ProcessModel,
|
||||
RetryPolicy,
|
||||
SchemaRef,
|
||||
StageKind,
|
||||
TrustMode,
|
||||
VerificationDecision,
|
||||
VerificationStatus,
|
||||
VersionRange,
|
||||
WorkloadDefinition,
|
||||
WorkloadRegistry,
|
||||
assert_manifest_round_trip,
|
||||
default_sdk_registry,
|
||||
default_sdk_runtime,
|
||||
similarity_search_sdk_adapter,
|
||||
)
|
||||
from scimesh.workloads.similarity_search import search_similar, write_search_results
|
||||
|
||||
|
||||
def _write_tiny_dataset(path: Path) -> None:
|
||||
path.write_text(
|
||||
"chembl_id\tcanonical_smiles\textra\n"
|
||||
"QUERY\tCCO\tquery\n"
|
||||
"ALCOHOL\tCCCO\talcohol\n"
|
||||
"ALKANE\tCCCC\talkane\n"
|
||||
"BROKEN\tnot-a-smiles\tinvalid\n"
|
||||
"DUPLICATE\tCCO\tduplicate\n"
|
||||
"AMINE\tCCN\tamine\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _registered_similarity_search(shard_rows: int = 2):
|
||||
registry = default_sdk_registry(shard_rows=shard_rows)
|
||||
runtime = default_sdk_runtime()
|
||||
descriptions = registry.descriptions()
|
||||
assert len(descriptions) == 1
|
||||
description = descriptions[0]
|
||||
definition, negotiated = registry.require(
|
||||
description.workload.name,
|
||||
description.workload.version,
|
||||
description.package_digest,
|
||||
runtime=runtime,
|
||||
)
|
||||
return registry, runtime, description, definition, negotiated
|
||||
|
||||
|
||||
def _request_for(
|
||||
dataset: Path,
|
||||
artifact_store: LocalArtifactStore,
|
||||
definition: WorkloadDefinition,
|
||||
) -> JobRequest:
|
||||
input_port = definition.manifest.inputs["input"]
|
||||
dataset_artifact = artifact_store.import_file(
|
||||
dataset,
|
||||
declaration=input_port.schema,
|
||||
)
|
||||
return JobRequest(
|
||||
workload=definition.manifest.workload,
|
||||
parameters={"query_id": "QUERY", "top_k": 3, "progress_every": 0},
|
||||
inputs={"input": ArtifactCollection.single(dataset_artifact)},
|
||||
)
|
||||
|
||||
|
||||
def test_builtin_similarity_search_manifest_is_registered_and_negotiable() -> None:
|
||||
_, _, description, definition, negotiated = _registered_similarity_search()
|
||||
manifest = definition.manifest
|
||||
|
||||
assert description.enabled is True
|
||||
assert manifest.workload.name == "similarity-search"
|
||||
assert manifest.workload.version == "1.0.0"
|
||||
assert manifest.determinism is DeterminismProfile.BYTE_EXACT
|
||||
assert manifest.conformance_profiles == ("core-batch-v1",)
|
||||
assert [stage.kind for stage in manifest.workflow.stages] == [
|
||||
StageKind.MAP,
|
||||
StageKind.REDUCE,
|
||||
]
|
||||
assert set(definition.runners) == {manifest.workflow.stages[0].entry_point}
|
||||
assert set(definition.reducers) == {manifest.workflow.stages[1].entry_point}
|
||||
assert negotiated is not None
|
||||
assert negotiated.manifest == manifest
|
||||
assert_manifest_round_trip(manifest)
|
||||
|
||||
|
||||
def test_local_sdk_executor_matches_similarity_search_reference(tmp_path: Path) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
registry, runtime, description, definition, _ = _registered_similarity_search()
|
||||
artifact_store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, artifact_store, definition)
|
||||
|
||||
result = LocalCoreBatchExecutor(
|
||||
registry,
|
||||
runtime,
|
||||
artifact_store,
|
||||
tmp_path / "sdk-work",
|
||||
).execute(request, description.package_digest)
|
||||
result_artifact = result.outputs["result"].items[0].artifact
|
||||
|
||||
reference_path = tmp_path / "reference.csv"
|
||||
query = find_molecule_by_id(dataset, "QUERY")
|
||||
reference = search_similar(dataset, query, top_k=3, progress_every=0)
|
||||
write_search_results(reference_path, reference.matches)
|
||||
|
||||
assert artifact_store.materialize(result_artifact).read_bytes() == reference_path.read_bytes()
|
||||
assert result.task_key == "reduce/final"
|
||||
assert result.metrics == {"matches_emitted": 3, "partial_count": 3}
|
||||
|
||||
|
||||
def test_legacy_adapter_planning_is_deterministic_ordered_and_path_free(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
registry, runtime, description, definition, _ = _registered_similarity_search()
|
||||
artifact_store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, artifact_store, definition)
|
||||
input_artifact = request.inputs["input"].items[0].artifact
|
||||
|
||||
first = registry.plan(
|
||||
request,
|
||||
description.package_digest,
|
||||
runtime,
|
||||
LocalPlanningContext(
|
||||
artifact_store,
|
||||
artifact_store,
|
||||
tmp_path / "first-plan",
|
||||
allowed_artifacts=(input_artifact,),
|
||||
),
|
||||
)
|
||||
second = registry.plan(
|
||||
request,
|
||||
description.package_digest,
|
||||
runtime,
|
||||
LocalPlanningContext(
|
||||
artifact_store,
|
||||
artifact_store,
|
||||
tmp_path / "second-plan",
|
||||
allowed_artifacts=(input_artifact,),
|
||||
),
|
||||
)
|
||||
|
||||
assert first.to_json() == second.to_json()
|
||||
assert first.digest == second.digest
|
||||
assert first.package_digest == definition.manifest.package.digest
|
||||
assert first.manifest_digest == definition.manifest.digest
|
||||
assert first.trust_mode is request.trust_mode
|
||||
assert JobRequest.from_json(request.to_json()) == request
|
||||
assert [task.task_key for task in first.tasks] == [
|
||||
"map/00000000",
|
||||
"map/00000001",
|
||||
"map/00000002",
|
||||
]
|
||||
assert all(task.stage_id == "map" for task in first.tasks)
|
||||
assert all(task.package_digest == first.package_digest for task in first.tasks)
|
||||
assert all(task.manifest_digest == first.manifest_digest for task in first.tasks)
|
||||
assert all(task.trust_mode is first.trust_mode for task in first.tasks)
|
||||
assert all("query_id" not in task.parameters for task in first.tasks)
|
||||
assert all(task.parameters["query_smiles"] == "CCO" for task in first.tasks)
|
||||
|
||||
shard_ids: list[list[str]] = []
|
||||
for task in first.tasks:
|
||||
artifact = task.inputs["input"].items[0].artifact
|
||||
with artifact_store.materialize(artifact).open(encoding="utf-8", newline="") as source:
|
||||
shard_ids.append(
|
||||
[row["chembl_id"] for row in csv.DictReader(source, delimiter="\t")]
|
||||
)
|
||||
assert set(artifact.to_dict()) == {
|
||||
"artifact_id",
|
||||
"sha256",
|
||||
"schema",
|
||||
"media_type",
|
||||
"size_bytes",
|
||||
"records",
|
||||
"dimensions",
|
||||
}
|
||||
assert shard_ids == [
|
||||
["QUERY", "ALCOHOL"],
|
||||
["ALKANE", "BROKEN"],
|
||||
["DUPLICATE", "AMINE"],
|
||||
]
|
||||
|
||||
wire_payload = first.to_json()
|
||||
assert str(tmp_path) not in wire_payload
|
||||
assert "file://" not in wire_payload
|
||||
assert "worker://" not in wire_payload
|
||||
assert "workspace" not in wire_payload
|
||||
|
||||
|
||||
def test_local_context_sink_cannot_seal_files_outside_the_attempt(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_, _, _, definition, _ = _registered_similarity_search()
|
||||
artifact_store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
workspace = tmp_path / "attempt"
|
||||
context = LocalPlanningContext(artifact_store, artifact_store, workspace)
|
||||
outside = tmp_path / "private.txt"
|
||||
outside.write_text("private", encoding="utf-8")
|
||||
schema = definition.manifest.inputs["input"].schema
|
||||
|
||||
with pytest.raises(ValueError, match="inside its workspace"):
|
||||
context.sink.seal(outside, declaration=schema)
|
||||
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
link = workspace / "result"
|
||||
link.symlink_to(outside)
|
||||
with pytest.raises(ValueError, match="real workspace directories"):
|
||||
context.sink.seal(link, declaration=schema)
|
||||
|
||||
|
||||
def test_local_store_rejects_malformed_content_before_publishing(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
malformed = tmp_path / "malformed.json"
|
||||
malformed.write_text('{"unfinished":', encoding="utf-8")
|
||||
declaration = ArtifactSchema(
|
||||
SchemaRef("json-result", 1),
|
||||
"application/json",
|
||||
"utf-8",
|
||||
max_bytes=1_024,
|
||||
validator=ComponentRef("json-document", 1),
|
||||
)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
|
||||
with pytest.raises(ValueError, match="not a valid bounded document"):
|
||||
store.import_file(malformed, declaration=declaration)
|
||||
assert tuple(path for path in store.root.iterdir() if not path.name.startswith(".seal-")) == ()
|
||||
|
||||
|
||||
def test_delimited_validator_rejects_headerless_data_and_enforces_record_limit(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
declaration = ArtifactSchema(
|
||||
SchemaRef("bounded-table", 1),
|
||||
"text/csv",
|
||||
"utf-8",
|
||||
max_bytes=1_024,
|
||||
validator=ComponentRef("delimited-table", 1),
|
||||
validator_configuration={"columns": ["value"]},
|
||||
max_records=1,
|
||||
)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
headerless = tmp_path / "headerless.csv"
|
||||
headerless.write_text("1\n2\n", encoding="utf-8")
|
||||
oversized = tmp_path / "oversized.csv"
|
||||
oversized.write_text("value\n1\n2\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="header does not match"):
|
||||
store.import_file(headerless, declaration=declaration)
|
||||
with pytest.raises(ValueError, match="record limit"):
|
||||
store.import_file(oversized, declaration=declaration)
|
||||
|
||||
|
||||
def test_custom_artifact_inspector_is_bound_to_schema_and_validator_identity(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
schema_ref = SchemaRef("matrix-result", 1)
|
||||
validator = ComponentRef("matrix-inspector", 1)
|
||||
declaration = ArtifactSchema(
|
||||
schema_ref,
|
||||
"application/x-matrix",
|
||||
None,
|
||||
max_bytes=1_024,
|
||||
validator=validator,
|
||||
validator_configuration={"layout": "row-major"},
|
||||
max_records=1,
|
||||
max_dimensions=(2, 2),
|
||||
)
|
||||
source = tmp_path / "matrix.bin"
|
||||
source.write_bytes(b"matrix")
|
||||
wrong = LocalArtifactStore(
|
||||
tmp_path / "wrong-store",
|
||||
inspectors={
|
||||
schema_ref.canonical: (
|
||||
ComponentRef("other-inspector", 1),
|
||||
lambda _path, _configuration: (1, (2, 2)),
|
||||
)
|
||||
},
|
||||
)
|
||||
with pytest.raises(ValueError, match="no matching registered validator"):
|
||||
wrong.import_file(source, declaration=declaration)
|
||||
|
||||
def inspect(_path: Path, configuration):
|
||||
assert dict(configuration) == {"layout": "row-major"}
|
||||
return 1, (2, 2)
|
||||
|
||||
store = LocalArtifactStore(
|
||||
tmp_path / "store",
|
||||
inspectors={schema_ref.canonical: (validator, inspect)},
|
||||
)
|
||||
artifact = store.import_file(source, declaration=declaration)
|
||||
assert artifact.records == 1
|
||||
assert artifact.dimensions == (2, 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("forgery", "message"),
|
||||
(
|
||||
("artifact", "artifacts sealed by its attempt"),
|
||||
("provenance", "provenance does not match"),
|
||||
),
|
||||
)
|
||||
def test_local_executor_rejects_handler_forged_outputs(
|
||||
tmp_path: Path,
|
||||
forgery: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
_, runtime, description, original, _ = _registered_similarity_search()
|
||||
map_stage = next(stage for stage in original.manifest.workflow.stages if stage.kind is StageKind.MAP)
|
||||
inner = original.runners[map_stage.entry_point]
|
||||
|
||||
class ForgingRunner:
|
||||
def run(self, context):
|
||||
result = inner.run(context)
|
||||
if forgery == "provenance":
|
||||
forged = replace(
|
||||
result.provenance,
|
||||
worker_runtime={"kind": "forged-runtime"},
|
||||
)
|
||||
return replace(result, provenance=forged)
|
||||
original_ref = result.outputs["partial"].items[0].artifact
|
||||
forged_ref = replace(original_ref, artifact_id=str(uuid4()))
|
||||
return replace(
|
||||
result,
|
||||
outputs={"partial": ArtifactCollection.single(forged_ref)},
|
||||
)
|
||||
|
||||
definition = WorkloadDefinition(
|
||||
original.manifest,
|
||||
original.planner,
|
||||
{map_stage.entry_point: ForgingRunner()},
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, store, definition)
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
description.package_digest,
|
||||
)
|
||||
|
||||
|
||||
def test_local_executor_rejects_profiles_that_claim_network_isolation(tmp_path: Path) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
_, runtime, description, original, _ = _registered_similarity_search()
|
||||
stages = tuple(
|
||||
replace(stage, execution=replace(stage.execution, network=NetworkPolicy.NONE))
|
||||
for stage in original.manifest.workflow.stages
|
||||
)
|
||||
workflow = replace(original.manifest.workflow, stages=stages)
|
||||
manifest = replace(original.manifest, workflow=workflow)
|
||||
definition = WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, store, definition)
|
||||
|
||||
with pytest.raises(CompatibilityError) as raised:
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
description.package_digest,
|
||||
)
|
||||
assert raised.value.code == "feature-undeclared"
|
||||
|
||||
|
||||
def test_local_executor_rejects_aliased_terminal_outputs_before_planning(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
_, runtime, description, original, _ = _registered_similarity_search()
|
||||
reducer = next(
|
||||
stage for stage in original.manifest.workflow.stages if stage.kind is StageKind.REDUCE
|
||||
)
|
||||
internal_name = next(iter(reducer.outputs))
|
||||
workflow = replace(
|
||||
original.manifest.workflow,
|
||||
outputs={"aliased": PortRef(internal_name, reducer.stage_id)},
|
||||
)
|
||||
manifest = replace(
|
||||
original.manifest,
|
||||
workflow=workflow,
|
||||
outputs={"aliased": reducer.outputs[internal_name]},
|
||||
)
|
||||
definition = WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, store, definition)
|
||||
|
||||
with pytest.raises(ValueError, match="identity-mapped reducer outputs"):
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
description.package_digest,
|
||||
)
|
||||
|
||||
|
||||
def test_local_executor_rejects_non_trusted_trust_modes(tmp_path: Path) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
_, runtime, _, original, _ = _registered_similarity_search()
|
||||
stages = tuple(
|
||||
replace(stage, trust_modes=("trusted", "verified"))
|
||||
for stage in original.manifest.workflow.stages
|
||||
)
|
||||
manifest = replace(
|
||||
original.manifest,
|
||||
workflow=replace(original.manifest.workflow, stages=stages),
|
||||
trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED),
|
||||
)
|
||||
definition = WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = replace(
|
||||
_request_for(dataset, store, definition),
|
||||
trust_mode=TrustMode.VERIFIED,
|
||||
)
|
||||
runtime = replace(runtime, trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED))
|
||||
|
||||
with pytest.raises(ValueError, match="supports only trusted workloads"):
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
manifest.package.digest,
|
||||
)
|
||||
|
||||
|
||||
def _advanced_execution_manifest(
|
||||
original: WorkloadDefinition,
|
||||
case: str,
|
||||
) -> tuple[tuple[str, ...], object]:
|
||||
"""Declare one negotiable advanced profile the local executor cannot enforce."""
|
||||
stages = original.manifest.workflow.stages
|
||||
if case == "process-pool":
|
||||
features = ("process-pools", "multi-process")
|
||||
changed = tuple(
|
||||
replace(
|
||||
stage,
|
||||
resources=replace(stage.resources, cpu_cores=2),
|
||||
execution=replace(
|
||||
stage.execution,
|
||||
process_model=ProcessModel.PROCESS_POOL,
|
||||
max_processes=2,
|
||||
),
|
||||
)
|
||||
for stage in stages
|
||||
)
|
||||
elif case == "checkpoints":
|
||||
features = ("checkpoints",)
|
||||
changed = tuple(
|
||||
replace(
|
||||
stage,
|
||||
execution=replace(
|
||||
stage.execution,
|
||||
checkpoint=CheckpointPolicy(
|
||||
enabled=True,
|
||||
schema=SchemaRef("task-state", 1),
|
||||
compatibility_version=1,
|
||||
),
|
||||
),
|
||||
)
|
||||
for stage in stages
|
||||
)
|
||||
elif case == "retries":
|
||||
features = ("retries",)
|
||||
changed = tuple(
|
||||
replace(stage, retry=RetryPolicy(max_attempts=2))
|
||||
for stage in stages
|
||||
)
|
||||
elif case == "secrets":
|
||||
features = ("secret-injection",)
|
||||
changed = tuple(
|
||||
replace(
|
||||
stage,
|
||||
execution=replace(stage.execution, secret_handles=("db-credential",)),
|
||||
)
|
||||
for stage in stages
|
||||
)
|
||||
elif case == "gang":
|
||||
features = ("gang-leases",)
|
||||
changed = tuple(
|
||||
replace(
|
||||
stage,
|
||||
gang=GangSpec(replicas=2, per_replica_resources=stage.resources),
|
||||
)
|
||||
for stage in stages
|
||||
)
|
||||
elif case == "network-isolation":
|
||||
features = ("network-isolation",)
|
||||
changed = tuple(
|
||||
replace(
|
||||
stage,
|
||||
execution=replace(stage.execution, network=NetworkPolicy.NONE),
|
||||
)
|
||||
for stage in stages
|
||||
)
|
||||
else:
|
||||
assert case == "service-stage"
|
||||
features = ("services",)
|
||||
changed = (replace(stages[0], kind=StageKind.SERVICE),) + stages[1:]
|
||||
manifest = replace(
|
||||
original.manifest,
|
||||
workflow=replace(original.manifest.workflow, stages=changed),
|
||||
required_features=original.manifest.required_features
|
||||
+ tuple(
|
||||
FeatureRequirement(feature, VersionRange(">=1,<2")) for feature in features
|
||||
),
|
||||
)
|
||||
return features, manifest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "message"),
|
||||
(
|
||||
("process-pool", "one non-nested host thread"),
|
||||
("checkpoints", "cannot enforce this stage profile"),
|
||||
("retries", "does not implement retries"),
|
||||
("secrets", "cannot enforce this stage profile"),
|
||||
("gang", "cannot enforce this stage profile"),
|
||||
("network-isolation", "cannot enforce a restricted network policy"),
|
||||
("service-stage", "does not implement advanced stages"),
|
||||
),
|
||||
)
|
||||
def test_local_executor_rejects_profiles_it_cannot_enforce(
|
||||
tmp_path: Path,
|
||||
case: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
_, runtime, _, original, _ = _registered_similarity_search()
|
||||
features, manifest = _advanced_execution_manifest(original, case)
|
||||
definition = WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, store, definition)
|
||||
runtime = replace(
|
||||
runtime,
|
||||
features={**runtime.features, **{feature: "1.0.0" for feature in features}},
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
manifest.package.digest,
|
||||
)
|
||||
|
||||
|
||||
def test_local_executor_fails_when_the_declared_verifier_rejects(tmp_path: Path) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
_, runtime, _, original, _ = _registered_similarity_search()
|
||||
|
||||
class RejectingVerifier:
|
||||
identity = ComponentRef("exact-artifact", 1)
|
||||
|
||||
def verify(self, context, candidates):
|
||||
return VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
self.identity,
|
||||
"forced-rejection",
|
||||
{},
|
||||
)
|
||||
|
||||
definition = WorkloadDefinition(
|
||||
original.manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
{ComponentRef("exact-artifact", 1).canonical: RejectingVerifier()},
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, store, definition)
|
||||
|
||||
with pytest.raises(ValueError, match="did not pass its declared verifier"):
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
original.manifest.package.digest,
|
||||
)
|
||||
|
||||
|
||||
def test_local_executor_enforces_the_declared_output_byte_budget(tmp_path: Path) -> None:
|
||||
dataset = tmp_path / "molecules.tsv"
|
||||
_write_tiny_dataset(dataset)
|
||||
runtime = default_sdk_runtime()
|
||||
adapter = similarity_search_sdk_adapter(shard_rows=2)
|
||||
# The planner pins its own manifest into every task, so the budget cut must
|
||||
# be applied to the adapter's manifest for plan and definition to agree.
|
||||
adapter.manifest = replace(
|
||||
adapter.manifest,
|
||||
workflow=replace(adapter.manifest.workflow, max_output_bytes=256),
|
||||
limits=replace(adapter.manifest.limits, max_output_bytes=256),
|
||||
)
|
||||
definition = adapter.definition()
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
request = _request_for(dataset, store, definition)
|
||||
|
||||
with pytest.raises(ValueError, match="bytes exceed their sink limit"):
|
||||
LocalCoreBatchExecutor(registry, runtime, store, tmp_path / "work").execute(
|
||||
request,
|
||||
definition.manifest.package.digest,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,609 @@
|
||||
"""Security and version-pinning tests for the installed SDK registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from importlib import metadata
|
||||
from pathlib import Path
|
||||
import py_compile
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from scimesh.sdk import (
|
||||
AllowedPackage,
|
||||
ArtifactCollection,
|
||||
ArtifactRef,
|
||||
CompatibilityError,
|
||||
FeatureRequirement,
|
||||
JobRequest,
|
||||
LocalArtifactStore,
|
||||
LocalPlanningContext,
|
||||
PackageSpec,
|
||||
TrustMode,
|
||||
VersionRange,
|
||||
WorkloadDefinition,
|
||||
WorkloadId,
|
||||
WorkloadRegistry,
|
||||
current_scimesh_package_digest,
|
||||
default_sdk_runtime,
|
||||
installed_distribution_digest,
|
||||
similarity_search_sdk_adapter,
|
||||
)
|
||||
from scimesh.sdk.schema import (
|
||||
ParameterValidationError,
|
||||
validate_parameter_instance,
|
||||
validate_schema_definition,
|
||||
)
|
||||
|
||||
|
||||
def _definition(*, version: str = "1.0.0", digest_character: str = "a") -> WorkloadDefinition:
|
||||
original = similarity_search_sdk_adapter(shard_rows=2).definition()
|
||||
manifest = replace(
|
||||
original.manifest,
|
||||
workload=WorkloadId("similarity-search", version),
|
||||
package=PackageSpec("scimesh", "sha256:" + digest_character * 64),
|
||||
)
|
||||
return WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
|
||||
|
||||
def test_registry_requires_an_explicit_enabled_version_and_digest() -> None:
|
||||
first = _definition(version="1.0.0", digest_character="a")
|
||||
second = _definition(version="2.0.0", digest_character="b")
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(first, enabled=True)
|
||||
registry.register(second)
|
||||
|
||||
resolved, _ = registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
|
||||
assert resolved is first
|
||||
with pytest.raises(ValueError, match="unknown workload version"):
|
||||
registry.require("similarity-search", "3.0.0", "sha256:" + "a" * 64)
|
||||
with pytest.raises(ValueError, match="not enabled"):
|
||||
registry.require("similarity-search", "2.0.0", "sha256:" + "b" * 64)
|
||||
with pytest.raises(ValueError, match="not enabled"):
|
||||
registry.require("similarity-search", "1.0.0", "sha256:" + "c" * 64)
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
registry.register(first)
|
||||
|
||||
registry.enable("similarity-search", "2.0.0", "sha256:" + "b" * 64)
|
||||
assert [item.workload.version for item in registry.descriptions()] == ["1.0.0", "2.0.0"]
|
||||
|
||||
|
||||
def test_compatibility_failure_occurs_before_planner_invocation(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
original = similarity_search_sdk_adapter(shard_rows=2).definition()
|
||||
|
||||
class CountingPlanner:
|
||||
calls = 0
|
||||
|
||||
def validate(self, request):
|
||||
self.calls += 1
|
||||
return original.planner.validate(request)
|
||||
|
||||
def plan(self, job, context):
|
||||
self.calls += 1
|
||||
return original.planner.plan(job, context)
|
||||
|
||||
planner = CountingPlanner()
|
||||
definition = WorkloadDefinition(
|
||||
original.manifest,
|
||||
planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
input_port = definition.manifest.inputs["input"]
|
||||
artifact = ArtifactRef(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"a" * 64,
|
||||
input_port.schema.ref,
|
||||
input_port.schema.media_type,
|
||||
1,
|
||||
)
|
||||
request = JobRequest(
|
||||
definition.manifest.workload,
|
||||
{"query_smiles": "CCO"},
|
||||
{"input": ArtifactCollection.single(artifact)},
|
||||
)
|
||||
incompatible = replace(default_sdk_runtime(), protocol_version="2.0.0")
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
|
||||
with pytest.raises(CompatibilityError) as raised:
|
||||
registry.plan(
|
||||
request,
|
||||
definition.manifest.package.digest,
|
||||
incompatible,
|
||||
LocalPlanningContext(store, store, tmp_path / "plan"),
|
||||
)
|
||||
assert raised.value.code == "protocol-mismatch"
|
||||
assert planner.calls == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_changes", "error_code"),
|
||||
(
|
||||
({"required_features": ("undeclared-feature",)}, "feature-undeclared"),
|
||||
({"trust_mode": TrustMode.VERIFIED}, "trust-mode-undeclared"),
|
||||
),
|
||||
)
|
||||
def test_job_selected_features_and_trust_mode_fail_closed_before_planning(
|
||||
tmp_path: Path,
|
||||
request_changes: dict[str, object],
|
||||
error_code: str,
|
||||
) -> None:
|
||||
definition = similarity_search_sdk_adapter(shard_rows=2).definition()
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
input_port = definition.manifest.inputs["input"]
|
||||
artifact = ArtifactRef(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"a" * 64,
|
||||
input_port.schema.ref,
|
||||
input_port.schema.media_type,
|
||||
1,
|
||||
records=1,
|
||||
)
|
||||
values: dict[str, object] = {
|
||||
"workload": definition.manifest.workload,
|
||||
"parameters": {"query_smiles": "CCO"},
|
||||
"inputs": {"input": ArtifactCollection.single(artifact)},
|
||||
}
|
||||
values.update(request_changes)
|
||||
request = JobRequest(**values) # type: ignore[arg-type]
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
|
||||
with pytest.raises(CompatibilityError) as raised:
|
||||
registry.plan(
|
||||
request,
|
||||
definition.manifest.package.digest,
|
||||
default_sdk_runtime(),
|
||||
LocalPlanningContext(store, store, tmp_path / "plan"),
|
||||
)
|
||||
assert raised.value.code == error_code
|
||||
|
||||
|
||||
class _EntryPoints(tuple):
|
||||
def select(self, *, group: str):
|
||||
assert group == WorkloadRegistry.ENTRY_POINT_GROUP
|
||||
return self
|
||||
|
||||
|
||||
def test_discovery_imports_only_an_exact_allowlisted_installed_entry_point(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
definition = similarity_search_sdk_adapter().definition()
|
||||
loaded: list[str] = []
|
||||
|
||||
class EntryPoint:
|
||||
def __init__(self, name: str, distribution: str) -> None:
|
||||
self.name = name
|
||||
self.dist = (
|
||||
metadata.distribution("scimesh")
|
||||
if distribution == "scimesh"
|
||||
else SimpleNamespace(name=distribution)
|
||||
)
|
||||
self.value = "scimesh.sdk.builtins:similarity_search_sdk_adapter"
|
||||
|
||||
@property
|
||||
def module(self) -> str:
|
||||
return self.value.partition(":")[0]
|
||||
|
||||
def load(self):
|
||||
loaded.append(self.name)
|
||||
return lambda: definition
|
||||
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.metadata.entry_points",
|
||||
lambda: _EntryPoints(
|
||||
(
|
||||
EntryPoint("evil-workload@1.0.0", "unapproved"),
|
||||
EntryPoint("similarity-search@1.0.0", "scimesh"),
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.installed_distribution_digest",
|
||||
lambda _distribution: definition.manifest.package.digest,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.discover_installed(
|
||||
(
|
||||
AllowedPackage(
|
||||
"scimesh",
|
||||
definition.manifest.workload,
|
||||
definition.manifest.package.digest,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert loaded == ["similarity-search@1.0.0"]
|
||||
assert registry.descriptions()[0].enabled
|
||||
|
||||
|
||||
def test_discovery_measures_package_before_importing_entry_point(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
definition = similarity_search_sdk_adapter().definition()
|
||||
loaded = False
|
||||
|
||||
class EntryPoint:
|
||||
name = "similarity-search@1.0.0"
|
||||
dist = metadata.distribution("scimesh")
|
||||
value = "scimesh.sdk.builtins:similarity_search_sdk_adapter"
|
||||
module = "scimesh.sdk.builtins"
|
||||
|
||||
def load(self):
|
||||
nonlocal loaded
|
||||
loaded = True
|
||||
return lambda: definition
|
||||
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.metadata.entry_points",
|
||||
lambda: _EntryPoints((EntryPoint(),)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.installed_distribution_digest",
|
||||
lambda _distribution: "sha256:" + "f" * 64,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="content does not match"):
|
||||
WorkloadRegistry().discover_installed(
|
||||
(
|
||||
AllowedPackage(
|
||||
"scimesh",
|
||||
definition.manifest.workload,
|
||||
definition.manifest.package.digest,
|
||||
),
|
||||
)
|
||||
)
|
||||
assert loaded is False
|
||||
|
||||
|
||||
def test_installed_digest_is_stable_when_python_generates_a_pycache(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
package = tmp_path / "fixture_pkg"
|
||||
package.mkdir()
|
||||
source = package / "__init__.py"
|
||||
source.write_text("VALUE = 1\n", encoding="utf-8")
|
||||
|
||||
class FixtureDistribution:
|
||||
name = "fixture-dist"
|
||||
files = (Path("fixture_pkg/__init__.py"),)
|
||||
entry_points = ()
|
||||
|
||||
@staticmethod
|
||||
def read_text(name: str) -> str | None:
|
||||
return "fixture_pkg\n" if name == "top_level.txt" else None
|
||||
|
||||
@staticmethod
|
||||
def locate_file(value: object) -> Path:
|
||||
return tmp_path / str(value)
|
||||
|
||||
distribution = FixtureDistribution()
|
||||
before = installed_distribution_digest(distribution) # type: ignore[arg-type]
|
||||
py_compile.compile(str(source), doraise=True)
|
||||
|
||||
assert installed_distribution_digest(distribution) == before # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_discovery_rejects_entry_point_module_owned_by_another_distribution(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
owned = tmp_path / "owned_pkg"
|
||||
owned.mkdir()
|
||||
(owned / "__init__.py").write_text("", encoding="utf-8")
|
||||
loaded = False
|
||||
|
||||
class Distribution:
|
||||
name = "allowed-dist"
|
||||
files = (Path("owned_pkg/__init__.py"),)
|
||||
entry_points = ()
|
||||
|
||||
@staticmethod
|
||||
def read_text(name: str) -> str | None:
|
||||
return "owned_pkg\n" if name == "top_level.txt" else None
|
||||
|
||||
@staticmethod
|
||||
def locate_file(value: object) -> Path:
|
||||
return tmp_path / str(value)
|
||||
|
||||
class EntryPoint:
|
||||
name = "similarity-search@1.0.0"
|
||||
dist = Distribution()
|
||||
value = "foreign_pkg.workload:factory"
|
||||
module = "foreign_pkg.workload"
|
||||
|
||||
def load(self):
|
||||
nonlocal loaded
|
||||
loaded = True
|
||||
raise AssertionError("foreign entry point must not load")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.metadata.entry_points",
|
||||
lambda: _EntryPoints((EntryPoint(),)),
|
||||
)
|
||||
definition = similarity_search_sdk_adapter().definition()
|
||||
with pytest.raises(ValueError, match="outside its distribution"):
|
||||
WorkloadRegistry().discover_installed(
|
||||
(
|
||||
AllowedPackage(
|
||||
"allowed-dist",
|
||||
definition.manifest.workload,
|
||||
"sha256:" + "a" * 64,
|
||||
),
|
||||
)
|
||||
)
|
||||
assert loaded is False
|
||||
|
||||
|
||||
def test_missing_allowlisted_entry_point_fails_without_loading_or_registering(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
loaded: list[str] = []
|
||||
|
||||
class EntryPoint:
|
||||
name = "job-selected-module@1.0.0"
|
||||
dist = SimpleNamespace(name="unapproved")
|
||||
|
||||
def load(self):
|
||||
loaded.append(self.name)
|
||||
raise AssertionError("unapproved entry point must not load")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.metadata.entry_points",
|
||||
lambda: _EntryPoints((EntryPoint(),)),
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
with pytest.raises(ValueError, match="were not installed"):
|
||||
registry.discover_installed(
|
||||
(
|
||||
AllowedPackage(
|
||||
"scimesh",
|
||||
WorkloadId("similarity-search", "1.0.0"),
|
||||
current_scimesh_package_digest(),
|
||||
),
|
||||
)
|
||||
)
|
||||
assert loaded == []
|
||||
assert registry.descriptions() == ()
|
||||
|
||||
|
||||
def test_parameter_schema_accepts_finite_big_integer_bounds() -> None:
|
||||
bound = 10**400
|
||||
schema = {"type": "integer", "minimum": -bound, "maximum": bound}
|
||||
|
||||
validate_schema_definition(schema)
|
||||
validate_parameter_instance(bound, schema)
|
||||
|
||||
with pytest.raises(ParameterValidationError, match="violates maximum"):
|
||||
validate_parameter_instance(bound + 1, schema)
|
||||
|
||||
|
||||
def test_job_parameters_reject_unbounded_json_integers_early() -> None:
|
||||
with pytest.raises(ValueError, match="4096-bit JSON bound"):
|
||||
JobRequest(
|
||||
WorkloadId("similarity-search", "1.0.0"),
|
||||
{"value": 10**2_000},
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "multiple", "accepted"),
|
||||
[
|
||||
(3 * 10**400, 3, True),
|
||||
(10**400, 3, False),
|
||||
(10**400, 0.1, True),
|
||||
(0.3, 0.1, True),
|
||||
(0.31, 0.1, False),
|
||||
],
|
||||
)
|
||||
def test_parameter_schema_multiple_of_is_exact_without_float_overflow(
|
||||
value: int | float,
|
||||
multiple: int | float,
|
||||
accepted: bool,
|
||||
) -> None:
|
||||
schema = {"type": "number", "multipleOf": multiple}
|
||||
validate_schema_definition(schema)
|
||||
|
||||
if accepted:
|
||||
validate_parameter_instance(value, schema)
|
||||
else:
|
||||
with pytest.raises(ParameterValidationError, match="violates multipleOf"):
|
||||
validate_parameter_instance(value, schema)
|
||||
|
||||
|
||||
def test_parameter_schema_equality_uses_json_types() -> None:
|
||||
validate_schema_definition({"enum": [True, 1]})
|
||||
with pytest.raises(ValueError, match="enum values must be unique"):
|
||||
validate_schema_definition({"enum": [1, 1.0]})
|
||||
|
||||
validate_parameter_instance(True, {"enum": [True]})
|
||||
with pytest.raises(ParameterValidationError, match="outside enum"):
|
||||
validate_parameter_instance(1, {"enum": [True]})
|
||||
validate_parameter_instance(1.0, {"enum": [1]})
|
||||
|
||||
validate_parameter_instance({"enabled": True}, {"const": {"enabled": True}})
|
||||
with pytest.raises(ParameterValidationError, match="does not match const"):
|
||||
validate_parameter_instance({"enabled": 1}, {"const": {"enabled": True}})
|
||||
|
||||
unique = {"type": "array", "uniqueItems": True}
|
||||
validate_parameter_instance([True, 1, {"enabled": True}, {"enabled": 1}], unique)
|
||||
with pytest.raises(ParameterValidationError, match="items must be unique"):
|
||||
validate_parameter_instance([1, 1.0], unique)
|
||||
|
||||
|
||||
def test_disabled_workload_is_not_resolvable_until_re_enabled() -> None:
|
||||
definition = _definition()
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
resolved, _ = registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
|
||||
assert resolved is definition
|
||||
|
||||
registry.disable("similarity-search", "1.0.0", "sha256:" + "a" * 64)
|
||||
with pytest.raises(ValueError, match="not enabled"):
|
||||
registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
|
||||
|
||||
registry.enable("similarity-search", "1.0.0", "sha256:" + "a" * 64)
|
||||
resolved, _ = registry.require("similarity-search", "1.0.0", "sha256:" + "a" * 64)
|
||||
assert resolved is definition
|
||||
|
||||
|
||||
def test_discovery_rechecks_the_package_digest_after_loading(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
definition = similarity_search_sdk_adapter().definition()
|
||||
digests = iter((definition.manifest.package.digest, "sha256:" + "e" * 64))
|
||||
|
||||
class EntryPoint:
|
||||
name = "similarity-search@1.0.0"
|
||||
dist = metadata.distribution("scimesh")
|
||||
value = "scimesh.sdk.builtins:similarity_search_sdk_adapter"
|
||||
module = "scimesh.sdk.builtins"
|
||||
|
||||
def load(self):
|
||||
return lambda: definition
|
||||
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.metadata.entry_points",
|
||||
lambda: _EntryPoints((EntryPoint(),)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"scimesh.sdk.registry.installed_distribution_digest",
|
||||
lambda _distribution: next(digests),
|
||||
)
|
||||
|
||||
registry = WorkloadRegistry()
|
||||
with pytest.raises(ValueError, match="changed while loading"):
|
||||
registry.discover_installed(
|
||||
(
|
||||
AllowedPackage(
|
||||
"scimesh",
|
||||
definition.manifest.workload,
|
||||
definition.manifest.package.digest,
|
||||
),
|
||||
)
|
||||
)
|
||||
assert registry.descriptions() == ()
|
||||
|
||||
|
||||
def test_request_trust_mode_must_be_enforceable_by_runtime_and_stages(tmp_path: Path) -> None:
|
||||
original = similarity_search_sdk_adapter(shard_rows=2).definition()
|
||||
manifest = replace(
|
||||
original.manifest,
|
||||
trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED),
|
||||
)
|
||||
definition = WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
input_port = definition.manifest.inputs["input"]
|
||||
artifact = ArtifactRef(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"a" * 64,
|
||||
input_port.schema.ref,
|
||||
input_port.schema.media_type,
|
||||
1,
|
||||
records=1,
|
||||
)
|
||||
request = JobRequest(
|
||||
definition.manifest.workload,
|
||||
{"query_smiles": "CCO"},
|
||||
{"input": ArtifactCollection.single(artifact)},
|
||||
trust_mode=TrustMode.VERIFIED,
|
||||
)
|
||||
store = LocalArtifactStore(tmp_path / "artifacts")
|
||||
|
||||
with pytest.raises(CompatibilityError) as raised:
|
||||
registry.plan(
|
||||
request,
|
||||
manifest.package.digest,
|
||||
default_sdk_runtime(),
|
||||
LocalPlanningContext(store, store, tmp_path / "runtime-plan"),
|
||||
)
|
||||
assert raised.value.code == "trust-mode-unavailable"
|
||||
|
||||
runtime = replace(
|
||||
default_sdk_runtime(),
|
||||
trust_modes=(TrustMode.TRUSTED, TrustMode.VERIFIED),
|
||||
)
|
||||
with pytest.raises(CompatibilityError) as raised:
|
||||
registry.plan(
|
||||
request,
|
||||
manifest.package.digest,
|
||||
runtime,
|
||||
LocalPlanningContext(store, store, tmp_path / "stage-plan"),
|
||||
)
|
||||
assert raised.value.code == "stage-trust-unavailable"
|
||||
|
||||
|
||||
def test_job_cannot_require_a_feature_outside_the_runtime(tmp_path: Path) -> None:
|
||||
original = similarity_search_sdk_adapter(shard_rows=2).definition()
|
||||
manifest = replace(
|
||||
original.manifest,
|
||||
optional_features=(
|
||||
FeatureRequirement("gpu-fastpath", VersionRange(">=1,<2"), "cpu-fallback"),
|
||||
),
|
||||
)
|
||||
definition = WorkloadDefinition(
|
||||
manifest,
|
||||
original.planner,
|
||||
original.runners,
|
||||
original.reducers,
|
||||
original.verifiers,
|
||||
)
|
||||
registry = WorkloadRegistry()
|
||||
registry.register(definition, enabled=True)
|
||||
input_port = definition.manifest.inputs["input"]
|
||||
artifact = ArtifactRef(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"a" * 64,
|
||||
input_port.schema.ref,
|
||||
input_port.schema.media_type,
|
||||
1,
|
||||
records=1,
|
||||
)
|
||||
request = JobRequest(
|
||||
definition.manifest.workload,
|
||||
{"query_smiles": "CCO"},
|
||||
{"input": ArtifactCollection.single(artifact)},
|
||||
required_features=("gpu-fastpath",),
|
||||
)
|
||||
|
||||
negotiated = registry.require(
|
||||
"similarity-search",
|
||||
"1.0.0",
|
||||
manifest.package.digest,
|
||||
runtime=default_sdk_runtime(),
|
||||
)[1]
|
||||
assert negotiated is not None
|
||||
assert negotiated.optional_fallbacks == {"gpu-fastpath": "cpu-fallback"}
|
||||
|
||||
with pytest.raises(CompatibilityError) as raised:
|
||||
registry.plan(
|
||||
request,
|
||||
manifest.package.digest,
|
||||
default_sdk_runtime(),
|
||||
LocalPlanningContext(
|
||||
LocalArtifactStore(tmp_path / "artifacts"),
|
||||
LocalArtifactStore(tmp_path / "artifacts"),
|
||||
tmp_path / "plan",
|
||||
),
|
||||
)
|
||||
assert raised.value.code == "feature-unavailable"
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Resource inventory and atomic local allocation tests for the SDK Agent layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Barrier
|
||||
|
||||
import pytest
|
||||
|
||||
from scimesh.sdk import (
|
||||
AcceleratorDevice,
|
||||
AcceleratorMode,
|
||||
ResourceInventory,
|
||||
ResourcePool,
|
||||
ResourceRequirements,
|
||||
ResourceUnavailableError,
|
||||
)
|
||||
|
||||
|
||||
ENVIRONMENT_DIGEST = "sha256:" + "d" * 64
|
||||
|
||||
|
||||
def gpu(device_id: str, *, topology_group: str = "socket-0") -> AcceleratorDevice:
|
||||
return AcceleratorDevice(
|
||||
kind="gpu",
|
||||
vendor="nvidia",
|
||||
device_id=device_id,
|
||||
model="Test GPU",
|
||||
memory_mb=16_384,
|
||||
modes=(AcceleratorMode.EXCLUSIVE_DEVICE,),
|
||||
capabilities={"compute": "9.0", "driver": "test"},
|
||||
topology_group=topology_group,
|
||||
)
|
||||
|
||||
|
||||
def cpu_requirements(*, cpu_cores: int = 1, memory_mb: int = 256) -> ResourceRequirements:
|
||||
return ResourceRequirements(
|
||||
profile="cpu-v1",
|
||||
cpu_cores=cpu_cores,
|
||||
memory_mb=memory_mb,
|
||||
scratch_mb=128,
|
||||
architecture="x86-64",
|
||||
environment_digest=ENVIRONMENT_DIGEST,
|
||||
max_duration_seconds=120,
|
||||
)
|
||||
|
||||
|
||||
def gpu_requirements(*, accelerator_count: int) -> ResourceRequirements:
|
||||
return ResourceRequirements(
|
||||
profile="gpu-v1",
|
||||
cpu_cores=1,
|
||||
memory_mb=512,
|
||||
scratch_mb=128,
|
||||
accelerator_count=accelerator_count,
|
||||
accelerator_kind="gpu",
|
||||
accelerator_memory_mb=8_192,
|
||||
accelerator_mode=AcceleratorMode.EXCLUSIVE_DEVICE,
|
||||
architecture="x86-64",
|
||||
environment_digest=ENVIRONMENT_DIGEST,
|
||||
max_duration_seconds=120,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_id", ("GPU-0,GPU-1", "file:/dev/gpu0", "/dev/gpu0"))
|
||||
def test_accelerator_ids_are_opaque_visibility_tokens(device_id: str) -> None:
|
||||
with pytest.raises(ValueError, match="opaque"):
|
||||
gpu(device_id)
|
||||
|
||||
|
||||
def test_resource_inventory_and_requirements_round_trip_without_mutable_aliases() -> None:
|
||||
capabilities = {"compute": "9.0"}
|
||||
device = AcceleratorDevice(
|
||||
kind="gpu",
|
||||
vendor="nvidia",
|
||||
device_id="gpu-0",
|
||||
model="Test GPU",
|
||||
memory_mb=16_384,
|
||||
modes=(AcceleratorMode.EXCLUSIVE_DEVICE,),
|
||||
capabilities=capabilities,
|
||||
topology_group="socket-0",
|
||||
)
|
||||
inventory = ResourceInventory(
|
||||
cpu_cores=8,
|
||||
memory_mb=32_768,
|
||||
scratch_mb=8_192,
|
||||
architecture="x86-64",
|
||||
accelerators=(device,),
|
||||
environment_digests=(ENVIRONMENT_DIGEST,),
|
||||
)
|
||||
requirements = gpu_requirements(accelerator_count=1)
|
||||
|
||||
capabilities["compute"] = "mutated"
|
||||
assert device.capabilities["compute"] == "9.0"
|
||||
with pytest.raises(TypeError):
|
||||
device.capabilities["compute"] = "mutated"
|
||||
assert ResourceInventory.from_dict(inventory.to_dict()) == inventory
|
||||
assert ResourceRequirements.from_dict(requirements.to_dict()) == requirements
|
||||
assert requirements.eligibility_errors(inventory) == ()
|
||||
|
||||
incompatible = ResourceRequirements.from_dict(
|
||||
{**requirements.to_dict(), "architecture": "arm64"}
|
||||
)
|
||||
assert incompatible.eligibility_errors(inventory) == ("architecture-mismatch",)
|
||||
|
||||
|
||||
def test_failed_multi_accelerator_reservation_is_atomic_and_releases_nothing_partial() -> None:
|
||||
inventory = ResourceInventory(
|
||||
cpu_cores=4,
|
||||
memory_mb=4_096,
|
||||
scratch_mb=2_048,
|
||||
architecture="x86-64",
|
||||
accelerators=(gpu("gpu-0"), gpu("gpu-1")),
|
||||
environment_digests=(ENVIRONMENT_DIGEST,),
|
||||
)
|
||||
pool = ResourcePool(inventory, max_concurrency=3)
|
||||
first = pool.reserve("task/first", gpu_requirements(accelerator_count=1))
|
||||
|
||||
with pytest.raises(ResourceUnavailableError, match="accelerator-unavailable"):
|
||||
pool.reserve("task/gang", gpu_requirements(accelerator_count=2))
|
||||
|
||||
assert pool.active_allocations() == (first,)
|
||||
assert pool.release(first.allocation_id)
|
||||
gang = pool.reserve("task/gang", gpu_requirements(accelerator_count=2))
|
||||
assert gang.accelerator_ids == ("gpu-0", "gpu-1")
|
||||
assert pool.active_allocations() == (gang,)
|
||||
|
||||
|
||||
def test_resource_pool_enforces_aggregate_limits_under_concurrent_reservations() -> None:
|
||||
inventory = ResourceInventory(
|
||||
cpu_cores=4,
|
||||
memory_mb=1_024,
|
||||
scratch_mb=512,
|
||||
architecture="x86-64",
|
||||
environment_digests=(ENVIRONMENT_DIGEST,),
|
||||
)
|
||||
pool = ResourcePool(inventory, max_concurrency=8)
|
||||
barrier = Barrier(8)
|
||||
|
||||
def attempt(index: int):
|
||||
barrier.wait()
|
||||
try:
|
||||
return pool.reserve(f"task/{index}", cpu_requirements())
|
||||
except ResourceUnavailableError:
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
results = tuple(executor.map(attempt, range(8)))
|
||||
|
||||
successful = tuple(result for result in results if result is not None)
|
||||
assert len(successful) == 4
|
||||
assert sum(item.cpu_cores for item in successful) == inventory.cpu_cores
|
||||
assert sum(item.memory_mb for item in successful) <= inventory.memory_mb
|
||||
assert sum(item.scratch_mb for item in successful) <= inventory.scratch_mb
|
||||
assert pool.active_allocations() == tuple(sorted(successful, key=lambda item: item.task_key))
|
||||
|
||||
|
||||
def test_resource_pool_slot_and_task_identity_limits_do_not_leak_capacity() -> None:
|
||||
inventory = ResourceInventory(
|
||||
cpu_cores=4,
|
||||
memory_mb=2_048,
|
||||
scratch_mb=1_024,
|
||||
architecture="x86-64",
|
||||
environment_digests=(ENVIRONMENT_DIGEST,),
|
||||
)
|
||||
pool = ResourcePool(inventory, max_concurrency=1)
|
||||
allocation = pool.reserve("task/one", cpu_requirements())
|
||||
|
||||
with pytest.raises(ValueError, match="already has"):
|
||||
pool.reserve("task/one", cpu_requirements())
|
||||
with pytest.raises(ResourceUnavailableError, match="execution-slot-unavailable"):
|
||||
pool.reserve("task/two", cpu_requirements())
|
||||
assert pool.active_allocations() == (allocation,)
|
||||
|
||||
assert pool.release(allocation.allocation_id)
|
||||
assert not pool.release(allocation.allocation_id)
|
||||
replacement = pool.reserve("task/two", cpu_requirements())
|
||||
assert replacement.task_key == "task/two"
|
||||
|
||||
|
||||
def test_exclusive_gpu_and_its_partitions_share_one_conflict_domain() -> None:
|
||||
full = AcceleratorDevice(
|
||||
kind="gpu",
|
||||
vendor="nvidia",
|
||||
device_id="gpu-0",
|
||||
model="Test GPU",
|
||||
memory_mb=16_384,
|
||||
modes=(AcceleratorMode.EXCLUSIVE_DEVICE, AcceleratorMode.PARTITION),
|
||||
capabilities={},
|
||||
)
|
||||
partitions = tuple(
|
||||
AcceleratorDevice(
|
||||
kind="gpu",
|
||||
vendor="nvidia",
|
||||
device_id="gpu-0",
|
||||
partition_id=f"mig-{index}",
|
||||
model="Test MIG",
|
||||
memory_mb=8_192,
|
||||
modes=(AcceleratorMode.PARTITION,),
|
||||
capabilities={},
|
||||
)
|
||||
for index in range(2)
|
||||
)
|
||||
inventory = ResourceInventory(
|
||||
cpu_cores=4,
|
||||
memory_mb=4_096,
|
||||
scratch_mb=2_048,
|
||||
architecture="x86-64",
|
||||
accelerators=(full, *partitions),
|
||||
environment_digests=(ENVIRONMENT_DIGEST,),
|
||||
)
|
||||
pool = ResourcePool(inventory, max_concurrency=3)
|
||||
exclusive = pool.reserve("task/exclusive", gpu_requirements(accelerator_count=1))
|
||||
partition_request = ResourceRequirements(
|
||||
**{
|
||||
**gpu_requirements(accelerator_count=1).to_dict(),
|
||||
"accelerator_mode": AcceleratorMode.PARTITION,
|
||||
}
|
||||
)
|
||||
with pytest.raises(ResourceUnavailableError, match="accelerator-unavailable"):
|
||||
pool.reserve("task/partition", partition_request)
|
||||
pool.release(exclusive.allocation_id)
|
||||
|
||||
first = pool.reserve("task/partition-0", partition_request)
|
||||
second = pool.reserve("task/partition-1", partition_request)
|
||||
assert set(first.accelerator_ids + second.accelerator_ids) == {"mig-0", "mig-1"}
|
||||
with pytest.raises(ResourceUnavailableError, match="accelerator-unavailable"):
|
||||
pool.reserve("task/full", gpu_requirements(accelerator_count=1))
|
||||
@@ -0,0 +1,744 @@
|
||||
"""Contract tests for SDK verifier decisions and built-in verifiers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import pytest
|
||||
|
||||
from scimesh.sdk import (
|
||||
ArtifactCollection,
|
||||
ArtifactRef,
|
||||
ArtifactSchema,
|
||||
CandidateOutput,
|
||||
CandidateOutputs,
|
||||
CanonicalRecordVerifier,
|
||||
ComponentRef,
|
||||
ExactArtifactVerifier,
|
||||
NumericTolerance,
|
||||
NumericToleranceVerifier,
|
||||
OutputManifest,
|
||||
PortSpec,
|
||||
Provenance,
|
||||
SchemaRef,
|
||||
TrustMode,
|
||||
VerificationDecision,
|
||||
VerificationBinding,
|
||||
VerificationStatus,
|
||||
VerifyContext,
|
||||
WorkloadId,
|
||||
)
|
||||
|
||||
|
||||
def _sha256(seed: str) -> str:
|
||||
return hashlib.sha256(seed.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
OUTPUT_SCHEMA = ArtifactSchema(
|
||||
ref=SchemaRef("verification-result", 1),
|
||||
media_type="application/json",
|
||||
encoding="utf-8",
|
||||
max_bytes=1_024,
|
||||
validator=ComponentRef("json-document", 1),
|
||||
)
|
||||
OUTPUT_PORT = PortSpec(OUTPUT_SCHEMA)
|
||||
AUTHENTICATION_KEY = b"sdk-verification-test-key-000001"
|
||||
JOB_ID = str(uuid5(NAMESPACE_URL, "verification-job"))
|
||||
TASK_ID = str(uuid5(NAMESPACE_URL, "verification-task"))
|
||||
EXECUTION_CONTRACT_DIGEST = _sha256("execution-contract")
|
||||
|
||||
|
||||
def _artifact(seed: str, *, size_bytes: int = 16) -> ArtifactRef:
|
||||
return ArtifactRef(
|
||||
artifact_id=str(uuid5(NAMESPACE_URL, f"artifact:{seed}")),
|
||||
sha256=_sha256(seed),
|
||||
schema=OUTPUT_SCHEMA.ref,
|
||||
media_type=OUTPUT_SCHEMA.media_type,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _provenance(attempt: str) -> Provenance:
|
||||
return Provenance(
|
||||
workload=WorkloadId("verification-fixture", "1.0.0"),
|
||||
sdk_api_version="1.0.0",
|
||||
protocol_version="1.0",
|
||||
manifest_schema_version=1,
|
||||
workflow_schema_version=1,
|
||||
verifier=ComponentRef("exact-artifact", 1),
|
||||
artifact_schemas=(OUTPUT_SCHEMA.ref,),
|
||||
package_digest=f"sha256:{_sha256('package')}",
|
||||
manifest_digest=_sha256("manifest"),
|
||||
environment_digest=f"sha256:{_sha256('environment')}",
|
||||
worker_runtime={"attempt": attempt},
|
||||
allocated_resource_ids=(f"cpu-{attempt}",),
|
||||
parameters_digest=_sha256("parameters"),
|
||||
input_collection_digest=_sha256("inputs"),
|
||||
execution_contract_digest=EXECUTION_CONTRACT_DIGEST,
|
||||
selected_features={"exact-verifier": "1.0.0"},
|
||||
optional_fallbacks={},
|
||||
job_id=JOB_ID,
|
||||
task_id=TASK_ID,
|
||||
started_at="2026-08-01T10:00:00Z",
|
||||
finished_at="2026-08-01T10:00:01Z",
|
||||
)
|
||||
|
||||
|
||||
def _manifest(
|
||||
output_seed: str,
|
||||
attempt: str,
|
||||
*,
|
||||
port_name: str = "result",
|
||||
size_bytes: int = 16,
|
||||
) -> OutputManifest:
|
||||
return OutputManifest(
|
||||
task_key="verify/0",
|
||||
outputs={port_name: ArtifactCollection.single(_artifact(output_seed, size_bytes=size_bytes))},
|
||||
metrics={"elapsed_seconds": 1.0},
|
||||
provenance=_provenance(attempt),
|
||||
)
|
||||
|
||||
|
||||
def _candidate(
|
||||
output_seed: str,
|
||||
attempt: str,
|
||||
*,
|
||||
owner: str | None,
|
||||
candidate_id: str | None = None,
|
||||
port_name: str = "result",
|
||||
size_bytes: int = 16,
|
||||
authenticated: bool = True,
|
||||
) -> CandidateOutput:
|
||||
resolved_candidate_id = candidate_id or str(
|
||||
uuid5(NAMESPACE_URL, f"candidate:{attempt}")
|
||||
)
|
||||
resolved_owner_id = None if owner is None else str(uuid5(NAMESPACE_URL, f"owner:{owner}"))
|
||||
manifest = _manifest(
|
||||
output_seed,
|
||||
attempt,
|
||||
port_name=port_name,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
if resolved_owner_id is not None and authenticated:
|
||||
return CandidateOutput.from_coordinator_record(
|
||||
resolved_candidate_id,
|
||||
resolved_owner_id,
|
||||
manifest,
|
||||
AUTHENTICATION_KEY,
|
||||
)
|
||||
return CandidateOutput(resolved_candidate_id, resolved_owner_id, manifest)
|
||||
|
||||
|
||||
def _context(
|
||||
*,
|
||||
minimum_matches: int = 1,
|
||||
reference: OutputManifest | None = None,
|
||||
require_distinct_owners: bool = False,
|
||||
trust_mode: TrustMode = TrustMode.TRUSTED,
|
||||
) -> VerifyContext:
|
||||
provenance = _provenance("binding")
|
||||
return VerifyContext(
|
||||
expected_outputs={"result": OUTPUT_PORT},
|
||||
max_output_bytes=1_024,
|
||||
minimum_matches=minimum_matches,
|
||||
reference=reference,
|
||||
require_distinct_owners=require_distinct_owners,
|
||||
binding=VerificationBinding(
|
||||
workload=provenance.workload,
|
||||
task_key="verify/0",
|
||||
package_digest=provenance.package_digest,
|
||||
manifest_digest=provenance.manifest_digest,
|
||||
environment_digest=provenance.environment_digest,
|
||||
parameters_digest=provenance.parameters_digest,
|
||||
input_collection_digest=provenance.input_collection_digest,
|
||||
execution_contract_digest=provenance.execution_contract_digest,
|
||||
selected_features=provenance.selected_features,
|
||||
optional_fallbacks=provenance.optional_fallbacks,
|
||||
job_id=provenance.job_id,
|
||||
task_id=provenance.task_id,
|
||||
verifier=provenance.verifier,
|
||||
sdk_api_version=provenance.sdk_api_version,
|
||||
protocol_version=provenance.protocol_version,
|
||||
manifest_schema_version=provenance.manifest_schema_version,
|
||||
workflow_schema_version=provenance.workflow_schema_version,
|
||||
artifact_schemas=provenance.artifact_schemas,
|
||||
trust_mode=trust_mode,
|
||||
),
|
||||
trust_mode=trust_mode,
|
||||
)
|
||||
|
||||
|
||||
def test_verification_decision_is_strict_immutable_and_round_trips() -> None:
|
||||
source = {"summary": {"counts": [1, 2]}}
|
||||
decision = VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"comparison-failed",
|
||||
source,
|
||||
)
|
||||
|
||||
source["summary"]["counts"].append(3) # type: ignore[index, union-attr]
|
||||
|
||||
assert decision.evidence["summary"]["counts"] == (1, 2)
|
||||
assert VerificationDecision.from_dict(decision.to_dict()) == decision
|
||||
with pytest.raises(TypeError):
|
||||
decision.evidence["new"] = True # type: ignore[index]
|
||||
with pytest.raises(TypeError):
|
||||
decision.evidence["summary"]["new"] = True # type: ignore[index]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "accepted_digest"),
|
||||
[
|
||||
(VerificationStatus.ACCEPTED, None),
|
||||
(VerificationStatus.REJECTED, "a" * 64),
|
||||
(VerificationStatus.INCONCLUSIVE, "a" * 64),
|
||||
],
|
||||
)
|
||||
def test_verification_decision_enforces_digest_status_invariant(
|
||||
status: VerificationStatus,
|
||||
accepted_digest: str | None,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="accepted_digest|accepted verification"):
|
||||
VerificationDecision(
|
||||
status,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"test-result",
|
||||
{},
|
||||
accepted_digest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsafe",
|
||||
[
|
||||
"/home/worker/private.log",
|
||||
"https://worker.invalid/evidence",
|
||||
"run-123/tasks/map/result.csv",
|
||||
"path=attempts/job-123/private.txt",
|
||||
"https%253A%252F%252Fworker.invalid%252Fevidence%253Ftoken%253Dsecret",
|
||||
],
|
||||
)
|
||||
def test_verification_decision_rejects_private_locations(unsafe: str) -> None:
|
||||
with pytest.raises(ValueError, match="URI or local path"):
|
||||
VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"unsafe-evidence",
|
||||
{"detail": unsafe},
|
||||
)
|
||||
|
||||
|
||||
def test_verification_decision_bounds_evidence_and_rejects_unknown_fields() -> None:
|
||||
with pytest.raises(ValueError, match="exceeds 16 KiB"):
|
||||
VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"oversized-evidence",
|
||||
{"detail": "x" * 17_000},
|
||||
)
|
||||
|
||||
payload = VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"test-result",
|
||||
{},
|
||||
).to_dict()
|
||||
payload["unexpected"] = True
|
||||
with pytest.raises(ValueError, match="unknown unexpected"):
|
||||
VerificationDecision.from_dict(payload)
|
||||
|
||||
|
||||
def test_candidate_output_envelope_is_strict_and_round_trips() -> None:
|
||||
candidate = _candidate("output", "attempt-one", owner="owner-one")
|
||||
|
||||
decoded_candidate = CandidateOutput.from_dict(candidate.to_dict())
|
||||
assert decoded_candidate.to_dict() == candidate.to_dict()
|
||||
assert not decoded_candidate.coordinator_authenticated
|
||||
candidates = CandidateOutputs(candidates=(candidate,))
|
||||
decoded = CandidateOutputs.from_dict(candidates.to_dict())
|
||||
assert not decoded.candidates[0].coordinator_authenticated
|
||||
assert CandidateOutputs.from_authenticated_dict(
|
||||
candidates.to_dict(), AUTHENTICATION_KEY
|
||||
) == candidates
|
||||
with pytest.raises(ValueError, match="authentication failed"):
|
||||
CandidateOutputs.from_authenticated_dict(candidates.to_dict(), b"x" * 32)
|
||||
assert candidates.manifests == (candidate.manifest,)
|
||||
|
||||
with pytest.raises(ValueError, match="opaque coordinator identity"):
|
||||
CandidateOutput("../worker-path", candidate.owner_id, candidate.manifest)
|
||||
|
||||
|
||||
def test_trusted_single_manifest_compatibility_uses_an_anonymous_envelope() -> None:
|
||||
manifest = _manifest("output", "trusted")
|
||||
candidates = CandidateOutputs((manifest,))
|
||||
|
||||
assert candidates.manifests == (manifest,)
|
||||
assert candidates.candidates[0].owner_id is None
|
||||
decision = ExactArtifactVerifier().verify(_context(), candidates)
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
|
||||
|
||||
def test_raw_manifests_cannot_form_a_quorum() -> None:
|
||||
with pytest.raises(ValueError, match="one trusted candidate"):
|
||||
CandidateOutputs(
|
||||
(
|
||||
_manifest("output", "trusted-one"),
|
||||
_manifest("output", "trusted-two"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_multi_vote_context_automatically_requires_distinct_owners() -> None:
|
||||
assert _context(minimum_matches=2).require_distinct_owners
|
||||
assert _context(require_distinct_owners=True).require_distinct_owners
|
||||
|
||||
with pytest.raises(ValueError, match="boolean"):
|
||||
_context(require_distinct_owners=1) # type: ignore[arg-type]
|
||||
|
||||
with pytest.raises(ValueError, match="coordinator binding"):
|
||||
VerifyContext(
|
||||
expected_outputs={"result": OUTPUT_PORT},
|
||||
max_output_bytes=1_024,
|
||||
minimum_matches=2,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="at least two"):
|
||||
_context(trust_mode=TrustMode.UNTRUSTED_QUORUM)
|
||||
|
||||
|
||||
def test_exact_verifier_reports_no_candidates_as_inconclusive() -> None:
|
||||
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs(()))
|
||||
|
||||
assert decision.status is VerificationStatus.INCONCLUSIVE
|
||||
assert decision.reason_code == "no-candidates"
|
||||
assert decision.evidence == {"candidate_count": 0, "invalid_count": 0}
|
||||
assert decision.accepted_digest is None
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_candidates_that_violate_output_contract() -> None:
|
||||
invalid = _manifest("same-output", "invalid", port_name="undeclared")
|
||||
|
||||
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((invalid,)))
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "no-valid-candidates"
|
||||
assert decision.evidence == {"candidate_count": 1, "invalid_count": 1}
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_candidate_from_another_scientific_binding() -> None:
|
||||
candidate = _candidate("same-output", "other-job", owner="owner-one")
|
||||
forged_provenance = replace(
|
||||
candidate.manifest.provenance,
|
||||
parameters_digest=_sha256("different-parameters"),
|
||||
)
|
||||
forged = replace(
|
||||
candidate,
|
||||
manifest=replace(candidate.manifest, provenance=forged_provenance),
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((forged,)))
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "no-valid-candidates"
|
||||
|
||||
|
||||
def test_exact_verifier_accepts_unique_quorum_and_ignores_invalid_candidates() -> None:
|
||||
first = _candidate("same-output", "one", owner="owner-one")
|
||||
second = _candidate("same-output", "two", owner="owner-two")
|
||||
minority = _candidate("different-output", "three", owner="owner-three")
|
||||
invalid = _candidate(
|
||||
"same-output",
|
||||
"four",
|
||||
owner="owner-four",
|
||||
port_name="undeclared",
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
CandidateOutputs((first, second, minority, invalid)),
|
||||
)
|
||||
|
||||
assert first.manifest.digest == second.manifest.digest
|
||||
assert first.manifest.manifest_digest != second.manifest.manifest_digest
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "quorum-match"
|
||||
assert decision.accepted_digest == first.manifest.digest
|
||||
assert decision.evidence == {
|
||||
"matched": 2,
|
||||
"required": 2,
|
||||
"distinct_digests": 2,
|
||||
"invalid_count": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_conflicting_quorums() -> None:
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("group-a", "a-one", owner="a-one"),
|
||||
_candidate("group-a", "a-two", owner="a-two"),
|
||||
_candidate("group-b", "b-one", owner="b-one"),
|
||||
_candidate("group-b", "b-two", owner="b-two"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(_context(minimum_matches=2), candidates)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "conflicting-quorums"
|
||||
assert decision.accepted_digest is None
|
||||
assert decision.evidence["largest_group"] == 2
|
||||
assert decision.evidence["distinct_digests"] == 2
|
||||
|
||||
|
||||
def test_exact_verifier_distinguishes_insufficient_evidence_from_reference_mismatch() -> None:
|
||||
reference = _manifest("reference", "reference")
|
||||
one_mismatch = CandidateOutputs((_candidate("other", "one", owner="owner-one"),))
|
||||
two_mismatches = CandidateOutputs(
|
||||
(
|
||||
_candidate("other-a", "two", owner="owner-two"),
|
||||
_candidate("other-b", "three", owner="owner-three"),
|
||||
)
|
||||
)
|
||||
|
||||
insufficient = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2, reference=reference),
|
||||
one_mismatch,
|
||||
)
|
||||
rejected = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2, reference=reference),
|
||||
two_mismatches,
|
||||
)
|
||||
|
||||
assert (insufficient.status, insufficient.reason_code) == (
|
||||
VerificationStatus.INCONCLUSIVE,
|
||||
"insufficient-evidence",
|
||||
)
|
||||
assert (rejected.status, rejected.reason_code) == (
|
||||
VerificationStatus.REJECTED,
|
||||
"reference-mismatch",
|
||||
)
|
||||
|
||||
|
||||
def test_exact_verifier_accepts_declared_reference_quorum() -> None:
|
||||
reference = _manifest("reference", "reference")
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("reference", "worker-one", owner="owner-one"),
|
||||
_candidate("reference", "worker-two", owner="owner-two"),
|
||||
_candidate("other", "worker-three", owner="owner-three"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2, reference=reference),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "reference-match"
|
||||
assert decision.accepted_digest == reference.digest
|
||||
assert decision.evidence == {"matched": 2, "required": 2, "invalid_count": 0}
|
||||
|
||||
|
||||
def test_candidate_outputs_rejects_duplicate_candidate_ids() -> None:
|
||||
candidate = _candidate("output", "one", owner="owner-one")
|
||||
replay = _candidate(
|
||||
"different-output",
|
||||
"two",
|
||||
owner="owner-two",
|
||||
candidate_id=candidate.candidate_id,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="candidate_id values must be unique"):
|
||||
CandidateOutputs((candidate, replay))
|
||||
|
||||
|
||||
def test_exact_quorum_rejects_candidates_without_authenticated_owners() -> None:
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("output", "one", owner=None),
|
||||
_candidate("output", "two", owner="owner-two"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "coordinator-authentication-required"
|
||||
assert decision.evidence == {"candidate_count": 2, "unauthenticated_count": 1}
|
||||
|
||||
|
||||
def test_exact_verifier_counts_at_most_one_vote_per_owner() -> None:
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("output", "one", owner="same-owner"),
|
||||
_candidate("output", "two", owner="same-owner"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.INCONCLUSIVE
|
||||
assert decision.reason_code == "insufficient-evidence"
|
||||
assert decision.evidence == {
|
||||
"largest_group": 1,
|
||||
"required": 2,
|
||||
"distinct_digests": 1,
|
||||
"invalid_count": 0,
|
||||
"duplicate_owner_candidates": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_owner_equivocation_without_leaking_identity() -> None:
|
||||
owner = "equivocating-owner"
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("output-a", "one", owner=owner),
|
||||
_candidate("output-b", "two", owner=owner),
|
||||
_candidate("output-a", "three", owner="honest-owner"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "owner-equivocation"
|
||||
assert decision.accepted_digest is None
|
||||
assert decision.evidence == {
|
||||
"candidate_count": 3,
|
||||
"equivocating_owner_count": 1,
|
||||
}
|
||||
assert candidates.candidates[0].owner_id not in json.dumps(decision.to_dict())
|
||||
|
||||
|
||||
def test_numeric_verifier_accepts_nested_values_with_absolute_and_relative_tolerance() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001, relative=0.01))
|
||||
|
||||
decision = verifier.verify_values(
|
||||
{"energies": [10.0, 0.05], "converged": True},
|
||||
{"energies": [10.05, 0.0505], "converged": True},
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "within-tolerance"
|
||||
assert decision.accepted_digest is not None
|
||||
assert decision.evidence == {
|
||||
"absolute": 0.001,
|
||||
"relative": 0.01,
|
||||
"max_ulps": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_numeric_verifier_supports_ulp_tolerance() -> None:
|
||||
adjacent = math.nextafter(1.0, 2.0)
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(max_ulps=1))
|
||||
|
||||
decision = verifier.verify_values(1.0, adjacent)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
|
||||
|
||||
def test_numeric_verifier_reports_bounded_location_and_error_evidence() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.1))
|
||||
|
||||
decision = verifier.verify_values(
|
||||
{"matrix": [[1.0, 2.0]]},
|
||||
{"matrix": [[1.0, 2.5]]},
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "numeric-mismatch"
|
||||
assert decision.evidence["location"] == "$.matrix[0][1]"
|
||||
assert decision.evidence["absolute_error"] == pytest.approx(0.5)
|
||||
assert decision.evidence["allowed_error"] == pytest.approx(0.1)
|
||||
assert isinstance(decision.evidence["ulp_distance"], int)
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_shape_changes_before_value_comparison() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values(
|
||||
{"energy": 1.0, "iterations": 4},
|
||||
{"energy": 1.0, "converged": True},
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "shape-mismatch"
|
||||
assert decision.evidence == {
|
||||
"location": "$",
|
||||
"missing_keys": ("iterations",),
|
||||
"extra_keys": ("converged",),
|
||||
}
|
||||
|
||||
|
||||
def test_numeric_verifier_applies_declared_nan_policy() -> None:
|
||||
reject = NumericToleranceVerifier(NumericTolerance(nan_policy="reject"))
|
||||
equal = NumericToleranceVerifier(NumericTolerance(nan_policy="equal"))
|
||||
|
||||
rejected = reject.verify_values(float("nan"), float("nan"))
|
||||
accepted = equal.verify_values(float("nan"), float("nan"))
|
||||
|
||||
assert (rejected.status, rejected.reason_code) == (
|
||||
VerificationStatus.REJECTED,
|
||||
"nan-policy",
|
||||
)
|
||||
assert accepted.status is VerificationStatus.ACCEPTED
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"factory",
|
||||
[
|
||||
lambda: NumericTolerance(absolute=-0.1),
|
||||
lambda: NumericTolerance(relative=float("inf")),
|
||||
lambda: NumericTolerance(max_ulps=True),
|
||||
lambda: NumericTolerance(nan_policy="propagate"),
|
||||
lambda: NumericTolerance(max_elements=0),
|
||||
],
|
||||
)
|
||||
def test_numeric_tolerance_rejects_ambiguous_or_non_finite_policy(
|
||||
factory: Callable[[], NumericTolerance],
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="numeric tolerance"):
|
||||
factory()
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_unrepresentable_integer_without_raising() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values(10**10_000, 10**10_000)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "numeric-range"
|
||||
assert decision.evidence["location"] == "$"
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_shapes_above_its_manifest_bound() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(max_elements=3))
|
||||
|
||||
decision = verifier.verify_values([1, 2, 3], [1, 2, 3])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "element-limit"
|
||||
assert decision.evidence == {"max_elements": 3}
|
||||
assert verifier.configuration["max_elements"] == 3
|
||||
|
||||
|
||||
def test_numeric_verifier_does_not_round_mixed_integer_and_float_values() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values(2**53 + 1, float(2**53))
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "numeric-mismatch"
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_non_json_mapping_keys() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values({1: 2.0}, {1: 2.0})
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "type-mismatch"
|
||||
|
||||
|
||||
def test_numeric_verifier_implements_manifest_verifier_protocol_with_loader() -> None:
|
||||
reference = _manifest("reference", "reference")
|
||||
candidate = _manifest("candidate", "candidate")
|
||||
values = {
|
||||
reference.outputs["result"].items[0].artifact.sha256: {"energy": 1.0},
|
||||
candidate.outputs["result"].items[0].artifact.sha256: {"energy": 1.0001},
|
||||
}
|
||||
|
||||
def load(manifest: OutputManifest) -> object:
|
||||
return values[manifest.outputs["result"].items[0].artifact.sha256]
|
||||
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001), load)
|
||||
decision = verifier.verify(
|
||||
_context(reference=reference),
|
||||
CandidateOutputs((candidate,)),
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.accepted_digest == reference.digest
|
||||
|
||||
|
||||
def test_manifest_verifiers_fail_closed_without_artifact_loaders() -> None:
|
||||
candidate = CandidateOutputs((_manifest("candidate", "candidate"),))
|
||||
|
||||
numeric = NumericToleranceVerifier(NumericTolerance()).verify(_context(), candidate)
|
||||
canonical = CanonicalRecordVerifier(_canonical_json_record).verify(_context(), candidate)
|
||||
|
||||
assert numeric.reason_code == "loader-unavailable"
|
||||
assert canonical.reason_code == "loader-unavailable"
|
||||
|
||||
|
||||
def _canonical_json_record(record: object) -> bytes:
|
||||
return json.dumps(record, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def test_canonical_record_verifier_accepts_normalized_records() -> None:
|
||||
verifier = CanonicalRecordVerifier(_canonical_json_record)
|
||||
expected = [{"name": "molecule", "score": 0.75}, {"id": 2}]
|
||||
actual = [{"score": 0.75, "name": "molecule"}, {"id": 2}]
|
||||
|
||||
decision = verifier.verify_records(expected, actual)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "canonical-match"
|
||||
assert decision.evidence == {"records": 2}
|
||||
assert decision.accepted_digest is not None
|
||||
|
||||
|
||||
def test_canonical_record_verifier_rejects_content_or_count_mismatch() -> None:
|
||||
verifier = CanonicalRecordVerifier(_canonical_json_record)
|
||||
|
||||
decision = verifier.verify_records([{"id": 1}, {"id": 2}], [{"id": 1}])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "canonical-mismatch"
|
||||
assert decision.evidence == {"expected_records": 2, "actual_records": 1}
|
||||
|
||||
|
||||
def test_canonical_record_verifier_enforces_record_limit() -> None:
|
||||
verifier = CanonicalRecordVerifier(_canonical_json_record, max_records=2)
|
||||
|
||||
decision = verifier.verify_records([1, 2, 3], [1, 2, 3])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "record-limit"
|
||||
assert decision.evidence == {"max_records": 2}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"canonicalizer",
|
||||
[
|
||||
lambda _record: "not-bytes",
|
||||
lambda _record: (_ for _ in ()).throw(ValueError("/private/worker/path")),
|
||||
],
|
||||
)
|
||||
def test_canonical_record_verifier_sanitizes_canonicalization_failures(
|
||||
canonicalizer: Callable[[object], object],
|
||||
) -> None:
|
||||
verifier = CanonicalRecordVerifier(canonicalizer) # type: ignore[arg-type]
|
||||
|
||||
decision = verifier.verify_records([{"id": 1}], [{"id": 1}])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "canonicalization-failed"
|
||||
assert decision.evidence == {}
|
||||
assert decision.accepted_digest is None
|
||||
Reference in New Issue
Block a user