745 lines
25 KiB
Python
745 lines
25 KiB
Python
"""Contract tests for SDK verifier decisions and built-in verifiers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from collections.abc import Callable
|
|
from dataclasses import replace
|
|
from uuid import NAMESPACE_URL, uuid5
|
|
|
|
import pytest
|
|
|
|
from scimesh.sdk import (
|
|
ArtifactCollection,
|
|
ArtifactRef,
|
|
ArtifactSchema,
|
|
CandidateOutput,
|
|
CandidateOutputs,
|
|
CanonicalRecordVerifier,
|
|
ComponentRef,
|
|
ExactArtifactVerifier,
|
|
NumericTolerance,
|
|
NumericToleranceVerifier,
|
|
OutputManifest,
|
|
PortSpec,
|
|
Provenance,
|
|
SchemaRef,
|
|
TrustMode,
|
|
VerificationDecision,
|
|
VerificationBinding,
|
|
VerificationStatus,
|
|
VerifyContext,
|
|
WorkloadId,
|
|
)
|
|
|
|
|
|
def _sha256(seed: str) -> str:
|
|
return hashlib.sha256(seed.encode("utf-8")).hexdigest()
|
|
|
|
|
|
OUTPUT_SCHEMA = ArtifactSchema(
|
|
ref=SchemaRef("verification-result", 1),
|
|
media_type="application/json",
|
|
encoding="utf-8",
|
|
max_bytes=1_024,
|
|
validator=ComponentRef("json-document", 1),
|
|
)
|
|
OUTPUT_PORT = PortSpec(OUTPUT_SCHEMA)
|
|
AUTHENTICATION_KEY = b"sdk-verification-test-key-000001"
|
|
JOB_ID = str(uuid5(NAMESPACE_URL, "verification-job"))
|
|
TASK_ID = str(uuid5(NAMESPACE_URL, "verification-task"))
|
|
EXECUTION_CONTRACT_DIGEST = _sha256("execution-contract")
|
|
|
|
|
|
def _artifact(seed: str, *, size_bytes: int = 16) -> ArtifactRef:
|
|
return ArtifactRef(
|
|
artifact_id=str(uuid5(NAMESPACE_URL, f"artifact:{seed}")),
|
|
sha256=_sha256(seed),
|
|
schema=OUTPUT_SCHEMA.ref,
|
|
media_type=OUTPUT_SCHEMA.media_type,
|
|
size_bytes=size_bytes,
|
|
)
|
|
|
|
|
|
def _provenance(attempt: str) -> Provenance:
|
|
return Provenance(
|
|
workload=WorkloadId("verification-fixture", "1.0.0"),
|
|
sdk_api_version="1.0.0",
|
|
protocol_version="1.0",
|
|
manifest_schema_version=1,
|
|
workflow_schema_version=1,
|
|
verifier=ComponentRef("exact-artifact", 1),
|
|
artifact_schemas=(OUTPUT_SCHEMA.ref,),
|
|
package_digest=f"sha256:{_sha256('package')}",
|
|
manifest_digest=_sha256("manifest"),
|
|
environment_digest=f"sha256:{_sha256('environment')}",
|
|
worker_runtime={"attempt": attempt},
|
|
allocated_resource_ids=(f"cpu-{attempt}",),
|
|
parameters_digest=_sha256("parameters"),
|
|
input_collection_digest=_sha256("inputs"),
|
|
execution_contract_digest=EXECUTION_CONTRACT_DIGEST,
|
|
selected_features={"exact-verifier": "1.0.0"},
|
|
optional_fallbacks={},
|
|
job_id=JOB_ID,
|
|
task_id=TASK_ID,
|
|
started_at="2026-08-01T10:00:00Z",
|
|
finished_at="2026-08-01T10:00:01Z",
|
|
)
|
|
|
|
|
|
def _manifest(
|
|
output_seed: str,
|
|
attempt: str,
|
|
*,
|
|
port_name: str = "result",
|
|
size_bytes: int = 16,
|
|
) -> OutputManifest:
|
|
return OutputManifest(
|
|
task_key="verify/0",
|
|
outputs={port_name: ArtifactCollection.single(_artifact(output_seed, size_bytes=size_bytes))},
|
|
metrics={"elapsed_seconds": 1.0},
|
|
provenance=_provenance(attempt),
|
|
)
|
|
|
|
|
|
def _candidate(
|
|
output_seed: str,
|
|
attempt: str,
|
|
*,
|
|
owner: str | None,
|
|
candidate_id: str | None = None,
|
|
port_name: str = "result",
|
|
size_bytes: int = 16,
|
|
authenticated: bool = True,
|
|
) -> CandidateOutput:
|
|
resolved_candidate_id = candidate_id or str(
|
|
uuid5(NAMESPACE_URL, f"candidate:{attempt}")
|
|
)
|
|
resolved_owner_id = None if owner is None else str(uuid5(NAMESPACE_URL, f"owner:{owner}"))
|
|
manifest = _manifest(
|
|
output_seed,
|
|
attempt,
|
|
port_name=port_name,
|
|
size_bytes=size_bytes,
|
|
)
|
|
if resolved_owner_id is not None and authenticated:
|
|
return CandidateOutput.from_coordinator_record(
|
|
resolved_candidate_id,
|
|
resolved_owner_id,
|
|
manifest,
|
|
AUTHENTICATION_KEY,
|
|
)
|
|
return CandidateOutput(resolved_candidate_id, resolved_owner_id, manifest)
|
|
|
|
|
|
def _context(
|
|
*,
|
|
minimum_matches: int = 1,
|
|
reference: OutputManifest | None = None,
|
|
require_distinct_owners: bool = False,
|
|
trust_mode: TrustMode = TrustMode.TRUSTED,
|
|
) -> VerifyContext:
|
|
provenance = _provenance("binding")
|
|
return VerifyContext(
|
|
expected_outputs={"result": OUTPUT_PORT},
|
|
max_output_bytes=1_024,
|
|
minimum_matches=minimum_matches,
|
|
reference=reference,
|
|
require_distinct_owners=require_distinct_owners,
|
|
binding=VerificationBinding(
|
|
workload=provenance.workload,
|
|
task_key="verify/0",
|
|
package_digest=provenance.package_digest,
|
|
manifest_digest=provenance.manifest_digest,
|
|
environment_digest=provenance.environment_digest,
|
|
parameters_digest=provenance.parameters_digest,
|
|
input_collection_digest=provenance.input_collection_digest,
|
|
execution_contract_digest=provenance.execution_contract_digest,
|
|
selected_features=provenance.selected_features,
|
|
optional_fallbacks=provenance.optional_fallbacks,
|
|
job_id=provenance.job_id,
|
|
task_id=provenance.task_id,
|
|
verifier=provenance.verifier,
|
|
sdk_api_version=provenance.sdk_api_version,
|
|
protocol_version=provenance.protocol_version,
|
|
manifest_schema_version=provenance.manifest_schema_version,
|
|
workflow_schema_version=provenance.workflow_schema_version,
|
|
artifact_schemas=provenance.artifact_schemas,
|
|
trust_mode=trust_mode,
|
|
),
|
|
trust_mode=trust_mode,
|
|
)
|
|
|
|
|
|
def test_verification_decision_is_strict_immutable_and_round_trips() -> None:
|
|
source = {"summary": {"counts": [1, 2]}}
|
|
decision = VerificationDecision(
|
|
VerificationStatus.REJECTED,
|
|
ComponentRef("test-verifier", 1),
|
|
"comparison-failed",
|
|
source,
|
|
)
|
|
|
|
source["summary"]["counts"].append(3) # type: ignore[index, union-attr]
|
|
|
|
assert decision.evidence["summary"]["counts"] == (1, 2)
|
|
assert VerificationDecision.from_dict(decision.to_dict()) == decision
|
|
with pytest.raises(TypeError):
|
|
decision.evidence["new"] = True # type: ignore[index]
|
|
with pytest.raises(TypeError):
|
|
decision.evidence["summary"]["new"] = True # type: ignore[index]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("status", "accepted_digest"),
|
|
[
|
|
(VerificationStatus.ACCEPTED, None),
|
|
(VerificationStatus.REJECTED, "a" * 64),
|
|
(VerificationStatus.INCONCLUSIVE, "a" * 64),
|
|
],
|
|
)
|
|
def test_verification_decision_enforces_digest_status_invariant(
|
|
status: VerificationStatus,
|
|
accepted_digest: str | None,
|
|
) -> None:
|
|
with pytest.raises(ValueError, match="accepted_digest|accepted verification"):
|
|
VerificationDecision(
|
|
status,
|
|
ComponentRef("test-verifier", 1),
|
|
"test-result",
|
|
{},
|
|
accepted_digest,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"unsafe",
|
|
[
|
|
"/home/worker/private.log",
|
|
"https://worker.invalid/evidence",
|
|
"run-123/tasks/map/result.csv",
|
|
"path=attempts/job-123/private.txt",
|
|
"https%253A%252F%252Fworker.invalid%252Fevidence%253Ftoken%253Dsecret",
|
|
],
|
|
)
|
|
def test_verification_decision_rejects_private_locations(unsafe: str) -> None:
|
|
with pytest.raises(ValueError, match="URI or local path"):
|
|
VerificationDecision(
|
|
VerificationStatus.REJECTED,
|
|
ComponentRef("test-verifier", 1),
|
|
"unsafe-evidence",
|
|
{"detail": unsafe},
|
|
)
|
|
|
|
|
|
def test_verification_decision_bounds_evidence_and_rejects_unknown_fields() -> None:
|
|
with pytest.raises(ValueError, match="exceeds 16 KiB"):
|
|
VerificationDecision(
|
|
VerificationStatus.REJECTED,
|
|
ComponentRef("test-verifier", 1),
|
|
"oversized-evidence",
|
|
{"detail": "x" * 17_000},
|
|
)
|
|
|
|
payload = VerificationDecision(
|
|
VerificationStatus.REJECTED,
|
|
ComponentRef("test-verifier", 1),
|
|
"test-result",
|
|
{},
|
|
).to_dict()
|
|
payload["unexpected"] = True
|
|
with pytest.raises(ValueError, match="unknown unexpected"):
|
|
VerificationDecision.from_dict(payload)
|
|
|
|
|
|
def test_candidate_output_envelope_is_strict_and_round_trips() -> None:
|
|
candidate = _candidate("output", "attempt-one", owner="owner-one")
|
|
|
|
decoded_candidate = CandidateOutput.from_dict(candidate.to_dict())
|
|
assert decoded_candidate.to_dict() == candidate.to_dict()
|
|
assert not decoded_candidate.coordinator_authenticated
|
|
candidates = CandidateOutputs(candidates=(candidate,))
|
|
decoded = CandidateOutputs.from_dict(candidates.to_dict())
|
|
assert not decoded.candidates[0].coordinator_authenticated
|
|
assert CandidateOutputs.from_authenticated_dict(
|
|
candidates.to_dict(), AUTHENTICATION_KEY
|
|
) == candidates
|
|
with pytest.raises(ValueError, match="authentication failed"):
|
|
CandidateOutputs.from_authenticated_dict(candidates.to_dict(), b"x" * 32)
|
|
assert candidates.manifests == (candidate.manifest,)
|
|
|
|
with pytest.raises(ValueError, match="opaque coordinator identity"):
|
|
CandidateOutput("../worker-path", candidate.owner_id, candidate.manifest)
|
|
|
|
|
|
def test_trusted_single_manifest_compatibility_uses_an_anonymous_envelope() -> None:
|
|
manifest = _manifest("output", "trusted")
|
|
candidates = CandidateOutputs((manifest,))
|
|
|
|
assert candidates.manifests == (manifest,)
|
|
assert candidates.candidates[0].owner_id is None
|
|
decision = ExactArtifactVerifier().verify(_context(), candidates)
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
|
|
|
|
def test_raw_manifests_cannot_form_a_quorum() -> None:
|
|
with pytest.raises(ValueError, match="one trusted candidate"):
|
|
CandidateOutputs(
|
|
(
|
|
_manifest("output", "trusted-one"),
|
|
_manifest("output", "trusted-two"),
|
|
)
|
|
)
|
|
|
|
|
|
def test_multi_vote_context_automatically_requires_distinct_owners() -> None:
|
|
assert _context(minimum_matches=2).require_distinct_owners
|
|
assert _context(require_distinct_owners=True).require_distinct_owners
|
|
|
|
with pytest.raises(ValueError, match="boolean"):
|
|
_context(require_distinct_owners=1) # type: ignore[arg-type]
|
|
|
|
with pytest.raises(ValueError, match="coordinator binding"):
|
|
VerifyContext(
|
|
expected_outputs={"result": OUTPUT_PORT},
|
|
max_output_bytes=1_024,
|
|
minimum_matches=2,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="at least two"):
|
|
_context(trust_mode=TrustMode.UNTRUSTED_QUORUM)
|
|
|
|
|
|
def test_exact_verifier_reports_no_candidates_as_inconclusive() -> None:
|
|
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs(()))
|
|
|
|
assert decision.status is VerificationStatus.INCONCLUSIVE
|
|
assert decision.reason_code == "no-candidates"
|
|
assert decision.evidence == {"candidate_count": 0, "invalid_count": 0}
|
|
assert decision.accepted_digest is None
|
|
|
|
|
|
def test_exact_verifier_rejects_candidates_that_violate_output_contract() -> None:
|
|
invalid = _manifest("same-output", "invalid", port_name="undeclared")
|
|
|
|
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((invalid,)))
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "no-valid-candidates"
|
|
assert decision.evidence == {"candidate_count": 1, "invalid_count": 1}
|
|
|
|
|
|
def test_exact_verifier_rejects_candidate_from_another_scientific_binding() -> None:
|
|
candidate = _candidate("same-output", "other-job", owner="owner-one")
|
|
forged_provenance = replace(
|
|
candidate.manifest.provenance,
|
|
parameters_digest=_sha256("different-parameters"),
|
|
)
|
|
forged = replace(
|
|
candidate,
|
|
manifest=replace(candidate.manifest, provenance=forged_provenance),
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((forged,)))
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "no-valid-candidates"
|
|
|
|
|
|
def test_exact_verifier_accepts_unique_quorum_and_ignores_invalid_candidates() -> None:
|
|
first = _candidate("same-output", "one", owner="owner-one")
|
|
second = _candidate("same-output", "two", owner="owner-two")
|
|
minority = _candidate("different-output", "three", owner="owner-three")
|
|
invalid = _candidate(
|
|
"same-output",
|
|
"four",
|
|
owner="owner-four",
|
|
port_name="undeclared",
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2),
|
|
CandidateOutputs((first, second, minority, invalid)),
|
|
)
|
|
|
|
assert first.manifest.digest == second.manifest.digest
|
|
assert first.manifest.manifest_digest != second.manifest.manifest_digest
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
assert decision.reason_code == "quorum-match"
|
|
assert decision.accepted_digest == first.manifest.digest
|
|
assert decision.evidence == {
|
|
"matched": 2,
|
|
"required": 2,
|
|
"distinct_digests": 2,
|
|
"invalid_count": 1,
|
|
}
|
|
|
|
|
|
def test_exact_verifier_rejects_conflicting_quorums() -> None:
|
|
candidates = CandidateOutputs(
|
|
(
|
|
_candidate("group-a", "a-one", owner="a-one"),
|
|
_candidate("group-a", "a-two", owner="a-two"),
|
|
_candidate("group-b", "b-one", owner="b-one"),
|
|
_candidate("group-b", "b-two", owner="b-two"),
|
|
)
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(_context(minimum_matches=2), candidates)
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "conflicting-quorums"
|
|
assert decision.accepted_digest is None
|
|
assert decision.evidence["largest_group"] == 2
|
|
assert decision.evidence["distinct_digests"] == 2
|
|
|
|
|
|
def test_exact_verifier_distinguishes_insufficient_evidence_from_reference_mismatch() -> None:
|
|
reference = _manifest("reference", "reference")
|
|
one_mismatch = CandidateOutputs((_candidate("other", "one", owner="owner-one"),))
|
|
two_mismatches = CandidateOutputs(
|
|
(
|
|
_candidate("other-a", "two", owner="owner-two"),
|
|
_candidate("other-b", "three", owner="owner-three"),
|
|
)
|
|
)
|
|
|
|
insufficient = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2, reference=reference),
|
|
one_mismatch,
|
|
)
|
|
rejected = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2, reference=reference),
|
|
two_mismatches,
|
|
)
|
|
|
|
assert (insufficient.status, insufficient.reason_code) == (
|
|
VerificationStatus.INCONCLUSIVE,
|
|
"insufficient-evidence",
|
|
)
|
|
assert (rejected.status, rejected.reason_code) == (
|
|
VerificationStatus.REJECTED,
|
|
"reference-mismatch",
|
|
)
|
|
|
|
|
|
def test_exact_verifier_accepts_declared_reference_quorum() -> None:
|
|
reference = _manifest("reference", "reference")
|
|
candidates = CandidateOutputs(
|
|
(
|
|
_candidate("reference", "worker-one", owner="owner-one"),
|
|
_candidate("reference", "worker-two", owner="owner-two"),
|
|
_candidate("other", "worker-three", owner="owner-three"),
|
|
)
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2, reference=reference),
|
|
candidates,
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
assert decision.reason_code == "reference-match"
|
|
assert decision.accepted_digest == reference.digest
|
|
assert decision.evidence == {"matched": 2, "required": 2, "invalid_count": 0}
|
|
|
|
|
|
def test_candidate_outputs_rejects_duplicate_candidate_ids() -> None:
|
|
candidate = _candidate("output", "one", owner="owner-one")
|
|
replay = _candidate(
|
|
"different-output",
|
|
"two",
|
|
owner="owner-two",
|
|
candidate_id=candidate.candidate_id,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="candidate_id values must be unique"):
|
|
CandidateOutputs((candidate, replay))
|
|
|
|
|
|
def test_exact_quorum_rejects_candidates_without_authenticated_owners() -> None:
|
|
candidates = CandidateOutputs(
|
|
(
|
|
_candidate("output", "one", owner=None),
|
|
_candidate("output", "two", owner="owner-two"),
|
|
)
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2),
|
|
candidates,
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "coordinator-authentication-required"
|
|
assert decision.evidence == {"candidate_count": 2, "unauthenticated_count": 1}
|
|
|
|
|
|
def test_exact_verifier_counts_at_most_one_vote_per_owner() -> None:
|
|
candidates = CandidateOutputs(
|
|
(
|
|
_candidate("output", "one", owner="same-owner"),
|
|
_candidate("output", "two", owner="same-owner"),
|
|
)
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2),
|
|
candidates,
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.INCONCLUSIVE
|
|
assert decision.reason_code == "insufficient-evidence"
|
|
assert decision.evidence == {
|
|
"largest_group": 1,
|
|
"required": 2,
|
|
"distinct_digests": 1,
|
|
"invalid_count": 0,
|
|
"duplicate_owner_candidates": 1,
|
|
}
|
|
|
|
|
|
def test_exact_verifier_rejects_owner_equivocation_without_leaking_identity() -> None:
|
|
owner = "equivocating-owner"
|
|
candidates = CandidateOutputs(
|
|
(
|
|
_candidate("output-a", "one", owner=owner),
|
|
_candidate("output-b", "two", owner=owner),
|
|
_candidate("output-a", "three", owner="honest-owner"),
|
|
)
|
|
)
|
|
|
|
decision = ExactArtifactVerifier().verify(
|
|
_context(minimum_matches=2),
|
|
candidates,
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "owner-equivocation"
|
|
assert decision.accepted_digest is None
|
|
assert decision.evidence == {
|
|
"candidate_count": 3,
|
|
"equivocating_owner_count": 1,
|
|
}
|
|
assert (candidates.candidates[0].owner_id or "") not in json.dumps(decision.to_dict())
|
|
|
|
|
|
def test_numeric_verifier_accepts_nested_values_with_absolute_and_relative_tolerance() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001, relative=0.01))
|
|
|
|
decision = verifier.verify_values(
|
|
{"energies": [10.0, 0.05], "converged": True},
|
|
{"energies": [10.05, 0.0505], "converged": True},
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
assert decision.reason_code == "within-tolerance"
|
|
assert decision.accepted_digest is not None
|
|
assert decision.evidence == {
|
|
"absolute": 0.001,
|
|
"relative": 0.01,
|
|
"max_ulps": 0,
|
|
}
|
|
|
|
|
|
def test_numeric_verifier_supports_ulp_tolerance() -> None:
|
|
adjacent = math.nextafter(1.0, 2.0)
|
|
verifier = NumericToleranceVerifier(NumericTolerance(max_ulps=1))
|
|
|
|
decision = verifier.verify_values(1.0, adjacent)
|
|
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
|
|
|
|
def test_numeric_verifier_reports_bounded_location_and_error_evidence() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.1))
|
|
|
|
decision = verifier.verify_values(
|
|
{"matrix": [[1.0, 2.0]]},
|
|
{"matrix": [[1.0, 2.5]]},
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "numeric-mismatch"
|
|
assert decision.evidence["location"] == "$.matrix[0][1]"
|
|
assert decision.evidence["absolute_error"] == pytest.approx(0.5)
|
|
assert decision.evidence["allowed_error"] == pytest.approx(0.1)
|
|
assert isinstance(decision.evidence["ulp_distance"], int)
|
|
|
|
|
|
def test_numeric_verifier_rejects_shape_changes_before_value_comparison() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance())
|
|
|
|
decision = verifier.verify_values(
|
|
{"energy": 1.0, "iterations": 4},
|
|
{"energy": 1.0, "converged": True},
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "shape-mismatch"
|
|
assert decision.evidence == {
|
|
"location": "$",
|
|
"missing_keys": ("iterations",),
|
|
"extra_keys": ("converged",),
|
|
}
|
|
|
|
|
|
def test_numeric_verifier_applies_declared_nan_policy() -> None:
|
|
reject = NumericToleranceVerifier(NumericTolerance(nan_policy="reject"))
|
|
equal = NumericToleranceVerifier(NumericTolerance(nan_policy="equal"))
|
|
|
|
rejected = reject.verify_values(float("nan"), float("nan"))
|
|
accepted = equal.verify_values(float("nan"), float("nan"))
|
|
|
|
assert (rejected.status, rejected.reason_code) == (
|
|
VerificationStatus.REJECTED,
|
|
"nan-policy",
|
|
)
|
|
assert accepted.status is VerificationStatus.ACCEPTED
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"factory",
|
|
[
|
|
lambda: NumericTolerance(absolute=-0.1),
|
|
lambda: NumericTolerance(relative=float("inf")),
|
|
lambda: NumericTolerance(max_ulps=True),
|
|
lambda: NumericTolerance(nan_policy="propagate"),
|
|
lambda: NumericTolerance(max_elements=0),
|
|
],
|
|
)
|
|
def test_numeric_tolerance_rejects_ambiguous_or_non_finite_policy(
|
|
factory: Callable[[], NumericTolerance],
|
|
) -> None:
|
|
with pytest.raises(ValueError, match="numeric tolerance"):
|
|
factory()
|
|
|
|
|
|
def test_numeric_verifier_rejects_unrepresentable_integer_without_raising() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance())
|
|
|
|
decision = verifier.verify_values(10**10_000, 10**10_000)
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "numeric-range"
|
|
assert decision.evidence["location"] == "$"
|
|
|
|
|
|
def test_numeric_verifier_rejects_shapes_above_its_manifest_bound() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance(max_elements=3))
|
|
|
|
decision = verifier.verify_values([1, 2, 3], [1, 2, 3])
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "element-limit"
|
|
assert decision.evidence == {"max_elements": 3}
|
|
assert verifier.configuration["max_elements"] == 3
|
|
|
|
|
|
def test_numeric_verifier_does_not_round_mixed_integer_and_float_values() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance())
|
|
|
|
decision = verifier.verify_values(2**53 + 1, float(2**53))
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "numeric-mismatch"
|
|
|
|
|
|
def test_numeric_verifier_rejects_non_json_mapping_keys() -> None:
|
|
verifier = NumericToleranceVerifier(NumericTolerance())
|
|
|
|
decision = verifier.verify_values({1: 2.0}, {1: 2.0})
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "type-mismatch"
|
|
|
|
|
|
def test_numeric_verifier_implements_manifest_verifier_protocol_with_loader() -> None:
|
|
reference = _manifest("reference", "reference")
|
|
candidate = _manifest("candidate", "candidate")
|
|
values = {
|
|
reference.outputs["result"].items[0].artifact.sha256: {"energy": 1.0},
|
|
candidate.outputs["result"].items[0].artifact.sha256: {"energy": 1.0001},
|
|
}
|
|
|
|
def load(manifest: OutputManifest) -> object:
|
|
return values[manifest.outputs["result"].items[0].artifact.sha256]
|
|
|
|
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001), load)
|
|
decision = verifier.verify(
|
|
_context(reference=reference),
|
|
CandidateOutputs((candidate,)),
|
|
)
|
|
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
assert decision.accepted_digest == reference.digest
|
|
|
|
|
|
def test_manifest_verifiers_fail_closed_without_artifact_loaders() -> None:
|
|
candidate = CandidateOutputs((_manifest("candidate", "candidate"),))
|
|
|
|
numeric = NumericToleranceVerifier(NumericTolerance()).verify(_context(), candidate)
|
|
canonical = CanonicalRecordVerifier(_canonical_json_record).verify(_context(), candidate)
|
|
|
|
assert numeric.reason_code == "loader-unavailable"
|
|
assert canonical.reason_code == "loader-unavailable"
|
|
|
|
|
|
def _canonical_json_record(record: object) -> bytes:
|
|
return json.dumps(record, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def test_canonical_record_verifier_accepts_normalized_records() -> None:
|
|
verifier = CanonicalRecordVerifier(_canonical_json_record)
|
|
expected = [{"name": "molecule", "score": 0.75}, {"id": 2}]
|
|
actual = [{"score": 0.75, "name": "molecule"}, {"id": 2}]
|
|
|
|
decision = verifier.verify_records(expected, actual)
|
|
|
|
assert decision.status is VerificationStatus.ACCEPTED
|
|
assert decision.reason_code == "canonical-match"
|
|
assert decision.evidence == {"records": 2}
|
|
assert decision.accepted_digest is not None
|
|
|
|
|
|
def test_canonical_record_verifier_rejects_content_or_count_mismatch() -> None:
|
|
verifier = CanonicalRecordVerifier(_canonical_json_record)
|
|
|
|
decision = verifier.verify_records([{"id": 1}, {"id": 2}], [{"id": 1}])
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "canonical-mismatch"
|
|
assert decision.evidence == {"expected_records": 2, "actual_records": 1}
|
|
|
|
|
|
def test_canonical_record_verifier_enforces_record_limit() -> None:
|
|
verifier = CanonicalRecordVerifier(_canonical_json_record, max_records=2)
|
|
|
|
decision = verifier.verify_records([1, 2, 3], [1, 2, 3])
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "record-limit"
|
|
assert decision.evidence == {"max_records": 2}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"canonicalizer",
|
|
[
|
|
lambda _record: "not-bytes",
|
|
lambda _record: (_ for _ in ()).throw(ValueError("/private/worker/path")),
|
|
],
|
|
)
|
|
def test_canonical_record_verifier_sanitizes_canonicalization_failures(
|
|
canonicalizer: Callable[[object], object],
|
|
) -> None:
|
|
verifier = CanonicalRecordVerifier(canonicalizer) # type: ignore[arg-type]
|
|
|
|
decision = verifier.verify_records([{"id": 1}], [{"id": 1}])
|
|
|
|
assert decision.status is VerificationStatus.REJECTED
|
|
assert decision.reason_code == "canonicalization-failed"
|
|
assert decision.evidence == {}
|
|
assert decision.accepted_digest is None
|