Add workload SDK foundation
This commit is contained in:
@@ -0,0 +1,744 @@
|
||||
"""Contract tests for SDK verifier decisions and built-in verifiers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import pytest
|
||||
|
||||
from scimesh.sdk import (
|
||||
ArtifactCollection,
|
||||
ArtifactRef,
|
||||
ArtifactSchema,
|
||||
CandidateOutput,
|
||||
CandidateOutputs,
|
||||
CanonicalRecordVerifier,
|
||||
ComponentRef,
|
||||
ExactArtifactVerifier,
|
||||
NumericTolerance,
|
||||
NumericToleranceVerifier,
|
||||
OutputManifest,
|
||||
PortSpec,
|
||||
Provenance,
|
||||
SchemaRef,
|
||||
TrustMode,
|
||||
VerificationDecision,
|
||||
VerificationBinding,
|
||||
VerificationStatus,
|
||||
VerifyContext,
|
||||
WorkloadId,
|
||||
)
|
||||
|
||||
|
||||
def _sha256(seed: str) -> str:
|
||||
return hashlib.sha256(seed.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
OUTPUT_SCHEMA = ArtifactSchema(
|
||||
ref=SchemaRef("verification-result", 1),
|
||||
media_type="application/json",
|
||||
encoding="utf-8",
|
||||
max_bytes=1_024,
|
||||
validator=ComponentRef("json-document", 1),
|
||||
)
|
||||
OUTPUT_PORT = PortSpec(OUTPUT_SCHEMA)
|
||||
AUTHENTICATION_KEY = b"sdk-verification-test-key-000001"
|
||||
JOB_ID = str(uuid5(NAMESPACE_URL, "verification-job"))
|
||||
TASK_ID = str(uuid5(NAMESPACE_URL, "verification-task"))
|
||||
EXECUTION_CONTRACT_DIGEST = _sha256("execution-contract")
|
||||
|
||||
|
||||
def _artifact(seed: str, *, size_bytes: int = 16) -> ArtifactRef:
|
||||
return ArtifactRef(
|
||||
artifact_id=str(uuid5(NAMESPACE_URL, f"artifact:{seed}")),
|
||||
sha256=_sha256(seed),
|
||||
schema=OUTPUT_SCHEMA.ref,
|
||||
media_type=OUTPUT_SCHEMA.media_type,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _provenance(attempt: str) -> Provenance:
|
||||
return Provenance(
|
||||
workload=WorkloadId("verification-fixture", "1.0.0"),
|
||||
sdk_api_version="1.0.0",
|
||||
protocol_version="1.0",
|
||||
manifest_schema_version=1,
|
||||
workflow_schema_version=1,
|
||||
verifier=ComponentRef("exact-artifact", 1),
|
||||
artifact_schemas=(OUTPUT_SCHEMA.ref,),
|
||||
package_digest=f"sha256:{_sha256('package')}",
|
||||
manifest_digest=_sha256("manifest"),
|
||||
environment_digest=f"sha256:{_sha256('environment')}",
|
||||
worker_runtime={"attempt": attempt},
|
||||
allocated_resource_ids=(f"cpu-{attempt}",),
|
||||
parameters_digest=_sha256("parameters"),
|
||||
input_collection_digest=_sha256("inputs"),
|
||||
execution_contract_digest=EXECUTION_CONTRACT_DIGEST,
|
||||
selected_features={"exact-verifier": "1.0.0"},
|
||||
optional_fallbacks={},
|
||||
job_id=JOB_ID,
|
||||
task_id=TASK_ID,
|
||||
started_at="2026-08-01T10:00:00Z",
|
||||
finished_at="2026-08-01T10:00:01Z",
|
||||
)
|
||||
|
||||
|
||||
def _manifest(
|
||||
output_seed: str,
|
||||
attempt: str,
|
||||
*,
|
||||
port_name: str = "result",
|
||||
size_bytes: int = 16,
|
||||
) -> OutputManifest:
|
||||
return OutputManifest(
|
||||
task_key="verify/0",
|
||||
outputs={port_name: ArtifactCollection.single(_artifact(output_seed, size_bytes=size_bytes))},
|
||||
metrics={"elapsed_seconds": 1.0},
|
||||
provenance=_provenance(attempt),
|
||||
)
|
||||
|
||||
|
||||
def _candidate(
|
||||
output_seed: str,
|
||||
attempt: str,
|
||||
*,
|
||||
owner: str | None,
|
||||
candidate_id: str | None = None,
|
||||
port_name: str = "result",
|
||||
size_bytes: int = 16,
|
||||
authenticated: bool = True,
|
||||
) -> CandidateOutput:
|
||||
resolved_candidate_id = candidate_id or str(
|
||||
uuid5(NAMESPACE_URL, f"candidate:{attempt}")
|
||||
)
|
||||
resolved_owner_id = None if owner is None else str(uuid5(NAMESPACE_URL, f"owner:{owner}"))
|
||||
manifest = _manifest(
|
||||
output_seed,
|
||||
attempt,
|
||||
port_name=port_name,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
if resolved_owner_id is not None and authenticated:
|
||||
return CandidateOutput.from_coordinator_record(
|
||||
resolved_candidate_id,
|
||||
resolved_owner_id,
|
||||
manifest,
|
||||
AUTHENTICATION_KEY,
|
||||
)
|
||||
return CandidateOutput(resolved_candidate_id, resolved_owner_id, manifest)
|
||||
|
||||
|
||||
def _context(
|
||||
*,
|
||||
minimum_matches: int = 1,
|
||||
reference: OutputManifest | None = None,
|
||||
require_distinct_owners: bool = False,
|
||||
trust_mode: TrustMode = TrustMode.TRUSTED,
|
||||
) -> VerifyContext:
|
||||
provenance = _provenance("binding")
|
||||
return VerifyContext(
|
||||
expected_outputs={"result": OUTPUT_PORT},
|
||||
max_output_bytes=1_024,
|
||||
minimum_matches=minimum_matches,
|
||||
reference=reference,
|
||||
require_distinct_owners=require_distinct_owners,
|
||||
binding=VerificationBinding(
|
||||
workload=provenance.workload,
|
||||
task_key="verify/0",
|
||||
package_digest=provenance.package_digest,
|
||||
manifest_digest=provenance.manifest_digest,
|
||||
environment_digest=provenance.environment_digest,
|
||||
parameters_digest=provenance.parameters_digest,
|
||||
input_collection_digest=provenance.input_collection_digest,
|
||||
execution_contract_digest=provenance.execution_contract_digest,
|
||||
selected_features=provenance.selected_features,
|
||||
optional_fallbacks=provenance.optional_fallbacks,
|
||||
job_id=provenance.job_id,
|
||||
task_id=provenance.task_id,
|
||||
verifier=provenance.verifier,
|
||||
sdk_api_version=provenance.sdk_api_version,
|
||||
protocol_version=provenance.protocol_version,
|
||||
manifest_schema_version=provenance.manifest_schema_version,
|
||||
workflow_schema_version=provenance.workflow_schema_version,
|
||||
artifact_schemas=provenance.artifact_schemas,
|
||||
trust_mode=trust_mode,
|
||||
),
|
||||
trust_mode=trust_mode,
|
||||
)
|
||||
|
||||
|
||||
def test_verification_decision_is_strict_immutable_and_round_trips() -> None:
|
||||
source = {"summary": {"counts": [1, 2]}}
|
||||
decision = VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"comparison-failed",
|
||||
source,
|
||||
)
|
||||
|
||||
source["summary"]["counts"].append(3) # type: ignore[index, union-attr]
|
||||
|
||||
assert decision.evidence["summary"]["counts"] == (1, 2)
|
||||
assert VerificationDecision.from_dict(decision.to_dict()) == decision
|
||||
with pytest.raises(TypeError):
|
||||
decision.evidence["new"] = True # type: ignore[index]
|
||||
with pytest.raises(TypeError):
|
||||
decision.evidence["summary"]["new"] = True # type: ignore[index]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "accepted_digest"),
|
||||
[
|
||||
(VerificationStatus.ACCEPTED, None),
|
||||
(VerificationStatus.REJECTED, "a" * 64),
|
||||
(VerificationStatus.INCONCLUSIVE, "a" * 64),
|
||||
],
|
||||
)
|
||||
def test_verification_decision_enforces_digest_status_invariant(
|
||||
status: VerificationStatus,
|
||||
accepted_digest: str | None,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="accepted_digest|accepted verification"):
|
||||
VerificationDecision(
|
||||
status,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"test-result",
|
||||
{},
|
||||
accepted_digest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsafe",
|
||||
[
|
||||
"/home/worker/private.log",
|
||||
"https://worker.invalid/evidence",
|
||||
"run-123/tasks/map/result.csv",
|
||||
"path=attempts/job-123/private.txt",
|
||||
"https%253A%252F%252Fworker.invalid%252Fevidence%253Ftoken%253Dsecret",
|
||||
],
|
||||
)
|
||||
def test_verification_decision_rejects_private_locations(unsafe: str) -> None:
|
||||
with pytest.raises(ValueError, match="URI or local path"):
|
||||
VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"unsafe-evidence",
|
||||
{"detail": unsafe},
|
||||
)
|
||||
|
||||
|
||||
def test_verification_decision_bounds_evidence_and_rejects_unknown_fields() -> None:
|
||||
with pytest.raises(ValueError, match="exceeds 16 KiB"):
|
||||
VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"oversized-evidence",
|
||||
{"detail": "x" * 17_000},
|
||||
)
|
||||
|
||||
payload = VerificationDecision(
|
||||
VerificationStatus.REJECTED,
|
||||
ComponentRef("test-verifier", 1),
|
||||
"test-result",
|
||||
{},
|
||||
).to_dict()
|
||||
payload["unexpected"] = True
|
||||
with pytest.raises(ValueError, match="unknown unexpected"):
|
||||
VerificationDecision.from_dict(payload)
|
||||
|
||||
|
||||
def test_candidate_output_envelope_is_strict_and_round_trips() -> None:
|
||||
candidate = _candidate("output", "attempt-one", owner="owner-one")
|
||||
|
||||
decoded_candidate = CandidateOutput.from_dict(candidate.to_dict())
|
||||
assert decoded_candidate.to_dict() == candidate.to_dict()
|
||||
assert not decoded_candidate.coordinator_authenticated
|
||||
candidates = CandidateOutputs(candidates=(candidate,))
|
||||
decoded = CandidateOutputs.from_dict(candidates.to_dict())
|
||||
assert not decoded.candidates[0].coordinator_authenticated
|
||||
assert CandidateOutputs.from_authenticated_dict(
|
||||
candidates.to_dict(), AUTHENTICATION_KEY
|
||||
) == candidates
|
||||
with pytest.raises(ValueError, match="authentication failed"):
|
||||
CandidateOutputs.from_authenticated_dict(candidates.to_dict(), b"x" * 32)
|
||||
assert candidates.manifests == (candidate.manifest,)
|
||||
|
||||
with pytest.raises(ValueError, match="opaque coordinator identity"):
|
||||
CandidateOutput("../worker-path", candidate.owner_id, candidate.manifest)
|
||||
|
||||
|
||||
def test_trusted_single_manifest_compatibility_uses_an_anonymous_envelope() -> None:
|
||||
manifest = _manifest("output", "trusted")
|
||||
candidates = CandidateOutputs((manifest,))
|
||||
|
||||
assert candidates.manifests == (manifest,)
|
||||
assert candidates.candidates[0].owner_id is None
|
||||
decision = ExactArtifactVerifier().verify(_context(), candidates)
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
|
||||
|
||||
def test_raw_manifests_cannot_form_a_quorum() -> None:
|
||||
with pytest.raises(ValueError, match="one trusted candidate"):
|
||||
CandidateOutputs(
|
||||
(
|
||||
_manifest("output", "trusted-one"),
|
||||
_manifest("output", "trusted-two"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_multi_vote_context_automatically_requires_distinct_owners() -> None:
|
||||
assert _context(minimum_matches=2).require_distinct_owners
|
||||
assert _context(require_distinct_owners=True).require_distinct_owners
|
||||
|
||||
with pytest.raises(ValueError, match="boolean"):
|
||||
_context(require_distinct_owners=1) # type: ignore[arg-type]
|
||||
|
||||
with pytest.raises(ValueError, match="coordinator binding"):
|
||||
VerifyContext(
|
||||
expected_outputs={"result": OUTPUT_PORT},
|
||||
max_output_bytes=1_024,
|
||||
minimum_matches=2,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="at least two"):
|
||||
_context(trust_mode=TrustMode.UNTRUSTED_QUORUM)
|
||||
|
||||
|
||||
def test_exact_verifier_reports_no_candidates_as_inconclusive() -> None:
|
||||
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs(()))
|
||||
|
||||
assert decision.status is VerificationStatus.INCONCLUSIVE
|
||||
assert decision.reason_code == "no-candidates"
|
||||
assert decision.evidence == {"candidate_count": 0, "invalid_count": 0}
|
||||
assert decision.accepted_digest is None
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_candidates_that_violate_output_contract() -> None:
|
||||
invalid = _manifest("same-output", "invalid", port_name="undeclared")
|
||||
|
||||
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((invalid,)))
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "no-valid-candidates"
|
||||
assert decision.evidence == {"candidate_count": 1, "invalid_count": 1}
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_candidate_from_another_scientific_binding() -> None:
|
||||
candidate = _candidate("same-output", "other-job", owner="owner-one")
|
||||
forged_provenance = replace(
|
||||
candidate.manifest.provenance,
|
||||
parameters_digest=_sha256("different-parameters"),
|
||||
)
|
||||
forged = replace(
|
||||
candidate,
|
||||
manifest=replace(candidate.manifest, provenance=forged_provenance),
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(_context(), CandidateOutputs((forged,)))
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "no-valid-candidates"
|
||||
|
||||
|
||||
def test_exact_verifier_accepts_unique_quorum_and_ignores_invalid_candidates() -> None:
|
||||
first = _candidate("same-output", "one", owner="owner-one")
|
||||
second = _candidate("same-output", "two", owner="owner-two")
|
||||
minority = _candidate("different-output", "three", owner="owner-three")
|
||||
invalid = _candidate(
|
||||
"same-output",
|
||||
"four",
|
||||
owner="owner-four",
|
||||
port_name="undeclared",
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
CandidateOutputs((first, second, minority, invalid)),
|
||||
)
|
||||
|
||||
assert first.manifest.digest == second.manifest.digest
|
||||
assert first.manifest.manifest_digest != second.manifest.manifest_digest
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "quorum-match"
|
||||
assert decision.accepted_digest == first.manifest.digest
|
||||
assert decision.evidence == {
|
||||
"matched": 2,
|
||||
"required": 2,
|
||||
"distinct_digests": 2,
|
||||
"invalid_count": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_conflicting_quorums() -> None:
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("group-a", "a-one", owner="a-one"),
|
||||
_candidate("group-a", "a-two", owner="a-two"),
|
||||
_candidate("group-b", "b-one", owner="b-one"),
|
||||
_candidate("group-b", "b-two", owner="b-two"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(_context(minimum_matches=2), candidates)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "conflicting-quorums"
|
||||
assert decision.accepted_digest is None
|
||||
assert decision.evidence["largest_group"] == 2
|
||||
assert decision.evidence["distinct_digests"] == 2
|
||||
|
||||
|
||||
def test_exact_verifier_distinguishes_insufficient_evidence_from_reference_mismatch() -> None:
|
||||
reference = _manifest("reference", "reference")
|
||||
one_mismatch = CandidateOutputs((_candidate("other", "one", owner="owner-one"),))
|
||||
two_mismatches = CandidateOutputs(
|
||||
(
|
||||
_candidate("other-a", "two", owner="owner-two"),
|
||||
_candidate("other-b", "three", owner="owner-three"),
|
||||
)
|
||||
)
|
||||
|
||||
insufficient = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2, reference=reference),
|
||||
one_mismatch,
|
||||
)
|
||||
rejected = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2, reference=reference),
|
||||
two_mismatches,
|
||||
)
|
||||
|
||||
assert (insufficient.status, insufficient.reason_code) == (
|
||||
VerificationStatus.INCONCLUSIVE,
|
||||
"insufficient-evidence",
|
||||
)
|
||||
assert (rejected.status, rejected.reason_code) == (
|
||||
VerificationStatus.REJECTED,
|
||||
"reference-mismatch",
|
||||
)
|
||||
|
||||
|
||||
def test_exact_verifier_accepts_declared_reference_quorum() -> None:
|
||||
reference = _manifest("reference", "reference")
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("reference", "worker-one", owner="owner-one"),
|
||||
_candidate("reference", "worker-two", owner="owner-two"),
|
||||
_candidate("other", "worker-three", owner="owner-three"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2, reference=reference),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "reference-match"
|
||||
assert decision.accepted_digest == reference.digest
|
||||
assert decision.evidence == {"matched": 2, "required": 2, "invalid_count": 0}
|
||||
|
||||
|
||||
def test_candidate_outputs_rejects_duplicate_candidate_ids() -> None:
|
||||
candidate = _candidate("output", "one", owner="owner-one")
|
||||
replay = _candidate(
|
||||
"different-output",
|
||||
"two",
|
||||
owner="owner-two",
|
||||
candidate_id=candidate.candidate_id,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="candidate_id values must be unique"):
|
||||
CandidateOutputs((candidate, replay))
|
||||
|
||||
|
||||
def test_exact_quorum_rejects_candidates_without_authenticated_owners() -> None:
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("output", "one", owner=None),
|
||||
_candidate("output", "two", owner="owner-two"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "coordinator-authentication-required"
|
||||
assert decision.evidence == {"candidate_count": 2, "unauthenticated_count": 1}
|
||||
|
||||
|
||||
def test_exact_verifier_counts_at_most_one_vote_per_owner() -> None:
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("output", "one", owner="same-owner"),
|
||||
_candidate("output", "two", owner="same-owner"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.INCONCLUSIVE
|
||||
assert decision.reason_code == "insufficient-evidence"
|
||||
assert decision.evidence == {
|
||||
"largest_group": 1,
|
||||
"required": 2,
|
||||
"distinct_digests": 1,
|
||||
"invalid_count": 0,
|
||||
"duplicate_owner_candidates": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_exact_verifier_rejects_owner_equivocation_without_leaking_identity() -> None:
|
||||
owner = "equivocating-owner"
|
||||
candidates = CandidateOutputs(
|
||||
(
|
||||
_candidate("output-a", "one", owner=owner),
|
||||
_candidate("output-b", "two", owner=owner),
|
||||
_candidate("output-a", "three", owner="honest-owner"),
|
||||
)
|
||||
)
|
||||
|
||||
decision = ExactArtifactVerifier().verify(
|
||||
_context(minimum_matches=2),
|
||||
candidates,
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "owner-equivocation"
|
||||
assert decision.accepted_digest is None
|
||||
assert decision.evidence == {
|
||||
"candidate_count": 3,
|
||||
"equivocating_owner_count": 1,
|
||||
}
|
||||
assert candidates.candidates[0].owner_id not in json.dumps(decision.to_dict())
|
||||
|
||||
|
||||
def test_numeric_verifier_accepts_nested_values_with_absolute_and_relative_tolerance() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001, relative=0.01))
|
||||
|
||||
decision = verifier.verify_values(
|
||||
{"energies": [10.0, 0.05], "converged": True},
|
||||
{"energies": [10.05, 0.0505], "converged": True},
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "within-tolerance"
|
||||
assert decision.accepted_digest is not None
|
||||
assert decision.evidence == {
|
||||
"absolute": 0.001,
|
||||
"relative": 0.01,
|
||||
"max_ulps": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_numeric_verifier_supports_ulp_tolerance() -> None:
|
||||
adjacent = math.nextafter(1.0, 2.0)
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(max_ulps=1))
|
||||
|
||||
decision = verifier.verify_values(1.0, adjacent)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
|
||||
|
||||
def test_numeric_verifier_reports_bounded_location_and_error_evidence() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.1))
|
||||
|
||||
decision = verifier.verify_values(
|
||||
{"matrix": [[1.0, 2.0]]},
|
||||
{"matrix": [[1.0, 2.5]]},
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "numeric-mismatch"
|
||||
assert decision.evidence["location"] == "$.matrix[0][1]"
|
||||
assert decision.evidence["absolute_error"] == pytest.approx(0.5)
|
||||
assert decision.evidence["allowed_error"] == pytest.approx(0.1)
|
||||
assert isinstance(decision.evidence["ulp_distance"], int)
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_shape_changes_before_value_comparison() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values(
|
||||
{"energy": 1.0, "iterations": 4},
|
||||
{"energy": 1.0, "converged": True},
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "shape-mismatch"
|
||||
assert decision.evidence == {
|
||||
"location": "$",
|
||||
"missing_keys": ("iterations",),
|
||||
"extra_keys": ("converged",),
|
||||
}
|
||||
|
||||
|
||||
def test_numeric_verifier_applies_declared_nan_policy() -> None:
|
||||
reject = NumericToleranceVerifier(NumericTolerance(nan_policy="reject"))
|
||||
equal = NumericToleranceVerifier(NumericTolerance(nan_policy="equal"))
|
||||
|
||||
rejected = reject.verify_values(float("nan"), float("nan"))
|
||||
accepted = equal.verify_values(float("nan"), float("nan"))
|
||||
|
||||
assert (rejected.status, rejected.reason_code) == (
|
||||
VerificationStatus.REJECTED,
|
||||
"nan-policy",
|
||||
)
|
||||
assert accepted.status is VerificationStatus.ACCEPTED
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"factory",
|
||||
[
|
||||
lambda: NumericTolerance(absolute=-0.1),
|
||||
lambda: NumericTolerance(relative=float("inf")),
|
||||
lambda: NumericTolerance(max_ulps=True),
|
||||
lambda: NumericTolerance(nan_policy="propagate"),
|
||||
lambda: NumericTolerance(max_elements=0),
|
||||
],
|
||||
)
|
||||
def test_numeric_tolerance_rejects_ambiguous_or_non_finite_policy(
|
||||
factory: Callable[[], NumericTolerance],
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="numeric tolerance"):
|
||||
factory()
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_unrepresentable_integer_without_raising() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values(10**10_000, 10**10_000)
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "numeric-range"
|
||||
assert decision.evidence["location"] == "$"
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_shapes_above_its_manifest_bound() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(max_elements=3))
|
||||
|
||||
decision = verifier.verify_values([1, 2, 3], [1, 2, 3])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "element-limit"
|
||||
assert decision.evidence == {"max_elements": 3}
|
||||
assert verifier.configuration["max_elements"] == 3
|
||||
|
||||
|
||||
def test_numeric_verifier_does_not_round_mixed_integer_and_float_values() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values(2**53 + 1, float(2**53))
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "numeric-mismatch"
|
||||
|
||||
|
||||
def test_numeric_verifier_rejects_non_json_mapping_keys() -> None:
|
||||
verifier = NumericToleranceVerifier(NumericTolerance())
|
||||
|
||||
decision = verifier.verify_values({1: 2.0}, {1: 2.0})
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "type-mismatch"
|
||||
|
||||
|
||||
def test_numeric_verifier_implements_manifest_verifier_protocol_with_loader() -> None:
|
||||
reference = _manifest("reference", "reference")
|
||||
candidate = _manifest("candidate", "candidate")
|
||||
values = {
|
||||
reference.outputs["result"].items[0].artifact.sha256: {"energy": 1.0},
|
||||
candidate.outputs["result"].items[0].artifact.sha256: {"energy": 1.0001},
|
||||
}
|
||||
|
||||
def load(manifest: OutputManifest) -> object:
|
||||
return values[manifest.outputs["result"].items[0].artifact.sha256]
|
||||
|
||||
verifier = NumericToleranceVerifier(NumericTolerance(absolute=0.001), load)
|
||||
decision = verifier.verify(
|
||||
_context(reference=reference),
|
||||
CandidateOutputs((candidate,)),
|
||||
)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.accepted_digest == reference.digest
|
||||
|
||||
|
||||
def test_manifest_verifiers_fail_closed_without_artifact_loaders() -> None:
|
||||
candidate = CandidateOutputs((_manifest("candidate", "candidate"),))
|
||||
|
||||
numeric = NumericToleranceVerifier(NumericTolerance()).verify(_context(), candidate)
|
||||
canonical = CanonicalRecordVerifier(_canonical_json_record).verify(_context(), candidate)
|
||||
|
||||
assert numeric.reason_code == "loader-unavailable"
|
||||
assert canonical.reason_code == "loader-unavailable"
|
||||
|
||||
|
||||
def _canonical_json_record(record: object) -> bytes:
|
||||
return json.dumps(record, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def test_canonical_record_verifier_accepts_normalized_records() -> None:
|
||||
verifier = CanonicalRecordVerifier(_canonical_json_record)
|
||||
expected = [{"name": "molecule", "score": 0.75}, {"id": 2}]
|
||||
actual = [{"score": 0.75, "name": "molecule"}, {"id": 2}]
|
||||
|
||||
decision = verifier.verify_records(expected, actual)
|
||||
|
||||
assert decision.status is VerificationStatus.ACCEPTED
|
||||
assert decision.reason_code == "canonical-match"
|
||||
assert decision.evidence == {"records": 2}
|
||||
assert decision.accepted_digest is not None
|
||||
|
||||
|
||||
def test_canonical_record_verifier_rejects_content_or_count_mismatch() -> None:
|
||||
verifier = CanonicalRecordVerifier(_canonical_json_record)
|
||||
|
||||
decision = verifier.verify_records([{"id": 1}, {"id": 2}], [{"id": 1}])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "canonical-mismatch"
|
||||
assert decision.evidence == {"expected_records": 2, "actual_records": 1}
|
||||
|
||||
|
||||
def test_canonical_record_verifier_enforces_record_limit() -> None:
|
||||
verifier = CanonicalRecordVerifier(_canonical_json_record, max_records=2)
|
||||
|
||||
decision = verifier.verify_records([1, 2, 3], [1, 2, 3])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "record-limit"
|
||||
assert decision.evidence == {"max_records": 2}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"canonicalizer",
|
||||
[
|
||||
lambda _record: "not-bytes",
|
||||
lambda _record: (_ for _ in ()).throw(ValueError("/private/worker/path")),
|
||||
],
|
||||
)
|
||||
def test_canonical_record_verifier_sanitizes_canonicalization_failures(
|
||||
canonicalizer: Callable[[object], object],
|
||||
) -> None:
|
||||
verifier = CanonicalRecordVerifier(canonicalizer) # type: ignore[arg-type]
|
||||
|
||||
decision = verifier.verify_records([{"id": 1}], [{"id": 1}])
|
||||
|
||||
assert decision.status is VerificationStatus.REJECTED
|
||||
assert decision.reason_code == "canonicalization-failed"
|
||||
assert decision.evidence == {}
|
||||
assert decision.accepted_digest is None
|
||||
Reference in New Issue
Block a user