Fix worker result and lease contracts

This commit is contained in:
Emil
2026-07-22 18:48:52 +03:00
parent 167b9e2aaa
commit bfdd234c2c
8 changed files with 171 additions and 37 deletions
+36 -8
View File
@@ -3,13 +3,14 @@ from __future__ import annotations
import hashlib
from pathlib import Path
import time
from datetime import datetime, timedelta, timezone
from urllib.request import Request
import pytest
from scimesh.worker.config import WorkerConfig
from scimesh.worker.coordinator import CoordinatorTransientError
from scimesh.worker.daemon import WorkerDaemon
from scimesh.worker.daemon import LeaseHeartbeat, WorkerDaemon
from scimesh.worker.models import ClaimedTask, InputArtifact, ProducedArtifact, RunResult
from scimesh.worker.artifacts import HttpArtifactClient, _SameOriginAuthRedirectHandler, _origin
from scimesh.worker.runners import SciMeshRunner
@@ -17,7 +18,7 @@ from scimesh.worker.runners import SciMeshRunner
class FakeCoordinator:
def __init__(self, task: ClaimedTask | None) -> None:
self.task, self.submissions, self.heartbeats = task, [], []
self.task, self.submissions, self.failures, self.heartbeats = task, [], [], []
def claim(self, worker_id: str, capabilities: tuple[str, ...]) -> ClaimedTask | None:
task, self.task = self.task, None
@@ -26,17 +27,25 @@ class FakeCoordinator:
def submit(self, task: ClaimedTask, payload: dict) -> None:
self.submissions.append(payload)
def heartbeat(self, task: ClaimedTask, worker_id: str) -> None:
def fail(self, task: ClaimedTask, payload: dict) -> None:
self.failures.append(payload)
def heartbeat(self, task: ClaimedTask, worker_id: str) -> str:
self.heartbeats.append((task.task_id, task.attempt, worker_id))
return (datetime.now(timezone.utc) + timedelta(seconds=1)).isoformat()
class FakeArtifacts:
def __init__(self, content: bytes) -> None:
self.content = content
self.content, self.uploaded = content, []
def download(self, uri: str, destination: Path) -> None:
destination.write_bytes(self.content)
def upload(self, task: ClaimedTask, worker_id: str, artifact: ProducedArtifact) -> str:
self.uploaded.append((task.task_id, worker_id, artifact.path))
return f"https://example.test/tasks/{task.task_id}/artifacts/{artifact.path.name}"
class FakeRunner:
def __init__(self) -> None:
self.calls = 0
@@ -49,7 +58,8 @@ class FakeRunner:
def make_task(content: bytes, checksum: str | None = None) -> ClaimedTask:
return ClaimedTask("task-1", 1, "2026-07-30T00:00:00Z", "similarity-search", InputArtifact("https://example.test/input", checksum or hashlib.sha256(content).hexdigest()), {"query_id": "CHEMBL1"})
lease = (datetime.now(timezone.utc) + timedelta(seconds=60)).isoformat()
return ClaimedTask("task-1", 1, lease, "similarity-search", InputArtifact("https://example.test/input", checksum or hashlib.sha256(content).hexdigest()), {"query_id": "CHEMBL1"})
def daemon(tmp_path: Path, task: ClaimedTask | None, content: bytes):
@@ -63,10 +73,11 @@ def test_claims_runs_uploads_and_submits_csv(tmp_path: Path) -> None:
worker, coordinator, artifacts, runner, _ = daemon(tmp_path, make_task(content), content)
assert worker.run_once() is True
assert runner.calls == 1
assert len(artifacts.uploaded) == 1
assert coordinator.heartbeats == [("task-1", 1, "worker-1")]
assert coordinator.submissions[0]["status"] == "completed"
assert coordinator.submissions[0]["result"]["content_type"] == "text/csv"
assert coordinator.submissions[0]["result"]["uri"].startswith("worker://worker-1/")
assert coordinator.submissions[0]["result"]["uri"].startswith("https://example.test/tasks/task-1/artifacts/")
def test_no_task_does_not_create_directory(tmp_path: Path) -> None:
@@ -80,8 +91,8 @@ def test_bad_checksum_reports_failure_without_running(tmp_path: Path) -> None:
worker, coordinator, _, runner, _ = daemon(tmp_path, make_task(b"actual", "not-the-hash"), b"actual")
assert worker.run_once() is True
assert runner.calls == 0
assert coordinator.submissions[0]["status"] == "failed"
assert coordinator.submissions[0]["error_code"] == "ValueError"
assert coordinator.failures[0]["error_code"] == "ValueError"
assert not coordinator.submissions
def test_transient_claim_error_is_propagated_for_bounded_backoff(tmp_path: Path) -> None:
@@ -137,6 +148,23 @@ def test_lease_is_renewed_while_a_runner_is_still_working(tmp_path: Path) -> Non
assert len(coordinator.heartbeats) >= 2
def test_heartbeat_reschedules_from_the_renewed_lease(tmp_path: Path) -> None:
class ShortLeaseCoordinator(FakeCoordinator):
def heartbeat(self, task: ClaimedTask, worker_id: str) -> str:
self.heartbeats.append((task.task_id, task.attempt, worker_id))
return (datetime.now(timezone.utc) + timedelta(seconds=0.02)).isoformat()
config = WorkerConfig(
"https://example.test", "worker-1", tmp_path / "work", heartbeat_interval=1
)
coordinator = ShortLeaseCoordinator(None)
heartbeat = LeaseHeartbeat(make_task(b"fixture"), coordinator, config)
heartbeat.start()
time.sleep(0.06)
heartbeat.stop()
assert len(coordinator.heartbeats) >= 3
def test_runner_maps_graph_and_smiles_search_parameters(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
commands: list[list[str]] = []