Replace Python worker daemon with the Go worker agent

This commit is contained in:
Emil
2026-08-02 16:53:04 +03:00
parent 9a8221163a
commit 706bc85e17
31 changed files with 516 additions and 2399 deletions
+42 -38
View File
@@ -46,9 +46,9 @@ from scimesh.sdk.runtime import (
)
from scimesh.sdk.workflow import StageKind
from .config import WorkerConfig
from .models import ClaimedTask, ProducedArtifact, RunResult
def _utc_now() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@@ -57,6 +57,38 @@ class Runner(Protocol):
def run(self, task: ClaimedTask, task_dir: Path) -> RunResult: ...
def _definitions_from_environment() -> dict[str, WorkloadDefinition]:
"""Load definitions from ``SCIMESH_WORKLOAD_ALLOWLIST`` or the built-ins.
The environment-driven discovery mirrors the former worker configuration;
the Go agent passes the allowlist through unchanged.
"""
import os
from scimesh.sdk.registry import workload_allowlist_from_json
allowlist = workload_allowlist_from_json(os.getenv("SCIMESH_WORKLOAD_ALLOWLIST"))
if allowlist:
registry = WorkloadRegistry()
registry.discover_installed(allowlist)
definitions: dict[str, WorkloadDefinition] = {}
for description in registry.descriptions():
definition, _ = registry.require(
description.workload.name,
description.workload.version,
description.package_digest,
)
definitions[description.workload.name] = definition
if not definitions:
raise ValueError("workload_allowlist discovered no workloads")
return definitions
from scimesh.workloads.search import similarity_search_sdk_definition
return {
"similarity-search": similarity_search_sdk_definition().definition(),
}
def _inventory_for(
definitions: Mapping[str, WorkloadDefinition],
*,
@@ -91,7 +123,12 @@ def _runtime_for(
class SciMeshRunner:
"""Execute claimed coordinator tasks through SDK-built workloads."""
"""Execute claimed coordinator tasks through SDK-built workloads.
Definitions come from ``SCIMESH_WORKLOAD_ALLOWLIST`` when set, otherwise
the built-in ``similarity-search``; callers may supply an explicit mapping
for tests.
"""
def __init__(
self,
@@ -100,13 +137,9 @@ class SciMeshRunner:
inventory: ResourceInventory | None = None,
runtime: RuntimeCapabilities | None = None,
) -> None:
self._definitions = dict(definitions or {})
if "similarity-search" not in self._definitions:
from scimesh.workloads.search import similarity_search_sdk_definition
self._definitions["similarity-search"] = (
similarity_search_sdk_definition().definition()
)
if definitions is None:
definitions = _definitions_from_environment()
self._definitions = dict(definitions)
self._inventory = inventory or _inventory_for(
self._definitions,
cpu_cores=1,
@@ -115,35 +148,6 @@ class SciMeshRunner:
self._runtime = runtime or _runtime_for(self._definitions, self._inventory)
self._pool = ResourcePool(self._runtime.inventory, max_concurrency=1)
@classmethod
def for_worker(cls, config: WorkerConfig) -> "SciMeshRunner":
"""Build a runner for one worker: discover allowlisted workloads or use built-ins."""
definitions: dict[str, WorkloadDefinition] = {}
if config.workload_allowlist:
registry = WorkloadRegistry()
registry.discover_installed(config.workload_allowlist)
for description in registry.descriptions():
definition, _ = registry.require(
description.workload.name,
description.workload.version,
description.package_digest,
)
definitions[description.workload.name] = definition
if not definitions:
raise ValueError("workload_allowlist discovered no workloads")
else:
from scimesh.workloads.search import similarity_search_sdk_definition
definitions["similarity-search"] = (
similarity_search_sdk_definition().definition()
)
inventory = _inventory_for(
definitions,
cpu_cores=config.cpu_count,
memory_mb=config.memory_mb or 1024,
)
return cls(definitions=definitions, inventory=inventory)
def run(self, task: ClaimedTask, task_dir: Path) -> RunResult:
task_dir = task_dir.resolve()
workload = task.workload.replace("_", "-")