Harden distributed pipeline

This commit is contained in:
Emil
2026-07-24 14:16:42 +03:00
parent 9ec8f50313
commit 19cbf7f113
35 changed files with 768 additions and 148 deletions
+4 -3
View File
@@ -38,13 +38,14 @@ class WorkerConfig:
cleanup_after_seconds: float | None = None
max_tasks: int | None = None
exit_when_idle: bool = False
# Distributed similarity-graph requires triangular block-pair planning and
# is deliberately not advertised until CTX-10. A normal worker must never
# make a multi-shard graph job appear scientifically complete.
# The local CLI uses hyphens; the first coordinator contract used
# underscores. Advertise both stable spellings while jobs are migrated.
# underscores, so retain the search alias during migration.
capabilities: tuple[str, ...] = (
"similarity-search",
"similarity-graph",
"similarity_search",
"similarity_graph",
)
def __post_init__(self) -> None:
+12 -1
View File
@@ -7,6 +7,7 @@ from dataclasses import replace
from dataclasses import dataclass
from pathlib import Path
import random
import re
import shutil
import threading
import time
@@ -199,7 +200,7 @@ class WorkerDaemon:
return RunOnceOutcome(claimed=True, completed=completed)
def _report_failure(self, task: ClaimedTask, error: Exception) -> None:
message = str(error).replace(str(self.config.work_dir), "<worker-dir>")[:300]
message = self._sanitize_error_message(error)
try:
self.coordinator.fail(task, {"worker_id": self._worker_id(), "attempt": task.attempt, "error_code": type(error).__name__, "error_message": message})
except CoordinatorTransientError:
@@ -207,6 +208,16 @@ class WorkerDaemon:
except Exception:
self._log("failed", task, error_type="FailureReportError")
def _sanitize_error_message(self, error: Exception) -> str:
"""Keep coordinator-visible failures useful without exposing local paths."""
message = str(error).replace(str(self.config.work_dir), "<worker-dir>")
# CalledProcessError includes the complete argv, including sys.executable
# outside work_dir. Replace POSIX and Windows absolute paths before the
# message reaches the coordinator database or operator UI.
message = re.sub(r"(?<![\w:])[A-Za-z]:\\[^\s'\"\],)]+", "<path>", message)
message = re.sub(r"(?<![\w:])/(?:[^\s'\"\],)]+)", "<path>", message)
return message[:300]
def _register_worker(self) -> None:
registered = self.coordinator.register(
self.config.worker_name,