Harden distributed pipeline
This commit is contained in:
@@ -38,13 +38,14 @@ class WorkerConfig:
|
||||
cleanup_after_seconds: float | None = None
|
||||
max_tasks: int | None = None
|
||||
exit_when_idle: bool = False
|
||||
# Distributed similarity-graph requires triangular block-pair planning and
|
||||
# is deliberately not advertised until CTX-10. A normal worker must never
|
||||
# make a multi-shard graph job appear scientifically complete.
|
||||
# The local CLI uses hyphens; the first coordinator contract used
|
||||
# underscores. Advertise both stable spellings while jobs are migrated.
|
||||
# underscores, so retain the search alias during migration.
|
||||
capabilities: tuple[str, ...] = (
|
||||
"similarity-search",
|
||||
"similarity-graph",
|
||||
"similarity_search",
|
||||
"similarity_graph",
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
|
||||
@@ -7,6 +7,7 @@ from dataclasses import replace
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
@@ -199,7 +200,7 @@ class WorkerDaemon:
|
||||
return RunOnceOutcome(claimed=True, completed=completed)
|
||||
|
||||
def _report_failure(self, task: ClaimedTask, error: Exception) -> None:
|
||||
message = str(error).replace(str(self.config.work_dir), "<worker-dir>")[:300]
|
||||
message = self._sanitize_error_message(error)
|
||||
try:
|
||||
self.coordinator.fail(task, {"worker_id": self._worker_id(), "attempt": task.attempt, "error_code": type(error).__name__, "error_message": message})
|
||||
except CoordinatorTransientError:
|
||||
@@ -207,6 +208,16 @@ class WorkerDaemon:
|
||||
except Exception:
|
||||
self._log("failed", task, error_type="FailureReportError")
|
||||
|
||||
def _sanitize_error_message(self, error: Exception) -> str:
|
||||
"""Keep coordinator-visible failures useful without exposing local paths."""
|
||||
message = str(error).replace(str(self.config.work_dir), "<worker-dir>")
|
||||
# CalledProcessError includes the complete argv, including sys.executable
|
||||
# outside work_dir. Replace POSIX and Windows absolute paths before the
|
||||
# message reaches the coordinator database or operator UI.
|
||||
message = re.sub(r"(?<![\w:])[A-Za-z]:\\[^\s'\"\],)]+", "<path>", message)
|
||||
message = re.sub(r"(?<![\w:])/(?:[^\s'\"\],)]+)", "<path>", message)
|
||||
return message[:300]
|
||||
|
||||
def _register_worker(self) -> None:
|
||||
registered = self.coordinator.register(
|
||||
self.config.worker_name,
|
||||
|
||||
Reference in New Issue
Block a user