Remove dead code and wire the two-worker smoke script

.opencode/session-goal.md

@@ -1,35 +1,23 @@
 COMPLETED
 # Session Goal
 
-пропиши абсолютно полную документацию scimesh в mkdocs.
+давай теперь почистим проект от линего кода
 
 ## Plan
 
-1. Каталог `mkdocs/` — самостоятельный источник сайта (docs_dir: mkdocs); проектные `docs/` не трогаем.
+1. Аудит: ruff/pyflakes — неиспользуемые импорты по scimesh/ и tests/; grep — неиспользуемые функции/модули (после рефакторингов могли остаться мёртвые экспорты, например в descriptors/search/graph core и sdk/_validation).
-2. mkdocs.yml: docs_dir, полный nav (Home → SDK guides → API reference (все модули scimesh.sdk) → Documentation approach), edit_uri.
+2. Удаление мёртвого кода: неиспользуемые импорты, функции, дубли (например write_descriptor_shards/concatenate_descriptor_shards, если вытеснены дефолтами batch), устаревшие файлы-обёртки.
-3. Контент:
+3. Проверка, что ничего публичного/API не сломано: pyright 0 ошибок, pytest зелёный, go test/vet, mkdocs build.
-   - index.md — лендинг: что такое SciMesh, архитектура, быстрый старт.
+4. Финал: полный прогон, COMPLETED.
-   - sdk/overview.md — концепция SDK (framework-only, core-batch-v1, безопасность, fail-closed).
-   - sdk/authoring-workloads.md — полный гайд MapReduceWorkload (атрибуты, хуки, полный пример molwt-filter, регистрация, тесты).
-   - sdk/cli.md — scimesh workload list/run/export + env.
-   - sdk/worker-integration.md — воркер: claim/heartbeat/upload, SDK-мост, allowlist, ограничения v1.
-   - api/ — автогенерация mkdocstrings для ВСЕХ модулей scimesh.sdk (batch, artifacts, manifest, workflow, plans, registry, verification, runtime, resources, conformance, execution, identity, integrity, schema, protocols) + index.md.
-   - approach.md — зафиксированные правила написания документации.
-4. Убрать тестовые страницы mkdocs из docs/ (docs/index.md, docs/api/).
-5. make docs — проверить рендер всех страниц; итерации по ошибкам mkdocstrings.
-6. pytest + pyright — зелёные; /ui/docs/ интеграция не ломается; README/STATUS обновить.
 
 ## Progress
 
-- [x] mkdocs.yml: `docs_dir: mkdocs`, полный nav, edit_uri; тема Material + mkdocstrings (handlers.python.options, show_if_no_docstring: true).
+- [x] Аудит: ruff F401/F811/F841 — 26 неиспользуемых импортов; vulture — кандидаты проверены grep'ом.
-- [x] index.md — лендинг (быстрый старт, карта сайта).
+- [x] Удалено:
-- [x] sdk/overview.md, sdk/authoring-workloads.md (полный гайд + molwt-filter пример + plan_tasks блоки), sdk/cli.md, sdk/worker-integration.md.
+  - 25 неиспользуемых импортов (ruff --fix) + 1 неиспользуемая локальная переменная в тесте;
-- [x] api/: 15 страниц `::: scimesh.sdk.<module>` + api/index.md (модульная карта) — ВСЕ модули SDK.
+  - мёртвые функции descriptors/core.py `write_descriptor_shards`/`concatenate_descriptor_shards` (вытеснены дефолтами MapReduceWorkload; ссылки только в собственном `__init__`) + их экспорты;
-- [x] approach.md — правила: типы страниц, reference только из docstrings (Google style), терминология, код-блоки, ссылки, когда обновлять, сборка.
+  - мёртвые атрибуты `MapReduceWorkload._resources/_execution` (записывались, нигде не читались);
-- [x] Удалены тестовые страницы mkdocs из docs/ (docs/index.md, docs/api/).
+  - мёртвый `CancellationFlag.cancel` (0 использований);
-- [x] Устранена проблема mkdocstrings-python 2.x: `default_options` → `handlers.python.options`; `show_if_no_docstring: true` (иначе члены без docstrings не рендерились).
+- [x] `scripts/two-worker-smoke.sh` (рабочий E2E, но без точки входа) подключён как `make smoke-two-worker` — не мёртвый, а доступный.
-- [x] Добавлены Google-style docstrings ВСЕМ публичным членам scimesh.sdk без них (~50: artifacts, manifest, plans, workflow, execution, identity, protocols, registry, resources, runtime, verification, schema, conformance); починено подавление pyright в schema.py (перенесённый форматтером `# type: ignore`).
+- [x] Проверено: ruff clean; pytest 260 passed; pyright 0 ошибок (scimesh+tests); go test 11 пакетов + vet; mkdocs build без warnings.
-- [x] make docs --strict: 0 ошибок/предупреждений; все 16 API-страниц + 4 гайда + approach рендерятся (проверено по содержимому HTML).
+- [x] Изменения не закоммичены.
-- [x] pytest 256 passed; pyright scimesh/tests 0 errors.
-- [x] README/STATUS/handoff обновлены (mkdocs/ сайт, /ui/docs/).
-- Изменения НЕ закоммичены (коммит по запросу).

Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := help
 
-.PHONY: help demo-ui demo-down demo-logs docs docs-serve
+.PHONY: help demo-ui demo-down demo-logs smoke-two-worker docs docs-serve
 
 help:
 	@printf '%s\n' \
@@ -9,6 +9,8 @@ help:
 	  '  make demo-ui WORKERS=3       Start the demo with 3 local workers.' \
 	  '  make demo-logs               Follow coordinator logs for the demo.' \
 	  '  make demo-down               Stop demo containers and workers.' \
+	  '  make smoke-two-worker        E2E: two workers process 4 shards and the' \
+	  '                              result must match the local CLI reference.' \
 	  '  make docs                    Build the MkDocs site into site/.' \
 	  '  make docs-serve              Serve the MkDocs site at http://localhost:8000.' \
 	  '' \
@@ -25,6 +27,9 @@ demo-down:
 demo-logs:
 	$(MAKE) -C coordinator demo-logs
 
+smoke-two-worker:
+	./scripts/two-worker-smoke.sh
+
 docs:
 	.venv/bin/mkdocs build
 

scimesh/sdk/artifacts.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from enum import Enum
 from types import MappingProxyType
-from typing import Any, Mapping, Sequence
+from typing import Any, Mapping
 
 from ._validation import (
     canonical_json,

scimesh/sdk/batch.py

@@ -24,9 +24,7 @@ from typing import Any, Mapping, Sequence
 
 from .artifacts import (
     ArtifactCollection,
-    ArtifactItem,
     ArtifactRef,
-    ArtifactSchema,
     Cardinality,
     CollectionKind,
     OutputManifest,
@@ -295,8 +293,6 @@ class MapReduceWorkload:
             conformance_profiles=("core-batch-v1",),
         )
         self._exact_verifier = _EXACT_VERIFIER
-        self._resources = resources
-        self._execution = execution
         self._limits = limits
 
     # ------------------------------------------------------------------

scimesh/sdk/conformance.py

@@ -24,7 +24,7 @@ from .artifacts import (
     OutputManifest,
     Provenance,
 )
-from .identity import ComponentRef, SDK_API_VERSION, SchemaRef
+from .identity import ComponentRef, SDK_API_VERSION
 from .execution import NetworkPolicy, ProcessModel
 from .manifest import TrustMode, WorkloadManifest
 from .plans import JobRequest, TaskSpec
@@ -686,9 +686,6 @@ class CancellationFlag:
     def __init__(self) -> None:
         self._event = Event()
 
-    def cancel(self) -> None:
-        self._event.set()
-
     def cancelled(self) -> bool:
         return self._event.is_set()
 

scimesh/sdk/integrity.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import hashlib
 import importlib.util
-import os
 from importlib import metadata
 from pathlib import Path
 

scimesh/sdk/protocols.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Mapping, Protocol, Sequence
+from typing import Mapping, Protocol
 
 from .artifacts import (
     ArtifactCollection,

scimesh/sdk/registry.py

@@ -26,7 +26,6 @@ from .manifest import WorkloadManifest
 from .plans import JobRequest, ValidatedJob, WorkflowPlan
 from .protocols import (
     Planner,
-    PlanningContext,
     PlanningResources,
     Reducer,
     Runner,

scimesh/sdk/resources.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 from dataclasses import dataclass
 from enum import Enum
 from threading import Lock
-from types import MappingProxyType
 from typing import Mapping
 from uuid import uuid4
 

scimesh/sdk/runtime.py

@@ -10,7 +10,6 @@ from ._validation import (
     require_identifier,
     require_string,
     validate_version_range,
-    version_in_range,
 )
 from .identity import SDK_API_VERSION
 from .execution import NetworkPolicy, ProcessModel

scimesh/sdk/schema.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import math
 import re
 from fractions import Fraction
-from typing import Mapping, Sequence
+from typing import Mapping
 
 
 _ANNOTATIONS = {

scimesh/sdk/verification.py

@@ -20,7 +20,6 @@ from ._validation import (
     freeze_json_mapping,
     require_exact_keys,
     require_identifier,
-    require_nonnegative_int,
     require_positive_int,
     require_sha256,
     require_string,

scimesh/sdk/workflow.py

@@ -15,7 +15,6 @@ from ._validation import (
     require_nonnegative_int,
     require_positive_int,
     require_schema_version,
-    require_string,
 )
 from .artifacts import PortSpec
 from .execution import ExecutionProfile, NetworkPolicy, RetryPolicy

scimesh/worker/config.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import json
 from dataclasses import dataclass
 from math import isfinite
 from pathlib import Path

scimesh/worker/daemon.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 import logging
 from dataclasses import replace
 from dataclasses import dataclass
-from pathlib import Path
 import random
 import re
 import shutil

scimesh/worker/runners.py

@@ -24,7 +24,6 @@ from uuid import NAMESPACE_URL, uuid5
 
 from scimesh.sdk.artifacts import (
     ArtifactCollection,
-    ArtifactRef,
     OutputManifest,
     Provenance,
 )

scimesh/workloads/descriptors/__init__.py

@@ -9,11 +9,9 @@ from .core import (
     DESCRIPTOR_NAMES,
     DescriptorRow,
     compute_descriptor_batch,
-    concatenate_descriptor_shards,
     descriptor_calculator,
     validate_descriptor_names,
     write_descriptor_rows,
-    write_descriptor_shards,
 )
 from .definition import (
     MAP_ENTRY_POINT,
@@ -31,11 +29,9 @@ __all__ = [
     "DescriptorBatchWorkload",
     "DescriptorRow",
     "compute_descriptor_batch",
-    "concatenate_descriptor_shards",
     "descriptor_batch_sdk_definition",
     "descriptor_calculator",
     "validate_descriptor_names",
     "workload_definition",
     "write_descriptor_rows",
-    "write_descriptor_shards",
 ]

scimesh/workloads/descriptors/core.py

@@ -19,7 +19,7 @@ import csv
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Iterator, Mapping, Sequence
+from typing import Iterator, Sequence
 
 from rdkit import Chem
 from rdkit.ML.Descriptors.MoleculeDescriptors import MolecularDescriptorCalculator
@@ -229,86 +229,3 @@ def compute_descriptor_batch(
     materialized = list(rows)
     write_descriptor_rows(output_path, materialized)
     return stats.as_metrics()
-
-
-def write_descriptor_shards(
-    input_path: Path,
-    workspace: Path,
-    shard_rows: int,
-) -> list[Path]:
-    """Split the input TSV into deterministic row-bounded shards with headers."""
-    if (
-        isinstance(shard_rows, bool)
-        or not isinstance(shard_rows, int)
-        or shard_rows < 1
-    ):
-        raise ValueError("shard_rows must be a positive integer")
-    paths: list[Path] = []
-    current: Path | None = None
-    destination = None
-    writer = None
-    rows_in_shard = 0
-    try:
-        with input_path.open("r", encoding="utf-8", newline="") as source:
-            reader = csv.DictReader(source, delimiter="\t")
-            fieldnames = tuple(reader.fieldnames or ())
-            if not {"chembl_id", "canonical_smiles"}.issubset(set(fieldnames)):
-                raise ValueError(
-                    "dataset is missing required columns: chembl_id, canonical_smiles"
-                )
-            for row in reader:
-                if destination is None or rows_in_shard == shard_rows:
-                    if destination is not None:
-                        destination.close()
-                    current = workspace / f"shard-{len(paths)}.tsv"
-                    destination = current.open("w", encoding="utf-8", newline="")
-                    writer = csv.DictWriter(
-                        destination,
-                        fieldnames=list(fieldnames),
-                        delimiter="\t",
-                        lineterminator="\n",
-                    )
-                    writer.writeheader()
-                    paths.append(current)
-                    rows_in_shard = 0
-                assert writer is not None
-                writer.writerow(row)
-                rows_in_shard += 1
-    finally:
-        if destination is not None:
-            destination.close()
-    if not paths:
-        raise ValueError("dataset has no data rows")
-    return paths
-
-
-def concatenate_descriptor_shards(
-    partial_paths: Sequence[Path],
-    output_path: Path,
-) -> dict[str, int]:
-    """Merge shard partial CSVs by shard index with exactly one header.
-
-    Every partial is a full CSV with the same header. The first partial is
-    copied verbatim; each later partial contributes only its data rows, so the
-    merged file is byte-identical to the single-process reference for the same
-    input rows.
-    """
-    if not partial_paths:
-        raise ValueError("descriptor reducer requires at least one partial")
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    rows_emitted = 0
-    with output_path.open("w", encoding="utf-8", newline="") as destination:
-        for index, partial in enumerate(partial_paths):
-            with partial.open("r", encoding="utf-8", newline="") as source:
-                for line_index, line in enumerate(source):
-                    if line_index == 0:
-                        if index > 0:
-                            continue
-                        if line.rstrip("\r\n") != ",".join(DESCRIPTOR_COLUMNS):
-                            raise ValueError(
-                                "partial descriptor CSV has an invalid header"
-                            )
-                    destination.write(line)
-                    if line_index > 0:
-                        rows_emitted += 1
-    return {"partial_count": len(partial_paths), "rows_emitted": rows_emitted}

scimesh/workloads/environment.py

@@ -8,7 +8,6 @@ workload definition packages (``search``, ``graph``, ``descriptors``) and the
 from __future__ import annotations
 
 import hashlib
-import os
 import platform
 import sys
 

scimesh/workloads/molwt_filter/core.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 
 import csv
 from pathlib import Path
-from typing import Mapping
 
 from rdkit import Chem
 from rdkit.Chem import Descriptors

scimesh/workloads/similarity_graph.py

@@ -12,7 +12,7 @@ from typing import Any
 
 from rdkit import DataStructs
 
-from scimesh.chemistry.dataset import DatasetStats, MoleculeRecord, iter_valid_molecules
+from scimesh.chemistry.dataset import DatasetStats, iter_valid_molecules
 from scimesh.chemistry.fingerprints import fingerprint
 
 

tests/test_sdk_descriptors.py

@@ -106,7 +106,6 @@ def test_local_sdk_executor_matches_descriptor_batch_reference(tmp_path: Path) -
     dataset = tmp_path / "molecules.tsv"
     _write_tiny_dataset(dataset)
     registry, runtime, workload, definition, _ = _registered_descriptor_batch()
-    manifest = workload.manifest
     artifact_store = LocalArtifactStore(tmp_path / "artifacts")
     request = _request_for(dataset, artifact_store, workload)
 

tests/test_sdk_graph.py

@@ -20,7 +20,6 @@ from scimesh.sdk import (
 from scimesh.workloads.graph import (
     check_pair_coverage,
     merge_edge_partials,
-    similarity_graph_sdk_definition,
 )
 from scimesh.workloads.library import default_sdk_registry, default_sdk_runtime
 from scimesh.workloads.similarity_graph import (

tests/test_sdk_search.py

@@ -15,11 +15,9 @@ from scimesh.sdk import (
     LocalCoreBatchExecutor,
     LocalPlanningContext,
     StageKind,
-    WorkloadRegistry,
     assert_manifest_round_trip,
 )
 from scimesh.workloads.library import default_sdk_registry, default_sdk_runtime
-from scimesh.workloads.search import similarity_search_sdk_definition
 from scimesh.workloads.similarity_search import (
     find_molecule_by_id,
     search_similar,

tests/test_worker_task.py

@@ -2,13 +2,11 @@
 
 from __future__ import annotations
 
-import hashlib
 import json
 import subprocess
 import sys
 from pathlib import Path
 
-import pytest
 
 from scimesh.worker.task import _EXIT_PERMANENT, _EXIT_RETRYABLE