diff --git a/.opencode/session-goal.md b/.opencode/session-goal.md index fd44d35..4945f1e 100644 --- a/.opencode/session-goal.md +++ b/.opencode/session-goal.md @@ -1,35 +1,23 @@ COMPLETED # Session Goal -пропиши абсолютно полную документацию scimesh в mkdocs. +давай теперь почистим проект от линего кода ## Plan -1. Каталог `mkdocs/` — самостоятельный источник сайта (docs_dir: mkdocs); проектные `docs/` не трогаем. -2. mkdocs.yml: docs_dir, полный nav (Home → SDK guides → API reference (все модули scimesh.sdk) → Documentation approach), edit_uri. -3. Контент: - - index.md — лендинг: что такое SciMesh, архитектура, быстрый старт. - - sdk/overview.md — концепция SDK (framework-only, core-batch-v1, безопасность, fail-closed). - - sdk/authoring-workloads.md — полный гайд MapReduceWorkload (атрибуты, хуки, полный пример molwt-filter, регистрация, тесты). - - sdk/cli.md — scimesh workload list/run/export + env. - - sdk/worker-integration.md — воркер: claim/heartbeat/upload, SDK-мост, allowlist, ограничения v1. - - api/ — автогенерация mkdocstrings для ВСЕХ модулей scimesh.sdk (batch, artifacts, manifest, workflow, plans, registry, verification, runtime, resources, conformance, execution, identity, integrity, schema, protocols) + index.md. - - approach.md — зафиксированные правила написания документации. -4. Убрать тестовые страницы mkdocs из docs/ (docs/index.md, docs/api/). -5. make docs — проверить рендер всех страниц; итерации по ошибкам mkdocstrings. -6. pytest + pyright — зелёные; /ui/docs/ интеграция не ломается; README/STATUS обновить. +1. Аудит: ruff/pyflakes — неиспользуемые импорты по scimesh/ и tests/; grep — неиспользуемые функции/модули (после рефакторингов могли остаться мёртвые экспорты, например в descriptors/search/graph core и sdk/_validation). +2. Удаление мёртвого кода: неиспользуемые импорты, функции, дубли (например write_descriptor_shards/concatenate_descriptor_shards, если вытеснены дефолтами batch), устаревшие файлы-обёртки. +3. Проверка, что ничего публичного/API не сломано: pyright 0 ошибок, pytest зелёный, go test/vet, mkdocs build. +4. Финал: полный прогон, COMPLETED. ## Progress -- [x] mkdocs.yml: `docs_dir: mkdocs`, полный nav, edit_uri; тема Material + mkdocstrings (handlers.python.options, show_if_no_docstring: true). -- [x] index.md — лендинг (быстрый старт, карта сайта). -- [x] sdk/overview.md, sdk/authoring-workloads.md (полный гайд + molwt-filter пример + plan_tasks блоки), sdk/cli.md, sdk/worker-integration.md. -- [x] api/: 15 страниц `::: scimesh.sdk.` + api/index.md (модульная карта) — ВСЕ модули SDK. -- [x] approach.md — правила: типы страниц, reference только из docstrings (Google style), терминология, код-блоки, ссылки, когда обновлять, сборка. -- [x] Удалены тестовые страницы mkdocs из docs/ (docs/index.md, docs/api/). -- [x] Устранена проблема mkdocstrings-python 2.x: `default_options` → `handlers.python.options`; `show_if_no_docstring: true` (иначе члены без docstrings не рендерились). -- [x] Добавлены Google-style docstrings ВСЕМ публичным членам scimesh.sdk без них (~50: artifacts, manifest, plans, workflow, execution, identity, protocols, registry, resources, runtime, verification, schema, conformance); починено подавление pyright в schema.py (перенесённый форматтером `# type: ignore`). -- [x] make docs --strict: 0 ошибок/предупреждений; все 16 API-страниц + 4 гайда + approach рендерятся (проверено по содержимому HTML). -- [x] pytest 256 passed; pyright scimesh/tests 0 errors. -- [x] README/STATUS/handoff обновлены (mkdocs/ сайт, /ui/docs/). -- Изменения НЕ закоммичены (коммит по запросу). +- [x] Аудит: ruff F401/F811/F841 — 26 неиспользуемых импортов; vulture — кандидаты проверены grep'ом. +- [x] Удалено: + - 25 неиспользуемых импортов (ruff --fix) + 1 неиспользуемая локальная переменная в тесте; + - мёртвые функции descriptors/core.py `write_descriptor_shards`/`concatenate_descriptor_shards` (вытеснены дефолтами MapReduceWorkload; ссылки только в собственном `__init__`) + их экспорты; + - мёртвые атрибуты `MapReduceWorkload._resources/_execution` (записывались, нигде не читались); + - мёртвый `CancellationFlag.cancel` (0 использований); +- [x] `scripts/two-worker-smoke.sh` (рабочий E2E, но без точки входа) подключён как `make smoke-two-worker` — не мёртвый, а доступный. +- [x] Проверено: ruff clean; pytest 260 passed; pyright 0 ошибок (scimesh+tests); go test 11 пакетов + vet; mkdocs build без warnings. +- [x] Изменения не закоммичены. diff --git a/Makefile b/Makefile index dfde28d..7b791aa 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -.PHONY: help demo-ui demo-down demo-logs docs docs-serve +.PHONY: help demo-ui demo-down demo-logs smoke-two-worker docs docs-serve help: @printf '%s\n' \ @@ -9,6 +9,8 @@ help: ' make demo-ui WORKERS=3 Start the demo with 3 local workers.' \ ' make demo-logs Follow coordinator logs for the demo.' \ ' make demo-down Stop demo containers and workers.' \ + ' make smoke-two-worker E2E: two workers process 4 shards and the' \ + ' result must match the local CLI reference.' \ ' make docs Build the MkDocs site into site/.' \ ' make docs-serve Serve the MkDocs site at http://localhost:8000.' \ '' \ @@ -25,6 +27,9 @@ demo-down: demo-logs: $(MAKE) -C coordinator demo-logs +smoke-two-worker: + ./scripts/two-worker-smoke.sh + docs: .venv/bin/mkdocs build diff --git a/scimesh/sdk/artifacts.py b/scimesh/sdk/artifacts.py index 2998d73..629ee5c 100644 --- a/scimesh/sdk/artifacts.py +++ b/scimesh/sdk/artifacts.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum from types import MappingProxyType -from typing import Any, Mapping, Sequence +from typing import Any, Mapping from ._validation import ( canonical_json, diff --git a/scimesh/sdk/batch.py b/scimesh/sdk/batch.py index ae51262..b7f30e1 100644 --- a/scimesh/sdk/batch.py +++ b/scimesh/sdk/batch.py @@ -24,9 +24,7 @@ from typing import Any, Mapping, Sequence from .artifacts import ( ArtifactCollection, - ArtifactItem, ArtifactRef, - ArtifactSchema, Cardinality, CollectionKind, OutputManifest, @@ -295,8 +293,6 @@ class MapReduceWorkload: conformance_profiles=("core-batch-v1",), ) self._exact_verifier = _EXACT_VERIFIER - self._resources = resources - self._execution = execution self._limits = limits # ------------------------------------------------------------------ diff --git a/scimesh/sdk/conformance.py b/scimesh/sdk/conformance.py index 5cf5b78..874a5e3 100644 --- a/scimesh/sdk/conformance.py +++ b/scimesh/sdk/conformance.py @@ -24,7 +24,7 @@ from .artifacts import ( OutputManifest, Provenance, ) -from .identity import ComponentRef, SDK_API_VERSION, SchemaRef +from .identity import ComponentRef, SDK_API_VERSION from .execution import NetworkPolicy, ProcessModel from .manifest import TrustMode, WorkloadManifest from .plans import JobRequest, TaskSpec @@ -686,9 +686,6 @@ class CancellationFlag: def __init__(self) -> None: self._event = Event() - def cancel(self) -> None: - self._event.set() - def cancelled(self) -> bool: return self._event.is_set() diff --git a/scimesh/sdk/integrity.py b/scimesh/sdk/integrity.py index 677f507..0e093bd 100644 --- a/scimesh/sdk/integrity.py +++ b/scimesh/sdk/integrity.py @@ -4,7 +4,6 @@ from __future__ import annotations import hashlib import importlib.util -import os from importlib import metadata from pathlib import Path diff --git a/scimesh/sdk/protocols.py b/scimesh/sdk/protocols.py index 247465c..18060f1 100644 --- a/scimesh/sdk/protocols.py +++ b/scimesh/sdk/protocols.py @@ -3,7 +3,7 @@ from __future__ import annotations from pathlib import Path -from typing import Mapping, Protocol, Sequence +from typing import Mapping, Protocol from .artifacts import ( ArtifactCollection, diff --git a/scimesh/sdk/registry.py b/scimesh/sdk/registry.py index 8d19849..aaa1c02 100644 --- a/scimesh/sdk/registry.py +++ b/scimesh/sdk/registry.py @@ -26,7 +26,6 @@ from .manifest import WorkloadManifest from .plans import JobRequest, ValidatedJob, WorkflowPlan from .protocols import ( Planner, - PlanningContext, PlanningResources, Reducer, Runner, diff --git a/scimesh/sdk/resources.py b/scimesh/sdk/resources.py index 9d4e7a9..d2d6f94 100644 --- a/scimesh/sdk/resources.py +++ b/scimesh/sdk/resources.py @@ -5,7 +5,6 @@ from __future__ import annotations from dataclasses import dataclass from enum import Enum from threading import Lock -from types import MappingProxyType from typing import Mapping from uuid import uuid4 diff --git a/scimesh/sdk/runtime.py b/scimesh/sdk/runtime.py index 08160ea..eed7bad 100644 --- a/scimesh/sdk/runtime.py +++ b/scimesh/sdk/runtime.py @@ -10,7 +10,6 @@ from ._validation import ( require_identifier, require_string, validate_version_range, - version_in_range, ) from .identity import SDK_API_VERSION from .execution import NetworkPolicy, ProcessModel diff --git a/scimesh/sdk/schema.py b/scimesh/sdk/schema.py index bf89cbd..66d32d4 100644 --- a/scimesh/sdk/schema.py +++ b/scimesh/sdk/schema.py @@ -5,7 +5,7 @@ from __future__ import annotations import math import re from fractions import Fraction -from typing import Mapping, Sequence +from typing import Mapping _ANNOTATIONS = { diff --git a/scimesh/sdk/verification.py b/scimesh/sdk/verification.py index 34f2c67..93e6f8b 100644 --- a/scimesh/sdk/verification.py +++ b/scimesh/sdk/verification.py @@ -20,7 +20,6 @@ from ._validation import ( freeze_json_mapping, require_exact_keys, require_identifier, - require_nonnegative_int, require_positive_int, require_sha256, require_string, diff --git a/scimesh/sdk/workflow.py b/scimesh/sdk/workflow.py index dc982dd..af5fc25 100644 --- a/scimesh/sdk/workflow.py +++ b/scimesh/sdk/workflow.py @@ -15,7 +15,6 @@ from ._validation import ( require_nonnegative_int, require_positive_int, require_schema_version, - require_string, ) from .artifacts import PortSpec from .execution import ExecutionProfile, NetworkPolicy, RetryPolicy diff --git a/scimesh/worker/config.py b/scimesh/worker/config.py index 86a16b0..f92c653 100644 --- a/scimesh/worker/config.py +++ b/scimesh/worker/config.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from dataclasses import dataclass from math import isfinite from pathlib import Path diff --git a/scimesh/worker/daemon.py b/scimesh/worker/daemon.py index e98c6ec..06b0b20 100644 --- a/scimesh/worker/daemon.py +++ b/scimesh/worker/daemon.py @@ -5,7 +5,6 @@ from __future__ import annotations import logging from dataclasses import replace from dataclasses import dataclass -from pathlib import Path import random import re import shutil diff --git a/scimesh/worker/runners.py b/scimesh/worker/runners.py index 3cc3225..8bb4946 100644 --- a/scimesh/worker/runners.py +++ b/scimesh/worker/runners.py @@ -24,7 +24,6 @@ from uuid import NAMESPACE_URL, uuid5 from scimesh.sdk.artifacts import ( ArtifactCollection, - ArtifactRef, OutputManifest, Provenance, ) diff --git a/scimesh/workloads/descriptors/__init__.py b/scimesh/workloads/descriptors/__init__.py index c0c7790..f815f59 100644 --- a/scimesh/workloads/descriptors/__init__.py +++ b/scimesh/workloads/descriptors/__init__.py @@ -9,11 +9,9 @@ from .core import ( DESCRIPTOR_NAMES, DescriptorRow, compute_descriptor_batch, - concatenate_descriptor_shards, descriptor_calculator, validate_descriptor_names, write_descriptor_rows, - write_descriptor_shards, ) from .definition import ( MAP_ENTRY_POINT, @@ -31,11 +29,9 @@ __all__ = [ "DescriptorBatchWorkload", "DescriptorRow", "compute_descriptor_batch", - "concatenate_descriptor_shards", "descriptor_batch_sdk_definition", "descriptor_calculator", "validate_descriptor_names", "workload_definition", "write_descriptor_rows", - "write_descriptor_shards", ] diff --git a/scimesh/workloads/descriptors/core.py b/scimesh/workloads/descriptors/core.py index 532e650..11a1bf1 100644 --- a/scimesh/workloads/descriptors/core.py +++ b/scimesh/workloads/descriptors/core.py @@ -19,7 +19,7 @@ import csv from dataclasses import dataclass from functools import lru_cache from pathlib import Path -from typing import Any, Iterator, Mapping, Sequence +from typing import Iterator, Sequence from rdkit import Chem from rdkit.ML.Descriptors.MoleculeDescriptors import MolecularDescriptorCalculator @@ -229,86 +229,3 @@ def compute_descriptor_batch( materialized = list(rows) write_descriptor_rows(output_path, materialized) return stats.as_metrics() - - -def write_descriptor_shards( - input_path: Path, - workspace: Path, - shard_rows: int, -) -> list[Path]: - """Split the input TSV into deterministic row-bounded shards with headers.""" - if ( - isinstance(shard_rows, bool) - or not isinstance(shard_rows, int) - or shard_rows < 1 - ): - raise ValueError("shard_rows must be a positive integer") - paths: list[Path] = [] - current: Path | None = None - destination = None - writer = None - rows_in_shard = 0 - try: - with input_path.open("r", encoding="utf-8", newline="") as source: - reader = csv.DictReader(source, delimiter="\t") - fieldnames = tuple(reader.fieldnames or ()) - if not {"chembl_id", "canonical_smiles"}.issubset(set(fieldnames)): - raise ValueError( - "dataset is missing required columns: chembl_id, canonical_smiles" - ) - for row in reader: - if destination is None or rows_in_shard == shard_rows: - if destination is not None: - destination.close() - current = workspace / f"shard-{len(paths)}.tsv" - destination = current.open("w", encoding="utf-8", newline="") - writer = csv.DictWriter( - destination, - fieldnames=list(fieldnames), - delimiter="\t", - lineterminator="\n", - ) - writer.writeheader() - paths.append(current) - rows_in_shard = 0 - assert writer is not None - writer.writerow(row) - rows_in_shard += 1 - finally: - if destination is not None: - destination.close() - if not paths: - raise ValueError("dataset has no data rows") - return paths - - -def concatenate_descriptor_shards( - partial_paths: Sequence[Path], - output_path: Path, -) -> dict[str, int]: - """Merge shard partial CSVs by shard index with exactly one header. - - Every partial is a full CSV with the same header. The first partial is - copied verbatim; each later partial contributes only its data rows, so the - merged file is byte-identical to the single-process reference for the same - input rows. - """ - if not partial_paths: - raise ValueError("descriptor reducer requires at least one partial") - output_path.parent.mkdir(parents=True, exist_ok=True) - rows_emitted = 0 - with output_path.open("w", encoding="utf-8", newline="") as destination: - for index, partial in enumerate(partial_paths): - with partial.open("r", encoding="utf-8", newline="") as source: - for line_index, line in enumerate(source): - if line_index == 0: - if index > 0: - continue - if line.rstrip("\r\n") != ",".join(DESCRIPTOR_COLUMNS): - raise ValueError( - "partial descriptor CSV has an invalid header" - ) - destination.write(line) - if line_index > 0: - rows_emitted += 1 - return {"partial_count": len(partial_paths), "rows_emitted": rows_emitted} diff --git a/scimesh/workloads/environment.py b/scimesh/workloads/environment.py index b03e1c9..8668c26 100644 --- a/scimesh/workloads/environment.py +++ b/scimesh/workloads/environment.py @@ -8,7 +8,6 @@ workload definition packages (``search``, ``graph``, ``descriptors``) and the from __future__ import annotations import hashlib -import os import platform import sys diff --git a/scimesh/workloads/molwt_filter/core.py b/scimesh/workloads/molwt_filter/core.py index 7cdd723..deed00f 100644 --- a/scimesh/workloads/molwt_filter/core.py +++ b/scimesh/workloads/molwt_filter/core.py @@ -10,7 +10,6 @@ from __future__ import annotations import csv from pathlib import Path -from typing import Mapping from rdkit import Chem from rdkit.Chem import Descriptors diff --git a/scimesh/workloads/similarity_graph.py b/scimesh/workloads/similarity_graph.py index 08df9a2..9134039 100644 --- a/scimesh/workloads/similarity_graph.py +++ b/scimesh/workloads/similarity_graph.py @@ -12,7 +12,7 @@ from typing import Any from rdkit import DataStructs -from scimesh.chemistry.dataset import DatasetStats, MoleculeRecord, iter_valid_molecules +from scimesh.chemistry.dataset import DatasetStats, iter_valid_molecules from scimesh.chemistry.fingerprints import fingerprint diff --git a/tests/test_sdk_descriptors.py b/tests/test_sdk_descriptors.py index ebda0a3..22c1523 100644 --- a/tests/test_sdk_descriptors.py +++ b/tests/test_sdk_descriptors.py @@ -106,7 +106,6 @@ def test_local_sdk_executor_matches_descriptor_batch_reference(tmp_path: Path) - dataset = tmp_path / "molecules.tsv" _write_tiny_dataset(dataset) registry, runtime, workload, definition, _ = _registered_descriptor_batch() - manifest = workload.manifest artifact_store = LocalArtifactStore(tmp_path / "artifacts") request = _request_for(dataset, artifact_store, workload) diff --git a/tests/test_sdk_graph.py b/tests/test_sdk_graph.py index daa4275..3ef757d 100644 --- a/tests/test_sdk_graph.py +++ b/tests/test_sdk_graph.py @@ -20,7 +20,6 @@ from scimesh.sdk import ( from scimesh.workloads.graph import ( check_pair_coverage, merge_edge_partials, - similarity_graph_sdk_definition, ) from scimesh.workloads.library import default_sdk_registry, default_sdk_runtime from scimesh.workloads.similarity_graph import ( diff --git a/tests/test_sdk_search.py b/tests/test_sdk_search.py index d4953cb..847e5ae 100644 --- a/tests/test_sdk_search.py +++ b/tests/test_sdk_search.py @@ -15,11 +15,9 @@ from scimesh.sdk import ( LocalCoreBatchExecutor, LocalPlanningContext, StageKind, - WorkloadRegistry, assert_manifest_round_trip, ) from scimesh.workloads.library import default_sdk_registry, default_sdk_runtime -from scimesh.workloads.search import similarity_search_sdk_definition from scimesh.workloads.similarity_search import ( find_molecule_by_id, search_similar, diff --git a/tests/test_worker_task.py b/tests/test_worker_task.py index e8d8253..46e26c8 100644 --- a/tests/test_worker_task.py +++ b/tests/test_worker_task.py @@ -2,13 +2,11 @@ from __future__ import annotations -import hashlib import json import subprocess import sys from pathlib import Path -import pytest from scimesh.worker.task import _EXIT_PERMANENT, _EXIT_RETRYABLE