From 6517145622b38fc665631da6e186d69dd3065da1 Mon Sep 17 00:00:00 2001 From: Efremenko Arhip Date: Wed, 22 Jul 2026 14:53:27 +0300 Subject: [PATCH] chore(coordinator): add API smoke script and request collection Two ways to exercise every endpoint, both living in the repo rather than in a personal Postman workspace: - scripts/smoke.sh walks the full lifecycle and asserts each status, exiting non-zero on the first surprise, so it works in CI as well as by hand; - api/requests.http drives the same calls from an editor's REST client, with later requests reusing ids captured from earlier responses. It doubles as API documentation for the worker author. The script claims until it sees its own job's chunks instead of assuming an empty queue: a shared development database usually holds pending tasks from earlier runs, and it takes the attempt number from the claim response, since a task requeued after an expired lease comes back with attempt 2 or 3. Note for whoever extends the validation cases: Go matches JSON field names case-insensitively, so "worker_ID" is accepted as "worker_id". Only a genuinely unknown key trips DisallowUnknownFields. --- coordinator/Makefile | 7 ++ coordinator/README.md | 12 +++ coordinator/api/requests.http | 157 ++++++++++++++++++++++++++++++++++ coordinator/scripts/smoke.sh | 122 ++++++++++++++++++++++++++ 4 files changed, 298 insertions(+) create mode 100644 coordinator/api/requests.http create mode 100755 coordinator/scripts/smoke.sh diff --git a/coordinator/Makefile b/coordinator/Makefile index 38e4b54..7dceb9b 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -63,3 +63,10 @@ rebuild: psql: docker compose exec postgres psql -U scimesh -d scimesh + +# --- api ------------------------------------------------------------------ +# Exercises every endpoint against a running coordinator; exits non-zero on the +# first unexpected status. See also api/requests.http for clicking through them +# one at a time in an editor. +smoke: + ./scripts/smoke.sh diff --git a/coordinator/README.md b/coordinator/README.md index ef54ec1..8a10bac 100644 --- a/coordinator/README.md +++ b/coordinator/README.md @@ -111,6 +111,18 @@ See `.env.example`; only `DATABASE_URL` is required. | GET | `/jobs/{job_id}` | Aggregate job progress | | GET | `/health` | Liveness (unauthenticated) | +## Poking the API + +Two ways, both checked in: + +```sh +make smoke # every endpoint, asserted; non-zero exit on failure +``` + +`api/requests.http` runs the same calls one at a time from an editor with a REST +client (VSCodium/VS Code "REST Client", JetBrains HTTP Client). Later requests +reuse ids captured from earlier responses, so it doubles as API documentation. + ## Status The queue works end to end: a job can be submitted, split into tasks, leased to diff --git a/coordinator/api/requests.http b/coordinator/api/requests.http new file mode 100644 index 0000000..5a4b356 --- /dev/null +++ b/coordinator/api/requests.http @@ -0,0 +1,157 @@ +# SciMesh Coordinator — API requests +# +# Runnable from any editor with a REST client (VSCodium/VS Code "REST Client", +# JetBrains HTTP Client). Click "Send Request" above each block, top to bottom: +# later requests reuse ids captured from earlier responses. +# +# Start the stack first: docker compose up -d + +@host = http://localhost:8080 +@token = change-me +@worker = worker-1 + +### Health — the only unauthenticated endpoint +GET {{host}}/health + +### Auth check — no token must be rejected with 401 +POST {{host}}/tasks/claim +Content-Type: application/json + +{ "worker_id": "{{worker}}" } + +### 1. Create a job and its chunks (201) +# The coordinator splits the submission into one task per chunk, transactionally. +# @name createJob +POST {{host}}/jobs +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "workload": "similarity_search", + "input_uri": "s3://chembl/full.sdf", + "parameters": { "top_k": 10 }, + "chunks": [ + { "chunk_index": 0, "input_uri": "s3://chembl/shard-0.sdf", "input_sha256": "aaa", "max_attempts": 3 }, + { "chunk_index": 1, "input_uri": "s3://chembl/shard-1.sdf", "input_sha256": "bbb", "max_attempts": 3 }, + { "chunk_index": 2, "input_uri": "s3://chembl/shard-2.sdf", "input_sha256": "ccc", "max_attempts": 3 } + ] +} + +@jobId = {{createJob.response.body.id}} + +### 2. Claim a task (200, or 204 when the queue is empty) +# Each call leases a different task; run it repeatedly to see chunk_index advance. +# @name claim +POST {{host}}/tasks/claim +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "{{worker}}", + "workloads": ["similarity_search"] +} + +@taskId = {{claim.response.body.task_id}} +@attempt = {{claim.response.body.attempt}} + +### 3. Heartbeat — renew the lease while the task is still running (200) +POST {{host}}/tasks/{{taskId}}/heartbeat +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "{{worker}}", + "attempt": {{attempt}} +} + +### 4. Submit the result (200) +POST {{host}}/tasks/{{taskId}}/result +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "{{worker}}", + "attempt": {{attempt}}, + "result_uri": "s3://results/shard-0.csv", + "result_sha256": "r0sha", + "metrics": { "elapsed_ms": 1234, "candidates": 50000 } +} + +### 4a. Replay the same result — must be idempotent (200, not 409) +POST {{host}}/tasks/{{taskId}}/result +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "{{worker}}", + "attempt": {{attempt}}, + "result_uri": "s3://results/shard-0.csv", + "result_sha256": "r0sha", + "metrics": { "elapsed_ms": 1234, "candidates": 50000 } +} + +### 4b. A different result for the same task — conflict (409) +POST {{host}}/tasks/{{taskId}}/result +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "{{worker}}", + "attempt": {{attempt}}, + "result_uri": "s3://results/SOMETHING-ELSE.csv", + "result_sha256": "different" +} + +### 4c. Another worker submitting for this task — conflict (409) +POST {{host}}/tasks/{{taskId}}/result +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "impostor", + "attempt": {{attempt}}, + "result_uri": "s3://results/x.csv", + "result_sha256": "x" +} + +### 5. Report a failure instead (200) +# retryable=true returns the task to the queue while attempts remain; +# retryable=false fails it terminally. +POST {{host}}/tasks/{{taskId}}/failure +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "worker_id": "{{worker}}", + "attempt": {{attempt}}, + "error_code": "download_failed", + "error_message": "checksum mismatch on shard", + "retryable": true +} + +### 6. Job progress (200) +GET {{host}}/jobs/{{jobId}} +Authorization: Bearer {{token}} + +### --- error cases ------------------------------------------------------- + +### Malformed UUID in the path (400) +POST {{host}}/tasks/not-a-uuid/result +Authorization: Bearer {{token}} +Content-Type: application/json + +{ "worker_id": "{{worker}}", "attempt": 1, "result_uri": "s3://x", "result_sha256": "x" } + +### Unknown field in the body (400) — a misspelled key must not pass silently +POST {{host}}/tasks/claim +Authorization: Bearer {{token}} +Content-Type: application/json + +{ "worker_ID": "{{worker}}" } + +### Unknown job (404) +GET {{host}}/jobs/00000000-0000-0000-0000-000000000000 +Authorization: Bearer {{token}} + +### Stitching is not implemented yet (501) +# Any endpoint whose use case is still a stub answers 501. diff --git a/coordinator/scripts/smoke.sh b/coordinator/scripts/smoke.sh new file mode 100755 index 0000000..16702f9 --- /dev/null +++ b/coordinator/scripts/smoke.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# +# End-to-end smoke test against a running coordinator. +# +# ./scripts/smoke.sh # localhost:8080, token from .env +# HOST=http://1.2.3.4:8080 TOKEN=x ./scripts/smoke.sh +# +# Exits non-zero on the first unexpected status, so it is usable in CI. + +set -uo pipefail + +HOST="${HOST:-http://localhost:8080}" +TOKEN="${TOKEN:-$(grep -s '^WORKER_AUTH_TOKEN=' .env | cut -d= -f2- || echo change-me)}" + +pass=0 +fail=0 + +# check