Harden distributed pipeline
This commit is contained in:
@@ -107,7 +107,8 @@ Content-Type: application/json
|
||||
```
|
||||
|
||||
`cpu_count`/`memory_mb` are accepted for forward compatibility and not yet
|
||||
persisted. `capabilities` must be non-empty (an allowlisted workload set).
|
||||
persisted. `capabilities` must be non-empty. A claim uses the capabilities
|
||||
stored at registration; the request cannot broaden them.
|
||||
|
||||
## Claim task
|
||||
|
||||
@@ -120,7 +121,8 @@ Content-Type: application/json
|
||||
```
|
||||
|
||||
- `204 No Content`: no compatible task.
|
||||
- `200 OK`: a task is leased atomically.
|
||||
- `200 OK`: a task is leased atomically. `worker_id` must be a registered UUID;
|
||||
its persisted capabilities, rather than this request field, decide eligibility.
|
||||
|
||||
```json
|
||||
{
|
||||
|
||||
@@ -46,24 +46,25 @@ coordinator was started with. Never log it, never send it in an error body.
|
||||
|
||||
```http
|
||||
POST /workers/register
|
||||
{ "name": "lab-worker-01", "capabilities": ["similarity_search"] }
|
||||
{ "name": "lab-worker-01", "capabilities": ["similarity-search"] }
|
||||
```
|
||||
|
||||
Response: `{ "worker_id": "<uuid>", "heartbeat_interval_seconds": 15 }`.
|
||||
|
||||
- `capabilities` are the workload names you can run — the coordinator only hands
|
||||
you matching tasks.
|
||||
- `capabilities` are fixed at registration — the coordinator only hands you
|
||||
matching tasks and a later claim cannot broaden that set.
|
||||
- **Keep `worker_id`**. Use it as your identity in every later call. Using the
|
||||
registered UUID is what lets the coordinator track your liveness (it marks
|
||||
workers offline after they go silent).
|
||||
- Current coordinator jobs use `similarity_search` / `similarity_graph`; the
|
||||
reference Python worker also accepts the public CLI spellings with hyphens.
|
||||
- Current diagnostic uploads use `similarity-search` with `query_smiles`. The
|
||||
reference worker accepts the legacy `similarity_search` spelling too. Do not
|
||||
advertise `similarity-graph` until CTX-10 implements cross-shard pair planning.
|
||||
|
||||
## 2. Claim a task
|
||||
|
||||
```http
|
||||
POST /tasks/claim
|
||||
{ "worker_id": "<uuid>", "capabilities": ["similarity_search"] }
|
||||
{ "worker_id": "<uuid>", "capabilities": ["similarity-search"] }
|
||||
```
|
||||
|
||||
- `200` → a leased task (below).
|
||||
@@ -74,9 +75,9 @@ POST /tasks/claim
|
||||
"task_id": "<uuid>",
|
||||
"attempt": 1,
|
||||
"lease_expires_at": "2026-07-22T12:05:00Z",
|
||||
"workload": "similarity_search",
|
||||
"workload": "similarity-search",
|
||||
"input": { "uri": "/tasks/<uuid>/input", "sha256": "<hex>" },
|
||||
"parameters": { "query_id": "CHEMBL939", "top_k": 20 }
|
||||
"parameters": { "query_smiles": "CCO", "top_k": 20 }
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
+9
-7
@@ -96,7 +96,9 @@ paths:
|
||||
description: >
|
||||
multipart/form-data. The text fields (`workload`, `parameters`,
|
||||
`chunk_rows`, `max_rows`) MUST precede the `file` part: the file is streamed, not
|
||||
buffered, so the fields have to be parsed before it arrives.
|
||||
buffered, so the fields have to be parsed before it arrives. Currently
|
||||
only diagnostic `similarity-search` with `parameters.query_smiles` is
|
||||
accepted; distributed graph planning is not implemented.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
@@ -388,7 +390,7 @@ components:
|
||||
type: array
|
||||
minItems: 1
|
||||
items: { type: string }
|
||||
example: [similarity_search, similarity_graph]
|
||||
example: [similarity-search]
|
||||
cpu_count:
|
||||
type: integer
|
||||
description: Accepted for forward compatibility; not yet persisted.
|
||||
@@ -419,7 +421,7 @@ components:
|
||||
type: object
|
||||
required: [workload, input_uri, chunks]
|
||||
properties:
|
||||
workload: { type: string, example: similarity_search }
|
||||
workload: { type: string, example: similarity-search }
|
||||
input_uri: { type: string }
|
||||
parameters: { type: object, additionalProperties: true }
|
||||
chunks:
|
||||
@@ -437,11 +439,11 @@ components:
|
||||
type: object
|
||||
required: [workload, file]
|
||||
properties:
|
||||
workload: { type: string, example: similarity_search }
|
||||
workload: { type: string, enum: [similarity-search], example: similarity-search }
|
||||
parameters:
|
||||
type: string
|
||||
description: JSON object, sent as a string form field.
|
||||
example: '{"top_k":10}'
|
||||
example: '{"query_smiles":"CCO","top_k":10}'
|
||||
chunk_rows:
|
||||
type: integer
|
||||
description: Data rows per shard. Default 1000.
|
||||
@@ -486,11 +488,11 @@ components:
|
||||
type: object
|
||||
required: [worker_id]
|
||||
properties:
|
||||
worker_id: { type: string }
|
||||
worker_id: { type: string, format: uuid, description: Registered worker identity. }
|
||||
capabilities:
|
||||
type: array
|
||||
items: { type: string }
|
||||
description: Workloads this worker can run. Empty means "any".
|
||||
description: Accepted for compatibility only; registration capabilities decide eligibility.
|
||||
max_concurrency:
|
||||
type: integer
|
||||
description: Accepted; the coordinator leases one task per call.
|
||||
|
||||
@@ -59,7 +59,7 @@ Content-Type: application/json
|
||||
|
||||
{
|
||||
"name": "lab-worker-01",
|
||||
"capabilities": ["similarity-search", "similarity-graph"],
|
||||
"capabilities": ["similarity-search"],
|
||||
"cpu_count": 8,
|
||||
"memory_mb": 16384
|
||||
}
|
||||
@@ -74,8 +74,8 @@ POST /tasks/claim
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"worker_id": "worker-01",
|
||||
"capabilities": ["similarity-search", "similarity-graph"],
|
||||
"worker_id": "<registered-uuid>",
|
||||
"capabilities": ["similarity-search"],
|
||||
"max_concurrency": 1
|
||||
}
|
||||
```
|
||||
@@ -95,7 +95,7 @@ When a task is available, it returns `200 OK`:
|
||||
"sha256": "..."
|
||||
},
|
||||
"parameters": {
|
||||
"query_id": "CHEMBL939",
|
||||
"query_smiles": "CCO",
|
||||
"top_k": 20
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user