feat(coordinator): upload a dataset and chunk it into shard tasks (CTX-05, part 4)

The coordinator can now ingest a dataset itself, not only accept client-supplied
chunk URIs.

- internal/chunk: a deterministic, generic TSV row splitter — repeats the header
  per shard, buffers one shard at a time, rejects header-only input. Unit-tested.
- POST /jobs/upload (multipart): streams the dataset into an input artifact,
  splits it into shard artifacts, and creates one shard task per shard, all in
  one transaction; blobs are cleaned up if the transaction fails.
- GET /tasks/{id}/input streams a task's input shard back to the worker.
- domain: NewUploadedJob, NewShardTask, Task/Job.InputArtifactID; a shard task's
  input is an artifact, not a URI. Claim response nests input:{uri,sha256} per
  the contract, with uri = /tasks/{id}/input for shards.
- migration 0005 makes input_uri nullable and adds a has-input check.
- The existing URI-based POST /jobs path is untouched; both coexist.
This commit is contained in:
Efremenko Arhip
2026-07-23 16:34:04 +03:00
parent 4a092d2e4e
commit c3243a6b7e
19 changed files with 741 additions and 42 deletions
+50
View File
@@ -147,6 +147,56 @@ check "unknown json field → 400" 400 -X POST "${HOST}/tasks/claim" "
-d '{"worker_id":"w1","totally_unknown":1}'
check "unknown job → 404" 404 "${HOST}/jobs/00000000-0000-0000-0000-000000000000" "${auth[@]}"
echo
echo "dataset upload → chunking"
# Upload a 5-row TSV split at 2 rows/shard → expect 3 shard tasks. The text
# fields precede the file part, which the coordinator streams.
up=$(curl -sS "${bearer[@]}" -X POST "${HOST}/jobs/upload" \
-F 'workload=similarity_search' \
-F 'parameters={"top_k":10}' \
-F 'chunk_rows=2' \
-F 'file=@-;filename=chembl.tsv;type=text/tab-separated-values' <<'TSV'
id smiles
A CC
B CCC
C CCCC
D CCCCC
E CCCCCC
TSV
)
up_job=$(printf '%s' "$up" | python3 -c 'import json,sys;print(json.load(sys.stdin)["job_id"])' 2>/dev/null)
up_count=$(printf '%s' "$up" | python3 -c 'import json,sys;print(json.load(sys.stdin)["task_count"])' 2>/dev/null)
if [[ "$up_count" == "3" ]]; then
printf ' \033[32m✓\033[0m %-46s task_count=3\n' "POST /jobs/upload (5 rows / 2)"
pass=$((pass + 1))
else
printf ' \033[31m✗\033[0m %-46s got task_count=%s, want 3\n' "POST /jobs/upload" "${up_count:-?}"
printf ' %s\n' "$up"
fail=$((fail + 1))
fi
# Claim one of this job's shard tasks and pull its input shard from the coordinator.
up_input=""
for _ in $(seq 1 30); do
c=$(curl -sS "${bearer[@]}" -H 'Content-Type: application/json' -X POST "${HOST}/tasks/claim" \
-d '{"worker_id":"up-w","capabilities":["similarity_search"]}')
[[ -z "$c" ]] && break
cj=$(printf '%s' "$c" | python3 -c 'import json,sys;print(json.load(sys.stdin)["job_id"])' 2>/dev/null)
[[ "$cj" != "$up_job" ]] && continue
up_input=$(printf '%s' "$c" | python3 -c 'import json,sys;print(json.load(sys.stdin)["input"]["uri"])' 2>/dev/null)
break
done
if [[ "$up_input" == /tasks/*/input ]]; then
printf ' \033[32m✓\033[0m %-46s %s\n' "claim → input.uri points at coordinator" "$up_input"
pass=$((pass + 1))
else
printf ' \033[31m✗\033[0m %-46s got %q\n' "claim shard input.uri" "$up_input"
fail=$((fail + 1))
fi
check "download shard input" 200 "${HOST}${up_input}" "${bearer[@]}"
echo
curl -sS "${HOST}/jobs/${job_id}" "${auth[@]}"
echo