From 5a1414bee91c41f2a7e14e9455ea03224afa18dd Mon Sep 17 00:00:00 2001 From: Emil Date: Fri, 24 Jul 2026 17:13:26 +0300 Subject: [PATCH] Add local pipeline demo launcher --- .gitignore | 1 + Makefile | 12 +++ coordinator/Makefile | 41 +++++++++- coordinator/README.md | 18 ++++ coordinator/scripts/demo-ui.sh | 145 +++++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100755 coordinator/scripts/demo-ui.sh diff --git a/.gitignore b/.gitignore index 93ddbb4..9ba4f7a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ test_structures/ # Local coordinator-worker execution state worker-data*/ scimesh-worker-data/ +coordinator/.demo/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..895f2c3 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +.PHONY: demo-ui demo-down demo-logs + +# Convenient entry points from the repository root. Extra settings are passed +# through, for example: make demo-ui DEMO_WORKERS=3 +demo-ui: + $(MAKE) -C coordinator demo-ui + +demo-down: + $(MAKE) -C coordinator demo-down + +demo-logs: + $(MAKE) -C coordinator demo-logs diff --git a/coordinator/Makefile b/coordinator/Makefile index 564b2cf..a5465bb 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -1,4 +1,4 @@ -.PHONY: build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke +.PHONY: build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke demo-ui demo-down demo-logs # `check` deliberately uses its own Compose project and host ports. This keeps # it from connecting to or replacing a developer's local PostgreSQL instance. @@ -10,6 +10,45 @@ CHECK_TOKEN ?= dev-token CHECK_DATABASE_URL ?= postgres://scimesh:scimesh@localhost:$(CHECK_POSTGRES_PORT)/scimesh?sslmode=disable CHECK_COMPOSE = POSTGRES_PORT=$(CHECK_POSTGRES_PORT) COORDINATOR_PORT=$(CHECK_COORDINATOR_PORT) docker compose -p $(CHECK_PROJECT) +# --- local manual demo --------------------------------------------------- +# A separate project and ports mean this demo cannot collide with the normal +# `make up` stack or a developer's local PostgreSQL on 5432. +DEMO_PROJECT ?= scimesh-demo +DEMO_POSTGRES_PORT ?= 55432 +DEMO_COORDINATOR_PORT ?= 18080 +DEMO_UI_TOKEN ?= demo-ui-secret +DEMO_WORKER_TOKEN ?= demo-worker-token +DEMO_WORKERS ?= 2 +DEMO_DIR ?= .demo + +demo-ui: + @DEMO_PROJECT="$(DEMO_PROJECT)" \ + DEMO_POSTGRES_PORT="$(DEMO_POSTGRES_PORT)" \ + DEMO_COORDINATOR_PORT="$(DEMO_COORDINATOR_PORT)" \ + DEMO_UI_TOKEN="$(DEMO_UI_TOKEN)" \ + DEMO_WORKER_TOKEN="$(DEMO_WORKER_TOKEN)" \ + DEMO_WORKERS="$(DEMO_WORKERS)" \ + DEMO_DIR="$(DEMO_DIR)" \ + ./scripts/demo-ui.sh start + +demo-down: + @DEMO_PROJECT="$(DEMO_PROJECT)" \ + DEMO_POSTGRES_PORT="$(DEMO_POSTGRES_PORT)" \ + DEMO_COORDINATOR_PORT="$(DEMO_COORDINATOR_PORT)" \ + DEMO_UI_TOKEN="$(DEMO_UI_TOKEN)" \ + DEMO_WORKER_TOKEN="$(DEMO_WORKER_TOKEN)" \ + DEMO_DIR="$(DEMO_DIR)" \ + ./scripts/demo-ui.sh stop + +demo-logs: + @DEMO_PROJECT="$(DEMO_PROJECT)" \ + DEMO_POSTGRES_PORT="$(DEMO_POSTGRES_PORT)" \ + DEMO_COORDINATOR_PORT="$(DEMO_COORDINATOR_PORT)" \ + DEMO_UI_TOKEN="$(DEMO_UI_TOKEN)" \ + DEMO_WORKER_TOKEN="$(DEMO_WORKER_TOKEN)" \ + DEMO_DIR="$(DEMO_DIR)" \ + ./scripts/demo-ui.sh logs + # --- build / run --------------------------------------------------------- build: go build ./... diff --git a/coordinator/README.md b/coordinator/README.md index c9d90f2..eb0e2ed 100644 --- a/coordinator/README.md +++ b/coordinator/README.md @@ -91,6 +91,24 @@ to inspect a bounded first page of a partial or completed final result before downloading it. The UI never exposes source datasets or shard inputs; partial CSVs remain available only as diagnostics. +### One-command manual demo + +From the repository root, create the Python environment once, then start a +self-contained UI demo with two local reference workers: + +```sh +python3 -m venv .venv +.venv/bin/pip install -e '.[dev]' +make demo-ui +``` + +This uses a separate Docker project and ports `18080` (coordinator) and +`55432` (PostgreSQL), so it does not conflict with the normal stack. Open +`http://localhost:18080/ui`, use username `operator` and password +`demo-ui-secret`, upload a small ChEMBL TSV, and observe the workers process +it. Change the worker count with `make demo-ui DEMO_WORKERS=3`; stop all demo +services and workers with `make demo-down`. + `up` starts three services in order: Postgres waits until `pg_isready` passes, a one-shot `migrate` container applies the schema and exits, and only then does the coordinator start — so it never queries a database that has no tables. diff --git a/coordinator/scripts/demo-ui.sh b/coordinator/scripts/demo-ui.sh new file mode 100755 index 0000000..f978ed1 --- /dev/null +++ b/coordinator/scripts/demo-ui.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Start a self-contained local UI demo with coordinator, PostgreSQL, and local +# reference workers. It is intentionally for a developer's machine only. +set -euo pipefail + +action=${1:-start} +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +coordinator_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +repo_dir=$(CDPATH= cd -- "$coordinator_dir/.." && pwd) + +project=${DEMO_PROJECT:-scimesh-demo} +postgres_port=${DEMO_POSTGRES_PORT:-55432} +coordinator_port=${DEMO_COORDINATOR_PORT:-18080} +ui_token=${DEMO_UI_TOKEN:-demo-ui-secret} +worker_token=${DEMO_WORKER_TOKEN:-demo-worker-token} +workers=${DEMO_WORKERS:-2} +demo_dir=${DEMO_DIR:-.demo} +case "$demo_dir" in + /*) ;; + *) demo_dir="$coordinator_dir/$demo_dir" ;; +esac +worker_bin=${SCIMESH_WORKER_BIN:-"$repo_dir/.venv/bin/scimesh-worker"} +pid_file="$demo_dir/workers.pids" +logs_dir="$demo_dir/logs" + +compose() { + POSTGRES_PORT="$postgres_port" \ + COORDINATOR_PORT="$coordinator_port" \ + UI_AUTH_TOKEN="$ui_token" \ + WORKER_AUTH_TOKEN="$worker_token" \ + docker compose -p "$project" -f "$coordinator_dir/docker-compose.yml" "$@" +} + +stop_workers() { + [[ -f "$pid_file" ]] || return 0 + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + command_line=$(ps -p "$pid" -o args= 2>/dev/null || true) + # Never kill a recycled PID or a worker launched outside this demo. + if [[ "$command_line" == *"$demo_dir/worker-"* ]]; then + kill "$pid" 2>/dev/null || true + fi + done < "$pid_file" + rm -f "$pid_file" +} + +wait_for_coordinator() { + local attempt=0 + until curl --fail --silent --show-error "http://localhost:$coordinator_port/health" >/dev/null; do + attempt=$((attempt + 1)) + if (( attempt >= 45 )); then + echo "Coordinator did not become ready. Recent logs:" >&2 + compose logs --tail=80 coordinator >&2 || true + exit 1 + fi + sleep 1 + done +} + +wait_for_workers() { + local attempt=0 registered overview + until false; do + overview=$(curl --fail --silent --show-error --user "operator:$ui_token" \ + "http://localhost:$coordinator_port/ui/api/overview" 2>/dev/null || true) + # The overview contains no jobs at demo startup, so every `id` belongs to + # a registered worker. Avoid adding jq just for this local helper. + registered=$(printf '%s' "$overview" | grep -o '"id"' | wc -l | tr -d ' ' || true) + if [[ "$registered" =~ ^[0-9]+$ ]] && (( registered >= workers )); then + return 0 + fi + attempt=$((attempt + 1)) + if (( attempt >= 20 )); then + echo "Only $registered of $workers demo workers registered. Recent worker logs:" >&2 + tail -n 40 "$logs_dir"/worker-*.log 2>/dev/null >&2 || true + exit 1 + fi + sleep 1 + done +} + +start() { + if ! [[ "$workers" =~ ^[1-9][0-9]*$ ]]; then + echo "DEMO_WORKERS must be a positive integer (got $workers)." >&2 + exit 2 + fi + if [[ ! -x "$worker_bin" ]]; then + echo "Reference worker not found: $worker_bin" >&2 + echo "Create it first from the repository root: python3 -m venv .venv && .venv/bin/pip install -e '.[dev]'" >&2 + exit 2 + fi + command -v docker >/dev/null || { echo "Docker is required." >&2; exit 2; } + command -v curl >/dev/null || { echo "curl is required." >&2; exit 2; } + + stop_workers + mkdir -p "$logs_dir" + compose up -d --build + echo "Waiting for the coordinator on http://localhost:$coordinator_port ..." + wait_for_coordinator + + : > "$pid_file" + for index in $(seq 1 "$workers"); do + work_dir="$demo_dir/worker-$index" + mkdir -p "$work_dir" + SCIMESH_COORDINATOR_URL="http://localhost:$coordinator_port" \ + SCIMESH_BEARER_TOKEN="$worker_token" \ + "$worker_bin" \ + --worker-name "demo-worker-$index" \ + --work-dir "$work_dir" \ + >"$logs_dir/worker-$index.log" 2>&1 & + echo "$!" >> "$pid_file" + done + wait_for_workers + + cat <&2 + exit 2 + ;; +esac