Add local pipeline demo launcher

.gitignore

@@ -15,3 +15,4 @@ test_structures/
 # Local coordinator-worker execution state
 worker-data*/
 scimesh-worker-data/
+coordinator/.demo/

Makefile

@@ -0,0 +1,12 @@
+.PHONY: demo-ui demo-down demo-logs
+
+# Convenient entry points from the repository root. Extra settings are passed
+# through, for example: make demo-ui DEMO_WORKERS=3
+demo-ui:
+	$(MAKE) -C coordinator demo-ui
+
+demo-down:
+	$(MAKE) -C coordinator demo-down
+
+demo-logs:
+	$(MAKE) -C coordinator demo-logs

coordinator/Makefile

@@ -1,4 +1,4 @@
-.PHONY: build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke
+.PHONY: build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke demo-ui demo-down demo-logs
 
 # `check` deliberately uses its own Compose project and host ports. This keeps
 # it from connecting to or replacing a developer's local PostgreSQL instance.
@@ -10,6 +10,45 @@ CHECK_TOKEN ?= dev-token
 CHECK_DATABASE_URL ?= postgres://scimesh:scimesh@localhost:$(CHECK_POSTGRES_PORT)/scimesh?sslmode=disable
 CHECK_COMPOSE = POSTGRES_PORT=$(CHECK_POSTGRES_PORT) COORDINATOR_PORT=$(CHECK_COORDINATOR_PORT) docker compose -p $(CHECK_PROJECT)
 
+# --- local manual demo ---------------------------------------------------
+# A separate project and ports mean this demo cannot collide with the normal
+# `make up` stack or a developer's local PostgreSQL on 5432.
+DEMO_PROJECT ?= scimesh-demo
+DEMO_POSTGRES_PORT ?= 55432
+DEMO_COORDINATOR_PORT ?= 18080
+DEMO_UI_TOKEN ?= demo-ui-secret
+DEMO_WORKER_TOKEN ?= demo-worker-token
+DEMO_WORKERS ?= 2
+DEMO_DIR ?= .demo
+
+demo-ui:
+	@DEMO_PROJECT="$(DEMO_PROJECT)" \
+	DEMO_POSTGRES_PORT="$(DEMO_POSTGRES_PORT)" \
+	DEMO_COORDINATOR_PORT="$(DEMO_COORDINATOR_PORT)" \
+	DEMO_UI_TOKEN="$(DEMO_UI_TOKEN)" \
+	DEMO_WORKER_TOKEN="$(DEMO_WORKER_TOKEN)" \
+	DEMO_WORKERS="$(DEMO_WORKERS)" \
+	DEMO_DIR="$(DEMO_DIR)" \
+	./scripts/demo-ui.sh start
+
+demo-down:
+	@DEMO_PROJECT="$(DEMO_PROJECT)" \
+	DEMO_POSTGRES_PORT="$(DEMO_POSTGRES_PORT)" \
+	DEMO_COORDINATOR_PORT="$(DEMO_COORDINATOR_PORT)" \
+	DEMO_UI_TOKEN="$(DEMO_UI_TOKEN)" \
+	DEMO_WORKER_TOKEN="$(DEMO_WORKER_TOKEN)" \
+	DEMO_DIR="$(DEMO_DIR)" \
+	./scripts/demo-ui.sh stop
+
+demo-logs:
+	@DEMO_PROJECT="$(DEMO_PROJECT)" \
+	DEMO_POSTGRES_PORT="$(DEMO_POSTGRES_PORT)" \
+	DEMO_COORDINATOR_PORT="$(DEMO_COORDINATOR_PORT)" \
+	DEMO_UI_TOKEN="$(DEMO_UI_TOKEN)" \
+	DEMO_WORKER_TOKEN="$(DEMO_WORKER_TOKEN)" \
+	DEMO_DIR="$(DEMO_DIR)" \
+	./scripts/demo-ui.sh logs
+
 # --- build / run ---------------------------------------------------------
 build:
 	go build ./...

coordinator/README.md

@@ -91,6 +91,24 @@ to inspect a bounded first page of a partial or completed final result before
 downloading it. The UI never exposes source datasets or shard inputs; partial
 CSVs remain available only as diagnostics.
 
+### One-command manual demo
+
+From the repository root, create the Python environment once, then start a
+self-contained UI demo with two local reference workers:
+
+```sh
+python3 -m venv .venv
+.venv/bin/pip install -e '.[dev]'
+make demo-ui
+```
+
+This uses a separate Docker project and ports `18080` (coordinator) and
+`55432` (PostgreSQL), so it does not conflict with the normal stack. Open
+`http://localhost:18080/ui`, use username `operator` and password
+`demo-ui-secret`, upload a small ChEMBL TSV, and observe the workers process
+it. Change the worker count with `make demo-ui DEMO_WORKERS=3`; stop all demo
+services and workers with `make demo-down`.
+
 `up` starts three services in order: Postgres waits until `pg_isready` passes, a
 one-shot `migrate` container applies the schema and exits, and only then does the
 coordinator start — so it never queries a database that has no tables.

coordinator/scripts/demo-ui.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+# Start a self-contained local UI demo with coordinator, PostgreSQL, and local
+# reference workers. It is intentionally for a developer's machine only.
+set -euo pipefail
+
+action=${1:-start}
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+coordinator_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
+repo_dir=$(CDPATH= cd -- "$coordinator_dir/.." && pwd)
+
+project=${DEMO_PROJECT:-scimesh-demo}
+postgres_port=${DEMO_POSTGRES_PORT:-55432}
+coordinator_port=${DEMO_COORDINATOR_PORT:-18080}
+ui_token=${DEMO_UI_TOKEN:-demo-ui-secret}
+worker_token=${DEMO_WORKER_TOKEN:-demo-worker-token}
+workers=${DEMO_WORKERS:-2}
+demo_dir=${DEMO_DIR:-.demo}
+case "$demo_dir" in
+  /*) ;;
+  *) demo_dir="$coordinator_dir/$demo_dir" ;;
+esac
+worker_bin=${SCIMESH_WORKER_BIN:-"$repo_dir/.venv/bin/scimesh-worker"}
+pid_file="$demo_dir/workers.pids"
+logs_dir="$demo_dir/logs"
+
+compose() {
+  POSTGRES_PORT="$postgres_port" \
+  COORDINATOR_PORT="$coordinator_port" \
+  UI_AUTH_TOKEN="$ui_token" \
+  WORKER_AUTH_TOKEN="$worker_token" \
+  docker compose -p "$project" -f "$coordinator_dir/docker-compose.yml" "$@"
+}
+
+stop_workers() {
+  [[ -f "$pid_file" ]] || return 0
+  while IFS= read -r pid; do
+    [[ "$pid" =~ ^[0-9]+$ ]] || continue
+    command_line=$(ps -p "$pid" -o args= 2>/dev/null || true)
+    # Never kill a recycled PID or a worker launched outside this demo.
+    if [[ "$command_line" == *"$demo_dir/worker-"* ]]; then
+      kill "$pid" 2>/dev/null || true
+    fi
+  done < "$pid_file"
+  rm -f "$pid_file"
+}
+
+wait_for_coordinator() {
+  local attempt=0
+  until curl --fail --silent --show-error "http://localhost:$coordinator_port/health" >/dev/null; do
+    attempt=$((attempt + 1))
+    if (( attempt >= 45 )); then
+      echo "Coordinator did not become ready. Recent logs:" >&2
+      compose logs --tail=80 coordinator >&2 || true
+      exit 1
+    fi
+    sleep 1
+  done
+}
+
+wait_for_workers() {
+  local attempt=0 registered overview
+  until false; do
+    overview=$(curl --fail --silent --show-error --user "operator:$ui_token" \
+      "http://localhost:$coordinator_port/ui/api/overview" 2>/dev/null || true)
+    # The overview contains no jobs at demo startup, so every `id` belongs to
+    # a registered worker. Avoid adding jq just for this local helper.
+    registered=$(printf '%s' "$overview" | grep -o '"id"' | wc -l | tr -d ' ' || true)
+    if [[ "$registered" =~ ^[0-9]+$ ]] && (( registered >= workers )); then
+      return 0
+    fi
+    attempt=$((attempt + 1))
+    if (( attempt >= 20 )); then
+      echo "Only $registered of $workers demo workers registered. Recent worker logs:" >&2
+      tail -n 40 "$logs_dir"/worker-*.log 2>/dev/null >&2 || true
+      exit 1
+    fi
+    sleep 1
+  done
+}
+
+start() {
+  if ! [[ "$workers" =~ ^[1-9][0-9]*$ ]]; then
+    echo "DEMO_WORKERS must be a positive integer (got $workers)." >&2
+    exit 2
+  fi
+  if [[ ! -x "$worker_bin" ]]; then
+    echo "Reference worker not found: $worker_bin" >&2
+    echo "Create it first from the repository root: python3 -m venv .venv && .venv/bin/pip install -e '.[dev]'" >&2
+    exit 2
+  fi
+  command -v docker >/dev/null || { echo "Docker is required." >&2; exit 2; }
+  command -v curl >/dev/null || { echo "curl is required." >&2; exit 2; }
+
+  stop_workers
+  mkdir -p "$logs_dir"
+  compose up -d --build
+  echo "Waiting for the coordinator on http://localhost:$coordinator_port ..."
+  wait_for_coordinator
+
+  : > "$pid_file"
+  for index in $(seq 1 "$workers"); do
+    work_dir="$demo_dir/worker-$index"
+    mkdir -p "$work_dir"
+    SCIMESH_COORDINATOR_URL="http://localhost:$coordinator_port" \
+    SCIMESH_BEARER_TOKEN="$worker_token" \
+    "$worker_bin" \
+      --worker-name "demo-worker-$index" \
+      --work-dir "$work_dir" \
+      >"$logs_dir/worker-$index.log" 2>&1 &
+    echo "$!" >> "$pid_file"
+  done
+  wait_for_workers
+
+  cat <<EOF
+
+SciMesh manual demo is ready.
+
+  UI:       http://localhost:$coordinator_port/ui
+  Username: operator
+  Password: $ui_token
+  Workers:  $workers local reference workers
+
+Upload a small ChEMBL TSV through “New similarity search”, then watch the job
+page update. Worker logs are in $logs_dir. Stop everything with:
+
+  make demo-down
+EOF
+}
+
+case "$action" in
+  start) start ;;
+  stop)
+    stop_workers
+    compose down
+    echo "SciMesh manual demo stopped."
+    ;;
+  logs)
+    echo "Worker logs: $logs_dir"
+    compose logs -f coordinator
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|logs}" >&2
+    exit 2
+    ;;
+esac