From 320f52615ea1473510a986271144c3c9520e8dea Mon Sep 17 00:00:00 2001 From: Emil Date: Sun, 2 Aug 2026 18:04:00 +0300 Subject: [PATCH] Embed schema migrations and self-provision the schema on startup --- .github/workflows/coordinator.yml | 2 +- PLAN.md | 36 +++++ coordinator/Makefile | 7 +- coordinator/cmd/coordinator/main.go | 9 ++ coordinator/docker-compose.yml | 19 +-- coordinator/internal/infra/config.go | 12 ++ .../storage/postgres/integration_test.go | 46 +++++- .../internal/storage/postgres/migrate.go | 145 ++++++++++++++++++ .../internal/storage/postgres/migrate_test.go | 60 ++++++++ .../postgres}/migrations/0001_init.down.sql | 0 .../postgres}/migrations/0001_init.up.sql | 0 .../migrations/0002_workers.down.sql | 0 .../postgres}/migrations/0002_workers.up.sql | 0 .../migrations/0003_artifacts.down.sql | 0 .../migrations/0003_artifacts.up.sql | 0 .../migrations/0004_result_artifact.down.sql | 0 .../migrations/0004_result_artifact.up.sql | 0 .../migrations/0005_uploaded_input.down.sql | 0 .../migrations/0005_uploaded_input.up.sql | 0 .../0006_task_running_enum.down.sql | 0 .../migrations/0006_task_running_enum.up.sql | 0 .../0007_task_running_lease.down.sql | 0 .../migrations/0007_task_running_lease.up.sql | 0 .../migrations/0008_artifact_attempt.down.sql | 0 .../migrations/0008_artifact_attempt.up.sql | 0 ...009_unique_partial_result_attempt.down.sql | 0 .../0009_unique_partial_result_attempt.up.sql | 0 .../migrations/0010_job_reduction.down.sql | 0 .../migrations/0010_job_reduction.up.sql | 0 .../migrations/0011_job_owner.down.sql | 0 .../migrations/0011_job_owner.up.sql | 0 .../migrations/0012_worker_trust.down.sql | 0 .../migrations/0012_worker_trust.up.sql | 0 .../migrations/0013_task_results.down.sql | 0 .../migrations/0013_task_results.up.sql | 0 mkdocs/index.md | 8 +- 36 files changed, 319 insertions(+), 25 deletions(-) create mode 100644 coordinator/internal/storage/postgres/migrate.go create mode 100644 coordinator/internal/storage/postgres/migrate_test.go rename coordinator/{ => internal/storage/postgres}/migrations/0001_init.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0001_init.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0002_workers.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0002_workers.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0003_artifacts.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0003_artifacts.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0004_result_artifact.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0004_result_artifact.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0005_uploaded_input.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0005_uploaded_input.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0006_task_running_enum.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0006_task_running_enum.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0007_task_running_lease.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0007_task_running_lease.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0008_artifact_attempt.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0008_artifact_attempt.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0009_unique_partial_result_attempt.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0009_unique_partial_result_attempt.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0010_job_reduction.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0010_job_reduction.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0011_job_owner.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0011_job_owner.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0012_worker_trust.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0012_worker_trust.up.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0013_task_results.down.sql (100%) rename coordinator/{ => internal/storage/postgres}/migrations/0013_task_results.up.sql (100%) diff --git a/.github/workflows/coordinator.yml b/.github/workflows/coordinator.yml index 11a5607..a7059c9 100644 --- a/.github/workflows/coordinator.yml +++ b/.github/workflows/coordinator.yml @@ -60,7 +60,7 @@ jobs: run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.17.1 - name: apply migrations - run: migrate -path migrations -database "$TEST_DATABASE_URL" up + run: migrate -path coordinator/internal/storage/postgres/migrations -database "$TEST_DATABASE_URL" up - name: integration tests run: go test -tags=integration ./internal/storage/postgres/ -v diff --git a/PLAN.md b/PLAN.md index 79bd59e..630511c 100644 --- a/PLAN.md +++ b/PLAN.md @@ -996,6 +996,35 @@ features. --- +### CTX-17 — Self-provisioning coordinator and setup wizard + +**Goal:** A downloaded coordinator binary should bring up a working platform +with as little manual configuration as possible: it provisions its own +schema, and a `setup` command walks the operator through the remaining +environment (database creation, secrets, admin account). + +**Depends on:** the Go coordinator and the release build pipeline. Step 1 +(embedded migrations, `AUTO_MIGRATE`) is implemented; steps 2–3 below are +the remaining work. + +**Acceptance criteria:** + +- the binary embeds the migrations and applies pending ones on startup by + default (`AUTO_MIGRATE=false` opts out for managed databases); applying is + idempotent and safe under concurrent starts; +- `coordinator setup` (interactive, then non-interactive with flags) checks + database reachability, offers to create the role/database when credentials + allow it, applies migrations, writes a `.env` with a generated `JWT_SECRET` + and storage path, and verifies readiness with `/health`; +- `coordinator --version` and the setup output agree on the release build; +- the wizard explains what it cannot do itself: running PostgreSQL and the + userservice, with concrete commands (docker compose, systemd) to finish; +- the Docker image keeps working without the separate migrate step, and the + release workflow publishes the binaries that support `setup`; +- setup fails closed on non-interactive input and never logs secrets. + +--- + ## 10. Suggested assignment bundles These bundles minimize overlap. Do not run tasks from the same bundle in @@ -1135,6 +1164,13 @@ Do not start these before CTX-12 is accepted. - Add shard caching and content-addressed input deduplication. - Add job priority and fair scheduling. - Add a CLI for submitting and monitoring remote jobs. +- Implement CTX-17 steps 2–3: the interactive `coordinator setup` wizard + (database creation, `.env` generation, admin seeding) and, later, a fully + embedded userservice (`coordinator userservice` subcommand) so one binary + can serve the whole platform without containers. +- Replace PostgreSQL with an embedded SQLite backend for fully self-contained + single-binary deployments (large storage-layer change; postgres row locks, + transactions, and integration tests must be re-derived). --- diff --git a/coordinator/Makefile b/coordinator/Makefile index 6a92c10..a652979 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -48,7 +48,8 @@ coordinator: @printf '%s\n' \ 'Built bin/coordinator. Configure via environment:' \ ' DATABASE_URL, COORDINATOR_ADDR, COORDINATOR_TOKEN, UI_AUTH_TOKEN,' \ - ' COORDINATOR_STORAGE_DIR, SCIMESH_DOCS_DIR, JWT_SECRET, USERSERVICE_URL' + ' COORDINATOR_STORAGE_DIR, SCIMESH_DOCS_DIR, JWT_SECRET, USERSERVICE_URL' \ + ' (embedded schema migrations run on startup; AUTO_MIGRATE=false disables)' workloads-export: cd .. && .venv/bin/scimesh workload export -o coordinator/$(WORKLOADS_JSON) @@ -155,10 +156,10 @@ tidy: # DATABASE_URL must be set, e.g.: # export DATABASE_URL='postgres://scimesh:scimesh@localhost:5432/scimesh?sslmode=disable' migrate-up: - migrate -path migrations -database "$(DATABASE_URL)" up + migrate -path internal/storage/postgres/migrations -database "$(DATABASE_URL)" up migrate-down: - migrate -path migrations -database "$(DATABASE_URL)" down 1 + migrate -path internal/storage/postgres/migrations -database "$(DATABASE_URL)" down 1 # --- docker -------------------------------------------------------------- # `up` starts Postgres, applies migrations, then launches the coordinator. diff --git a/coordinator/cmd/coordinator/main.go b/coordinator/cmd/coordinator/main.go index afc0477..45cc4fb 100644 --- a/coordinator/cmd/coordinator/main.go +++ b/coordinator/cmd/coordinator/main.go @@ -66,6 +66,15 @@ func run() error { } defer pool.Close() + // A downloaded binary provisions its own schema; AUTO_MIGRATE=false keeps + // out-of-band migration workflows (the migrate CLI, CI, managed databases). + if cfg.AutoMigrate { + if err := postgres.Migrate(ctx, cfg.DatabaseURL, log); err != nil { + log.Error("apply migrations", "err", err) + return err + } + } + blobStore, err := blob.NewFSStore(cfg.StorageDir) if err != nil { log.Error("init blob storage", "err", err) diff --git a/coordinator/docker-compose.yml b/coordinator/docker-compose.yml index c3f7709..067cf33 100644 --- a/coordinator/docker-compose.yml +++ b/coordinator/docker-compose.yml @@ -20,20 +20,8 @@ services: retries: 10 start_period: 5s - # One-shot: applies migrations, then exits. Schema changes stay an explicit - # deployment step — the coordinator binary never migrates on startup. - migrate: - image: migrate/migrate:v4.17.1 - depends_on: - postgres: - condition: service_healthy - volumes: - - ./migrations:/migrations:ro - command: - - -path=/migrations - - -database=postgres://${POSTGRES_USER:-scimesh}:${POSTGRES_PASSWORD:-scimesh}@postgres:5432/${POSTGRES_DB:-scimesh}?sslmode=disable - - up - restart: on-failure + # The coordinator applies its embedded schema migrations on startup + # (AUTO_MIGRATE, on by default), so no separate migration step is needed. coordinator: build: @@ -41,9 +29,6 @@ services: depends_on: postgres: condition: service_healthy - # Start only once the schema exists, otherwise the first query fails. - migrate: - condition: service_completed_successfully environment: COORDINATOR_ADDR: ":8080" # Host is the service name: compose resolves it on the project network. diff --git a/coordinator/internal/infra/config.go b/coordinator/internal/infra/config.go index 9cb1755..2fe1efc 100644 --- a/coordinator/internal/infra/config.go +++ b/coordinator/internal/infra/config.go @@ -78,6 +78,10 @@ type Config struct { ReaperInterval time.Duration // A worker silent for longer than this is marked offline by the reaper. WorkerOfflineAfter time.Duration + // Whether the binary applies its embedded schema migrations on startup. + // On by default so a downloaded binary provisions its own database; set + // AUTO_MIGRATE=false when an operator manages migrations out of band. + AutoMigrate bool } // Load reads the environment and fails fast on anything required-but-missing @@ -173,6 +177,14 @@ func LoadConfig() (Config, error) { if cfg.DefaultMaxAttempts < 1 { return Config{}, fmt.Errorf("DEFAULT_MAX_ATTEMPTS must be positive") } + cfg.AutoMigrate = true + if raw := os.Getenv("AUTO_MIGRATE"); raw != "" { + parsed, err := strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("AUTO_MIGRATE must be true or false") + } + cfg.AutoMigrate = parsed + } return cfg, nil } diff --git a/coordinator/internal/storage/postgres/integration_test.go b/coordinator/internal/storage/postgres/integration_test.go index ba974c3..b48304e 100644 --- a/coordinator/internal/storage/postgres/integration_test.go +++ b/coordinator/internal/storage/postgres/integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/emil28092005/SciMesh/coordinator/internal/domain" + "github.com/emil28092005/SciMesh/coordinator/internal/workloads" "github.com/emil28092005/SciMesh/coordinator/internal/usecase" ) @@ -408,7 +409,7 @@ func TestCompleteTaskReplayIsIdempotent(t *testing.T) { tasks, jobs, artifacts, tx := NewTaskRepo(pool), NewJobRepo(pool), NewArtifactRepo(pool), NewTxManager(pool) workers, results := NewWorkerRepo(pool), NewTaskResultRepo(pool) clk := fixedClock{now: time.Now().UTC()} - uc := usecase.NewCompleteTask(tasks, jobs, artifacts, workers, results, tx, clk, 2) + uc := usecase.NewCompleteTask(tasks, jobs, artifacts, workers, results, tx, clk, 2, integrationCatalog()) claimed, err := tasks.ClaimNext(ctx, usecase.ClaimFilter{ Owner: "worker-1", Now: clk.now, LeaseUntil: clk.now.Add(time.Minute), @@ -657,3 +658,46 @@ func TestExpireLeasesRequeuesElapsedTasks(t *testing.T) { t.Errorf("pending = %d, want 1 — a dead worker must not strand its task", counts[domain.TaskPending]) } } + +func TestMigrateProvisionsAndIsIdempotent(t *testing.T) { + ctx := context.Background() + url := os.Getenv("TEST_DATABASE_URL") + if url == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + if err := Migrate(ctx, url, nil); err != nil { + t.Fatalf("first migrate: %v", err) + } + if err := Migrate(ctx, url, nil); err != nil { + t.Fatalf("second migrate (idempotent): %v", err) + } + pool := testPool(t) + var count int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM schema_migrations").Scan(&count); err != nil { + t.Fatalf("read schema_migrations: %v", err) + } + migrations, err := listMigrations() + if err != nil { + t.Fatal(err) + } + if count != len(migrations) { + t.Errorf("schema_migrations has %d rows, want %d", count, len(migrations)) + } + var hasJobs bool + if err := pool.QueryRow(ctx, + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'jobs')", + ).Scan(&hasJobs); err != nil { + t.Fatal(err) + } + if !hasJobs { + t.Error("jobs table was not created by the embedded migrations") + } +} + +func integrationCatalog() *workloads.Catalog { + catalog, err := workloads.Load() + if err != nil { + panic(err) + } + return catalog +} diff --git a/coordinator/internal/storage/postgres/migrate.go b/coordinator/internal/storage/postgres/migrate.go new file mode 100644 index 0000000..896efae --- /dev/null +++ b/coordinator/internal/storage/postgres/migrate.go @@ -0,0 +1,145 @@ +package postgres + +import ( + "context" + "embed" + "fmt" + "log/slog" + "regexp" + "sort" + "strconv" + + "github.com/jackc/pgx/v5" +) + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +var migrationNamePattern = regexp.MustCompile(`^([0-9]+)_[a-z0-9_]+\.(up|down)\.sql$`) + +// migration is one parsed embedded migration file. +type migration struct { + version int + name string + sql string +} + +// listMigrations parses and orders the embedded .up.sql files by version. +func listMigrations() ([]migration, error) { + entries, err := migrationFiles.ReadDir("migrations") + if err != nil { + return nil, fmt.Errorf("read embedded migrations: %w", err) + } + up := map[int]migration{} + for _, entry := range entries { + match := migrationNamePattern.FindStringSubmatch(entry.Name()) + if match == nil { + continue + } + if match[2] != "up" { + continue + } + version, err := strconv.Atoi(match[1]) + if err != nil { + return nil, fmt.Errorf("migration %q has an invalid version: %w", entry.Name(), err) + } + if _, duplicate := up[version]; duplicate { + return nil, fmt.Errorf("migration version %d is duplicated", version) + } + body, err := migrationFiles.ReadFile("migrations/" + entry.Name()) + if err != nil { + return nil, fmt.Errorf("read migration %q: %w", entry.Name(), err) + } + up[version] = migration{version: version, name: entry.Name(), sql: string(body)} + } + if len(up) == 0 { + return nil, fmt.Errorf("no .up.sql migrations are embedded") + } + versions := make([]int, 0, len(up)) + for version := range up { + versions = append(versions, version) + } + sort.Ints(versions) + migrations := make([]migration, 0, len(versions)) + for _, version := range versions { + migrations = append(migrations, up[version]) + } + for index, item := range migrations { + if item.version != index+1 { + return nil, fmt.Errorf("embedded migrations are not contiguous: version %d at position %d", item.version, index+1) + } + } + return migrations, nil +} + +// Migrate applies every embedded migration that is not yet recorded in the +// schema_migrations table, so the binary provisions its own schema. It is +// idempotent and safe to run concurrently: a PostgreSQL advisory lock +// serializes migrators, and each migration file runs as its own transaction +// (the files carry explicit BEGIN/COMMIT, matching the golang-migrate format +// the CLI and CI still use). +func Migrate(ctx context.Context, databaseURL string, log *slog.Logger) error { + migrations, err := listMigrations() + if err != nil { + return err + } + connConfig, err := pgx.ParseConfig(databaseURL) + if err != nil { + return fmt.Errorf("parse database url: %w", err) + } + // Migration files contain multiple statements (BEGIN...COMMIT), which the + // extended query protocol rejects; run them with the simple protocol. + connConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol + conn, err := pgx.ConnectConfig(ctx, connConfig) + if err != nil { + return fmt.Errorf("connect for migration: %w", err) + } + defer func() { _ = conn.Close(ctx) }() + + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock(82473911)"); err != nil { + return fmt.Errorf("acquire migration lock: %w", err) + } + defer func() { _, _ = conn.Exec(ctx, "SELECT pg_advisory_unlock(82473911)") }() + + if _, err := conn.Exec(ctx, + "CREATE TABLE IF NOT EXISTS schema_migrations (version bigint PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())", + ); err != nil { + return fmt.Errorf("ensure schema_migrations: %w", err) + } + + applied := map[int64]bool{} + rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return fmt.Errorf("read applied migrations: %w", err) + } + for rows.Next() { + var version int64 + if err := rows.Scan(&version); err != nil { + rows.Close() + return fmt.Errorf("scan applied migration: %w", err) + } + applied[version] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("read applied migrations: %w", err) + } + + for _, item := range migrations { + if applied[int64(item.version)] { + continue + } + if log != nil { + log.Info("applying migration", "version", item.version, "file", item.name) + } + if _, err := conn.Exec(ctx, item.sql); err != nil { + return fmt.Errorf("apply migration %s: %w", item.name, err) + } + if _, err := conn.Exec(ctx, + "INSERT INTO schema_migrations (version) VALUES ($1)", item.version, + ); err != nil { + return fmt.Errorf("record migration %s: %w", item.name, err) + } + } + return nil +} diff --git a/coordinator/internal/storage/postgres/migrate_test.go b/coordinator/internal/storage/postgres/migrate_test.go new file mode 100644 index 0000000..7b241f2 --- /dev/null +++ b/coordinator/internal/storage/postgres/migrate_test.go @@ -0,0 +1,60 @@ +package postgres + +import ( + "strings" + "testing" +) + +func TestListMigrationsParsesAndOrdersEmbeddedFiles(t *testing.T) { + migrations, err := listMigrations() + if err != nil { + t.Fatalf("list migrations: %v", err) + } + if len(migrations) == 0 { + t.Fatal("no embedded migrations") + } + for index, item := range migrations { + if item.version != index+1 { + t.Errorf("migration %d has version %d, want contiguous ordering", index, item.version) + } + if item.name != expectedMigrationName(item.version) { + t.Errorf("migration %d file is %q, want %q", item.version, item.name, expectedMigrationName(item.version)) + } + if strings.TrimSpace(item.sql) == "" { + t.Errorf("migration %d is empty", item.version) + } + } +} + +func expectedMigrationName(version int) string { + switch version { + case 1: + return "0001_init.up.sql" + case 2: + return "0002_workers.up.sql" + case 3: + return "0003_artifacts.up.sql" + case 4: + return "0004_result_artifact.up.sql" + case 5: + return "0005_uploaded_input.up.sql" + case 6: + return "0006_task_running_enum.up.sql" + case 7: + return "0007_task_running_lease.up.sql" + case 8: + return "0008_artifact_attempt.up.sql" + case 9: + return "0009_unique_partial_result_attempt.up.sql" + case 10: + return "0010_job_reduction.up.sql" + case 11: + return "0011_job_owner.up.sql" + case 12: + return "0012_worker_trust.up.sql" + case 13: + return "0013_task_results.up.sql" + default: + return "" + } +} diff --git a/coordinator/migrations/0001_init.down.sql b/coordinator/internal/storage/postgres/migrations/0001_init.down.sql similarity index 100% rename from coordinator/migrations/0001_init.down.sql rename to coordinator/internal/storage/postgres/migrations/0001_init.down.sql diff --git a/coordinator/migrations/0001_init.up.sql b/coordinator/internal/storage/postgres/migrations/0001_init.up.sql similarity index 100% rename from coordinator/migrations/0001_init.up.sql rename to coordinator/internal/storage/postgres/migrations/0001_init.up.sql diff --git a/coordinator/migrations/0002_workers.down.sql b/coordinator/internal/storage/postgres/migrations/0002_workers.down.sql similarity index 100% rename from coordinator/migrations/0002_workers.down.sql rename to coordinator/internal/storage/postgres/migrations/0002_workers.down.sql diff --git a/coordinator/migrations/0002_workers.up.sql b/coordinator/internal/storage/postgres/migrations/0002_workers.up.sql similarity index 100% rename from coordinator/migrations/0002_workers.up.sql rename to coordinator/internal/storage/postgres/migrations/0002_workers.up.sql diff --git a/coordinator/migrations/0003_artifacts.down.sql b/coordinator/internal/storage/postgres/migrations/0003_artifacts.down.sql similarity index 100% rename from coordinator/migrations/0003_artifacts.down.sql rename to coordinator/internal/storage/postgres/migrations/0003_artifacts.down.sql diff --git a/coordinator/migrations/0003_artifacts.up.sql b/coordinator/internal/storage/postgres/migrations/0003_artifacts.up.sql similarity index 100% rename from coordinator/migrations/0003_artifacts.up.sql rename to coordinator/internal/storage/postgres/migrations/0003_artifacts.up.sql diff --git a/coordinator/migrations/0004_result_artifact.down.sql b/coordinator/internal/storage/postgres/migrations/0004_result_artifact.down.sql similarity index 100% rename from coordinator/migrations/0004_result_artifact.down.sql rename to coordinator/internal/storage/postgres/migrations/0004_result_artifact.down.sql diff --git a/coordinator/migrations/0004_result_artifact.up.sql b/coordinator/internal/storage/postgres/migrations/0004_result_artifact.up.sql similarity index 100% rename from coordinator/migrations/0004_result_artifact.up.sql rename to coordinator/internal/storage/postgres/migrations/0004_result_artifact.up.sql diff --git a/coordinator/migrations/0005_uploaded_input.down.sql b/coordinator/internal/storage/postgres/migrations/0005_uploaded_input.down.sql similarity index 100% rename from coordinator/migrations/0005_uploaded_input.down.sql rename to coordinator/internal/storage/postgres/migrations/0005_uploaded_input.down.sql diff --git a/coordinator/migrations/0005_uploaded_input.up.sql b/coordinator/internal/storage/postgres/migrations/0005_uploaded_input.up.sql similarity index 100% rename from coordinator/migrations/0005_uploaded_input.up.sql rename to coordinator/internal/storage/postgres/migrations/0005_uploaded_input.up.sql diff --git a/coordinator/migrations/0006_task_running_enum.down.sql b/coordinator/internal/storage/postgres/migrations/0006_task_running_enum.down.sql similarity index 100% rename from coordinator/migrations/0006_task_running_enum.down.sql rename to coordinator/internal/storage/postgres/migrations/0006_task_running_enum.down.sql diff --git a/coordinator/migrations/0006_task_running_enum.up.sql b/coordinator/internal/storage/postgres/migrations/0006_task_running_enum.up.sql similarity index 100% rename from coordinator/migrations/0006_task_running_enum.up.sql rename to coordinator/internal/storage/postgres/migrations/0006_task_running_enum.up.sql diff --git a/coordinator/migrations/0007_task_running_lease.down.sql b/coordinator/internal/storage/postgres/migrations/0007_task_running_lease.down.sql similarity index 100% rename from coordinator/migrations/0007_task_running_lease.down.sql rename to coordinator/internal/storage/postgres/migrations/0007_task_running_lease.down.sql diff --git a/coordinator/migrations/0007_task_running_lease.up.sql b/coordinator/internal/storage/postgres/migrations/0007_task_running_lease.up.sql similarity index 100% rename from coordinator/migrations/0007_task_running_lease.up.sql rename to coordinator/internal/storage/postgres/migrations/0007_task_running_lease.up.sql diff --git a/coordinator/migrations/0008_artifact_attempt.down.sql b/coordinator/internal/storage/postgres/migrations/0008_artifact_attempt.down.sql similarity index 100% rename from coordinator/migrations/0008_artifact_attempt.down.sql rename to coordinator/internal/storage/postgres/migrations/0008_artifact_attempt.down.sql diff --git a/coordinator/migrations/0008_artifact_attempt.up.sql b/coordinator/internal/storage/postgres/migrations/0008_artifact_attempt.up.sql similarity index 100% rename from coordinator/migrations/0008_artifact_attempt.up.sql rename to coordinator/internal/storage/postgres/migrations/0008_artifact_attempt.up.sql diff --git a/coordinator/migrations/0009_unique_partial_result_attempt.down.sql b/coordinator/internal/storage/postgres/migrations/0009_unique_partial_result_attempt.down.sql similarity index 100% rename from coordinator/migrations/0009_unique_partial_result_attempt.down.sql rename to coordinator/internal/storage/postgres/migrations/0009_unique_partial_result_attempt.down.sql diff --git a/coordinator/migrations/0009_unique_partial_result_attempt.up.sql b/coordinator/internal/storage/postgres/migrations/0009_unique_partial_result_attempt.up.sql similarity index 100% rename from coordinator/migrations/0009_unique_partial_result_attempt.up.sql rename to coordinator/internal/storage/postgres/migrations/0009_unique_partial_result_attempt.up.sql diff --git a/coordinator/migrations/0010_job_reduction.down.sql b/coordinator/internal/storage/postgres/migrations/0010_job_reduction.down.sql similarity index 100% rename from coordinator/migrations/0010_job_reduction.down.sql rename to coordinator/internal/storage/postgres/migrations/0010_job_reduction.down.sql diff --git a/coordinator/migrations/0010_job_reduction.up.sql b/coordinator/internal/storage/postgres/migrations/0010_job_reduction.up.sql similarity index 100% rename from coordinator/migrations/0010_job_reduction.up.sql rename to coordinator/internal/storage/postgres/migrations/0010_job_reduction.up.sql diff --git a/coordinator/migrations/0011_job_owner.down.sql b/coordinator/internal/storage/postgres/migrations/0011_job_owner.down.sql similarity index 100% rename from coordinator/migrations/0011_job_owner.down.sql rename to coordinator/internal/storage/postgres/migrations/0011_job_owner.down.sql diff --git a/coordinator/migrations/0011_job_owner.up.sql b/coordinator/internal/storage/postgres/migrations/0011_job_owner.up.sql similarity index 100% rename from coordinator/migrations/0011_job_owner.up.sql rename to coordinator/internal/storage/postgres/migrations/0011_job_owner.up.sql diff --git a/coordinator/migrations/0012_worker_trust.down.sql b/coordinator/internal/storage/postgres/migrations/0012_worker_trust.down.sql similarity index 100% rename from coordinator/migrations/0012_worker_trust.down.sql rename to coordinator/internal/storage/postgres/migrations/0012_worker_trust.down.sql diff --git a/coordinator/migrations/0012_worker_trust.up.sql b/coordinator/internal/storage/postgres/migrations/0012_worker_trust.up.sql similarity index 100% rename from coordinator/migrations/0012_worker_trust.up.sql rename to coordinator/internal/storage/postgres/migrations/0012_worker_trust.up.sql diff --git a/coordinator/migrations/0013_task_results.down.sql b/coordinator/internal/storage/postgres/migrations/0013_task_results.down.sql similarity index 100% rename from coordinator/migrations/0013_task_results.down.sql rename to coordinator/internal/storage/postgres/migrations/0013_task_results.down.sql diff --git a/coordinator/migrations/0013_task_results.up.sql b/coordinator/internal/storage/postgres/migrations/0013_task_results.up.sql similarity index 100% rename from coordinator/migrations/0013_task_results.up.sql rename to coordinator/internal/storage/postgres/migrations/0013_task_results.up.sql diff --git a/mkdocs/index.md b/mkdocs/index.md index 21f3c96..bca66b6 100644 --- a/mkdocs/index.md +++ b/mkdocs/index.md @@ -83,9 +83,11 @@ chmod +x coordinator `python -m scimesh.worker.task`, so the machine needs the `scimesh` package in a venv (`pip install scimesh`) and the usual environment: `COORDINATOR_URL`, `WORKER_AUTH_TOKEN`, `WORK_DIR`. -- **coordinator** needs PostgreSQL with the applied migrations (the binary - never migrates on startup) plus `DATABASE_URL`, `COORDINATOR_STORAGE_DIR`, - and `JWT_SECRET`; the UI login additionally requires `USERSERVICE_URL`. +- **coordinator** needs PostgreSQL running (`DATABASE_URL`, + `COORDINATOR_STORAGE_DIR`, `JWT_SECRET`); the binary applies its embedded + schema migrations itself on startup (`AUTO_MIGRATE=false` opts out), so no + separate migration step is needed. The UI login additionally requires + `USERSERVICE_URL`. `coordinator --version` / `worker-agent --version` print the build tag. Build and serve this documentation site: