feat(coordinator): scaffold task-queue service in Go

Adds the SciMesh coordinator: a durable task-queue server on PostgreSQL that owns all database access, with workers reaching it over HTTP only. Structured as a modular monolith following Clean Architecture: domain entities and their invariants, no I/O usecase business operations + repository/clock ports transport HTTP handlers, DTOs, auth, error mapping storage PostgreSQL repositories, transactions carried in context infra config, pool, clock, server, lease reaper Dependencies point strictly inward; domain imports nothing from the module. Working: layer wiring, routing, shared-token auth, access logging, request IDs, domain-error to status-code mapping, transactional boundaries, graceful shutdown (HTTP drain -> reaper stop -> pool close), migrations, and a Compose stack starting Postgres -> migrations -> coordinator. The domain is complete and covered by unit tests that need no database: lease ownership, stale attempts, idempotent result replay, retry budgets, and lease expiry. Repository methods are stubs returning ErrNotImplemented (HTTP 501). The SQL for atomic claiming (FOR UPDATE SKIP LOCKED) and for lease expiry is written and ready to wire up. See coordinator/ARCHITECTURE.md for the layer map and a request traced through every layer.
2026-07-22 13:49:01 +03:00
parent ccf11403bf
commit bda22666d7
37 changed files with 3005 additions and 0 deletions
@@ -0,0 +1,40 @@
+package postgres
+
+import (
+	"context"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"github.com/emil28092005/SciMesh/coordinator/internal/domain"
+	"github.com/emil28092005/SciMesh/coordinator/internal/usecase"
+)
+
+// JobRepo implements usecase.JobRepository.
+type JobRepo struct {
+	pool *pgxpool.Pool
+}
+
+func NewJobRepo(pool *pgxpool.Pool) *JobRepo {
+	return &JobRepo{pool: pool}
+}
+
+var _ usecase.JobRepository = (*JobRepo)(nil)
+
+// TODO(phase 2-3): replace stubs with real pgx queries.
+
+func (r *JobRepo) Insert(ctx context.Context, j *domain.Job) error {
+	// Phase 2: INSERT INTO jobs ...; runs inside the caller's transaction.
+	return usecase.ErrNotImplemented
+}
+
+func (r *JobRepo) Get(ctx context.Context, id uuid.UUID) (*domain.Job, error) {
+	// Phase 3: SELECT ... WHERE id = $1; no rows -> domain.ErrJobNotFound.
+	return nil, usecase.ErrNotImplemented
+}
+
+func (r *JobRepo) UpdateStatus(ctx context.Context, id uuid.UUID, status domain.JobStatus, completedAt *time.Time) error {
+	// Phase 3: UPDATE jobs SET status = $2, completed_at = $3 WHERE id = $1.
+	return usecase.ErrNotImplemented
+}
@@ -0,0 +1,82 @@
+package postgres
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+// Transient PostgreSQL failures. Under concurrent claiming these are expected
+// rather than exceptional: two coordinators touching neighbouring rows can
+// deadlock or fail to serialize, and the correct response is to try again.
+const (
+	codeSerializationFailure = "40001"
+	codeDeadlockDetected     = "40P01"
+	codeTooManyConnections   = "53300"
+	codeCannotConnectNow     = "57P03"
+)
+
+// Retry budget: short and bounded. A worker polling for tasks would rather get
+// a fast error and poll again than have its request hang for half a minute.
+const (
+	retryInitialInterval = 50 * time.Millisecond
+	retryMaxInterval     = 1 * time.Second
+	retryMaxElapsedTime  = 5 * time.Second
+)
+
+// isTransient reports whether err is worth retrying.
+//
+// The default is *not* to retry: a constraint violation or a syntax error will
+// fail identically every time, and retrying it only multiplies the damage.
+func isTransient(err error) bool {
+	if err == nil {
+		return false
+	}
+	// A cancelled caller does not want another attempt.
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return false
+	}
+
+	var pgErr *pgconn.PgError
+	if errors.As(err, &pgErr) {
+		switch pgErr.Code {
+		case codeSerializationFailure, codeDeadlockDetected,
+			codeTooManyConnections, codeCannotConnectNow:
+			return true
+		default:
+			return false
+		}
+	}
+
+	// Connection-level trouble (dropped socket, closed pool). pgconn knows
+	// whether the query could have been executed before the failure — retrying
+	// a maybe-executed write would risk duplicating it.
+	return pgconn.SafeToRetry(err)
+}
+
+// withRetry runs op, retrying only transient database failures with
+// exponential backoff and jitter, and giving up as soon as ctx is done.
+//
+// Jitter matters here: without it, several coordinators that collide once will
+// retry in lockstep and collide again at exactly the same moment.
+func withRetry(ctx context.Context, op func(context.Context) error) error {
+	b := backoff.NewExponentialBackOff()
+	b.InitialInterval = retryInitialInterval
+	b.MaxInterval = retryMaxInterval
+	b.MaxElapsedTime = retryMaxElapsedTime
+	// RandomizationFactor defaults to 0.5, which is the jitter.
+
+	return backoff.Retry(func() error {
+		err := op(ctx)
+		if err == nil {
+			return nil
+		}
+		if !isTransient(err) {
+			return backoff.Permanent(err) // stop now, do not burn the budget
+		}
+		return err
+	}, backoff.WithContext(b, ctx))
+}
@@ -0,0 +1,98 @@
+package postgres
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+func TestIsTransient(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"serialization failure", &pgconn.PgError{Code: codeSerializationFailure}, true},
+		{"deadlock", &pgconn.PgError{Code: codeDeadlockDetected}, true},
+		{"too many connections", &pgconn.PgError{Code: codeTooManyConnections}, true},
+		// A unique-violation repeats identically forever — retrying is pointless.
+		{"unique violation", &pgconn.PgError{Code: "23505"}, false},
+		{"syntax error", &pgconn.PgError{Code: "42601"}, false},
+		{"context cancelled", context.Canceled, false},
+		{"deadline exceeded", context.DeadlineExceeded, false},
+		{"unknown error", errors.New("boom"), false},
+		// Wrapping must not hide the cause: errors.As walks the chain.
+		{"wrapped deadlock", errors2Wrap(&pgconn.PgError{Code: codeDeadlockDetected}), true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isTransient(tt.err); got != tt.want {
+				t.Errorf("isTransient(%v) = %v, want %v", tt.err, got, tt.want)
+			}
+		})
+	}
+}
+
+func errors2Wrap(err error) error {
+	return errors.Join(errors.New("query failed"), err)
+}
+
+func TestWithRetrySucceedsAfterTransientFailures(t *testing.T) {
+	calls := 0
+	err := withRetry(context.Background(), func(context.Context) error {
+		calls++
+		if calls < 3 {
+			return &pgconn.PgError{Code: codeSerializationFailure}
+		}
+		return nil
+	})
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if calls != 3 {
+		t.Errorf("calls = %d, want 3", calls)
+	}
+}
+
+func TestWithRetryStopsOnPermanentError(t *testing.T) {
+	permanent := &pgconn.PgError{Code: "23505"} // unique violation
+	calls := 0
+
+	err := withRetry(context.Background(), func(context.Context) error {
+		calls++
+		return permanent
+	})
+
+	if !errors.Is(err, permanent) {
+		t.Errorf("err = %v, want the original error", err)
+	}
+	if calls != 1 {
+		t.Errorf("calls = %d, want 1 — a permanent error must not be retried", calls)
+	}
+}
+
+func TestWithRetryHonoursContextCancellation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	calls := 0
+	start := time.Now()
+	err := withRetry(ctx, func(context.Context) error {
+		calls++
+		return &pgconn.PgError{Code: codeDeadlockDetected}
+	})
+
+	if err == nil {
+		t.Fatal("expected an error once the context expired")
+	}
+	// Must abort at the deadline, not run the full 5s retry budget.
+	if elapsed := time.Since(start); elapsed > time.Second {
+		t.Errorf("took %v, expected to stop at the context deadline", elapsed)
+	}
+}
@@ -0,0 +1,114 @@
+package postgres
+
+import (
+	"context"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"github.com/emil28092005/SciMesh/coordinator/internal/domain"
+	"github.com/emil28092005/SciMesh/coordinator/internal/usecase"
+)
+
+// TaskRepo implements usecase.TaskRepository.
+type TaskRepo struct {
+	pool *pgxpool.Pool
+}
+
+func NewTaskRepo(pool *pgxpool.Pool) *TaskRepo {
+	return &TaskRepo{pool: pool}
+}
+
+var _ usecase.TaskRepository = (*TaskRepo)(nil)
+
+// claimNextSQL leases one task in a single statement.
+//
+// FOR UPDATE SKIP LOCKED is what makes concurrent coordinators safe: each
+// process locks a different candidate row instead of queueing on the same one,
+// so no task is ever handed to two workers and no claim blocks behind another.
+// Splitting this into SELECT + UPDATE would reintroduce exactly that race.
+//
+//nolint:unused // wired up in phase 2; kept beside the repository it belongs to
+const claimNextSQL = `
+WITH candidate AS (
+    SELECT id
+    FROM tasks
+    WHERE status = 'pending'
+      AND attempt < max_attempts
+      AND (cardinality($1::text[]) = 0 OR workload = ANY($1))
+    ORDER BY created_at, chunk_index
+    FOR UPDATE SKIP LOCKED
+    LIMIT 1
+)
+UPDATE tasks
+SET status           = 'leased',
+    attempt          = attempt + 1,
+    lease_owner      = $2,
+    lease_expires_at = $3,
+    started_at       = COALESCE(started_at, $4),
+    version          = version + 1
+FROM candidate
+WHERE tasks.id = candidate.id
+RETURNING tasks.id, tasks.job_id, tasks.chunk_index, tasks.workload,
+          tasks.input_uri, tasks.input_sha256, tasks.parameters,
+          tasks.status, tasks.attempt, tasks.max_attempts,
+          tasks.lease_owner, tasks.lease_expires_at, tasks.version;
+`
+
+// expireLeasesSQL applies the lease-expiry rule set-based, mirroring
+// domain.Task.ExpireLease: requeue while attempts remain, otherwise fail.
+//
+//nolint:unused // wired up in phase 5; mirrors domain.Task.ExpireLease
+const expireLeasesSQL = `
+UPDATE tasks
+SET status           = CASE WHEN attempt < max_attempts THEN 'pending'::task_status
+                            ELSE 'failed'::task_status END,
+    lease_owner      = NULL,
+    lease_expires_at = NULL,
+    error_code       = CASE WHEN attempt >= max_attempts THEN $2 ELSE error_code END,
+    error_message    = CASE WHEN attempt >= max_attempts
+                            THEN 'lease expired after the final attempt'
+                            ELSE error_message END,
+    completed_at     = CASE WHEN attempt >= max_attempts THEN $1 ELSE completed_at END,
+    version          = version + 1
+WHERE status = 'leased' AND lease_expires_at < $1;
+`
+
+// TODO(phase 2-6): replace stubs with real pgx queries; the SQL above is ready
+// to wire up. Each method maps 1:1 to a roadmap phase.
+
+func (r *TaskRepo) ClaimNext(ctx context.Context, f usecase.ClaimFilter) (*domain.Task, error) {
+	// Phase 2: run claimNextSQL; pgx.ErrNoRows -> (nil, nil).
+	return nil, usecase.ErrNotImplemented
+}
+
+func (r *TaskRepo) GetForUpdate(ctx context.Context, id uuid.UUID) (*domain.Task, error) {
+	// Phase 3: SELECT ... WHERE id = $1 FOR UPDATE; no rows -> domain.ErrTaskNotFound.
+	return nil, usecase.ErrNotImplemented
+}
+
+func (r *TaskRepo) Update(ctx context.Context, t *domain.Task) error {
+	// Phase 3: UPDATE ... WHERE id = $1 AND version = $2 (optimistic concurrency).
+	return usecase.ErrNotImplemented
+}
+
+func (r *TaskRepo) InsertBatch(ctx context.Context, tasks []*domain.Task) error {
+	// Phase 2: pgx.Batch or COPY; runs inside the caller's transaction.
+	return usecase.ErrNotImplemented
+}
+
+func (r *TaskRepo) ListCompleted(ctx context.Context, jobID uuid.UUID) ([]*domain.Task, error) {
+	// Phase 6: WHERE job_id = $1 AND status = 'completed' ORDER BY chunk_index.
+	return nil, usecase.ErrNotImplemented
+}
+
+func (r *TaskRepo) CountByStatus(ctx context.Context, jobID uuid.UUID) (map[domain.TaskStatus]int, error) {
+	// Phase 3: SELECT status, count(*) ... GROUP BY status.
+	return nil, usecase.ErrNotImplemented
+}
+
+func (r *TaskRepo) ExpireLeases(ctx context.Context, now time.Time) (int64, error) {
+	// Phase 5: run expireLeasesSQL, return the affected row count.
+	return 0, usecase.ErrNotImplemented
+}
@@ -0,0 +1,69 @@
+// Package postgres implements the usecase repository ports on PostgreSQL.
+// SQL and pgx types never escape this package.
+package postgres
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// querier is satisfied by both *pgxpool.Pool and pgx.Tx, letting every
+// repository method run identically inside or outside a transaction.
+//
+//nolint:unused // used by repository methods once phase 2 replaces the stubs
+type querier interface {
+	Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
+	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
+	Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)
+}
+
+// txKey is an unexported struct type, so no other package can collide with it
+// or reach the transaction we stash in the context.
+type txKey struct{}
+
+// TxManager implements usecase.TxManager.
+type TxManager struct {
+	pool *pgxpool.Pool
+}
+
+func NewTxManager(pool *pgxpool.Pool) *TxManager {
+	return &TxManager{pool: pool}
+}
+
+// WithinTx runs fn inside one transaction, committing on success and rolling
+// back on any error or panic.
+//
+// The transaction travels in the context rather than in fn's signature, which
+// is what lets the usecase layer express "do these repository calls atomically"
+// without its port ever mentioning pgx.
+func (m *TxManager) WithinTx(ctx context.Context, fn func(ctx context.Context) error) error {
+	if _, ok := ctx.Value(txKey{}).(pgx.Tx); ok {
+		return fn(ctx) // already inside a transaction — join it, don't nest
+	}
+
+	tx, err := m.pool.Begin(ctx)
+	if err != nil {
+		return err
+	}
+	// Rollback after a successful Commit is a no-op, so this defer is safe and
+	// also covers the panic path.
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	if err := fn(context.WithValue(ctx, txKey{}, tx)); err != nil {
+		return err
+	}
+	return tx.Commit(ctx)
+}
+
+// conn returns the transaction bound to ctx, or the pool when there is none.
+//
+//nolint:unused // every repository method will route through this in phase 2
+func conn(ctx context.Context, pool *pgxpool.Pool) querier {
+	if tx, ok := ctx.Value(txKey{}).(pgx.Tx); ok {
+		return tx
+	}
+	return pool
+}