feat(coordinator): running state, worker liveness, request-size limits

Polish pass hardening the queue and closing plan gaps.

- Task state machine gains `running`: the first heartbeat moves a task from
  leased to running (migrations 0006/0007 add the enum value and extend the
  lease-integrity check). verifyLease, ExpireLease, the reaper SQL, and job
  progress all treat leased and running alike.
- Worker liveness: a heartbeat from a registered worker (UUID worker_id) bumps
  its last_heartbeat_at online; a second background reaper marks workers offline
  after WORKER_OFFLINE_AFTER of silence (RunReaper generalized to RunPeriodic).
- Request-size limits: JSON bodies capped at 1 MiB; dataset/artifact uploads
  capped at MAX_UPLOAD_BYTES (default 1 GiB) via http.MaxBytesReader.
- Tests cover the running transition, liveness + offline reaper (unit over
  memstore and integration over Postgres).

coordinator/internal/usecase/job.go

@@ -143,9 +143,10 @@ func progressFrom(job domain.Job, counts map[domain.TaskStatus]int) domain.JobPr
 	p := domain.JobProgress{
 		Job:     job,
 		Pending: counts[domain.TaskPending],
-		Leased:  counts[domain.TaskLeased],
-		Done:    counts[domain.TaskCompleted],
-		Failed:  counts[domain.TaskFailed],
+		// Leased and running are both "in flight" for progress purposes.
+		Leased: counts[domain.TaskLeased] + counts[domain.TaskRunning],
+		Done:   counts[domain.TaskCompleted],
+		Failed: counts[domain.TaskFailed],
 	}
 	for _, n := range counts {
 		p.Total += n

coordinator/internal/usecase/ports.go

@@ -72,6 +72,12 @@ type JobRepository interface {
 type WorkerRepository interface {
 	Insert(ctx context.Context, w *domain.Worker) error
 	Get(ctx context.Context, id uuid.UUID) (*domain.Worker, error)
+	// Touch records liveness for a heartbeating worker, marking it online. A
+	// no-op for an id that is not a registered worker.
+	Touch(ctx context.Context, id uuid.UUID, at time.Time) error
+	// MarkStaleOffline flips every worker last seen before cutoff to offline and
+	// reports how many changed.
+	MarkStaleOffline(ctx context.Context, cutoff time.Time) (int64, error)
 }
 
 // ArtifactRepository persists artifact metadata. The bytes live in a BlobStore;

coordinator/internal/usecase/task.go

@@ -69,13 +69,15 @@ func (uc *ClaimTask) Execute(ctx context.Context, in ClaimTaskInput) (*domain.Cl
 
 type RenewLease struct {
 	tasks         TaskRepository
+	workers       WorkerRepository
 	tx            TxManager
 	clock         Clock
 	leaseDuration time.Duration
 }
 
-func NewRenewLease(tasks TaskRepository, tx TxManager, clock Clock, leaseDuration time.Duration) *RenewLease {
-	return &RenewLease{tasks: tasks, tx: tx, clock: clock, leaseDuration: leaseDuration}
+func NewRenewLease(tasks TaskRepository, workers WorkerRepository, tx TxManager,
+	clock Clock, leaseDuration time.Duration) *RenewLease {
+	return &RenewLease{tasks: tasks, workers: workers, tx: tx, clock: clock, leaseDuration: leaseDuration}
 }
 
 // Execute is a read-modify-write, so it runs inside a transaction with the row
@@ -101,6 +103,12 @@ func (uc *RenewLease) Execute(ctx context.Context, in RenewLeaseInput) (*domain.
 	if err != nil {
 		return nil, err
 	}
+
+	// Best-effort worker liveness, outside the task transaction so it can never
+	// fail the heartbeat. Only registered workers (a UUID worker_id) are tracked.
+	if id, perr := uuid.Parse(in.WorkerID); perr == nil {
+		_ = uc.workers.Touch(ctx, id, uc.clock.Now())
+	}
 	return &claimed, nil
 }
 

coordinator/internal/usecase/usecase_test.go

@@ -57,7 +57,7 @@ func newHarness() *harness {
 	h.createJob = usecase.NewCreateJob(h.jobs, h.tasks, tx, h.clk)
 	h.submit = usecase.NewSubmitDataset(h.blobs, h.arts, h.jobs, h.tasks, tx, h.clk)
 	h.claim = usecase.NewClaimTask(h.tasks, h.clk, lease)
-	h.renew = usecase.NewRenewLease(h.tasks, tx, h.clk, lease)
+	h.renew = usecase.NewRenewLease(h.tasks, h.work, tx, h.clk, lease)
 	h.complete = usecase.NewCompleteTask(h.tasks, h.jobs, h.arts, tx, h.clk)
 	h.fail = usecase.NewFailTask(h.tasks, h.jobs, tx, h.clk)
 	h.status = usecase.NewGetJobStatus(h.jobs, h.tasks)
@@ -172,6 +172,26 @@ func TestRenewExtendsForHolder(t *testing.T) {
 	}
 }
 
+func TestHeartbeatThenCompleteViaRunning(t *testing.T) {
+	h := newHarness()
+	jobID := h.seedJob(t, "w", 1)
+	taskID, attempt := h.leaseOne(t, "w1", "w")
+
+	// Heartbeat moves the task to running; completion must still work from there.
+	if _, err := h.renew.Execute(ctx, usecase.RenewLeaseInput{TaskID: taskID, WorkerID: "w1", Attempt: attempt}); err != nil {
+		t.Fatalf("heartbeat: %v", err)
+	}
+	artID := h.uploadResult(t, taskID, "w1", attempt)
+	if _, err := h.complete.Execute(ctx, usecase.CompleteTaskInput{
+		TaskID: taskID, WorkerID: "w1", Attempt: attempt, ResultArtifactID: artID,
+	}); err != nil {
+		t.Fatalf("complete after heartbeat: %v", err)
+	}
+	if prog, _ := h.status.Execute(ctx, jobID); prog.DeriveStatus() != domain.JobCompleted {
+		t.Errorf("job status = %q, want completed", prog.DeriveStatus())
+	}
+}
+
 func TestRenewRejectsForeignWorker(t *testing.T) {
 	h := newHarness()
 	h.seedJob(t, "w", 1)
@@ -285,6 +305,38 @@ func TestRegisterWorkerPersists(t *testing.T) {
 	}
 }
 
+func TestHeartbeatTracksWorkerLivenessAndReaperMarksOffline(t *testing.T) {
+	h := newHarness()
+	w, err := h.register.Execute(ctx, usecase.RegisterWorkerInput{Name: "lab", Capabilities: []string{"w"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	wid := w.ID.String() // a registered worker heartbeats with its UUID
+	h.seedJob(t, "w", 1)
+
+	c, err := h.claim.Execute(ctx, usecase.ClaimTaskInput{WorkerID: wid, Workloads: []string{"w"}})
+	if err != nil || c == nil {
+		t.Fatalf("claim: %v", err)
+	}
+	if _, err := h.renew.Execute(ctx, usecase.RenewLeaseInput{TaskID: c.TaskID, WorkerID: wid, Attempt: c.Attempt}); err != nil {
+		t.Fatalf("heartbeat: %v", err)
+	}
+	if got, _ := h.work.Get(ctx, w.ID); got.Status != domain.WorkerOnline {
+		t.Errorf("worker status = %q, want online after heartbeat", got.Status)
+	}
+
+	// Go silent past the threshold; the reaper marks it offline.
+	offline := usecase.NewMarkWorkersOffline(h.work, h.clk, 30*time.Second)
+	h.clk.Advance(time.Minute)
+	n, err := offline.Execute(ctx)
+	if err != nil || n != 1 {
+		t.Fatalf("reaper marked %d offline (err %v), want 1", n, err)
+	}
+	if got, _ := h.work.Get(ctx, w.ID); got.Status != domain.WorkerOffline {
+		t.Errorf("worker status = %q, want offline after reaper", got.Status)
+	}
+}
+
 func TestRegisterWorkerRejectsNoCapabilities(t *testing.T) {
 	h := newHarness()
 	if _, err := h.register.Execute(ctx, usecase.RegisterWorkerInput{Name: "lab"}); !errors.Is(err, domain.ErrInvalidInput) {

coordinator/internal/usecase/worker.go

@@ -2,6 +2,7 @@ package usecase
 
 import (
 	"context"
+	"time"
 
 	"github.com/emil28092005/SciMesh/coordinator/internal/domain"
 )
@@ -26,3 +27,19 @@ func (uc *RegisterWorker) Execute(ctx context.Context, in RegisterWorkerInput) (
 	}
 	return w, nil
 }
+
+// MarkWorkersOffline is the liveness reaper: workers that stopped heartbeating
+// longer ago than `after` are flipped to offline.
+type MarkWorkersOffline struct {
+	workers WorkerRepository
+	clk     Clock
+	after   time.Duration
+}
+
+func NewMarkWorkersOffline(workers WorkerRepository, clk Clock, after time.Duration) *MarkWorkersOffline {
+	return &MarkWorkersOffline{workers: workers, clk: clk, after: after}
+}
+
+func (uc *MarkWorkersOffline) Execute(ctx context.Context) (int64, error) {
+	return uc.workers.MarkStaleOffline(ctx, uc.clk.Now().Add(-uc.after))
+}