feat(coordinator): running state, worker liveness, request-size limits

Polish pass hardening the queue and closing plan gaps.

- Task state machine gains `running`: the first heartbeat moves a task from
  leased to running (migrations 0006/0007 add the enum value and extend the
  lease-integrity check). verifyLease, ExpireLease, the reaper SQL, and job
  progress all treat leased and running alike.
- Worker liveness: a heartbeat from a registered worker (UUID worker_id) bumps
  its last_heartbeat_at online; a second background reaper marks workers offline
  after WORKER_OFFLINE_AFTER of silence (RunReaper generalized to RunPeriodic).
- Request-size limits: JSON bodies capped at 1 MiB; dataset/artifact uploads
  capped at MAX_UPLOAD_BYTES (default 1 GiB) via http.MaxBytesReader.
- Tests cover the running transition, liveness + offline reaper (unit over
  memstore and integration over Postgres).
This commit is contained in:
Efremenko Arhip
2026-07-23 17:36:26 +03:00
parent 4fc3c69fdf
commit 3b41455b20
23 changed files with 332 additions and 32 deletions
+11 -3
View File
@@ -15,6 +15,7 @@ type TaskStatus string
const (
TaskPending TaskStatus = "pending"
TaskLeased TaskStatus = "leased"
TaskRunning TaskStatus = "running"
TaskCompleted TaskStatus = "completed"
TaskFailed TaskStatus = "failed"
TaskCancelled TaskStatus = "cancelled"
@@ -146,7 +147,9 @@ func (t *Task) AsClaimed() ClaimedTask {
// verifyLease is the guard every worker-driven transition shares: the caller
// must own the lease and reference the attempt it was granted.
func (t *Task) verifyLease(worker string, attempt int) error {
if t.Status != TaskLeased {
// A task is worker-owned while leased or running: the first heartbeat moves
// it from leased to running, but ownership rules are identical for both.
if t.Status != TaskLeased && t.Status != TaskRunning {
return ErrTaskNotLeased
}
if t.LeaseOwner == nil || *t.LeaseOwner != worker {
@@ -158,12 +161,16 @@ func (t *Task) verifyLease(worker string, attempt int) error {
return nil
}
// RenewLease extends the lease of the worker that holds it.
// RenewLease extends the lease of the worker that holds it. The first heartbeat
// also acknowledges start, moving the task from leased to running.
func (t *Task) RenewLease(worker string, attempt int, until time.Time) error {
if err := t.verifyLease(worker, attempt); err != nil {
return err
}
t.LeaseExpiresAt = &until
if t.Status == TaskLeased {
t.Status = TaskRunning
}
t.Version++
return nil
}
@@ -228,7 +235,8 @@ func (t *Task) Fail(worker string, attempt int, code, message string, retryable
// ExpireLease is applied by the reaper when a lease elapses without a
// heartbeat: requeue while attempts remain, otherwise fail terminally.
func (t *Task) ExpireLease(now time.Time) {
if t.Status != TaskLeased {
// Both a leased and a running task can go silent and must be reclaimed.
if t.Status != TaskLeased && t.Status != TaskRunning {
return
}
t.LeaseOwner = nil