feat(coordinator): running state, worker liveness, request-size limits
Polish pass hardening the queue and closing plan gaps. - Task state machine gains `running`: the first heartbeat moves a task from leased to running (migrations 0006/0007 add the enum value and extend the lease-integrity check). verifyLease, ExpireLease, the reaper SQL, and job progress all treat leased and running alike. - Worker liveness: a heartbeat from a registered worker (UUID worker_id) bumps its last_heartbeat_at online; a second background reaper marks workers offline after WORKER_OFFLINE_AFTER of silence (RunReaper generalized to RunPeriodic). - Request-size limits: JSON bodies capped at 1 MiB; dataset/artifact uploads capped at MAX_UPLOAD_BYTES (default 1 GiB) via http.MaxBytesReader. - Tests cover the running transition, liveness + offline reaper (unit over memstore and integration over Postgres).
This commit is contained in:
@@ -15,6 +15,7 @@ type TaskStatus string
|
||||
const (
|
||||
TaskPending TaskStatus = "pending"
|
||||
TaskLeased TaskStatus = "leased"
|
||||
TaskRunning TaskStatus = "running"
|
||||
TaskCompleted TaskStatus = "completed"
|
||||
TaskFailed TaskStatus = "failed"
|
||||
TaskCancelled TaskStatus = "cancelled"
|
||||
@@ -146,7 +147,9 @@ func (t *Task) AsClaimed() ClaimedTask {
|
||||
// verifyLease is the guard every worker-driven transition shares: the caller
|
||||
// must own the lease and reference the attempt it was granted.
|
||||
func (t *Task) verifyLease(worker string, attempt int) error {
|
||||
if t.Status != TaskLeased {
|
||||
// A task is worker-owned while leased or running: the first heartbeat moves
|
||||
// it from leased to running, but ownership rules are identical for both.
|
||||
if t.Status != TaskLeased && t.Status != TaskRunning {
|
||||
return ErrTaskNotLeased
|
||||
}
|
||||
if t.LeaseOwner == nil || *t.LeaseOwner != worker {
|
||||
@@ -158,12 +161,16 @@ func (t *Task) verifyLease(worker string, attempt int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RenewLease extends the lease of the worker that holds it.
|
||||
// RenewLease extends the lease of the worker that holds it. The first heartbeat
|
||||
// also acknowledges start, moving the task from leased to running.
|
||||
func (t *Task) RenewLease(worker string, attempt int, until time.Time) error {
|
||||
if err := t.verifyLease(worker, attempt); err != nil {
|
||||
return err
|
||||
}
|
||||
t.LeaseExpiresAt = &until
|
||||
if t.Status == TaskLeased {
|
||||
t.Status = TaskRunning
|
||||
}
|
||||
t.Version++
|
||||
return nil
|
||||
}
|
||||
@@ -228,7 +235,8 @@ func (t *Task) Fail(worker string, attempt int, code, message string, retryable
|
||||
// ExpireLease is applied by the reaper when a lease elapses without a
|
||||
// heartbeat: requeue while attempts remain, otherwise fail terminally.
|
||||
func (t *Task) ExpireLease(now time.Time) {
|
||||
if t.Status != TaskLeased {
|
||||
// Both a leased and a running task can go silent and must be reclaimed.
|
||||
if t.Status != TaskLeased && t.Status != TaskRunning {
|
||||
return
|
||||
}
|
||||
t.LeaseOwner = nil
|
||||
|
||||
@@ -168,6 +168,42 @@ func TestExpireLeaseIgnoresUnleasedTasks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstHeartbeatMovesLeasedToRunning(t *testing.T) {
|
||||
task := leasedTask(1, 3)
|
||||
until := testLater.Add(time.Hour)
|
||||
|
||||
if err := task.RenewLease(testWorker, 1, until); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.Status != TaskRunning {
|
||||
t.Errorf("status = %q, want running after first heartbeat", task.Status)
|
||||
}
|
||||
// A second heartbeat keeps it running.
|
||||
if err := task.RenewLease(testWorker, 1, until); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.Status != TaskRunning {
|
||||
t.Errorf("status = %q, want running", task.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunningTaskCanBeCompletedAndExpired(t *testing.T) {
|
||||
// Complete works from running.
|
||||
task := leasedTask(1, 3)
|
||||
_ = task.RenewLease(testWorker, 1, testLater) // -> running
|
||||
if err := task.CompleteWith(testResult, nil, testWorker, 1, testNow); err != nil {
|
||||
t.Errorf("complete from running: %v", err)
|
||||
}
|
||||
|
||||
// Expire reclaims a running task too.
|
||||
task2 := leasedTask(1, 3)
|
||||
_ = task2.RenewLease(testWorker, 1, testLater) // -> running
|
||||
task2.ExpireLease(testNow)
|
||||
if task2.Status != TaskPending {
|
||||
t.Errorf("status = %q, want pending after a running lease expires", task2.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenewLeaseExtendsOnlyForHolder(t *testing.T) {
|
||||
task := leasedTask(1, 3)
|
||||
until := testLater.Add(time.Hour)
|
||||
|
||||
Reference in New Issue
Block a user