Add job cancellation and dataset row limit
coordinator / test (push) Canceled after 0s

This commit is contained in:
Emil
2026-07-23 23:14:25 +03:00
parent 6bac7dad3c
commit 7547a30bde
30 changed files with 438 additions and 54 deletions
+10 -9
View File
@@ -8,13 +8,14 @@ import "errors"
//
// Always compare with errors.Is — outer layers may wrap these with %w.
var (
ErrJobNotFound = errors.New("job not found")
ErrTaskNotFound = errors.New("task not found")
ErrWorkerNotFound = errors.New("worker not found")
ErrArtifactNotFound = errors.New("artifact not found")
ErrLeaseConflict = errors.New("task leased to another worker")
ErrStaleAttempt = errors.New("attempt does not match lease")
ErrResultConflict = errors.New("different result already recorded")
ErrInvalidInput = errors.New("invalid input")
ErrTaskNotLeased = errors.New("task is not currently leased")
ErrJobNotFound = errors.New("job not found")
ErrTaskNotFound = errors.New("task not found")
ErrWorkerNotFound = errors.New("worker not found")
ErrArtifactNotFound = errors.New("artifact not found")
ErrJobNotCancellable = errors.New("job cannot be cancelled")
ErrLeaseConflict = errors.New("task leased to another worker")
ErrStaleAttempt = errors.New("attempt does not match lease")
ErrResultConflict = errors.New("different result already recorded")
ErrInvalidInput = errors.New("invalid input")
ErrTaskNotLeased = errors.New("task is not currently leased")
)
+9 -6
View File
@@ -99,18 +99,21 @@ func NewJobWithTasks(workload, inputURI string, params map[string]any,
// JobProgress is the aggregate view of a job and the state of its tasks.
type JobProgress struct {
Job Job
Total int
Pending int
Leased int
Done int
Failed int
Job Job
Total int
Pending int
Leased int
Done int
Failed int
Cancelled int
}
// DeriveStatus computes what the job's status should be from its task counts,
// so the rule lives here rather than in a SQL trigger or a handler.
func (p JobProgress) DeriveStatus() JobStatus {
switch {
case p.Job.Status == JobCancelled:
return JobCancelled
case p.Total == 0:
return JobPending
case p.Done == p.Total:
+1
View File
@@ -81,6 +81,7 @@ func TestDeriveStatus(t *testing.T) {
{"all done", JobProgress{Total: 3, Done: 3}, JobCompleted},
{"done and failed", JobProgress{Total: 3, Done: 2, Failed: 1}, JobFailed},
{"failed but work remains", JobProgress{Total: 3, Pending: 1, Failed: 2}, JobRunning},
{"cancelled job wins over task histogram", JobProgress{Job: Job{Status: JobCancelled}, Total: 3, Done: 1, Cancelled: 2}, JobCancelled},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
+18
View File
@@ -259,6 +259,24 @@ func (t *Task) ExpireLease(now time.Time) {
t.CompletedAt = &now
}
// Cancel prevents any further worker transition for a task that has not
// reached a terminal result. A cancelled lease deliberately becomes invalid:
// a worker still running locally must not upload or complete after its job was
// stopped by the operator.
func (t *Task) Cancel(now time.Time) bool {
if t.Status == TaskCompleted || t.Status == TaskFailed || t.Status == TaskCancelled {
return false
}
t.Status = TaskCancelled
t.LeaseOwner = nil
t.LeaseExpiresAt = nil
t.ErrorCode = nil
t.ErrorMessage = nil
t.CompletedAt = &now
t.Version++
return true
}
// ClaimedTask is the worker-facing projection of a leased task. Input is either
// an external URI or a coordinator-stored shard (InputArtifactID set); the
// transport turns the latter into a coordinator download URL.
+17
View File
@@ -168,6 +168,23 @@ func TestExpireLeaseIgnoresUnleasedTasks(t *testing.T) {
}
}
func TestCancelInvalidatesLeaseButPreservesTerminalTask(t *testing.T) {
task := leasedTask(1, 3)
if !task.Cancel(testNow) {
t.Fatal("leased task should be cancelled")
}
if task.Status != TaskCancelled || task.LeaseOwner != nil || task.LeaseExpiresAt != nil {
t.Errorf("cancelled task = %+v", task)
}
if task.Cancel(testLater) {
t.Error("cancelled task must not be changed twice")
}
completed := &Task{Status: TaskCompleted}
if completed.Cancel(testNow) {
t.Error("completed task must remain terminal")
}
}
func TestFirstHeartbeatMovesLeasedToRunning(t *testing.T) {
task := leasedTask(1, 3)
until := testLater.Add(time.Hour)