This commit is contained in:
@@ -8,13 +8,14 @@ import "errors"
|
||||
//
|
||||
// Always compare with errors.Is — outer layers may wrap these with %w.
|
||||
var (
|
||||
ErrJobNotFound = errors.New("job not found")
|
||||
ErrTaskNotFound = errors.New("task not found")
|
||||
ErrWorkerNotFound = errors.New("worker not found")
|
||||
ErrArtifactNotFound = errors.New("artifact not found")
|
||||
ErrLeaseConflict = errors.New("task leased to another worker")
|
||||
ErrStaleAttempt = errors.New("attempt does not match lease")
|
||||
ErrResultConflict = errors.New("different result already recorded")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrTaskNotLeased = errors.New("task is not currently leased")
|
||||
ErrJobNotFound = errors.New("job not found")
|
||||
ErrTaskNotFound = errors.New("task not found")
|
||||
ErrWorkerNotFound = errors.New("worker not found")
|
||||
ErrArtifactNotFound = errors.New("artifact not found")
|
||||
ErrJobNotCancellable = errors.New("job cannot be cancelled")
|
||||
ErrLeaseConflict = errors.New("task leased to another worker")
|
||||
ErrStaleAttempt = errors.New("attempt does not match lease")
|
||||
ErrResultConflict = errors.New("different result already recorded")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrTaskNotLeased = errors.New("task is not currently leased")
|
||||
)
|
||||
|
||||
@@ -99,18 +99,21 @@ func NewJobWithTasks(workload, inputURI string, params map[string]any,
|
||||
|
||||
// JobProgress is the aggregate view of a job and the state of its tasks.
|
||||
type JobProgress struct {
|
||||
Job Job
|
||||
Total int
|
||||
Pending int
|
||||
Leased int
|
||||
Done int
|
||||
Failed int
|
||||
Job Job
|
||||
Total int
|
||||
Pending int
|
||||
Leased int
|
||||
Done int
|
||||
Failed int
|
||||
Cancelled int
|
||||
}
|
||||
|
||||
// DeriveStatus computes what the job's status should be from its task counts,
|
||||
// so the rule lives here rather than in a SQL trigger or a handler.
|
||||
func (p JobProgress) DeriveStatus() JobStatus {
|
||||
switch {
|
||||
case p.Job.Status == JobCancelled:
|
||||
return JobCancelled
|
||||
case p.Total == 0:
|
||||
return JobPending
|
||||
case p.Done == p.Total:
|
||||
|
||||
@@ -81,6 +81,7 @@ func TestDeriveStatus(t *testing.T) {
|
||||
{"all done", JobProgress{Total: 3, Done: 3}, JobCompleted},
|
||||
{"done and failed", JobProgress{Total: 3, Done: 2, Failed: 1}, JobFailed},
|
||||
{"failed but work remains", JobProgress{Total: 3, Pending: 1, Failed: 2}, JobRunning},
|
||||
{"cancelled job wins over task histogram", JobProgress{Job: Job{Status: JobCancelled}, Total: 3, Done: 1, Cancelled: 2}, JobCancelled},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
@@ -259,6 +259,24 @@ func (t *Task) ExpireLease(now time.Time) {
|
||||
t.CompletedAt = &now
|
||||
}
|
||||
|
||||
// Cancel prevents any further worker transition for a task that has not
|
||||
// reached a terminal result. A cancelled lease deliberately becomes invalid:
|
||||
// a worker still running locally must not upload or complete after its job was
|
||||
// stopped by the operator.
|
||||
func (t *Task) Cancel(now time.Time) bool {
|
||||
if t.Status == TaskCompleted || t.Status == TaskFailed || t.Status == TaskCancelled {
|
||||
return false
|
||||
}
|
||||
t.Status = TaskCancelled
|
||||
t.LeaseOwner = nil
|
||||
t.LeaseExpiresAt = nil
|
||||
t.ErrorCode = nil
|
||||
t.ErrorMessage = nil
|
||||
t.CompletedAt = &now
|
||||
t.Version++
|
||||
return true
|
||||
}
|
||||
|
||||
// ClaimedTask is the worker-facing projection of a leased task. Input is either
|
||||
// an external URI or a coordinator-stored shard (InputArtifactID set); the
|
||||
// transport turns the latter into a coordinator download URL.
|
||||
|
||||
@@ -168,6 +168,23 @@ func TestExpireLeaseIgnoresUnleasedTasks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelInvalidatesLeaseButPreservesTerminalTask(t *testing.T) {
|
||||
task := leasedTask(1, 3)
|
||||
if !task.Cancel(testNow) {
|
||||
t.Fatal("leased task should be cancelled")
|
||||
}
|
||||
if task.Status != TaskCancelled || task.LeaseOwner != nil || task.LeaseExpiresAt != nil {
|
||||
t.Errorf("cancelled task = %+v", task)
|
||||
}
|
||||
if task.Cancel(testLater) {
|
||||
t.Error("cancelled task must not be changed twice")
|
||||
}
|
||||
completed := &Task{Status: TaskCompleted}
|
||||
if completed.Cancel(testNow) {
|
||||
t.Error("completed task must remain terminal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstHeartbeatMovesLeasedToRunning(t *testing.T) {
|
||||
task := leasedTask(1, 3)
|
||||
until := testLater.Add(time.Hour)
|
||||
|
||||
Reference in New Issue
Block a user