Replaces the repository stubs with real pgx queries, so the queue now works end
to end: a job is split into tasks, leased to workers one at a time, heartbeated,
completed, and reflected in job progress.
Task claiming is a single statement — SELECT ... FOR UPDATE SKIP LOCKED feeding
an UPDATE — so concurrent coordinators lease different rows instead of blocking
on the same one. Writes use optimistic concurrency: the entity increments its
version in memory, and the UPDATE guards on the previous value.
Retries moved to the transaction level. Once Postgres aborts a transaction with
a serialization failure, replaying one statement inside it cannot help; the unit
of retry is Begin -> fn -> Commit, which is safe because each attempt re-reads
its rows through GetForUpdate.
Adds integration tests behind the `integration` build tag, run against a real
PostgreSQL through TEST_DATABASE_URL: concurrent claiming hands each task to
exactly one worker, job creation rolls back whole, stale writes are refused,
completed results keep chunk order, and expired leases return to the queue.
Two bugs the tests caught:
- a nil parameters map reached a NOT NULL jsonb column as SQL NULL, since pgx
sends NULL rather than omitting the column and letting DEFAULT '{}' apply;
- replaying an already-recorded result returned 409. The idempotent path leaves
the entity untouched, so the version guard matched nothing and a successful
no-op looked like a conflict. CompleteTask now skips the write when the
entity did not change.
216 lines
6.4 KiB
Go
216 lines
6.4 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/emil28092005/SciMesh/coordinator/internal/domain"
|
|
)
|
|
|
|
// Task operations: the worker-facing lifecycle of a single chunk.
|
|
//
|
|
// ClaimTask lease the next available task
|
|
// RenewLease extend a held lease (heartbeat)
|
|
// CompleteTask record a successful result
|
|
// FailTask record a failure
|
|
// ExpireLeases reclaim leases that elapsed without a heartbeat
|
|
|
|
// --- ClaimTask -----------------------------------------------------------
|
|
|
|
type ClaimTask struct {
|
|
tasks TaskRepository
|
|
clock Clock
|
|
leaseDuration time.Duration
|
|
}
|
|
|
|
func NewClaimTask(tasks TaskRepository, clock Clock, leaseDuration time.Duration) *ClaimTask {
|
|
return &ClaimTask{tasks: tasks, clock: clock, leaseDuration: leaseDuration}
|
|
}
|
|
|
|
// Execute reclaims elapsed leases first, then hands out one task.
|
|
//
|
|
// Sweeping before claiming matters: otherwise a task abandoned by a dead worker
|
|
// stays invisible until the reaper's next tick, and a waiting worker is told the
|
|
// queue is empty while work sits idle.
|
|
//
|
|
// This use case is thin by design — the atomicity that makes claiming correct
|
|
// lives in one SQL statement behind ClaimNext, and splitting it across the layer
|
|
// boundary would break it.
|
|
func (uc *ClaimTask) Execute(ctx context.Context, in ClaimTaskInput) (*domain.ClaimedTask, error) {
|
|
if in.WorkerID == "" {
|
|
return nil, domain.ErrInvalidInput
|
|
}
|
|
now := uc.clock.Now()
|
|
|
|
if _, err := uc.tasks.ExpireLeases(ctx, now); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
task, err := uc.tasks.ClaimNext(ctx, ClaimFilter{
|
|
Workloads: in.Workloads,
|
|
Owner: in.WorkerID,
|
|
Now: now,
|
|
LeaseUntil: now.Add(uc.leaseDuration),
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if task == nil {
|
|
return nil, nil // empty queue is a normal state, not an error
|
|
}
|
|
|
|
claimed := task.AsClaimed()
|
|
return &claimed, nil
|
|
}
|
|
|
|
// --- RenewLease ----------------------------------------------------------
|
|
|
|
type RenewLease struct {
|
|
tasks TaskRepository
|
|
tx TxManager
|
|
clock Clock
|
|
leaseDuration time.Duration
|
|
}
|
|
|
|
func NewRenewLease(tasks TaskRepository, tx TxManager, clock Clock, leaseDuration time.Duration) *RenewLease {
|
|
return &RenewLease{tasks: tasks, tx: tx, clock: clock, leaseDuration: leaseDuration}
|
|
}
|
|
|
|
// Execute is a read-modify-write, so it runs inside a transaction with the row
|
|
// locked: two concurrent heartbeats must not interleave into a lost update.
|
|
// Whether the caller may renew at all is decided by the entity, not here.
|
|
func (uc *RenewLease) Execute(ctx context.Context, in RenewLeaseInput) (*domain.ClaimedTask, error) {
|
|
var claimed domain.ClaimedTask
|
|
|
|
err := uc.tx.WithinTx(ctx, func(ctx context.Context) error {
|
|
task, err := uc.tasks.GetForUpdate(ctx, in.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := task.RenewLease(in.WorkerID, in.Attempt, uc.clock.Now().Add(uc.leaseDuration)); err != nil {
|
|
return err
|
|
}
|
|
if err := uc.tasks.Update(ctx, task); err != nil {
|
|
return err
|
|
}
|
|
claimed = task.AsClaimed()
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &claimed, nil
|
|
}
|
|
|
|
// --- CompleteTask --------------------------------------------------------
|
|
|
|
type CompleteTask struct {
|
|
tasks TaskRepository
|
|
jobs JobRepository
|
|
tx TxManager
|
|
clock Clock
|
|
}
|
|
|
|
func NewCompleteTask(tasks TaskRepository, jobs JobRepository, tx TxManager, clock Clock) *CompleteTask {
|
|
return &CompleteTask{tasks: tasks, jobs: jobs, tx: tx, clock: clock}
|
|
}
|
|
|
|
// Execute applies the result and, when that was the job's last outstanding
|
|
// task, closes the job in the same transaction — so a caller who sees a
|
|
// completed task never observes its job still marked running.
|
|
//
|
|
// Lease ownership, staleness, and idempotent replays are all decided by
|
|
// Task.CompleteWith; this use case only orchestrates.
|
|
func (uc *CompleteTask) Execute(ctx context.Context, in CompleteTaskInput) (*domain.Task, error) {
|
|
var out *domain.Task
|
|
|
|
err := uc.tx.WithinTx(ctx, func(ctx context.Context) error {
|
|
task, err := uc.tasks.GetForUpdate(ctx, in.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := uc.clock.Now()
|
|
before := task.Version
|
|
if err := task.CompleteWith(in.ResultURI, in.ResultSHA256, in.Metrics,
|
|
in.WorkerID, in.Attempt, now); err != nil {
|
|
return err
|
|
}
|
|
out = task
|
|
|
|
// A replay of an already-recorded result leaves the entity untouched.
|
|
// Writing anyway would fail the optimistic-concurrency guard (the stored
|
|
// version already equals ours) and turn an idempotent call into a 409.
|
|
if task.Version == before {
|
|
return nil
|
|
}
|
|
|
|
if err := uc.tasks.Update(ctx, task); err != nil {
|
|
return err
|
|
}
|
|
return syncJobStatus(ctx, uc.jobs, uc.tasks, task.JobID, now)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// --- FailTask ------------------------------------------------------------
|
|
|
|
type FailTask struct {
|
|
tasks TaskRepository
|
|
jobs JobRepository
|
|
tx TxManager
|
|
clock Clock
|
|
}
|
|
|
|
func NewFailTask(tasks TaskRepository, jobs JobRepository, tx TxManager, clock Clock) *FailTask {
|
|
return &FailTask{tasks: tasks, jobs: jobs, tx: tx, clock: clock}
|
|
}
|
|
|
|
// Execute delegates the requeue-or-terminate decision to Task.Fail, then keeps
|
|
// the parent job's status consistent in the same transaction.
|
|
func (uc *FailTask) Execute(ctx context.Context, in FailTaskInput) (*domain.Task, error) {
|
|
var out *domain.Task
|
|
|
|
err := uc.tx.WithinTx(ctx, func(ctx context.Context) error {
|
|
task, err := uc.tasks.GetForUpdate(ctx, in.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := uc.clock.Now()
|
|
if err := task.Fail(in.WorkerID, in.Attempt, in.ErrorCode, in.ErrorMessage, in.Retryable, now); err != nil {
|
|
return err
|
|
}
|
|
if err := uc.tasks.Update(ctx, task); err != nil {
|
|
return err
|
|
}
|
|
out = task
|
|
return syncJobStatus(ctx, uc.jobs, uc.tasks, task.JobID, now)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// --- ExpireLeases --------------------------------------------------------
|
|
|
|
type ExpireLeases struct {
|
|
tasks TaskRepository
|
|
clock Clock
|
|
}
|
|
|
|
func NewExpireLeases(tasks TaskRepository, clock Clock) *ExpireLeases {
|
|
return &ExpireLeases{tasks: tasks, clock: clock}
|
|
}
|
|
|
|
// Execute reports how many tasks were reclaimed.
|
|
//
|
|
// The sweep is one set-based statement rather than a load-decide-save loop:
|
|
// several coordinators run it concurrently, and a single atomic UPDATE makes
|
|
// the duplicate work harmless — the loser simply updates 0 rows.
|
|
func (uc *ExpireLeases) Execute(ctx context.Context) (int64, error) {
|
|
return uc.tasks.ExpireLeases(ctx, uc.clock.Now())
|
|
}
|