Add worker stats to the wizard, artifact pruning and offline-worker removal to the admin console
This commit is contained in:
@@ -521,3 +521,20 @@ func (a *Admin) RevealWorkerToken(ctx context.Context, actor string) string {
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
// RemoveWorker deletes an offline worker from the registry. Online or busy
|
||||
// workers are refused: an admin console must never yank a live machine out
|
||||
// from under a running task.
|
||||
func (a *Admin) RemoveWorker(ctx context.Context, id uuid.UUID) error {
|
||||
if a.workers == nil {
|
||||
return domain.ErrWorkerNotFound
|
||||
}
|
||||
worker, err := a.workers.Get(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if worker.Status != domain.WorkerOffline {
|
||||
return domain.ErrInvalidInput
|
||||
}
|
||||
return a.workers.Delete(ctx, id)
|
||||
}
|
||||
|
||||
@@ -336,3 +336,30 @@ func TestAdminRevealToken(t *testing.T) {
|
||||
t.Errorf("token = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type removableWorkerRepo struct {
|
||||
WorkerRepository
|
||||
deleted uuid.UUID
|
||||
}
|
||||
|
||||
func (f *removableWorkerRepo) Get(ctx context.Context, id uuid.UUID) (*domain.Worker, error) {
|
||||
return &domain.Worker{ID: id, Status: domain.WorkerOffline}, nil
|
||||
}
|
||||
|
||||
func (f *removableWorkerRepo) Delete(ctx context.Context, id uuid.UUID) error {
|
||||
f.deleted = id
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAdminRemoveWorker(t *testing.T) {
|
||||
a := adminFixture()
|
||||
repo := &removableWorkerRepo{}
|
||||
a.workers = repo
|
||||
id := uuid.New()
|
||||
if err := a.RemoveWorker(context.Background(), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repo.deleted != id {
|
||||
t.Error("offline worker must be deleted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,12 @@ type JobRepository interface {
|
||||
ClaimReduction(ctx context.Context, id uuid.UUID, startedAt time.Time) (bool, error)
|
||||
CompleteWithResult(ctx context.Context, id, resultArtifactID uuid.UUID, completedAt time.Time) error
|
||||
FailReduction(ctx context.Context, id uuid.UUID, code, message string, completedAt time.Time) error
|
||||
// ListCompletedBefore returns jobs that finished (completed or failed)
|
||||
// before the cutoff, for the admin artifact pruner.
|
||||
ListCompletedBefore(ctx context.Context, cutoff time.Time) ([]domain.Job, error)
|
||||
// Delete removes a job row; the engine cascades its tasks, artifacts and
|
||||
// quorum votes. Blob files must be removed separately.
|
||||
Delete(ctx context.Context, id uuid.UUID) error
|
||||
}
|
||||
|
||||
// WorkerRepository persists the worker registry.
|
||||
@@ -97,6 +103,9 @@ type WorkerRepository interface {
|
||||
// SetTrust reclassifies a worker's trust level (trusted/untrusted). Returns
|
||||
// ErrNotFound when the id is unknown.
|
||||
SetTrust(ctx context.Context, id uuid.UUID, trust domain.WorkerTrust) error
|
||||
// Delete removes a worker from the registry. Returns ErrNotFound when the
|
||||
// id is unknown.
|
||||
Delete(ctx context.Context, id uuid.UUID) error
|
||||
}
|
||||
|
||||
// ArtifactRepository persists artifact metadata. The bytes live in a BlobStore;
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PruneArtifacts removes completed or failed jobs older than `olderThan` and
|
||||
// every artifact they own: database rows cascade, blob files are deleted
|
||||
// explicitly. It returns what was freed so the admin console can report it.
|
||||
type PruneArtifacts struct {
|
||||
jobs JobRepository
|
||||
read UIReadRepository
|
||||
blobs BlobStore
|
||||
clk Clock
|
||||
}
|
||||
|
||||
func NewPruneArtifacts(jobs JobRepository, read UIReadRepository, blobs BlobStore, clk Clock) *PruneArtifacts {
|
||||
return &PruneArtifacts{jobs: jobs, read: read, blobs: blobs, clk: clk}
|
||||
}
|
||||
|
||||
type PruneResult struct {
|
||||
Jobs int `json:"jobs"`
|
||||
Artifacts int `json:"artifacts"`
|
||||
FreedBytes int64 `json:"freed_bytes"`
|
||||
}
|
||||
|
||||
// Execute deletes finished jobs whose completion timestamp is older than the
|
||||
// cutoff. Jobs that are still active are never touched.
|
||||
func (uc *PruneArtifacts) Execute(ctx context.Context, olderThan time.Duration) (PruneResult, error) {
|
||||
cutoff := uc.clk.Now().Add(-olderThan)
|
||||
jobs, err := uc.jobs.ListCompletedBefore(ctx, cutoff)
|
||||
if err != nil {
|
||||
return PruneResult{}, err
|
||||
}
|
||||
out := PruneResult{}
|
||||
for _, job := range jobs {
|
||||
artifacts, err := uc.read.ListArtifactsByJob(ctx, job.ID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, artifact := range artifacts {
|
||||
if err := uc.blobs.Delete(ctx, artifact.StorageKey); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.FreedBytes += artifact.SizeBytes
|
||||
out.Artifacts++
|
||||
}
|
||||
if err := uc.jobs.Delete(ctx, job.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Jobs++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -980,3 +980,40 @@ func TestSubmitDatasetRejectsDisabledWorkload(t *testing.T) {
|
||||
t.Fatalf("submit after re-enable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneArtifactsRemovesOldFinishedJobs(t *testing.T) {
|
||||
h := newHarness()
|
||||
old := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
job := &domain.Job{ID: uuid.New(), Workload: "similarity-search", Status: domain.JobCompleted, CreatedAt: old, CompletedAt: &old}
|
||||
if err := h.jobs.Insert(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
art, err := domain.NewArtifact(job.ID, nil, domain.ArtifactFinalResult, "r.csv", "text/csv", old)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
art.SetContent("sha", 42)
|
||||
if err := h.arts.Insert(context.Background(), art); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An active job must survive the prune.
|
||||
active := &domain.Job{ID: uuid.New(), Workload: "similarity-search", Status: domain.JobRunning, CreatedAt: old}
|
||||
if err := h.jobs.Insert(context.Background(), active); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
prune := usecase.NewPruneArtifacts(h.jobs, memstore.NewUIReadRepo(h.jobs, h.tasks, h.work, h.arts), h.blobs, h.clk)
|
||||
result, err := prune.Execute(context.Background(), 7*24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Jobs != 1 || result.Artifacts != 1 || result.FreedBytes != 42 {
|
||||
t.Errorf("prune = %+v, want 1 job / 1 artifact / 42 bytes", result)
|
||||
}
|
||||
if _, err := h.jobs.Get(context.Background(), job.ID); err == nil {
|
||||
t.Error("finished job must be gone")
|
||||
}
|
||||
if _, err := h.jobs.Get(context.Background(), active.ID); err != nil {
|
||||
t.Error("active job must survive the prune")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user