This commit is contained in:
@@ -77,6 +77,7 @@ func run() error {
|
||||
CompleteTask: usecase.NewCompleteTask(taskRepo, jobRepo, artifactRepo, tx, clk),
|
||||
FailTask: usecase.NewFailTask(taskRepo, jobRepo, tx, clk),
|
||||
GetJobStatus: usecase.NewGetJobStatus(jobRepo, taskRepo),
|
||||
CancelJob: usecase.NewCancelJob(jobRepo, taskRepo, tx, clk),
|
||||
UploadArtifact: usecase.NewUploadArtifact(taskRepo, artifactRepo, blobStore, clk),
|
||||
DownloadArtifact: usecase.NewDownloadArtifact(artifactRepo, blobStore),
|
||||
GetTaskInput: usecase.NewGetTaskInput(taskRepo, artifactRepo, blobStore),
|
||||
|
||||
@@ -26,9 +26,19 @@ var ErrNoRows = fmt.Errorf("input has no data rows")
|
||||
// Only one shard is buffered at a time, so memory is bounded by shard size (a
|
||||
// worker-sized slice of the data), not by the size of the whole dataset.
|
||||
func SplitTSV(r io.Reader, rowsPerShard int, emit func(index int, shard io.Reader) error) error {
|
||||
return SplitTSVLimit(r, rowsPerShard, 0, emit)
|
||||
}
|
||||
|
||||
// SplitTSVLimit behaves like SplitTSV but emits no more than maxRows data rows.
|
||||
// A maxRows value of zero means unlimited. This lets an operator make a small,
|
||||
// representative pipeline check without materialising a second dataset file.
|
||||
func SplitTSVLimit(r io.Reader, rowsPerShard, maxRows int, emit func(index int, shard io.Reader) error) error {
|
||||
if rowsPerShard <= 0 {
|
||||
return fmt.Errorf("rowsPerShard must be positive, got %d", rowsPerShard)
|
||||
}
|
||||
if maxRows < 0 {
|
||||
return fmt.Errorf("maxRows must be non-negative, got %d", maxRows)
|
||||
}
|
||||
|
||||
sc := bufio.NewScanner(r)
|
||||
// Allow long lines: a SMILES row can be far wider than bufio's 64 KB default.
|
||||
@@ -73,6 +83,9 @@ func SplitTSV(r io.Reader, rowsPerShard int, emit func(index int, shard io.Reade
|
||||
return err
|
||||
}
|
||||
}
|
||||
if maxRows > 0 && index*rowsPerShard+rows == maxRows {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return fmt.Errorf("read rows: %w", err)
|
||||
|
||||
@@ -103,6 +103,22 @@ func TestSplitSingleShardWhenSizeExceedsRows(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitLimitUsesOnlyLeadingDataRows(t *testing.T) {
|
||||
input := "h\nr1\nr2\nr3\nr4\nr5\n"
|
||||
var shards []string
|
||||
err := SplitTSVLimit(strings.NewReader(input), 2, 3, func(_ int, shard io.Reader) error {
|
||||
b, _ := io.ReadAll(shard)
|
||||
shards = append(shards, string(b))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := strings.Join(shards, ""), "h\nr1\nr2\nh\nr3\n"; got != want {
|
||||
t.Errorf("limited shards = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The scanned bytes are reused by bufio; the shard buffer must copy them, or a
|
||||
// later row would corrupt an earlier one. This guards that copy.
|
||||
func TestSplitDoesNotAliasScannerBuffer(t *testing.T) {
|
||||
|
||||
@@ -8,13 +8,14 @@ import "errors"
|
||||
//
|
||||
// Always compare with errors.Is — outer layers may wrap these with %w.
|
||||
var (
|
||||
ErrJobNotFound = errors.New("job not found")
|
||||
ErrTaskNotFound = errors.New("task not found")
|
||||
ErrWorkerNotFound = errors.New("worker not found")
|
||||
ErrArtifactNotFound = errors.New("artifact not found")
|
||||
ErrLeaseConflict = errors.New("task leased to another worker")
|
||||
ErrStaleAttempt = errors.New("attempt does not match lease")
|
||||
ErrResultConflict = errors.New("different result already recorded")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrTaskNotLeased = errors.New("task is not currently leased")
|
||||
ErrJobNotFound = errors.New("job not found")
|
||||
ErrTaskNotFound = errors.New("task not found")
|
||||
ErrWorkerNotFound = errors.New("worker not found")
|
||||
ErrArtifactNotFound = errors.New("artifact not found")
|
||||
ErrJobNotCancellable = errors.New("job cannot be cancelled")
|
||||
ErrLeaseConflict = errors.New("task leased to another worker")
|
||||
ErrStaleAttempt = errors.New("attempt does not match lease")
|
||||
ErrResultConflict = errors.New("different result already recorded")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrTaskNotLeased = errors.New("task is not currently leased")
|
||||
)
|
||||
|
||||
@@ -99,18 +99,21 @@ func NewJobWithTasks(workload, inputURI string, params map[string]any,
|
||||
|
||||
// JobProgress is the aggregate view of a job and the state of its tasks.
|
||||
type JobProgress struct {
|
||||
Job Job
|
||||
Total int
|
||||
Pending int
|
||||
Leased int
|
||||
Done int
|
||||
Failed int
|
||||
Job Job
|
||||
Total int
|
||||
Pending int
|
||||
Leased int
|
||||
Done int
|
||||
Failed int
|
||||
Cancelled int
|
||||
}
|
||||
|
||||
// DeriveStatus computes what the job's status should be from its task counts,
|
||||
// so the rule lives here rather than in a SQL trigger or a handler.
|
||||
func (p JobProgress) DeriveStatus() JobStatus {
|
||||
switch {
|
||||
case p.Job.Status == JobCancelled:
|
||||
return JobCancelled
|
||||
case p.Total == 0:
|
||||
return JobPending
|
||||
case p.Done == p.Total:
|
||||
|
||||
@@ -81,6 +81,7 @@ func TestDeriveStatus(t *testing.T) {
|
||||
{"all done", JobProgress{Total: 3, Done: 3}, JobCompleted},
|
||||
{"done and failed", JobProgress{Total: 3, Done: 2, Failed: 1}, JobFailed},
|
||||
{"failed but work remains", JobProgress{Total: 3, Pending: 1, Failed: 2}, JobRunning},
|
||||
{"cancelled job wins over task histogram", JobProgress{Job: Job{Status: JobCancelled}, Total: 3, Done: 1, Cancelled: 2}, JobCancelled},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
@@ -259,6 +259,24 @@ func (t *Task) ExpireLease(now time.Time) {
|
||||
t.CompletedAt = &now
|
||||
}
|
||||
|
||||
// Cancel prevents any further worker transition for a task that has not
|
||||
// reached a terminal result. A cancelled lease deliberately becomes invalid:
|
||||
// a worker still running locally must not upload or complete after its job was
|
||||
// stopped by the operator.
|
||||
func (t *Task) Cancel(now time.Time) bool {
|
||||
if t.Status == TaskCompleted || t.Status == TaskFailed || t.Status == TaskCancelled {
|
||||
return false
|
||||
}
|
||||
t.Status = TaskCancelled
|
||||
t.LeaseOwner = nil
|
||||
t.LeaseExpiresAt = nil
|
||||
t.ErrorCode = nil
|
||||
t.ErrorMessage = nil
|
||||
t.CompletedAt = &now
|
||||
t.Version++
|
||||
return true
|
||||
}
|
||||
|
||||
// ClaimedTask is the worker-facing projection of a leased task. Input is either
|
||||
// an external URI or a coordinator-stored shard (InputArtifactID set); the
|
||||
// transport turns the latter into a coordinator download URL.
|
||||
|
||||
@@ -168,6 +168,23 @@ func TestExpireLeaseIgnoresUnleasedTasks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelInvalidatesLeaseButPreservesTerminalTask(t *testing.T) {
|
||||
task := leasedTask(1, 3)
|
||||
if !task.Cancel(testNow) {
|
||||
t.Fatal("leased task should be cancelled")
|
||||
}
|
||||
if task.Status != TaskCancelled || task.LeaseOwner != nil || task.LeaseExpiresAt != nil {
|
||||
t.Errorf("cancelled task = %+v", task)
|
||||
}
|
||||
if task.Cancel(testLater) {
|
||||
t.Error("cancelled task must not be changed twice")
|
||||
}
|
||||
completed := &Task{Status: TaskCompleted}
|
||||
if completed.Cancel(testNow) {
|
||||
t.Error("completed task must remain terminal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstHeartbeatMovesLeasedToRunning(t *testing.T) {
|
||||
task := leasedTask(1, 3)
|
||||
until := testLater.Add(time.Hour)
|
||||
|
||||
@@ -148,6 +148,18 @@ func (r *TaskRepo) CountByStatus(ctx context.Context, jobID uuid.UUID) (map[doma
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
func (r *TaskRepo) CancelByJob(_ context.Context, jobID uuid.UUID, now time.Time) (int64, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var cancelled int64
|
||||
for _, task := range r.tasks {
|
||||
if task.JobID == jobID && task.Cancel(now) {
|
||||
cancelled++
|
||||
}
|
||||
}
|
||||
return cancelled, nil
|
||||
}
|
||||
|
||||
func (r *TaskRepo) ExpireLeases(ctx context.Context, now time.Time) (int64, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -206,6 +206,27 @@ func TestClaimNextReturnsNilOnEmptyQueue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelJobCancelsEveryUnfinishedTask(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
job, _ := seedJob(t, pool, 3)
|
||||
clk := fixedClock{now: time.Now().UTC()}
|
||||
uc := usecase.NewCancelJob(NewJobRepo(pool), NewTaskRepo(pool), NewTxManager(pool), clk)
|
||||
|
||||
cancelled, err := uc.Execute(ctx, job.ID)
|
||||
if err != nil || cancelled != 3 {
|
||||
t.Fatalf("cancel = (%d, %v), want (3, nil)", cancelled, err)
|
||||
}
|
||||
stored, err := NewJobRepo(pool).Get(ctx, job.ID)
|
||||
if err != nil || stored.Status != domain.JobCancelled {
|
||||
t.Fatalf("job after cancel = (%+v, %v)", stored, err)
|
||||
}
|
||||
counts, err := NewTaskRepo(pool).CountByStatus(ctx, job.ID)
|
||||
if err != nil || counts[domain.TaskCancelled] != 3 {
|
||||
t.Fatalf("cancelled tasks = %d, err = %v", counts[domain.TaskCancelled], err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateRejectsStaleVersion(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -295,6 +295,29 @@ func (r *TaskRepo) CountByStatus(ctx context.Context, jobID uuid.UUID) (map[doma
|
||||
return counts, rows.Err()
|
||||
}
|
||||
|
||||
// cancelByJobSQL mirrors domain.Task.Cancel in one set-based update. It runs in
|
||||
// the same transaction as the job-status update, so no claimable shard remains
|
||||
// after an operator receives a successful cancellation response.
|
||||
const cancelByJobSQL = `
|
||||
UPDATE tasks
|
||||
SET status = 'cancelled'::task_status,
|
||||
lease_owner = NULL,
|
||||
lease_expires_at = NULL,
|
||||
error_code = NULL,
|
||||
error_message = NULL,
|
||||
completed_at = $2,
|
||||
version = version + 1
|
||||
WHERE job_id = $1
|
||||
AND status IN ('pending','leased','running')`
|
||||
|
||||
func (r *TaskRepo) CancelByJob(ctx context.Context, jobID uuid.UUID, now time.Time) (int64, error) {
|
||||
tag, err := conn(ctx, r.pool).Exec(ctx, cancelByJobSQL, jobID, now)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// expireLeasesSQL applies the lease-expiry rule set-based, mirroring
|
||||
// domain.Task.ExpireLease: requeue while attempts remain, otherwise fail.
|
||||
//
|
||||
|
||||
@@ -111,13 +111,14 @@ type uploadJobResponse struct {
|
||||
}
|
||||
|
||||
type jobProgressResponse struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Pending int `json:"pending"`
|
||||
Leased int `json:"leased"`
|
||||
Done int `json:"completed"`
|
||||
Failed int `json:"failed"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Pending int `json:"pending"`
|
||||
Leased int `json:"leased"`
|
||||
Done int `json:"completed"`
|
||||
Failed int `json:"failed"`
|
||||
Cancelled int `json:"cancelled"`
|
||||
}
|
||||
|
||||
type uploadArtifactResponse struct {
|
||||
@@ -153,12 +154,13 @@ func toClaimedTaskResponse(c domain.ClaimedTask) claimedTaskResponse {
|
||||
|
||||
func toJobProgressResponse(p domain.JobProgress) jobProgressResponse {
|
||||
return jobProgressResponse{
|
||||
ID: p.Job.ID,
|
||||
Status: string(p.DeriveStatus()),
|
||||
Total: p.Total,
|
||||
Pending: p.Pending,
|
||||
Leased: p.Leased,
|
||||
Done: p.Done,
|
||||
Failed: p.Failed,
|
||||
ID: p.Job.ID,
|
||||
Status: string(p.DeriveStatus()),
|
||||
Total: p.Total,
|
||||
Pending: p.Pending,
|
||||
Leased: p.Leased,
|
||||
Done: p.Done,
|
||||
Failed: p.Failed,
|
||||
Cancelled: p.Cancelled,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,8 @@ func (s *Server) writeError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
case errors.Is(err, domain.ErrLeaseConflict),
|
||||
errors.Is(err, domain.ErrStaleAttempt),
|
||||
errors.Is(err, domain.ErrResultConflict),
|
||||
errors.Is(err, domain.ErrTaskNotLeased):
|
||||
errors.Is(err, domain.ErrTaskNotLeased),
|
||||
errors.Is(err, domain.ErrJobNotCancellable):
|
||||
status = http.StatusConflict
|
||||
case errors.Is(err, usecase.ErrNotImplemented):
|
||||
status = http.StatusNotImplemented
|
||||
|
||||
@@ -182,7 +182,7 @@ func (s *Server) handleFailure(w http.ResponseWriter, r *http.Request) {
|
||||
const defaultChunkRows = 1000
|
||||
|
||||
// handleUploadDataset accepts a multipart submission — the dataset file plus the
|
||||
// workload/parameters/chunk_rows fields — and hands the file, streamed, to the
|
||||
// workload/parameters/chunk_rows/max_rows fields — and hands the file, streamed, to the
|
||||
// chunker. The text fields MUST precede the file part: the file is streamed, not
|
||||
// buffered, so by the time it arrives the other fields are already parsed.
|
||||
func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -197,11 +197,13 @@ func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
|
||||
workload string
|
||||
params map[string]any
|
||||
rows = defaultChunkRows
|
||||
maxRows int
|
||||
result usecase.SubmitDatasetResult
|
||||
gotDataset bool
|
||||
gotWorkload bool
|
||||
gotParams bool
|
||||
gotRows bool
|
||||
gotMaxRows bool
|
||||
)
|
||||
|
||||
for {
|
||||
@@ -249,6 +251,19 @@ func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
rows = n
|
||||
gotRows = true
|
||||
case "max_rows":
|
||||
if gotDataset || gotMaxRows {
|
||||
s.writeError(w, r, domain.ErrInvalidInput)
|
||||
return
|
||||
}
|
||||
b, _ := io.ReadAll(io.LimitReader(part, 32))
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(b)))
|
||||
if err != nil || n < 1 {
|
||||
s.writeError(w, r, domain.ErrInvalidInput)
|
||||
return
|
||||
}
|
||||
maxRows = n
|
||||
gotMaxRows = true
|
||||
case "file", "dataset":
|
||||
if gotDataset || workload == "" {
|
||||
s.writeError(w, r, domain.ErrInvalidInput)
|
||||
@@ -262,6 +277,7 @@ func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
|
||||
Workload: workload,
|
||||
Parameters: params,
|
||||
RowsPerShard: rows,
|
||||
MaxRows: maxRows,
|
||||
Filename: filename,
|
||||
ContentType: part.Header.Get("Content-Type"),
|
||||
Body: part,
|
||||
@@ -379,6 +395,27 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, toJobProgressResponse(progress))
|
||||
}
|
||||
|
||||
// handleCancelJob stops all non-terminal shards for an operator-requested job.
|
||||
// It is available to both the bearer API and the separately authenticated UI.
|
||||
func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := s.reqCtx(r)
|
||||
defer cancel()
|
||||
jobID, ok := s.pathUUID(w, r, "job_id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
cancelled, err := s.uc.CancelJob.Execute(ctx, jobID)
|
||||
if err != nil {
|
||||
s.writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"job_id": jobID,
|
||||
"status": domain.JobCancelled,
|
||||
"cancelled_tasks": cancelled,
|
||||
})
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func (s *Server) reqCtx(r *http.Request) (context.Context, context.CancelFunc) {
|
||||
|
||||
@@ -24,6 +24,7 @@ type UseCases struct {
|
||||
CompleteTask *usecase.CompleteTask
|
||||
FailTask *usecase.FailTask
|
||||
GetJobStatus *usecase.GetJobStatus
|
||||
CancelJob *usecase.CancelJob
|
||||
UploadArtifact *usecase.UploadArtifact
|
||||
DownloadArtifact *usecase.DownloadArtifact
|
||||
GetTaskInput *usecase.GetTaskInput
|
||||
@@ -61,6 +62,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
|
||||
protected.HandleFunc("POST /jobs", s.handleCreateJob)
|
||||
protected.HandleFunc("POST /jobs/upload", s.handleUploadDataset)
|
||||
protected.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
|
||||
protected.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
|
||||
protected.HandleFunc("POST /tasks/claim", s.handleClaim)
|
||||
protected.HandleFunc("GET /tasks/{task_id}/input", s.handleGetTaskInput)
|
||||
protected.HandleFunc("POST /tasks/{task_id}/heartbeat", s.handleHeartbeat)
|
||||
@@ -77,6 +79,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
|
||||
ui.HandleFunc("GET /ui/jobs/new", s.handleUINewJob)
|
||||
ui.HandleFunc("GET /ui/jobs/{job_id}", s.handleUIJob)
|
||||
ui.HandleFunc("GET /ui/api/jobs/{job_id}", s.handleUIJobJSON)
|
||||
ui.HandleFunc("POST /ui/api/jobs/{job_id}/cancel", s.handleCancelJob)
|
||||
ui.HandleFunc("POST /ui/api/jobs/upload", s.handleUploadDataset)
|
||||
ui.HandleFunc("GET /ui/jobs/{job_id}/artifacts/{artifact_id}", s.handleUIArtifactDownload)
|
||||
mux.Handle("/ui", chain(ui, withRequestID, withAccessLog(s.log), withBasicAuth(uiToken[0]), withSameOrigin))
|
||||
|
||||
@@ -51,6 +51,7 @@ func newEnvWithUIToken(t *testing.T, ready func(context.Context) error, configur
|
||||
CompleteTask: usecase.NewCompleteTask(tasks, jobs, arts, tx, clk),
|
||||
FailTask: usecase.NewFailTask(tasks, jobs, tx, clk),
|
||||
GetJobStatus: usecase.NewGetJobStatus(jobs, tasks),
|
||||
CancelJob: usecase.NewCancelJob(jobs, tasks, tx, clk),
|
||||
UploadArtifact: usecase.NewUploadArtifact(tasks, arts, blobs, clk),
|
||||
DownloadArtifact: usecase.NewDownloadArtifact(arts, blobs),
|
||||
GetTaskInput: usecase.NewGetTaskInput(tasks, arts, blobs),
|
||||
@@ -200,6 +201,39 @@ func TestUIUploadDatasetCreatesJob(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelJobStopsUnfinishedTasks(t *testing.T) {
|
||||
e := newEnv(t, healthy)
|
||||
code, job := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c0","input_sha256":"sha"},{"chunk_index":1,"input_uri":"s3://c1","input_sha256":"sha"}]}`)
|
||||
if code != http.StatusCreated {
|
||||
t.Fatalf("create: %d", code)
|
||||
}
|
||||
jobID := job["id"].(string)
|
||||
if code, body := e.do(t, "POST", "/jobs/"+jobID+"/cancel", ""); code != http.StatusOK || body["cancelled_tasks"].(float64) != 2 {
|
||||
t.Fatalf("cancel = (%d, %v)", code, body)
|
||||
}
|
||||
if code, progress := e.do(t, "GET", "/jobs/"+jobID, ""); code != http.StatusOK || progress["status"] != "cancelled" || progress["cancelled"].(float64) != 2 {
|
||||
t.Fatalf("cancelled job progress = (%d, %v)", code, progress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUICancelJobUsesOperatorCredential(t *testing.T) {
|
||||
e := newEnv(t, healthy)
|
||||
code, job := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c0","input_sha256":"sha"}]}`)
|
||||
if code != http.StatusCreated {
|
||||
t.Fatalf("create: %d", code)
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(context.Background(), "POST", e.ts.URL+"/ui/api/jobs/"+job["id"].(string)+"/cancel", nil)
|
||||
req.SetBasicAuth("operator", uiToken)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("UI cancel = %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIJobAndArtifactAreScopedToTheirJob(t *testing.T) {
|
||||
e := newEnv(t, healthy)
|
||||
code, job := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c","input_sha256":"sha"}]}`)
|
||||
@@ -392,6 +426,31 @@ func TestUploadDatasetChunksAndServesInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadDatasetLimitsRows(t *testing.T) {
|
||||
e := newEnv(t, healthy)
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
_ = mw.WriteField("workload", "w")
|
||||
_ = mw.WriteField("chunk_rows", "2")
|
||||
_ = mw.WriteField("max_rows", "3")
|
||||
fw, _ := mw.CreateFormFile("file", "chembl.tsv")
|
||||
_, _ = io.Copy(fw, strings.NewReader("id\tsmiles\nA\tCC\nB\tCCC\nC\tCCCC\nD\tCCCCC\n"))
|
||||
_ = mw.Close()
|
||||
req, _ := http.NewRequestWithContext(context.Background(), "POST", e.ts.URL+"/jobs/upload", &buf)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var result map[string]any
|
||||
_ = json.NewDecoder(resp.Body).Decode(&result)
|
||||
if resp.StatusCode != http.StatusCreated || result["task_count"].(float64) != 2 {
|
||||
t.Fatalf("limited upload = (%d, %v)", resp.StatusCode, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorMappings(t *testing.T) {
|
||||
e := newEnv(t, healthy)
|
||||
zero := "00000000-0000-0000-0000-000000000000"
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<header class="top"><div><p class="eyebrow">Local coordinator</p><h1>SciMesh operator dashboard</h1><p class="lead">See where a computation is and what should happen next.</p></div><a class="button" href="/ui/jobs/new">Start a check</a></header>
|
||||
<section class="notice" aria-label="Current pipeline limitation"><strong>This screen currently diagnoses shard jobs.</strong><span>Workers upload partial CSVs to the coordinator. Until a reducer is implemented, those files are not one final scientific result.</span><div class="steps"><div class="step"><b>1. Upload TSV</b>The coordinator splits the file into shard tasks.</div><div class="step"><b>2. Wait for a worker</b>A worker claims a shard, calculates similarity, and returns a CSV.</div><div class="step"><b>3. Inspect artifacts</b>Download a partial result from the job page.</div></div></section>
|
||||
<h2>Recent jobs</h2>
|
||||
<div class="table-wrap"><table><tr><th>Computation</th><th>State</th><th>Progress</th><th class="hide-mobile">Created</th></tr>{{range .Jobs}}<tr><td><a href="/ui/jobs/{{.ID}}"><strong>{{workloadLabel .Workload}}</strong></a><br><small>Open job details</small></td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><br><small>{{statusHint .Status}}</small></td><td class="kicker"><strong>{{.Completed}} / {{.Total}}</strong> complete{{if gt .Failed 0}} · <span style="color:#a31135">failed: {{.Failed}}</span>{{end}}<div class="bar"><span style="width:{{progressPercent .Completed .Failed .Total}}%"></span></div></td><td class="hide-mobile"><small>{{time .CreatedAt}}</small></td></tr>{{else}}<tr><td colspan="4" class="empty"><strong>No jobs yet.</strong><br>Click “Start a check”, upload a small TSV, and leave a worker running.</td></tr>{{end}}</table></div>
|
||||
<div class="table-wrap"><table><tr><th>Computation</th><th>State</th><th>Progress</th><th class="hide-mobile">Created</th></tr>{{range .Jobs}}<tr><td><a href="/ui/jobs/{{.ID}}"><strong>{{workloadLabel .Workload}}</strong></a><br><small>Open job details</small></td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><br><small>{{statusHint .Status}}</small></td><td class="kicker"><strong>{{.Completed}} / {{.Total}}</strong> complete{{if gt .Failed 0}} · <span style="color:#a31135">failed: {{.Failed}}</span>{{end}}{{if gt .Cancelled 0}} · <span>stopped: {{.Cancelled}}</span>{{end}}<div class="bar"><span style="width:{{progressPercent .Completed .Failed .Cancelled .Total}}%"></span></div></td><td class="hide-mobile"><small>{{time .CreatedAt}}</small></td></tr>{{else}}<tr><td colspan="4" class="empty"><strong>No jobs yet.</strong><br>Click “Start a check”, upload a small TSV, and leave a worker running.</td></tr>{{end}}</table></div>
|
||||
<h2>Workers</h2>
|
||||
<section class="card">{{range .Workers}}<div class="worker"><div><strong>{{.Name}}</strong><br><small>{{.ID}}</small></div><div><span class="status status-{{if eq .Status "online"}}success{{else}}waiting{{end}}">{{workerStatusLabel .Status}}</span></div><div>{{range .Capabilities}}<code>{{.}}</code> {{end}}</div><div class="muted">Last signal<br>{{time .LastHeartbeatAt}}</div></div>{{else}}<div class="empty"><strong>No worker is registered yet.</strong><br>Run <code>scimesh-worker</code> with the coordinator URL and worker token.</div>{{end}}</section>
|
||||
</main>
|
||||
|
||||
@@ -6,13 +6,13 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>SciMesh job</title>
|
||||
<style>
|
||||
:root{color:#172033;background:#f6f8fc;font:16px/1.5 system-ui,sans-serif}body{margin:0}.page{max-width:1180px;margin:auto;padding:32px 20px 56px}a{color:#174ecf}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#50617d;font-size:.86rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em}h1{margin:0;font-size:2rem}h2{margin:32px 0 12px;font-size:1.3rem}.summary,.card{padding:20px;border:1px solid #dfe5f0;border-radius:12px;background:#fff}.summary-head{display:flex;justify-content:space-between;gap:16px;align-items:start}.status{display:inline-block;border-radius:999px;padding:4px 10px;font-size:.9rem;font-weight:700}.status-success{background:#dff6e9;color:#126b3d}.status-danger{background:#ffe4e6;color:#a31135}.status-active{background:#e4edff;color:#174ecf}.status-waiting{background:#edf0f5;color:#50617d}.hint{margin:8px 0 0;color:#56657c}.bar{height:10px;margin:20px 0 8px;overflow:hidden;border-radius:999px;background:#e6eaf1}.bar>span{display:block;height:100%;background:#1f5eff;transition:width .3s}.numbers{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}.number{padding:12px;border-radius:8px;background:#f6f8fc}.number b{display:block;font-size:1.35rem}.notice{margin:20px 0;padding:15px 17px;border:1px solid #f2cb72;border-radius:10px;background:#fff8e6}.table-wrap{overflow-x:auto;border:1px solid #dfe5f0;border-radius:10px;background:#fff}table{width:100%;border-collapse:collapse}td,th{padding:12px 13px;border-bottom:1px solid #e8ecf4;text-align:left;vertical-align:top}th{color:#50617d;font-size:.78rem;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small,.muted{color:#68758b}.error{color:#a31135;max-width:360px;word-break:break-word}.download{display:inline-block;padding:7px 10px;border:1px solid #b9c9ee;border-radius:7px;text-decoration:none}.empty{padding:24px;text-align:center;color:#68758b}details{margin-top:18px;color:#56657c}code{word-break:break-all}@media(max-width:700px){.summary-head{display:block}.numbers{grid-template-columns:repeat(2,1fr)}}
|
||||
:root{color:#172033;background:#f6f8fc;font:16px/1.5 system-ui,sans-serif}body{margin:0}.page{max-width:1180px;margin:auto;padding:32px 20px 56px}a{color:#174ecf}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#50617d;font-size:.86rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em}h1{margin:0;font-size:2rem}h2{margin:32px 0 12px;font-size:1.3rem}.summary,.card{padding:20px;border:1px solid #dfe5f0;border-radius:12px;background:#fff}.summary-head{display:flex;justify-content:space-between;gap:16px;align-items:start}.status{display:inline-block;border-radius:999px;padding:4px 10px;font-size:.9rem;font-weight:700}.status-success{background:#dff6e9;color:#126b3d}.status-danger{background:#ffe4e6;color:#a31135}.status-active{background:#e4edff;color:#174ecf}.status-waiting{background:#edf0f5;color:#50617d}.hint{margin:8px 0 0;color:#56657c}.bar{height:10px;margin:20px 0 8px;overflow:hidden;border-radius:999px;background:#e6eaf1}.bar>span{display:block;height:100%;background:#1f5eff;transition:width .3s}.numbers{display:grid;grid-template-columns:repeat(6,1fr);gap:10px}.number{padding:12px;border-radius:8px;background:#f6f8fc}.number b{display:block;font-size:1.35rem}.stop{display:block;margin-left:auto;border:1px solid #d43b51;border-radius:7px;padding:8px 11px;background:#fff;color:#b2223a;font:inherit;font-weight:700;cursor:pointer}.stop:disabled{opacity:.6}.notice{margin:20px 0;padding:15px 17px;border:1px solid #f2cb72;border-radius:10px;background:#fff8e6}.table-wrap{overflow-x:auto;border:1px solid #dfe5f0;border-radius:10px;background:#fff}table{width:100%;border-collapse:collapse}td,th{padding:12px 13px;border-bottom:1px solid #e8ecf4;text-align:left;vertical-align:top}th{color:#50617d;font-size:.78rem;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small,.muted{color:#68758b}.error{color:#a31135;max-width:360px;word-break:break-word}.download{display:inline-block;padding:7px 10px;border:1px solid #b9c9ee;border-radius:7px;text-decoration:none}.empty{padding:24px;text-align:center;color:#68758b}details{margin-top:18px;color:#56657c}code{word-break:break-all}@media(max-width:700px){.summary-head{display:block}.numbers{grid-template-columns:repeat(2,1fr)}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="page">
|
||||
<a class="back" href="/ui">← Back to jobs</a><p class="eyebrow">{{workloadLabel .Workload}}</p><h1>Execution progress</h1>
|
||||
<section class="summary"><div class="summary-head"><div><span id="status" class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><p id="hint" class="hint">{{statusHint .Status}}</p></div><small>Summary refreshes automatically every two seconds.</small></div><div class="bar" aria-label="Progress"><span id="progress-bar" style="width:{{progressPercent .Completed .Failed .Total}}%"></span></div><p id="progress" class="muted">{{.Completed}} of {{.Total}} tasks complete</p><div class="numbers"><div class="number"><b id="total">{{.Total}}</b><small>total shards</small></div><div class="number"><b id="completed">{{.Completed}}</b><small>complete</small></div><div class="number"><b id="pending">{{.Pending}}</b><small>waiting</small></div><div class="number"><b id="active">{{add .Leased .Running}}</b><small>with workers</small></div><div class="number"><b id="failed">{{.Failed}}</b><small>failed</small></div></div><details><summary>Technical details</summary><p>Job ID: <code>{{.ID}}</code><br>Workload: <code>{{.Workload}}</code><br>Created: {{time .CreatedAt}}</p></details></section>
|
||||
<section class="summary"><div class="summary-head"><div><span id="status" class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><p id="hint" class="hint">{{statusHint .Status}}</p></div><div>{{if cancellable .Status}}<button id="stop-job" class="stop" type="button">Stop job</button><small>This cancels every shard that is not finished yet.</small>{{else}}<small>Summary refreshes automatically every two seconds.</small>{{end}}</div></div><div class="bar" aria-label="Progress"><span id="progress-bar" style="width:{{progressPercent .Completed .Failed .Cancelled .Total}}%"></span></div><p id="progress" class="muted">{{.Completed}} of {{.Total}} tasks complete</p><div class="numbers"><div class="number"><b id="total">{{.Total}}</b><small>total shards</small></div><div class="number"><b id="completed">{{.Completed}}</b><small>complete</small></div><div class="number"><b id="pending">{{.Pending}}</b><small>waiting</small></div><div class="number"><b id="active">{{add .Leased .Running}}</b><small>with workers</small></div><div class="number"><b id="failed">{{.Failed}}</b><small>failed</small></div><div class="number"><b id="cancelled">{{.Cancelled}}</b><small>stopped</small></div></div><details><summary>Technical details</summary><p>Job ID: <code>{{.ID}}</code><br>Workload: <code>{{.Workload}}</code><br>Created: {{time .CreatedAt}}</p></details></section>
|
||||
<section class="notice"><strong>What can be downloaded now?</strong><br><code>partial_result</code> files come from individual shards. They are useful for checking the pipeline, but are not a merged final CSV because the reducer is not implemented yet.</section>
|
||||
<h2>Shard tasks</h2><p class="muted">If a task fails, its code and message appear here. Refresh the page to update the detailed rows.</p>
|
||||
<div class="table-wrap"><table><tr><th>Shard</th><th>State</th><th>Attempt</th><th>Worker / lease</th><th>Error</th></tr>{{range .Tasks}}<tr><td>#{{.ChunkIndex}}</td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span></td><td>{{.Attempt}} / {{.MaxAttempts}}</td><td>{{if .LeaseOwner}}<code>{{.LeaseOwner}}</code>{{if .LeaseExpiresAt}}<br><small>until {{time .LeaseExpiresAt}}</small>{{end}}{{else}}<span class="muted">—</span>{{end}}</td><td class="error">{{if .ErrorCode}}<strong>{{taskErrorLabel .ErrorCode}}</strong><br><small>{{taskErrorHint .ErrorCode}}</small>{{else}}<span class="muted">—</span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No tasks have appeared yet.</td></tr>{{end}}</table></div>
|
||||
@@ -20,8 +20,9 @@
|
||||
<div class="table-wrap"><table><tr><th>Type</th><th>File</th><th>Size</th><th>Integrity check</th><th></th></tr>{{range .Artifacts}}<tr><td>{{if .Diagnostic}}<strong>Partial result</strong><br><small>diagnostic</small>{{else}}{{.Kind}}{{end}}</td><td>{{.Filename}}</td><td>{{bytes .SizeBytes}}</td><td><code>{{.SHA256}}</code></td><td>{{if .Downloadable}}<a class="download" href="/ui/jobs/{{$.ID}}/artifacts/{{.ID}}">Download CSV</a>{{else}}<span class="muted">Unavailable</span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No artifacts yet. The worker uploads a CSV after it completes a shard.</td></tr>{{end}}</table></div>
|
||||
</main>
|
||||
<script>
|
||||
const id={{printf "%q" .ID}},state={pending:['Waiting for a worker','waiting','Waiting for an available worker with the required capability.'],leased:['Assigned to a worker','active','A worker has claimed the task and should begin processing shortly.'],running:['Running','active','A worker is reading a shard, calculating fingerprints, and uploading its result through the coordinator.'],completed:['Tasks complete','success','Every shard task is complete. Files below are still partial results.'],failed:['Needs attention','danger','One or more shard tasks failed. Open the task list below for details.']};
|
||||
setInterval(async()=>{try{const response=await fetch('/ui/api/jobs/'+id);if(!response.ok)return;const job=await response.json(),info=state[job.status]||[job.status,'waiting','Status reported by the coordinator.'],done=job.completed+job.failed,percent=job.total?Math.min(100,Math.floor(done*100/job.total)):0,badge=document.querySelector('#status');badge.textContent=info[0];badge.className='status status-'+info[1];document.querySelector('#hint').textContent=info[2];document.querySelector('#progress').textContent=job.completed+' of '+job.total+' tasks complete'+(job.failed?' · failed: '+job.failed:'');document.querySelector('#progress-bar').style.width=percent+'%';for(const key of ['total','completed','pending','failed'])document.querySelector('#'+key).textContent=job[key];document.querySelector('#active').textContent=job.leased+job.running}catch(_){}} ,2000);
|
||||
const id={{printf "%q" .ID}},state={pending:['Waiting for a worker','waiting','Waiting for an available worker with the required capability.'],leased:['Assigned to a worker','active','A worker has claimed the task and should begin processing shortly.'],running:['Running','active','A worker is reading a shard, calculating fingerprints, and uploading its result through the coordinator.'],completed:['Tasks complete','success','Every shard task is complete. Files below are still partial results.'],failed:['Needs attention','danger','One or more shard tasks failed. Open the task list below for details.'],cancelled:['Stopped','waiting','The operator stopped this job. No new shards can be claimed.']};
|
||||
const stop=document.querySelector('#stop-job');if(stop)stop.addEventListener('click',async()=>{if(!confirm('Stop this job? Unfinished shards will be cancelled.'))return;stop.disabled=true;const response=await fetch('/ui/api/jobs/'+id+'/cancel',{method:'POST'});if(!response.ok){stop.disabled=false;alert('Unable to stop this job.');return}location.reload()});
|
||||
setInterval(async()=>{try{const response=await fetch('/ui/api/jobs/'+id);if(!response.ok)return;const job=await response.json(),info=state[job.status]||[job.status,'waiting','Status reported by the coordinator.'],done=job.completed+job.failed+job.cancelled,percent=job.total?Math.min(100,Math.floor(done*100/job.total)):0,badge=document.querySelector('#status');badge.textContent=info[0];badge.className='status status-'+info[1];document.querySelector('#hint').textContent=info[2];document.querySelector('#progress').textContent=job.completed+' of '+job.total+' tasks complete'+(job.failed?' · failed: '+job.failed:'')+(job.cancelled?' · stopped: '+job.cancelled:'');document.querySelector('#progress-bar').style.width=percent+'%';for(const key of ['total','completed','pending','failed','cancelled'])document.querySelector('#'+key).textContent=job[key];document.querySelector('#active').textContent=job.leased+job.running}catch(_){}} ,2000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -18,12 +18,13 @@
|
||||
<label for="query-smiles">Target molecule (SMILES)</label><input id="query-smiles" name="query_smiles" required maxlength="200" value="CCO" autocomplete="off"><p class="hint"><code>CCO</code> is ethanol. For gefitinib, use its SMILES here or the local CLI with <code>--query-id</code>.</p>
|
||||
<label for="top-k">Matches to return</label><input id="top-k" name="top_k" type="number" min="1" max="100000" value="20" required><p class="hint">This is the top-k within each shard, not a global top-k for the whole dataset yet.</p>
|
||||
<label for="chunk-rows">Rows per shard</label><input id="chunk-rows" name="chunk_rows" type="number" min="1" max="100000" value="1000" required><p class="hint">Fewer rows mean more tasks and more visible progress; more rows mean fewer, longer tasks.</p>
|
||||
<label for="max-rows">Maximum dataset rows to process <small>(optional)</small></label><input id="max-rows" name="max_rows" type="number" min="1" max="10000000" placeholder="For example: 500"><p class="hint">Useful for a quick check of a large TSV. The coordinator creates shards from only the first N data rows; it still stores the original upload.</p>
|
||||
<button class="button" id="submit" type="submit">Upload file and create job</button><p id="working" class="working" hidden aria-live="polite">Uploading the file and creating shard tasks… Keep this page open.</p><p id="error" class="error" role="alert"></p>
|
||||
</form>
|
||||
</main>
|
||||
<script>
|
||||
const form=document.querySelector('#run'),button=document.querySelector('#submit'),working=document.querySelector('#working'),error=document.querySelector('#error');
|
||||
form.addEventListener('submit',async event=>{event.preventDefault();error.textContent='';const fields=new FormData(form),file=fields.get('file');if(!(file instanceof File)||file.size===0){error.textContent='Choose a non-empty TSV file.';return}const parameters={query_smiles:fields.get('query_smiles'),top_k:Number(fields.get('top_k')),progress_every:0},upload=new FormData();upload.append('workload','similarity-search');upload.append('parameters',JSON.stringify(parameters));upload.append('chunk_rows',fields.get('chunk_rows'));upload.append('file',file,file.name);button.disabled=true;working.hidden=false;try{const response=await fetch('/ui/api/jobs/upload',{method:'POST',body:upload}),data=await response.json();if(!response.ok)throw Error(data.error||'Unable to create the job.');location.href='/ui/jobs/'+data.job_id}catch(err){error.textContent=err.message==='invalid input'?'Check the TSV and fields: the coordinator could not accept this request.':err.message;button.disabled=false;working.hidden=true}});
|
||||
form.addEventListener('submit',async event=>{event.preventDefault();error.textContent='';const fields=new FormData(form),file=fields.get('file'),maxRows=String(fields.get('max_rows')||'').trim();if(!(file instanceof File)||file.size===0){error.textContent='Choose a non-empty TSV file.';return}const parameters={query_smiles:fields.get('query_smiles'),top_k:Number(fields.get('top_k')),progress_every:0},upload=new FormData();upload.append('workload','similarity-search');upload.append('parameters',JSON.stringify(parameters));upload.append('chunk_rows',fields.get('chunk_rows'));if(maxRows)upload.append('max_rows',maxRows);upload.append('file',file,file.name);button.disabled=true;working.hidden=false;try{const response=await fetch('/ui/api/jobs/upload',{method:'POST',body:upload}),data=await response.json();if(!response.ok)throw Error(data.error||'Unable to create the job.');location.href='/ui/jobs/'+data.job_id}catch(err){error.textContent=err.message==='invalid input'?'Check the TSV and fields: the coordinator could not accept this request.':err.message;button.disabled=false;working.hidden=true}});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -28,6 +28,7 @@ var uiTemplates = template.Must(template.New("ui").Funcs(template.FuncMap{
|
||||
"workerStatusLabel": uiWorkerStatusLabel,
|
||||
"workloadLabel": uiWorkloadLabel,
|
||||
"progressPercent": uiProgressPercent,
|
||||
"cancellable": uiCancellable,
|
||||
"bytes": uiBytes,
|
||||
"add": func(a, b int) int { return a + b },
|
||||
}).ParseFS(uiFiles, "templates/*.html"))
|
||||
@@ -51,6 +52,8 @@ func uiStatusLabel(status string) string {
|
||||
return "Tasks complete"
|
||||
case "failed":
|
||||
return "Needs attention"
|
||||
case "cancelled":
|
||||
return "Stopped"
|
||||
default:
|
||||
return status
|
||||
}
|
||||
@@ -68,6 +71,8 @@ func uiStatusHint(status string) string {
|
||||
return "Every shard task is complete. Files below are still partial results."
|
||||
case "failed":
|
||||
return "One or more shard tasks failed. Open the task list below for details."
|
||||
case "cancelled":
|
||||
return "The operator stopped this job. No new shards can be claimed."
|
||||
default:
|
||||
return "Status reported by the coordinator."
|
||||
}
|
||||
@@ -79,6 +84,8 @@ func uiStatusClass(status string) string {
|
||||
return "success"
|
||||
case "failed":
|
||||
return "danger"
|
||||
case "cancelled":
|
||||
return "waiting"
|
||||
case "running", "leased":
|
||||
return "active"
|
||||
default:
|
||||
@@ -145,11 +152,15 @@ func uiWorkloadLabel(workload string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func uiProgressPercent(completed, failed, total int) int {
|
||||
func uiCancellable(status string) bool {
|
||||
return status == "pending" || status == "running"
|
||||
}
|
||||
|
||||
func uiProgressPercent(completed, failed, cancelled, total int) int {
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
percent := (completed + failed) * 100 / total
|
||||
percent := (completed + failed + cancelled) * 100 / total
|
||||
if percent > 100 {
|
||||
return 100
|
||||
}
|
||||
|
||||
@@ -26,10 +26,10 @@ func TestUIStatusPresentation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUIProgressPercent(t *testing.T) {
|
||||
if got := uiProgressPercent(3, 1, 8); got != 50 {
|
||||
if got := uiProgressPercent(3, 1, 0, 8); got != 50 {
|
||||
t.Errorf("progress = %d, want 50", got)
|
||||
}
|
||||
if got := uiProgressPercent(1, 1, 0); got != 0 {
|
||||
if got := uiProgressPercent(1, 1, 0, 0); got != 0 {
|
||||
t.Errorf("empty progress = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,9 +53,12 @@ type SubmitDatasetInput struct {
|
||||
Workload string
|
||||
Parameters map[string]any
|
||||
RowsPerShard int
|
||||
Filename string
|
||||
ContentType string
|
||||
Body io.Reader
|
||||
// MaxRows limits how many data rows are turned into shards. Zero means the
|
||||
// whole uploaded dataset; the input artifact itself remains stored intact.
|
||||
MaxRows int
|
||||
Filename string
|
||||
ContentType string
|
||||
Body io.Reader
|
||||
}
|
||||
|
||||
type SubmitDatasetResult struct {
|
||||
|
||||
@@ -62,6 +62,45 @@ type GetJobStatus struct {
|
||||
tasks TaskRepository
|
||||
}
|
||||
|
||||
// --- CancelJob -----------------------------------------------------------
|
||||
|
||||
type CancelJob struct {
|
||||
jobs JobRepository
|
||||
tasks TaskRepository
|
||||
tx TxManager
|
||||
clock Clock
|
||||
}
|
||||
|
||||
func NewCancelJob(jobs JobRepository, tasks TaskRepository, tx TxManager, clock Clock) *CancelJob {
|
||||
return &CancelJob{jobs: jobs, tasks: tasks, tx: tx, clock: clock}
|
||||
}
|
||||
|
||||
// Execute stops a job atomically. Completed and finally failed tasks are kept
|
||||
// as historical evidence; all other tasks are cancelled, including leased and
|
||||
// running ones. A repeated cancel of an already cancelled job is idempotent.
|
||||
func (uc *CancelJob) Execute(ctx context.Context, jobID uuid.UUID) (int64, error) {
|
||||
now := uc.clock.Now()
|
||||
var cancelled int64
|
||||
err := uc.tx.WithinTx(ctx, func(ctx context.Context) error {
|
||||
job, err := uc.jobs.Get(ctx, jobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if job.Status == domain.JobCancelled {
|
||||
return nil
|
||||
}
|
||||
if job.Status == domain.JobCompleted || job.Status == domain.JobFailed {
|
||||
return domain.ErrJobNotCancellable
|
||||
}
|
||||
cancelled, err = uc.tasks.CancelByJob(ctx, jobID, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return uc.jobs.UpdateStatus(ctx, jobID, domain.JobCancelled, &now)
|
||||
})
|
||||
return cancelled, err
|
||||
}
|
||||
|
||||
func NewGetJobStatus(jobs JobRepository, tasks TaskRepository) *GetJobStatus {
|
||||
return &GetJobStatus{jobs: jobs, tasks: tasks}
|
||||
}
|
||||
@@ -144,9 +183,10 @@ func progressFrom(job domain.Job, counts map[domain.TaskStatus]int) domain.JobPr
|
||||
Job: job,
|
||||
Pending: counts[domain.TaskPending],
|
||||
// Leased and running are both "in flight" for progress purposes.
|
||||
Leased: counts[domain.TaskLeased] + counts[domain.TaskRunning],
|
||||
Done: counts[domain.TaskCompleted],
|
||||
Failed: counts[domain.TaskFailed],
|
||||
Leased: counts[domain.TaskLeased] + counts[domain.TaskRunning],
|
||||
Done: counts[domain.TaskCompleted],
|
||||
Failed: counts[domain.TaskFailed],
|
||||
Cancelled: counts[domain.TaskCancelled],
|
||||
}
|
||||
for _, n := range counts {
|
||||
p.Total += n
|
||||
|
||||
@@ -56,6 +56,10 @@ type TaskRepository interface {
|
||||
// CountByStatus aggregates a job's tasks for progress reporting.
|
||||
CountByStatus(ctx context.Context, jobID uuid.UUID) (map[domain.TaskStatus]int, error)
|
||||
|
||||
// CancelByJob marks every non-terminal task as cancelled and invalidates its
|
||||
// lease. It returns how many tasks changed.
|
||||
CancelByJob(ctx context.Context, jobID uuid.UUID, now time.Time) (int64, error)
|
||||
|
||||
// ExpireLeases applies the lease-expiry rule to every elapsed task and
|
||||
// reports how many were affected.
|
||||
ExpireLeases(ctx context.Context, now time.Time) (int64, error)
|
||||
|
||||
@@ -30,6 +30,7 @@ type JobCard struct {
|
||||
Running int `json:"running"`
|
||||
Completed int `json:"completed"`
|
||||
Failed int `json:"failed"`
|
||||
Cancelled int `json:"cancelled"`
|
||||
}
|
||||
|
||||
type TaskCard struct {
|
||||
@@ -166,9 +167,11 @@ func jobCard(job domain.Job, tasks []domain.Task) JobCard {
|
||||
c.Completed++
|
||||
case domain.TaskFailed:
|
||||
c.Failed++
|
||||
case domain.TaskCancelled:
|
||||
c.Cancelled++
|
||||
}
|
||||
}
|
||||
p := domain.JobProgress{Job: job, Total: c.Total, Pending: c.Pending, Leased: c.Leased + c.Running, Done: c.Completed, Failed: c.Failed}
|
||||
p := domain.JobProgress{Job: job, Total: c.Total, Pending: c.Pending, Leased: c.Leased + c.Running, Done: c.Completed, Failed: c.Failed, Cancelled: c.Cancelled}
|
||||
c.Status = string(p.DeriveStatus())
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func (uc *SubmitDataset) Execute(ctx context.Context, in SubmitDatasetInput) (Su
|
||||
cleanup()
|
||||
return SubmitDatasetResult{}, err
|
||||
}
|
||||
splitErr := chunk.SplitTSV(rc, in.RowsPerShard, func(index int, shard io.Reader) error {
|
||||
splitErr := chunk.SplitTSVLimit(rc, in.RowsPerShard, in.MaxRows, func(index int, shard io.Reader) error {
|
||||
art, err := domain.NewArtifact(job.ID, nil, domain.ArtifactShard,
|
||||
fmt.Sprintf("shard-%d.tsv", index), in.ContentType, now)
|
||||
if err != nil {
|
||||
|
||||
@@ -54,6 +54,7 @@ type harness struct {
|
||||
downloadArt *usecase.DownloadArtifact
|
||||
getInput *usecase.GetTaskInput
|
||||
expire *usecase.ExpireLeases
|
||||
cancel *usecase.CancelJob
|
||||
}
|
||||
|
||||
func newHarness() *harness {
|
||||
@@ -79,6 +80,7 @@ func newHarness() *harness {
|
||||
h.downloadArt = usecase.NewDownloadArtifact(h.arts, h.blobs)
|
||||
h.getInput = usecase.NewGetTaskInput(h.tasks, h.arts, h.blobs)
|
||||
h.expire = usecase.NewExpireLeases(h.tasks, h.clk)
|
||||
h.cancel = usecase.NewCancelJob(h.jobs, h.tasks, tx, h.clk)
|
||||
return h
|
||||
}
|
||||
|
||||
@@ -468,6 +470,44 @@ func TestSubmitDatasetChunksAndServesInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitDatasetLimitsRowsBeforeCreatingShards(t *testing.T) {
|
||||
h := newHarness()
|
||||
tsv := "id\tsmiles\nA\tCC\nB\tCCC\nC\tCCCC\nD\tCCCCC\nE\tCCCCCC\n"
|
||||
res, err := h.submit.Execute(ctx, usecase.SubmitDatasetInput{
|
||||
Workload: "w", RowsPerShard: 2, MaxRows: 3, Filename: "chembl.tsv",
|
||||
ContentType: "text/tab-separated-values", Body: strings.NewReader(tsv),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.TaskCount != 2 {
|
||||
t.Fatalf("task_count = %d, want 2", res.TaskCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelJobInvalidatesClaimedAndPendingTasks(t *testing.T) {
|
||||
h := newHarness()
|
||||
jobID := h.seedJob(t, "w", 3)
|
||||
claimed, err := h.claim.Execute(ctx, usecase.ClaimTaskInput{WorkerID: "w1", Workloads: []string{"w"}})
|
||||
if err != nil || claimed == nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
cancelled, err := h.cancel.Execute(ctx, jobID)
|
||||
if err != nil || cancelled != 3 {
|
||||
t.Fatalf("cancel = (%d, %v), want (3, nil)", cancelled, err)
|
||||
}
|
||||
if _, err := h.renew.Execute(ctx, usecase.RenewLeaseInput{TaskID: claimed.TaskID, WorkerID: "w1", Attempt: claimed.Attempt}); !errors.Is(err, domain.ErrTaskNotLeased) {
|
||||
t.Errorf("cancelled lease heartbeat = %v, want ErrTaskNotLeased", err)
|
||||
}
|
||||
progress, err := h.status.Execute(ctx, jobID)
|
||||
if err != nil || progress.DeriveStatus() != domain.JobCancelled || progress.Cancelled != 3 {
|
||||
t.Errorf("cancelled progress = %+v, err = %v", progress, err)
|
||||
}
|
||||
if cancelled, err := h.cancel.Execute(ctx, jobID); err != nil || cancelled != 0 {
|
||||
t.Errorf("second cancel = (%d, %v), want (0, nil)", cancelled, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTaskInputMissingForURITask(t *testing.T) {
|
||||
h := newHarness()
|
||||
h.seedJob(t, "w", 1) // URI-based task, no coordinator-stored input
|
||||
|
||||
+23
-3
@@ -52,9 +52,11 @@ Content-Type: multipart/form-data
|
||||
```
|
||||
|
||||
Fields, in order (text fields first, file last — the file is streamed):
|
||||
`workload`, `parameters` (JSON), `chunk_rows` (int, default 1000), and the file
|
||||
part `file`. The coordinator stores the input, splits the TSV into shard
|
||||
artifacts (header repeated per shard), and creates one task per shard.
|
||||
`workload`, `parameters` (JSON), `chunk_rows` (int, default 1000), optional
|
||||
`max_rows` (positive int), and the file part `file`. `max_rows` limits the
|
||||
leading data rows that become shards; it does not change the stored source
|
||||
artifact. The coordinator splits the selected TSV rows into shard artifacts
|
||||
(header repeated per shard) and creates one task per shard.
|
||||
|
||||
`201`:
|
||||
|
||||
@@ -65,6 +67,24 @@ artifacts (header repeated per shard), and creates one task per shard.
|
||||
Each resulting task's claim response carries `input.uri = /tasks/{id}/input`,
|
||||
served by §5.4.
|
||||
|
||||
## Stop a job
|
||||
|
||||
```http
|
||||
POST /jobs/{job_id}/cancel
|
||||
Authorization: Bearer <token>
|
||||
```
|
||||
|
||||
The coordinator transactionally marks every pending, leased, or running shard
|
||||
as `cancelled`, invalidates its lease, and marks the job `cancelled`. Completed
|
||||
and terminally failed shards remain as history. Repeating a cancellation of an
|
||||
already cancelled job is safe.
|
||||
|
||||
`200`:
|
||||
|
||||
```json
|
||||
{ "job_id": "uuid", "status": "cancelled", "cancelled_tasks": 12 }
|
||||
```
|
||||
|
||||
## Register worker
|
||||
|
||||
```http
|
||||
|
||||
+32
-2
@@ -95,7 +95,7 @@ paths:
|
||||
summary: Upload a dataset; the coordinator chunks it into shard tasks
|
||||
description: >
|
||||
multipart/form-data. The text fields (`workload`, `parameters`,
|
||||
`chunk_rows`) MUST precede the `file` part: the file is streamed, not
|
||||
`chunk_rows`, `max_rows`) MUST precede the `file` part: the file is streamed, not
|
||||
buffered, so the fields have to be parsed before it arrives.
|
||||
requestBody:
|
||||
required: true
|
||||
@@ -130,6 +130,23 @@ paths:
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"404": { $ref: "#/components/responses/NotFound" }
|
||||
|
||||
/jobs/{job_id}/cancel:
|
||||
post:
|
||||
tags: [jobs]
|
||||
summary: Cancel a job and invalidate all unfinished task leases
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/JobID"
|
||||
responses:
|
||||
"200":
|
||||
description: The job is cancelled. Completed and terminally failed tasks remain unchanged.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/CancelJobResponse" }
|
||||
"400": { $ref: "#/components/responses/BadRequest" }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"404": { $ref: "#/components/responses/NotFound" }
|
||||
"409": { $ref: "#/components/responses/Conflict" }
|
||||
|
||||
/tasks/claim:
|
||||
post:
|
||||
tags: [tasks]
|
||||
@@ -429,6 +446,11 @@ components:
|
||||
type: integer
|
||||
description: Data rows per shard. Default 1000.
|
||||
example: 1000
|
||||
max_rows:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: Optional leading data-row limit for a small pipeline check.
|
||||
example: 500
|
||||
file:
|
||||
type: string
|
||||
format: binary
|
||||
@@ -441,6 +463,13 @@ components:
|
||||
task_count: { type: integer, example: 3 }
|
||||
input_artifact_id: { type: string, format: uuid }
|
||||
|
||||
CancelJobResponse:
|
||||
type: object
|
||||
properties:
|
||||
job_id: { type: string, format: uuid }
|
||||
status: { type: string, enum: [cancelled] }
|
||||
cancelled_tasks: { type: integer }
|
||||
|
||||
JobProgress:
|
||||
type: object
|
||||
properties:
|
||||
@@ -451,6 +480,7 @@ components:
|
||||
leased: { type: integer }
|
||||
completed: { type: integer }
|
||||
failed: { type: integer }
|
||||
cancelled: { type: integer }
|
||||
|
||||
ClaimRequest:
|
||||
type: object
|
||||
@@ -549,4 +579,4 @@ components:
|
||||
|
||||
TaskStatus:
|
||||
type: string
|
||||
enum: [pending, leased, completed, failed, cancelled]
|
||||
enum: [pending, leased, running, completed, failed, cancelled]
|
||||
|
||||
@@ -173,6 +173,9 @@ Rules:
|
||||
- TSV file, required, streamed; show expected columns
|
||||
`chembl_id` and `canonical_smiles`.
|
||||
- `chunk_rows`: integer 1--100000, default 1000.
|
||||
- `max_rows`: optional positive integer. The coordinator creates shards only
|
||||
from the first N data rows, so a user can test a large upload without
|
||||
creating thousands of tasks. It does not truncate the stored source blob.
|
||||
- optional human-readable run name is a later schema/API addition; v1 does not
|
||||
silently store it.
|
||||
- display file name and client-side size only as convenience; server limits and
|
||||
|
||||
Reference in New Issue
Block a user