Add job cancellation and dataset row limit
coordinator / test (push) Canceled after 0s

This commit is contained in:
Emil
2026-07-23 23:14:25 +03:00
parent 6bac7dad3c
commit 7547a30bde
30 changed files with 438 additions and 54 deletions
+1
View File
@@ -77,6 +77,7 @@ func run() error {
CompleteTask: usecase.NewCompleteTask(taskRepo, jobRepo, artifactRepo, tx, clk),
FailTask: usecase.NewFailTask(taskRepo, jobRepo, tx, clk),
GetJobStatus: usecase.NewGetJobStatus(jobRepo, taskRepo),
CancelJob: usecase.NewCancelJob(jobRepo, taskRepo, tx, clk),
UploadArtifact: usecase.NewUploadArtifact(taskRepo, artifactRepo, blobStore, clk),
DownloadArtifact: usecase.NewDownloadArtifact(artifactRepo, blobStore),
GetTaskInput: usecase.NewGetTaskInput(taskRepo, artifactRepo, blobStore),
+13
View File
@@ -26,9 +26,19 @@ var ErrNoRows = fmt.Errorf("input has no data rows")
// Only one shard is buffered at a time, so memory is bounded by shard size (a
// worker-sized slice of the data), not by the size of the whole dataset.
func SplitTSV(r io.Reader, rowsPerShard int, emit func(index int, shard io.Reader) error) error {
return SplitTSVLimit(r, rowsPerShard, 0, emit)
}
// SplitTSVLimit behaves like SplitTSV but emits no more than maxRows data rows.
// A maxRows value of zero means unlimited. This lets an operator make a small,
// representative pipeline check without materialising a second dataset file.
func SplitTSVLimit(r io.Reader, rowsPerShard, maxRows int, emit func(index int, shard io.Reader) error) error {
if rowsPerShard <= 0 {
return fmt.Errorf("rowsPerShard must be positive, got %d", rowsPerShard)
}
if maxRows < 0 {
return fmt.Errorf("maxRows must be non-negative, got %d", maxRows)
}
sc := bufio.NewScanner(r)
// Allow long lines: a SMILES row can be far wider than bufio's 64 KB default.
@@ -73,6 +83,9 @@ func SplitTSV(r io.Reader, rowsPerShard int, emit func(index int, shard io.Reade
return err
}
}
if maxRows > 0 && index*rowsPerShard+rows == maxRows {
break
}
}
if err := sc.Err(); err != nil {
return fmt.Errorf("read rows: %w", err)
+16
View File
@@ -103,6 +103,22 @@ func TestSplitSingleShardWhenSizeExceedsRows(t *testing.T) {
}
}
func TestSplitLimitUsesOnlyLeadingDataRows(t *testing.T) {
input := "h\nr1\nr2\nr3\nr4\nr5\n"
var shards []string
err := SplitTSVLimit(strings.NewReader(input), 2, 3, func(_ int, shard io.Reader) error {
b, _ := io.ReadAll(shard)
shards = append(shards, string(b))
return nil
})
if err != nil {
t.Fatal(err)
}
if got, want := strings.Join(shards, ""), "h\nr1\nr2\nh\nr3\n"; got != want {
t.Errorf("limited shards = %q, want %q", got, want)
}
}
// The scanned bytes are reused by bufio; the shard buffer must copy them, or a
// later row would corrupt an earlier one. This guards that copy.
func TestSplitDoesNotAliasScannerBuffer(t *testing.T) {
+10 -9
View File
@@ -8,13 +8,14 @@ import "errors"
//
// Always compare with errors.Is — outer layers may wrap these with %w.
var (
ErrJobNotFound = errors.New("job not found")
ErrTaskNotFound = errors.New("task not found")
ErrWorkerNotFound = errors.New("worker not found")
ErrArtifactNotFound = errors.New("artifact not found")
ErrLeaseConflict = errors.New("task leased to another worker")
ErrStaleAttempt = errors.New("attempt does not match lease")
ErrResultConflict = errors.New("different result already recorded")
ErrInvalidInput = errors.New("invalid input")
ErrTaskNotLeased = errors.New("task is not currently leased")
ErrJobNotFound = errors.New("job not found")
ErrTaskNotFound = errors.New("task not found")
ErrWorkerNotFound = errors.New("worker not found")
ErrArtifactNotFound = errors.New("artifact not found")
ErrJobNotCancellable = errors.New("job cannot be cancelled")
ErrLeaseConflict = errors.New("task leased to another worker")
ErrStaleAttempt = errors.New("attempt does not match lease")
ErrResultConflict = errors.New("different result already recorded")
ErrInvalidInput = errors.New("invalid input")
ErrTaskNotLeased = errors.New("task is not currently leased")
)
+9 -6
View File
@@ -99,18 +99,21 @@ func NewJobWithTasks(workload, inputURI string, params map[string]any,
// JobProgress is the aggregate view of a job and the state of its tasks.
type JobProgress struct {
Job Job
Total int
Pending int
Leased int
Done int
Failed int
Job Job
Total int
Pending int
Leased int
Done int
Failed int
Cancelled int
}
// DeriveStatus computes what the job's status should be from its task counts,
// so the rule lives here rather than in a SQL trigger or a handler.
func (p JobProgress) DeriveStatus() JobStatus {
switch {
case p.Job.Status == JobCancelled:
return JobCancelled
case p.Total == 0:
return JobPending
case p.Done == p.Total:
+1
View File
@@ -81,6 +81,7 @@ func TestDeriveStatus(t *testing.T) {
{"all done", JobProgress{Total: 3, Done: 3}, JobCompleted},
{"done and failed", JobProgress{Total: 3, Done: 2, Failed: 1}, JobFailed},
{"failed but work remains", JobProgress{Total: 3, Pending: 1, Failed: 2}, JobRunning},
{"cancelled job wins over task histogram", JobProgress{Job: Job{Status: JobCancelled}, Total: 3, Done: 1, Cancelled: 2}, JobCancelled},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
+18
View File
@@ -259,6 +259,24 @@ func (t *Task) ExpireLease(now time.Time) {
t.CompletedAt = &now
}
// Cancel prevents any further worker transition for a task that has not
// reached a terminal result. A cancelled lease deliberately becomes invalid:
// a worker still running locally must not upload or complete after its job was
// stopped by the operator.
func (t *Task) Cancel(now time.Time) bool {
if t.Status == TaskCompleted || t.Status == TaskFailed || t.Status == TaskCancelled {
return false
}
t.Status = TaskCancelled
t.LeaseOwner = nil
t.LeaseExpiresAt = nil
t.ErrorCode = nil
t.ErrorMessage = nil
t.CompletedAt = &now
t.Version++
return true
}
// ClaimedTask is the worker-facing projection of a leased task. Input is either
// an external URI or a coordinator-stored shard (InputArtifactID set); the
// transport turns the latter into a coordinator download URL.
+17
View File
@@ -168,6 +168,23 @@ func TestExpireLeaseIgnoresUnleasedTasks(t *testing.T) {
}
}
func TestCancelInvalidatesLeaseButPreservesTerminalTask(t *testing.T) {
task := leasedTask(1, 3)
if !task.Cancel(testNow) {
t.Fatal("leased task should be cancelled")
}
if task.Status != TaskCancelled || task.LeaseOwner != nil || task.LeaseExpiresAt != nil {
t.Errorf("cancelled task = %+v", task)
}
if task.Cancel(testLater) {
t.Error("cancelled task must not be changed twice")
}
completed := &Task{Status: TaskCompleted}
if completed.Cancel(testNow) {
t.Error("completed task must remain terminal")
}
}
func TestFirstHeartbeatMovesLeasedToRunning(t *testing.T) {
task := leasedTask(1, 3)
until := testLater.Add(time.Hour)
+12
View File
@@ -148,6 +148,18 @@ func (r *TaskRepo) CountByStatus(ctx context.Context, jobID uuid.UUID) (map[doma
return counts, nil
}
func (r *TaskRepo) CancelByJob(_ context.Context, jobID uuid.UUID, now time.Time) (int64, error) {
r.mu.Lock()
defer r.mu.Unlock()
var cancelled int64
for _, task := range r.tasks {
if task.JobID == jobID && task.Cancel(now) {
cancelled++
}
}
return cancelled, nil
}
func (r *TaskRepo) ExpireLeases(ctx context.Context, now time.Time) (int64, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -206,6 +206,27 @@ func TestClaimNextReturnsNilOnEmptyQueue(t *testing.T) {
}
}
func TestCancelJobCancelsEveryUnfinishedTask(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
job, _ := seedJob(t, pool, 3)
clk := fixedClock{now: time.Now().UTC()}
uc := usecase.NewCancelJob(NewJobRepo(pool), NewTaskRepo(pool), NewTxManager(pool), clk)
cancelled, err := uc.Execute(ctx, job.ID)
if err != nil || cancelled != 3 {
t.Fatalf("cancel = (%d, %v), want (3, nil)", cancelled, err)
}
stored, err := NewJobRepo(pool).Get(ctx, job.ID)
if err != nil || stored.Status != domain.JobCancelled {
t.Fatalf("job after cancel = (%+v, %v)", stored, err)
}
counts, err := NewTaskRepo(pool).CountByStatus(ctx, job.ID)
if err != nil || counts[domain.TaskCancelled] != 3 {
t.Fatalf("cancelled tasks = %d, err = %v", counts[domain.TaskCancelled], err)
}
}
func TestUpdateRejectsStaleVersion(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
@@ -295,6 +295,29 @@ func (r *TaskRepo) CountByStatus(ctx context.Context, jobID uuid.UUID) (map[doma
return counts, rows.Err()
}
// cancelByJobSQL mirrors domain.Task.Cancel in one set-based update. It runs in
// the same transaction as the job-status update, so no claimable shard remains
// after an operator receives a successful cancellation response.
const cancelByJobSQL = `
UPDATE tasks
SET status = 'cancelled'::task_status,
lease_owner = NULL,
lease_expires_at = NULL,
error_code = NULL,
error_message = NULL,
completed_at = $2,
version = version + 1
WHERE job_id = $1
AND status IN ('pending','leased','running')`
func (r *TaskRepo) CancelByJob(ctx context.Context, jobID uuid.UUID, now time.Time) (int64, error) {
tag, err := conn(ctx, r.pool).Exec(ctx, cancelByJobSQL, jobID, now)
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
// expireLeasesSQL applies the lease-expiry rule set-based, mirroring
// domain.Task.ExpireLease: requeue while attempts remain, otherwise fail.
//
+16 -14
View File
@@ -111,13 +111,14 @@ type uploadJobResponse struct {
}
type jobProgressResponse struct {
ID uuid.UUID `json:"id"`
Status string `json:"status"`
Total int `json:"total"`
Pending int `json:"pending"`
Leased int `json:"leased"`
Done int `json:"completed"`
Failed int `json:"failed"`
ID uuid.UUID `json:"id"`
Status string `json:"status"`
Total int `json:"total"`
Pending int `json:"pending"`
Leased int `json:"leased"`
Done int `json:"completed"`
Failed int `json:"failed"`
Cancelled int `json:"cancelled"`
}
type uploadArtifactResponse struct {
@@ -153,12 +154,13 @@ func toClaimedTaskResponse(c domain.ClaimedTask) claimedTaskResponse {
func toJobProgressResponse(p domain.JobProgress) jobProgressResponse {
return jobProgressResponse{
ID: p.Job.ID,
Status: string(p.DeriveStatus()),
Total: p.Total,
Pending: p.Pending,
Leased: p.Leased,
Done: p.Done,
Failed: p.Failed,
ID: p.Job.ID,
Status: string(p.DeriveStatus()),
Total: p.Total,
Pending: p.Pending,
Leased: p.Leased,
Done: p.Done,
Failed: p.Failed,
Cancelled: p.Cancelled,
}
}
@@ -50,7 +50,8 @@ func (s *Server) writeError(w http.ResponseWriter, r *http.Request, err error) {
case errors.Is(err, domain.ErrLeaseConflict),
errors.Is(err, domain.ErrStaleAttempt),
errors.Is(err, domain.ErrResultConflict),
errors.Is(err, domain.ErrTaskNotLeased):
errors.Is(err, domain.ErrTaskNotLeased),
errors.Is(err, domain.ErrJobNotCancellable):
status = http.StatusConflict
case errors.Is(err, usecase.ErrNotImplemented):
status = http.StatusNotImplemented
@@ -182,7 +182,7 @@ func (s *Server) handleFailure(w http.ResponseWriter, r *http.Request) {
const defaultChunkRows = 1000
// handleUploadDataset accepts a multipart submission — the dataset file plus the
// workload/parameters/chunk_rows fields — and hands the file, streamed, to the
// workload/parameters/chunk_rows/max_rows fields — and hands the file, streamed, to the
// chunker. The text fields MUST precede the file part: the file is streamed, not
// buffered, so by the time it arrives the other fields are already parsed.
func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
@@ -197,11 +197,13 @@ func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
workload string
params map[string]any
rows = defaultChunkRows
maxRows int
result usecase.SubmitDatasetResult
gotDataset bool
gotWorkload bool
gotParams bool
gotRows bool
gotMaxRows bool
)
for {
@@ -249,6 +251,19 @@ func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
}
rows = n
gotRows = true
case "max_rows":
if gotDataset || gotMaxRows {
s.writeError(w, r, domain.ErrInvalidInput)
return
}
b, _ := io.ReadAll(io.LimitReader(part, 32))
n, err := strconv.Atoi(strings.TrimSpace(string(b)))
if err != nil || n < 1 {
s.writeError(w, r, domain.ErrInvalidInput)
return
}
maxRows = n
gotMaxRows = true
case "file", "dataset":
if gotDataset || workload == "" {
s.writeError(w, r, domain.ErrInvalidInput)
@@ -262,6 +277,7 @@ func (s *Server) handleUploadDataset(w http.ResponseWriter, r *http.Request) {
Workload: workload,
Parameters: params,
RowsPerShard: rows,
MaxRows: maxRows,
Filename: filename,
ContentType: part.Header.Get("Content-Type"),
Body: part,
@@ -379,6 +395,27 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, toJobProgressResponse(progress))
}
// handleCancelJob stops all non-terminal shards for an operator-requested job.
// It is available to both the bearer API and the separately authenticated UI.
func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
ctx, cancel := s.reqCtx(r)
defer cancel()
jobID, ok := s.pathUUID(w, r, "job_id")
if !ok {
return
}
cancelled, err := s.uc.CancelJob.Execute(ctx, jobID)
if err != nil {
s.writeError(w, r, err)
return
}
writeJSON(w, http.StatusOK, map[string]any{
"job_id": jobID,
"status": domain.JobCancelled,
"cancelled_tasks": cancelled,
})
}
// --- helpers ---
func (s *Server) reqCtx(r *http.Request) (context.Context, context.CancelFunc) {
@@ -24,6 +24,7 @@ type UseCases struct {
CompleteTask *usecase.CompleteTask
FailTask *usecase.FailTask
GetJobStatus *usecase.GetJobStatus
CancelJob *usecase.CancelJob
UploadArtifact *usecase.UploadArtifact
DownloadArtifact *usecase.DownloadArtifact
GetTaskInput *usecase.GetTaskInput
@@ -61,6 +62,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
protected.HandleFunc("POST /jobs", s.handleCreateJob)
protected.HandleFunc("POST /jobs/upload", s.handleUploadDataset)
protected.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
protected.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
protected.HandleFunc("POST /tasks/claim", s.handleClaim)
protected.HandleFunc("GET /tasks/{task_id}/input", s.handleGetTaskInput)
protected.HandleFunc("POST /tasks/{task_id}/heartbeat", s.handleHeartbeat)
@@ -77,6 +79,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
ui.HandleFunc("GET /ui/jobs/new", s.handleUINewJob)
ui.HandleFunc("GET /ui/jobs/{job_id}", s.handleUIJob)
ui.HandleFunc("GET /ui/api/jobs/{job_id}", s.handleUIJobJSON)
ui.HandleFunc("POST /ui/api/jobs/{job_id}/cancel", s.handleCancelJob)
ui.HandleFunc("POST /ui/api/jobs/upload", s.handleUploadDataset)
ui.HandleFunc("GET /ui/jobs/{job_id}/artifacts/{artifact_id}", s.handleUIArtifactDownload)
mux.Handle("/ui", chain(ui, withRequestID, withAccessLog(s.log), withBasicAuth(uiToken[0]), withSameOrigin))
@@ -51,6 +51,7 @@ func newEnvWithUIToken(t *testing.T, ready func(context.Context) error, configur
CompleteTask: usecase.NewCompleteTask(tasks, jobs, arts, tx, clk),
FailTask: usecase.NewFailTask(tasks, jobs, tx, clk),
GetJobStatus: usecase.NewGetJobStatus(jobs, tasks),
CancelJob: usecase.NewCancelJob(jobs, tasks, tx, clk),
UploadArtifact: usecase.NewUploadArtifact(tasks, arts, blobs, clk),
DownloadArtifact: usecase.NewDownloadArtifact(arts, blobs),
GetTaskInput: usecase.NewGetTaskInput(tasks, arts, blobs),
@@ -200,6 +201,39 @@ func TestUIUploadDatasetCreatesJob(t *testing.T) {
}
}
func TestCancelJobStopsUnfinishedTasks(t *testing.T) {
e := newEnv(t, healthy)
code, job := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c0","input_sha256":"sha"},{"chunk_index":1,"input_uri":"s3://c1","input_sha256":"sha"}]}`)
if code != http.StatusCreated {
t.Fatalf("create: %d", code)
}
jobID := job["id"].(string)
if code, body := e.do(t, "POST", "/jobs/"+jobID+"/cancel", ""); code != http.StatusOK || body["cancelled_tasks"].(float64) != 2 {
t.Fatalf("cancel = (%d, %v)", code, body)
}
if code, progress := e.do(t, "GET", "/jobs/"+jobID, ""); code != http.StatusOK || progress["status"] != "cancelled" || progress["cancelled"].(float64) != 2 {
t.Fatalf("cancelled job progress = (%d, %v)", code, progress)
}
}
func TestUICancelJobUsesOperatorCredential(t *testing.T) {
e := newEnv(t, healthy)
code, job := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c0","input_sha256":"sha"}]}`)
if code != http.StatusCreated {
t.Fatalf("create: %d", code)
}
req, _ := http.NewRequestWithContext(context.Background(), "POST", e.ts.URL+"/ui/api/jobs/"+job["id"].(string)+"/cancel", nil)
req.SetBasicAuth("operator", uiToken)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("UI cancel = %d", resp.StatusCode)
}
}
func TestUIJobAndArtifactAreScopedToTheirJob(t *testing.T) {
e := newEnv(t, healthy)
code, job := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c","input_sha256":"sha"}]}`)
@@ -392,6 +426,31 @@ func TestUploadDatasetChunksAndServesInput(t *testing.T) {
}
}
func TestUploadDatasetLimitsRows(t *testing.T) {
e := newEnv(t, healthy)
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
_ = mw.WriteField("workload", "w")
_ = mw.WriteField("chunk_rows", "2")
_ = mw.WriteField("max_rows", "3")
fw, _ := mw.CreateFormFile("file", "chembl.tsv")
_, _ = io.Copy(fw, strings.NewReader("id\tsmiles\nA\tCC\nB\tCCC\nC\tCCCC\nD\tCCCCC\n"))
_ = mw.Close()
req, _ := http.NewRequestWithContext(context.Background(), "POST", e.ts.URL+"/jobs/upload", &buf)
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", mw.FormDataContentType())
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
var result map[string]any
_ = json.NewDecoder(resp.Body).Decode(&result)
if resp.StatusCode != http.StatusCreated || result["task_count"].(float64) != 2 {
t.Fatalf("limited upload = (%d, %v)", resp.StatusCode, result)
}
}
func TestErrorMappings(t *testing.T) {
e := newEnv(t, healthy)
zero := "00000000-0000-0000-0000-000000000000"
@@ -14,7 +14,7 @@
<header class="top"><div><p class="eyebrow">Local coordinator</p><h1>SciMesh operator dashboard</h1><p class="lead">See where a computation is and what should happen next.</p></div><a class="button" href="/ui/jobs/new">Start a check</a></header>
<section class="notice" aria-label="Current pipeline limitation"><strong>This screen currently diagnoses shard jobs.</strong><span>Workers upload partial CSVs to the coordinator. Until a reducer is implemented, those files are not one final scientific result.</span><div class="steps"><div class="step"><b>1. Upload TSV</b>The coordinator splits the file into shard tasks.</div><div class="step"><b>2. Wait for a worker</b>A worker claims a shard, calculates similarity, and returns a CSV.</div><div class="step"><b>3. Inspect artifacts</b>Download a partial result from the job page.</div></div></section>
<h2>Recent jobs</h2>
<div class="table-wrap"><table><tr><th>Computation</th><th>State</th><th>Progress</th><th class="hide-mobile">Created</th></tr>{{range .Jobs}}<tr><td><a href="/ui/jobs/{{.ID}}"><strong>{{workloadLabel .Workload}}</strong></a><br><small>Open job details</small></td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><br><small>{{statusHint .Status}}</small></td><td class="kicker"><strong>{{.Completed}} / {{.Total}}</strong> complete{{if gt .Failed 0}} · <span style="color:#a31135">failed: {{.Failed}}</span>{{end}}<div class="bar"><span style="width:{{progressPercent .Completed .Failed .Total}}%"></span></div></td><td class="hide-mobile"><small>{{time .CreatedAt}}</small></td></tr>{{else}}<tr><td colspan="4" class="empty"><strong>No jobs yet.</strong><br>Click “Start a check”, upload a small TSV, and leave a worker running.</td></tr>{{end}}</table></div>
<div class="table-wrap"><table><tr><th>Computation</th><th>State</th><th>Progress</th><th class="hide-mobile">Created</th></tr>{{range .Jobs}}<tr><td><a href="/ui/jobs/{{.ID}}"><strong>{{workloadLabel .Workload}}</strong></a><br><small>Open job details</small></td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><br><small>{{statusHint .Status}}</small></td><td class="kicker"><strong>{{.Completed}} / {{.Total}}</strong> complete{{if gt .Failed 0}} · <span style="color:#a31135">failed: {{.Failed}}</span>{{end}}{{if gt .Cancelled 0}} · <span>stopped: {{.Cancelled}}</span>{{end}}<div class="bar"><span style="width:{{progressPercent .Completed .Failed .Cancelled .Total}}%"></span></div></td><td class="hide-mobile"><small>{{time .CreatedAt}}</small></td></tr>{{else}}<tr><td colspan="4" class="empty"><strong>No jobs yet.</strong><br>Click “Start a check”, upload a small TSV, and leave a worker running.</td></tr>{{end}}</table></div>
<h2>Workers</h2>
<section class="card">{{range .Workers}}<div class="worker"><div><strong>{{.Name}}</strong><br><small>{{.ID}}</small></div><div><span class="status status-{{if eq .Status "online"}}success{{else}}waiting{{end}}">{{workerStatusLabel .Status}}</span></div><div>{{range .Capabilities}}<code>{{.}}</code> {{end}}</div><div class="muted">Last signal<br>{{time .LastHeartbeatAt}}</div></div>{{else}}<div class="empty"><strong>No worker is registered yet.</strong><br>Run <code>scimesh-worker</code> with the coordinator URL and worker token.</div>{{end}}</section>
</main>
@@ -6,13 +6,13 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>SciMesh job</title>
<style>
:root{color:#172033;background:#f6f8fc;font:16px/1.5 system-ui,sans-serif}body{margin:0}.page{max-width:1180px;margin:auto;padding:32px 20px 56px}a{color:#174ecf}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#50617d;font-size:.86rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em}h1{margin:0;font-size:2rem}h2{margin:32px 0 12px;font-size:1.3rem}.summary,.card{padding:20px;border:1px solid #dfe5f0;border-radius:12px;background:#fff}.summary-head{display:flex;justify-content:space-between;gap:16px;align-items:start}.status{display:inline-block;border-radius:999px;padding:4px 10px;font-size:.9rem;font-weight:700}.status-success{background:#dff6e9;color:#126b3d}.status-danger{background:#ffe4e6;color:#a31135}.status-active{background:#e4edff;color:#174ecf}.status-waiting{background:#edf0f5;color:#50617d}.hint{margin:8px 0 0;color:#56657c}.bar{height:10px;margin:20px 0 8px;overflow:hidden;border-radius:999px;background:#e6eaf1}.bar>span{display:block;height:100%;background:#1f5eff;transition:width .3s}.numbers{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}.number{padding:12px;border-radius:8px;background:#f6f8fc}.number b{display:block;font-size:1.35rem}.notice{margin:20px 0;padding:15px 17px;border:1px solid #f2cb72;border-radius:10px;background:#fff8e6}.table-wrap{overflow-x:auto;border:1px solid #dfe5f0;border-radius:10px;background:#fff}table{width:100%;border-collapse:collapse}td,th{padding:12px 13px;border-bottom:1px solid #e8ecf4;text-align:left;vertical-align:top}th{color:#50617d;font-size:.78rem;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small,.muted{color:#68758b}.error{color:#a31135;max-width:360px;word-break:break-word}.download{display:inline-block;padding:7px 10px;border:1px solid #b9c9ee;border-radius:7px;text-decoration:none}.empty{padding:24px;text-align:center;color:#68758b}details{margin-top:18px;color:#56657c}code{word-break:break-all}@media(max-width:700px){.summary-head{display:block}.numbers{grid-template-columns:repeat(2,1fr)}}
:root{color:#172033;background:#f6f8fc;font:16px/1.5 system-ui,sans-serif}body{margin:0}.page{max-width:1180px;margin:auto;padding:32px 20px 56px}a{color:#174ecf}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#50617d;font-size:.86rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em}h1{margin:0;font-size:2rem}h2{margin:32px 0 12px;font-size:1.3rem}.summary,.card{padding:20px;border:1px solid #dfe5f0;border-radius:12px;background:#fff}.summary-head{display:flex;justify-content:space-between;gap:16px;align-items:start}.status{display:inline-block;border-radius:999px;padding:4px 10px;font-size:.9rem;font-weight:700}.status-success{background:#dff6e9;color:#126b3d}.status-danger{background:#ffe4e6;color:#a31135}.status-active{background:#e4edff;color:#174ecf}.status-waiting{background:#edf0f5;color:#50617d}.hint{margin:8px 0 0;color:#56657c}.bar{height:10px;margin:20px 0 8px;overflow:hidden;border-radius:999px;background:#e6eaf1}.bar>span{display:block;height:100%;background:#1f5eff;transition:width .3s}.numbers{display:grid;grid-template-columns:repeat(6,1fr);gap:10px}.number{padding:12px;border-radius:8px;background:#f6f8fc}.number b{display:block;font-size:1.35rem}.stop{display:block;margin-left:auto;border:1px solid #d43b51;border-radius:7px;padding:8px 11px;background:#fff;color:#b2223a;font:inherit;font-weight:700;cursor:pointer}.stop:disabled{opacity:.6}.notice{margin:20px 0;padding:15px 17px;border:1px solid #f2cb72;border-radius:10px;background:#fff8e6}.table-wrap{overflow-x:auto;border:1px solid #dfe5f0;border-radius:10px;background:#fff}table{width:100%;border-collapse:collapse}td,th{padding:12px 13px;border-bottom:1px solid #e8ecf4;text-align:left;vertical-align:top}th{color:#50617d;font-size:.78rem;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small,.muted{color:#68758b}.error{color:#a31135;max-width:360px;word-break:break-word}.download{display:inline-block;padding:7px 10px;border:1px solid #b9c9ee;border-radius:7px;text-decoration:none}.empty{padding:24px;text-align:center;color:#68758b}details{margin-top:18px;color:#56657c}code{word-break:break-all}@media(max-width:700px){.summary-head{display:block}.numbers{grid-template-columns:repeat(2,1fr)}}
</style>
</head>
<body>
<main class="page">
<a class="back" href="/ui">← Back to jobs</a><p class="eyebrow">{{workloadLabel .Workload}}</p><h1>Execution progress</h1>
<section class="summary"><div class="summary-head"><div><span id="status" class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><p id="hint" class="hint">{{statusHint .Status}}</p></div><small>Summary refreshes automatically every two seconds.</small></div><div class="bar" aria-label="Progress"><span id="progress-bar" style="width:{{progressPercent .Completed .Failed .Total}}%"></span></div><p id="progress" class="muted">{{.Completed}} of {{.Total}} tasks complete</p><div class="numbers"><div class="number"><b id="total">{{.Total}}</b><small>total shards</small></div><div class="number"><b id="completed">{{.Completed}}</b><small>complete</small></div><div class="number"><b id="pending">{{.Pending}}</b><small>waiting</small></div><div class="number"><b id="active">{{add .Leased .Running}}</b><small>with workers</small></div><div class="number"><b id="failed">{{.Failed}}</b><small>failed</small></div></div><details><summary>Technical details</summary><p>Job ID: <code>{{.ID}}</code><br>Workload: <code>{{.Workload}}</code><br>Created: {{time .CreatedAt}}</p></details></section>
<section class="summary"><div class="summary-head"><div><span id="status" class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><p id="hint" class="hint">{{statusHint .Status}}</p></div><div>{{if cancellable .Status}}<button id="stop-job" class="stop" type="button">Stop job</button><small>This cancels every shard that is not finished yet.</small>{{else}}<small>Summary refreshes automatically every two seconds.</small>{{end}}</div></div><div class="bar" aria-label="Progress"><span id="progress-bar" style="width:{{progressPercent .Completed .Failed .Cancelled .Total}}%"></span></div><p id="progress" class="muted">{{.Completed}} of {{.Total}} tasks complete</p><div class="numbers"><div class="number"><b id="total">{{.Total}}</b><small>total shards</small></div><div class="number"><b id="completed">{{.Completed}}</b><small>complete</small></div><div class="number"><b id="pending">{{.Pending}}</b><small>waiting</small></div><div class="number"><b id="active">{{add .Leased .Running}}</b><small>with workers</small></div><div class="number"><b id="failed">{{.Failed}}</b><small>failed</small></div><div class="number"><b id="cancelled">{{.Cancelled}}</b><small>stopped</small></div></div><details><summary>Technical details</summary><p>Job ID: <code>{{.ID}}</code><br>Workload: <code>{{.Workload}}</code><br>Created: {{time .CreatedAt}}</p></details></section>
<section class="notice"><strong>What can be downloaded now?</strong><br><code>partial_result</code> files come from individual shards. They are useful for checking the pipeline, but are not a merged final CSV because the reducer is not implemented yet.</section>
<h2>Shard tasks</h2><p class="muted">If a task fails, its code and message appear here. Refresh the page to update the detailed rows.</p>
<div class="table-wrap"><table><tr><th>Shard</th><th>State</th><th>Attempt</th><th>Worker / lease</th><th>Error</th></tr>{{range .Tasks}}<tr><td>#{{.ChunkIndex}}</td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span></td><td>{{.Attempt}} / {{.MaxAttempts}}</td><td>{{if .LeaseOwner}}<code>{{.LeaseOwner}}</code>{{if .LeaseExpiresAt}}<br><small>until {{time .LeaseExpiresAt}}</small>{{end}}{{else}}<span class="muted"></span>{{end}}</td><td class="error">{{if .ErrorCode}}<strong>{{taskErrorLabel .ErrorCode}}</strong><br><small>{{taskErrorHint .ErrorCode}}</small>{{else}}<span class="muted"></span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No tasks have appeared yet.</td></tr>{{end}}</table></div>
@@ -20,8 +20,9 @@
<div class="table-wrap"><table><tr><th>Type</th><th>File</th><th>Size</th><th>Integrity check</th><th></th></tr>{{range .Artifacts}}<tr><td>{{if .Diagnostic}}<strong>Partial result</strong><br><small>diagnostic</small>{{else}}{{.Kind}}{{end}}</td><td>{{.Filename}}</td><td>{{bytes .SizeBytes}}</td><td><code>{{.SHA256}}</code></td><td>{{if .Downloadable}}<a class="download" href="/ui/jobs/{{$.ID}}/artifacts/{{.ID}}">Download CSV</a>{{else}}<span class="muted">Unavailable</span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No artifacts yet. The worker uploads a CSV after it completes a shard.</td></tr>{{end}}</table></div>
</main>
<script>
const id={{printf "%q" .ID}},state={pending:['Waiting for a worker','waiting','Waiting for an available worker with the required capability.'],leased:['Assigned to a worker','active','A worker has claimed the task and should begin processing shortly.'],running:['Running','active','A worker is reading a shard, calculating fingerprints, and uploading its result through the coordinator.'],completed:['Tasks complete','success','Every shard task is complete. Files below are still partial results.'],failed:['Needs attention','danger','One or more shard tasks failed. Open the task list below for details.']};
setInterval(async()=>{try{const response=await fetch('/ui/api/jobs/'+id);if(!response.ok)return;const job=await response.json(),info=state[job.status]||[job.status,'waiting','Status reported by the coordinator.'],done=job.completed+job.failed,percent=job.total?Math.min(100,Math.floor(done*100/job.total)):0,badge=document.querySelector('#status');badge.textContent=info[0];badge.className='status status-'+info[1];document.querySelector('#hint').textContent=info[2];document.querySelector('#progress').textContent=job.completed+' of '+job.total+' tasks complete'+(job.failed?' · failed: '+job.failed:'');document.querySelector('#progress-bar').style.width=percent+'%';for(const key of ['total','completed','pending','failed'])document.querySelector('#'+key).textContent=job[key];document.querySelector('#active').textContent=job.leased+job.running}catch(_){}} ,2000);
const id={{printf "%q" .ID}},state={pending:['Waiting for a worker','waiting','Waiting for an available worker with the required capability.'],leased:['Assigned to a worker','active','A worker has claimed the task and should begin processing shortly.'],running:['Running','active','A worker is reading a shard, calculating fingerprints, and uploading its result through the coordinator.'],completed:['Tasks complete','success','Every shard task is complete. Files below are still partial results.'],failed:['Needs attention','danger','One or more shard tasks failed. Open the task list below for details.'],cancelled:['Stopped','waiting','The operator stopped this job. No new shards can be claimed.']};
const stop=document.querySelector('#stop-job');if(stop)stop.addEventListener('click',async()=>{if(!confirm('Stop this job? Unfinished shards will be cancelled.'))return;stop.disabled=true;const response=await fetch('/ui/api/jobs/'+id+'/cancel',{method:'POST'});if(!response.ok){stop.disabled=false;alert('Unable to stop this job.');return}location.reload()});
setInterval(async()=>{try{const response=await fetch('/ui/api/jobs/'+id);if(!response.ok)return;const job=await response.json(),info=state[job.status]||[job.status,'waiting','Status reported by the coordinator.'],done=job.completed+job.failed+job.cancelled,percent=job.total?Math.min(100,Math.floor(done*100/job.total)):0,badge=document.querySelector('#status');badge.textContent=info[0];badge.className='status status-'+info[1];document.querySelector('#hint').textContent=info[2];document.querySelector('#progress').textContent=job.completed+' of '+job.total+' tasks complete'+(job.failed?' · failed: '+job.failed:'')+(job.cancelled?' · stopped: '+job.cancelled:'');document.querySelector('#progress-bar').style.width=percent+'%';for(const key of ['total','completed','pending','failed','cancelled'])document.querySelector('#'+key).textContent=job[key];document.querySelector('#active').textContent=job.leased+job.running}catch(_){}} ,2000);
</script>
</body>
</html>
@@ -18,12 +18,13 @@
<label for="query-smiles">Target molecule (SMILES)</label><input id="query-smiles" name="query_smiles" required maxlength="200" value="CCO" autocomplete="off"><p class="hint"><code>CCO</code> is ethanol. For gefitinib, use its SMILES here or the local CLI with <code>--query-id</code>.</p>
<label for="top-k">Matches to return</label><input id="top-k" name="top_k" type="number" min="1" max="100000" value="20" required><p class="hint">This is the top-k within each shard, not a global top-k for the whole dataset yet.</p>
<label for="chunk-rows">Rows per shard</label><input id="chunk-rows" name="chunk_rows" type="number" min="1" max="100000" value="1000" required><p class="hint">Fewer rows mean more tasks and more visible progress; more rows mean fewer, longer tasks.</p>
<label for="max-rows">Maximum dataset rows to process <small>(optional)</small></label><input id="max-rows" name="max_rows" type="number" min="1" max="10000000" placeholder="For example: 500"><p class="hint">Useful for a quick check of a large TSV. The coordinator creates shards from only the first N data rows; it still stores the original upload.</p>
<button class="button" id="submit" type="submit">Upload file and create job</button><p id="working" class="working" hidden aria-live="polite">Uploading the file and creating shard tasks… Keep this page open.</p><p id="error" class="error" role="alert"></p>
</form>
</main>
<script>
const form=document.querySelector('#run'),button=document.querySelector('#submit'),working=document.querySelector('#working'),error=document.querySelector('#error');
form.addEventListener('submit',async event=>{event.preventDefault();error.textContent='';const fields=new FormData(form),file=fields.get('file');if(!(file instanceof File)||file.size===0){error.textContent='Choose a non-empty TSV file.';return}const parameters={query_smiles:fields.get('query_smiles'),top_k:Number(fields.get('top_k')),progress_every:0},upload=new FormData();upload.append('workload','similarity-search');upload.append('parameters',JSON.stringify(parameters));upload.append('chunk_rows',fields.get('chunk_rows'));upload.append('file',file,file.name);button.disabled=true;working.hidden=false;try{const response=await fetch('/ui/api/jobs/upload',{method:'POST',body:upload}),data=await response.json();if(!response.ok)throw Error(data.error||'Unable to create the job.');location.href='/ui/jobs/'+data.job_id}catch(err){error.textContent=err.message==='invalid input'?'Check the TSV and fields: the coordinator could not accept this request.':err.message;button.disabled=false;working.hidden=true}});
form.addEventListener('submit',async event=>{event.preventDefault();error.textContent='';const fields=new FormData(form),file=fields.get('file'),maxRows=String(fields.get('max_rows')||'').trim();if(!(file instanceof File)||file.size===0){error.textContent='Choose a non-empty TSV file.';return}const parameters={query_smiles:fields.get('query_smiles'),top_k:Number(fields.get('top_k')),progress_every:0},upload=new FormData();upload.append('workload','similarity-search');upload.append('parameters',JSON.stringify(parameters));upload.append('chunk_rows',fields.get('chunk_rows'));if(maxRows)upload.append('max_rows',maxRows);upload.append('file',file,file.name);button.disabled=true;working.hidden=false;try{const response=await fetch('/ui/api/jobs/upload',{method:'POST',body:upload}),data=await response.json();if(!response.ok)throw Error(data.error||'Unable to create the job.');location.href='/ui/jobs/'+data.job_id}catch(err){error.textContent=err.message==='invalid input'?'Check the TSV and fields: the coordinator could not accept this request.':err.message;button.disabled=false;working.hidden=true}});
</script>
</body>
</html>
+13 -2
View File
@@ -28,6 +28,7 @@ var uiTemplates = template.Must(template.New("ui").Funcs(template.FuncMap{
"workerStatusLabel": uiWorkerStatusLabel,
"workloadLabel": uiWorkloadLabel,
"progressPercent": uiProgressPercent,
"cancellable": uiCancellable,
"bytes": uiBytes,
"add": func(a, b int) int { return a + b },
}).ParseFS(uiFiles, "templates/*.html"))
@@ -51,6 +52,8 @@ func uiStatusLabel(status string) string {
return "Tasks complete"
case "failed":
return "Needs attention"
case "cancelled":
return "Stopped"
default:
return status
}
@@ -68,6 +71,8 @@ func uiStatusHint(status string) string {
return "Every shard task is complete. Files below are still partial results."
case "failed":
return "One or more shard tasks failed. Open the task list below for details."
case "cancelled":
return "The operator stopped this job. No new shards can be claimed."
default:
return "Status reported by the coordinator."
}
@@ -79,6 +84,8 @@ func uiStatusClass(status string) string {
return "success"
case "failed":
return "danger"
case "cancelled":
return "waiting"
case "running", "leased":
return "active"
default:
@@ -145,11 +152,15 @@ func uiWorkloadLabel(workload string) string {
}
}
func uiProgressPercent(completed, failed, total int) int {
func uiCancellable(status string) bool {
return status == "pending" || status == "running"
}
func uiProgressPercent(completed, failed, cancelled, total int) int {
if total <= 0 {
return 0
}
percent := (completed + failed) * 100 / total
percent := (completed + failed + cancelled) * 100 / total
if percent > 100 {
return 100
}
@@ -26,10 +26,10 @@ func TestUIStatusPresentation(t *testing.T) {
}
func TestUIProgressPercent(t *testing.T) {
if got := uiProgressPercent(3, 1, 8); got != 50 {
if got := uiProgressPercent(3, 1, 0, 8); got != 50 {
t.Errorf("progress = %d, want 50", got)
}
if got := uiProgressPercent(1, 1, 0); got != 0 {
if got := uiProgressPercent(1, 1, 0, 0); got != 0 {
t.Errorf("empty progress = %d, want 0", got)
}
}
+6 -3
View File
@@ -53,9 +53,12 @@ type SubmitDatasetInput struct {
Workload string
Parameters map[string]any
RowsPerShard int
Filename string
ContentType string
Body io.Reader
// MaxRows limits how many data rows are turned into shards. Zero means the
// whole uploaded dataset; the input artifact itself remains stored intact.
MaxRows int
Filename string
ContentType string
Body io.Reader
}
type SubmitDatasetResult struct {
+43 -3
View File
@@ -62,6 +62,45 @@ type GetJobStatus struct {
tasks TaskRepository
}
// --- CancelJob -----------------------------------------------------------
type CancelJob struct {
jobs JobRepository
tasks TaskRepository
tx TxManager
clock Clock
}
func NewCancelJob(jobs JobRepository, tasks TaskRepository, tx TxManager, clock Clock) *CancelJob {
return &CancelJob{jobs: jobs, tasks: tasks, tx: tx, clock: clock}
}
// Execute stops a job atomically. Completed and finally failed tasks are kept
// as historical evidence; all other tasks are cancelled, including leased and
// running ones. A repeated cancel of an already cancelled job is idempotent.
func (uc *CancelJob) Execute(ctx context.Context, jobID uuid.UUID) (int64, error) {
now := uc.clock.Now()
var cancelled int64
err := uc.tx.WithinTx(ctx, func(ctx context.Context) error {
job, err := uc.jobs.Get(ctx, jobID)
if err != nil {
return err
}
if job.Status == domain.JobCancelled {
return nil
}
if job.Status == domain.JobCompleted || job.Status == domain.JobFailed {
return domain.ErrJobNotCancellable
}
cancelled, err = uc.tasks.CancelByJob(ctx, jobID, now)
if err != nil {
return err
}
return uc.jobs.UpdateStatus(ctx, jobID, domain.JobCancelled, &now)
})
return cancelled, err
}
func NewGetJobStatus(jobs JobRepository, tasks TaskRepository) *GetJobStatus {
return &GetJobStatus{jobs: jobs, tasks: tasks}
}
@@ -144,9 +183,10 @@ func progressFrom(job domain.Job, counts map[domain.TaskStatus]int) domain.JobPr
Job: job,
Pending: counts[domain.TaskPending],
// Leased and running are both "in flight" for progress purposes.
Leased: counts[domain.TaskLeased] + counts[domain.TaskRunning],
Done: counts[domain.TaskCompleted],
Failed: counts[domain.TaskFailed],
Leased: counts[domain.TaskLeased] + counts[domain.TaskRunning],
Done: counts[domain.TaskCompleted],
Failed: counts[domain.TaskFailed],
Cancelled: counts[domain.TaskCancelled],
}
for _, n := range counts {
p.Total += n
+4
View File
@@ -56,6 +56,10 @@ type TaskRepository interface {
// CountByStatus aggregates a job's tasks for progress reporting.
CountByStatus(ctx context.Context, jobID uuid.UUID) (map[domain.TaskStatus]int, error)
// CancelByJob marks every non-terminal task as cancelled and invalidates its
// lease. It returns how many tasks changed.
CancelByJob(ctx context.Context, jobID uuid.UUID, now time.Time) (int64, error)
// ExpireLeases applies the lease-expiry rule to every elapsed task and
// reports how many were affected.
ExpireLeases(ctx context.Context, now time.Time) (int64, error)
+4 -1
View File
@@ -30,6 +30,7 @@ type JobCard struct {
Running int `json:"running"`
Completed int `json:"completed"`
Failed int `json:"failed"`
Cancelled int `json:"cancelled"`
}
type TaskCard struct {
@@ -166,9 +167,11 @@ func jobCard(job domain.Job, tasks []domain.Task) JobCard {
c.Completed++
case domain.TaskFailed:
c.Failed++
case domain.TaskCancelled:
c.Cancelled++
}
}
p := domain.JobProgress{Job: job, Total: c.Total, Pending: c.Pending, Leased: c.Leased + c.Running, Done: c.Completed, Failed: c.Failed}
p := domain.JobProgress{Job: job, Total: c.Total, Pending: c.Pending, Leased: c.Leased + c.Running, Done: c.Completed, Failed: c.Failed, Cancelled: c.Cancelled}
c.Status = string(p.DeriveStatus())
return c
}
+1 -1
View File
@@ -64,7 +64,7 @@ func (uc *SubmitDataset) Execute(ctx context.Context, in SubmitDatasetInput) (Su
cleanup()
return SubmitDatasetResult{}, err
}
splitErr := chunk.SplitTSV(rc, in.RowsPerShard, func(index int, shard io.Reader) error {
splitErr := chunk.SplitTSVLimit(rc, in.RowsPerShard, in.MaxRows, func(index int, shard io.Reader) error {
art, err := domain.NewArtifact(job.ID, nil, domain.ArtifactShard,
fmt.Sprintf("shard-%d.tsv", index), in.ContentType, now)
if err != nil {
@@ -54,6 +54,7 @@ type harness struct {
downloadArt *usecase.DownloadArtifact
getInput *usecase.GetTaskInput
expire *usecase.ExpireLeases
cancel *usecase.CancelJob
}
func newHarness() *harness {
@@ -79,6 +80,7 @@ func newHarness() *harness {
h.downloadArt = usecase.NewDownloadArtifact(h.arts, h.blobs)
h.getInput = usecase.NewGetTaskInput(h.tasks, h.arts, h.blobs)
h.expire = usecase.NewExpireLeases(h.tasks, h.clk)
h.cancel = usecase.NewCancelJob(h.jobs, h.tasks, tx, h.clk)
return h
}
@@ -468,6 +470,44 @@ func TestSubmitDatasetChunksAndServesInput(t *testing.T) {
}
}
func TestSubmitDatasetLimitsRowsBeforeCreatingShards(t *testing.T) {
h := newHarness()
tsv := "id\tsmiles\nA\tCC\nB\tCCC\nC\tCCCC\nD\tCCCCC\nE\tCCCCCC\n"
res, err := h.submit.Execute(ctx, usecase.SubmitDatasetInput{
Workload: "w", RowsPerShard: 2, MaxRows: 3, Filename: "chembl.tsv",
ContentType: "text/tab-separated-values", Body: strings.NewReader(tsv),
})
if err != nil {
t.Fatal(err)
}
if res.TaskCount != 2 {
t.Fatalf("task_count = %d, want 2", res.TaskCount)
}
}
func TestCancelJobInvalidatesClaimedAndPendingTasks(t *testing.T) {
h := newHarness()
jobID := h.seedJob(t, "w", 3)
claimed, err := h.claim.Execute(ctx, usecase.ClaimTaskInput{WorkerID: "w1", Workloads: []string{"w"}})
if err != nil || claimed == nil {
t.Fatalf("claim: %v", err)
}
cancelled, err := h.cancel.Execute(ctx, jobID)
if err != nil || cancelled != 3 {
t.Fatalf("cancel = (%d, %v), want (3, nil)", cancelled, err)
}
if _, err := h.renew.Execute(ctx, usecase.RenewLeaseInput{TaskID: claimed.TaskID, WorkerID: "w1", Attempt: claimed.Attempt}); !errors.Is(err, domain.ErrTaskNotLeased) {
t.Errorf("cancelled lease heartbeat = %v, want ErrTaskNotLeased", err)
}
progress, err := h.status.Execute(ctx, jobID)
if err != nil || progress.DeriveStatus() != domain.JobCancelled || progress.Cancelled != 3 {
t.Errorf("cancelled progress = %+v, err = %v", progress, err)
}
if cancelled, err := h.cancel.Execute(ctx, jobID); err != nil || cancelled != 0 {
t.Errorf("second cancel = (%d, %v), want (0, nil)", cancelled, err)
}
}
func TestGetTaskInputMissingForURITask(t *testing.T) {
h := newHarness()
h.seedJob(t, "w", 1) // URI-based task, no coordinator-stored input
+23 -3
View File
@@ -52,9 +52,11 @@ Content-Type: multipart/form-data
```
Fields, in order (text fields first, file last — the file is streamed):
`workload`, `parameters` (JSON), `chunk_rows` (int, default 1000), and the file
part `file`. The coordinator stores the input, splits the TSV into shard
artifacts (header repeated per shard), and creates one task per shard.
`workload`, `parameters` (JSON), `chunk_rows` (int, default 1000), optional
`max_rows` (positive int), and the file part `file`. `max_rows` limits the
leading data rows that become shards; it does not change the stored source
artifact. The coordinator splits the selected TSV rows into shard artifacts
(header repeated per shard) and creates one task per shard.
`201`:
@@ -65,6 +67,24 @@ artifacts (header repeated per shard), and creates one task per shard.
Each resulting task's claim response carries `input.uri = /tasks/{id}/input`,
served by §5.4.
## Stop a job
```http
POST /jobs/{job_id}/cancel
Authorization: Bearer <token>
```
The coordinator transactionally marks every pending, leased, or running shard
as `cancelled`, invalidates its lease, and marks the job `cancelled`. Completed
and terminally failed shards remain as history. Repeating a cancellation of an
already cancelled job is safe.
`200`:
```json
{ "job_id": "uuid", "status": "cancelled", "cancelled_tasks": 12 }
```
## Register worker
```http
+32 -2
View File
@@ -95,7 +95,7 @@ paths:
summary: Upload a dataset; the coordinator chunks it into shard tasks
description: >
multipart/form-data. The text fields (`workload`, `parameters`,
`chunk_rows`) MUST precede the `file` part: the file is streamed, not
`chunk_rows`, `max_rows`) MUST precede the `file` part: the file is streamed, not
buffered, so the fields have to be parsed before it arrives.
requestBody:
required: true
@@ -130,6 +130,23 @@ paths:
"401": { $ref: "#/components/responses/Unauthorized" }
"404": { $ref: "#/components/responses/NotFound" }
/jobs/{job_id}/cancel:
post:
tags: [jobs]
summary: Cancel a job and invalidate all unfinished task leases
parameters:
- $ref: "#/components/parameters/JobID"
responses:
"200":
description: The job is cancelled. Completed and terminally failed tasks remain unchanged.
content:
application/json:
schema: { $ref: "#/components/schemas/CancelJobResponse" }
"400": { $ref: "#/components/responses/BadRequest" }
"401": { $ref: "#/components/responses/Unauthorized" }
"404": { $ref: "#/components/responses/NotFound" }
"409": { $ref: "#/components/responses/Conflict" }
/tasks/claim:
post:
tags: [tasks]
@@ -429,6 +446,11 @@ components:
type: integer
description: Data rows per shard. Default 1000.
example: 1000
max_rows:
type: integer
minimum: 1
description: Optional leading data-row limit for a small pipeline check.
example: 500
file:
type: string
format: binary
@@ -441,6 +463,13 @@ components:
task_count: { type: integer, example: 3 }
input_artifact_id: { type: string, format: uuid }
CancelJobResponse:
type: object
properties:
job_id: { type: string, format: uuid }
status: { type: string, enum: [cancelled] }
cancelled_tasks: { type: integer }
JobProgress:
type: object
properties:
@@ -451,6 +480,7 @@ components:
leased: { type: integer }
completed: { type: integer }
failed: { type: integer }
cancelled: { type: integer }
ClaimRequest:
type: object
@@ -549,4 +579,4 @@ components:
TaskStatus:
type: string
enum: [pending, leased, completed, failed, cancelled]
enum: [pending, leased, running, completed, failed, cancelled]
+3
View File
@@ -173,6 +173,9 @@ Rules:
- TSV file, required, streamed; show expected columns
`chembl_id` and `canonical_smiles`.
- `chunk_rows`: integer 1--100000, default 1000.
- `max_rows`: optional positive integer. The coordinator creates shards only
from the first N data rows, so a user can test a large upload without
creating thousands of tasks. It does not truncate the stored source blob.
- optional human-readable run name is a later schema/API addition; v1 does not
silently store it.
- display file name and client-side size only as convenience; server limits and