diff --git a/coordinator/README.md b/coordinator/README.md index 02ef5d7..c9d90f2 100644 --- a/coordinator/README.md +++ b/coordinator/README.md @@ -86,9 +86,10 @@ or exposes worker processes. For a hands-on run, open `/ui`, choose **New similarity search**, select a small ChEMBL-style TSV, then leave one or more `scimesh-worker` processes running in separate terminals. The detail page updates every two seconds and -stops polling after a completed, failed, or cancelled job. Download the -`final_result` artifact only after the job reaches **Completed**; shard partial -CSVs remain available as diagnostics. +stops polling after a completed, failed, or cancelled job. Use **Preview CSV** +to inspect a bounded first page of a partial or completed final result before +downloading it. The UI never exposes source datasets or shard inputs; partial +CSVs remain available only as diagnostics. `up` starts three services in order: Postgres waits until `pg_isready` passes, a one-shot `migrate` container applies the schema and exits, and only then does the diff --git a/coordinator/cmd/coordinator/main.go b/coordinator/cmd/coordinator/main.go index a200584..46b077c 100644 --- a/coordinator/cmd/coordinator/main.go +++ b/coordinator/cmd/coordinator/main.go @@ -84,6 +84,7 @@ func run() error { GetJobResult: usecase.NewGetJobResult(jobRepo, usecase.NewDownloadArtifact(artifactRepo, blobStore)), GetTaskInput: usecase.NewGetTaskInput(taskRepo, artifactRepo, blobStore), Dashboard: usecase.NewDashboard(uiReadRepo), + PreviewArtifact: usecase.NewPreviewArtifact(uiReadRepo, blobStore), } // Background reapers are tracked so shutdown can wait for them. Without this diff --git a/coordinator/internal/domain/job.go b/coordinator/internal/domain/job.go index 94ff189..16ec023 100644 --- a/coordinator/internal/domain/job.go +++ b/coordinator/internal/domain/job.go @@ -119,6 +119,10 @@ func (p JobProgress) DeriveStatus() JobStatus { switch { case p.Job.Status == JobCancelled: return JobCancelled + case p.Job.Status == JobFailed: + // A reducer may fail after every shard has completed. That terminal + // failure must not be overwritten by an otherwise-complete task count. + return JobFailed case p.Job.Status == JobReducing: return JobReducing case p.Total == 0: diff --git a/coordinator/internal/domain/job_test.go b/coordinator/internal/domain/job_test.go index 54a221d..6ee06aa 100644 --- a/coordinator/internal/domain/job_test.go +++ b/coordinator/internal/domain/job_test.go @@ -82,6 +82,7 @@ func TestDeriveStatus(t *testing.T) { {"done and failed", JobProgress{Total: 3, Done: 2, Failed: 1}, JobFailed}, {"failed but work remains", JobProgress{Total: 3, Pending: 1, Failed: 2}, JobRunning}, {"cancelled job wins over task histogram", JobProgress{Job: Job{Status: JobCancelled}, Total: 3, Done: 1, Cancelled: 2}, JobCancelled}, + {"persisted reducer failure wins over completed tasks", JobProgress{Job: Job{Status: JobFailed}, Total: 3, Done: 3}, JobFailed}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/coordinator/internal/transport/http/server.go b/coordinator/internal/transport/http/server.go index a159ab2..9eb56e2 100644 --- a/coordinator/internal/transport/http/server.go +++ b/coordinator/internal/transport/http/server.go @@ -31,6 +31,7 @@ type UseCases struct { DownloadArtifact *usecase.DownloadArtifact GetTaskInput *usecase.GetTaskInput Dashboard *usecase.Dashboard + PreviewArtifact *usecase.PreviewArtifact } type Server struct { @@ -86,6 +87,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler { ui.HandleFunc("POST /ui/api/jobs/{job_id}/cancel", s.handleCancelJob) ui.HandleFunc("POST /ui/api/jobs/upload", s.handleUploadDataset) ui.HandleFunc("GET /ui/jobs/{job_id}/artifacts/{artifact_id}", s.handleUIArtifactDownload) + ui.HandleFunc("GET /ui/jobs/{job_id}/artifacts/{artifact_id}/preview", s.handleUIArtifactPreview) mux.Handle("/ui", chain(ui, withRequestID, withAccessLog(s.log), withBasicAuth(uiToken[0]), withSameOrigin)) mux.Handle("/ui/", chain(ui, withRequestID, withAccessLog(s.log), withBasicAuth(uiToken[0]), withSameOrigin)) } else { diff --git a/coordinator/internal/transport/http/server_test.go b/coordinator/internal/transport/http/server_test.go index 1146344..29bbd8a 100644 --- a/coordinator/internal/transport/http/server_test.go +++ b/coordinator/internal/transport/http/server_test.go @@ -60,6 +60,7 @@ func newEnvWithUIToken(t *testing.T, ready func(context.Context) error, configur DownloadArtifact: downloadArtifact, GetTaskInput: usecase.NewGetTaskInput(tasks, arts, blobs), Dashboard: usecase.NewDashboard(memstore.NewUIReadRepo(jobs, tasks, work, arts)), + PreviewArtifact: usecase.NewPreviewArtifact(memstore.NewUIReadRepo(jobs, tasks, work, arts), blobs), } worker, err := uc.RegisterWorker.Execute(context.Background(), usecase.RegisterWorkerInput{ Name: "test-worker", Capabilities: []string{"w", "similarity-search"}, @@ -489,6 +490,50 @@ func TestSimilaritySearchLifecyclePublishesFinalResult(t *testing.T) { if jsonResponse.StatusCode != http.StatusOK || detail["final_result_available"] != true { t.Fatalf("final UI JSON = (%d, %v)", jsonResponse.StatusCode, detail) } + artifacts := detail["artifacts"].([]any) + var finalID string + for _, raw := range artifacts { + artifact := raw.(map[string]any) + if artifact["kind"] == "final_result" && artifact["downloadable"] == true { + finalID = artifact["id"].(string) + break + } + } + if finalID == "" { + t.Fatalf("artifacts = %v, want downloadable final result", artifacts) + } + var inputID string + for _, raw := range artifacts { + artifact := raw.(map[string]any) + if artifact["kind"] == "input" { + inputID = artifact["id"].(string) + break + } + } + if inputID == "" { + t.Fatalf("artifacts = %v, want input artifact", artifacts) + } + inputRequest, _ := http.NewRequestWithContext(context.Background(), "GET", e.ts.URL+"/ui/jobs/"+jobID+"/artifacts/"+inputID, nil) + inputRequest.SetBasicAuth("operator", uiToken) + inputResponse, err := http.DefaultClient.Do(inputRequest) + if err != nil { + t.Fatal(err) + } + defer inputResponse.Body.Close() + if inputResponse.StatusCode != http.StatusNotFound { + t.Fatalf("UI input download = %d, want 404", inputResponse.StatusCode) + } + previewRequest, _ := http.NewRequestWithContext(context.Background(), "GET", e.ts.URL+"/ui/jobs/"+jobID+"/artifacts/"+finalID+"/preview", nil) + previewRequest.SetBasicAuth("operator", uiToken) + previewResponse, err := http.DefaultClient.Do(previewRequest) + if err != nil { + t.Fatal(err) + } + defer previewResponse.Body.Close() + previewBody, _ := io.ReadAll(previewResponse.Body) + if previewResponse.StatusCode != http.StatusOK || !strings.Contains(string(previewBody), "Final result preview") || !strings.Contains(string(previewBody), "0.900000") { + t.Fatalf("final preview = (%d, %q)", previewResponse.StatusCode, previewBody) + } } func TestForeignArtifactResultConflict(t *testing.T) { diff --git a/coordinator/internal/transport/http/templates/artifact-preview.html b/coordinator/internal/transport/http/templates/artifact-preview.html new file mode 100644 index 0000000..fa5fbd7 --- /dev/null +++ b/coordinator/internal/transport/http/templates/artifact-preview.html @@ -0,0 +1,35 @@ +{{define "artifact-preview.html"}} + + + + + + SciMesh · artifact preview + + + +
+ ← Back to job +

Preview: {{.Filename}}

+ {{if .Diagnostic}} +

Diagnostic preview — a shard-level partial result, not the final molecular-search answer. At most {{.RowLimit}} rows and {{.ByteLimit}} bytes are read from storage.

+ {{else}} +

Final result preview. At most {{.RowLimit}} rows and {{.ByteLimit}} bytes are read from storage.

+ {{end}} + {{if not .Previewable}} +
{{.Reason}}
+ {{else}} + {{if .Truncated}}
Truncated: showing at most the first {{.RowLimit}} rows or {{.ByteLimit}} bytes. Download the artifact for its full contents.
{{end}} +
+ + {{range .Headers}}{{end}} + {{range .Rows}}{{range .}}{{end}}{{else}}{{end}} +
{{.}}
{{.}}
No data rows.
+
+ {{end}} +
+ + +{{end}} diff --git a/coordinator/internal/transport/http/templates/job.html b/coordinator/internal/transport/http/templates/job.html index 43a390a..63cfbce 100644 --- a/coordinator/internal/transport/http/templates/job.html +++ b/coordinator/internal/transport/http/templates/job.html @@ -21,7 +21,7 @@

Shard activity

Every task is one input partition. The table refreshes while work is in progress.

{{range .Tasks}}{{else}}{{end}}
ShardStateAttemptWorker / leaseOutcome
#{{.ChunkIndex}}{{statusLabel .Status}}{{.Attempt}} / {{.MaxAttempts}}{{if .LeaseOwner}}{{.LeaseOwner}}{{if .LeaseExpiresAt}}
lease until {{time .LeaseExpiresAt}}{{end}}{{else}}{{end}}
{{if .ErrorCode}}{{taskErrorLabel .ErrorCode}}
{{taskErrorHint .ErrorCode}}{{else if eq .Status "completed"}}Partial CSV uploaded{{else}}{{end}}
No shard tasks are present yet.
-

Coordinator artifacts

All files stay coordinator-owned; checksums make downloads auditable.

{{range .Artifacts}}
{{if .Diagnostic}}Partial result · diagnostic{{else}}{{.Kind}}{{end}}
{{.Filename}}{{bytes .SizeBytes}}SHA-256 {{.SHA256}}{{if .Downloadable}}Download CSV{{end}}
{{else}}
Artifacts appear here as the coordinator stores input, shards, partial results, and the final CSV.
{{end}}
+

Coordinator artifacts

All files stay coordinator-owned; checksums make downloads auditable.

{{range .Artifacts}}
{{if .Diagnostic}}Partial result · diagnostic{{else}}{{.Kind}}{{end}}
{{.Filename}}{{bytes .SizeBytes}}SHA-256 {{.SHA256}}{{if .Downloadable}}Preview CSVDownload CSV{{end}}
{{else}}
Artifacts appear here as the coordinator stores input, shards, partial results, and the final CSV.
{{end}}
Technical details

Job ID: {{.ID}}
Created: {{time .CreatedAt}}
Workload: {{.Workload}}