This commit is contained in:
@@ -38,7 +38,7 @@ real PostgreSQL smoke test) passed on 2026-07-24.
|
||||
| CTX-08 Distributed similarity-search | Implemented | Python planner resolves `query_id` once, creates deterministic shard plans, worker adapter emits exact partial top-k CSVs/metrics, and reducer matches the local reference. |
|
||||
| CTX-09 Reducer and final-result API | Implemented | Atomic `reducing` claim, deterministic coordinator-side top-k reducer, sanitized reducer failure, final artifact persistence, `result_uri`, and final CSV download. |
|
||||
| CTX-10 Distributed similarity-graph | Not started | Local reference exists. |
|
||||
| CTX-11 Dashboard/operator view | Implemented | Protected local view: job/task/worker status, validated similarity-search upload, partial-artifact diagnostics, final-result download, and bounded polling. |
|
||||
| CTX-11 Dashboard/operator view | Implemented | Protected live control room: recent-run/worker overview, real pipeline-stage visualization, shard attempts and safe failures, validated similarity-search upload, coordinator artifacts, final-result download, and bounded polling. |
|
||||
| CTX-12 Reliability, security, CI | In progress | Unit, race, PostgreSQL integration, and smoke checks exist; CI hardening remains. |
|
||||
|
||||
## Next recommended assignment
|
||||
|
||||
+13
-4
@@ -76,10 +76,19 @@ UI_AUTH_TOKEN='local-ui-secret' make up
|
||||
```
|
||||
|
||||
The UI is disabled by default and never accepts the worker bearer token.
|
||||
It shows recent jobs, task/worker state, diagnostic shard artifacts, and the
|
||||
final CSV for completed similarity-search jobs. The coordinator enters
|
||||
`reducing` after the last shard completes, then exposes the final deterministic
|
||||
global top-k result when merging succeeds.
|
||||
The **control room** shows live workers, recent runs, shard state/attempts,
|
||||
safe failures, coordinator artifacts, and the final CSV for completed
|
||||
similarity-search jobs. The job page follows the real stages: TSV accepted →
|
||||
shards execute → workers return CSVs → `reducing` → final deterministic global
|
||||
top-k result. It polls only its own coordinator read-model and never controls
|
||||
or exposes worker processes.
|
||||
|
||||
For a hands-on run, open `/ui`, choose **New similarity search**, select a
|
||||
small ChEMBL-style TSV, then leave one or more `scimesh-worker` processes
|
||||
running in separate terminals. The detail page updates every two seconds and
|
||||
stops polling after a completed, failed, or cancelled job. Download the
|
||||
`final_result` artifact only after the job reaches **Completed**; shard partial
|
||||
CSVs remain available as diagnostics.
|
||||
|
||||
`up` starts three services in order: Postgres waits until `pg_isready` passes, a
|
||||
one-shot `migrate` container applies the schema and exits, and only then does the
|
||||
|
||||
@@ -133,6 +133,33 @@ func TestClaimReductionIsAtomic(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIReadRepoListsReducerFields(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
job, _ := seedJob(t, pool, 1)
|
||||
jobs := NewJobRepo(pool)
|
||||
ctx := context.Background()
|
||||
if err := jobs.UpdateStatus(ctx, job.ID, domain.JobReducing, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claimed, err := jobs.ClaimReduction(ctx, job.ID, time.Now().UTC()); err != nil || !claimed {
|
||||
t.Fatalf("claim reduction = (%v, %v)", claimed, err)
|
||||
}
|
||||
listed, err := NewUIReadRepo(pool).ListJobs(ctx, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("list UI jobs: %v", err)
|
||||
}
|
||||
for _, item := range listed {
|
||||
if item.ID != job.ID {
|
||||
continue
|
||||
}
|
||||
if item.Status != domain.JobReducing || item.ReducerStartedAt == nil {
|
||||
t.Fatalf("UI reducer projection = %+v", item)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("seeded job %s is missing from UI list", job.ID)
|
||||
}
|
||||
|
||||
// A job must land whole or not at all: a half-created job leaves chunks no
|
||||
// worker could ever complete.
|
||||
func TestCreateJobRollsBackOnFailure(t *testing.T) {
|
||||
|
||||
@@ -41,13 +41,12 @@ func (r *UIReadRepo) ListJobs(ctx context.Context, limit int) ([]domain.Job, err
|
||||
for rows.Next() {
|
||||
var j domain.Job
|
||||
var status string
|
||||
var inputURI *string
|
||||
if err := rows.Scan(&j.ID, &j.Workload, &inputURI, &j.Parameters, &status, &j.CreatedAt, &j.CompletedAt); err != nil {
|
||||
if err := rows.Scan(
|
||||
&j.ID, &j.Workload, &j.InputURI, &j.Parameters, &status, &j.CreatedAt, &j.CompletedAt,
|
||||
&j.InputArtifactID, &j.ResultArtifactID, &j.ErrorCode, &j.ErrorMessage, &j.ReducerStartedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if inputURI != nil {
|
||||
j.InputURI = *inputURI
|
||||
}
|
||||
j.Status = domain.JobStatus(status)
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
|
||||
@@ -81,6 +81,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
|
||||
ui.HandleFunc("GET /ui", s.handleUIHome)
|
||||
ui.HandleFunc("GET /ui/jobs/new", s.handleUINewJob)
|
||||
ui.HandleFunc("GET /ui/jobs/{job_id}", s.handleUIJob)
|
||||
ui.HandleFunc("GET /ui/api/overview", s.handleUIOverviewJSON)
|
||||
ui.HandleFunc("GET /ui/api/jobs/{job_id}", s.handleUIJobJSON)
|
||||
ui.HandleFunc("POST /ui/api/jobs/{job_id}/cancel", s.handleCancelJob)
|
||||
ui.HandleFunc("POST /ui/api/jobs/upload", s.handleUploadDataset)
|
||||
|
||||
@@ -152,11 +152,36 @@ func TestUIRequiresDistinctCredentialAndRendersDashboard(t *testing.T) {
|
||||
t.Fatalf("UI status: %d", resp.StatusCode)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if !strings.Contains(string(body), "SciMesh operator dashboard") {
|
||||
if !strings.Contains(string(body), "SciMesh control room") {
|
||||
t.Errorf("dashboard body missing title")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIOverviewReturnsLiveSafeProjection(t *testing.T) {
|
||||
e := newEnv(t, healthy)
|
||||
code, _ := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c","input_sha256":"sha"}]}`)
|
||||
if code != http.StatusCreated {
|
||||
t.Fatalf("create job: %d", code)
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(context.Background(), "GET", e.ts.URL+"/ui/api/overview", nil)
|
||||
req.SetBasicAuth("operator", uiToken)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var overview map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&overview); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || overview["active_jobs"].(float64) != 1 || overview["online_workers"].(float64) != 1 {
|
||||
t.Fatalf("overview = (%d, %v)", resp.StatusCode, overview)
|
||||
}
|
||||
if _, leaked := overview["worker_auth_token"]; leaked {
|
||||
t.Fatal("overview must not expose authentication configuration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIDisabledReturnsNotFound(t *testing.T) {
|
||||
e := newEnvWithUIToken(t, healthy, "")
|
||||
resp := e.get(t, "/ui")
|
||||
@@ -437,6 +462,33 @@ func TestSimilaritySearchLifecyclePublishesFinalResult(t *testing.T) {
|
||||
if resp.StatusCode != http.StatusOK || string(body) != "rank,chembl_id,canonical_smiles,similarity\n1,A,CC,0.900000\n" {
|
||||
t.Fatalf("final result = (%d, %q)", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
uiRequest, _ := http.NewRequestWithContext(context.Background(), "GET", e.ts.URL+"/ui/jobs/"+jobID, nil)
|
||||
uiRequest.SetBasicAuth("operator", uiToken)
|
||||
uiResponse, err := http.DefaultClient.Do(uiRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer uiResponse.Body.Close()
|
||||
uiBody, _ := io.ReadAll(uiResponse.Body)
|
||||
if uiResponse.StatusCode != http.StatusOK || !strings.Contains(string(uiBody), "Final result ready") {
|
||||
t.Fatalf("final UI = (%d, %q)", uiResponse.StatusCode, uiBody)
|
||||
}
|
||||
|
||||
jsonRequest, _ := http.NewRequestWithContext(context.Background(), "GET", e.ts.URL+"/ui/api/jobs/"+jobID, nil)
|
||||
jsonRequest.SetBasicAuth("operator", uiToken)
|
||||
jsonResponse, err := http.DefaultClient.Do(jsonRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer jsonResponse.Body.Close()
|
||||
var detail map[string]any
|
||||
if err := json.NewDecoder(jsonResponse.Body).Decode(&detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jsonResponse.StatusCode != http.StatusOK || detail["final_result_available"] != true {
|
||||
t.Fatalf("final UI JSON = (%d, %v)", jsonResponse.StatusCode, detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForeignArtifactResultConflict(t *testing.T) {
|
||||
|
||||
@@ -4,20 +4,36 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>SciMesh operator dashboard</title>
|
||||
<title>SciMesh control room</title>
|
||||
<style>
|
||||
:root{color:#172033;background:#f6f8fc;font:16px/1.5 system-ui,sans-serif}body{margin:0}.page{max-width:1180px;margin:auto;padding:32px 20px 56px}.top{display:flex;justify-content:space-between;gap:24px;align-items:start}.eyebrow{margin:0;color:#50617d;font-size:.86rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em}h1{margin:.2rem 0;font-size:2rem}h2{margin:32px 0 12px;font-size:1.28rem}.lead{margin:0;color:#56657c}.button{display:inline-block;border:0;border-radius:8px;padding:11px 15px;background:#1f5eff;color:#fff;font-weight:700;text-decoration:none;white-space:nowrap}.notice{margin-top:24px;padding:16px 18px;border:1px solid #f2cb72;border-radius:10px;background:#fff8e6}.notice strong{display:block}.steps{display:grid;grid-template-columns:repeat(3,1fr);gap:12px;margin-top:14px}.step,.card{padding:16px;border:1px solid #dfe5f0;border-radius:10px;background:#fff}.step b{display:block;color:#1f5eff}.table-wrap{overflow-x:auto;background:#fff;border:1px solid #dfe5f0;border-radius:10px}table{width:100%;border-collapse:collapse}td,th{padding:13px 14px;border-bottom:1px solid #e8ecf4;text-align:left;vertical-align:top}th{color:#50617d;font-size:.78rem;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}a{color:#174ecf}small,.muted{color:#68758b}.status{display:inline-block;border-radius:999px;padding:3px 9px;font-size:.84rem;font-weight:700}.status-success{background:#dff6e9;color:#126b3d}.status-danger{background:#ffe4e6;color:#a31135}.status-active{background:#e4edff;color:#174ecf}.status-waiting{background:#edf0f5;color:#50617d}.bar{height:7px;min-width:120px;margin-top:7px;overflow:hidden;border-radius:999px;background:#e6eaf1}.bar>span{display:block;height:100%;background:#1f5eff}.kicker{font-variant-numeric:tabular-nums}.empty{padding:28px;text-align:center;color:#68758b}.worker{display:grid;grid-template-columns:1.3fr .8fr 2fr 1fr;gap:12px;align-items:center}.worker+.worker{border-top:1px solid #e8ecf4;padding-top:12px;margin-top:12px}@media(max-width:760px){.top,.steps{display:block}.button{margin-top:12px}.step{margin-top:10px}.worker{grid-template-columns:1fr}.hide-mobile{display:none}}
|
||||
:root{color:#dce8ff;background:#08111f;font:16px/1.5 Inter,ui-sans-serif,system-ui,sans-serif;color-scheme:dark}*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 15% -10%,#163d77 0,transparent 32rem),radial-gradient(circle at 95% 5%,#123e39 0,transparent 29rem),#08111f}.page{max-width:1240px;margin:auto;padding:28px 22px 64px}.top{display:flex;align-items:flex-start;justify-content:space-between;gap:22px}.eyebrow{margin:0;color:#78a9ff;font-size:.77rem;font-weight:750;letter-spacing:.14em;text-transform:uppercase}.title{max-width:700px;margin:8px 0;font-size:clamp(2rem,5vw,3.6rem);line-height:1.04;letter-spacing:-.055em}.lead{max-width:690px;margin:0;color:#aabbd5;font-size:1.05rem}.button{display:inline-flex;align-items:center;gap:8px;border:0;border-radius:10px;padding:12px 16px;background:#4f8cff;color:#071224;font:inherit;font-weight:800;text-decoration:none;box-shadow:0 12px 30px #163d7766}.live{display:inline-flex;align-items:center;gap:7px;margin-top:18px;color:#8ba2c2;font-size:.87rem}.pulse{width:8px;height:8px;border-radius:50%;background:#5ee6a6;box-shadow:0 0 0 5px #5ee6a622}.summary{display:grid;grid-template-columns:1.4fr repeat(3,1fr);gap:13px;margin:32px 0}.panel,.metric,.flow-step,.job,.worker{border:1px solid #26415f;border-radius:15px;background:#0d1a2cdd;box-shadow:0 20px 45px #0000001f}.panel{padding:20px}.pipeline{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin-top:16px}.flow-step{position:relative;padding:14px;min-height:116px}.flow-step:not(:last-child):after{content:"";position:absolute;top:35px;right:-10px;width:10px;height:2px;background:#3c5d83}.flow-step b{display:block;color:#f2f7ff}.flow-step small{display:block;margin-top:6px;color:#91a8c6}.flow-step .dot{display:inline-block;width:9px;height:9px;margin-right:7px;border-radius:50%;background:#5ee6a6}.metric{padding:16px}.metric b{display:block;margin-top:7px;color:#f5f8ff;font-size:2rem;line-height:1}.metric span{color:#9bb0cc;font-size:.84rem}.section-head{display:flex;align-items:baseline;justify-content:space-between;gap:12px;margin:36px 0 12px}.section-head h2{margin:0;color:#f3f7ff;font-size:1.18rem}.section-head p{margin:0;color:#8fa6c3;font-size:.9rem}.jobs{display:grid;gap:10px}.job{display:grid;grid-template-columns:minmax(210px,1.35fr) minmax(150px,.9fr) minmax(180px,1fr) auto;gap:18px;align-items:center;padding:17px 18px;text-decoration:none;color:inherit;transition:border-color .15s,transform .15s}.job:hover{border-color:#5d96ee;transform:translateY(-1px)}.job-name{color:#f3f7ff;font-weight:750}.job-id{margin-top:3px;color:#8196b3;font-family:ui-monospace,SFMono-Regular,monospace;font-size:.76rem}.badge{display:inline-flex;align-items:center;border-radius:999px;padding:4px 9px;font-size:.78rem;font-weight:800}.badge-waiting{background:#23344d;color:#b9cce9}.badge-active{background:#173d77;color:#9fc7ff}.badge-success{background:#123f34;color:#76efb5}.badge-danger{background:#552334;color:#ff9bad}.bar{height:7px;margin-top:8px;overflow:hidden;border-radius:999px;background:#20344e}.bar span{display:block;height:100%;border-radius:inherit;background:linear-gradient(90deg,#5a92ff,#61e1bd)}.numbers{color:#afc0d9;font-size:.88rem}.arrow{color:#79aaff;font-size:1.35rem}.workers{display:grid;grid-template-columns:repeat(3,1fr);gap:11px}.worker{padding:15px}.worker-head{display:flex;justify-content:space-between;gap:8px}.worker strong{color:#f3f7ff}.worker small,.worker p{color:#95a9c4}.worker p{margin:12px 0 0}.cap{display:inline-block;margin:4px 5px 0 0;border:1px solid #365576;border-radius:5px;padding:2px 6px;color:#a9c9f4;font:.75rem ui-monospace,SFMono-Regular,monospace}.empty{padding:30px;border:1px dashed #35516f;border-radius:14px;color:#9ab0cb;text-align:center}.offline{color:#faafbd}.sr{position:absolute;width:1px;height:1px;clip:rect(0,0,0,0);overflow:hidden;white-space:nowrap}@media(max-width:820px){.top,.section-head{display:block}.button{margin-top:18px}.summary{grid-template-columns:1fr 1fr}.panel{grid-column:span 2}.pipeline{grid-template-columns:1fr 1fr}.flow-step:not(:last-child):after{display:none}.job{grid-template-columns:1fr 1fr}.arrow{display:none}.workers{grid-template-columns:1fr 1fr}}@media(max-width:540px){.page{padding:22px 14px}.summary,.workers{grid-template-columns:1fr}.panel{grid-column:auto}.pipeline,.job{grid-template-columns:1fr}.title{font-size:2.35rem}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="page">
|
||||
<header class="top"><div><p class="eyebrow">Local coordinator</p><h1>SciMesh operator dashboard</h1><p class="lead">See where a computation is and what should happen next.</p></div><a class="button" href="/ui/jobs/new">Start a check</a></header>
|
||||
<section class="notice" aria-label="Pipeline flow"><strong>Similarity-search jobs produce a final CSV.</strong><span>Workers return shard-level candidates; after every shard succeeds, the coordinator deterministically merges them into one global top-k result.</span><div class="steps"><div class="step"><b>1. Upload TSV</b>The coordinator splits the file into shard tasks.</div><div class="step"><b>2. Wait for a worker</b>A worker claims a shard, calculates similarity, and returns a CSV.</div><div class="step"><b>3. Download result</b>When merging finishes, download the final CSV from the job page.</div></div></section>
|
||||
<h2>Recent jobs</h2>
|
||||
<div class="table-wrap"><table><tr><th>Computation</th><th>State</th><th>Progress</th><th class="hide-mobile">Created</th></tr>{{range .Jobs}}<tr><td><a href="/ui/jobs/{{.ID}}"><strong>{{workloadLabel .Workload}}</strong></a><br><small>Open job details</small></td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><br><small>{{statusHint .Status}}</small></td><td class="kicker"><strong>{{.Completed}} / {{.Total}}</strong> complete{{if gt .Failed 0}} · <span style="color:#a31135">failed: {{.Failed}}</span>{{end}}{{if gt .Cancelled 0}} · <span>stopped: {{.Cancelled}}</span>{{end}}<div class="bar"><span style="width:{{progressPercent .Completed .Failed .Cancelled .Total}}%"></span></div></td><td class="hide-mobile"><small>{{time .CreatedAt}}</small></td></tr>{{else}}<tr><td colspan="4" class="empty"><strong>No jobs yet.</strong><br>Click “Start a check”, upload a small TSV, and leave a worker running.</td></tr>{{end}}</table></div>
|
||||
<h2>Workers</h2>
|
||||
<section class="card">{{range .Workers}}<div class="worker"><div><strong>{{.Name}}</strong><br><small>{{.ID}}</small></div><div><span class="status status-{{if eq .Status "online"}}success{{else}}waiting{{end}}">{{workerStatusLabel .Status}}</span></div><div>{{range .Capabilities}}<code>{{.}}</code> {{end}}</div><div class="muted">Last signal<br>{{time .LastHeartbeatAt}}</div></div>{{else}}<div class="empty"><strong>No worker is registered yet.</strong><br>Run <code>scimesh-worker</code> with the coordinator URL and worker token.</div>{{end}}</section>
|
||||
<header class="top">
|
||||
<div><p class="eyebrow">Local scientific compute</p><h1 class="title">SciMesh control room</h1><p class="lead">Follow the real path from a molecular TSV to a globally reduced similarity result—without reading coordinator logs.</p><div class="live"><i class="pulse"></i><span id="refresh-state">Live overview · refreshes every 2 seconds</span></div></div>
|
||||
<a class="button" href="/ui/jobs/new">+ New similarity search</a>
|
||||
</header>
|
||||
<section class="summary" aria-label="Pipeline summary">
|
||||
<div class="panel"><strong>How a search becomes a result</strong><div class="pipeline"><div class="flow-step"><span><i class="dot"></i>01</span><b>Upload TSV</b><small>The coordinator validates and slices the dataset.</small></div><div class="flow-step"><span><i class="dot"></i>02</span><b>Run shards</b><small>Workers fingerprint molecules and return shard top-k CSVs.</small></div><div class="flow-step"><span><i class="dot"></i>03</span><b>Merge exactly</b><small>The coordinator ranks retained candidates deterministically.</small></div><div class="flow-step"><span><i class="dot"></i>04</span><b>Download CSV</b><small>A checksum-protected global result is ready.</small></div></div></div>
|
||||
<div class="metric"><span>Active runs</span><b id="active-jobs">{{.ActiveJobs}}</b><small>waiting, running, or merging</small></div>
|
||||
<div class="metric"><span>Available workers</span><b id="online-workers">{{.OnlineWorkers}}</b><small>recently registered</small></div>
|
||||
<div class="metric"><span>Finished runs</span><b id="finished-jobs">{{.FinishedJobs}}</b><small>in the latest 20</small></div>
|
||||
</section>
|
||||
|
||||
<section><div class="section-head"><h2>Recent computations</h2><p id="job-count">{{len .Jobs}} shown · newest first</p></div><div id="jobs" class="jobs">{{range .Jobs}}<a class="job" href="/ui/jobs/{{.ID}}"><div><div class="job-name">{{workloadLabel .Workload}}</div><div class="job-id">{{.ID}}</div></div><div><span class="badge badge-{{statusClass .Status}}">{{statusLabel .Status}}</span><div class="job-id">{{statusHint .Status}}</div></div><div><div class="numbers"><b>{{.Completed}}</b> / {{.Total}} shards complete{{if gt .Failed 0}} · <span class="offline">{{.Failed}} failed</span>{{end}}</div><div class="bar"><span style="width:{{progressPercent .Completed .Failed .Cancelled .Total}}%"></span></div></div><span class="arrow" aria-hidden="true">→</span></a>{{else}}<div class="empty"><strong>No computations yet.</strong><br>Start a small similarity search, then keep one or more workers running to watch this dashboard come alive.</div>{{end}}</div></section>
|
||||
<section><div class="section-head"><h2>Worker fleet</h2><p>Workers register themselves; this page never controls their processes.</p></div><div id="workers" class="workers">{{range .Workers}}<article class="worker"><div class="worker-head"><strong>{{.Name}}</strong><span class="badge badge-{{workerStatusClass .Status}}">{{workerStatusLabel .Status}}</span></div><small>{{.ID}}</small><p>{{range .Capabilities}}<span class="cap">{{.}}</span>{{end}}</p><p>Last signal · {{time .LastHeartbeatAt}}</p></article>{{else}}<div class="empty"><strong>No worker is registered.</strong><br>Start <code>scimesh-worker</code> in another terminal, then return here.</div>{{end}}</div></section>
|
||||
</main>
|
||||
<script>
|
||||
const statusInfo={pending:['Waiting for a worker','waiting'],leased:['Assigned to a worker','active'],running:['Running','active'],reducing:['Merging results','active'],completed:['Completed','success'],failed:['Needs attention','danger'],cancelled:['Stopped','waiting']};
|
||||
const pct=j=>j.total?Math.min(100,Math.floor((j.completed+j.failed+j.cancelled)*100/j.total)):0;
|
||||
const node=(tag,text,cls)=>{const n=document.createElement(tag);if(text!==undefined)n.textContent=text;if(cls)n.className=cls;return n};
|
||||
const renderJobs=jobs=>{const box=document.querySelector('#jobs');box.replaceChildren();if(!jobs.length){const empty=node('div',undefined,'empty');empty.append(node('strong','No computations yet.'),document.createElement('br'),document.createTextNode('Start a small similarity search, then keep one or more workers running to watch this dashboard come alive.'));box.append(empty);return}for(const job of jobs){const info=statusInfo[job.status]||[job.status,'waiting'],link=node('a',undefined,'job');link.href='/ui/jobs/'+encodeURIComponent(job.id);const intro=node('div');intro.append(node('div',job.workload==='similarity-search'?'Molecule similarity search':job.workload,'job-name'),node('div',job.id,'job-id'));const state=node('div');state.append(node('span',info[0],'badge badge-'+info[1]),node('div',job.status==='reducing'?'Every shard is complete; coordinator is ranking the global top-k.':'Live coordinator state','job-id'));const progress=node('div'),numbers=node('div',undefined,'numbers');numbers.append(node('b',String(job.completed)),document.createTextNode(' / '+job.total+' shards complete'));if(job.failed){numbers.append(document.createTextNode(' · '),node('span',job.failed+' failed','offline'))}const bar=node('div',undefined,'bar'),fill=node('span');fill.style.width=pct(job)+'%';bar.append(fill);progress.append(numbers,bar);link.append(intro,state,progress,node('span','→','arrow'));box.append(link)}};
|
||||
const renderWorkers=workers=>{const box=document.querySelector('#workers');box.replaceChildren();if(!workers.length){const empty=node('div',undefined,'empty');empty.append(node('strong','No worker is registered.'),document.createElement('br'),document.createTextNode('Start scimesh-worker in another terminal, then return here.'));box.append(empty);return}for(const worker of workers){const card=node('article',undefined,'worker'),head=node('div',undefined,'worker-head'),left=node('div'),workerInfo=worker.status==='online'?['Available','success']:worker.status==='busy'?['Busy','active']:['Offline','waiting'];left.append(node('strong',worker.name),node('small',worker.id));head.append(left,node('span',workerInfo[0],'badge badge-'+workerInfo[1]));const caps=node('p');for(const capability of worker.capabilities||[])caps.append(node('span',capability,'cap'));card.append(head,caps,node('p','Last signal · '+new Date(worker.last_heartbeat_at).toLocaleString()));box.append(card)}};
|
||||
let timer;const refresh=async()=>{try{const response=await fetch('/ui/api/overview',{headers:{Accept:'application/json'}});if(!response.ok)throw Error();const view=await response.json();document.querySelector('#active-jobs').textContent=view.active_jobs;document.querySelector('#online-workers').textContent=view.online_workers;document.querySelector('#finished-jobs').textContent=view.finished_jobs;document.querySelector('#job-count').textContent=view.jobs.length+' shown · newest first';renderJobs(view.jobs);renderWorkers(view.workers);document.querySelector('#refresh-state').textContent='Live overview · updated just now'}catch(_){document.querySelector('#refresh-state').textContent='Connection interrupted · trying again automatically'}};
|
||||
const start=()=>{if(!timer&&!document.hidden)timer=setInterval(refresh,2000)};document.addEventListener('visibilitychange',()=>{if(document.hidden&&timer){clearInterval(timer);timer=undefined}else start()});start();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
{{end}}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -4,27 +4,20 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Create a check — SciMesh</title>
|
||||
<title>New similarity search · SciMesh</title>
|
||||
<style>
|
||||
:root{color:#172033;background:#f6f8fc;font:16px/1.5 system-ui,sans-serif}body{margin:0}.page{max-width:760px;margin:auto;padding:32px 20px 56px}a{color:#174ecf}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#50617d;font-size:.86rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em}h1{margin:0;font-size:2rem}.lead{color:#56657c}.notice{margin:22px 0;padding:16px 18px;border:1px solid #f2cb72;border-radius:10px;background:#fff8e6}.notice strong{display:block}.card{padding:22px;border:1px solid #dfe5f0;border-radius:12px;background:#fff}label{display:block;margin:18px 0 4px;font-weight:700}input{box-sizing:border-box;width:100%;padding:10px;border:1px solid #bac5d8;border-radius:7px;font:inherit}input[type=file]{padding:8px;background:#f8faff}.hint{margin:4px 0;color:#68758b;font-size:.9rem}.button{margin-top:22px;border:0;border-radius:8px;padding:11px 16px;background:#1f5eff;color:#fff;font:inherit;font-weight:700;cursor:pointer}.button:disabled{opacity:.6;cursor:wait}.error{margin-top:16px;color:#a31135}.working{margin-top:16px;color:#174ecf}.checklist{margin:8px 0;padding-left:20px;color:#56657c}.checklist li{margin:5px 0}
|
||||
:root{color:#e5efff;background:#08111f;font:16px/1.5 Inter,ui-sans-serif,system-ui,sans-serif;color-scheme:dark}*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 10% -8%,#183f77 0,transparent 32rem),#08111f}.page{max-width:980px;margin:auto;padding:28px 22px 64px}a{color:#94bdff}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#7baaff;font-size:.78rem;font-weight:800;letter-spacing:.14em;text-transform:uppercase}h1{margin:0;color:#f4f8ff;font-size:clamp(2rem,5vw,3.25rem);letter-spacing:-.055em}.lead{max-width:720px;margin:10px 0 0;color:#aabed9;font-size:1.06rem}.layout{display:grid;grid-template-columns:1.45fr .8fr;gap:15px;margin-top:28px}.card,.aside,.notice{border:1px solid #294662;border-radius:15px;background:#0d1a2cdd;box-shadow:0 20px 45px #00000021}.card{padding:22px}.aside,.notice{padding:18px}.aside h2,.notice h2{margin:0;color:#f1f6ff;font-size:1rem}.aside p,.notice p{color:#9fb3cf}.aside ol{margin:13px 0 0;padding-left:20px;color:#aebfda}.aside li{margin:10px 0}label{display:block;margin:18px 0 5px;color:#eaf2ff;font-weight:750}input,select{width:100%;border:1px solid #42617f;border-radius:9px;padding:10px 11px;background:#0a1626;color:#e5efff;font:inherit}input:focus,select:focus{outline:2px solid #5d97f5;outline-offset:1px}input[type=file]{padding:8px}.hint{margin:5px 0;color:#92a9c6;font-size:.87rem}.split{display:grid;grid-template-columns:1fr 1fr;gap:12px}.run-preview{margin-top:18px;border:1px solid #2c8c70;border-radius:11px;padding:13px;background:#0c2b2a;color:#a8f1d0}.run-preview strong{color:#e6fff4}.button{display:inline-flex;margin-top:22px;border:0;border-radius:10px;padding:12px 16px;background:#67e3b8;color:#062018;font:inherit;font-weight:850;cursor:pointer}.button:disabled{opacity:.6;cursor:wait}.working{margin:14px 0 0;color:#9fc5ff}.error{margin:12px 0 0;color:#ffacba}.hidden{display:none}.cap{display:inline-block;margin:4px 5px 0 0;border:1px solid #375978;border-radius:5px;padding:2px 6px;color:#b5d3f5;font:.76rem ui-monospace,SFMono-Regular,monospace}@media(max-width:720px){.layout,.split{grid-template-columns:1fr}.page{padding:22px 14px}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="page">
|
||||
<a class="back" href="/ui">← Back to jobs</a><p class="eyebrow">Guided run</p><h1>Search for similar molecules</h1><p class="lead">Creates a diagnostic <code>similarity-search</code> job: a worker finds the top-k molecules most similar to a target SMILES.</p>
|
||||
<section class="notice"><strong>Before starting</strong><ul class="checklist"><li>Keep at least one <code>scimesh-worker</code> running.</li><li>Use a small TSV for a hands-on check.</li><li><b>“Rows per shard” does not limit the file size.</b> It splits the entire upload into tasks: a full ChEMBL TSV at 1,000 rows per shard creates thousands of tasks.</li></ul></section>
|
||||
<form id="run" class="card">
|
||||
<label for="file">ChEMBL TSV</label><input id="file" type="file" name="file" required accept=".tsv,.txt,text/tab-separated-values"><p class="hint">Expected columns: <code>chembl_id</code> and <code>canonical_smiles</code>.</p>
|
||||
<label for="query-smiles">Target molecule (SMILES)</label><input id="query-smiles" name="query_smiles" required maxlength="200" value="CCO" autocomplete="off"><p class="hint"><code>CCO</code> is ethanol. For gefitinib, use its SMILES here or the local CLI with <code>--query-id</code>.</p>
|
||||
<label for="top-k">Matches to return</label><input id="top-k" name="top_k" type="number" min="1" max="100000" value="20" required><p class="hint">This is the top-k within each shard, not a global top-k for the whole dataset yet.</p>
|
||||
<label for="chunk-rows">Rows per shard</label><input id="chunk-rows" name="chunk_rows" type="number" min="1" max="100000" value="1000" required><p class="hint">Fewer rows mean more tasks and more visible progress; more rows mean fewer, longer tasks.</p>
|
||||
<label for="max-rows">Maximum dataset rows to process <small>(optional)</small></label><input id="max-rows" name="max_rows" type="number" min="1" max="10000000" placeholder="For example: 500"><p class="hint">Useful for a quick check of a large TSV. The coordinator creates shards from only the first N data rows; it still stores the original upload.</p>
|
||||
<button class="button" id="submit" type="submit">Upload file and create job</button><p id="working" class="working" hidden aria-live="polite">Uploading the file and creating shard tasks… Keep this page open.</p><p id="error" class="error" role="alert"></p>
|
||||
</form>
|
||||
<a class="back" href="/ui">← Back to control room</a><p class="eyebrow">New computation</p><h1>Similarity search, end to end</h1><p class="lead">Upload a ChEMBL-style TSV. Workers calculate shard candidates; after every shard succeeds, SciMesh merges the exact global top-k into one final CSV.</p>
|
||||
<div class="layout"><form id="run" class="card" novalidate><label for="file">ChEMBL TSV</label><input id="file" type="file" name="file" required accept=".tsv,.txt,text/tab-separated-values"><p class="hint">Required columns: <code>chembl_id</code> and <code>canonical_smiles</code>.</p><label for="query-smiles">Target molecule (SMILES)</label><input id="query-smiles" name="query_smiles" required maxlength="200" value="CCO" autocomplete="off"><p class="hint">Use a valid SMILES. The coordinator shares this exact query with every shard.</p><div class="split"><div><label for="top-k">Global top-k</label><input id="top-k" name="top_k" type="number" min="1" max="100000" value="20" required><p class="hint">How many final molecules to retain.</p></div><div><label for="chunk-rows">Rows per shard</label><input id="chunk-rows" name="chunk_rows" type="number" min="1" max="100000" value="1000" required><p class="hint">Smaller shards make more visible tasks.</p></div></div><div class="split"><div><label for="threshold">Similarity threshold <small>(optional)</small></label><input id="threshold" name="threshold" type="number" min="0" max="1" step="0.01" placeholder="For example: 0.70"><p class="hint">Leave blank to rank every valid candidate.</p></div><div><label for="direction">Keep molecules</label><select id="direction" name="threshold_direction"><option value="greater">more similar (≥ threshold)</option><option value="less">less similar (≤ threshold)</option></select><p class="hint">“Less” helps explore dissimilar molecules.</p></div></div><label for="max-rows">Maximum dataset rows <small>(optional quick run)</small></label><input id="max-rows" name="max_rows" type="number" min="1" max="10000000" placeholder="For example: 500"><p class="hint">Only the first N data rows become shards; the original upload remains stored by the coordinator.</p><div id="preview" class="run-preview"><strong>Ready to plan a run.</strong><br>Select a TSV to see the file that will be sent to the coordinator.</div><button class="button" id="submit" type="submit">Create pipeline run →</button><p id="working" class="working hidden" aria-live="polite">Uploading TSV and creating coordinator-owned shards…</p><p id="error" class="error" role="alert"></p></form><aside class="aside"><h2>What you will observe</h2><ol><li><strong>Input accepted</strong><br>TSV is validated and split into durable shard artifacts.</li><li><strong>Workers claim tasks</strong><br>Each worker downloads one shard, fingerprints it, and uploads a partial CSV.</li><li><strong>Global reduction</strong><br>The coordinator compares exact scores from all partial results.</li><li><strong>Final download</strong><br>The result page exposes a checksum-protected global CSV.</li></ol><h2 style="margin-top:26px">Before you submit</h2><p>Keep at least one worker running in another terminal. The browser cannot start or control worker processes.</p><p><span class="cap">similarity-search</span> is currently the only distributed workload available here.</p></aside></div>
|
||||
</main>
|
||||
<script>
|
||||
const form=document.querySelector('#run'),button=document.querySelector('#submit'),working=document.querySelector('#working'),error=document.querySelector('#error');
|
||||
form.addEventListener('submit',async event=>{event.preventDefault();error.textContent='';const fields=new FormData(form),file=fields.get('file'),maxRows=String(fields.get('max_rows')||'').trim();if(!(file instanceof File)||file.size===0){error.textContent='Choose a non-empty TSV file.';return}const parameters={query_smiles:fields.get('query_smiles'),top_k:Number(fields.get('top_k')),progress_every:0},upload=new FormData();upload.append('workload','similarity-search');upload.append('parameters',JSON.stringify(parameters));upload.append('chunk_rows',fields.get('chunk_rows'));if(maxRows)upload.append('max_rows',maxRows);upload.append('file',file,file.name);button.disabled=true;working.hidden=false;try{const response=await fetch('/ui/api/jobs/upload',{method:'POST',body:upload}),data=await response.json();if(!response.ok)throw Error(data.error||'Unable to create the job.');location.href='/ui/jobs/'+data.job_id}catch(err){error.textContent=err.message==='invalid input'?'Check the TSV and fields: the coordinator could not accept this request.':err.message;button.disabled=false;working.hidden=true}});
|
||||
const form=document.querySelector('#run'),button=document.querySelector('#submit'),working=document.querySelector('#working'),error=document.querySelector('#error'),preview=document.querySelector('#preview'),fileInput=document.querySelector('#file');
|
||||
const bytes=size=>size<1024?size+' B':size<1024*1024?(size/1024).toFixed(1)+' KiB':(size/(1024*1024)).toFixed(1)+' MiB';fileInput.addEventListener('change',()=>{const file=fileInput.files&&fileInput.files[0];preview.replaceChildren();if(!file){preview.append(document.createTextNode('Select a TSV to see the file that will be sent to the coordinator.'));return}const strong=document.createElement('strong');strong.textContent='Source ready: '+file.name;preview.append(strong,document.createElement('br'),document.createTextNode(bytes(file.size)+' · the coordinator will validate its header before creating tasks.'))});
|
||||
form.addEventListener('submit',async event=>{event.preventDefault();error.textContent='';const fields=new FormData(form),file=fields.get('file'),query=String(fields.get('query_smiles')||'').trim(),topK=Number(fields.get('top_k')),chunkRows=Number(fields.get('chunk_rows')),threshold=String(fields.get('threshold')||'').trim(),maxRows=String(fields.get('max_rows')||'').trim();if(!(file instanceof File)||file.size===0){error.textContent='Choose a non-empty TSV file.';return}if(!query||query.length>200||!Number.isInteger(topK)||topK<1||!Number.isInteger(chunkRows)||chunkRows<1){error.textContent='Enter a target SMILES, a positive global top-k, and a positive rows-per-shard value.';return}if(threshold&&(Number.isNaN(Number(threshold))||Number(threshold)<0||Number(threshold)>1)){error.textContent='Similarity threshold must be between 0 and 1.';return}const parameters={query_smiles:query,top_k:topK,threshold_direction:fields.get('threshold_direction'),progress_every:0};if(threshold)parameters.threshold=Number(threshold);const upload=new FormData();upload.append('workload','similarity-search');upload.append('parameters',JSON.stringify(parameters));upload.append('chunk_rows',String(chunkRows));if(maxRows)upload.append('max_rows',maxRows);upload.append('file',file,file.name);button.disabled=true;working.classList.remove('hidden');try{const response=await fetch('/ui/api/jobs/upload',{method:'POST',body:upload}),data=await response.json();if(!response.ok)throw Error(data.error||'Unable to create the job.');location.href='/ui/jobs/'+encodeURIComponent(data.job_id)}catch(err){error.textContent=err.message==='invalid input'?'The coordinator could not accept this run. Check the TSV columns and form values.':err.message;button.disabled=false;working.classList.add('hidden')}});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -26,6 +26,7 @@ var uiTemplates = template.Must(template.New("ui").Funcs(template.FuncMap{
|
||||
"taskErrorLabel": uiTaskErrorLabel,
|
||||
"taskErrorHint": uiTaskErrorHint,
|
||||
"workerStatusLabel": uiWorkerStatusLabel,
|
||||
"workerStatusClass": uiWorkerStatusClass,
|
||||
"workloadLabel": uiWorkloadLabel,
|
||||
"progressPercent": uiProgressPercent,
|
||||
"cancellable": uiCancellable,
|
||||
@@ -101,6 +102,8 @@ func uiWorkerStatusLabel(status string) string {
|
||||
switch status {
|
||||
case "online":
|
||||
return "Available"
|
||||
case "busy":
|
||||
return "Busy"
|
||||
case "offline":
|
||||
return "Offline"
|
||||
default:
|
||||
@@ -108,6 +111,17 @@ func uiWorkerStatusLabel(status string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func uiWorkerStatusClass(status string) string {
|
||||
switch status {
|
||||
case "online":
|
||||
return "success"
|
||||
case "busy":
|
||||
return "active"
|
||||
default:
|
||||
return "waiting"
|
||||
}
|
||||
}
|
||||
|
||||
// uiTaskErrorLabel deliberately maps worker implementation errors to an
|
||||
// operator-facing diagnosis. Raw subprocess commands and local paths belong in
|
||||
// the worker terminal, not in the web UI.
|
||||
@@ -207,6 +221,20 @@ func (s *Server) handleUIHome(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderUI(w, "dashboard.html", view)
|
||||
}
|
||||
|
||||
// handleUIOverviewJSON is the bounded polling projection used by the operator
|
||||
// dashboard. It intentionally returns only the safe UI read model, never
|
||||
// worker tokens, storage keys, or database entities.
|
||||
func (s *Server) handleUIOverviewJSON(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := s.reqCtx(r)
|
||||
defer cancel()
|
||||
view, err := s.uc.Dashboard.Overview(ctx, 20)
|
||||
if err != nil {
|
||||
s.writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, view)
|
||||
}
|
||||
|
||||
func (s *Server) handleUINewJob(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderUI(w, "new-job.html", nil)
|
||||
}
|
||||
|
||||
@@ -43,3 +43,12 @@ func TestUITaskErrorPresentationDoesNotExposeCommand(t *testing.T) {
|
||||
t.Error("error hint must explain the failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIWorkerStatusPresentation(t *testing.T) {
|
||||
if got := uiWorkerStatusLabel("busy"); got != "Busy" {
|
||||
t.Errorf("busy worker label = %q", got)
|
||||
}
|
||||
if got := uiWorkerStatusClass("busy"); got != "active" {
|
||||
t.Errorf("busy worker class = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -21,17 +22,21 @@ type UIReadRepository interface {
|
||||
}
|
||||
|
||||
type JobCard struct {
|
||||
ID string `json:"id"`
|
||||
Workload string `json:"workload"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Total int `json:"total"`
|
||||
Pending int `json:"pending"`
|
||||
Leased int `json:"leased"`
|
||||
Running int `json:"running"`
|
||||
Completed int `json:"completed"`
|
||||
Failed int `json:"failed"`
|
||||
Cancelled int `json:"cancelled"`
|
||||
ID string `json:"id"`
|
||||
Workload string `json:"workload"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ReducerStartedAt *time.Time `json:"reducer_started_at,omitempty"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
Total int `json:"total"`
|
||||
Pending int `json:"pending"`
|
||||
Leased int `json:"leased"`
|
||||
Running int `json:"running"`
|
||||
Completed int `json:"completed"`
|
||||
Failed int `json:"failed"`
|
||||
Cancelled int `json:"cancelled"`
|
||||
}
|
||||
|
||||
type TaskCard struct {
|
||||
@@ -42,10 +47,20 @@ type TaskCard struct {
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
LeaseOwner string `json:"lease_owner,omitempty"`
|
||||
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
}
|
||||
|
||||
// ParameterCard is an intentionally small allowlist of run configuration that
|
||||
// helps an operator verify what is being computed without exposing arbitrary
|
||||
// job payloads to the browser.
|
||||
type ParameterCard struct {
|
||||
Label string `json:"label"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type ArtifactCard struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
@@ -65,14 +80,18 @@ type WorkerCard struct {
|
||||
}
|
||||
|
||||
type DashboardView struct {
|
||||
Jobs []JobCard
|
||||
Workers []WorkerCard
|
||||
Jobs []JobCard `json:"jobs"`
|
||||
Workers []WorkerCard `json:"workers"`
|
||||
ActiveJobs int `json:"active_jobs"`
|
||||
FinishedJobs int `json:"finished_jobs"`
|
||||
OnlineWorkers int `json:"online_workers"`
|
||||
}
|
||||
type JobDetailView struct {
|
||||
JobCard
|
||||
Tasks []TaskCard `json:"tasks"`
|
||||
Artifacts []ArtifactCard `json:"artifacts"`
|
||||
FinalResultAvailable bool `json:"final_result_available"`
|
||||
Tasks []TaskCard `json:"tasks"`
|
||||
Artifacts []ArtifactCard `json:"artifacts"`
|
||||
Parameters []ParameterCard `json:"parameters"`
|
||||
FinalResultAvailable bool `json:"final_result_available"`
|
||||
}
|
||||
|
||||
type Dashboard struct{ read UIReadRepository }
|
||||
@@ -98,10 +117,20 @@ func (d *Dashboard) Overview(ctx context.Context, limit int) (DashboardView, err
|
||||
return DashboardView{}, err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
out.Jobs = append(out.Jobs, jobCard(job, tasksByJob[job.ID]))
|
||||
card := jobCard(job, tasksByJob[job.ID])
|
||||
out.Jobs = append(out.Jobs, card)
|
||||
switch card.Status {
|
||||
case string(domain.JobCompleted), string(domain.JobFailed), string(domain.JobCancelled):
|
||||
out.FinishedJobs++
|
||||
default:
|
||||
out.ActiveJobs++
|
||||
}
|
||||
}
|
||||
for _, worker := range workers {
|
||||
out.Workers = append(out.Workers, WorkerCard{ID: worker.ID.String(), Name: worker.Name, Status: string(worker.Status), Capabilities: worker.Capabilities, LastHeartbeatAt: worker.LastHeartbeatAt})
|
||||
if worker.Status == domain.WorkerOnline || worker.Status == domain.WorkerBusy {
|
||||
out.OnlineWorkers++
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -119,11 +148,27 @@ func (d *Dashboard) JobDetail(ctx context.Context, jobID uuid.UUID) (JobDetailVi
|
||||
if err != nil {
|
||||
return JobDetailView{}, err
|
||||
}
|
||||
out := JobDetailView{JobCard: jobCard(*job, tasks), Tasks: make([]TaskCard, 0, len(tasks)), Artifacts: make([]ArtifactCard, 0, len(artifacts))}
|
||||
workers, err := d.read.ListWorkers(ctx, 100)
|
||||
if err != nil {
|
||||
return JobDetailView{}, err
|
||||
}
|
||||
workerNames := make(map[string]string, len(workers))
|
||||
for _, worker := range workers {
|
||||
workerNames[worker.ID.String()] = worker.Name
|
||||
}
|
||||
out := JobDetailView{
|
||||
JobCard: jobCard(*job, tasks),
|
||||
Tasks: make([]TaskCard, 0, len(tasks)),
|
||||
Artifacts: make([]ArtifactCard, 0, len(artifacts)),
|
||||
Parameters: uiParameters(job.Parameters),
|
||||
}
|
||||
for _, task := range tasks {
|
||||
card := TaskCard{ID: task.ID.String(), ChunkIndex: task.ChunkIndex, Status: string(task.Status), Attempt: task.Attempt, MaxAttempts: task.MaxAttempts, LeaseExpiresAt: task.LeaseExpiresAt}
|
||||
card := TaskCard{ID: task.ID.String(), ChunkIndex: task.ChunkIndex, Status: string(task.Status), Attempt: task.Attempt, MaxAttempts: task.MaxAttempts, LeaseExpiresAt: task.LeaseExpiresAt, StartedAt: task.StartedAt, CompletedAt: task.CompletedAt}
|
||||
if task.LeaseOwner != nil {
|
||||
card.LeaseOwner = *task.LeaseOwner
|
||||
card.LeaseOwner = workerNames[*task.LeaseOwner]
|
||||
if card.LeaseOwner == "" {
|
||||
card.LeaseOwner = "Worker " + shortID(*task.LeaseOwner)
|
||||
}
|
||||
}
|
||||
if task.ErrorCode != nil {
|
||||
card.ErrorCode = *task.ErrorCode
|
||||
@@ -158,7 +203,13 @@ func (d *Dashboard) ArtifactBelongsToJob(ctx context.Context, jobID, artifactID
|
||||
}
|
||||
|
||||
func jobCard(job domain.Job, tasks []domain.Task) JobCard {
|
||||
c := JobCard{ID: job.ID.String(), Workload: job.Workload, CreatedAt: job.CreatedAt}
|
||||
c := JobCard{ID: job.ID.String(), Workload: job.Workload, CreatedAt: job.CreatedAt, CompletedAt: job.CompletedAt, ReducerStartedAt: job.ReducerStartedAt}
|
||||
if job.ErrorCode != nil {
|
||||
c.ErrorCode = *job.ErrorCode
|
||||
}
|
||||
if job.ErrorMessage != nil {
|
||||
c.ErrorMessage = *job.ErrorMessage
|
||||
}
|
||||
for _, task := range tasks {
|
||||
c.Total++
|
||||
switch task.Status {
|
||||
@@ -180,3 +231,46 @@ func jobCard(job domain.Job, tasks []domain.Task) JobCard {
|
||||
c.Status = string(p.DeriveStatus())
|
||||
return c
|
||||
}
|
||||
|
||||
func uiParameters(parameters map[string]any) []ParameterCard {
|
||||
keys := []struct {
|
||||
key string
|
||||
label string
|
||||
}{
|
||||
{"query_smiles", "Target SMILES"},
|
||||
{"query_id", "Target ChEMBL ID"},
|
||||
{"top_k", "Global top-k"},
|
||||
{"threshold", "Similarity threshold"},
|
||||
{"threshold_direction", "Threshold direction"},
|
||||
}
|
||||
out := make([]ParameterCard, 0, len(keys))
|
||||
for _, entry := range keys {
|
||||
value, ok := parameters[entry.key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
formatted, ok := formatUIParameter(value)
|
||||
if ok {
|
||||
out = append(out, ParameterCard{Label: entry.label, Value: formatted})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func formatUIParameter(value any) (string, bool) {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return typed, true
|
||||
case int, int64, float64, bool:
|
||||
return fmt.Sprint(typed), true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func shortID(value string) string {
|
||||
if len(value) <= 8 {
|
||||
return value
|
||||
}
|
||||
return value[:8]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package usecase
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestUIParametersAreAllowlisted(t *testing.T) {
|
||||
parameters := uiParameters(map[string]any{
|
||||
"query_smiles": "CCO",
|
||||
"top_k": float64(20),
|
||||
"internal_storage_key": "must-not-reach-browser",
|
||||
"nested": map[string]any{"secret": "no"},
|
||||
})
|
||||
if len(parameters) != 2 {
|
||||
t.Fatalf("parameters = %#v, want only two allowlisted values", parameters)
|
||||
}
|
||||
if parameters[0] != (ParameterCard{Label: "Target SMILES", Value: "CCO"}) ||
|
||||
parameters[1] != (ParameterCard{Label: "Global top-k", Value: "20"}) {
|
||||
t.Fatalf("parameters = %#v", parameters)
|
||||
}
|
||||
}
|
||||
+29
-17
@@ -12,16 +12,28 @@ for a trusted local team. The coordinator remains the only process with direct
|
||||
database and artifact-storage access; the browser never calls PostgreSQL and
|
||||
never receives a worker bearer token.
|
||||
|
||||
The first release must be useful before CTX-07--CTX-10 are complete. Therefore
|
||||
it has two visibly different modes:
|
||||
## Current delivered scope
|
||||
|
||||
The initial operator UI and CTX-09 final reduction are now implemented. The
|
||||
control room polls a bounded, coordinator-owned read model every two seconds
|
||||
while a tab is visible. It shows the worker fleet, recent jobs, safe shard
|
||||
diagnostics, the actual `reducing` phase, and final-result availability. A job
|
||||
detail page renders the concrete pipeline stages—input accepted, shards,
|
||||
worker CSVs, reduction, final CSV—from coordinator state and replaces task and
|
||||
artifact views as work changes. All browser mutations remain limited to
|
||||
validated dataset upload and operator cancellation.
|
||||
|
||||
The interface must distinguish an in-progress distributed search from a run
|
||||
whose reducer has produced a durable final result:
|
||||
|
||||
| Mode | What it proves | What it must not claim |
|
||||
| --- | --- | --- |
|
||||
| **Pipeline check** | Upload, task creation, claim, heartbeat, artifact upload, task completion, retries, and downloads work end-to-end. | That multiple shard results have been scientifically reduced into one answer. |
|
||||
| **Final run** | A reducer has produced a durable final CSV for the full job. | Available only after CTX-09, and for graph only after CTX-10. |
|
||||
| **In-progress run** | Upload, task creation, claim, heartbeat, artifact upload, task completion, retries, and shard diagnostics work end-to-end. | That the partial CSVs are a global scientific answer. |
|
||||
| **Final run** | A reducer has produced a durable final CSV for the full job. | Available for `similarity-search` after CTX-09; graph remains unavailable until CTX-10. |
|
||||
|
||||
Never label a partial artifact as a final molecular result. The UI must show a
|
||||
clear `Pipeline check — partial results` badge while a reducer is unavailable.
|
||||
clear waiting or `reducing` stage until a final artifact exists and the job is
|
||||
`completed`.
|
||||
|
||||
## 2. Constraints and decisions
|
||||
|
||||
@@ -74,7 +86,7 @@ clear `Pipeline check — partial results` badge while a reducer is unavailable.
|
||||
| Worker registration/lease flow | Implemented | Add a read-only worker list; no browser worker controls. |
|
||||
| Task diagnostics | No public list/detail response | Add sanitized job task list with attempt, status, lease owner, expiry and error. |
|
||||
| Artifact download | Worker endpoint exists | Add UI-authorized, job-scoped download proxy. |
|
||||
| Final result | Reducer is not implemented | Gate behind CTX-09; show partial diagnostic artifacts meanwhile. |
|
||||
| Final result | CTX-09 final artifact and download route exist | Show the `reducing` stage, then make the final CSV prominent only for `completed`. |
|
||||
| Distributed graph correctness | Planner/reducer unavailable | Do not advertise a multi-shard graph as final until CTX-10. |
|
||||
|
||||
## 5. Proposed structure
|
||||
@@ -186,11 +198,10 @@ Rules:
|
||||
Inputs: exactly one `query_smiles` or `query_id`, `top_k`, optional threshold,
|
||||
threshold direction, `max_rows`, and `progress_every`.
|
||||
|
||||
For a runnable manual pipeline check before CTX-08, offer `query_smiles` and
|
||||
default `chunk_rows` large enough to create one shard. A `query_id` across
|
||||
multiple shards is disabled with an explanation until CTX-07 resolves it once
|
||||
before fan-out. The detail page calls an artifact a **partial top-k CSV**, not
|
||||
a global top-k, until CTX-09 reduction exists.
|
||||
The current upload form accepts `query_smiles`, because resolving a
|
||||
cross-shard `query_id` has not yet been connected to coordinator uploads. The
|
||||
detail page calls an artifact a **partial top-k CSV** until all shards are
|
||||
complete and CTX-09 reduction stores the final global result.
|
||||
|
||||
### 8.3 Similarity graph
|
||||
|
||||
@@ -283,7 +294,7 @@ checksum/size metadata display, and prominent partial/final labels.
|
||||
file; `Content-Disposition` is safe; preview never loads an unbounded CSV; no
|
||||
final-result button exists before CTX-09.
|
||||
|
||||
### WUI-06 — Final-result UX after CTX-09
|
||||
### WUI-06 — Final-result UX after CTX-09 — implemented
|
||||
|
||||
**Depends on:** CTX-09 and WUI-05.
|
||||
|
||||
@@ -365,8 +376,9 @@ that the interface exists today.
|
||||
|
||||
## 13. Definition of done for the first hand-testable release
|
||||
|
||||
WUI-00 through WUI-05 are complete when a clean local checkout can run a
|
||||
trusted, authenticated local UI; display coordinator readiness, workers, jobs,
|
||||
tasks and safe errors; submit a valid small search pipeline check; poll it to a
|
||||
terminal task state; and download/preview the coordinator-owned partial CSV.
|
||||
The page must make the absence of final reduction impossible to miss.
|
||||
The hand-testable release is complete when a clean local checkout can run a
|
||||
trusted, authenticated local UI; display workers, jobs, pipeline stages, tasks
|
||||
and safe errors; submit a valid small search; poll it through `reducing`; and
|
||||
download the coordinator-owned final CSV only after completion. The page must
|
||||
make the distinction between partial diagnostics and the final result
|
||||
impossible to miss.
|
||||
|
||||
Reference in New Issue
Block a user