Add workload library page to the operator UI
This commit is contained in:
Emil
@@ -60,8 +60,12 @@ Open `http://localhost:18080/ui` and sign in with username `operator` and
|
||||
password `demo-ui-secret`. The command starts PostgreSQL, the coordinator, and
|
||||
two local reference workers. Upload a small ChEMBL TSV, then use the job page
|
||||
to follow shard progress, inspect bounded **Preview CSV** results, and see a
|
||||
live processing-speed chart in shards per minute. To change the worker count,
|
||||
run `make demo-ui WORKERS=3`; stop everything with `make demo-down`.
|
||||
live processing-speed chart in shards per minute. The **Workloads** page shows
|
||||
the installed SDK workload library (descriptions, parameters, and artifact
|
||||
schemas) from the embedded catalog; regenerate it with
|
||||
`make workloads-export` (or `scimesh workload export`) whenever workloads
|
||||
change. To change the worker count, run `make demo-ui WORKERS=3`; stop
|
||||
everything with `make demo-down`.
|
||||
|
||||
Run `make help` to display these commands in the terminal.
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ the complete result-artifact SHA-256 before a task is accepted.
|
||||
| CTX-08 Distributed similarity-search | Implemented | Python planner resolves `query_id` once, creates deterministic shard plans, worker adapter emits exact partial top-k CSVs/metrics, and reducer matches the local reference. |
|
||||
| CTX-09 Reducer and final-result API | Implemented | Atomic `reducing` claim, deterministic coordinator-side top-k reducer, sanitized reducer failure, final artifact persistence, `result_uri`, and final CSV download. |
|
||||
| CTX-10 Distributed similarity-graph | Not started | Local reference exists; the SDK-built local graph workload already enforces the pair-coverage invariant. |
|
||||
| CTX-11 Dashboard/operator view | Implemented | Protected live control room: recent-run/worker overview, real pipeline-stage visualization, shard attempts and safe failures, validated similarity-search upload, coordinator artifacts, final-result download, and bounded polling. |
|
||||
| CTX-11 Dashboard/operator view | Implemented | Protected live control room: recent-run/worker overview, real pipeline-stage visualization, shard attempts and safe failures, validated similarity-search upload, coordinator artifacts, final-result download, bounded polling, and a Workload library page rendering the embedded catalog from `scimesh workload export` (`/ui/workloads`, regenerated via `make workloads-export`). |
|
||||
| CTX-12 Reliability, security, CI | In progress | Unit, race, PostgreSQL integration, and smoke checks exist; CI hardening remains. |
|
||||
| CTX-15 User Service and access control | Implemented | User/owner scoping, verified contributors, worker keys, self-service enrollment, and quorum-backed untrusted workers are merged; local Go/Python and Docker/PostgreSQL checks passed. |
|
||||
| CTX-16 Workload SDK foundation | Implemented | `scimesh.sdk` provides strict immutable manifests/plans/artifacts, digest/trust-pinned tasks, typed DAGs, compatibility negotiation, verifier primitives with owner/binding-safe quorum inputs, resource eligibility/local allocation, measured package discovery, a trusted local core-batch conformance harness, and strict package discovery. Enforcing coordinator/Worker profiles remain fail-closed. |
|
||||
|
||||
+10
-1
@@ -1,6 +1,6 @@
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
.PHONY: help build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke demo-ui demo-down demo-reset demo-logs
|
||||
.PHONY: help build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke workloads-export demo-ui demo-down demo-reset demo-logs
|
||||
|
||||
# `check` deliberately uses its own Compose project and host ports. This keeps
|
||||
# it from connecting to or replacing a developer's local PostgreSQL instance.
|
||||
@@ -26,6 +26,14 @@ DEMO_WORKERS ?= 2
|
||||
WORKERS ?= $(DEMO_WORKERS)
|
||||
DEMO_DIR ?= .demo
|
||||
|
||||
# workloads.json is the UI workload catalog, generated from the Python SDK
|
||||
# workload library. It is checked in so the binary embeds it; regenerate it
|
||||
# whenever workloads or their manifests change (requires the Python venv).
|
||||
WORKLOADS_JSON := internal/transport/http/workloads.json
|
||||
|
||||
workloads-export:
|
||||
cd .. && .venv/bin/scimesh workload export -o coordinator/$(WORKLOADS_JSON)
|
||||
|
||||
help:
|
||||
@printf '%s\n' \
|
||||
'SciMesh coordinator commands:' \
|
||||
@@ -34,6 +42,7 @@ help:
|
||||
' make demo-logs Follow coordinator logs for the UI demo.' \
|
||||
' make demo-down Stop the demo services and workers.' \
|
||||
' make demo-reset Stop the demo and wipe its data volumes.' \
|
||||
' make workloads-export Regenerate the embedded UI workload catalog.' \
|
||||
' make test / make vet Run Go verification.' \
|
||||
'' \
|
||||
'Demo UI: http://localhost:18080/ui (login page; admin root@scimesh.local / rootpassword).'
|
||||
|
||||
@@ -135,6 +135,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
|
||||
}{
|
||||
{"GET /ui", s.handleUIHome},
|
||||
{"GET /ui/jobs/new", s.handleUINewJob},
|
||||
{"GET /ui/workloads", s.handleUIWorkloads},
|
||||
{"GET /ui/jobs/{job_id}", s.handleUIJob},
|
||||
{"GET /ui/api/overview", s.handleUIOverviewJSON},
|
||||
{"GET /ui/api/jobs/{job_id}", s.handleUIJobJSON},
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<main class="page">
|
||||
<header class="top">
|
||||
<div><p class="eyebrow">Local scientific compute</p><h1 class="title">SciMesh control room</h1><p class="lead">Follow the real path from a molecular TSV to a globally reduced similarity result—without reading coordinator logs.</p><div class="live"><i class="pulse"></i><span id="refresh-state">Live overview · refreshes every 2 seconds</span></div></div>
|
||||
<div style="display:flex;gap:10px;align-items:center;flex-wrap:wrap">{{if .Session}}<span class="live" style="margin-top:0">Signed in · {{.Session.Role}}</span>{{end}}{{if .Session}}<a class="button" href="/ui/profile" style="background:#23344d;color:#dce8ff;box-shadow:none">Profile</a>{{end}}{{if and .Session (eq .Session.Role "admin")}}<a class="button" href="/ui/admin" style="background:#23344d;color:#dce8ff;box-shadow:none">Admin</a>{{end}}{{if .Session}}<a class="button" href="/ui/workers/new" style="background:#23344d;color:#dce8ff;box-shadow:none">🖥 Add your machine</a>{{end}}<a class="button" href="/ui/jobs/new">＋ New similarity search</a>{{if .Session}}<form method="post" action="/ui/logout" style="margin:0"><button class="button" type="submit" style="background:#23344d;color:#dce8ff;box-shadow:none">Log out</button></form>{{end}}</div>
|
||||
<div style="display:flex;gap:10px;align-items:center;flex-wrap:wrap">{{if .Session}}<span class="live" style="margin-top:0">Signed in · {{.Session.Role}}</span>{{end}}{{if .Session}}<a class="button" href="/ui/profile" style="background:#23344d;color:#dce8ff;box-shadow:none">Profile</a>{{end}}{{if and .Session (eq .Session.Role "admin")}}<a class="button" href="/ui/admin" style="background:#23344d;color:#dce8ff;box-shadow:none">Admin</a>{{end}}{{if .Session}}<a class="button" href="/ui/workloads" style="background:#23344d;color:#dce8ff;box-shadow:none">Workloads</a>{{end}}{{if .Session}}<a class="button" href="/ui/workers/new" style="background:#23344d;color:#dce8ff;box-shadow:none">🖥 Add your machine</a>{{end}}<a class="button" href="/ui/jobs/new">＋ New similarity search</a>{{if .Session}}<form method="post" action="/ui/logout" style="margin:0"><button class="button" type="submit" style="background:#23344d;color:#dce8ff;box-shadow:none">Log out</button></form>{{end}}</div>
|
||||
</header>
|
||||
<section class="summary" aria-label="Pipeline summary">
|
||||
<div class="panel"><strong>How a search becomes a result</strong><div class="pipeline"><div class="flow-step"><span><i class="dot"></i>01</span><b>Upload TSV</b><small>The coordinator validates and slices the dataset.</small></div><div class="flow-step"><span><i class="dot"></i>02</span><b>Run shards</b><small>Workers fingerprint molecules and return shard top-k CSVs.</small></div><div class="flow-step"><span><i class="dot"></i>03</span><b>Merge exactly</b><small>The coordinator ranks retained candidates deterministically.</small></div><div class="flow-step"><span><i class="dot"></i>04</span><b>Download CSV</b><small>A checksum-protected global result is ready.</small></div></div></div>
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
{{define "workloads.html"}}
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Workload library · SciMesh</title>
|
||||
<style>
|
||||
:root{color:#e5efff;background:#08111f;font:16px/1.5 Inter,ui-sans-serif,system-ui,sans-serif;color-scheme:dark}*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 12% -8%,#183f77 0,transparent 32rem),radial-gradient(circle at 95% 5%,#123e39 0,transparent 29rem),#08111f}.page{max-width:1240px;margin:auto;padding:28px 22px 64px}a{color:#94bdff}.back{text-decoration:none}.eyebrow{margin:28px 0 4px;color:#7baaff;font-size:.78rem;font-weight:800;letter-spacing:.14em;text-transform:uppercase}h1{margin:0;color:#f4f8ff;font-size:clamp(2rem,5vw,3.25rem);letter-spacing:-.055em}.lead{max-width:760px;margin:10px 0 0;color:#aabed9;font-size:1.06rem}.library{display:grid;gap:14px;margin-top:30px}.workload{border:1px solid #294662;border-radius:15px;background:#0d1a2cdd;box-shadow:0 20px 45px #00000021;padding:20px 22px}.workload-head{display:flex;align-items:baseline;justify-content:space-between;gap:14px;flex-wrap:wrap}.workload-head h2{margin:0;color:#f2f7ff;font-size:1.22rem;letter-spacing:-.02em}.version{margin:0;color:#7d93b2;font:0.82rem ui-monospace,SFMono-Regular,monospace}.description{margin:8px 0 0;color:#b9c9e2;max-width:860px}.cap{display:inline-block;margin:12px 5px 0 0;border:1px solid #375978;border-radius:5px;padding:2px 7px;color:#b5d3f5;font:.76rem ui-monospace,SFMono-Regular,monospace}.badge{display:inline-flex;align-items:center;border-radius:999px;padding:4px 10px;font-size:.78rem;font-weight:800}.badge-success{background:#123f34;color:#76efb5}.badge-waiting{background:#23344d;color:#b9cce9}.meta{display:flex;gap:9px;flex-wrap:wrap;margin-top:14px}.meta span{border:1px solid #2b4a6b;border-radius:7px;padding:3px 8px;color:#a9c3e2;font-size:.8rem}.meta b{color:#dbe9fb;font-weight:750}.schema-grid{display:grid;grid-template-columns:1fr 1fr;gap:13px;margin-top:16px}.schema{border:1px solid #233e5c;border-radius:11px;background:#091627;padding:13px}.schema h3{margin:0 0 8px;color:#cfe1f7;font-size:.86rem;letter-spacing:.04em;text-transform:uppercase}.schema pre{margin:0;overflow:auto;max-height:300px;color:#9fc1e8;font:.76rem ui-monospace,SFMono-Regular,monospace;white-space:pre-wrap;word-break:break-word}.params{padding:13px}.params h3{margin:0 0 8px;color:#cfe1f7;font-size:.86rem;letter-spacing:.04em;text-transform:uppercase}.param{margin:0;padding:6px 0;border-bottom:1px dashed #223a56;color:#b9c9e2;font-size:.9rem}.param:last-child{border-bottom:0}.param b{color:#e8f2ff}.param small{display:block;margin-top:2px;color:#7f96b5}.empty{padding:30px;border:1px dashed #35516f;border-radius:14px;color:#9ab0cb;text-align:center}@media(max-width:820px){.schema-grid{grid-template-columns:1fr}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="page">
|
||||
<p class="eyebrow">Installed packages</p>
|
||||
<h1>Workload library</h1>
|
||||
<p class="lead">Every SDK-built workload installed in this SciMesh deployment, as declared by <code>scimesh workload export</code>. The coordinator stores this catalog as presentation metadata only; it never executes workload code.</p>
|
||||
<div class="library">
|
||||
{{range .Workloads}}
|
||||
<article class="workload">
|
||||
<div class="workload-head">
|
||||
<h2>{{.Name}}</h2>
|
||||
<p class="version">{{.Version}}</p>
|
||||
{{if .Enabled}}<span class="badge badge-success">enabled</span>{{else}}<span class="badge badge-waiting">disabled</span>{{end}}
|
||||
</div>
|
||||
<p class="description">{{.Description}}</p>
|
||||
<div>{{range .Capabilities}}<span class="cap">{{.}}</span>{{end}}</div>
|
||||
<div class="meta">
|
||||
<span>determinism <b>{{.Determinism}}</b></span>
|
||||
<span>verifier <b>{{.Verifier}}</b></span>
|
||||
<span>trust <b>{{range $i, $mode := .TrustModes}}{{if $i}}, {{end}}{{$mode}}{{end}}</b></span>
|
||||
</div>
|
||||
<div class="schema-grid">
|
||||
<div class="schema params">
|
||||
<h3>Parameters</h3>
|
||||
<pre>{{.Parameters}}</pre>
|
||||
</div>
|
||||
<div>
|
||||
{{range .Inputs}}
|
||||
<div class="schema"><h3>Input · {{.Name}}</h3><pre>{{.Schema}}</pre></div>
|
||||
{{end}}
|
||||
{{range .Outputs}}
|
||||
<div class="schema"><h3>Output · {{.Name}}</h3><pre>{{.Schema}}</pre></div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
{{else}}
|
||||
<div class="empty"><strong>No workloads are installed.</strong><br>Install an SDK workload package and run <code>scimesh workload export</code> to republish this catalog.</div>
|
||||
{{end}}
|
||||
</div>
|
||||
<p class="lead" style="margin-top:26px"><a class="back" href="/ui">← Back to the control room</a></p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
{{end}}
|
||||
@@ -0,0 +1,131 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
//go:embed workloads.json
|
||||
var workloadLibraryFile embed.FS
|
||||
|
||||
// uiWorkloadLibrary is the catalog written by `scimesh workload export`. The
|
||||
// coordinator never evaluates the schemas in it; it is presentation metadata
|
||||
// for the operator UI, kept in sync by `make workloads-export`.
|
||||
type uiWorkloadLibrary struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
GeneratedBy string `json:"generated_by"`
|
||||
Workloads []uiWorkloadRaw `json:"workloads"`
|
||||
}
|
||||
|
||||
type uiWorkloadRaw struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Description string `json:"description"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
TrustModes []string `json:"trust_modes"`
|
||||
Determinism string `json:"determinism"`
|
||||
Verifier string `json:"verifier"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Parameters map[string]any `json:"parameters_schema"`
|
||||
Inputs map[string]map[string]any `json:"inputs"`
|
||||
Outputs map[string]map[string]any `json:"outputs"`
|
||||
}
|
||||
|
||||
type uiPortView struct {
|
||||
Name string
|
||||
Schema string
|
||||
}
|
||||
|
||||
type uiWorkloadView struct {
|
||||
Name string
|
||||
Version string
|
||||
Description string
|
||||
Capabilities []string
|
||||
TrustModes []string
|
||||
Determinism string
|
||||
Verifier string
|
||||
Enabled bool
|
||||
Parameters string
|
||||
Inputs []uiPortView
|
||||
Outputs []uiPortView
|
||||
}
|
||||
|
||||
type uiWorkloadsView struct {
|
||||
Workloads []uiWorkloadView
|
||||
}
|
||||
|
||||
var (
|
||||
workloadLibraryOnce sync.Once
|
||||
workloadLibraryView uiWorkloadsView
|
||||
workloadLibraryErr error
|
||||
)
|
||||
|
||||
func loadWorkloadLibrary() (uiWorkloadsView, error) {
|
||||
workloadLibraryOnce.Do(func() {
|
||||
data, err := workloadLibraryFile.ReadFile("workloads.json")
|
||||
if err != nil {
|
||||
workloadLibraryErr = err
|
||||
return
|
||||
}
|
||||
var raw uiWorkloadLibrary
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
workloadLibraryErr = err
|
||||
return
|
||||
}
|
||||
view := uiWorkloadsView{Workloads: make([]uiWorkloadView, 0, len(raw.Workloads))}
|
||||
for _, item := range raw.Workloads {
|
||||
view.Workloads = append(view.Workloads, uiWorkloadView{
|
||||
Name: item.Name,
|
||||
Version: item.Version,
|
||||
Description: item.Description,
|
||||
Capabilities: item.Capabilities,
|
||||
TrustModes: item.TrustModes,
|
||||
Determinism: item.Determinism,
|
||||
Verifier: item.Verifier,
|
||||
Enabled: item.Enabled,
|
||||
Parameters: prettyJSON(item.Parameters),
|
||||
Inputs: portViews(item.Inputs),
|
||||
Outputs: portViews(item.Outputs),
|
||||
})
|
||||
}
|
||||
workloadLibraryView = view
|
||||
})
|
||||
return workloadLibraryView, workloadLibraryErr
|
||||
}
|
||||
|
||||
func prettyJSON(value any) string {
|
||||
if value == nil {
|
||||
return "{}"
|
||||
}
|
||||
encoded, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
func portViews(ports map[string]map[string]any) []uiPortView {
|
||||
names := make([]string, 0, len(ports))
|
||||
for name := range ports {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
views := make([]uiPortView, 0, len(names))
|
||||
for _, name := range names {
|
||||
views = append(views, uiPortView{Name: name, Schema: prettyJSON(ports[name])})
|
||||
}
|
||||
return views
|
||||
}
|
||||
|
||||
func (s *Server) handleUIWorkloads(w http.ResponseWriter, r *http.Request) {
|
||||
view, err := loadWorkloadLibrary()
|
||||
if err != nil {
|
||||
s.log.Error("load workload library", "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.renderUI(w, "workloads.html", view)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWorkloadLibraryLoadsAndListsEveryWorkload(t *testing.T) {
|
||||
view, err := loadWorkloadLibrary()
|
||||
if err != nil {
|
||||
t.Fatalf("load workload library: %v", err)
|
||||
}
|
||||
names := make(map[string]bool)
|
||||
for _, workload := range view.Workloads {
|
||||
if workload.Name == "" || workload.Version == "" {
|
||||
t.Errorf("workload with empty name or version: %+v", workload)
|
||||
}
|
||||
if workload.Description == "" {
|
||||
t.Errorf("workload %s has no description", workload.Name)
|
||||
}
|
||||
if workload.Parameters == "" {
|
||||
t.Errorf("workload %s has no parameter schema", workload.Name)
|
||||
}
|
||||
if len(workload.Inputs) == 0 || len(workload.Outputs) == 0 {
|
||||
t.Errorf("workload %s has no input or output ports", workload.Name)
|
||||
}
|
||||
names[workload.Name] = true
|
||||
}
|
||||
for _, expected := range []string{"similarity-search", "similarity-graph", "descriptor-batch", "molwt-filter"} {
|
||||
if !names[expected] {
|
||||
t.Errorf("workload library is missing %s", expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkloadLibraryPageRendersWorkloads(t *testing.T) {
|
||||
view, err := loadWorkloadLibrary()
|
||||
if err != nil {
|
||||
t.Fatalf("load workload library: %v", err)
|
||||
}
|
||||
var builder strings.Builder
|
||||
if err := uiTemplates.ExecuteTemplate(&builder, "workloads.html", view); err != nil {
|
||||
t.Fatalf("render workloads page: %v", err)
|
||||
}
|
||||
page := builder.String()
|
||||
for _, expected := range []string{"Workload library", "descriptor-batch", "molwt-filter", "byte_exact", "exact-artifact@1"} {
|
||||
if !strings.Contains(page, expected) {
|
||||
t.Errorf("workloads page is missing %q", expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
{
|
||||
"generated_by": "scimesh workload export",
|
||||
"schema_version": 1,
|
||||
"workloads": [
|
||||
{
|
||||
"capabilities": [
|
||||
"descriptor-batch"
|
||||
],
|
||||
"description": "Compute a pinned set of RDKit 2D descriptors, one canonical CSV row per input molecule, in deterministic input order.",
|
||||
"determinism": "byte_exact",
|
||||
"enabled": true,
|
||||
"inputs": {
|
||||
"input": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "scimesh-tsv-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 10737418240,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000000,
|
||||
"media_type": "text/tab-separated-values",
|
||||
"privacy_class": "project",
|
||||
"ref": "molecule-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"required_columns": [
|
||||
"canonical_smiles",
|
||||
"chembl_id"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"name": "descriptor-batch",
|
||||
"outputs": {
|
||||
"result": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "descriptor-table-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 107374182400,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000000,
|
||||
"media_type": "text/csv",
|
||||
"privacy_class": "project",
|
||||
"ref": "descriptor-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"columns": [
|
||||
"chembl_id",
|
||||
"canonical_smiles",
|
||||
"ExactMolWt",
|
||||
"MolWt",
|
||||
"HeavyAtomMolWt",
|
||||
"HeavyAtomCount",
|
||||
"NumHDonors",
|
||||
"NumHAcceptors",
|
||||
"NumRotatableBonds",
|
||||
"NumHeteroatoms",
|
||||
"NumRadicalElectrons",
|
||||
"NumValenceElectrons",
|
||||
"FractionCSP3",
|
||||
"RingCount",
|
||||
"NumAromaticRings",
|
||||
"NumSaturatedRings",
|
||||
"NumAliphaticRings",
|
||||
"NumAromaticHeterocycles",
|
||||
"NumSaturatedHeterocycles",
|
||||
"NumAliphaticHeterocycles",
|
||||
"NumAromaticCarbocycles",
|
||||
"NumSaturatedCarbocycles",
|
||||
"NumAliphaticCarbocycles",
|
||||
"TPSA",
|
||||
"LabuteASA",
|
||||
"MolLogP",
|
||||
"MolMR",
|
||||
"BalabanJ",
|
||||
"BertzCT",
|
||||
"HallKierAlpha",
|
||||
"Kappa1",
|
||||
"Kappa2",
|
||||
"Kappa3",
|
||||
"Chi0",
|
||||
"Chi1",
|
||||
"Chi0n",
|
||||
"Chi1n",
|
||||
"Chi2n",
|
||||
"Chi3n",
|
||||
"Chi4n",
|
||||
"Chi0v",
|
||||
"Chi1v",
|
||||
"Chi2v",
|
||||
"Chi3v",
|
||||
"Chi4v",
|
||||
"PEOE_VSA1",
|
||||
"PEOE_VSA2",
|
||||
"PEOE_VSA3",
|
||||
"PEOE_VSA4",
|
||||
"PEOE_VSA5",
|
||||
"PEOE_VSA6",
|
||||
"PEOE_VSA7",
|
||||
"PEOE_VSA8",
|
||||
"PEOE_VSA9",
|
||||
"PEOE_VSA10",
|
||||
"PEOE_VSA11",
|
||||
"PEOE_VSA12",
|
||||
"PEOE_VSA13",
|
||||
"PEOE_VSA14",
|
||||
"SMR_VSA1",
|
||||
"SMR_VSA2",
|
||||
"SMR_VSA3",
|
||||
"SMR_VSA4",
|
||||
"SMR_VSA5",
|
||||
"SMR_VSA6",
|
||||
"SMR_VSA7",
|
||||
"SMR_VSA8",
|
||||
"SMR_VSA9",
|
||||
"SMR_VSA10",
|
||||
"SlogP_VSA1",
|
||||
"SlogP_VSA2",
|
||||
"SlogP_VSA3",
|
||||
"SlogP_VSA4",
|
||||
"SlogP_VSA5",
|
||||
"SlogP_VSA6",
|
||||
"SlogP_VSA7",
|
||||
"SlogP_VSA8",
|
||||
"SlogP_VSA9",
|
||||
"SlogP_VSA10",
|
||||
"SlogP_VSA11",
|
||||
"SlogP_VSA12",
|
||||
"NHOHCount",
|
||||
"NOCount"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters_schema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"skip_invalid": {
|
||||
"default": true,
|
||||
"description": "Skip rows with invalid SMILES instead of failing",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"trust_modes": [
|
||||
"trusted",
|
||||
"untrusted_quorum"
|
||||
],
|
||||
"verifier": "exact-artifact@1",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"capabilities": [
|
||||
"molwt-filter"
|
||||
],
|
||||
"description": "Filter molecules by exact RDKit molecular weight, one canonical CSV row per kept input molecule, in deterministic input order.",
|
||||
"determinism": "byte_exact",
|
||||
"enabled": true,
|
||||
"inputs": {
|
||||
"input": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "scimesh-tsv-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 10737418240,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000000,
|
||||
"media_type": "text/tab-separated-values",
|
||||
"privacy_class": "project",
|
||||
"ref": "molecule-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"required_columns": [
|
||||
"canonical_smiles",
|
||||
"chembl_id"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"name": "molwt-filter",
|
||||
"outputs": {
|
||||
"result": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "molwt-filtered-table-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 107374182400,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000000,
|
||||
"media_type": "text/csv",
|
||||
"privacy_class": "project",
|
||||
"ref": "molwt-filtered-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"columns": [
|
||||
"chembl_id",
|
||||
"canonical_smiles",
|
||||
"molwt"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters_schema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"max_molwt": {
|
||||
"description": "Keep molecules with MolWt <= this value",
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
"min_molwt": {
|
||||
"description": "Keep molecules with MolWt >= this value",
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
"skip_invalid": {
|
||||
"default": true,
|
||||
"description": "Skip rows with invalid SMILES instead of failing",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"trust_modes": [
|
||||
"trusted",
|
||||
"untrusted_quorum"
|
||||
],
|
||||
"verifier": "exact-artifact@1",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"capabilities": [
|
||||
"similarity-graph"
|
||||
],
|
||||
"description": "Exact sparse Tanimoto similarity graph over deterministic block pairs with a duplicate-safe, coverage-checked merge.",
|
||||
"determinism": "byte_exact",
|
||||
"enabled": true,
|
||||
"inputs": {
|
||||
"input": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "scimesh-tsv-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 10737418240,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000000,
|
||||
"media_type": "text/tab-separated-values",
|
||||
"privacy_class": "project",
|
||||
"ref": "molecule-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"required_columns": [
|
||||
"canonical_smiles",
|
||||
"chembl_id"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"name": "similarity-graph",
|
||||
"outputs": {
|
||||
"result": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "similarity-edge-table-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 107374182400,
|
||||
"max_dimensions": [],
|
||||
"max_records": 1000000000,
|
||||
"media_type": "text/csv",
|
||||
"privacy_class": "project",
|
||||
"ref": "similarity-edge-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"columns": [
|
||||
"source_id",
|
||||
"target_id",
|
||||
"similarity"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters_schema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"block_size": {
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
"max_rows": {
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
"threshold": {
|
||||
"maximum": 1,
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
"threshold_direction": {
|
||||
"enum": [
|
||||
"greater",
|
||||
"less"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"threshold"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"trust_modes": [
|
||||
"trusted",
|
||||
"untrusted_quorum"
|
||||
],
|
||||
"verifier": "exact-artifact@1",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"capabilities": [
|
||||
"similarity-search"
|
||||
],
|
||||
"description": "Exact top-k Tanimoto molecular similarity search over deterministic TSV shards with a bounded merge.",
|
||||
"determinism": "byte_exact",
|
||||
"enabled": true,
|
||||
"inputs": {
|
||||
"input": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "scimesh-tsv-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 10737418240,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000000,
|
||||
"media_type": "text/tab-separated-values",
|
||||
"privacy_class": "project",
|
||||
"ref": "molecule-table@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"required_columns": [
|
||||
"canonical_smiles",
|
||||
"chembl_id"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"name": "similarity-search",
|
||||
"outputs": {
|
||||
"result": {
|
||||
"allow_nested_collections": false,
|
||||
"canonicalizer": "scimesh-search-result-v1",
|
||||
"encoding": "utf-8",
|
||||
"max_bytes": 1073741824,
|
||||
"max_dimensions": [],
|
||||
"max_records": 100000,
|
||||
"media_type": "text/csv",
|
||||
"privacy_class": "project",
|
||||
"ref": "similarity-search-result@1",
|
||||
"retention_class": "durable",
|
||||
"streaming": false,
|
||||
"validator": "delimited-table@1",
|
||||
"validator_configuration": {
|
||||
"columns": [
|
||||
"rank",
|
||||
"chembl_id",
|
||||
"canonical_smiles",
|
||||
"similarity"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters_schema": {
|
||||
"additionalProperties": false,
|
||||
"oneOf": [
|
||||
{
|
||||
"not": {
|
||||
"required": [
|
||||
"query_smiles"
|
||||
]
|
||||
},
|
||||
"required": [
|
||||
"query_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"not": {
|
||||
"required": [
|
||||
"query_id"
|
||||
]
|
||||
},
|
||||
"required": [
|
||||
"query_smiles"
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"max_rows": {
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
"progress_every": {
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
"query_id": {
|
||||
"maxLength": 200,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
"query_smiles": {
|
||||
"maxLength": 200,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
"threshold": {
|
||||
"maximum": 1,
|
||||
"minimum": 0,
|
||||
"type": "number"
|
||||
},
|
||||
"threshold_direction": {
|
||||
"enum": [
|
||||
"greater",
|
||||
"less"
|
||||
]
|
||||
},
|
||||
"top_k": {
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"trust_modes": [
|
||||
"trusted",
|
||||
"untrusted_quorum"
|
||||
],
|
||||
"verifier": "exact-artifact@1",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -28,7 +28,7 @@ class WorkloadCLI:
|
||||
"""Inspect and run SDK-built workloads from the command line."""
|
||||
|
||||
name = "workload"
|
||||
help = "List and run SDK-built workloads locally."
|
||||
help = "List, run, and export SDK-built workloads."
|
||||
|
||||
def configure_parser(self, parser: argparse.ArgumentParser) -> None:
|
||||
subparsers = parser.add_subparsers(dest="workload_command", required=True)
|
||||
@@ -38,6 +38,18 @@ class WorkloadCLI:
|
||||
)
|
||||
list_parser.set_defaults(workload_handler=self.list_workloads)
|
||||
|
||||
export_parser = subparsers.add_parser(
|
||||
"export", help="Write the workload library as JSON for the coordinator UI."
|
||||
)
|
||||
export_parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("workloads.json"),
|
||||
help="Output JSON path (default: workloads.json)",
|
||||
)
|
||||
export_parser.set_defaults(workload_handler=self.export_workloads)
|
||||
|
||||
run_parser = subparsers.add_parser(
|
||||
"run", help="Run one SDK workload locally against an input file."
|
||||
)
|
||||
@@ -107,6 +119,56 @@ class WorkloadCLI:
|
||||
)
|
||||
return 0
|
||||
|
||||
def export_workloads(self, args: argparse.Namespace) -> int:
|
||||
"""Write the workload library as a JSON catalog for the coordinator UI."""
|
||||
import json
|
||||
|
||||
from scimesh.sdk._validation import thaw_json
|
||||
|
||||
registry = self._registry(args)
|
||||
workloads: list[dict[str, object]] = []
|
||||
for item in sorted(
|
||||
registry.descriptions(), key=lambda value: value.workload.name
|
||||
):
|
||||
definition, _ = registry.require(
|
||||
item.workload.name,
|
||||
item.workload.version,
|
||||
item.package_digest,
|
||||
)
|
||||
manifest = definition.manifest
|
||||
workloads.append(
|
||||
{
|
||||
"name": manifest.workload.name,
|
||||
"version": manifest.workload.version,
|
||||
"description": manifest.description,
|
||||
"capabilities": list(manifest.capabilities),
|
||||
"trust_modes": [mode.value for mode in manifest.trust_modes],
|
||||
"determinism": manifest.determinism.value,
|
||||
"verifier": manifest.verifier.verifier.canonical,
|
||||
"enabled": item.enabled,
|
||||
"parameters_schema": thaw_json(manifest.parameters_schema),
|
||||
"inputs": {
|
||||
name: port.schema.to_dict()
|
||||
for name, port in manifest.inputs.items()
|
||||
},
|
||||
"outputs": {
|
||||
name: port.schema.to_dict()
|
||||
for name, port in manifest.outputs.items()
|
||||
},
|
||||
}
|
||||
)
|
||||
payload: dict[str, object] = {
|
||||
"schema_version": 1,
|
||||
"generated_by": "scimesh workload export",
|
||||
"workloads": workloads,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("w", encoding="utf-8") as destination:
|
||||
json.dump(payload, destination, indent=2, sort_keys=True)
|
||||
destination.write("\n")
|
||||
print(f"Exported {len(workloads)} workloads to {args.output}")
|
||||
return 0
|
||||
|
||||
def run_workload(self, args: argparse.Namespace) -> int:
|
||||
registry = self._registry(args)
|
||||
descriptions = registry.descriptions()
|
||||
|
||||
@@ -227,3 +227,41 @@ def test_workload_cli_runs_an_allowlisted_custom_workload(
|
||||
== 0
|
||||
)
|
||||
assert output.read_text(encoding="utf-8") == "id,rows\nshard,1\nshard,1\n"
|
||||
|
||||
|
||||
def test_workload_cli_exports_the_library_as_json(tmp_path: Path) -> None:
|
||||
import json
|
||||
|
||||
output = tmp_path / "workloads.json"
|
||||
assert main(["workload", "export", "-o", str(output)]) == 0
|
||||
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||
assert payload["schema_version"] == 1
|
||||
names = [item["name"] for item in payload["workloads"]]
|
||||
assert names == sorted(
|
||||
["descriptor-batch", "molwt-filter", "similarity-graph", "similarity-search"]
|
||||
)
|
||||
for item in payload["workloads"]:
|
||||
assert item["version"] == "1.0.0"
|
||||
assert item["enabled"] is True
|
||||
assert item["determinism"] == "byte_exact"
|
||||
assert item["verifier"] == "exact-artifact@1"
|
||||
assert "input" in item["inputs"]
|
||||
assert "result" in item["outputs"]
|
||||
molwt = next(item for item in payload["workloads"] if item["name"] == "molwt-filter")
|
||||
assert molwt["parameters_schema"]["properties"] == {
|
||||
"min_molwt": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Keep molecules with MolWt >= this value",
|
||||
},
|
||||
"max_molwt": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Keep molecules with MolWt <= this value",
|
||||
},
|
||||
"skip_invalid": {
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "Skip rows with invalid SMILES instead of failing",
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user