diff --git a/README.md b/README.md index 7a3fbbd..94e55a5 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,12 @@ Open `http://localhost:18080/ui` and sign in with username `operator` and password `demo-ui-secret`. The command starts PostgreSQL, the coordinator, and two local reference workers. Upload a small ChEMBL TSV, then use the job page to follow shard progress, inspect bounded **Preview CSV** results, and see a -live processing-speed chart in shards per minute. To change the worker count, -run `make demo-ui WORKERS=3`; stop everything with `make demo-down`. +live processing-speed chart in shards per minute. The **Workloads** page shows +the installed SDK workload library (descriptions, parameters, and artifact +schemas) from the embedded catalog; regenerate it with +`make workloads-export` (or `scimesh workload export`) whenever workloads +change. To change the worker count, run `make demo-ui WORKERS=3`; stop +everything with `make demo-down`. Run `make help` to display these commands in the terminal. diff --git a/STATUS.md b/STATUS.md index dcddfa5..cfd8186 100644 --- a/STATUS.md +++ b/STATUS.md @@ -46,7 +46,7 @@ the complete result-artifact SHA-256 before a task is accepted. | CTX-08 Distributed similarity-search | Implemented | Python planner resolves `query_id` once, creates deterministic shard plans, worker adapter emits exact partial top-k CSVs/metrics, and reducer matches the local reference. | | CTX-09 Reducer and final-result API | Implemented | Atomic `reducing` claim, deterministic coordinator-side top-k reducer, sanitized reducer failure, final artifact persistence, `result_uri`, and final CSV download. | | CTX-10 Distributed similarity-graph | Not started | Local reference exists; the SDK-built local graph workload already enforces the pair-coverage invariant. | -| CTX-11 Dashboard/operator view | Implemented | Protected live control room: recent-run/worker overview, real pipeline-stage visualization, shard attempts and safe failures, validated similarity-search upload, coordinator artifacts, final-result download, and bounded polling. | +| CTX-11 Dashboard/operator view | Implemented | Protected live control room: recent-run/worker overview, real pipeline-stage visualization, shard attempts and safe failures, validated similarity-search upload, coordinator artifacts, final-result download, bounded polling, and a Workload library page rendering the embedded catalog from `scimesh workload export` (`/ui/workloads`, regenerated via `make workloads-export`). | | CTX-12 Reliability, security, CI | In progress | Unit, race, PostgreSQL integration, and smoke checks exist; CI hardening remains. | | CTX-15 User Service and access control | Implemented | User/owner scoping, verified contributors, worker keys, self-service enrollment, and quorum-backed untrusted workers are merged; local Go/Python and Docker/PostgreSQL checks passed. | | CTX-16 Workload SDK foundation | Implemented | `scimesh.sdk` provides strict immutable manifests/plans/artifacts, digest/trust-pinned tasks, typed DAGs, compatibility negotiation, verifier primitives with owner/binding-safe quorum inputs, resource eligibility/local allocation, measured package discovery, a trusted local core-batch conformance harness, and strict package discovery. Enforcing coordinator/Worker profiles remain fail-closed. | diff --git a/coordinator/Makefile b/coordinator/Makefile index faab4ee..371037a 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -.PHONY: help build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke demo-ui demo-down demo-reset demo-logs +.PHONY: help build run test test-integration vet lint tidy check migrate-up migrate-down up down down-clean logs ps rebuild psql smoke workloads-export demo-ui demo-down demo-reset demo-logs # `check` deliberately uses its own Compose project and host ports. This keeps # it from connecting to or replacing a developer's local PostgreSQL instance. @@ -26,6 +26,14 @@ DEMO_WORKERS ?= 2 WORKERS ?= $(DEMO_WORKERS) DEMO_DIR ?= .demo +# workloads.json is the UI workload catalog, generated from the Python SDK +# workload library. It is checked in so the binary embeds it; regenerate it +# whenever workloads or their manifests change (requires the Python venv). +WORKLOADS_JSON := internal/transport/http/workloads.json + +workloads-export: + cd .. && .venv/bin/scimesh workload export -o coordinator/$(WORKLOADS_JSON) + help: @printf '%s\n' \ 'SciMesh coordinator commands:' \ @@ -34,6 +42,7 @@ help: ' make demo-logs Follow coordinator logs for the UI demo.' \ ' make demo-down Stop the demo services and workers.' \ ' make demo-reset Stop the demo and wipe its data volumes.' \ + ' make workloads-export Regenerate the embedded UI workload catalog.' \ ' make test / make vet Run Go verification.' \ '' \ 'Demo UI: http://localhost:18080/ui (login page; admin root@scimesh.local / rootpassword).' diff --git a/coordinator/internal/transport/http/server.go b/coordinator/internal/transport/http/server.go index a2967e7..2edd6dc 100644 --- a/coordinator/internal/transport/http/server.go +++ b/coordinator/internal/transport/http/server.go @@ -135,6 +135,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler { }{ {"GET /ui", s.handleUIHome}, {"GET /ui/jobs/new", s.handleUINewJob}, + {"GET /ui/workloads", s.handleUIWorkloads}, {"GET /ui/jobs/{job_id}", s.handleUIJob}, {"GET /ui/api/overview", s.handleUIOverviewJSON}, {"GET /ui/api/jobs/{job_id}", s.handleUIJobJSON}, diff --git a/coordinator/internal/transport/http/templates/dashboard.html b/coordinator/internal/transport/http/templates/dashboard.html index cbbd118..a1ee301 100644 --- a/coordinator/internal/transport/http/templates/dashboard.html +++ b/coordinator/internal/transport/http/templates/dashboard.html @@ -13,7 +13,7 @@

Local scientific compute

SciMesh control room

Follow the real path from a molecular TSV to a globally reduced similarity result—without reading coordinator logs.

Live overview · refreshes every 2 seconds
-
{{if .Session}}Signed in · {{.Session.Role}}{{end}}{{if .Session}}Profile{{end}}{{if and .Session (eq .Session.Role "admin")}}Admin{{end}}{{if .Session}}🖥 Add your machine{{end}}＋ New similarity search{{if .Session}}
{{end}}
+
{{if .Session}}Signed in · {{.Session.Role}}{{end}}{{if .Session}}Profile{{end}}{{if and .Session (eq .Session.Role "admin")}}Admin{{end}}{{if .Session}}Workloads{{end}}{{if .Session}}🖥 Add your machine{{end}}＋ New similarity search{{if .Session}}
{{end}}
How a search becomes a result
01Upload TSVThe coordinator validates and slices the dataset.
02Run shardsWorkers fingerprint molecules and return shard top-k CSVs.
03Merge exactlyThe coordinator ranks retained candidates deterministically.
04Download CSVA checksum-protected global result is ready.
diff --git a/coordinator/internal/transport/http/templates/workloads.html b/coordinator/internal/transport/http/templates/workloads.html new file mode 100644 index 0000000..a5f21c1 --- /dev/null +++ b/coordinator/internal/transport/http/templates/workloads.html @@ -0,0 +1,55 @@ +{{define "workloads.html"}} + + + + + + Workload library · SciMesh + + + +
+

Installed packages

+

Workload library

+

Every SDK-built workload installed in this SciMesh deployment, as declared by scimesh workload export. The coordinator stores this catalog as presentation metadata only; it never executes workload code.

+
+ {{range .Workloads}} +
+
+

{{.Name}}

+

{{.Version}}

+ {{if .Enabled}}enabled{{else}}disabled{{end}} +
+

{{.Description}}

+
{{range .Capabilities}}{{.}}{{end}}
+
+ determinism {{.Determinism}} + verifier {{.Verifier}} + trust {{range $i, $mode := .TrustModes}}{{if $i}}, {{end}}{{$mode}}{{end}} +
+
+
+

Parameters

+ 
{{.Parameters}}
+
+
+ {{range .Inputs}} +

Input · {{.Name}}

{{.Schema}}
+ {{end}} + {{range .Outputs}} +

Output · {{.Name}}

{{.Schema}}
+ {{end}} +
+
+
+ {{else}} +
No workloads are installed.
Install an SDK workload package and run scimesh workload export to republish this catalog.
+ {{end}} +
+

← Back to the control room

+
+ + +{{end}} diff --git a/coordinator/internal/transport/http/ui_workloads.go b/coordinator/internal/transport/http/ui_workloads.go new file mode 100644 index 0000000..e61a596 --- /dev/null +++ b/coordinator/internal/transport/http/ui_workloads.go @@ -0,0 +1,131 @@ +package http + +import ( + "embed" + "encoding/json" + "net/http" + "sort" + "sync" +) + +//go:embed workloads.json +var workloadLibraryFile embed.FS + +// uiWorkloadLibrary is the catalog written by `scimesh workload export`. The +// coordinator never evaluates the schemas in it; it is presentation metadata +// for the operator UI, kept in sync by `make workloads-export`. +type uiWorkloadLibrary struct { + SchemaVersion int `json:"schema_version"` + GeneratedBy string `json:"generated_by"` + Workloads []uiWorkloadRaw `json:"workloads"` +} + +type uiWorkloadRaw struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Capabilities []string `json:"capabilities"` + TrustModes []string `json:"trust_modes"` + Determinism string `json:"determinism"` + Verifier string `json:"verifier"` + Enabled bool `json:"enabled"` + Parameters map[string]any `json:"parameters_schema"` + Inputs map[string]map[string]any `json:"inputs"` + Outputs map[string]map[string]any `json:"outputs"` +} + +type uiPortView struct { + Name string + Schema string +} + +type uiWorkloadView struct { + Name string + Version string + Description string + Capabilities []string + TrustModes []string + Determinism string + Verifier string + Enabled bool + Parameters string + Inputs []uiPortView + Outputs []uiPortView +} + +type uiWorkloadsView struct { + Workloads []uiWorkloadView +} + +var ( + workloadLibraryOnce sync.Once + workloadLibraryView uiWorkloadsView + workloadLibraryErr error +) + +func loadWorkloadLibrary() (uiWorkloadsView, error) { + workloadLibraryOnce.Do(func() { + data, err := workloadLibraryFile.ReadFile("workloads.json") + if err != nil { + workloadLibraryErr = err + return + } + var raw uiWorkloadLibrary + if err := json.Unmarshal(data, &raw); err != nil { + workloadLibraryErr = err + return + } + view := uiWorkloadsView{Workloads: make([]uiWorkloadView, 0, len(raw.Workloads))} + for _, item := range raw.Workloads { + view.Workloads = append(view.Workloads, uiWorkloadView{ + Name: item.Name, + Version: item.Version, + Description: item.Description, + Capabilities: item.Capabilities, + TrustModes: item.TrustModes, + Determinism: item.Determinism, + Verifier: item.Verifier, + Enabled: item.Enabled, + Parameters: prettyJSON(item.Parameters), + Inputs: portViews(item.Inputs), + Outputs: portViews(item.Outputs), + }) + } + workloadLibraryView = view + }) + return workloadLibraryView, workloadLibraryErr +} + +func prettyJSON(value any) string { + if value == nil { + return "{}" + } + encoded, err := json.MarshalIndent(value, "", " ") + if err != nil { + return "{}" + } + return string(encoded) +} + +func portViews(ports map[string]map[string]any) []uiPortView { + names := make([]string, 0, len(ports)) + for name := range ports { + names = append(names, name) + } + sort.Strings(names) + views := make([]uiPortView, 0, len(names)) + for _, name := range names { + views = append(views, uiPortView{Name: name, Schema: prettyJSON(ports[name])}) + } + return views +} + +func (s *Server) handleUIWorkloads(w http.ResponseWriter, r *http.Request) { + view, err := loadWorkloadLibrary() + if err != nil { + s.log.Error("load workload library", "err", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + s.renderUI(w, "workloads.html", view) +} diff --git a/coordinator/internal/transport/http/ui_workloads_internal_test.go b/coordinator/internal/transport/http/ui_workloads_internal_test.go new file mode 100644 index 0000000..e5b7490 --- /dev/null +++ b/coordinator/internal/transport/http/ui_workloads_internal_test.go @@ -0,0 +1,51 @@ +package http + +import ( + "strings" + "testing" +) + +func TestWorkloadLibraryLoadsAndListsEveryWorkload(t *testing.T) { + view, err := loadWorkloadLibrary() + if err != nil { + t.Fatalf("load workload library: %v", err) + } + names := make(map[string]bool) + for _, workload := range view.Workloads { + if workload.Name == "" || workload.Version == "" { + t.Errorf("workload with empty name or version: %+v", workload) + } + if workload.Description == "" { + t.Errorf("workload %s has no description", workload.Name) + } + if workload.Parameters == "" { + t.Errorf("workload %s has no parameter schema", workload.Name) + } + if len(workload.Inputs) == 0 || len(workload.Outputs) == 0 { + t.Errorf("workload %s has no input or output ports", workload.Name) + } + names[workload.Name] = true + } + for _, expected := range []string{"similarity-search", "similarity-graph", "descriptor-batch", "molwt-filter"} { + if !names[expected] { + t.Errorf("workload library is missing %s", expected) + } + } +} + +func TestWorkloadLibraryPageRendersWorkloads(t *testing.T) { + view, err := loadWorkloadLibrary() + if err != nil { + t.Fatalf("load workload library: %v", err) + } + var builder strings.Builder + if err := uiTemplates.ExecuteTemplate(&builder, "workloads.html", view); err != nil { + t.Fatalf("render workloads page: %v", err) + } + page := builder.String() + for _, expected := range []string{"Workload library", "descriptor-batch", "molwt-filter", "byte_exact", "exact-artifact@1"} { + if !strings.Contains(page, expected) { + t.Errorf("workloads page is missing %q", expected) + } + } +} diff --git a/coordinator/internal/transport/http/workloads.json b/coordinator/internal/transport/http/workloads.json new file mode 100644 index 0000000..a4fff29 --- /dev/null +++ b/coordinator/internal/transport/http/workloads.json @@ -0,0 +1,448 @@ +{ + "generated_by": "scimesh workload export", + "schema_version": 1, + "workloads": [ + { + "capabilities": [ + "descriptor-batch" + ], + "description": "Compute a pinned set of RDKit 2D descriptors, one canonical CSV row per input molecule, in deterministic input order.", + "determinism": "byte_exact", + "enabled": true, + "inputs": { + "input": { + "allow_nested_collections": false, + "canonicalizer": "scimesh-tsv-v1", + "encoding": "utf-8", + "max_bytes": 10737418240, + "max_dimensions": [], + "max_records": 100000000, + "media_type": "text/tab-separated-values", + "privacy_class": "project", + "ref": "molecule-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "required_columns": [ + "canonical_smiles", + "chembl_id" + ] + } + } + }, + "name": "descriptor-batch", + "outputs": { + "result": { + "allow_nested_collections": false, + "canonicalizer": "descriptor-table-v1", + "encoding": "utf-8", + "max_bytes": 107374182400, + "max_dimensions": [], + "max_records": 100000000, + "media_type": "text/csv", + "privacy_class": "project", + "ref": "descriptor-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "columns": [ + "chembl_id", + "canonical_smiles", + "ExactMolWt", + "MolWt", + "HeavyAtomMolWt", + "HeavyAtomCount", + "NumHDonors", + "NumHAcceptors", + "NumRotatableBonds", + "NumHeteroatoms", + "NumRadicalElectrons", + "NumValenceElectrons", + "FractionCSP3", + "RingCount", + "NumAromaticRings", + "NumSaturatedRings", + "NumAliphaticRings", + "NumAromaticHeterocycles", + "NumSaturatedHeterocycles", + "NumAliphaticHeterocycles", + "NumAromaticCarbocycles", + "NumSaturatedCarbocycles", + "NumAliphaticCarbocycles", + "TPSA", + "LabuteASA", + "MolLogP", + "MolMR", + "BalabanJ", + "BertzCT", + "HallKierAlpha", + "Kappa1", + "Kappa2", + "Kappa3", + "Chi0", + "Chi1", + "Chi0n", + "Chi1n", + "Chi2n", + "Chi3n", + "Chi4n", + "Chi0v", + "Chi1v", + "Chi2v", + "Chi3v", + "Chi4v", + "PEOE_VSA1", + "PEOE_VSA2", + "PEOE_VSA3", + "PEOE_VSA4", + "PEOE_VSA5", + "PEOE_VSA6", + "PEOE_VSA7", + "PEOE_VSA8", + "PEOE_VSA9", + "PEOE_VSA10", + "PEOE_VSA11", + "PEOE_VSA12", + "PEOE_VSA13", + "PEOE_VSA14", + "SMR_VSA1", + "SMR_VSA2", + "SMR_VSA3", + "SMR_VSA4", + "SMR_VSA5", + "SMR_VSA6", + "SMR_VSA7", + "SMR_VSA8", + "SMR_VSA9", + "SMR_VSA10", + "SlogP_VSA1", + "SlogP_VSA2", + "SlogP_VSA3", + "SlogP_VSA4", + "SlogP_VSA5", + "SlogP_VSA6", + "SlogP_VSA7", + "SlogP_VSA8", + "SlogP_VSA9", + "SlogP_VSA10", + "SlogP_VSA11", + "SlogP_VSA12", + "NHOHCount", + "NOCount" + ] + } + } + }, + "parameters_schema": { + "additionalProperties": false, + "properties": { + "skip_invalid": { + "default": true, + "description": "Skip rows with invalid SMILES instead of failing", + "type": "boolean" + } + }, + "type": "object" + }, + "trust_modes": [ + "trusted", + "untrusted_quorum" + ], + "verifier": "exact-artifact@1", + "version": "1.0.0" + }, + { + "capabilities": [ + "molwt-filter" + ], + "description": "Filter molecules by exact RDKit molecular weight, one canonical CSV row per kept input molecule, in deterministic input order.", + "determinism": "byte_exact", + "enabled": true, + "inputs": { + "input": { + "allow_nested_collections": false, + "canonicalizer": "scimesh-tsv-v1", + "encoding": "utf-8", + "max_bytes": 10737418240, + "max_dimensions": [], + "max_records": 100000000, + "media_type": "text/tab-separated-values", + "privacy_class": "project", + "ref": "molecule-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "required_columns": [ + "canonical_smiles", + "chembl_id" + ] + } + } + }, + "name": "molwt-filter", + "outputs": { + "result": { + "allow_nested_collections": false, + "canonicalizer": "molwt-filtered-table-v1", + "encoding": "utf-8", + "max_bytes": 107374182400, + "max_dimensions": [], + "max_records": 100000000, + "media_type": "text/csv", + "privacy_class": "project", + "ref": "molwt-filtered-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "columns": [ + "chembl_id", + "canonical_smiles", + "molwt" + ] + } + } + }, + "parameters_schema": { + "additionalProperties": false, + "properties": { + "max_molwt": { + "description": "Keep molecules with MolWt <= this value", + "minimum": 0, + "type": "number" + }, + "min_molwt": { + "description": "Keep molecules with MolWt >= this value", + "minimum": 0, + "type": "number" + }, + "skip_invalid": { + "default": true, + "description": "Skip rows with invalid SMILES instead of failing", + "type": "boolean" + } + }, + "type": "object" + }, + "trust_modes": [ + "trusted", + "untrusted_quorum" + ], + "verifier": "exact-artifact@1", + "version": "1.0.0" + }, + { + "capabilities": [ + "similarity-graph" + ], + "description": "Exact sparse Tanimoto similarity graph over deterministic block pairs with a duplicate-safe, coverage-checked merge.", + "determinism": "byte_exact", + "enabled": true, + "inputs": { + "input": { + "allow_nested_collections": false, + "canonicalizer": "scimesh-tsv-v1", + "encoding": "utf-8", + "max_bytes": 10737418240, + "max_dimensions": [], + "max_records": 100000000, + "media_type": "text/tab-separated-values", + "privacy_class": "project", + "ref": "molecule-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "required_columns": [ + "canonical_smiles", + "chembl_id" + ] + } + } + }, + "name": "similarity-graph", + "outputs": { + "result": { + "allow_nested_collections": false, + "canonicalizer": "similarity-edge-table-v1", + "encoding": "utf-8", + "max_bytes": 107374182400, + "max_dimensions": [], + "max_records": 1000000000, + "media_type": "text/csv", + "privacy_class": "project", + "ref": "similarity-edge-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "columns": [ + "source_id", + "target_id", + "similarity" + ] + } + } + }, + "parameters_schema": { + "additionalProperties": false, + "properties": { + "block_size": { + "minimum": 1, + "type": "integer" + }, + "max_rows": { + "minimum": 1, + "type": "integer" + }, + "threshold": { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "threshold_direction": { + "enum": [ + "greater", + "less" + ] + } + }, + "required": [ + "threshold" + ], + "type": "object" + }, + "trust_modes": [ + "trusted", + "untrusted_quorum" + ], + "verifier": "exact-artifact@1", + "version": "1.0.0" + }, + { + "capabilities": [ + "similarity-search" + ], + "description": "Exact top-k Tanimoto molecular similarity search over deterministic TSV shards with a bounded merge.", + "determinism": "byte_exact", + "enabled": true, + "inputs": { + "input": { + "allow_nested_collections": false, + "canonicalizer": "scimesh-tsv-v1", + "encoding": "utf-8", + "max_bytes": 10737418240, + "max_dimensions": [], + "max_records": 100000000, + "media_type": "text/tab-separated-values", + "privacy_class": "project", + "ref": "molecule-table@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "required_columns": [ + "canonical_smiles", + "chembl_id" + ] + } + } + }, + "name": "similarity-search", + "outputs": { + "result": { + "allow_nested_collections": false, + "canonicalizer": "scimesh-search-result-v1", + "encoding": "utf-8", + "max_bytes": 1073741824, + "max_dimensions": [], + "max_records": 100000, + "media_type": "text/csv", + "privacy_class": "project", + "ref": "similarity-search-result@1", + "retention_class": "durable", + "streaming": false, + "validator": "delimited-table@1", + "validator_configuration": { + "columns": [ + "rank", + "chembl_id", + "canonical_smiles", + "similarity" + ] + } + } + }, + "parameters_schema": { + "additionalProperties": false, + "oneOf": [ + { + "not": { + "required": [ + "query_smiles" + ] + }, + "required": [ + "query_id" + ] + }, + { + "not": { + "required": [ + "query_id" + ] + }, + "required": [ + "query_smiles" + ] + } + ], + "properties": { + "max_rows": { + "minimum": 1, + "type": "integer" + }, + "progress_every": { + "minimum": 0, + "type": "integer" + }, + "query_id": { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + "query_smiles": { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + "threshold": { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "threshold_direction": { + "enum": [ + "greater", + "less" + ] + }, + "top_k": { + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + }, + "trust_modes": [ + "trusted", + "untrusted_quorum" + ], + "verifier": "exact-artifact@1", + "version": "1.0.0" + } + ] +} diff --git a/scimesh/workloads/workload_cli.py b/scimesh/workloads/workload_cli.py index 488bee4..5026a5e 100644 --- a/scimesh/workloads/workload_cli.py +++ b/scimesh/workloads/workload_cli.py @@ -28,7 +28,7 @@ class WorkloadCLI: """Inspect and run SDK-built workloads from the command line.""" name = "workload" - help = "List and run SDK-built workloads locally." + help = "List, run, and export SDK-built workloads." def configure_parser(self, parser: argparse.ArgumentParser) -> None: subparsers = parser.add_subparsers(dest="workload_command", required=True) @@ -38,6 +38,18 @@ class WorkloadCLI: ) list_parser.set_defaults(workload_handler=self.list_workloads) + export_parser = subparsers.add_parser( + "export", help="Write the workload library as JSON for the coordinator UI." + ) + export_parser.add_argument( + "-o", + "--output", + type=Path, + default=Path("workloads.json"), + help="Output JSON path (default: workloads.json)", + ) + export_parser.set_defaults(workload_handler=self.export_workloads) + run_parser = subparsers.add_parser( "run", help="Run one SDK workload locally against an input file." ) @@ -107,6 +119,56 @@ class WorkloadCLI: ) return 0 + def export_workloads(self, args: argparse.Namespace) -> int: + """Write the workload library as a JSON catalog for the coordinator UI.""" + import json + + from scimesh.sdk._validation import thaw_json + + registry = self._registry(args) + workloads: list[dict[str, object]] = [] + for item in sorted( + registry.descriptions(), key=lambda value: value.workload.name + ): + definition, _ = registry.require( + item.workload.name, + item.workload.version, + item.package_digest, + ) + manifest = definition.manifest + workloads.append( + { + "name": manifest.workload.name, + "version": manifest.workload.version, + "description": manifest.description, + "capabilities": list(manifest.capabilities), + "trust_modes": [mode.value for mode in manifest.trust_modes], + "determinism": manifest.determinism.value, + "verifier": manifest.verifier.verifier.canonical, + "enabled": item.enabled, + "parameters_schema": thaw_json(manifest.parameters_schema), + "inputs": { + name: port.schema.to_dict() + for name, port in manifest.inputs.items() + }, + "outputs": { + name: port.schema.to_dict() + for name, port in manifest.outputs.items() + }, + } + ) + payload: dict[str, object] = { + "schema_version": 1, + "generated_by": "scimesh workload export", + "workloads": workloads, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as destination: + json.dump(payload, destination, indent=2, sort_keys=True) + destination.write("\n") + print(f"Exported {len(workloads)} workloads to {args.output}") + return 0 + def run_workload(self, args: argparse.Namespace) -> int: registry = self._registry(args) descriptions = registry.descriptions() diff --git a/tests/test_cli_workload.py b/tests/test_cli_workload.py index 9284fd2..29ec950 100644 --- a/tests/test_cli_workload.py +++ b/tests/test_cli_workload.py @@ -227,3 +227,41 @@ def test_workload_cli_runs_an_allowlisted_custom_workload( == 0 ) assert output.read_text(encoding="utf-8") == "id,rows\nshard,1\nshard,1\n" + + +def test_workload_cli_exports_the_library_as_json(tmp_path: Path) -> None: + import json + + output = tmp_path / "workloads.json" + assert main(["workload", "export", "-o", str(output)]) == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["schema_version"] == 1 + names = [item["name"] for item in payload["workloads"]] + assert names == sorted( + ["descriptor-batch", "molwt-filter", "similarity-graph", "similarity-search"] + ) + for item in payload["workloads"]: + assert item["version"] == "1.0.0" + assert item["enabled"] is True + assert item["determinism"] == "byte_exact" + assert item["verifier"] == "exact-artifact@1" + assert "input" in item["inputs"] + assert "result" in item["outputs"] + molwt = next(item for item in payload["workloads"] if item["name"] == "molwt-filter") + assert molwt["parameters_schema"]["properties"] == { + "min_molwt": { + "type": "number", + "minimum": 0, + "description": "Keep molecules with MolWt >= this value", + }, + "max_molwt": { + "type": "number", + "minimum": 0, + "description": "Keep molecules with MolWt <= this value", + }, + "skip_invalid": { + "type": "boolean", + "default": True, + "description": "Skip rows with invalid SMILES instead of failing", + }, + }