Create the worker directory before the wizard downloads the release wheel

README.md

@@ -225,6 +225,10 @@ The coordinator and worker agent are Go modules under `coordinator/` and `users/
 cd coordinator && make coordinator agent && go test ./...
 ```
 
+On headless servers (no desktop environment), RDKit needs a few X11
+libraries that desktops already ship: `sudo apt-get install -y libxrender1
+libxext6 libxcursor1 libxfixes3 libxi6 libxrandr2`.
+
 `make check` runs the full gate: vet, lint, race tests, the PostgreSQL
 integration suite, and the two-worker end-to-end smoke script.
 

coordinator/cmd/worker-agent/main.go

@@ -58,7 +58,7 @@ func main() {
 			fmt.Println("check: no coordinator URL (pass --coordinator-url or set COORDINATOR_URL)")
 			os.Exit(1)
 		}
-		report := agent.RunCheck(ctx, url)
+		report := agent.RunCheck(ctx, url, "")
 		printCheck(report)
 		if !report.Coordinator.OK || !report.Python.OK || !report.Scimesh.OK {
 			os.Exit(1)

coordinator/internal/agent/check.go

@@ -70,17 +70,23 @@ func CheckCoordinator(ctx context.Context, url string, timeout time.Duration) Ch
 	return report
 }
 
-// CheckEnvironment verifies the local runtime: Python present and the scimesh
-// package importable.
+// CheckEnvironment verifies the local runtime against the python3 found on
+// PATH.
 func CheckEnvironment(ctx context.Context) CheckReport {
-	report := CheckReport{Agent: Version}
 	python, err := exec.LookPath("python3")
 	if err != nil {
-		report.Python = CheckItem{Name: "python", OK: false, Detail: "python3 not found on PATH"}
-		return report
+		return CheckReport{Agent: Version, Python: CheckItem{Name: "python", OK: false, Detail: "python3 not found on PATH"}}
 	}
-	report.Python = CheckItem{Name: "python", OK: true, Detail: python}
-	//nolint:gosec // G204: python comes from LookPath, the argument list is constant
+	return CheckEnvironmentWithPython(ctx, python)
+}
+
+// CheckEnvironmentWithPython verifies the local runtime against a specific
+// interpreter — the wizard's managed venv python when the runtime installer
+// has created one, so the preflight reflects what the worker will actually
+// execute with.
+func CheckEnvironmentWithPython(ctx context.Context, python string) CheckReport {
+	report := CheckReport{Agent: Version, Python: CheckItem{Name: "python", OK: true, Detail: python}}
+	//nolint:gosec // G204: python is a resolved interpreter path, the argument list is constant
 	cmd := exec.CommandContext(ctx, python, "-c", "import scimesh; print(scimesh.__version__ if hasattr(scimesh, '__version__') else 'installed')")
 	out, err := cmd.Output()
 	if err != nil {
@@ -96,10 +102,17 @@ func CheckEnvironment(ctx context.Context) CheckReport {
 }
 
 // RunCheck combines the coordinator probe and the local environment probe; it
-// is the body behind `worker-agent --check` and the wizard's test step.
-func RunCheck(ctx context.Context, coordinatorURL string) CheckReport {
+// is the body behind `worker-agent --check` and the wizard's test step. A
+// non-empty python overrides the interpreter probed for the scimesh package
+// (the managed venv after a runtime install).
+func RunCheck(ctx context.Context, coordinatorURL, python string) CheckReport {
 	report := CheckCoordinator(ctx, coordinatorURL, 15*time.Second)
-	env := CheckEnvironment(ctx)
+	var env CheckReport
+	if python != "" {
+		env = CheckEnvironmentWithPython(ctx, python)
+	} else {
+		env = CheckEnvironment(ctx)
+	}
 	report.Python = env.Python
 	report.Scimesh = env.Scimesh
 	return report

coordinator/internal/agent/releasewheel.go

@@ -38,6 +38,9 @@ func NormalizePEP440(version string) string {
 // with a generous timeout: wheels can be several MB.
 func DownloadWheel(ctx context.Context, url, dir string) (string, error) {
 	target := filepath.Join(dir, wheelNameFromURL(url))
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)

coordinator/internal/agent/setupui/server.go

@@ -300,6 +300,25 @@ type statusView struct {
 	TokenSet      bool   `json:"token_set"`
 }
 
+// ensureVenvTaskRunner rewrites the saved config so its task runner uses the
+// managed venv python when one exists and the config does not already pin one.
+func (s *Server) ensureVenvTaskRunner() {
+	raw, err := os.ReadFile(s.cfgPath)
+	if err != nil {
+		return
+	}
+	var file agent.ConfigFile
+	if json.Unmarshal(raw, &file) != nil || len(file.TaskRunner) > 0 {
+		return
+	}
+	if venv := s.venvPython(); venv != "" {
+		file.TaskRunner = []string{venv, "-m", "scimesh.worker.task"}
+		if payload, err := json.MarshalIndent(file, "", "  "); err == nil {
+			_ = os.WriteFile(s.cfgPath, append(payload, '\n'), 0o600)
+		}
+	}
+}
+
 func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
 	view := statusView{ConfigPath: s.cfgPath, LogPath: s.logPath, Running: s.sup.Alive(), Pid: s.sup.Pid()}
 	if raw, err := os.ReadFile(s.cfgPath); err == nil {
@@ -365,6 +384,14 @@ func (s *Server) handleSaveConfig(w http.ResponseWriter, r *http.Request) {
 	if file.CPUCount < 1 {
 		file.CPUCount = 1
 	}
+	// The wizard UI bakes the venv python into the runner after an install;
+	// an API-driven or scripted flow may not, so the server guarantees it:
+	// workloads execute through scimesh's task runner, which lives in the venv.
+	if len(file.TaskRunner) == 0 {
+		if venv := s.venvPython(); venv != "" {
+			file.TaskRunner = []string{venv, "-m", "scimesh.worker.task"}
+		}
+	}
 	if err := agent.SaveConfigFile(s.cfgPath, file); err != nil {
 		s.log.Error("save wizard config", "err", err)
 		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not write the config file"})
@@ -384,7 +411,10 @@ func (s *Server) handleTest(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "coordinator_url is required"})
 		return
 	}
-	report := agent.RunCheck(r.Context(), url)
+	// After the runtime installer created the venv, probe that interpreter:
+	// checking the bare system python3 would keep reporting scimesh as
+	// missing even though the worker would run with the venv.
+	report := agent.RunCheck(r.Context(), url, s.venvPython())
 	writeJSON(w, http.StatusOK, report)
 }
 
@@ -393,6 +423,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "no configuration saved yet"})
 		return
 	}
+	s.ensureVenvTaskRunner()
 	pid, err := s.sup.Start(s.cfgPath, s.logPath)
 	if err != nil {
 		writeJSON(w, http.StatusConflict, map[string]string{"error": err.Error()})
@@ -448,6 +479,12 @@ type installRuntimeResponse struct {
 // step. The venv python path is returned for the wizard to bake into the
 // task runner.
 func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) {
+	// The wizard may run the install before any config was saved, so the
+	// worker directory (venv, wheel) may not exist yet.
+	if err := os.MkdirAll(s.dir, 0o700); err != nil {
+		writeJSON(w, http.StatusConflict, map[string]string{"error": "could not create the worker directory"})
+		return
+	}
 	var req installRuntimeRequest
 	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
 		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON body"})
@@ -514,6 +551,20 @@ func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, installRuntimeResponse{OK: true, Python: venvPython, Installed: true})
 }
 
+// venvPython returns the managed venv python when the runtime installer has
+// created one, so the task runner can be pointed at it automatically.
+func (s *Server) venvPython() string {
+	for _, candidate := range []string{
+		filepath.Join(s.dir, "venv", "bin", "python"),
+		filepath.Join(s.dir, "venv", "Scripts", "python.exe"),
+	} {
+		if _, err := os.Stat(candidate); err == nil {
+			return candidate
+		}
+	}
+	return ""
+}
+
 // installScimeshWithPip installs the package with the venv's own pip,
 // streaming into the agent log so a long build is not silent.
 func installScimeshWithPip(ctx context.Context, venvPython, pkg string) error {

coordinator/internal/agent/setupui/server_test.go

@@ -409,3 +409,73 @@ func TestInstallRuntimeWheelDownloadFailureIsExplained(t *testing.T) {
 		t.Errorf("error = %v, want a SCIMESH_PIP_PACKAGE hint", data["error"])
 	}
 }
+
+func TestStartPinsTheVenvTaskRunner(t *testing.T) {
+	sup := &fakeSup{}
+	server, base := newTestServer(t, sup)
+	postJSON(t, base, "/api/config", map[string]any{
+		"coordinator_url": "http://coord:8080", "token": "t", "work_dir": ".",
+	})
+	// Simulate the runtime installer: create the venv python marker.
+	venvPython := filepath.Join(server.dir, "venv", "bin", "python")
+	_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+	_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
+
+	rec, _ := postJSON(t, base, "/api/start", map[string]any{})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("start: got %d, want 200", rec.Code)
+	}
+	config, err := agent.LoadConfigFile(server.cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(config.TaskRunner) != 3 || config.TaskRunner[0] != venvPython || config.TaskRunner[1] != "-m" || config.TaskRunner[2] != "scimesh.worker.task" {
+		t.Errorf("task runner = %v, want the venv python runner", config.TaskRunner)
+	}
+}
+
+func TestSaveConfigPinsVenvRunnerWhenPresent(t *testing.T) {
+	server, base := newTestServer(t, &fakeSup{})
+	venvPython := filepath.Join(server.dir, "venv", "bin", "python")
+	_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+	_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
+
+	rec, _ := postJSON(t, base, "/api/config", map[string]any{
+		"coordinator_url": "http://coord:8080", "token": "t", "work_dir": ".",
+	})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("config: got %d", rec.Code)
+	}
+	config, err := agent.LoadConfigFile(server.cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(config.TaskRunner) != 3 || config.TaskRunner[0] != venvPython {
+		t.Errorf("task runner = %v, want the venv python", config.TaskRunner)
+	}
+}
+
+func TestTestProbesTheVenvPythonAfterInstall(t *testing.T) {
+	sup := &fakeSup{}
+	server, base := newTestServer(t, sup)
+	// The runtime installer leaves a venv python; make it a stub that reports
+	// a fake scimesh version so the preflight goes green through the venv.
+	venvPython := filepath.Join(server.dir, "venv", "bin", "python")
+	_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+	_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nif [ \"$1\" = \"-c\" ]; then echo 9.9.9-test; exit 0; fi\nexit 0\n"), 0o755)
+
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodPost, base+"/api/test", strings.NewReader(`{"coordinator_url":"http://127.0.0.1:1"}`))
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	var report agent.CheckReport
+	if err := json.NewDecoder(resp.Body).Decode(&report); err != nil {
+		t.Fatal(err)
+	}
+	if !report.Scimesh.OK || report.Scimesh.Detail != "9.9.9-test" {
+		t.Errorf("scimesh check = %+v, want the venv interpreter reporting 9.9.9-test", report.Scimesh)
+	}
+}

coordinator/internal/agent/setupui/template.html

@@ -249,8 +249,8 @@ $('b4').onclick=async()=>{
   showStatus();
 };
 
-const checkRow=(name,ok,detail,ms)=>
-  '<div class="check-row"><div class="check-ic '+(ok===null?'check-wait':ok?'check-ok':'check-bad')+'"><svg viewBox="0 0 24 24" fill="none" stroke-linecap="round">'+(ok===null?'<path d="M12 7v5l3 3"/>':ok?'<path d="M4 12l5 5L20 6"/>':'<path d="M6 6l12 12M18 6L6 18"/>')+'</svg></div><div><b>'+name+'</b><span>'+(detail||'')+'</span></div>'+(ms?'<span class="ms">'+ms+' ms</span>':'')+'</div>';
+const checkRow=(name,ok,detail,ms,action)=>
+  '<div class="check-row"><div class="check-ic '+(ok===null?'check-wait':ok?'check-ok':'check-bad')+'"><svg viewBox="0 0 24 24" fill="none" stroke-linecap="round">'+(ok===null?'<path d="M12 7v5l3 3"/>':ok?'<path d="M4 12l5 5L20 6"/>':'<path d="M6 6l12 12M18 6L6 18"/>')+'</svg></div><div><b>'+name+'</b><span>'+(detail||'')+'</span></div>'+(ms?'<span class="ms">'+ms+' ms</span>':'')+(action?action:'')+'</div>';
 async function runChecks(){
   const box=$('checks');
   box.innerHTML=checkRow('Coordinator reachable','',null,null)+checkRow('Python 3','',null,null)+checkRow('scimesh package','',null,null);
@@ -260,8 +260,21 @@ async function runChecks(){
   const items=[r.data.coordinator,r.data.python,r.data.scimesh];
   for(const item of items){
     if(item&&!item.ok)checksOk=false;
-    box.insertAdjacentHTML('beforeend',checkRow(item?item.name:'?',item?item.ok:null,item?item.detail:'',item?item.latency_ms:null));
+    let action='';
+    if(item&&item.name==='scimesh'&&!item.ok){
+      action='<div style="margin-left:auto;display:flex;gap:8px;align-items:center"><input id="in-pkg" placeholder="wheel path / checkout / index URL (optional)" style="width:230px;padding:7px 10px;font-size:12px"><button class="btn btn-primary" style="padding:6px 12px;font-size:12px" id="install-scimesh">Install</button></div>';
+    }
+    box.insertAdjacentHTML('beforeend',checkRow(item?item.name:'?',item?item.ok:null,item?item.detail:'',item?item.latency_ms:null,action));
   }
+  const btn=$('install-scimesh');
+  if(btn)btn.onclick=async()=>{
+    const src=$('in-pkg')?$('in-pkg').value.trim():'';
+    btn.disabled=true;btn.textContent='Installing…';
+    const ir=await postJSON('/api/runtime/install',{scimesh_package:src});
+    if(ir.status!==200){err('Could not install scimesh: '+(ir.data.error||'unknown error'));btn.disabled=false;btn.textContent='Install';return}
+    state.venvPython=ir.data.python;
+    runChecks();
+  };
   $('b3').disabled=!checksOk;
 }
 

mkdocs/standalone.md

@@ -59,6 +59,13 @@ curl -fsSL https://raw.githubusercontent.com/emil28092005/SciMesh/main/install.s
 # the installer opens the local wizard at http://127.0.0.1:12700 automatically
 ```
 
+On a headless server (no desktop environment), RDKit needs a few X11
+libraries that desktops already ship — install them once with apt:
+
+```bash
+sudo apt-get install -y libxrender1 libxext6 libxcursor1 libxfixes3 libxi6 libxrandr2
+```
+
 Or configure by hand:
 
 ```bash