Probe the managed venv in the wizard preflight and document RDKit's X11 deps

.github/workflows/release.yml

@@ -54,8 +54,39 @@ jobs:
           path: coordinator/dist/*
           if-no-files-found: error
 
+  wheel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: build the scimesh wheel
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: |
+          # The tag (v1.1.0-alpha.10) becomes the package version in its
+          # PEP 440 form (1.1.0a10); the wheel is then version-locked to the
+          # binaries of the same release.
+          WHEEL_VERSION="${VERSION#v}"
+          WHEEL_VERSION="${WHEEL_VERSION/-alpha./a}"
+          WHEEL_VERSION="${WHEEL_VERSION/-beta./b}"
+          WHEEL_VERSION="${WHEEL_VERSION/-rc./rc}"
+          sed -i "s/^version = .*/version = \"${WHEEL_VERSION}\"/" pyproject.toml
+          python -m pip install --quiet build
+          python -m build --wheel --outdir dist
+          ls -la dist/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel
+          path: dist/*.whl
+          if-no-files-found: error
+
   release:
-    needs: binaries
+    needs: [binaries, wheel]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -68,6 +99,11 @@ jobs:
           pattern: binaries-*
           merge-multiple: true
 
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheel
+          path: artifacts
+
       - name: checksums
         working-directory: artifacts
         run: sha256sum * > SHA256SUMS.txt

README.md

@@ -225,6 +225,10 @@ The coordinator and worker agent are Go modules under `coordinator/` and `users/
 cd coordinator && make coordinator agent && go test ./...
 ```
 
+On headless servers (no desktop environment), RDKit needs a few X11
+libraries that desktops already ship: `sudo apt-get install -y libxrender1
+libxext6 libxcursor1 libxfixes3 libxi6 libxrandr2`.
+
 `make check` runs the full gate: vet, lint, race tests, the PostgreSQL
 integration suite, and the two-worker end-to-end smoke script.
 

coordinator/cmd/coordinator/serve_cmd.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"context"
+
+	"github.com/emil28092005/SciMesh/coordinator/internal/agent"
+
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
@@ -259,13 +262,25 @@ func ensureRuntime(log *slog.Logger, dataDir, venvPython string) {
 	}
 	pip := filepath.Join(venvDir, binName("bin/pip"))
 	// The scimesh package is installed from an explicit source only: the PyPI
-	// name is not ours yet, so `pip install scimesh` would fetch a stranger's
-	// package. Operators publish a wheel or index via SCIMESH_PIP_PACKAGE.
+	// name belongs to an unrelated project, so `pip install scimesh` would
+	// fetch a stranger's package. Default: download the wheel attached to our
+	// own GitHub release for this binary version; SCIMESH_PIP_PACKAGE
+	// overrides with a custom wheel, checkout or index.
 	source := os.Getenv("SCIMESH_PIP_PACKAGE")
 	if source == "" {
-		log.Warn("scientific runtime venv created, but scimesh is not installed",
-			"hint", pip+" install <your scimesh wheel or index> (or set SCIMESH_PIP_PACKAGE)")
-		return
+		url, _, err := agent.ReleaseWheelURL(version)
+		if err != nil {
+			log.Warn("scientific runtime venv created, but scimesh is not installed",
+				"hint", "set SCIMESH_PIP_PACKAGE to your wheel or index")
+			return
+		}
+		downloaded, err := agent.DownloadWheel(context.Background(), url, venvDir)
+		if err != nil {
+			log.Warn("could not download the scimesh wheel for this release",
+				"err", err, "hint", "set SCIMESH_PIP_PACKAGE to your wheel or index")
+			return
+		}
+		source = downloaded
 	}
 	// #nosec G204,G702 -- pip and source are operator-configured paths.
 	install := exec.CommandContext(context.Background(), pip, "install", source)

coordinator/cmd/worker-agent/main.go

@@ -27,6 +27,10 @@ import (
 var version = "dev"
 
 func main() {
+	// The wizard and --check need the injected build version too (they resolve
+	// the release wheel matching this binary), so it is set before dispatch.
+	agent.Version = version
+
 	if len(os.Args) > 1 && os.Args[1] == "setup" {
 		os.Exit(runSetup(os.Args[2:]))
 	}
@@ -38,8 +42,6 @@ func main() {
 	checkURL := fs.String("coordinator-url", "", "coordinator URL to probe in --check mode")
 	_ = fs.Parse(os.Args[1:])
 
-	agent.Version = version
-
 	if *showVersion {
 		fmt.Println("worker-agent " + version)
 		return
@@ -56,7 +58,7 @@ func main() {
 			fmt.Println("check: no coordinator URL (pass --coordinator-url or set COORDINATOR_URL)")
 			os.Exit(1)
 		}
-		report := agent.RunCheck(ctx, url)
+		report := agent.RunCheck(ctx, url, "")
 		printCheck(report)
 		if !report.Coordinator.OK || !report.Python.OK || !report.Scimesh.OK {
 			os.Exit(1)

coordinator/internal/agent/check.go

@@ -70,21 +70,31 @@ func CheckCoordinator(ctx context.Context, url string, timeout time.Duration) Ch
 	return report
 }
 
-// CheckEnvironment verifies the local runtime: Python present and the scimesh
-// package importable.
+// CheckEnvironment verifies the local runtime against the python3 found on
+// PATH.
 func CheckEnvironment(ctx context.Context) CheckReport {
-	report := CheckReport{Agent: Version}
 	python, err := exec.LookPath("python3")
 	if err != nil {
-		report.Python = CheckItem{Name: "python", OK: false, Detail: "python3 not found on PATH"}
-		return report
+		return CheckReport{Agent: Version, Python: CheckItem{Name: "python", OK: false, Detail: "python3 not found on PATH"}}
 	}
-	report.Python = CheckItem{Name: "python", OK: true, Detail: python}
-	//nolint:gosec // G204: python comes from LookPath, the argument list is constant
+	return CheckEnvironmentWithPython(ctx, python)
+}
+
+// CheckEnvironmentWithPython verifies the local runtime against a specific
+// interpreter — the wizard's managed venv python when the runtime installer
+// has created one, so the preflight reflects what the worker will actually
+// execute with.
+func CheckEnvironmentWithPython(ctx context.Context, python string) CheckReport {
+	report := CheckReport{Agent: Version, Python: CheckItem{Name: "python", OK: true, Detail: python}}
+	//nolint:gosec // G204: python is a resolved interpreter path, the argument list is constant
 	cmd := exec.CommandContext(ctx, python, "-c", "import scimesh; print(scimesh.__version__ if hasattr(scimesh, '__version__') else 'installed')")
 	out, err := cmd.Output()
 	if err != nil {
-		report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "install with: pip install scimesh"}
+		// The worker executes workloads by spawning scimesh's task runner, so
+		// the package is a hard requirement, not an optimisation. The PyPI
+		// name belongs to a different project, so the wizard installs from
+		// SCIMESH_PIP_PACKAGE instead of suggesting a bare pip install.
+		report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "the worker runs workloads through scimesh — install it from your wheel or index (SCIMESH_PIP_PACKAGE)"}
 		return report
 	}
 	report.Scimesh = CheckItem{Name: "scimesh", OK: true, Detail: strings.TrimSpace(string(out))}
@@ -92,10 +102,17 @@ func CheckEnvironment(ctx context.Context) CheckReport {
 }
 
 // RunCheck combines the coordinator probe and the local environment probe; it
-// is the body behind `worker-agent --check` and the wizard's test step.
-func RunCheck(ctx context.Context, coordinatorURL string) CheckReport {
+// is the body behind `worker-agent --check` and the wizard's test step. A
+// non-empty python overrides the interpreter probed for the scimesh package
+// (the managed venv after a runtime install).
+func RunCheck(ctx context.Context, coordinatorURL, python string) CheckReport {
 	report := CheckCoordinator(ctx, coordinatorURL, 15*time.Second)
-	env := CheckEnvironment(ctx)
+	var env CheckReport
+	if python != "" {
+		env = CheckEnvironmentWithPython(ctx, python)
+	} else {
+		env = CheckEnvironment(ctx)
+	}
 	report.Python = env.Python
 	report.Scimesh = env.Scimesh
 	return report

coordinator/internal/agent/releasewheel.go

@@ -0,0 +1,70 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ReleaseWheelURL returns the download URL of the scimesh wheel attached to
+// the GitHub release that matches the given binary version (for example
+// "1.1.0-alpha.10"), plus the wheel file name. The wheel is version-locked to
+// the binary so a worker's catalog always matches its task runner.
+func ReleaseWheelURL(version string) (string, string, error) {
+	if version == "" || version == "dev" {
+		return "", "", fmt.Errorf("no release wheel for build %q", version)
+	}
+	filename := fmt.Sprintf("scimesh-%s-py3-none-any.whl", NormalizePEP440(version))
+	return fmt.Sprintf("https://github.com/emil28092005/SciMesh/releases/download/v%s/%s", version, filename), filename, nil
+}
+
+// NormalizePEP440 turns our release tag suffixes into the PEP 440 form
+// setuptools uses for wheel names: 1.1.0-alpha.10 -> 1.1.0a10,
+// 1.1.0-beta.2 -> 1.1.0b2, 1.1.0-rc.1 -> 1.1.0rc1. Stable tags pass through.
+func NormalizePEP440(version string) string {
+	for from, to := range map[string]string{"-alpha.": "a", "-beta.": "b", "-rc.": "rc"} {
+		version = strings.ReplaceAll(version, from, to)
+	}
+	return version
+}
+
+// DownloadWheel fetches the release wheel into dir (config directory of the
+// wizard / serve data dir) and returns the local path. Best-effort download
+// with a generous timeout: wheels can be several MB.
+func DownloadWheel(ctx context.Context, url, dir string) (string, error) {
+	target := filepath.Join(dir, wheelNameFromURL(url))
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("download wheel: HTTP %d", resp.StatusCode)
+	}
+	//nolint:gosec // G304: target is our own config dir + a fixed wheel name
+	out, err := os.Create(target)
+	if err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
+	defer func() { _ = out.Close() }()
+	if _, err := io.Copy(out, resp.Body); err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
+	return target, nil
+}
+
+// wheelNameFromURL extracts the trailing file name of a wheel URL.
+func wheelNameFromURL(url string) string {
+	return url[strings.LastIndex(url, "/")+1:]
+}

coordinator/internal/agent/releasewheel_test.go

@@ -0,0 +1,81 @@
+package agent
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestNormalizePEP440(t *testing.T) {
+	cases := map[string]string{
+		"1.1.0":          "1.1.0",
+		"1.1.0-alpha.10": "1.1.0a10",
+		"1.1.0-beta.2":   "1.1.0b2",
+		"1.1.0-rc.1":     "1.1.0rc1",
+		"1.0.0":          "1.0.0",
+	}
+	for in, want := range cases {
+		if got := NormalizePEP440(in); got != want {
+			t.Errorf("NormalizePEP440(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestReleaseWheelURL(t *testing.T) {
+	url, name, err := ReleaseWheelURL("1.1.0-alpha.10")
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantURL := "https://github.com/emil28092005/SciMesh/releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl"
+	if url != wantURL {
+		t.Errorf("url = %q, want %q", url, wantURL)
+	}
+	if name != "scimesh-1.1.0a10-py3-none-any.whl" {
+		t.Errorf("name = %q", name)
+	}
+
+	// A dev build has no release wheel.
+	if _, _, err := ReleaseWheelURL("dev"); err == nil {
+		t.Error("dev build must not resolve a release wheel")
+	}
+	if _, _, err := ReleaseWheelURL(""); err == nil {
+		t.Error("empty version must not resolve a release wheel")
+	}
+}
+
+func TestDownloadWheel(t *testing.T) {
+	payload := []byte("fake wheel bytes")
+	stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(payload)
+	}))
+	defer stub.Close()
+
+	dir := t.TempDir()
+	path, err := DownloadWheel(context.Background(), stub.URL+"/scimesh-1.1.0a10-py3-none-any.whl", dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasSuffix(path, "scimesh-1.1.0a10-py3-none-any.whl") {
+		t.Errorf("path = %q", path)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != string(payload) {
+		t.Error("wheel bytes mismatch")
+	}
+}
+
+func TestDownloadWheelReportsHTTPErrors(t *testing.T) {
+	stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.NotFound(w, r)
+	}))
+	defer stub.Close()
+	if _, err := DownloadWheel(context.Background(), stub.URL+"/missing.whl", t.TempDir()); err == nil {
+		t.Error("404 must fail the download")
+	}
+}

coordinator/internal/agent/setupui/server.go

@@ -176,13 +176,15 @@ func (s *PIDSupervisor) Stop() error {
 
 // Server is the wizard HTTP server, bound to the loopback interface only.
 type Server struct {
-	log         *slog.Logger
-	cfgPath     string
-	logPath     string
-	dir         string
-	sup         Supervisor
-	openBrowser func(string)
-	port        int
+	log           *slog.Logger
+	cfgPath       string
+	logPath       string
+	dir           string
+	sup           Supervisor
+	openBrowser   func(string)
+	port          int
+	install       func(ctx context.Context, venvPython, pkg string) error
+	downloadWheel func(ctx context.Context, url, dir string) (string, error)
 }
 
 // Options customises the wizard for tests and embedding.
@@ -192,6 +194,12 @@ type Options struct {
 	OpenBrowser func(url string)
 	Supervisor  Supervisor
 	Dir         string // directory for pid/log files; defaults to the config dir
+	// InstallScimesh overrides the pip step of the runtime installer (tests
+	// substitute a fake); nil uses the real pip inside the managed venv.
+	InstallScimesh func(ctx context.Context, venvPython, pkg string) error
+	// DownloadWheel overrides the release-wheel download (tests substitute a
+	// fake); nil downloads from the GitHub release matching the agent version.
+	DownloadWheel func(ctx context.Context, url, dir string) (string, error)
 }
 
 func New(log *slog.Logger, opts Options) *Server {
@@ -215,7 +223,15 @@ func New(log *slog.Logger, opts Options) *Server {
 	if port == 0 {
 		port = defaultPort
 	}
-	return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port}
+	install := opts.InstallScimesh
+	if install == nil {
+		install = installScimeshWithPip
+	}
+	downloadWheel := opts.DownloadWheel
+	if downloadWheel == nil {
+		downloadWheel = agent.DownloadWheel
+	}
+	return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install, downloadWheel: downloadWheel}
 }
 
 // Listen binds the loopback listener and returns it; Serve runs the server on
@@ -234,6 +250,7 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error {
 	mux.HandleFunc("GET /api/status", s.handleStatus)
 	mux.HandleFunc("POST /api/config", s.handleSaveConfig)
 	mux.HandleFunc("POST /api/test", s.handleTest)
+	mux.HandleFunc("POST /api/runtime/install", s.handleInstallRuntime)
 	mux.HandleFunc("POST /api/start", s.handleStart)
 	mux.HandleFunc("POST /api/stop", s.handleStop)
 	mux.HandleFunc("GET /api/logs", s.handleLogs)
@@ -283,6 +300,25 @@ type statusView struct {
 	TokenSet      bool   `json:"token_set"`
 }
 
+// ensureVenvTaskRunner rewrites the saved config so its task runner uses the
+// managed venv python when one exists and the config does not already pin one.
+func (s *Server) ensureVenvTaskRunner() {
+	raw, err := os.ReadFile(s.cfgPath)
+	if err != nil {
+		return
+	}
+	var file agent.ConfigFile
+	if json.Unmarshal(raw, &file) != nil || len(file.TaskRunner) > 0 {
+		return
+	}
+	if venv := s.venvPython(); venv != "" {
+		file.TaskRunner = []string{venv, "-m", "scimesh.worker.task"}
+		if payload, err := json.MarshalIndent(file, "", "  "); err == nil {
+			_ = os.WriteFile(s.cfgPath, append(payload, '\n'), 0o600)
+		}
+	}
+}
+
 func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
 	view := statusView{ConfigPath: s.cfgPath, LogPath: s.logPath, Running: s.sup.Alive(), Pid: s.sup.Pid()}
 	if raw, err := os.ReadFile(s.cfgPath); err == nil {
@@ -348,6 +384,14 @@ func (s *Server) handleSaveConfig(w http.ResponseWriter, r *http.Request) {
 	if file.CPUCount < 1 {
 		file.CPUCount = 1
 	}
+	// The wizard UI bakes the venv python into the runner after an install;
+	// an API-driven or scripted flow may not, so the server guarantees it:
+	// workloads execute through scimesh's task runner, which lives in the venv.
+	if len(file.TaskRunner) == 0 {
+		if venv := s.venvPython(); venv != "" {
+			file.TaskRunner = []string{venv, "-m", "scimesh.worker.task"}
+		}
+	}
 	if err := agent.SaveConfigFile(s.cfgPath, file); err != nil {
 		s.log.Error("save wizard config", "err", err)
 		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not write the config file"})
@@ -367,7 +411,10 @@ func (s *Server) handleTest(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "coordinator_url is required"})
 		return
 	}
-	report := agent.RunCheck(r.Context(), url)
+	// After the runtime installer created the venv, probe that interpreter:
+	// checking the bare system python3 would keep reporting scimesh as
+	// missing even though the worker would run with the venv.
+	report := agent.RunCheck(r.Context(), url, s.venvPython())
 	writeJSON(w, http.StatusOK, report)
 }
 
@@ -376,6 +423,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "no configuration saved yet"})
 		return
 	}
+	s.ensureVenvTaskRunner()
 	pid, err := s.sup.Start(s.cfgPath, s.logPath)
 	if err != nil {
 		writeJSON(w, http.StatusConflict, map[string]string{"error": err.Error()})
@@ -412,3 +460,126 @@ func (s *Server) handleLogs(w http.ResponseWriter, r *http.Request) {
 // ErrCanceled mirrors context.Canceled for callers that treat a cancelled
 // wizard as a clean exit.
 var ErrCanceled = errors.New("setup wizard cancelled")
+
+type installRuntimeRequest struct {
+	// ScimeshPackage overrides where the scimesh wheel comes from: a local
+	// wheel/index path or the PyPI name. Defaults to SCIMESH_PIP_PACKAGE, then
+	// to the PyPI name.
+	ScimeshPackage string `json:"scimesh_package"`
+}
+
+type installRuntimeResponse struct {
+	OK        bool   `json:"ok"`
+	Python    string `json:"python,omitempty"` // venv python to use as TASK_RUNNER[0]
+	Installed bool   `json:"installed"`
+}
+
+// handleInstallRuntime creates a managed venv next to the worker config and
+// installs the scimesh package into it, so the machine needs no manual pip
+// step. The venv python path is returned for the wizard to bake into the
+// task runner.
+func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) {
+	var req installRuntimeRequest
+	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON body"})
+		return
+	}
+	pkg := strings.TrimSpace(req.ScimeshPackage)
+	if pkg == "" {
+		pkg = os.Getenv("SCIMESH_PIP_PACKAGE")
+	}
+	if pkg == "" {
+		// No PyPI default on purpose: the PyPI name "scimesh" belongs to an
+		// unrelated project. Instead we ship the wheel in our own GitHub
+		// release, version-locked to this binary, and download it from there.
+		url, _, err := agent.ReleaseWheelURL(agent.Version)
+		if err != nil {
+			writeJSON(w, http.StatusConflict, map[string]string{
+				"error": "no scimesh source configured: set SCIMESH_PIP_PACKAGE to your wheel, checkout or index, then retry",
+			})
+			return
+		}
+		local, err := s.downloadWheel(r.Context(), url, s.dir)
+		if err != nil {
+			s.log.Error("download release wheel", "err", err, "url", url)
+			writeJSON(w, http.StatusConflict, map[string]string{
+				"error": "could not download the scimesh wheel for this release: " + err.Error() + ". Set SCIMESH_PIP_PACKAGE to your wheel, checkout or index and retry.",
+			})
+			return
+		}
+		pkg = local
+	}
+
+	python3, err := exec.LookPath("python3")
+	if err != nil {
+		writeJSON(w, http.StatusConflict, map[string]string{"error": "python3 is not installed on this machine"})
+		return
+	}
+	venvDir := filepath.Join(s.dir, "venv")
+	venvPython := filepath.Join(venvDir, "bin", "python")
+	if _, err := os.Stat(venvPython); err != nil {
+		// Windows layout: Scripts/python.exe.
+		if win := filepath.Join(venvDir, "Scripts", "python.exe"); stat(win) {
+			venvPython = win
+		}
+	}
+	if _, err := os.Stat(venvPython); err != nil {
+		ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
+		defer cancel()
+		cmd := exec.CommandContext(ctx, python3, "-m", "venv", venvDir) //nolint:gosec // G204: python3 from LookPath, venvDir is our own dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			s.log.Error("create runtime venv", "err", err, "out", truncate(string(out), 500))
+			writeJSON(w, http.StatusConflict, map[string]string{"error": "could not create the python venv"})
+			return
+		}
+	}
+
+	if err := s.install(r.Context(), venvPython, pkg); err != nil {
+		s.log.Error("install scimesh runtime", "err", err)
+		writeJSON(w, http.StatusConflict, map[string]string{
+			"error": "pip install " + pkg + " failed: " + err.Error() +
+				". Set SCIMESH_PIP_PACKAGE to your scimesh wheel or index and retry.",
+		})
+		return
+	}
+	writeJSON(w, http.StatusOK, installRuntimeResponse{OK: true, Python: venvPython, Installed: true})
+}
+
+// venvPython returns the managed venv python when the runtime installer has
+// created one, so the task runner can be pointed at it automatically.
+func (s *Server) venvPython() string {
+	for _, candidate := range []string{
+		filepath.Join(s.dir, "venv", "bin", "python"),
+		filepath.Join(s.dir, "venv", "Scripts", "python.exe"),
+	} {
+		if _, err := os.Stat(candidate); err == nil {
+			return candidate
+		}
+	}
+	return ""
+}
+
+// installScimeshWithPip installs the package with the venv's own pip,
+// streaming into the agent log so a long build is not silent.
+func installScimeshWithPip(ctx context.Context, venvPython, pkg string) error {
+	pip := filepath.Join(filepath.Dir(venvPython), "pip")
+	if _, err := os.Stat(pip); err != nil {
+		pip += ".exe"
+	}
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, pip, "install", pkg) //nolint:gosec // G204: pip from our venv, pkg is operator-set or a fixed default
+	return cmd.Run()
+}
+
+func stat(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}

coordinator/internal/agent/setupui/server_test.go

@@ -3,6 +3,7 @@ package setupui
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"io"
 	"log/slog"
 	"net"
@@ -23,17 +24,29 @@ func testLogger() *slog.Logger {
 }
 
 func newTestServer(t *testing.T, sup Supervisor) (*Server, string) {
+	t.Helper()
+	return newTestServerWithInstall(t, sup, nil)
+}
+
+func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error) (*Server, string) {
+	t.Helper()
+	return newTestServerWithInstallAndWheel(t, sup, install, nil)
+}
+
+func newTestServerWithInstallAndWheel(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error, wheel func(ctx context.Context, url, dir string) (string, error)) (*Server, string) {
 	t.Helper()
 	dir := t.TempDir()
 	server := New(testLogger(), Options{
 		// A distinct random port per test: Port 0 means "the default 12700" in
 		// the server, which would let the shared http.Client pool reuse a stale
 		// keep-alive connection across tests (EOF after a Shutdown).
-		Port:        freePort(t),
-		ConfigPath:  filepath.Join(dir, "config.json"),
-		Dir:         dir,
-		Supervisor:  sup,
-		OpenBrowser: func(string) {},
+		Port:           freePort(t),
+		ConfigPath:     filepath.Join(dir, "config.json"),
+		Dir:            dir,
+		Supervisor:     sup,
+		OpenBrowser:    func(string) {},
+		InstallScimesh: install,
+		DownloadWheel:  wheel,
 	})
 	listener, err := server.Listen()
 	if err != nil {
@@ -291,3 +304,178 @@ func TestConfigFileDefaultsAndEnvOverride(t *testing.T) {
 		t.Errorf("cpu = %d", config.CPUCount)
 	}
 }
+
+func TestInstallRuntimeCreatesVenvAndReportsPython(t *testing.T) {
+	var installedPkg string
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
+		installedPkg = pkg
+		// Prove the venv python path really exists by creating a marker file
+		// where the real venv python would be.
+		_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+		_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
+		return nil
+	})
+
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{"scimesh_package": "/wheels/scimesh.whl"})
+	if rec.Code != http.StatusOK || data["ok"] != true {
+		t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
+	}
+	if installedPkg != "/wheels/scimesh.whl" {
+		t.Errorf("package = %q, want the requested wheel", installedPkg)
+	}
+	if !strings.HasSuffix(data["python"].(string), "venv/bin/python") {
+		t.Errorf("python = %v, want the venv python", data["python"])
+	}
+}
+
+func TestInstallRuntimeRequiresASource(t *testing.T) {
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
+		t.Fatal("install must not run without a package source")
+		return nil
+	})
+	// No source anywhere (SCIMESH_PIP_PACKAGE unset, request empty): 409 with
+	// guidance. The PyPI name is another project, so no silent fallback.
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusConflict {
+		t.Fatalf("install without source: got %d, want 409", rec.Code)
+	}
+	if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
+		t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
+	}
+}
+
+func TestInstallRuntimeFailureIsExplained(t *testing.T) {
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
+		return errors.New("no matching distribution found")
+	})
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{"scimesh_package": "/wheels/scimesh.whl"})
+	if rec.Code != http.StatusConflict {
+		t.Fatalf("install failure: got %d, want 409", rec.Code)
+	}
+	if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
+		t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
+	}
+}
+
+func TestInstallRuntimeDownloadsReleaseWheelWhenNoSource(t *testing.T) {
+	oldVersion := agent.Version
+	agent.Version = "1.1.0-alpha.10"
+	t.Cleanup(func() { agent.Version = oldVersion })
+
+	var downloadedURL, installedPkg string
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstallAndWheel(t, sup,
+		func(ctx context.Context, venvPython, pkg string) error {
+			installedPkg = pkg
+			return nil
+		},
+		func(ctx context.Context, url, dir string) (string, error) {
+			downloadedURL = url
+			return filepath.Join(dir, "scimesh-1.1.0a10-py3-none-any.whl"), nil
+		})
+
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusOK || data["ok"] != true {
+		t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
+	}
+	if !strings.Contains(downloadedURL, "releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl") {
+		t.Errorf("download url = %q, want the release wheel of this version", downloadedURL)
+	}
+	if !strings.HasSuffix(installedPkg, "scimesh-1.1.0a10-py3-none-any.whl") {
+		t.Errorf("pip received %q, want the downloaded wheel", installedPkg)
+	}
+}
+
+func TestInstallRuntimeWheelDownloadFailureIsExplained(t *testing.T) {
+	oldVersion := agent.Version
+	agent.Version = "1.1.0-alpha.10"
+	t.Cleanup(func() { agent.Version = oldVersion })
+
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstallAndWheel(t, sup,
+		func(ctx context.Context, venvPython, pkg string) error { t.Fatal("pip must not run"); return nil },
+		func(ctx context.Context, url, dir string) (string, error) {
+			return "", errors.New("HTTP 404")
+		})
+
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusConflict {
+		t.Fatalf("got %d, want 409", rec.Code)
+	}
+	if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
+		t.Errorf("error = %v, want a SCIMESH_PIP_PACKAGE hint", data["error"])
+	}
+}
+
+func TestStartPinsTheVenvTaskRunner(t *testing.T) {
+	sup := &fakeSup{}
+	server, base := newTestServer(t, sup)
+	postJSON(t, base, "/api/config", map[string]any{
+		"coordinator_url": "http://coord:8080", "token": "t", "work_dir": ".",
+	})
+	// Simulate the runtime installer: create the venv python marker.
+	venvPython := filepath.Join(server.dir, "venv", "bin", "python")
+	_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+	_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
+
+	rec, _ := postJSON(t, base, "/api/start", map[string]any{})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("start: got %d, want 200", rec.Code)
+	}
+	config, err := agent.LoadConfigFile(server.cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(config.TaskRunner) != 3 || config.TaskRunner[0] != venvPython || config.TaskRunner[1] != "-m" || config.TaskRunner[2] != "scimesh.worker.task" {
+		t.Errorf("task runner = %v, want the venv python runner", config.TaskRunner)
+	}
+}
+
+func TestSaveConfigPinsVenvRunnerWhenPresent(t *testing.T) {
+	server, base := newTestServer(t, &fakeSup{})
+	venvPython := filepath.Join(server.dir, "venv", "bin", "python")
+	_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+	_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
+
+	rec, _ := postJSON(t, base, "/api/config", map[string]any{
+		"coordinator_url": "http://coord:8080", "token": "t", "work_dir": ".",
+	})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("config: got %d", rec.Code)
+	}
+	config, err := agent.LoadConfigFile(server.cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(config.TaskRunner) != 3 || config.TaskRunner[0] != venvPython {
+		t.Errorf("task runner = %v, want the venv python", config.TaskRunner)
+	}
+}
+
+func TestTestProbesTheVenvPythonAfterInstall(t *testing.T) {
+	sup := &fakeSup{}
+	server, base := newTestServer(t, sup)
+	// The runtime installer leaves a venv python; make it a stub that reports
+	// a fake scimesh version so the preflight goes green through the venv.
+	venvPython := filepath.Join(server.dir, "venv", "bin", "python")
+	_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+	_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nif [ \"$1\" = \"-c\" ]; then echo 9.9.9-test; exit 0; fi\nexit 0\n"), 0o755)
+
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodPost, base+"/api/test", strings.NewReader(`{"coordinator_url":"http://127.0.0.1:1"}`))
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	var report agent.CheckReport
+	if err := json.NewDecoder(resp.Body).Decode(&report); err != nil {
+		t.Fatal(err)
+	}
+	if !report.Scimesh.OK || report.Scimesh.Detail != "9.9.9-test" {
+		t.Errorf("scimesh check = %+v, want the venv interpreter reporting 9.9.9-test", report.Scimesh)
+	}
+}

coordinator/internal/agent/setupui/template.html

@@ -183,7 +183,7 @@ code{font-family:var(--mono);font-size:.86em}
 
 <script>
 const $=id=>document.getElementById(id);
-let state={mode:'token',cpu:'auto'};
+let state={mode:'token',cpu:'auto',venvPython:null};
 let checksOk=false;
 
 function err(msg){$('error-box').innerHTML=msg?'<div class="error-strip">'+msg+'</div>':''}
@@ -215,7 +215,7 @@ async function postJSON(path,body){
   return {status:r.status,data};
 }
 function draftConfig(){
-  return {
+  const cfg={
     coordinator_url:$('in-url').value.trim(),
     token:state.mode==='token'?$('in-token').value.trim():'',
     worker_key:state.mode==='key'?$('in-key').value.trim():'',
@@ -224,6 +224,8 @@ function draftConfig(){
     worker_name:$('in-name').value.trim(),
     cpu_count:state.cpu==='custom'?parseInt($('in-cpu').value||'1',10):0
   };
+  if(state.venvPython)cfg.task_runner=[state.venvPython,'-m','scimesh.worker.task'];
+  return cfg;
 }
 
 $('b1').onclick=()=>{

coordinator/internal/transport/http/server.go

@@ -158,6 +158,7 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler {
 			// Public auth pages — reachable without a session so a user can log in.
 			ui.HandleFunc("GET /ui/login", s.handleUILoginForm)
 			ui.HandleFunc("POST /ui/login", s.handleUILogin)
+			ui.HandleFunc("GET /ui/logout-form", s.handleUILogoutForm)
 			ui.HandleFunc("GET /ui/register", s.handleUIRegisterForm)
 			ui.HandleFunc("POST /ui/register", s.handleUIRegister)
 			ui.HandleFunc("POST /ui/logout", s.handleUILogout)

coordinator/internal/transport/http/templates/login.html

@@ -20,7 +20,10 @@
       <input id="password" name="password" type="password" autocomplete="current-password" required>
       <button class="button" type="submit">Sign in</button>
     </form>
-    {{if .Error}}<p class="error">{{.Error}}</p>{{end}}
+    {{if eq .Error "admin role required"}}
+    <p class="error">The admin console is reserved for the cluster administrator.</p>
+    <p class="alt">You are signed in as a non-admin. <a href="/ui/logout-form">Log out</a>, then sign in with the admin account — its login is printed by <code>coordinator serve</code> on first start and stored in <code>~/.scimesh/admin.password</code>.</p>
+    {{else if .Error}}<p class="error">{{.Error}}</p>{{end}}
     <p class="alt">No account? <a href="/ui/register">Register</a></p>
   </main>
 </body>

coordinator/internal/transport/http/templates/logout-form.html

@@ -0,0 +1,24 @@
+{{define "logout-form.html"}}
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Sign out · SciMesh</title>
+  <style>
+    :root{color:#e5efff;background:#08111f;font:16px/1.5 Inter,ui-sans-serif,system-ui,sans-serif;color-scheme:dark}*{box-sizing:border-box}body{margin:0;min-height:100vh;display:grid;place-items:center;background:radial-gradient(circle at 10% -8%,#183f77 0,transparent 32rem),#08111f}a{color:#94bdff}.card{width:min(92vw,380px);border:1px solid #294662;border-radius:15px;background:#0d1a2cdd;box-shadow:0 20px 45px #00000021;padding:28px}.eyebrow{margin:0 0 4px;color:#7baaff;font-size:.78rem;font-weight:800;letter-spacing:.14em;text-transform:uppercase}h1{margin:0 0 12px;color:#f4f8ff;font-size:1.5rem;letter-spacing:-.03em}p{margin:0 0 18px;color:#9fb3cf;font-size:.92rem}code{color:#cfe0ff}.button{display:block;width:100%;margin-top:4px;border:0;border-radius:10px;padding:12px 16px;background:#ff7d92;color:#230810;font:inherit;font-weight:850;cursor:pointer}.alt{margin:18px 0 0;color:#9fb3cf;font-size:.92rem}
+  </style>
+</head>
+<body>
+  <main class="card">
+    <p class="eyebrow">SciMesh</p>
+    <h1>Sign out</h1>
+    <p>End the current session so you can sign in with a different account (for example the cluster administrator).</p>
+    <form method="post" action="/ui/logout">
+      <button class="button" type="submit">Log out</button>
+    </form>
+    <p class="alt">Changed your mind? <a href="/ui/login">Back to sign in</a></p>
+  </main>
+</body>
+</html>
+{{end}}

coordinator/internal/transport/http/ui_admin.go

@@ -33,7 +33,12 @@ var adminUserActions = map[string]bool{
 func requireAdmin(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if req, ok := authctx.From(r.Context()); !ok || !req.IsAdmin() {
-			http.Redirect(w, r, "/ui/login?error=admin+role+required", http.StatusSeeOther)
+			target := "/ui/login?error=admin+role+required"
+			// Keep the destination so a successful login lands straight back.
+			if strings.HasPrefix(r.URL.Path, "/ui/") {
+				target += "&next=" + url.QueryEscape(r.URL.Path)
+			}
+			http.Redirect(w, r, target, http.StatusSeeOther)
 			return
 		}
 		next.ServeHTTP(w, r)

coordinator/internal/transport/http/ui_admin_internal_test.go

@@ -29,15 +29,15 @@ func TestRequireAdminAllowsAdminOnly(t *testing.T) {
 		t.Error("admin must reach the handler")
 	}
 
-	// Plain user is redirected to the login with the reason.
+	// Plain user is redirected to the login with the reason and the destination.
 	reached = false
 	rec := httptest.NewRecorder()
 	h.ServeHTTP(rec, adminReq(t, "user"))
 	if reached {
 		t.Error("non-admin must not reach the handler")
 	}
-	if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/ui/login?error=admin+role+required" {
-		t.Errorf("non-admin got %d -> %q, want 303 -> login with the admin-required error", rec.Code, rec.Header().Get("Location"))
+	if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/ui/login?error=admin+role+required&next=%2Fui%2Fadmin" {
+		t.Errorf("non-admin got %d -> %q, want 303 -> login with the admin-required error and next", rec.Code, rec.Header().Get("Location"))
 	}
 }
 
@@ -99,3 +99,25 @@ func TestAdminUserActionRejectsBadID(t *testing.T) {
 		t.Errorf("bad id redirect = %q, want an error", rec.Header().Get("Location"))
 	}
 }
+
+func TestLoginPageExplainsAdminRequiredError(t *testing.T) {
+	html := render(t, "login.html", map[string]any{"Error": "admin role required"})
+	if !strings.Contains(html, "/ui/logout-form") {
+		t.Error("the admin-required error must offer a logout path to switch accounts")
+	}
+	if !strings.Contains(html, "cluster administrator") {
+		t.Error("the admin-required error must name the admin account")
+	}
+	// Other errors keep the plain message, no logout teaser.
+	plain := render(t, "login.html", map[string]any{"Error": "invalid email or password"})
+	if strings.Contains(plain, "/ui/logout-form") {
+		t.Error("plain login errors must not advertise logout")
+	}
+}
+
+func TestLogoutFormRendersPostButton(t *testing.T) {
+	html := render(t, "logout-form.html", map[string]any{})
+	if !strings.Contains(html, `action="/ui/logout"`) || !strings.Contains(html, "Log out") {
+		t.Error("logout form must POST /ui/logout")
+	}
+}

coordinator/internal/transport/http/ui_auth.go

@@ -58,6 +58,13 @@ func (s *Server) handleUILoginForm(w http.ResponseWriter, r *http.Request) {
 	s.renderUI(w, "login.html", map[string]any{"Error": r.URL.Query().Get("error"), "Next": r.URL.Query().Get("next")})
 }
 
+// handleUILogoutForm renders a small confirm page for ending the current
+// session. The actual logout stays a POST (/ui/logout); this page exists so a
+// signed-in non-admin who hit an admin-only page can switch accounts.
+func (s *Server) handleUILogoutForm(w http.ResponseWriter, r *http.Request) {
+	s.renderUI(w, "logout-form.html", map[string]any{})
+}
+
 func (s *Server) handleUIRegisterForm(w http.ResponseWriter, r *http.Request) {
 	s.renderUI(w, "register.html", map[string]any{"Error": r.URL.Query().Get("error")})
 }

install.ps1

@@ -97,6 +97,7 @@ if ($Component -eq "coordinator") {
         Write-Host "For a coordinator started with 'coordinator serve', the worker token is"
         Write-Host "in ~\.scimesh\worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
         Write-Host "install scimesh into a managed venv, or install it yourself:"
-        Write-Host "  pip install scimesh"
+        Write-Host "  set SCIMESH_PIP_PACKAGE=<your wheel or index>"
+        Write-Host "  $Target setup"
     }
 }

install.sh

@@ -112,5 +112,5 @@ else
   echo "For a coordinator started with 'coordinator serve', the worker token is"
   echo "in ~/.scimesh/worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
   echo "install scimesh into a managed venv, or install it yourself:"
-  echo "  pip install scimesh"
+  echo "  SCIMESH_PIP_PACKAGE=<your wheel or index> worker-agent setup"
 fi

mkdocs/index.md

@@ -113,10 +113,12 @@ chmod +x coordinator
   ```
 
   It spawns `python -m scimesh.worker.task`, so the machine needs Python 3
-  with the `scimesh` package (`pip install scimesh`, or let the managed venv
-  do it via `SCIMESH_PIP_PACKAGE`). For a `coordinator serve` instance, the
-  worker token is in `~/.scimesh/worker.token`. On Windows set
-  `SCIMESH_COMPONENT=worker` for `install.ps1`.
+  with the `scimesh` package. The wizard installs it into its own venv; the
+  package must come from your wheel, checkout or index — point
+  `SCIMESH_PIP_PACKAGE` at it (the PyPI name `scimesh` belongs to an unrelated
+  project). For a `coordinator serve` instance, the worker token is in
+  `~/.scimesh/worker.token`. On Windows set `SCIMESH_COMPONENT=worker` for
+  `install.ps1`.
 - **coordinator** needs no external services at all in its default mode:
   `coordinator serve` embeds SQLite (both databases), the userservice, and
   local workers. The `SCIMESH_DB=postgres` engine remains for cluster

mkdocs/standalone.md

@@ -59,6 +59,13 @@ curl -fsSL https://raw.githubusercontent.com/emil28092005/SciMesh/main/install.s
 # the installer opens the local wizard at http://127.0.0.1:12700 automatically
 ```
 
+On a headless server (no desktop environment), RDKit needs a few X11
+libraries that desktops already ship — install them once with apt:
+
+```bash
+sudo apt-get install -y libxrender1 libxext6 libxcursor1 libxfixes3 libxi6 libxrandr2
+```
+
 Or configure by hand:
 
 ```bash