Let the worker wizard install the scimesh runtime into a managed venv

coordinator/internal/agent/check.go

@@ -84,7 +84,9 @@ func CheckEnvironment(ctx context.Context) CheckReport {
 	cmd := exec.CommandContext(ctx, python, "-c", "import scimesh; print(scimesh.__version__ if hasattr(scimesh, '__version__') else 'installed')")
 	out, err := cmd.Output()
 	if err != nil {
-		report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "install with: pip install scimesh"}
+		// The worker executes workloads by spawning scimesh's task runner, so
+		// the package is a hard requirement, not an optimisation.
+		report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "the worker runs workloads through scimesh — install with: pip install scimesh"}
 		return report
 	}
 	report.Scimesh = CheckItem{Name: "scimesh", OK: true, Detail: strings.TrimSpace(string(out))}

coordinator/internal/agent/setupui/server.go

@@ -183,6 +183,7 @@ type Server struct {
 	sup         Supervisor
 	openBrowser func(string)
 	port        int
+	install     func(ctx context.Context, venvPython, pkg string) error
 }
 
 // Options customises the wizard for tests and embedding.
@@ -192,6 +193,9 @@ type Options struct {
 	OpenBrowser func(url string)
 	Supervisor  Supervisor
 	Dir         string // directory for pid/log files; defaults to the config dir
+	// InstallScimesh overrides the pip step of the runtime installer (tests
+	// substitute a fake); nil uses the real pip inside the managed venv.
+	InstallScimesh func(ctx context.Context, venvPython, pkg string) error
 }
 
 func New(log *slog.Logger, opts Options) *Server {
@@ -215,7 +219,11 @@ func New(log *slog.Logger, opts Options) *Server {
 	if port == 0 {
 		port = defaultPort
 	}
-	return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port}
+	install := opts.InstallScimesh
+	if install == nil {
+		install = installScimeshWithPip
+	}
+	return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install}
 }
 
 // Listen binds the loopback listener and returns it; Serve runs the server on
@@ -234,6 +242,7 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error {
 	mux.HandleFunc("GET /api/status", s.handleStatus)
 	mux.HandleFunc("POST /api/config", s.handleSaveConfig)
 	mux.HandleFunc("POST /api/test", s.handleTest)
+	mux.HandleFunc("POST /api/runtime/install", s.handleInstallRuntime)
 	mux.HandleFunc("POST /api/start", s.handleStart)
 	mux.HandleFunc("POST /api/stop", s.handleStop)
 	mux.HandleFunc("GET /api/logs", s.handleLogs)
@@ -412,3 +421,94 @@ func (s *Server) handleLogs(w http.ResponseWriter, r *http.Request) {
 // ErrCanceled mirrors context.Canceled for callers that treat a cancelled
 // wizard as a clean exit.
 var ErrCanceled = errors.New("setup wizard cancelled")
+
+type installRuntimeRequest struct {
+	// ScimeshPackage overrides where the scimesh wheel comes from: a local
+	// wheel/index path or the PyPI name. Defaults to SCIMESH_PIP_PACKAGE, then
+	// to the PyPI name.
+	ScimeshPackage string `json:"scimesh_package"`
+}
+
+type installRuntimeResponse struct {
+	OK        bool   `json:"ok"`
+	Python    string `json:"python,omitempty"` // venv python to use as TASK_RUNNER[0]
+	Installed bool   `json:"installed"`
+}
+
+// handleInstallRuntime creates a managed venv next to the worker config and
+// installs the scimesh package into it, so the machine needs no manual pip
+// step. The venv python path is returned for the wizard to bake into the
+// task runner.
+func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) {
+	var req installRuntimeRequest
+	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON body"})
+		return
+	}
+	pkg := strings.TrimSpace(req.ScimeshPackage)
+	if pkg == "" {
+		pkg = os.Getenv("SCIMESH_PIP_PACKAGE")
+	}
+	if pkg == "" {
+		pkg = "scimesh"
+	}
+
+	python3, err := exec.LookPath("python3")
+	if err != nil {
+		writeJSON(w, http.StatusConflict, map[string]string{"error": "python3 is not installed on this machine"})
+		return
+	}
+	venvDir := filepath.Join(s.dir, "venv")
+	venvPython := filepath.Join(venvDir, "bin", "python")
+	if _, err := os.Stat(venvPython); err != nil {
+		// Windows layout: Scripts/python.exe.
+		if win := filepath.Join(venvDir, "Scripts", "python.exe"); stat(win) {
+			venvPython = win
+		}
+	}
+	if _, err := os.Stat(venvPython); err != nil {
+		ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
+		defer cancel()
+		cmd := exec.CommandContext(ctx, python3, "-m", "venv", venvDir) //nolint:gosec // G204: python3 from LookPath, venvDir is our own dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			s.log.Error("create runtime venv", "err", err, "out", truncate(string(out), 500))
+			writeJSON(w, http.StatusConflict, map[string]string{"error": "could not create the python venv"})
+			return
+		}
+	}
+
+	if err := s.install(r.Context(), venvPython, pkg); err != nil {
+		s.log.Error("install scimesh runtime", "err", err)
+		writeJSON(w, http.StatusConflict, map[string]string{
+			"error": "pip install " + pkg + " failed: " + err.Error() +
+				". Set SCIMESH_PIP_PACKAGE to your scimesh wheel or index and retry.",
+		})
+		return
+	}
+	writeJSON(w, http.StatusOK, installRuntimeResponse{OK: true, Python: venvPython, Installed: true})
+}
+
+// installScimeshWithPip installs the package with the venv's own pip,
+// streaming into the agent log so a long build is not silent.
+func installScimeshWithPip(ctx context.Context, venvPython, pkg string) error {
+	pip := filepath.Join(filepath.Dir(venvPython), "pip")
+	if _, err := os.Stat(pip); err != nil {
+		pip += ".exe"
+	}
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, pip, "install", pkg) //nolint:gosec // G204: pip from our venv, pkg is operator-set or a fixed default
+	return cmd.Run()
+}
+
+func stat(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}

coordinator/internal/agent/setupui/server_test.go

@@ -3,6 +3,7 @@ package setupui
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"io"
 	"log/slog"
 	"net"
@@ -23,17 +24,23 @@ func testLogger() *slog.Logger {
 }
 
 func newTestServer(t *testing.T, sup Supervisor) (*Server, string) {
+	t.Helper()
+	return newTestServerWithInstall(t, sup, nil)
+}
+
+func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error) (*Server, string) {
 	t.Helper()
 	dir := t.TempDir()
 	server := New(testLogger(), Options{
 		// A distinct random port per test: Port 0 means "the default 12700" in
 		// the server, which would let the shared http.Client pool reuse a stale
 		// keep-alive connection across tests (EOF after a Shutdown).
-		Port:        freePort(t),
+		Port:           freePort(t),
-		ConfigPath:  filepath.Join(dir, "config.json"),
+		ConfigPath:     filepath.Join(dir, "config.json"),
-		Dir:         dir,
+		Dir:            dir,
-		Supervisor:  sup,
+		Supervisor:     sup,
-		OpenBrowser: func(string) {},
+		OpenBrowser:    func(string) {},
+		InstallScimesh: install,
 	})
 	listener, err := server.Listen()
 	if err != nil {
@@ -291,3 +298,41 @@ func TestConfigFileDefaultsAndEnvOverride(t *testing.T) {
 		t.Errorf("cpu = %d", config.CPUCount)
 	}
 }
+
+func TestInstallRuntimeCreatesVenvAndReportsPython(t *testing.T) {
+	var installedPkg string
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
+		installedPkg = pkg
+		// Prove the venv python path really exists by creating a marker file
+		// where the real venv python would be.
+		_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
+		_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
+		return nil
+	})
+
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusOK || data["ok"] != true {
+		t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
+	}
+	if installedPkg != "scimesh" {
+		t.Errorf("package = %q, want the default scimesh", installedPkg)
+	}
+	if !strings.HasSuffix(data["python"].(string), "venv/bin/python") {
+		t.Errorf("python = %v, want the venv python", data["python"])
+	}
+}
+
+func TestInstallRuntimeFailureIsExplained(t *testing.T) {
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
+		return errors.New("no matching distribution found")
+	})
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{"scimesh_package": "/wheels/scimesh.whl"})
+	if rec.Code != http.StatusConflict {
+		t.Fatalf("install failure: got %d, want 409", rec.Code)
+	}
+	if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
+		t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
+	}
+}

coordinator/internal/agent/setupui/template.html

@@ -183,7 +183,7 @@ code{font-family:var(--mono);font-size:.86em}
 
 <script>
 const $=id=>document.getElementById(id);
-let state={mode:'token',cpu:'auto'};
+let state={mode:'token',cpu:'auto',venvPython:null};
 let checksOk=false;
 
 function err(msg){$('error-box').innerHTML=msg?'<div class="error-strip">'+msg+'</div>':''}
@@ -215,7 +215,7 @@ async function postJSON(path,body){
   return {status:r.status,data};
 }
 function draftConfig(){
-  return {
+  const cfg={
     coordinator_url:$('in-url').value.trim(),
     token:state.mode==='token'?$('in-token').value.trim():'',
     worker_key:state.mode==='key'?$('in-key').value.trim():'',
@@ -224,6 +224,8 @@ function draftConfig(){
     worker_name:$('in-name').value.trim(),
     cpu_count:state.cpu==='custom'?parseInt($('in-cpu').value||'1',10):0
   };
+  if(state.venvPython)cfg.task_runner=[state.venvPython,'-m','scimesh.worker.task'];
+  return cfg;
 }
 
 $('b1').onclick=()=>{