Ship the scimesh wheel in releases; wizard and serve install it version-locked

.github/workflows/release.yml

@@ -54,8 +54,39 @@ jobs:
           path: coordinator/dist/*
           if-no-files-found: error
 
+  wheel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: build the scimesh wheel
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: |
+          # The tag (v1.1.0-alpha.10) becomes the package version in its
+          # PEP 440 form (1.1.0a10); the wheel is then version-locked to the
+          # binaries of the same release.
+          WHEEL_VERSION="${VERSION#v}"
+          WHEEL_VERSION="${WHEEL_VERSION/-alpha./a}"
+          WHEEL_VERSION="${WHEEL_VERSION/-beta./b}"
+          WHEEL_VERSION="${WHEEL_VERSION/-rc./rc}"
+          sed -i "s/^version = .*/version = \"${WHEEL_VERSION}\"/" pyproject.toml
+          python -m pip install --quiet build
+          python -m build --wheel --outdir dist
+          ls -la dist/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel
+          path: dist/*.whl
+          if-no-files-found: error
+
   release:
-    needs: binaries
+    needs: [binaries, wheel]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -68,6 +99,11 @@ jobs:
           pattern: binaries-*
           merge-multiple: true
 
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheel
+          path: artifacts
+
       - name: checksums
         working-directory: artifacts
         run: sha256sum * > SHA256SUMS.txt

coordinator/cmd/coordinator/serve_cmd.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"context"
+
+	"github.com/emil28092005/SciMesh/coordinator/internal/agent"
+
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
@@ -259,13 +262,25 @@ func ensureRuntime(log *slog.Logger, dataDir, venvPython string) {
 	}
 	pip := filepath.Join(venvDir, binName("bin/pip"))
 	// The scimesh package is installed from an explicit source only: the PyPI
-	// name is not ours yet, so `pip install scimesh` would fetch a stranger's
-	// package. Operators publish a wheel or index via SCIMESH_PIP_PACKAGE.
+	// name belongs to an unrelated project, so `pip install scimesh` would
+	// fetch a stranger's package. Default: download the wheel attached to our
+	// own GitHub release for this binary version; SCIMESH_PIP_PACKAGE
+	// overrides with a custom wheel, checkout or index.
 	source := os.Getenv("SCIMESH_PIP_PACKAGE")
 	if source == "" {
-		log.Warn("scientific runtime venv created, but scimesh is not installed",
-			"hint", pip+" install <your scimesh wheel or index> (or set SCIMESH_PIP_PACKAGE)")
-		return
+		url, _, err := agent.ReleaseWheelURL(version)
+		if err != nil {
+			log.Warn("scientific runtime venv created, but scimesh is not installed",
+				"hint", "set SCIMESH_PIP_PACKAGE to your wheel or index")
+			return
+		}
+		downloaded, err := agent.DownloadWheel(context.Background(), url, venvDir)
+		if err != nil {
+			log.Warn("could not download the scimesh wheel for this release",
+				"err", err, "hint", "set SCIMESH_PIP_PACKAGE to your wheel or index")
+			return
+		}
+		source = downloaded
 	}
 	// #nosec G204,G702 -- pip and source are operator-configured paths.
 	install := exec.CommandContext(context.Background(), pip, "install", source)

coordinator/internal/agent/check.go

@@ -85,8 +85,10 @@ func CheckEnvironment(ctx context.Context) CheckReport {
 	out, err := cmd.Output()
 	if err != nil {
 		// The worker executes workloads by spawning scimesh's task runner, so
-		// the package is a hard requirement, not an optimisation.
-		report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "the worker runs workloads through scimesh — install with: pip install scimesh"}
+		// the package is a hard requirement, not an optimisation. The PyPI
+		// name belongs to a different project, so the wizard installs from
+		// SCIMESH_PIP_PACKAGE instead of suggesting a bare pip install.
+		report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "the worker runs workloads through scimesh — install it from your wheel or index (SCIMESH_PIP_PACKAGE)"}
 		return report
 	}
 	report.Scimesh = CheckItem{Name: "scimesh", OK: true, Detail: strings.TrimSpace(string(out))}

coordinator/internal/agent/releasewheel.go

@@ -0,0 +1,70 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ReleaseWheelURL returns the download URL of the scimesh wheel attached to
+// the GitHub release that matches the given binary version (for example
+// "1.1.0-alpha.10"), plus the wheel file name. The wheel is version-locked to
+// the binary so a worker's catalog always matches its task runner.
+func ReleaseWheelURL(version string) (string, string, error) {
+	if version == "" || version == "dev" {
+		return "", "", fmt.Errorf("no release wheel for build %q", version)
+	}
+	filename := fmt.Sprintf("scimesh-%s-py3-none-any.whl", NormalizePEP440(version))
+	return fmt.Sprintf("https://github.com/emil28092005/SciMesh/releases/download/v%s/%s", version, filename), filename, nil
+}
+
+// NormalizePEP440 turns our release tag suffixes into the PEP 440 form
+// setuptools uses for wheel names: 1.1.0-alpha.10 -> 1.1.0a10,
+// 1.1.0-beta.2 -> 1.1.0b2, 1.1.0-rc.1 -> 1.1.0rc1. Stable tags pass through.
+func NormalizePEP440(version string) string {
+	for from, to := range map[string]string{"-alpha.": "a", "-beta.": "b", "-rc.": "rc"} {
+		version = strings.ReplaceAll(version, from, to)
+	}
+	return version
+}
+
+// DownloadWheel fetches the release wheel into dir (config directory of the
+// wizard / serve data dir) and returns the local path. Best-effort download
+// with a generous timeout: wheels can be several MB.
+func DownloadWheel(ctx context.Context, url, dir string) (string, error) {
+	target := filepath.Join(dir, wheelNameFromURL(url))
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("download wheel: HTTP %d", resp.StatusCode)
+	}
+	//nolint:gosec // G304: target is our own config dir + a fixed wheel name
+	out, err := os.Create(target)
+	if err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
+	defer func() { _ = out.Close() }()
+	if _, err := io.Copy(out, resp.Body); err != nil {
+		return "", fmt.Errorf("download wheel: %w", err)
+	}
+	return target, nil
+}
+
+// wheelNameFromURL extracts the trailing file name of a wheel URL.
+func wheelNameFromURL(url string) string {
+	return url[strings.LastIndex(url, "/")+1:]
+}

coordinator/internal/agent/releasewheel_test.go

@@ -0,0 +1,81 @@
+package agent
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestNormalizePEP440(t *testing.T) {
+	cases := map[string]string{
+		"1.1.0":          "1.1.0",
+		"1.1.0-alpha.10": "1.1.0a10",
+		"1.1.0-beta.2":   "1.1.0b2",
+		"1.1.0-rc.1":     "1.1.0rc1",
+		"1.0.0":          "1.0.0",
+	}
+	for in, want := range cases {
+		if got := NormalizePEP440(in); got != want {
+			t.Errorf("NormalizePEP440(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestReleaseWheelURL(t *testing.T) {
+	url, name, err := ReleaseWheelURL("1.1.0-alpha.10")
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantURL := "https://github.com/emil28092005/SciMesh/releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl"
+	if url != wantURL {
+		t.Errorf("url = %q, want %q", url, wantURL)
+	}
+	if name != "scimesh-1.1.0a10-py3-none-any.whl" {
+		t.Errorf("name = %q", name)
+	}
+
+	// A dev build has no release wheel.
+	if _, _, err := ReleaseWheelURL("dev"); err == nil {
+		t.Error("dev build must not resolve a release wheel")
+	}
+	if _, _, err := ReleaseWheelURL(""); err == nil {
+		t.Error("empty version must not resolve a release wheel")
+	}
+}
+
+func TestDownloadWheel(t *testing.T) {
+	payload := []byte("fake wheel bytes")
+	stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(payload)
+	}))
+	defer stub.Close()
+
+	dir := t.TempDir()
+	path, err := DownloadWheel(context.Background(), stub.URL+"/scimesh-1.1.0a10-py3-none-any.whl", dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasSuffix(path, "scimesh-1.1.0a10-py3-none-any.whl") {
+		t.Errorf("path = %q", path)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != string(payload) {
+		t.Error("wheel bytes mismatch")
+	}
+}
+
+func TestDownloadWheelReportsHTTPErrors(t *testing.T) {
+	stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.NotFound(w, r)
+	}))
+	defer stub.Close()
+	if _, err := DownloadWheel(context.Background(), stub.URL+"/missing.whl", t.TempDir()); err == nil {
+		t.Error("404 must fail the download")
+	}
+}

coordinator/internal/agent/setupui/server.go

@@ -176,14 +176,15 @@ func (s *PIDSupervisor) Stop() error {
 
 // Server is the wizard HTTP server, bound to the loopback interface only.
 type Server struct {
-	log         *slog.Logger
-	cfgPath     string
-	logPath     string
-	dir         string
-	sup         Supervisor
-	openBrowser func(string)
-	port        int
-	install     func(ctx context.Context, venvPython, pkg string) error
+	log           *slog.Logger
+	cfgPath       string
+	logPath       string
+	dir           string
+	sup           Supervisor
+	openBrowser   func(string)
+	port          int
+	install       func(ctx context.Context, venvPython, pkg string) error
+	downloadWheel func(ctx context.Context, url, dir string) (string, error)
 }
 
 // Options customises the wizard for tests and embedding.
@@ -196,6 +197,9 @@ type Options struct {
 	// InstallScimesh overrides the pip step of the runtime installer (tests
 	// substitute a fake); nil uses the real pip inside the managed venv.
 	InstallScimesh func(ctx context.Context, venvPython, pkg string) error
+	// DownloadWheel overrides the release-wheel download (tests substitute a
+	// fake); nil downloads from the GitHub release matching the agent version.
+	DownloadWheel func(ctx context.Context, url, dir string) (string, error)
 }
 
 func New(log *slog.Logger, opts Options) *Server {
@@ -223,7 +227,11 @@ func New(log *slog.Logger, opts Options) *Server {
 	if install == nil {
 		install = installScimeshWithPip
 	}
-	return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install}
+	downloadWheel := opts.DownloadWheel
+	if downloadWheel == nil {
+		downloadWheel = agent.DownloadWheel
+	}
+	return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install, downloadWheel: downloadWheel}
 }
 
 // Listen binds the loopback listener and returns it; Serve runs the server on
@@ -450,7 +458,25 @@ func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) {
 		pkg = os.Getenv("SCIMESH_PIP_PACKAGE")
 	}
 	if pkg == "" {
-		pkg = "scimesh"
+		// No PyPI default on purpose: the PyPI name "scimesh" belongs to an
+		// unrelated project. Instead we ship the wheel in our own GitHub
+		// release, version-locked to this binary, and download it from there.
+		url, _, err := agent.ReleaseWheelURL(agent.Version)
+		if err != nil {
+			writeJSON(w, http.StatusConflict, map[string]string{
+				"error": "no scimesh source configured: set SCIMESH_PIP_PACKAGE to your wheel, checkout or index, then retry",
+			})
+			return
+		}
+		local, err := s.downloadWheel(r.Context(), url, s.dir)
+		if err != nil {
+			s.log.Error("download release wheel", "err", err, "url", url)
+			writeJSON(w, http.StatusConflict, map[string]string{
+				"error": "could not download the scimesh wheel for this release: " + err.Error() + ". Set SCIMESH_PIP_PACKAGE to your wheel, checkout or index and retry.",
+			})
+			return
+		}
+		pkg = local
 	}
 
 	python3, err := exec.LookPath("python3")

coordinator/internal/agent/setupui/server_test.go

@@ -29,6 +29,11 @@ func newTestServer(t *testing.T, sup Supervisor) (*Server, string) {
 }
 
 func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error) (*Server, string) {
+	t.Helper()
+	return newTestServerWithInstallAndWheel(t, sup, install, nil)
+}
+
+func newTestServerWithInstallAndWheel(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error, wheel func(ctx context.Context, url, dir string) (string, error)) (*Server, string) {
 	t.Helper()
 	dir := t.TempDir()
 	server := New(testLogger(), Options{
@@ -41,6 +46,7 @@ func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx con
 		Supervisor:     sup,
 		OpenBrowser:    func(string) {},
 		InstallScimesh: install,
+		DownloadWheel:  wheel,
 	})
 	listener, err := server.Listen()
 	if err != nil {
@@ -311,18 +317,35 @@ func TestInstallRuntimeCreatesVenvAndReportsPython(t *testing.T) {
 		return nil
 	})
 
-	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{"scimesh_package": "/wheels/scimesh.whl"})
 	if rec.Code != http.StatusOK || data["ok"] != true {
 		t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
 	}
-	if installedPkg != "scimesh" {
-		t.Errorf("package = %q, want the default scimesh", installedPkg)
+	if installedPkg != "/wheels/scimesh.whl" {
+		t.Errorf("package = %q, want the requested wheel", installedPkg)
 	}
 	if !strings.HasSuffix(data["python"].(string), "venv/bin/python") {
 		t.Errorf("python = %v, want the venv python", data["python"])
 	}
 }
 
+func TestInstallRuntimeRequiresASource(t *testing.T) {
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
+		t.Fatal("install must not run without a package source")
+		return nil
+	})
+	// No source anywhere (SCIMESH_PIP_PACKAGE unset, request empty): 409 with
+	// guidance. The PyPI name is another project, so no silent fallback.
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusConflict {
+		t.Fatalf("install without source: got %d, want 409", rec.Code)
+	}
+	if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
+		t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
+	}
+}
+
 func TestInstallRuntimeFailureIsExplained(t *testing.T) {
 	sup := &fakeSup{}
 	_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
@@ -336,3 +359,53 @@ func TestInstallRuntimeFailureIsExplained(t *testing.T) {
 		t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
 	}
 }
+
+func TestInstallRuntimeDownloadsReleaseWheelWhenNoSource(t *testing.T) {
+	oldVersion := agent.Version
+	agent.Version = "1.1.0-alpha.10"
+	t.Cleanup(func() { agent.Version = oldVersion })
+
+	var downloadedURL, installedPkg string
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstallAndWheel(t, sup,
+		func(ctx context.Context, venvPython, pkg string) error {
+			installedPkg = pkg
+			return nil
+		},
+		func(ctx context.Context, url, dir string) (string, error) {
+			downloadedURL = url
+			return filepath.Join(dir, "scimesh-1.1.0a10-py3-none-any.whl"), nil
+		})
+
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusOK || data["ok"] != true {
+		t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
+	}
+	if !strings.Contains(downloadedURL, "releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl") {
+		t.Errorf("download url = %q, want the release wheel of this version", downloadedURL)
+	}
+	if !strings.HasSuffix(installedPkg, "scimesh-1.1.0a10-py3-none-any.whl") {
+		t.Errorf("pip received %q, want the downloaded wheel", installedPkg)
+	}
+}
+
+func TestInstallRuntimeWheelDownloadFailureIsExplained(t *testing.T) {
+	oldVersion := agent.Version
+	agent.Version = "1.1.0-alpha.10"
+	t.Cleanup(func() { agent.Version = oldVersion })
+
+	sup := &fakeSup{}
+	_, base := newTestServerWithInstallAndWheel(t, sup,
+		func(ctx context.Context, venvPython, pkg string) error { t.Fatal("pip must not run"); return nil },
+		func(ctx context.Context, url, dir string) (string, error) {
+			return "", errors.New("HTTP 404")
+		})
+
+	rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
+	if rec.Code != http.StatusConflict {
+		t.Fatalf("got %d, want 409", rec.Code)
+	}
+	if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
+		t.Errorf("error = %v, want a SCIMESH_PIP_PACKAGE hint", data["error"])
+	}
+}

install.ps1

@@ -97,6 +97,7 @@ if ($Component -eq "coordinator") {
         Write-Host "For a coordinator started with 'coordinator serve', the worker token is"
         Write-Host "in ~\.scimesh\worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
         Write-Host "install scimesh into a managed venv, or install it yourself:"
-        Write-Host "  pip install scimesh"
+        Write-Host "  set SCIMESH_PIP_PACKAGE=<your wheel or index>"
+        Write-Host "  $Target setup"
     }
 }

install.sh

@@ -112,5 +112,5 @@ else
   echo "For a coordinator started with 'coordinator serve', the worker token is"
   echo "in ~/.scimesh/worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
   echo "install scimesh into a managed venv, or install it yourself:"
-  echo "  pip install scimesh"
+  echo "  SCIMESH_PIP_PACKAGE=<your wheel or index> worker-agent setup"
 fi

mkdocs/index.md

@@ -113,10 +113,12 @@ chmod +x coordinator
   ```
 
   It spawns `python -m scimesh.worker.task`, so the machine needs Python 3
-  with the `scimesh` package (`pip install scimesh`, or let the managed venv
-  do it via `SCIMESH_PIP_PACKAGE`). For a `coordinator serve` instance, the
-  worker token is in `~/.scimesh/worker.token`. On Windows set
-  `SCIMESH_COMPONENT=worker` for `install.ps1`.
+  with the `scimesh` package. The wizard installs it into its own venv; the
+  package must come from your wheel, checkout or index — point
+  `SCIMESH_PIP_PACKAGE` at it (the PyPI name `scimesh` belongs to an unrelated
+  project). For a `coordinator serve` instance, the worker token is in
+  `~/.scimesh/worker.token`. On Windows set `SCIMESH_COMPONENT=worker` for
+  `install.ps1`.
 - **coordinator** needs no external services at all in its default mode:
   `coordinator serve` embeds SQLite (both databases), the userservice, and
   local workers. The `SCIMESH_DB=postgres` engine remains for cluster