From 15dd9651a88f41ec3426bdec9264303d695aa607 Mon Sep 17 00:00:00 2001 From: Emil Date: Mon, 3 Aug 2026 02:59:22 +0300 Subject: [PATCH] Ship the scimesh wheel in releases; wizard and serve install it version-locked --- .github/workflows/release.yml | 38 ++++++++- coordinator/cmd/coordinator/serve_cmd.go | 25 ++++-- coordinator/internal/agent/releasewheel.go | 70 ++++++++++++++++ .../internal/agent/releasewheel_test.go | 81 +++++++++++++++++++ coordinator/internal/agent/setupui/server.go | 50 ++++++++---- .../internal/agent/setupui/server_test.go | 56 +++++++++++++ 6 files changed, 299 insertions(+), 21 deletions(-) create mode 100644 coordinator/internal/agent/releasewheel.go create mode 100644 coordinator/internal/agent/releasewheel_test.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39bd509..64100d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,8 +54,39 @@ jobs: path: coordinator/dist/* if-no-files-found: error + wheel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: build the scimesh wheel + env: + VERSION: ${{ github.ref_name }} + run: | + # The tag (v1.1.0-alpha.10) becomes the package version in its + # PEP 440 form (1.1.0a10); the wheel is then version-locked to the + # binaries of the same release. + WHEEL_VERSION="${VERSION#v}" + WHEEL_VERSION="${WHEEL_VERSION/-alpha./a}" + WHEEL_VERSION="${WHEEL_VERSION/-beta./b}" + WHEEL_VERSION="${WHEEL_VERSION/-rc./rc}" + sed -i "s/^version = .*/version = \"${WHEEL_VERSION}\"/" pyproject.toml + python -m pip install --quiet build + python -m build --wheel --outdir dist + ls -la dist/ + + - uses: actions/upload-artifact@v4 + with: + name: wheel + path: dist/*.whl + if-no-files-found: error + release: - needs: binaries + needs: [binaries, wheel] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -68,6 +99,11 @@ jobs: pattern: binaries-* merge-multiple: true + - uses: actions/download-artifact@v4 + with: + name: wheel + path: artifacts + - name: checksums working-directory: artifacts run: sha256sum * > SHA256SUMS.txt diff --git a/coordinator/cmd/coordinator/serve_cmd.go b/coordinator/cmd/coordinator/serve_cmd.go index 7f0756c..9cefe73 100644 --- a/coordinator/cmd/coordinator/serve_cmd.go +++ b/coordinator/cmd/coordinator/serve_cmd.go @@ -2,6 +2,9 @@ package main import ( "context" + + "github.com/emil28092005/SciMesh/coordinator/internal/agent" + "crypto/rand" "encoding/hex" "flag" @@ -259,13 +262,25 @@ func ensureRuntime(log *slog.Logger, dataDir, venvPython string) { } pip := filepath.Join(venvDir, binName("bin/pip")) // The scimesh package is installed from an explicit source only: the PyPI - // name is not ours yet, so `pip install scimesh` would fetch a stranger's - // package. Operators publish a wheel or index via SCIMESH_PIP_PACKAGE. + // name belongs to an unrelated project, so `pip install scimesh` would + // fetch a stranger's package. Default: download the wheel attached to our + // own GitHub release for this binary version; SCIMESH_PIP_PACKAGE + // overrides with a custom wheel, checkout or index. source := os.Getenv("SCIMESH_PIP_PACKAGE") if source == "" { - log.Warn("scientific runtime venv created, but scimesh is not installed", - "hint", pip+" install (or set SCIMESH_PIP_PACKAGE)") - return + url, _, err := agent.ReleaseWheelURL(version) + if err != nil { + log.Warn("scientific runtime venv created, but scimesh is not installed", + "hint", "set SCIMESH_PIP_PACKAGE to your wheel or index") + return + } + downloaded, err := agent.DownloadWheel(context.Background(), url, venvDir) + if err != nil { + log.Warn("could not download the scimesh wheel for this release", + "err", err, "hint", "set SCIMESH_PIP_PACKAGE to your wheel or index") + return + } + source = downloaded } // #nosec G204,G702 -- pip and source are operator-configured paths. install := exec.CommandContext(context.Background(), pip, "install", source) diff --git a/coordinator/internal/agent/releasewheel.go b/coordinator/internal/agent/releasewheel.go new file mode 100644 index 0000000..a8f0f45 --- /dev/null +++ b/coordinator/internal/agent/releasewheel.go @@ -0,0 +1,70 @@ +package agent + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +// ReleaseWheelURL returns the download URL of the scimesh wheel attached to +// the GitHub release that matches the given binary version (for example +// "1.1.0-alpha.10"), plus the wheel file name. The wheel is version-locked to +// the binary so a worker's catalog always matches its task runner. +func ReleaseWheelURL(version string) (string, string, error) { + if version == "" || version == "dev" { + return "", "", fmt.Errorf("no release wheel for build %q", version) + } + filename := fmt.Sprintf("scimesh-%s-py3-none-any.whl", NormalizePEP440(version)) + return fmt.Sprintf("https://github.com/emil28092005/SciMesh/releases/download/v%s/%s", version, filename), filename, nil +} + +// NormalizePEP440 turns our release tag suffixes into the PEP 440 form +// setuptools uses for wheel names: 1.1.0-alpha.10 -> 1.1.0a10, +// 1.1.0-beta.2 -> 1.1.0b2, 1.1.0-rc.1 -> 1.1.0rc1. Stable tags pass through. +func NormalizePEP440(version string) string { + for from, to := range map[string]string{"-alpha.": "a", "-beta.": "b", "-rc.": "rc"} { + version = strings.ReplaceAll(version, from, to) + } + return version +} + +// DownloadWheel fetches the release wheel into dir (config directory of the +// wizard / serve data dir) and returns the local path. Best-effort download +// with a generous timeout: wheels can be several MB. +func DownloadWheel(ctx context.Context, url, dir string) (string, error) { + target := filepath.Join(dir, wheelNameFromURL(url)) + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("download wheel: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("download wheel: HTTP %d", resp.StatusCode) + } + //nolint:gosec // G304: target is our own config dir + a fixed wheel name + out, err := os.Create(target) + if err != nil { + return "", fmt.Errorf("download wheel: %w", err) + } + defer func() { _ = out.Close() }() + if _, err := io.Copy(out, resp.Body); err != nil { + return "", fmt.Errorf("download wheel: %w", err) + } + return target, nil +} + +// wheelNameFromURL extracts the trailing file name of a wheel URL. +func wheelNameFromURL(url string) string { + return url[strings.LastIndex(url, "/")+1:] +} diff --git a/coordinator/internal/agent/releasewheel_test.go b/coordinator/internal/agent/releasewheel_test.go new file mode 100644 index 0000000..9d4af31 --- /dev/null +++ b/coordinator/internal/agent/releasewheel_test.go @@ -0,0 +1,81 @@ +package agent + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestNormalizePEP440(t *testing.T) { + cases := map[string]string{ + "1.1.0": "1.1.0", + "1.1.0-alpha.10": "1.1.0a10", + "1.1.0-beta.2": "1.1.0b2", + "1.1.0-rc.1": "1.1.0rc1", + "1.0.0": "1.0.0", + } + for in, want := range cases { + if got := NormalizePEP440(in); got != want { + t.Errorf("NormalizePEP440(%q) = %q, want %q", in, got, want) + } + } +} + +func TestReleaseWheelURL(t *testing.T) { + url, name, err := ReleaseWheelURL("1.1.0-alpha.10") + if err != nil { + t.Fatal(err) + } + wantURL := "https://github.com/emil28092005/SciMesh/releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl" + if url != wantURL { + t.Errorf("url = %q, want %q", url, wantURL) + } + if name != "scimesh-1.1.0a10-py3-none-any.whl" { + t.Errorf("name = %q", name) + } + + // A dev build has no release wheel. + if _, _, err := ReleaseWheelURL("dev"); err == nil { + t.Error("dev build must not resolve a release wheel") + } + if _, _, err := ReleaseWheelURL(""); err == nil { + t.Error("empty version must not resolve a release wheel") + } +} + +func TestDownloadWheel(t *testing.T) { + payload := []byte("fake wheel bytes") + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(payload) + })) + defer stub.Close() + + dir := t.TempDir() + path, err := DownloadWheel(context.Background(), stub.URL+"/scimesh-1.1.0a10-py3-none-any.whl", dir) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(path, "scimesh-1.1.0a10-py3-none-any.whl") { + t.Errorf("path = %q", path) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != string(payload) { + t.Error("wheel bytes mismatch") + } +} + +func TestDownloadWheelReportsHTTPErrors(t *testing.T) { + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer stub.Close() + if _, err := DownloadWheel(context.Background(), stub.URL+"/missing.whl", t.TempDir()); err == nil { + t.Error("404 must fail the download") + } +} diff --git a/coordinator/internal/agent/setupui/server.go b/coordinator/internal/agent/setupui/server.go index 45161f7..7d9cca7 100644 --- a/coordinator/internal/agent/setupui/server.go +++ b/coordinator/internal/agent/setupui/server.go @@ -176,14 +176,15 @@ func (s *PIDSupervisor) Stop() error { // Server is the wizard HTTP server, bound to the loopback interface only. type Server struct { - log *slog.Logger - cfgPath string - logPath string - dir string - sup Supervisor - openBrowser func(string) - port int - install func(ctx context.Context, venvPython, pkg string) error + log *slog.Logger + cfgPath string + logPath string + dir string + sup Supervisor + openBrowser func(string) + port int + install func(ctx context.Context, venvPython, pkg string) error + downloadWheel func(ctx context.Context, url, dir string) (string, error) } // Options customises the wizard for tests and embedding. @@ -196,6 +197,9 @@ type Options struct { // InstallScimesh overrides the pip step of the runtime installer (tests // substitute a fake); nil uses the real pip inside the managed venv. InstallScimesh func(ctx context.Context, venvPython, pkg string) error + // DownloadWheel overrides the release-wheel download (tests substitute a + // fake); nil downloads from the GitHub release matching the agent version. + DownloadWheel func(ctx context.Context, url, dir string) (string, error) } func New(log *slog.Logger, opts Options) *Server { @@ -223,7 +227,11 @@ func New(log *slog.Logger, opts Options) *Server { if install == nil { install = installScimeshWithPip } - return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install} + downloadWheel := opts.DownloadWheel + if downloadWheel == nil { + downloadWheel = agent.DownloadWheel + } + return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install, downloadWheel: downloadWheel} } // Listen binds the loopback listener and returns it; Serve runs the server on @@ -451,12 +459,24 @@ func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) { } if pkg == "" { // No PyPI default on purpose: the PyPI name "scimesh" belongs to an - // unrelated project, so a silent `pip install scimesh` would install - // the wrong software. - writeJSON(w, http.StatusConflict, map[string]string{ - "error": "no scimesh source configured: set SCIMESH_PIP_PACKAGE to your wheel, checkout or index, then retry", - }) - return + // unrelated project. Instead we ship the wheel in our own GitHub + // release, version-locked to this binary, and download it from there. + url, _, err := agent.ReleaseWheelURL(agent.Version) + if err != nil { + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "no scimesh source configured: set SCIMESH_PIP_PACKAGE to your wheel, checkout or index, then retry", + }) + return + } + local, err := s.downloadWheel(r.Context(), url, s.dir) + if err != nil { + s.log.Error("download release wheel", "err", err, "url", url) + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "could not download the scimesh wheel for this release: " + err.Error() + ". Set SCIMESH_PIP_PACKAGE to your wheel, checkout or index and retry.", + }) + return + } + pkg = local } python3, err := exec.LookPath("python3") diff --git a/coordinator/internal/agent/setupui/server_test.go b/coordinator/internal/agent/setupui/server_test.go index ff63217..44e1ab7 100644 --- a/coordinator/internal/agent/setupui/server_test.go +++ b/coordinator/internal/agent/setupui/server_test.go @@ -29,6 +29,11 @@ func newTestServer(t *testing.T, sup Supervisor) (*Server, string) { } func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error) (*Server, string) { + t.Helper() + return newTestServerWithInstallAndWheel(t, sup, install, nil) +} + +func newTestServerWithInstallAndWheel(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error, wheel func(ctx context.Context, url, dir string) (string, error)) (*Server, string) { t.Helper() dir := t.TempDir() server := New(testLogger(), Options{ @@ -41,6 +46,7 @@ func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx con Supervisor: sup, OpenBrowser: func(string) {}, InstallScimesh: install, + DownloadWheel: wheel, }) listener, err := server.Listen() if err != nil { @@ -353,3 +359,53 @@ func TestInstallRuntimeFailureIsExplained(t *testing.T) { t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"]) } } + +func TestInstallRuntimeDownloadsReleaseWheelWhenNoSource(t *testing.T) { + oldVersion := agent.Version + agent.Version = "1.1.0-alpha.10" + t.Cleanup(func() { agent.Version = oldVersion }) + + var downloadedURL, installedPkg string + sup := &fakeSup{} + _, base := newTestServerWithInstallAndWheel(t, sup, + func(ctx context.Context, venvPython, pkg string) error { + installedPkg = pkg + return nil + }, + func(ctx context.Context, url, dir string) (string, error) { + downloadedURL = url + return filepath.Join(dir, "scimesh-1.1.0a10-py3-none-any.whl"), nil + }) + + rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{}) + if rec.Code != http.StatusOK || data["ok"] != true { + t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data) + } + if !strings.Contains(downloadedURL, "releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl") { + t.Errorf("download url = %q, want the release wheel of this version", downloadedURL) + } + if !strings.HasSuffix(installedPkg, "scimesh-1.1.0a10-py3-none-any.whl") { + t.Errorf("pip received %q, want the downloaded wheel", installedPkg) + } +} + +func TestInstallRuntimeWheelDownloadFailureIsExplained(t *testing.T) { + oldVersion := agent.Version + agent.Version = "1.1.0-alpha.10" + t.Cleanup(func() { agent.Version = oldVersion }) + + sup := &fakeSup{} + _, base := newTestServerWithInstallAndWheel(t, sup, + func(ctx context.Context, venvPython, pkg string) error { t.Fatal("pip must not run"); return nil }, + func(ctx context.Context, url, dir string) (string, error) { + return "", errors.New("HTTP 404") + }) + + rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{}) + if rec.Code != http.StatusConflict { + t.Fatalf("got %d, want 409", rec.Code) + } + if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") { + t.Errorf("error = %v, want a SCIMESH_PIP_PACKAGE hint", data["error"]) + } +}