Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d309145290 | ||
|
|
15dd9651a8 | ||
|
|
30c441a7e9 | ||
|
|
cb51172885 |
@@ -54,8 +54,39 @@ jobs:
|
|||||||
path: coordinator/dist/*
|
path: coordinator/dist/*
|
||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
|
|
||||||
|
wheel:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
|
||||||
|
- name: build the scimesh wheel
|
||||||
|
env:
|
||||||
|
VERSION: ${{ github.ref_name }}
|
||||||
|
run: |
|
||||||
|
# The tag (v1.1.0-alpha.10) becomes the package version in its
|
||||||
|
# PEP 440 form (1.1.0a10); the wheel is then version-locked to the
|
||||||
|
# binaries of the same release.
|
||||||
|
WHEEL_VERSION="${VERSION#v}"
|
||||||
|
WHEEL_VERSION="${WHEEL_VERSION/-alpha./a}"
|
||||||
|
WHEEL_VERSION="${WHEEL_VERSION/-beta./b}"
|
||||||
|
WHEEL_VERSION="${WHEEL_VERSION/-rc./rc}"
|
||||||
|
sed -i "s/^version = .*/version = \"${WHEEL_VERSION}\"/" pyproject.toml
|
||||||
|
python -m pip install --quiet build
|
||||||
|
python -m build --wheel --outdir dist
|
||||||
|
ls -la dist/
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: wheel
|
||||||
|
path: dist/*.whl
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
release:
|
release:
|
||||||
needs: binaries
|
needs: [binaries, wheel]
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
@@ -68,6 +99,11 @@ jobs:
|
|||||||
pattern: binaries-*
|
pattern: binaries-*
|
||||||
merge-multiple: true
|
merge-multiple: true
|
||||||
|
|
||||||
|
- uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: wheel
|
||||||
|
path: artifacts
|
||||||
|
|
||||||
- name: checksums
|
- name: checksums
|
||||||
working-directory: artifacts
|
working-directory: artifacts
|
||||||
run: sha256sum * > SHA256SUMS.txt
|
run: sha256sum * > SHA256SUMS.txt
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
|
"github.com/emil28092005/SciMesh/coordinator/internal/agent"
|
||||||
|
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"flag"
|
"flag"
|
||||||
@@ -259,13 +262,25 @@ func ensureRuntime(log *slog.Logger, dataDir, venvPython string) {
|
|||||||
}
|
}
|
||||||
pip := filepath.Join(venvDir, binName("bin/pip"))
|
pip := filepath.Join(venvDir, binName("bin/pip"))
|
||||||
// The scimesh package is installed from an explicit source only: the PyPI
|
// The scimesh package is installed from an explicit source only: the PyPI
|
||||||
// name is not ours yet, so `pip install scimesh` would fetch a stranger's
|
// name belongs to an unrelated project, so `pip install scimesh` would
|
||||||
// package. Operators publish a wheel or index via SCIMESH_PIP_PACKAGE.
|
// fetch a stranger's package. Default: download the wheel attached to our
|
||||||
|
// own GitHub release for this binary version; SCIMESH_PIP_PACKAGE
|
||||||
|
// overrides with a custom wheel, checkout or index.
|
||||||
source := os.Getenv("SCIMESH_PIP_PACKAGE")
|
source := os.Getenv("SCIMESH_PIP_PACKAGE")
|
||||||
if source == "" {
|
if source == "" {
|
||||||
log.Warn("scientific runtime venv created, but scimesh is not installed",
|
url, _, err := agent.ReleaseWheelURL(version)
|
||||||
"hint", pip+" install <your scimesh wheel or index> (or set SCIMESH_PIP_PACKAGE)")
|
if err != nil {
|
||||||
return
|
log.Warn("scientific runtime venv created, but scimesh is not installed",
|
||||||
|
"hint", "set SCIMESH_PIP_PACKAGE to your wheel or index")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
downloaded, err := agent.DownloadWheel(context.Background(), url, venvDir)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("could not download the scimesh wheel for this release",
|
||||||
|
"err", err, "hint", "set SCIMESH_PIP_PACKAGE to your wheel or index")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
source = downloaded
|
||||||
}
|
}
|
||||||
// #nosec G204,G702 -- pip and source are operator-configured paths.
|
// #nosec G204,G702 -- pip and source are operator-configured paths.
|
||||||
install := exec.CommandContext(context.Background(), pip, "install", source)
|
install := exec.CommandContext(context.Background(), pip, "install", source)
|
||||||
|
|||||||
@@ -27,6 +27,10 @@ import (
|
|||||||
var version = "dev"
|
var version = "dev"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
// The wizard and --check need the injected build version too (they resolve
|
||||||
|
// the release wheel matching this binary), so it is set before dispatch.
|
||||||
|
agent.Version = version
|
||||||
|
|
||||||
if len(os.Args) > 1 && os.Args[1] == "setup" {
|
if len(os.Args) > 1 && os.Args[1] == "setup" {
|
||||||
os.Exit(runSetup(os.Args[2:]))
|
os.Exit(runSetup(os.Args[2:]))
|
||||||
}
|
}
|
||||||
@@ -38,8 +42,6 @@ func main() {
|
|||||||
checkURL := fs.String("coordinator-url", "", "coordinator URL to probe in --check mode")
|
checkURL := fs.String("coordinator-url", "", "coordinator URL to probe in --check mode")
|
||||||
_ = fs.Parse(os.Args[1:])
|
_ = fs.Parse(os.Args[1:])
|
||||||
|
|
||||||
agent.Version = version
|
|
||||||
|
|
||||||
if *showVersion {
|
if *showVersion {
|
||||||
fmt.Println("worker-agent " + version)
|
fmt.Println("worker-agent " + version)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -84,7 +84,11 @@ func CheckEnvironment(ctx context.Context) CheckReport {
|
|||||||
cmd := exec.CommandContext(ctx, python, "-c", "import scimesh; print(scimesh.__version__ if hasattr(scimesh, '__version__') else 'installed')")
|
cmd := exec.CommandContext(ctx, python, "-c", "import scimesh; print(scimesh.__version__ if hasattr(scimesh, '__version__') else 'installed')")
|
||||||
out, err := cmd.Output()
|
out, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "install with: pip install scimesh"}
|
// The worker executes workloads by spawning scimesh's task runner, so
|
||||||
|
// the package is a hard requirement, not an optimisation. The PyPI
|
||||||
|
// name belongs to a different project, so the wizard installs from
|
||||||
|
// SCIMESH_PIP_PACKAGE instead of suggesting a bare pip install.
|
||||||
|
report.Scimesh = CheckItem{Name: "scimesh", OK: false, Detail: "the worker runs workloads through scimesh — install it from your wheel or index (SCIMESH_PIP_PACKAGE)"}
|
||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
report.Scimesh = CheckItem{Name: "scimesh", OK: true, Detail: strings.TrimSpace(string(out))}
|
report.Scimesh = CheckItem{Name: "scimesh", OK: true, Detail: strings.TrimSpace(string(out))}
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReleaseWheelURL returns the download URL of the scimesh wheel attached to
|
||||||
|
// the GitHub release that matches the given binary version (for example
|
||||||
|
// "1.1.0-alpha.10"), plus the wheel file name. The wheel is version-locked to
|
||||||
|
// the binary so a worker's catalog always matches its task runner.
|
||||||
|
func ReleaseWheelURL(version string) (string, string, error) {
|
||||||
|
if version == "" || version == "dev" {
|
||||||
|
return "", "", fmt.Errorf("no release wheel for build %q", version)
|
||||||
|
}
|
||||||
|
filename := fmt.Sprintf("scimesh-%s-py3-none-any.whl", NormalizePEP440(version))
|
||||||
|
return fmt.Sprintf("https://github.com/emil28092005/SciMesh/releases/download/v%s/%s", version, filename), filename, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizePEP440 turns our release tag suffixes into the PEP 440 form
|
||||||
|
// setuptools uses for wheel names: 1.1.0-alpha.10 -> 1.1.0a10,
|
||||||
|
// 1.1.0-beta.2 -> 1.1.0b2, 1.1.0-rc.1 -> 1.1.0rc1. Stable tags pass through.
|
||||||
|
func NormalizePEP440(version string) string {
|
||||||
|
for from, to := range map[string]string{"-alpha.": "a", "-beta.": "b", "-rc.": "rc"} {
|
||||||
|
version = strings.ReplaceAll(version, from, to)
|
||||||
|
}
|
||||||
|
return version
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadWheel fetches the release wheel into dir (config directory of the
|
||||||
|
// wizard / serve data dir) and returns the local path. Best-effort download
|
||||||
|
// with a generous timeout: wheels can be several MB.
|
||||||
|
func DownloadWheel(ctx context.Context, url, dir string) (string, error) {
|
||||||
|
target := filepath.Join(dir, wheelNameFromURL(url))
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("download wheel: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", fmt.Errorf("download wheel: HTTP %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
//nolint:gosec // G304: target is our own config dir + a fixed wheel name
|
||||||
|
out, err := os.Create(target)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("download wheel: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = out.Close() }()
|
||||||
|
if _, err := io.Copy(out, resp.Body); err != nil {
|
||||||
|
return "", fmt.Errorf("download wheel: %w", err)
|
||||||
|
}
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// wheelNameFromURL extracts the trailing file name of a wheel URL.
|
||||||
|
func wheelNameFromURL(url string) string {
|
||||||
|
return url[strings.LastIndex(url, "/")+1:]
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizePEP440(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"1.1.0": "1.1.0",
|
||||||
|
"1.1.0-alpha.10": "1.1.0a10",
|
||||||
|
"1.1.0-beta.2": "1.1.0b2",
|
||||||
|
"1.1.0-rc.1": "1.1.0rc1",
|
||||||
|
"1.0.0": "1.0.0",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := NormalizePEP440(in); got != want {
|
||||||
|
t.Errorf("NormalizePEP440(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReleaseWheelURL(t *testing.T) {
|
||||||
|
url, name, err := ReleaseWheelURL("1.1.0-alpha.10")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantURL := "https://github.com/emil28092005/SciMesh/releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl"
|
||||||
|
if url != wantURL {
|
||||||
|
t.Errorf("url = %q, want %q", url, wantURL)
|
||||||
|
}
|
||||||
|
if name != "scimesh-1.1.0a10-py3-none-any.whl" {
|
||||||
|
t.Errorf("name = %q", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A dev build has no release wheel.
|
||||||
|
if _, _, err := ReleaseWheelURL("dev"); err == nil {
|
||||||
|
t.Error("dev build must not resolve a release wheel")
|
||||||
|
}
|
||||||
|
if _, _, err := ReleaseWheelURL(""); err == nil {
|
||||||
|
t.Error("empty version must not resolve a release wheel")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadWheel(t *testing.T) {
|
||||||
|
payload := []byte("fake wheel bytes")
|
||||||
|
stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write(payload)
|
||||||
|
}))
|
||||||
|
defer stub.Close()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
path, err := DownloadWheel(context.Background(), stub.URL+"/scimesh-1.1.0a10-py3-none-any.whl", dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(path, "scimesh-1.1.0a10-py3-none-any.whl") {
|
||||||
|
t.Errorf("path = %q", path)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(got) != string(payload) {
|
||||||
|
t.Error("wheel bytes mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadWheelReportsHTTPErrors(t *testing.T) {
|
||||||
|
stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer stub.Close()
|
||||||
|
if _, err := DownloadWheel(context.Background(), stub.URL+"/missing.whl", t.TempDir()); err == nil {
|
||||||
|
t.Error("404 must fail the download")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -176,13 +176,15 @@ func (s *PIDSupervisor) Stop() error {
|
|||||||
|
|
||||||
// Server is the wizard HTTP server, bound to the loopback interface only.
|
// Server is the wizard HTTP server, bound to the loopback interface only.
|
||||||
type Server struct {
|
type Server struct {
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
cfgPath string
|
cfgPath string
|
||||||
logPath string
|
logPath string
|
||||||
dir string
|
dir string
|
||||||
sup Supervisor
|
sup Supervisor
|
||||||
openBrowser func(string)
|
openBrowser func(string)
|
||||||
port int
|
port int
|
||||||
|
install func(ctx context.Context, venvPython, pkg string) error
|
||||||
|
downloadWheel func(ctx context.Context, url, dir string) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Options customises the wizard for tests and embedding.
|
// Options customises the wizard for tests and embedding.
|
||||||
@@ -192,6 +194,12 @@ type Options struct {
|
|||||||
OpenBrowser func(url string)
|
OpenBrowser func(url string)
|
||||||
Supervisor Supervisor
|
Supervisor Supervisor
|
||||||
Dir string // directory for pid/log files; defaults to the config dir
|
Dir string // directory for pid/log files; defaults to the config dir
|
||||||
|
// InstallScimesh overrides the pip step of the runtime installer (tests
|
||||||
|
// substitute a fake); nil uses the real pip inside the managed venv.
|
||||||
|
InstallScimesh func(ctx context.Context, venvPython, pkg string) error
|
||||||
|
// DownloadWheel overrides the release-wheel download (tests substitute a
|
||||||
|
// fake); nil downloads from the GitHub release matching the agent version.
|
||||||
|
DownloadWheel func(ctx context.Context, url, dir string) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(log *slog.Logger, opts Options) *Server {
|
func New(log *slog.Logger, opts Options) *Server {
|
||||||
@@ -215,7 +223,15 @@ func New(log *slog.Logger, opts Options) *Server {
|
|||||||
if port == 0 {
|
if port == 0 {
|
||||||
port = defaultPort
|
port = defaultPort
|
||||||
}
|
}
|
||||||
return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port}
|
install := opts.InstallScimesh
|
||||||
|
if install == nil {
|
||||||
|
install = installScimeshWithPip
|
||||||
|
}
|
||||||
|
downloadWheel := opts.DownloadWheel
|
||||||
|
if downloadWheel == nil {
|
||||||
|
downloadWheel = agent.DownloadWheel
|
||||||
|
}
|
||||||
|
return &Server{log: log, cfgPath: cfgPath, logPath: filepath.Join(dir, logFileName), dir: dir, sup: sup, openBrowser: open, port: port, install: install, downloadWheel: downloadWheel}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Listen binds the loopback listener and returns it; Serve runs the server on
|
// Listen binds the loopback listener and returns it; Serve runs the server on
|
||||||
@@ -234,6 +250,7 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error {
|
|||||||
mux.HandleFunc("GET /api/status", s.handleStatus)
|
mux.HandleFunc("GET /api/status", s.handleStatus)
|
||||||
mux.HandleFunc("POST /api/config", s.handleSaveConfig)
|
mux.HandleFunc("POST /api/config", s.handleSaveConfig)
|
||||||
mux.HandleFunc("POST /api/test", s.handleTest)
|
mux.HandleFunc("POST /api/test", s.handleTest)
|
||||||
|
mux.HandleFunc("POST /api/runtime/install", s.handleInstallRuntime)
|
||||||
mux.HandleFunc("POST /api/start", s.handleStart)
|
mux.HandleFunc("POST /api/start", s.handleStart)
|
||||||
mux.HandleFunc("POST /api/stop", s.handleStop)
|
mux.HandleFunc("POST /api/stop", s.handleStop)
|
||||||
mux.HandleFunc("GET /api/logs", s.handleLogs)
|
mux.HandleFunc("GET /api/logs", s.handleLogs)
|
||||||
@@ -412,3 +429,112 @@ func (s *Server) handleLogs(w http.ResponseWriter, r *http.Request) {
|
|||||||
// ErrCanceled mirrors context.Canceled for callers that treat a cancelled
|
// ErrCanceled mirrors context.Canceled for callers that treat a cancelled
|
||||||
// wizard as a clean exit.
|
// wizard as a clean exit.
|
||||||
var ErrCanceled = errors.New("setup wizard cancelled")
|
var ErrCanceled = errors.New("setup wizard cancelled")
|
||||||
|
|
||||||
|
type installRuntimeRequest struct {
|
||||||
|
// ScimeshPackage overrides where the scimesh wheel comes from: a local
|
||||||
|
// wheel/index path or the PyPI name. Defaults to SCIMESH_PIP_PACKAGE, then
|
||||||
|
// to the PyPI name.
|
||||||
|
ScimeshPackage string `json:"scimesh_package"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type installRuntimeResponse struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Python string `json:"python,omitempty"` // venv python to use as TASK_RUNNER[0]
|
||||||
|
Installed bool `json:"installed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleInstallRuntime creates a managed venv next to the worker config and
|
||||||
|
// installs the scimesh package into it, so the machine needs no manual pip
|
||||||
|
// step. The venv python path is returned for the wizard to bake into the
|
||||||
|
// task runner.
|
||||||
|
func (s *Server) handleInstallRuntime(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req installRuntimeRequest
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
|
||||||
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON body"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pkg := strings.TrimSpace(req.ScimeshPackage)
|
||||||
|
if pkg == "" {
|
||||||
|
pkg = os.Getenv("SCIMESH_PIP_PACKAGE")
|
||||||
|
}
|
||||||
|
if pkg == "" {
|
||||||
|
// No PyPI default on purpose: the PyPI name "scimesh" belongs to an
|
||||||
|
// unrelated project. Instead we ship the wheel in our own GitHub
|
||||||
|
// release, version-locked to this binary, and download it from there.
|
||||||
|
url, _, err := agent.ReleaseWheelURL(agent.Version)
|
||||||
|
if err != nil {
|
||||||
|
writeJSON(w, http.StatusConflict, map[string]string{
|
||||||
|
"error": "no scimesh source configured: set SCIMESH_PIP_PACKAGE to your wheel, checkout or index, then retry",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
local, err := s.downloadWheel(r.Context(), url, s.dir)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Error("download release wheel", "err", err, "url", url)
|
||||||
|
writeJSON(w, http.StatusConflict, map[string]string{
|
||||||
|
"error": "could not download the scimesh wheel for this release: " + err.Error() + ". Set SCIMESH_PIP_PACKAGE to your wheel, checkout or index and retry.",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pkg = local
|
||||||
|
}
|
||||||
|
|
||||||
|
python3, err := exec.LookPath("python3")
|
||||||
|
if err != nil {
|
||||||
|
writeJSON(w, http.StatusConflict, map[string]string{"error": "python3 is not installed on this machine"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
venvDir := filepath.Join(s.dir, "venv")
|
||||||
|
venvPython := filepath.Join(venvDir, "bin", "python")
|
||||||
|
if _, err := os.Stat(venvPython); err != nil {
|
||||||
|
// Windows layout: Scripts/python.exe.
|
||||||
|
if win := filepath.Join(venvDir, "Scripts", "python.exe"); stat(win) {
|
||||||
|
venvPython = win
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(venvPython); err != nil {
|
||||||
|
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, python3, "-m", "venv", venvDir) //nolint:gosec // G204: python3 from LookPath, venvDir is our own dir
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
s.log.Error("create runtime venv", "err", err, "out", truncate(string(out), 500))
|
||||||
|
writeJSON(w, http.StatusConflict, map[string]string{"error": "could not create the python venv"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.install(r.Context(), venvPython, pkg); err != nil {
|
||||||
|
s.log.Error("install scimesh runtime", "err", err)
|
||||||
|
writeJSON(w, http.StatusConflict, map[string]string{
|
||||||
|
"error": "pip install " + pkg + " failed: " + err.Error() +
|
||||||
|
". Set SCIMESH_PIP_PACKAGE to your scimesh wheel or index and retry.",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, installRuntimeResponse{OK: true, Python: venvPython, Installed: true})
|
||||||
|
}
|
||||||
|
|
||||||
|
// installScimeshWithPip installs the package with the venv's own pip,
|
||||||
|
// streaming into the agent log so a long build is not silent.
|
||||||
|
func installScimeshWithPip(ctx context.Context, venvPython, pkg string) error {
|
||||||
|
pip := filepath.Join(filepath.Dir(venvPython), "pip")
|
||||||
|
if _, err := os.Stat(pip); err != nil {
|
||||||
|
pip += ".exe"
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, pip, "install", pkg) //nolint:gosec // G204: pip from our venv, pkg is operator-set or a fixed default
|
||||||
|
return cmd.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func stat(path string) bool {
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncate(s string, n int) string {
|
||||||
|
if len(s) <= n {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return s[:n] + "…"
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package setupui
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -23,17 +24,29 @@ func testLogger() *slog.Logger {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newTestServer(t *testing.T, sup Supervisor) (*Server, string) {
|
func newTestServer(t *testing.T, sup Supervisor) (*Server, string) {
|
||||||
|
t.Helper()
|
||||||
|
return newTestServerWithInstall(t, sup, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestServerWithInstall(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error) (*Server, string) {
|
||||||
|
t.Helper()
|
||||||
|
return newTestServerWithInstallAndWheel(t, sup, install, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestServerWithInstallAndWheel(t *testing.T, sup Supervisor, install func(ctx context.Context, venvPython, pkg string) error, wheel func(ctx context.Context, url, dir string) (string, error)) (*Server, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
server := New(testLogger(), Options{
|
server := New(testLogger(), Options{
|
||||||
// A distinct random port per test: Port 0 means "the default 12700" in
|
// A distinct random port per test: Port 0 means "the default 12700" in
|
||||||
// the server, which would let the shared http.Client pool reuse a stale
|
// the server, which would let the shared http.Client pool reuse a stale
|
||||||
// keep-alive connection across tests (EOF after a Shutdown).
|
// keep-alive connection across tests (EOF after a Shutdown).
|
||||||
Port: freePort(t),
|
Port: freePort(t),
|
||||||
ConfigPath: filepath.Join(dir, "config.json"),
|
ConfigPath: filepath.Join(dir, "config.json"),
|
||||||
Dir: dir,
|
Dir: dir,
|
||||||
Supervisor: sup,
|
Supervisor: sup,
|
||||||
OpenBrowser: func(string) {},
|
OpenBrowser: func(string) {},
|
||||||
|
InstallScimesh: install,
|
||||||
|
DownloadWheel: wheel,
|
||||||
})
|
})
|
||||||
listener, err := server.Listen()
|
listener, err := server.Listen()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -291,3 +304,108 @@ func TestConfigFileDefaultsAndEnvOverride(t *testing.T) {
|
|||||||
t.Errorf("cpu = %d", config.CPUCount)
|
t.Errorf("cpu = %d", config.CPUCount)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInstallRuntimeCreatesVenvAndReportsPython(t *testing.T) {
|
||||||
|
var installedPkg string
|
||||||
|
sup := &fakeSup{}
|
||||||
|
_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
|
||||||
|
installedPkg = pkg
|
||||||
|
// Prove the venv python path really exists by creating a marker file
|
||||||
|
// where the real venv python would be.
|
||||||
|
_ = os.MkdirAll(filepath.Dir(venvPython), 0o755)
|
||||||
|
_ = os.WriteFile(venvPython, []byte("#!/bin/sh\nexit 0\n"), 0o755)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{"scimesh_package": "/wheels/scimesh.whl"})
|
||||||
|
if rec.Code != http.StatusOK || data["ok"] != true {
|
||||||
|
t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
|
||||||
|
}
|
||||||
|
if installedPkg != "/wheels/scimesh.whl" {
|
||||||
|
t.Errorf("package = %q, want the requested wheel", installedPkg)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(data["python"].(string), "venv/bin/python") {
|
||||||
|
t.Errorf("python = %v, want the venv python", data["python"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstallRuntimeRequiresASource(t *testing.T) {
|
||||||
|
sup := &fakeSup{}
|
||||||
|
_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
|
||||||
|
t.Fatal("install must not run without a package source")
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
// No source anywhere (SCIMESH_PIP_PACKAGE unset, request empty): 409 with
|
||||||
|
// guidance. The PyPI name is another project, so no silent fallback.
|
||||||
|
rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
|
||||||
|
if rec.Code != http.StatusConflict {
|
||||||
|
t.Fatalf("install without source: got %d, want 409", rec.Code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
|
||||||
|
t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstallRuntimeFailureIsExplained(t *testing.T) {
|
||||||
|
sup := &fakeSup{}
|
||||||
|
_, base := newTestServerWithInstall(t, sup, func(ctx context.Context, venvPython, pkg string) error {
|
||||||
|
return errors.New("no matching distribution found")
|
||||||
|
})
|
||||||
|
rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{"scimesh_package": "/wheels/scimesh.whl"})
|
||||||
|
if rec.Code != http.StatusConflict {
|
||||||
|
t.Fatalf("install failure: got %d, want 409", rec.Code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
|
||||||
|
t.Errorf("error = %v, want a hint about SCIMESH_PIP_PACKAGE", data["error"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstallRuntimeDownloadsReleaseWheelWhenNoSource(t *testing.T) {
|
||||||
|
oldVersion := agent.Version
|
||||||
|
agent.Version = "1.1.0-alpha.10"
|
||||||
|
t.Cleanup(func() { agent.Version = oldVersion })
|
||||||
|
|
||||||
|
var downloadedURL, installedPkg string
|
||||||
|
sup := &fakeSup{}
|
||||||
|
_, base := newTestServerWithInstallAndWheel(t, sup,
|
||||||
|
func(ctx context.Context, venvPython, pkg string) error {
|
||||||
|
installedPkg = pkg
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
func(ctx context.Context, url, dir string) (string, error) {
|
||||||
|
downloadedURL = url
|
||||||
|
return filepath.Join(dir, "scimesh-1.1.0a10-py3-none-any.whl"), nil
|
||||||
|
})
|
||||||
|
|
||||||
|
rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
|
||||||
|
if rec.Code != http.StatusOK || data["ok"] != true {
|
||||||
|
t.Fatalf("install: got %d %v, want 200 ok", rec.Code, data)
|
||||||
|
}
|
||||||
|
if !strings.Contains(downloadedURL, "releases/download/v1.1.0-alpha.10/scimesh-1.1.0a10-py3-none-any.whl") {
|
||||||
|
t.Errorf("download url = %q, want the release wheel of this version", downloadedURL)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(installedPkg, "scimesh-1.1.0a10-py3-none-any.whl") {
|
||||||
|
t.Errorf("pip received %q, want the downloaded wheel", installedPkg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstallRuntimeWheelDownloadFailureIsExplained(t *testing.T) {
|
||||||
|
oldVersion := agent.Version
|
||||||
|
agent.Version = "1.1.0-alpha.10"
|
||||||
|
t.Cleanup(func() { agent.Version = oldVersion })
|
||||||
|
|
||||||
|
sup := &fakeSup{}
|
||||||
|
_, base := newTestServerWithInstallAndWheel(t, sup,
|
||||||
|
func(ctx context.Context, venvPython, pkg string) error { t.Fatal("pip must not run"); return nil },
|
||||||
|
func(ctx context.Context, url, dir string) (string, error) {
|
||||||
|
return "", errors.New("HTTP 404")
|
||||||
|
})
|
||||||
|
|
||||||
|
rec, data := postJSON(t, base, "/api/runtime/install", map[string]any{})
|
||||||
|
if rec.Code != http.StatusConflict {
|
||||||
|
t.Fatalf("got %d, want 409", rec.Code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(data["error"].(string), "SCIMESH_PIP_PACKAGE") {
|
||||||
|
t.Errorf("error = %v, want a SCIMESH_PIP_PACKAGE hint", data["error"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -183,7 +183,7 @@ code{font-family:var(--mono);font-size:.86em}
|
|||||||
|
|
||||||
<script>
|
<script>
|
||||||
const $=id=>document.getElementById(id);
|
const $=id=>document.getElementById(id);
|
||||||
let state={mode:'token',cpu:'auto'};
|
let state={mode:'token',cpu:'auto',venvPython:null};
|
||||||
let checksOk=false;
|
let checksOk=false;
|
||||||
|
|
||||||
function err(msg){$('error-box').innerHTML=msg?'<div class="error-strip">'+msg+'</div>':''}
|
function err(msg){$('error-box').innerHTML=msg?'<div class="error-strip">'+msg+'</div>':''}
|
||||||
@@ -215,7 +215,7 @@ async function postJSON(path,body){
|
|||||||
return {status:r.status,data};
|
return {status:r.status,data};
|
||||||
}
|
}
|
||||||
function draftConfig(){
|
function draftConfig(){
|
||||||
return {
|
const cfg={
|
||||||
coordinator_url:$('in-url').value.trim(),
|
coordinator_url:$('in-url').value.trim(),
|
||||||
token:state.mode==='token'?$('in-token').value.trim():'',
|
token:state.mode==='token'?$('in-token').value.trim():'',
|
||||||
worker_key:state.mode==='key'?$('in-key').value.trim():'',
|
worker_key:state.mode==='key'?$('in-key').value.trim():'',
|
||||||
@@ -224,6 +224,8 @@ function draftConfig(){
|
|||||||
worker_name:$('in-name').value.trim(),
|
worker_name:$('in-name').value.trim(),
|
||||||
cpu_count:state.cpu==='custom'?parseInt($('in-cpu').value||'1',10):0
|
cpu_count:state.cpu==='custom'?parseInt($('in-cpu').value||'1',10):0
|
||||||
};
|
};
|
||||||
|
if(state.venvPython)cfg.task_runner=[state.venvPython,'-m','scimesh.worker.task'];
|
||||||
|
return cfg;
|
||||||
}
|
}
|
||||||
|
|
||||||
$('b1').onclick=()=>{
|
$('b1').onclick=()=>{
|
||||||
|
|||||||
+2
-1
@@ -97,6 +97,7 @@ if ($Component -eq "coordinator") {
|
|||||||
Write-Host "For a coordinator started with 'coordinator serve', the worker token is"
|
Write-Host "For a coordinator started with 'coordinator serve', the worker token is"
|
||||||
Write-Host "in ~\.scimesh\worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
|
Write-Host "in ~\.scimesh\worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
|
||||||
Write-Host "install scimesh into a managed venv, or install it yourself:"
|
Write-Host "install scimesh into a managed venv, or install it yourself:"
|
||||||
Write-Host " pip install scimesh"
|
Write-Host " set SCIMESH_PIP_PACKAGE=<your wheel or index>"
|
||||||
|
Write-Host " $Target setup"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -112,5 +112,5 @@ else
|
|||||||
echo "For a coordinator started with 'coordinator serve', the worker token is"
|
echo "For a coordinator started with 'coordinator serve', the worker token is"
|
||||||
echo "in ~/.scimesh/worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
|
echo "in ~/.scimesh/worker.token on that machine. Set SCIMESH_PIP_PACKAGE to"
|
||||||
echo "install scimesh into a managed venv, or install it yourself:"
|
echo "install scimesh into a managed venv, or install it yourself:"
|
||||||
echo " pip install scimesh"
|
echo " SCIMESH_PIP_PACKAGE=<your wheel or index> worker-agent setup"
|
||||||
fi
|
fi
|
||||||
|
|||||||
+6
-4
@@ -113,10 +113,12 @@ chmod +x coordinator
|
|||||||
```
|
```
|
||||||
|
|
||||||
It spawns `python -m scimesh.worker.task`, so the machine needs Python 3
|
It spawns `python -m scimesh.worker.task`, so the machine needs Python 3
|
||||||
with the `scimesh` package (`pip install scimesh`, or let the managed venv
|
with the `scimesh` package. The wizard installs it into its own venv; the
|
||||||
do it via `SCIMESH_PIP_PACKAGE`). For a `coordinator serve` instance, the
|
package must come from your wheel, checkout or index — point
|
||||||
worker token is in `~/.scimesh/worker.token`. On Windows set
|
`SCIMESH_PIP_PACKAGE` at it (the PyPI name `scimesh` belongs to an unrelated
|
||||||
`SCIMESH_COMPONENT=worker` for `install.ps1`.
|
project). For a `coordinator serve` instance, the worker token is in
|
||||||
|
`~/.scimesh/worker.token`. On Windows set `SCIMESH_COMPONENT=worker` for
|
||||||
|
`install.ps1`.
|
||||||
- **coordinator** needs no external services at all in its default mode:
|
- **coordinator** needs no external services at all in its default mode:
|
||||||
`coordinator serve` embeds SQLite (both databases), the userservice, and
|
`coordinator serve` embeds SQLite (both databases), the userservice, and
|
||||||
local workers. The `SCIMESH_DB=postgres` engine remains for cluster
|
local workers. The `SCIMESH_DB=postgres` engine remains for cluster
|
||||||
|
|||||||
Reference in New Issue
Block a user