diff --git a/README.md b/README.md index 431808e..038ee4e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ curl -fsSL https://raw.githubusercontent.com/emil28092005/SciMesh/main/install.s powershell -ExecutionPolicy Bypass -c "irm https://raw.githubusercontent.com/emil28092005/SciMesh/main/install.ps1 | iex" ``` -Set `SCIMESH_AUTO_START=0` to install without starting anything. A standalone +Set `SCIMESH_AUTO_START=0` to install without starting anything. The old demo +control room was removed: `/ui` is the admin console. A standalone worker is installed the same way (`bash -s worker`, or `SCIMESH_COMPONENT=worker` on Windows); its installer opens the local setup wizard (`worker-agent setup`) in the browser automatically. @@ -73,12 +74,12 @@ PostgreSQL, no Docker, no environment variables. The scientific runtime is a managed venv (`~/.scimesh/venv`); point `SCIMESH_PIP_PACKAGE` at your scimesh wheel to install it automatically. -The coordinator serves two operator surfaces: the **control room** (jobs, -workloads, docs) and the **admin console** at `/ui/admin` — cluster health +The coordinator's UI is the **admin console** at `/ui/admin` — cluster health and storage, paginated job table, worker fleet with trust controls, users and -worker keys, workload enable/disable, metrics and the worker token -(`serve --open` lands on the admin console; login returns you to the page you -asked for). The **worker** binary (`worker-agent`) carries its own local setup +worker keys, workload enable/disable, metrics and the worker token. The job +form (`/ui/jobs/new`), job detail pages and the workload library complete the +operator surface; `/ui` redirects to the console, `serve --open` lands on it, +and login returns you to the page you asked for. The **worker** binary (`worker-agent`) carries its own local setup wizard for machines that run only a worker: `worker-agent setup` opens a browser wizard at `127.0.0.1` that collects the coordinator URL and credential, runs a preflight check, saves `~/.scimesh-worker/config.json` and diff --git a/coordinator/internal/storage/postgres/integration_test.go b/coordinator/internal/storage/postgres/integration_test.go index 5caa8d4..c9b6fe4 100644 --- a/coordinator/internal/storage/postgres/integration_test.go +++ b/coordinator/internal/storage/postgres/integration_test.go @@ -145,20 +145,20 @@ func TestUIReadRepoListsReducerFields(t *testing.T) { if claimed, err := jobs.ClaimReduction(ctx, job.ID, time.Now().UTC()); err != nil || !claimed { t.Fatalf("claim reduction = (%v, %v)", claimed, err) } - listed, err := NewUIReadRepo(pool).ListJobs(ctx, nil, 20) + listed, _, err := NewAdminReadRepo(pool).ListJobsPaginated(ctx, "", 20, 0) if err != nil { - t.Fatalf("list UI jobs: %v", err) + t.Fatalf("list admin jobs: %v", err) } for _, item := range listed { if item.ID != job.ID { continue } if item.Status != domain.JobReducing || item.ReducerStartedAt == nil { - t.Fatalf("UI reducer projection = %+v", item) + t.Fatalf("admin reducer projection = %+v", item) } return } - t.Fatalf("seeded job %s is missing from UI list", job.ID) + t.Fatalf("seeded job %s is missing from the admin list", job.ID) } // A job must land whole or not at all: a half-created job leaves chunks no diff --git a/coordinator/internal/storage/postgres/ui_read_repo.go b/coordinator/internal/storage/postgres/ui_read_repo.go index 1382ee4..95cec83 100644 --- a/coordinator/internal/storage/postgres/ui_read_repo.go +++ b/coordinator/internal/storage/postgres/ui_read_repo.go @@ -24,40 +24,6 @@ func (r *UIReadRepo) GetJob(ctx context.Context, id uuid.UUID) (*domain.Job, err return job, err } -func (r *UIReadRepo) ListJobs(ctx context.Context, owner *uuid.UUID, limit int) ([]domain.Job, error) { - if limit < 1 || limit > 100 { - return nil, domain.ErrInvalidInput - } - q := psql.Select(jobColumns...).From("jobs") - if owner != nil { - q = q.Where(sq.Eq{"owner_id": *owner}) - } - sql, args, err := q.OrderBy("created_at DESC", "id DESC").Limit(uint64(limit)).ToSql() - if err != nil { - return nil, err - } - rows, err := conn(ctx, r.pool).Query(ctx, sql, args...) - if err != nil { - return nil, fmt.Errorf("list jobs: %w", err) - } - defer rows.Close() - jobs := make([]domain.Job, 0) - for rows.Next() { - var j domain.Job - var status string - if err := rows.Scan( - &j.ID, &j.Workload, &j.InputURI, &j.Parameters, &status, &j.CreatedAt, &j.CompletedAt, - &j.InputArtifactID, &j.ResultArtifactID, &j.ErrorCode, &j.ErrorMessage, &j.ReducerStartedAt, - &j.OwnerID, - ); err != nil { - return nil, err - } - j.Status = domain.JobStatus(status) - jobs = append(jobs, j) - } - return jobs, rows.Err() -} - func (r *UIReadRepo) ListTasksByJob(ctx context.Context, jobID uuid.UUID) ([]domain.Task, error) { sql, args, err := psql.Select(taskColumns...).From("tasks").Where(sq.Eq{"job_id": jobID}).OrderBy("chunk_index ASC").ToSql() if err != nil { @@ -79,68 +45,18 @@ func (r *UIReadRepo) ListTasksByJob(ctx context.Context, jobID uuid.UUID) ([]dom return tasks, rows.Err() } -func (r *UIReadRepo) ListTasksByJobs(ctx context.Context, jobIDs []uuid.UUID) (map[uuid.UUID][]domain.Task, error) { - out := make(map[uuid.UUID][]domain.Task, len(jobIDs)) - if len(jobIDs) == 0 { - return out, nil - } - sql, args, err := psql.Select(taskColumns...).From("tasks"). - Where(sq.Eq{"job_id": jobIDs}).OrderBy("job_id ASC", "chunk_index ASC").ToSql() - if err != nil { - return nil, err - } - rows, err := conn(ctx, r.pool).Query(ctx, sql, args...) - if err != nil { - return nil, fmt.Errorf("list tasks by jobs: %w", err) - } - defer rows.Close() - for rows.Next() { - task, err := scanTask(rows) - if err != nil { - return nil, err - } - out[task.JobID] = append(out[task.JobID], *task) - } - return out, rows.Err() -} - func (r *UIReadRepo) ListWorkers(ctx context.Context, limit int) ([]domain.Worker, error) { - if limit < 1 || limit > 100 { - return nil, domain.ErrInvalidInput - } - sql, args, err := psql.Select(workerColumns...).From("workers").OrderBy("last_heartbeat_at DESC", "id DESC").Limit(uint64(limit)).ToSql() - if err != nil { - return nil, err - } - rows, err := conn(ctx, r.pool).Query(ctx, sql, args...) - if err != nil { - return nil, fmt.Errorf("list workers: %w", err) - } - defer rows.Close() - workers := make([]domain.Worker, 0) - for rows.Next() { - worker, err := scanWorker(rows) - if err != nil { - return nil, err - } - workers = append(workers, *worker) - } - return workers, rows.Err() -} - -func (r *UIReadRepo) ListWorkersByOwner(ctx context.Context, owner uuid.UUID, limit int) ([]domain.Worker, error) { if limit < 1 || limit > 100 { return nil, domain.ErrInvalidInput } sql, args, err := psql.Select(workerColumns...).From("workers"). - Where(sq.Eq{"owner_id": owner}). OrderBy("last_heartbeat_at DESC", "id DESC").Limit(uint64(limit)).ToSql() if err != nil { return nil, err } rows, err := conn(ctx, r.pool).Query(ctx, sql, args...) if err != nil { - return nil, fmt.Errorf("list workers by owner: %w", err) + return nil, fmt.Errorf("list workers: %w", err) } defer rows.Close() workers := make([]domain.Worker, 0) diff --git a/coordinator/internal/storage/sqlite/ui_read_repo.go b/coordinator/internal/storage/sqlite/ui_read_repo.go index 5806321..a92448a 100644 --- a/coordinator/internal/storage/sqlite/ui_read_repo.go +++ b/coordinator/internal/storage/sqlite/ui_read_repo.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "strings" "github.com/google/uuid" @@ -20,34 +19,6 @@ func (r *UIReadRepo) GetJob(ctx context.Context, id uuid.UUID) (*domain.Job, err return NewJobRepo(r.db).Get(ctx, id) } -func (r *UIReadRepo) ListJobs(ctx context.Context, owner *uuid.UUID, limit int) ([]domain.Job, error) { - if limit < 1 || limit > 100 { - return nil, domain.ErrInvalidInput - } - query := "SELECT " + jobColumns + " FROM jobs" - args := []any{} - if owner != nil { - query += " WHERE owner_id = ?" - args = append(args, owner.String()) - } - query += " ORDER BY created_at DESC, id DESC LIMIT ?" - args = append(args, limit) - rows, err := conn(ctx, r.db).QueryContext(ctx, query, args...) - if err != nil { - return nil, fmt.Errorf("list jobs: %w", err) - } - defer func() { _ = rows.Close() }() - jobs := make([]domain.Job, 0) - for rows.Next() { - job, err := scanJob(rows) - if err != nil { - return nil, err - } - jobs = append(jobs, *job) - } - return jobs, rows.Err() -} - func (r *UIReadRepo) ListTasksByJob(ctx context.Context, jobID uuid.UUID) ([]domain.Task, error) { rows, err := conn(ctx, r.db).QueryContext(ctx, "SELECT "+taskColumns+" FROM tasks WHERE job_id = ? ORDER BY chunk_index ASC", jobID.String()) @@ -66,50 +37,12 @@ func (r *UIReadRepo) ListTasksByJob(ctx context.Context, jobID uuid.UUID) ([]dom return tasks, rows.Err() } -func (r *UIReadRepo) ListTasksByJobs(ctx context.Context, jobIDs []uuid.UUID) (map[uuid.UUID][]domain.Task, error) { - out := make(map[uuid.UUID][]domain.Task, len(jobIDs)) - if len(jobIDs) == 0 { - return out, nil - } - placeholders := make([]string, 0, len(jobIDs)) - args := make([]any, 0, len(jobIDs)) - for _, id := range jobIDs { - placeholders = append(placeholders, "?") - args = append(args, id.String()) - } - rows, err := conn(ctx, r.db).QueryContext(ctx, - "SELECT "+taskColumns+" FROM tasks WHERE job_id IN ("+strings.Join(placeholders, ", ")+") ORDER BY job_id ASC, chunk_index ASC", - args...) - if err != nil { - return nil, fmt.Errorf("list tasks by jobs: %w", err) - } - defer func() { _ = rows.Close() }() - for rows.Next() { - task, err := scanTask(rows) - if err != nil { - return nil, err - } - out[task.JobID] = append(out[task.JobID], *task) - } - return out, rows.Err() -} - func (r *UIReadRepo) ListWorkers(ctx context.Context, limit int) ([]domain.Worker, error) { - return r.listWorkers(ctx, "", nil, limit) -} - -func (r *UIReadRepo) ListWorkersByOwner(ctx context.Context, owner uuid.UUID, limit int) ([]domain.Worker, error) { - return r.listWorkers(ctx, " WHERE owner_id = ?", []any{owner.String()}, limit) -} - -func (r *UIReadRepo) listWorkers(ctx context.Context, clause string, args []any, limit int) ([]domain.Worker, error) { if limit < 1 || limit > 100 { return nil, domain.ErrInvalidInput } - query := "SELECT " + workerColumns + " FROM workers" + clause + - " ORDER BY last_heartbeat_at DESC, id DESC LIMIT ?" - fullArgs := append(args, limit) - rows, err := conn(ctx, r.db).QueryContext(ctx, query, fullArgs...) + rows, err := conn(ctx, r.db).QueryContext(ctx, + "SELECT "+workerColumns+" FROM workers ORDER BY last_heartbeat_at DESC, id DESC LIMIT ?", limit) if err != nil { return nil, fmt.Errorf("list workers: %w", err) } diff --git a/coordinator/internal/transport/http/server.go b/coordinator/internal/transport/http/server.go index b919846..2f2d51e 100644 --- a/coordinator/internal/transport/http/server.go +++ b/coordinator/internal/transport/http/server.go @@ -147,7 +147,6 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler { {"GET /ui/docs", s.handleUIDocsIndex}, {"GET /ui/docs/{path...}", s.handleUIDocs}, {"GET /ui/jobs/{job_id}", s.handleUIJob}, - {"GET /ui/api/overview", s.handleUIOverviewJSON}, {"GET /ui/api/jobs/{job_id}", s.handleUIJobJSON}, {"POST /ui/api/jobs/{job_id}/cancel", s.handleCancelJob}, {"POST /ui/api/jobs/upload", s.handleUploadDataset}, diff --git a/coordinator/internal/transport/http/server_test.go b/coordinator/internal/transport/http/server_test.go index 3c401b6..ac6d077 100644 --- a/coordinator/internal/transport/http/server_test.go +++ b/coordinator/internal/transport/http/server_test.go @@ -145,42 +145,17 @@ func TestUIRequiresDistinctCredentialAndRendersDashboard(t *testing.T) { req = request() req.SetBasicAuth("operator", uiToken) - resp, err = http.DefaultClient.Do(req) + // Do not follow the redirect: we assert it, not its target. + client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err = client.Do(req) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("UI status: %d", resp.StatusCode) - } - body, _ := io.ReadAll(resp.Body) - if !strings.Contains(string(body), "SciMesh control room") { - t.Errorf("dashboard body missing title") - } -} - -func TestUIOverviewReturnsLiveSafeProjection(t *testing.T) { - e := newEnv(t, healthy) - code, _ := e.do(t, "POST", "/jobs", `{"workload":"w","input_uri":"s3://in","chunks":[{"chunk_index":0,"input_uri":"s3://c","input_sha256":"sha"}]}`) - if code != http.StatusCreated { - t.Fatalf("create job: %d", code) - } - req, _ := http.NewRequestWithContext(context.Background(), "GET", e.ts.URL+"/ui/api/overview", nil) - req.SetBasicAuth("operator", uiToken) - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - var overview map[string]any - if err := json.NewDecoder(resp.Body).Decode(&overview); err != nil { - t.Fatal(err) - } - if resp.StatusCode != http.StatusOK || overview["active_jobs"].(float64) != 1 || overview["online_workers"].(float64) != 1 { - t.Fatalf("overview = (%d, %v)", resp.StatusCode, overview) - } - if _, leaked := overview["worker_auth_token"]; leaked { - t.Fatal("overview must not expose authentication configuration") + // In basic-auth mode (no userservice) /ui lands on the job form: the admin + // console exists only in session mode. + if resp.StatusCode != http.StatusSeeOther || resp.Header.Get("Location") != "/ui/jobs/new" { + t.Fatalf("UI status: %d -> %s, want 303 -> /ui/jobs/new", resp.StatusCode, resp.Header.Get("Location")) } } diff --git a/coordinator/internal/transport/http/templates/add-worker.html b/coordinator/internal/transport/http/templates/add-worker.html index 9ec7be5..fe57d04 100644 --- a/coordinator/internal/transport/http/templates/add-worker.html +++ b/coordinator/internal/transport/http/templates/add-worker.html @@ -11,7 +11,7 @@
- ← Back to control room

Contribute compute

Turn this computer into a worker

Create a key, install the worker, and run one command. The worker binds to your account and pulls tasks whenever it is online.

+ ← Back to control room

Contribute compute

Turn this computer into a worker

Create a key, install the worker, and run one command. The worker binds to your account and pulls tasks whenever it is online.

Your worker keys

@@ -29,7 +29,7 @@

Set it up

  1. Create a key
    Use the form; copy the command it generates.
    2. -
  2. Paste it in a terminal
    The command installs the worker, points it at this coordinator, and starts it. The machine then appears under My machines.
    3. +
  3. Paste it in a terminal
    The command installs the worker, points it at this coordinator, and starts it. The machine then appears under Workers.

Single-binary mode?

If the coordinator runs as coordinator serve, skip the key: diff --git a/coordinator/internal/transport/http/templates/admin.html b/coordinator/internal/transport/http/templates/admin.html index e920f57..9f2dfb4 100644 --- a/coordinator/internal/transport/http/templates/admin.html +++ b/coordinator/internal/transport/http/templates/admin.html @@ -143,14 +143,14 @@ tbody tr:hover{background:var(--panel-2)}

{{.Role}}
Signed in as {{.Role}}cluster administrator
- ← Back to control room + ＋ New computation

System

Cluster state and node information

-
admin console
+
＋ New computation
admin console
@@ -198,7 +198,7 @@ tbody tr:hover{background:var(--panel-2)}
- +
JobWorkloadOwnerStatusProgressSubmitted
JobWorkloadOwnerStatusProgressSubmitted
—
@@ -380,7 +380,8 @@ async function loadJobs(){ ''+esc(j.owner)+''+ ''+pill(statusLabel[j.status]||j.status,statusClass[j.status]||'pill-waiting',null)+''+ '
'+j.completed+' / '+j.total+' shards'+(j.failed?' · '+j.failed+' failed':'')+'
'+ - ''+fmtTime(j.created_at)+''; + ''+fmtTime(j.created_at)+''+ + 'Open'; rows.append(tr); } const from=(v.page-1)*v.per_page+1,to=Math.min(v.page*v.per_page,v.total); diff --git a/coordinator/internal/transport/http/templates/dashboard.html b/coordinator/internal/transport/http/templates/dashboard.html deleted file mode 100644 index ffce034..0000000 --- a/coordinator/internal/transport/http/templates/dashboard.html +++ /dev/null @@ -1,42 +0,0 @@ -{{define "dashboard.html"}} - - - - - - SciMesh control room - - - -
-
-

Local scientific compute

SciMesh control room

Follow the real path from a molecular TSV to a globally reduced similarity result—without reading coordinator logs.

Live overview · refreshes every 2 seconds
-
{{if .Session}}Signed in · {{.Session.Role}}{{end}}{{if .Session}}Profile{{end}}{{if and .Session (eq .Session.Role "admin")}}Admin{{end}}{{if .Session}}Workloads{{end}}{{if .Session}}Docs{{end}}{{if .Session}}🖥 Add your machine{{end}}＋ New computation{{if .Session}}
{{end}}
-
-
-
How a search becomes a result
01Upload TSVThe coordinator validates and slices the dataset.
02Run shardsWorkers fingerprint molecules and return shard top-k CSVs.
03Merge exactlyThe coordinator ranks retained candidates deterministically.
04Download CSVA checksum-protected global result is ready.
-
Active runs{{.ActiveJobs}}waiting, running, or merging
-
Available workers{{.OnlineWorkers}}recently registered
-
Finished runs{{.FinishedJobs}}in the latest 20
-
- -

Recent computations

{{len .Jobs}} shown · newest first

{{range .Jobs}}
{{workloadLabel .Workload}}
{{.ID}}
{{statusLabel .Status}}
{{statusHint .Status}}
{{.Completed}} / {{.Total}} shards complete{{if gt .Failed 0}} · {{.Failed}} failed{{end}}
{{else}}
No computations yet.
Start a computation, then keep one or more workers running to watch this dashboard come alive.
{{end}}
- {{if and .Session (ne .Session.Role "admin")}}

My machines

Workers you registered. Add your machine →

{{range .MyWorkers}}
{{.Name}}{{workerStatusLabel .Status}}
{{.ID}}

{{range .Capabilities}}{{.}}{{end}}

Last signal · {{time .LastHeartbeatAt}}

{{else}}
No machine of yours is connected.
Turn this computer into a worker →
{{end}}
{{end}} -

Worker fleet

Workers register themselves; this page never controls their processes.

{{range .Workers}}
{{.Name}}{{workerStatusLabel .Status}}
{{.ID}}

{{range .Capabilities}}{{.}}{{end}}

Last signal · {{time .LastHeartbeatAt}}

{{else}}
No worker is registered.
Start scimesh-worker in another terminal, then return here.
{{end}}
-
- - - -{{end}} diff --git a/coordinator/internal/transport/http/templates/docs-unavailable.html b/coordinator/internal/transport/http/templates/docs-unavailable.html index 2cbcf14..f772390 100644 --- a/coordinator/internal/transport/http/templates/docs-unavailable.html +++ b/coordinator/internal/transport/http/templates/docs-unavailable.html @@ -16,7 +16,7 @@

The documentation site has not been built or the coordinator has not been pointed at it. From the repository root, run:

make docs  then restart the coordinator with SCIMESH_DOCS_DIR set to the generated site/ directory (the make demo-ui demo does this automatically).

- ← Back to the control room + ← Back to the control room
diff --git a/coordinator/internal/transport/http/templates/job.html b/coordinator/internal/transport/http/templates/job.html index 0fa8218..f4a1229 100644 --- a/coordinator/internal/transport/http/templates/job.html +++ b/coordinator/internal/transport/http/templates/job.html @@ -12,7 +12,7 @@
-
← Back to control room{{if .Session}}
Profile
{{end}}
+
← Back to control room{{if .Session}}
Profile
{{end}}

{{workloadLabel .Workload}}

Live pipeline

One job, shown from accepted input through its final coordinator-owned scientific result.

Live · refreshes every 2 seconds
{{statusLabel .Status}}

{{statusHint .Status}}

Completed shards are preserved.

{{.Completed}} of {{.Total}} shards complete

{{.Total}}total shards
{{.Completed}}completed
{{.Pending}}waiting
{{add .Leased .Running}}with workers
{{.Failed}}failed
{{.Cancelled}}stopped
diff --git a/coordinator/internal/transport/http/templates/new-job.html b/coordinator/internal/transport/http/templates/new-job.html index da7c43e..51795bc 100644 --- a/coordinator/internal/transport/http/templates/new-job.html +++ b/coordinator/internal/transport/http/templates/new-job.html @@ -11,7 +11,7 @@
- ← Back to control room

New computation

Any workload, end to end

Choose a workload from the installed library. Upload its dataset; workers compute shards; the coordinator reduces the partials into one final artifact.

+ ← Back to control room

New computation

Any workload, end to end

Choose a workload from the installed library. Upload its dataset; workers compute shards; the coordinator reduces the partials into one final artifact.

Smaller shards make more visible tasks.

Only the first N data rows become shards; the upload stays stored.

A delimited table with a header row. The workload defines the required columns.

Ready to plan a run.
Select a dataset to see the file that will be sent to the coordinator.