feat(coordinator): business metrics — jobs/tasks/workers by status

A scrape-time collector reports scimesh_tasks/jobs/workers gauges keyed by
status, sourced from cheap GROUP BY queries (StatsRepo), zero-filled across all
known statuses so the dashboard shows flat zeros instead of gaps. A failed
query yields no samples for that scrape rather than crashing it.

Metrics is now built in main so the DB-backed collector can be registered
(NewServer takes *metrics.Metrics; nil self-provisions for tests). Grafana
dashboard gains a Domain state row: tasks/jobs/workers by status and a queue-
depth stat.

coordinator/internal/metrics/business.go

@@ -0,0 +1,66 @@
+package metrics
+
+import (
+	"context"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Stats is a point-in-time snapshot of the coordinator's domain state: counts of
+// tasks, jobs, and workers keyed by their status. Maps are expected to be
+// zero-filled by the provider so every known status is always present, giving
+// the dashboard flat zero lines instead of gaps.
+type Stats struct {
+	Tasks   map[string]int
+	Jobs    map[string]int
+	Workers map[string]int
+}
+
+// StatsFunc returns the current snapshot. It is called on every scrape, so it
+// must be a cheap aggregate query.
+type StatsFunc func(context.Context) (Stats, error)
+
+// RegisterBusiness registers a collector that reports domain-state gauges
+// (scimesh_tasks/jobs/workers by status) sourced from collect on each scrape.
+// Deriving the gauges at scrape time keeps them fresh without a background
+// goroutine, and a failed query simply yields no samples for that scrape.
+func (m *Metrics) RegisterBusiness(collect StatsFunc) {
+	m.reg.MustRegister(&businessCollector{
+		collect: collect,
+		tasks:   prometheus.NewDesc("scimesh_tasks", "Tasks by status.", []string{"status"}, nil),
+		jobs:    prometheus.NewDesc("scimesh_jobs", "Jobs by status.", []string{"status"}, nil),
+		workers: prometheus.NewDesc("scimesh_workers", "Workers by status.", []string{"status"}, nil),
+	})
+}
+
+type businessCollector struct {
+	collect              StatsFunc
+	tasks, jobs, workers *prometheus.Desc
+}
+
+func (c *businessCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.tasks
+	ch <- c.jobs
+	ch <- c.workers
+}
+
+func (c *businessCollector) Collect(ch chan<- prometheus.Metric) {
+	// A bounded query so one slow scrape cannot stall Prometheus.
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	s, err := c.collect(ctx)
+	if err != nil {
+		return // no samples this scrape; Prometheus keeps the last value
+	}
+	emit(ch, c.tasks, s.Tasks)
+	emit(ch, c.jobs, s.Jobs)
+	emit(ch, c.workers, s.Workers)
+}
+
+func emit(ch chan<- prometheus.Metric, desc *prometheus.Desc, counts map[string]int) {
+	for status, n := range counts {
+		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(n), status)
+	}
+}

coordinator/internal/metrics/business_test.go

@@ -0,0 +1,51 @@
+package metrics
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func scrape(t *testing.T, m *Metrics) string {
+	t.Helper()
+	rec := httptest.NewRecorder()
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/metrics", nil)
+	m.Handler().ServeHTTP(rec, req)
+	return rec.Body.String()
+}
+
+func TestBusinessCollectorEmitsGauges(t *testing.T) {
+	m := New()
+	m.RegisterBusiness(func(context.Context) (Stats, error) {
+		return Stats{
+			Tasks:   map[string]int{"pending": 3, "running": 1, "completed": 0},
+			Jobs:    map[string]int{"running": 2},
+			Workers: map[string]int{"online": 4},
+		}, nil
+	})
+
+	body := scrape(t, m)
+	for _, want := range []string{
+		`scimesh_tasks{status="pending"} 3`,
+		`scimesh_tasks{status="completed"} 0`,
+		`scimesh_jobs{status="running"} 2`,
+		`scimesh_workers{status="online"} 4`,
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("metrics missing %q\n%s", want, body)
+		}
+	}
+}
+
+func TestBusinessCollectorSkipsOnError(t *testing.T) {
+	m := New()
+	m.RegisterBusiness(func(context.Context) (Stats, error) {
+		return Stats{}, errors.New("db down")
+	})
+	if strings.Contains(scrape(t, m), "scimesh_tasks") {
+		t.Error("a failed snapshot must emit no business samples")
+	}
+}