From e584cfc48110c331621600e8d47c1a4b9996624b Mon Sep 17 00:00:00 2001 From: Efremenko Arhip Date: Sun, 26 Jul 2026 21:47:27 +0300 Subject: [PATCH] feat(coordinator): Prometheus /metrics endpoint Adds internal/metrics: a private registry with the Go runtime + process collectors and HTTP RED instrumentation (scimesh_http_requests_total and request_duration_seconds), labelled by method/status and a normalized route so per-id paths collapse to {id} and never blow up label cardinality. /metrics is unauthenticated (like /health) for a Prometheus scraper; the middleware wraps the whole router so every request is measured once. --- coordinator/go.mod | 17 ++- coordinator/go.sum | 44 +++++-- coordinator/internal/metrics/metrics.go | 112 ++++++++++++++++++ coordinator/internal/metrics/metrics_test.go | 47 ++++++++ coordinator/internal/transport/http/server.go | 9 +- 5 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 coordinator/internal/metrics/metrics.go create mode 100644 coordinator/internal/metrics/metrics_test.go diff --git a/coordinator/go.mod b/coordinator/go.mod index 3b6d8e7..9d805cc 100644 --- a/coordinator/go.mod +++ b/coordinator/go.mod @@ -1,24 +1,33 @@ module github.com/emil28092005/SciMesh/coordinator -go 1.22 +go 1.25.0 require ( github.com/Masterminds/squirrel v1.5.4 github.com/cenkalti/backoff/v4 v4.3.0 - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.6.0 github.com/joho/godotenv v1.5.1 + github.com/prometheus/client_golang v1.24.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect golang.org/x/crypto v0.17.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/coordinator/go.sum b/coordinator/go.sum index 8eea60b..198473a 100644 --- a/coordinator/go.sum +++ b/coordinator/go.sum @@ -1,12 +1,18 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -19,24 +25,46 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= diff --git a/coordinator/internal/metrics/metrics.go b/coordinator/internal/metrics/metrics.go new file mode 100644 index 0000000..1070e72 --- /dev/null +++ b/coordinator/internal/metrics/metrics.go @@ -0,0 +1,112 @@ +// Package metrics exposes Prometheus instrumentation for the coordinator: an +// HTTP RED middleware (rate, errors, duration) plus the standard Go runtime and +// process collectors, all on a private registry so nothing leaks in from global +// state. +package metrics + +import ( + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type Metrics struct { + reg *prometheus.Registry + requests *prometheus.CounterVec + duration *prometheus.HistogramVec +} + +// New builds the registry and registers the runtime, process, and HTTP metrics. +func New() *Metrics { + reg := prometheus.NewRegistry() + reg.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + + requests := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "scimesh", + Subsystem: "http", + Name: "requests_total", + Help: "HTTP requests, labelled by method, normalized route, and status.", + }, []string{"method", "route", "status"}) + + duration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "scimesh", + Subsystem: "http", + Name: "request_duration_seconds", + Help: "HTTP request duration in seconds.", + Buckets: prometheus.DefBuckets, + }, []string{"method", "route"}) + + reg.MustRegister(requests, duration) + return &Metrics{reg: reg, requests: requests, duration: duration} +} + +// Handler serves the metrics in Prometheus text format. +func (m *Metrics) Handler() http.Handler { + return promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{}) +} + +// Registry exposes the registry so callers can register extra collectors. +func (m *Metrics) Registry() *prometheus.Registry { return m.reg } + +// Middleware records one request into the RED metrics. It normalizes the path +// so per-id routes collapse to a single low-cardinality label. +func (m *Metrics) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(rec, r) + + route := normalizeRoute(r.URL.Path) + m.requests.WithLabelValues(r.Method, route, strconv.Itoa(rec.status)).Inc() + m.duration.WithLabelValues(r.Method, route).Observe(time.Since(start).Seconds()) + }) +} + +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (s *statusRecorder) WriteHeader(code int) { + s.status = code + s.ResponseWriter.WriteHeader(code) +} + +var uuidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// normalizeRoute collapses uuid and numeric path segments to {id}, keeping the +// route label cardinality bounded (otherwise every job/task id would be its own +// time series). +func normalizeRoute(path string) string { + if path == "" { + return "/" + } + segs := strings.Split(path, "/") + for i, s := range segs { + if s == "" { + continue + } + if uuidRe.MatchString(s) || isAllDigits(s) { + segs[i] = "{id}" + } + } + return strings.Join(segs, "/") +} + +func isAllDigits(s string) bool { + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return s != "" +} diff --git a/coordinator/internal/metrics/metrics_test.go b/coordinator/internal/metrics/metrics_test.go new file mode 100644 index 0000000..75729ee --- /dev/null +++ b/coordinator/internal/metrics/metrics_test.go @@ -0,0 +1,47 @@ +package metrics + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNormalizeRoute(t *testing.T) { + cases := map[string]string{ + "/health": "/health", + "/jobs/3f2504e0-4f89-41d3-9a0c-0305e82c3301": "/jobs/{id}", + "/tasks/3f2504e0-4f89-41d3-9a0c-0305e82c3301/result": "/tasks/{id}/result", + "/ui/jobs/12345": "/ui/jobs/{id}", + "/": "/", + } + for in, want := range cases { + if got := normalizeRoute(in); got != want { + t.Errorf("normalizeRoute(%q) = %q, want %q", in, got, want) + } + } +} + +func TestMiddlewareAndHandler(t *testing.T) { + m := New() + h := m.Middleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + })) + + req, _ := http.NewRequestWithContext(context.Background(), http.MethodPost, "/jobs/3f2504e0-4f89-41d3-9a0c-0305e82c3301", nil) + h.ServeHTTP(httptest.NewRecorder(), req) + + // Scrape and confirm the request was recorded under the normalized route. + rec := httptest.NewRecorder() + greq, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/metrics", nil) + m.Handler().ServeHTTP(rec, greq) + + body := rec.Body.String() + if !strings.Contains(body, `scimesh_http_requests_total{method="POST",route="/jobs/{id}",status="201"}`) { + t.Errorf("requests_total not recorded as expected; body:\n%s", body) + } + if !strings.Contains(body, "go_goroutines") { + t.Error("Go runtime collector not registered") + } +} diff --git a/coordinator/internal/transport/http/server.go b/coordinator/internal/transport/http/server.go index 7598f1a..79d080a 100644 --- a/coordinator/internal/transport/http/server.go +++ b/coordinator/internal/transport/http/server.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/emil28092005/SciMesh/coordinator/internal/metrics" tokenpkg "github.com/emil28092005/SciMesh/coordinator/internal/token" "github.com/emil28092005/SciMesh/coordinator/internal/usecase" ) @@ -50,6 +51,8 @@ type Server struct { userserviceURL string // httpClient makes the login/register calls to the userservice. httpClient *http.Client + // metrics holds the Prometheus registry and HTTP instrumentation. + metrics *metrics.Metrics // ready probes downstream dependencies (the database) for /health. Kept as // a func so the transport layer never imports pgx. ready func(context.Context) error @@ -66,6 +69,7 @@ func NewServer(uc UseCases, log *slog.Logger, requestTimeout, heartbeatInterval verifier: tokenpkg.NewVerifier(jwtSecret), userserviceURL: strings.TrimRight(userserviceURL, "/"), httpClient: &http.Client{Timeout: 10 * time.Second}, + metrics: metrics.New(), ready: ready, } } @@ -97,6 +101,8 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /health", s.handleHealth) + // Unauthenticated like /health, so a Prometheus scraper needs no credential. + mux.Handle("GET /metrics", s.metrics.Handler()) hasBasicAuth := len(uiToken) > 0 && uiToken[0] != "" if s.uc.Dashboard != nil && (s.uiSessionMode() || hasBasicAuth) { @@ -157,7 +163,8 @@ func (s *Server) Handler(token string, uiToken ...string) http.Handler { withAccessLog(s.log), // including the 401s below withAuth(token, s.verifier), )) - return mux + // Measure every request once, outermost, with a normalized route label. + return s.metrics.Middleware(mux) } // handleHealth reports readiness. It probes the database so an orchestrator