feat(coordinator): running state, worker liveness, request-size limits

Polish pass hardening the queue and closing plan gaps.

- Task state machine gains `running`: the first heartbeat moves a task from
  leased to running (migrations 0006/0007 add the enum value and extend the
  lease-integrity check). verifyLease, ExpireLease, the reaper SQL, and job
  progress all treat leased and running alike.
- Worker liveness: a heartbeat from a registered worker (UUID worker_id) bumps
  its last_heartbeat_at online; a second background reaper marks workers offline
  after WORKER_OFFLINE_AFTER of silence (RunReaper generalized to RunPeriodic).
- Request-size limits: JSON bodies capped at 1 MiB; dataset/artifact uploads
  capped at MAX_UPLOAD_BYTES (default 1 GiB) via http.MaxBytesReader.
- Tests cover the running transition, liveness + offline reaper (unit over
  memstore and integration over Postgres).
This commit is contained in:
Efremenko Arhip
2026-07-23 17:36:26 +03:00
parent 4fc3c69fdf
commit 3b41455b20
23 changed files with 332 additions and 32 deletions
+24
View File
@@ -31,6 +31,8 @@ type Config struct {
LogFile string
// Directory where artifact bytes are stored.
StorageDir string
// Upper bound on an uploaded dataset or artifact body, in bytes.
MaxUploadBytes int64
// Connection pool upper bound.
DBMaxConns int32
@@ -48,6 +50,8 @@ type Config struct {
DefaultMaxAttempts int
// How often the background lease-reaper runs.
ReaperInterval time.Duration
// A worker silent for longer than this is marked offline by the reaper.
WorkerOfflineAfter time.Duration
}
// Load reads the environment and fails fast on anything required-but-missing
@@ -76,6 +80,7 @@ func LoadConfig() (Config, error) {
LogLevel: getEnv("LOG_LEVEL", "info"),
LogFile: os.Getenv("LOG_FILE"),
StorageDir: getEnv("COORDINATOR_STORAGE_DIR", "./data"),
MaxUploadBytes: 1 << 30, // 1 GiB
DBMaxConns: 10,
DBConnectTimeout: 30 * time.Second,
RequestTimeout: 15 * time.Second,
@@ -83,6 +88,7 @@ func LoadConfig() (Config, error) {
LeaseDuration: 2 * time.Minute,
DefaultMaxAttempts: 3,
ReaperInterval: 30 * time.Second,
WorkerOfflineAfter: 1 * time.Minute,
}
if cfg.DatabaseURL == "" {
@@ -96,6 +102,9 @@ func LoadConfig() (Config, error) {
if cfg.DBConnectTimeout, err = getEnvDuration("DB_CONNECT_TIMEOUT", cfg.DBConnectTimeout); err != nil {
return Config{}, err
}
if cfg.MaxUploadBytes, err = getEnvInt64("MAX_UPLOAD_BYTES", cfg.MaxUploadBytes); err != nil {
return Config{}, err
}
if cfg.RequestTimeout, err = getEnvDuration("REQUEST_TIMEOUT", cfg.RequestTimeout); err != nil {
return Config{}, err
}
@@ -108,6 +117,9 @@ func LoadConfig() (Config, error) {
if cfg.ReaperInterval, err = getEnvDuration("REAPER_INTERVAL", cfg.ReaperInterval); err != nil {
return Config{}, err
}
if cfg.WorkerOfflineAfter, err = getEnvDuration("WORKER_OFFLINE_AFTER", cfg.WorkerOfflineAfter); err != nil {
return Config{}, err
}
if cfg.DefaultMaxAttempts, err = getEnvInt("DEFAULT_MAX_ATTEMPTS", cfg.DefaultMaxAttempts); err != nil {
return Config{}, err
}
@@ -147,6 +159,18 @@ func getEnvInt32(key string, def int32) (int32, error) {
return int32(n), nil
}
func getEnvInt64(key string, def int64) (int64, error) {
v := os.Getenv(key)
if v == "" {
return def, nil
}
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return 0, fmt.Errorf("%s: %w", key, err)
}
return n, nil
}
func getEnvDuration(key string, def time.Duration) (time.Duration, error) {
v := os.Getenv(key)
if v == "" {
+10 -8
View File
@@ -8,8 +8,6 @@ import (
"log/slog"
"net/http"
"time"
"github.com/emil28092005/SciMesh/coordinator/internal/usecase"
)
const shutdownGrace = 15 * time.Second
@@ -48,7 +46,13 @@ func RunServer(ctx context.Context, log *slog.Logger, addr string, handler http.
// RunReaper periodically reclaims tasks whose lease elapsed, so a worker that
// died without a heartbeat cannot strand its task in 'leased' forever.
func RunReaper(ctx context.Context, log *slog.Logger, uc *usecase.ExpireLeases, interval time.Duration) {
// RunPeriodic invokes fn on an interval until ctx is done, logging how many rows
// each tick affected. It backs the background reapers (expired leases, offline
// workers) — each is a set-based UPDATE that is safe to run repeatedly and
// concurrently across coordinators.
func RunPeriodic(ctx context.Context, log *slog.Logger, name string, interval time.Duration,
fn func(context.Context) (int64, error)) {
t := time.NewTicker(interval)
defer t.Stop()
@@ -57,15 +61,13 @@ func RunReaper(ctx context.Context, log *slog.Logger, uc *usecase.ExpireLeases,
case <-ctx.Done():
return
case <-t.C:
n, err := uc.Execute(ctx)
n, err := fn(ctx)
if err != nil {
// Demoted to debug while the repository is still a stub;
// raise to Warn once phase 5 lands.
log.Debug("reaper skipped", "err", err)
log.Debug(name+" skipped", "err", err)
continue
}
if n > 0 {
log.Info("reaper requeued expired leases", "count", n)
log.Info(name, "count", n)
}
}
}