feat(coordinator): running state, worker liveness, request-size limits
Polish pass hardening the queue and closing plan gaps. - Task state machine gains `running`: the first heartbeat moves a task from leased to running (migrations 0006/0007 add the enum value and extend the lease-integrity check). verifyLease, ExpireLease, the reaper SQL, and job progress all treat leased and running alike. - Worker liveness: a heartbeat from a registered worker (UUID worker_id) bumps its last_heartbeat_at online; a second background reaper marks workers offline after WORKER_OFFLINE_AFTER of silence (RunReaper generalized to RunPeriodic). - Request-size limits: JSON bodies capped at 1 MiB; dataset/artifact uploads capped at MAX_UPLOAD_BYTES (default 1 GiB) via http.MaxBytesReader. - Tests cover the running transition, liveness + offline reaper (unit over memstore and integration over Postgres).
This commit is contained in:
@@ -31,6 +31,8 @@ type Config struct {
|
||||
LogFile string
|
||||
// Directory where artifact bytes are stored.
|
||||
StorageDir string
|
||||
// Upper bound on an uploaded dataset or artifact body, in bytes.
|
||||
MaxUploadBytes int64
|
||||
|
||||
// Connection pool upper bound.
|
||||
DBMaxConns int32
|
||||
@@ -48,6 +50,8 @@ type Config struct {
|
||||
DefaultMaxAttempts int
|
||||
// How often the background lease-reaper runs.
|
||||
ReaperInterval time.Duration
|
||||
// A worker silent for longer than this is marked offline by the reaper.
|
||||
WorkerOfflineAfter time.Duration
|
||||
}
|
||||
|
||||
// Load reads the environment and fails fast on anything required-but-missing
|
||||
@@ -76,6 +80,7 @@ func LoadConfig() (Config, error) {
|
||||
LogLevel: getEnv("LOG_LEVEL", "info"),
|
||||
LogFile: os.Getenv("LOG_FILE"),
|
||||
StorageDir: getEnv("COORDINATOR_STORAGE_DIR", "./data"),
|
||||
MaxUploadBytes: 1 << 30, // 1 GiB
|
||||
DBMaxConns: 10,
|
||||
DBConnectTimeout: 30 * time.Second,
|
||||
RequestTimeout: 15 * time.Second,
|
||||
@@ -83,6 +88,7 @@ func LoadConfig() (Config, error) {
|
||||
LeaseDuration: 2 * time.Minute,
|
||||
DefaultMaxAttempts: 3,
|
||||
ReaperInterval: 30 * time.Second,
|
||||
WorkerOfflineAfter: 1 * time.Minute,
|
||||
}
|
||||
|
||||
if cfg.DatabaseURL == "" {
|
||||
@@ -96,6 +102,9 @@ func LoadConfig() (Config, error) {
|
||||
if cfg.DBConnectTimeout, err = getEnvDuration("DB_CONNECT_TIMEOUT", cfg.DBConnectTimeout); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.MaxUploadBytes, err = getEnvInt64("MAX_UPLOAD_BYTES", cfg.MaxUploadBytes); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.RequestTimeout, err = getEnvDuration("REQUEST_TIMEOUT", cfg.RequestTimeout); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -108,6 +117,9 @@ func LoadConfig() (Config, error) {
|
||||
if cfg.ReaperInterval, err = getEnvDuration("REAPER_INTERVAL", cfg.ReaperInterval); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.WorkerOfflineAfter, err = getEnvDuration("WORKER_OFFLINE_AFTER", cfg.WorkerOfflineAfter); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.DefaultMaxAttempts, err = getEnvInt("DEFAULT_MAX_ATTEMPTS", cfg.DefaultMaxAttempts); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -147,6 +159,18 @@ func getEnvInt32(key string, def int32) (int32, error) {
|
||||
return int32(n), nil
|
||||
}
|
||||
|
||||
func getEnvInt64(key string, def int64) (int64, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def, nil
|
||||
}
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func getEnvDuration(key string, def time.Duration) (time.Duration, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/emil28092005/SciMesh/coordinator/internal/usecase"
|
||||
)
|
||||
|
||||
const shutdownGrace = 15 * time.Second
|
||||
@@ -48,7 +46,13 @@ func RunServer(ctx context.Context, log *slog.Logger, addr string, handler http.
|
||||
|
||||
// RunReaper periodically reclaims tasks whose lease elapsed, so a worker that
|
||||
// died without a heartbeat cannot strand its task in 'leased' forever.
|
||||
func RunReaper(ctx context.Context, log *slog.Logger, uc *usecase.ExpireLeases, interval time.Duration) {
|
||||
// RunPeriodic invokes fn on an interval until ctx is done, logging how many rows
|
||||
// each tick affected. It backs the background reapers (expired leases, offline
|
||||
// workers) — each is a set-based UPDATE that is safe to run repeatedly and
|
||||
// concurrently across coordinators.
|
||||
func RunPeriodic(ctx context.Context, log *slog.Logger, name string, interval time.Duration,
|
||||
fn func(context.Context) (int64, error)) {
|
||||
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
|
||||
@@ -57,15 +61,13 @@ func RunReaper(ctx context.Context, log *slog.Logger, uc *usecase.ExpireLeases,
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
n, err := uc.Execute(ctx)
|
||||
n, err := fn(ctx)
|
||||
if err != nil {
|
||||
// Demoted to debug while the repository is still a stub;
|
||||
// raise to Warn once phase 5 lands.
|
||||
log.Debug("reaper skipped", "err", err)
|
||||
log.Debug(name+" skipped", "err", err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Info("reaper requeued expired leases", "count", n)
|
||||
log.Info(name, "count", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user