feat(coordinator): running state, worker liveness, request-size limits
Polish pass hardening the queue and closing plan gaps. - Task state machine gains `running`: the first heartbeat moves a task from leased to running (migrations 0006/0007 add the enum value and extend the lease-integrity check). verifyLease, ExpireLease, the reaper SQL, and job progress all treat leased and running alike. - Worker liveness: a heartbeat from a registered worker (UUID worker_id) bumps its last_heartbeat_at online; a second background reaper marks workers offline after WORKER_OFFLINE_AFTER of silence (RunReaper generalized to RunPeriodic). - Request-size limits: JSON bodies capped at 1 MiB; dataset/artifact uploads capped at MAX_UPLOAD_BYTES (default 1 GiB) via http.MaxBytesReader. - Tests cover the running transition, liveness + offline reaper (unit over memstore and integration over Postgres).
This commit is contained in:
@@ -31,6 +31,8 @@ type Config struct {
|
||||
LogFile string
|
||||
// Directory where artifact bytes are stored.
|
||||
StorageDir string
|
||||
// Upper bound on an uploaded dataset or artifact body, in bytes.
|
||||
MaxUploadBytes int64
|
||||
|
||||
// Connection pool upper bound.
|
||||
DBMaxConns int32
|
||||
@@ -48,6 +50,8 @@ type Config struct {
|
||||
DefaultMaxAttempts int
|
||||
// How often the background lease-reaper runs.
|
||||
ReaperInterval time.Duration
|
||||
// A worker silent for longer than this is marked offline by the reaper.
|
||||
WorkerOfflineAfter time.Duration
|
||||
}
|
||||
|
||||
// Load reads the environment and fails fast on anything required-but-missing
|
||||
@@ -76,6 +80,7 @@ func LoadConfig() (Config, error) {
|
||||
LogLevel: getEnv("LOG_LEVEL", "info"),
|
||||
LogFile: os.Getenv("LOG_FILE"),
|
||||
StorageDir: getEnv("COORDINATOR_STORAGE_DIR", "./data"),
|
||||
MaxUploadBytes: 1 << 30, // 1 GiB
|
||||
DBMaxConns: 10,
|
||||
DBConnectTimeout: 30 * time.Second,
|
||||
RequestTimeout: 15 * time.Second,
|
||||
@@ -83,6 +88,7 @@ func LoadConfig() (Config, error) {
|
||||
LeaseDuration: 2 * time.Minute,
|
||||
DefaultMaxAttempts: 3,
|
||||
ReaperInterval: 30 * time.Second,
|
||||
WorkerOfflineAfter: 1 * time.Minute,
|
||||
}
|
||||
|
||||
if cfg.DatabaseURL == "" {
|
||||
@@ -96,6 +102,9 @@ func LoadConfig() (Config, error) {
|
||||
if cfg.DBConnectTimeout, err = getEnvDuration("DB_CONNECT_TIMEOUT", cfg.DBConnectTimeout); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.MaxUploadBytes, err = getEnvInt64("MAX_UPLOAD_BYTES", cfg.MaxUploadBytes); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.RequestTimeout, err = getEnvDuration("REQUEST_TIMEOUT", cfg.RequestTimeout); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -108,6 +117,9 @@ func LoadConfig() (Config, error) {
|
||||
if cfg.ReaperInterval, err = getEnvDuration("REAPER_INTERVAL", cfg.ReaperInterval); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.WorkerOfflineAfter, err = getEnvDuration("WORKER_OFFLINE_AFTER", cfg.WorkerOfflineAfter); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.DefaultMaxAttempts, err = getEnvInt("DEFAULT_MAX_ATTEMPTS", cfg.DefaultMaxAttempts); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -147,6 +159,18 @@ func getEnvInt32(key string, def int32) (int32, error) {
|
||||
return int32(n), nil
|
||||
}
|
||||
|
||||
func getEnvInt64(key string, def int64) (int64, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def, nil
|
||||
}
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func getEnvDuration(key string, def time.Duration) (time.Duration, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
|
||||
Reference in New Issue
Block a user