Emil_Shanaty/SciMesh
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(coordinator): quorum verification for untrusted (volunteer) results

Browse Source 
Replaces the C1 quarantine with real verification. Trusted results (lab token,
verified, or admin worker) are accepted directly as before. An untrusted
worker's result is recorded as one vote per (task, owner) in a new task_results
table; the task only completes once QUORUM_SIZE distinct owners submit the same
result hash, otherwise it returns to the queue for another independent compute.

- migration 0013 task_results (one vote per owner, quorum by result_sha256)
- CompleteTask branches on worker trust; unknown worker defaults trusted (safe:
  completing needs the lease, whose owner is always a known registered worker)
- claim drops the quarantine and excludes chunks the owner already voted on
- domain Task.ReleaseAfterVote; QUORUM_SIZE config (default 2)
- unit tests: trusted direct-complete, untrusted needs-quorum, can-claim

Reducer and job done/total logic untouched — still one completed task per chunk.
This commit is contained in:
Efremenko Arhip
2026-07-26 22:55:51 +03:00
parent dcabfcd0c3
commit 18d58cce84
12 changed files with 356 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coordinator/internal/usecase/task.go
+106 -23
View File
@@ -2,6 +2,7 @@ package usecase
import (
"context"
"errors"
"time"
"github.com/google/uuid"
@@ -47,6 +48,7 @@ func (uc *ClaimTask) Execute(ctx context.Context, in ClaimTaskInput) (*domain.Cl
return nil, domain.ErrInvalidInput
}
workloads := in.Workloads
var voterOwner *uuid.UUID
if workerID, err := uuid.Parse(in.WorkerID); err == nil {
worker, err := uc.workers.Get(ctx, workerID)
if err != nil {
@@ -55,21 +57,18 @@ func (uc *ClaimTask) Execute(ctx context.Context, in ClaimTaskInput) (*domain.Cl
// Bind the caller to the worker it claims as. A JWT-authenticated
// volunteer may operate only its own workers; without this the trust
// tier would be read off a caller-supplied worker_id, letting anyone who
// knows a trusted worker's id claim as it and bypass the quarantine
// below. A shared-token caller (no requester) is a lab operator and may
// act as any worker, preserving the original behaviour.
// knows a trusted worker's id claim as it. A shared-token caller (no
// requester) is a lab operator and may act as any worker.
if r, ok := authctx.From(ctx); ok {
if worker.OwnerID == nil || *worker.OwnerID != r.UserID {
// Don't disclose that another user's worker exists.
return nil, domain.ErrWorkerNotFound
}
}
// C1 quarantine: an untrusted volunteer worker may register but receives
// no tasks, because there is not yet (until quorum, C2) any way to verify
// its results. Report an empty queue rather than an error, so its poller
// simply idles.
// An untrusted volunteer may claim, but never a chunk its owner has
// already voted on — so quorum needs genuinely independent computations.
if worker.TrustLevel == domain.WorkerUntrusted {
return nil, nil
voterOwner = worker.OwnerID
}
// Never trust caller-supplied capabilities: registration is the durable
// worker identity and its allowlist.
@@ -91,6 +90,7 @@ func (uc *ClaimTask) Execute(ctx context.Context, in ClaimTaskInput) (*domain.Cl
Owner: in.WorkerID,
Now: now,
LeaseUntil: now.Add(uc.leaseDuration),
VoterOwner: voterOwner,
})
if err != nil {
return err
@@ -161,13 +161,22 @@ type CompleteTask struct {
tasks TaskRepository
jobs JobRepository
artifacts ArtifactRepository
workers WorkerRepository
results TaskResultRepository
tx TxManager
clock Clock
// quorum is how many distinct owners must agree on an untrusted result
// before it is accepted; a trusted worker's result is accepted directly.
quorum int
}
func NewCompleteTask(tasks TaskRepository, jobs JobRepository, artifacts ArtifactRepository,
tx TxManager, clock Clock) *CompleteTask {
return &CompleteTask{tasks: tasks, jobs: jobs, artifacts: artifacts, tx: tx, clock: clock}
workers WorkerRepository, results TaskResultRepository, tx TxManager, clock Clock, quorum int) *CompleteTask {
if quorum < 1 {
quorum = 2
}
return &CompleteTask{tasks: tasks, jobs: jobs, artifacts: artifacts, workers: workers,
results: results, tx: tx, clock: clock, quorum: quorum}
}
// Execute applies the result and, when that was the job's last outstanding
@@ -186,24 +195,35 @@ func (uc *CompleteTask) Execute(ctx context.Context, in CompleteTaskInput) (*dom
}
// Rule 10: never trust a worker-supplied artifact reference. The result
// must be an artifact the coordinator itself stored for *this* task.
if err := uc.verifyResultArtifact(ctx, in.TaskID, in.Attempt, in.ResultArtifactID); err != nil {
art, err := uc.verifyResultArtifact(ctx, in.TaskID, in.Attempt, in.ResultArtifactID)
if err != nil {
return err
}
trusted, ownerID, err := uc.workerTrust(ctx, in.WorkerID)
if err != nil {
return err
}
now := uc.clock.Now()
// Untrusted (volunteer) worker: record a vote and only complete once a
// quorum of distinct owners agree; otherwise return the task to the queue.
if !trusted {
return uc.recordVote(ctx, task, in, art, ownerID, now, &out)
}
// Trusted worker (lab token, verified, or admin): accept directly.
before := task.Version
if err := task.CompleteWith(in.ResultArtifactID, in.Metrics,
in.WorkerID, in.Attempt, now); err != nil {
if err := task.CompleteWith(in.ResultArtifactID, in.Metrics, in.WorkerID, in.Attempt, now); err != nil {
return err
}
out = task
// A replay of an already-recorded result leaves the entity untouched.
// Writing anyway would fail the optimistic-concurrency guard (the stored
// version already equals ours) and turn an idempotent call into a 409.
if task.Version == before {
return nil
}
if err := uc.tasks.Update(ctx, task); err != nil {
return err
}
@@ -215,18 +235,81 @@ func (uc *CompleteTask) Execute(ctx context.Context, in CompleteTaskInput) (*dom
return out, nil
}
// verifyResultArtifact enforces that the referenced artifact was stored by the
// coordinator for this exact task. It stops a worker from completing task B with
// an artifact it uploaded for task A, and from naming an id that isn't a result.
func (uc *CompleteTask) verifyResultArtifact(ctx context.Context, taskID uuid.UUID, attempt int, artifactID uuid.UUID) error {
art, err := uc.artifacts.Get(ctx, artifactID)
// recordVote handles an untrusted result: it stores the vote, then completes the
// task when the submitter's result hash has reached quorum, or returns the task
// to the queue so another owner can compute it independently.
func (uc *CompleteTask) recordVote(ctx context.Context, task *domain.Task, in CompleteTaskInput,
art *domain.Artifact, ownerID uuid.UUID, now time.Time, out **domain.Task) error {
*out = task
if task.Status == domain.TaskCompleted {
return nil // already settled by an earlier quorum; nothing to record
}
if err := uc.results.RecordVote(ctx, task.ID, ownerID, art.SHA256, in.ResultArtifactID); err != nil {
return err
}
agree, err := uc.results.CountAgreeing(ctx, task.ID, art.SHA256)
if err != nil {
return err
}
if art.TaskID == nil || *art.TaskID != taskID || art.Attempt == nil || *art.Attempt != attempt || art.Kind != domain.ArtifactPartialResult {
return domain.ErrResultConflict
if agree >= uc.quorum {
// The submitter's own (already verified) artifact carries the winning
// hash, so complete with it.
if err := task.CompleteWith(in.ResultArtifactID, in.Metrics, in.WorkerID, in.Attempt, now); err != nil {
return err
}
} else if err := task.ReleaseAfterVote(in.WorkerID, in.Attempt, now); err != nil {
return err
}
return nil
if err := uc.tasks.Update(ctx, task); err != nil {
return err
}
return syncJobStatus(ctx, uc.jobs, uc.tasks, task.JobID, now)
}
// workerTrust reports whether the worker's results are accepted directly, and
// the owner to attribute a vote to when they are not.
func (uc *CompleteTask) workerTrust(ctx context.Context, workerID string) (trusted bool, ownerID uuid.UUID, err error) {
// When the worker can't be resolved, default to trusted — the pre-quorum
// behaviour. This is safe because completing a task requires holding its
// lease, and the lease owner is always a real registered worker whose trust
// is therefore known; only an untrusted worker ever takes the quorum path.
id, err := uuid.Parse(workerID)
if err != nil {
return true, uuid.Nil, nil
}
w, err := uc.workers.Get(ctx, id)
if err != nil {
if errors.Is(err, domain.ErrWorkerNotFound) {
return true, uuid.Nil, nil
}
return false, uuid.Nil, err
}
if w.TrustLevel != domain.WorkerUntrusted {
return true, uuid.Nil, nil
}
if w.OwnerID == nil {
// An untrusted worker always has an owner (it registered via a user JWT);
// a missing one is a data error, not a silent trust upgrade.
return false, uuid.Nil, domain.ErrInvalidInput
}
return false, *w.OwnerID, nil
}
// verifyResultArtifact enforces that the referenced artifact was stored by the
// coordinator for this exact task. It stops a worker from completing task B with
// an artifact it uploaded for task A, and from naming an id that isn't a result.
func (uc *CompleteTask) verifyResultArtifact(ctx context.Context, taskID uuid.UUID, attempt int, artifactID uuid.UUID) (*domain.Artifact, error) {
art, err := uc.artifacts.Get(ctx, artifactID)
if err != nil {
return nil, err
}
if art.TaskID == nil || *art.TaskID != taskID || art.Attempt == nil || *art.Attempt != attempt || art.Kind != domain.ArtifactPartialResult {
return nil, domain.ErrResultConflict
}
return art, nil
}
// --- FailTask ------------------------------------------------------------