Efremenko Arhip
c3243a6b7e
The coordinator can now ingest a dataset itself, not only accept client-supplied
chunk URIs.
- internal/chunk: a deterministic, generic TSV row splitter — repeats the header
per shard, buffers one shard at a time, rejects header-only input. Unit-tested.
- POST /jobs/upload (multipart): streams the dataset into an input artifact,
splits it into shard artifacts, and creates one shard task per shard, all in
one transaction; blobs are cleaned up if the transaction fails.
- GET /tasks/{id}/input streams a task's input shard back to the worker.
- domain: NewUploadedJob, NewShardTask, Task/Job.InputArtifactID; a shard task's
input is an artifact, not a URI. Claim response nests input:{uri,sha256} per
the contract, with uri = /tasks/{id}/input for shards.
- migration 0005 makes input_uri nullable and adds a has-input check.
- The existing URI-based POST /jobs path is untouched; both coexist.
84 lines
1.4 KiB
Go
84 lines
1.4 KiB
Go
package usecase
|
|
|
|
import (
|
|
"io"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
// Use-case boundary types. Adapters map their wire formats onto these, so the
|
|
// HTTP shape can change without touching business code.
|
|
|
|
type CreateJobInput struct {
|
|
Workload string
|
|
InputURI string
|
|
Parameters map[string]any
|
|
Chunks []ChunkInput
|
|
}
|
|
|
|
type ChunkInput struct {
|
|
ChunkIndex int
|
|
Workload string
|
|
InputURI string
|
|
InputSHA256 string
|
|
Parameters map[string]any
|
|
MaxAttempts int
|
|
}
|
|
|
|
type RegisterWorkerInput struct {
|
|
Name string
|
|
Capabilities []string
|
|
}
|
|
|
|
type ClaimTaskInput struct {
|
|
WorkerID string
|
|
Workloads []string
|
|
}
|
|
|
|
type RenewLeaseInput struct {
|
|
TaskID uuid.UUID
|
|
WorkerID string
|
|
Attempt int
|
|
}
|
|
|
|
type CompleteTaskInput struct {
|
|
TaskID uuid.UUID
|
|
WorkerID string
|
|
Attempt int
|
|
ResultArtifactID uuid.UUID
|
|
Metrics map[string]any
|
|
}
|
|
|
|
type SubmitDatasetInput struct {
|
|
Workload string
|
|
Parameters map[string]any
|
|
RowsPerShard int
|
|
Filename string
|
|
ContentType string
|
|
Body io.Reader
|
|
}
|
|
|
|
type SubmitDatasetResult struct {
|
|
JobID uuid.UUID
|
|
TaskCount int
|
|
InputArtifactID uuid.UUID
|
|
}
|
|
|
|
type UploadArtifactInput struct {
|
|
TaskID uuid.UUID
|
|
WorkerID string
|
|
Attempt int
|
|
Filename string
|
|
ContentType string
|
|
Body io.Reader
|
|
}
|
|
|
|
type FailTaskInput struct {
|
|
TaskID uuid.UUID
|
|
WorkerID string
|
|
Attempt int
|
|
ErrorCode string
|
|
ErrorMessage string
|
|
Retryable bool
|
|
}
|