58 lines
1.5 KiB
Go
58 lines
1.5 KiB
Go
// Package reducer contains deterministic, coordinator-side result reductions.
|
|
package reducer
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/csv"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
)
|
|
|
|
// ReduceOrderedConcat concatenates worker partial tables in shard order into a
|
|
// single table with one header. Every partial must carry the same header as the
|
|
// first partial and rows of the same width; anything else fails the job closed.
|
|
func ReduceOrderedConcat(partials []io.Reader) ([]byte, error) {
|
|
var out bytes.Buffer
|
|
writer := csv.NewWriter(&out)
|
|
var firstHeader []string
|
|
for _, partial := range partials {
|
|
reader := csv.NewReader(partial)
|
|
header, err := reader.Read()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read partial header: %w", err)
|
|
}
|
|
if len(header) == 0 {
|
|
return nil, fmt.Errorf("partial result has an empty header")
|
|
}
|
|
if firstHeader == nil {
|
|
firstHeader = header
|
|
if err := writer.Write(header); err != nil {
|
|
return nil, err
|
|
}
|
|
} else if !equalStrings(header, firstHeader) {
|
|
return nil, fmt.Errorf("partial result has an inconsistent header")
|
|
}
|
|
for {
|
|
row, err := reader.Read()
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read partial row: %w", err)
|
|
}
|
|
if len(row) != len(header) {
|
|
return nil, fmt.Errorf("partial result has a row with an inconsistent width")
|
|
}
|
|
if err := writer.Write(row); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
writer.Flush()
|
|
if err := writer.Error(); err != nil {
|
|
return nil, err
|
|
}
|
|
return out.Bytes(), nil
|
|
}
|