forked from longtao.wu/droboticflow
853 lines
21 KiB
Go
853 lines
21 KiB
Go
package store
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/workforcestats"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
)
|
|
|
|
const outputBatchTarget = 8 << 20
|
|
|
|
// A build spills projected metadata to private temporary files. The directory
|
|
// retains identities, not revision/decision history. Only one bounded canonical
|
|
// part partition is interpreted at a time; a part never spans partitions.
|
|
type outputSourceSpool struct {
|
|
ctx context.Context
|
|
root string
|
|
guard *os.File
|
|
mu sync.Mutex
|
|
files map[string]*os.File
|
|
disk, peakDisk int64
|
|
directoryBytes int
|
|
index *outputSpoolIndex
|
|
directories workforcestats.OutputSnapshot
|
|
}
|
|
type outputTaskRef struct {
|
|
ID string `bson:"id"`
|
|
Project string `bson:"projectId"`
|
|
Run string `bson:"runId"`
|
|
Data string `bson:"dataTaskId"`
|
|
}
|
|
type outputItemRef struct {
|
|
ID string `bson:"id"`
|
|
Task string `bson:"legacyTaskId"`
|
|
Release string `bson:"releasePartitionId"`
|
|
}
|
|
type outputRunRef struct {
|
|
ID string `bson:"id"`
|
|
Task string `bson:"taskId"`
|
|
}
|
|
type outputEnvelope struct {
|
|
Kind string `bson:"k"`
|
|
Parts []string `bson:"p"`
|
|
Video string `bson:"v,omitempty"`
|
|
Data []byte `bson:"d"`
|
|
}
|
|
|
|
func newOutputSourceSpool(ctx context.Context) (*outputSourceSpool, error) {
|
|
cleanupOutputSpools(os.TempDir(), time.Now())
|
|
root, err := os.MkdirTemp("", "droboticflow-output-")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
p := &outputSourceSpool{ctx: ctx, root: root, files: map[string]*os.File{}}
|
|
p.guard, err = os.OpenFile(filepath.Join(root, ".build-lock"), os.O_CREATE|os.O_RDWR, 0600)
|
|
if err == nil {
|
|
err = syscall.Flock(int(p.guard.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
|
}
|
|
if err != nil {
|
|
if p.guard != nil {
|
|
_ = p.guard.Close()
|
|
}
|
|
_ = os.RemoveAll(root)
|
|
return nil, err
|
|
}
|
|
p.index, err = newOutputSpoolIndex(filepath.Join(root, "associations.db"), p.reserve)
|
|
if err != nil {
|
|
_ = p.guard.Close()
|
|
_ = os.RemoveAll(root)
|
|
return nil, err
|
|
}
|
|
return p, nil
|
|
}
|
|
func (p *outputSourceSpool) close() {
|
|
for _, f := range p.files {
|
|
_ = f.Close()
|
|
}
|
|
if p.index != nil {
|
|
_ = p.index.close()
|
|
}
|
|
_ = os.RemoveAll(p.root)
|
|
if p.guard != nil {
|
|
_ = p.guard.Close()
|
|
}
|
|
}
|
|
func (p *outputSourceSpool) reserve(n int64) error {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.disk += n
|
|
if p.disk > p.peakDisk {
|
|
p.peakDisk = p.disk
|
|
}
|
|
if p.disk > workforcestats.OutputSpoolMaxBytes {
|
|
return fmt.Errorf("%w: temporary spool budget", workforcestats.ErrOutputSourceBudget)
|
|
}
|
|
return nil
|
|
}
|
|
func writeOutputRecord(w io.Writer, raw []byte) error {
|
|
var size [4]byte
|
|
binary.LittleEndian.PutUint32(size[:], uint32(len(raw)))
|
|
if _, e := w.Write(size[:]); e != nil {
|
|
return e
|
|
}
|
|
_, e := w.Write(raw)
|
|
return e
|
|
}
|
|
func readOutputRecords(ctx context.Context, path string, visit func([]byte) error) error {
|
|
f, e := os.Open(path)
|
|
if errors.Is(e, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
defer f.Close()
|
|
r := bufio.NewReaderSize(f, 64<<10)
|
|
for {
|
|
if e := ctx.Err(); e != nil {
|
|
return e
|
|
}
|
|
var size [4]byte
|
|
_, e := io.ReadFull(r, size[:])
|
|
if e == io.EOF {
|
|
return nil
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n := int(binary.LittleEndian.Uint32(size[:]))
|
|
if n < 5 || n > 32<<20 {
|
|
return fmt.Errorf("invalid reporting spool record size")
|
|
}
|
|
raw := make([]byte, n)
|
|
if _, e = io.ReadFull(r, raw); e != nil {
|
|
return e
|
|
}
|
|
if e = visit(raw); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
}
|
|
func (p *outputSourceSpool) add(kind string, raw []byte) error {
|
|
if e := p.ctx.Err(); e != nil {
|
|
return e
|
|
}
|
|
directory := false
|
|
switch kind {
|
|
case "annotationTasks":
|
|
var v outputTaskRef
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
if e := p.index.put("tasks", v.ID, raw); e != nil {
|
|
return e
|
|
}
|
|
if v.Run != "" {
|
|
if e := p.index.put("runRefs", v.Run, []byte{1}); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
case "humanItems":
|
|
var v outputItemRef
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
if e := p.index.put("items", v.ID, raw); e != nil {
|
|
return e
|
|
}
|
|
case "captureRecords":
|
|
var v struct {
|
|
ID string `bson:"id"`
|
|
Run string `bson:"runId"`
|
|
}
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
if e := p.index.put("captures", v.ID, raw); e != nil {
|
|
return e
|
|
}
|
|
if v.Run != "" {
|
|
if e := p.index.put("captureByRun", v.Run, []byte(v.ID)); e != nil {
|
|
return e
|
|
}
|
|
if e := p.index.put("runRefs", v.Run, []byte{1}); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
case "runs":
|
|
var v outputRunRef
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
if e := p.index.put("runs", v.ID, raw); e != nil {
|
|
return e
|
|
}
|
|
has, e := p.index.contains("captures", v.Task)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if has {
|
|
if e = p.index.put("captureByRun", v.ID, []byte(v.Task)); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
case "users":
|
|
var v domain.UserRecord
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
p.directories.Users = append(p.directories.Users, v)
|
|
directory = true
|
|
case "suppliers":
|
|
var v domain.SupplierRecord
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
p.directories.Suppliers = append(p.directories.Suppliers, v)
|
|
directory = true
|
|
case "projects":
|
|
var v domain.ProjectRecord
|
|
if e := bson.Unmarshal(raw, &v); e != nil {
|
|
return e
|
|
}
|
|
p.directories.Projects = append(p.directories.Projects, v)
|
|
directory = true
|
|
}
|
|
if directory {
|
|
p.mu.Lock()
|
|
p.directoryBytes += len(raw) + 512
|
|
over := p.directoryBytes > workforcestats.OutputDirectoryMaxBytes
|
|
p.mu.Unlock()
|
|
if over {
|
|
return fmt.Errorf("%w: display directory budget", workforcestats.ErrOutputSourceBudget)
|
|
}
|
|
}
|
|
if kind == "captureRecords" || kind == "users" || kind == "suppliers" || kind == "projects" {
|
|
return nil
|
|
}
|
|
if e := p.reserve(int64(len(raw) + 4)); e != nil {
|
|
return e
|
|
}
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
f := p.files[kind]
|
|
if f == nil {
|
|
var e error
|
|
f, e = os.OpenFile(filepath.Join(p.root, kind+".raw"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
p.files[kind] = f
|
|
}
|
|
return writeOutputRecord(f, raw)
|
|
}
|
|
func (p *outputSourceSpool) identities() error {
|
|
if e := p.index.eachKeys(p.ctx, "tasks", func(ids []string) error {
|
|
for _, id := range ids {
|
|
raw, e := p.index.get("tasks", id)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var t outputTaskRef
|
|
if e = bson.Unmarshal(raw, &t); e != nil {
|
|
return e
|
|
}
|
|
capID, e := p.index.get("captureByRun", t.Run)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
cRaw, e := p.index.get("captures", string(capID))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if len(cRaw) == 0 {
|
|
cRaw, e = p.index.get("captures", t.Data)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
}
|
|
var capture *domain.CaptureRecord
|
|
if len(cRaw) > 0 {
|
|
var c domain.CaptureRecord
|
|
if e = bson.Unmarshal(cRaw, &c); e != nil {
|
|
return e
|
|
}
|
|
capture = &c
|
|
}
|
|
identity := workforcestats.OutputPartIdentityForCapture(t.Project, t.ID, capture)
|
|
value, e := bson.Marshal(identity)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if e = p.index.put("taskParts", id, value); e != nil {
|
|
return e
|
|
}
|
|
if e = p.index.addParts("runParts", t.Run, []string{identity.ID}); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return nil
|
|
}); e != nil {
|
|
return e
|
|
}
|
|
return p.index.eachKeys(p.ctx, "items", func(ids []string) error {
|
|
for _, id := range ids {
|
|
raw, e := p.index.get("items", id)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var i outputItemRef
|
|
if e = bson.Unmarshal(raw, &i); e != nil {
|
|
return e
|
|
}
|
|
identity, e := p.taskIdentity(i.Task)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if identity.ID != "" {
|
|
if e = p.index.put("itemParts", id, []byte(identity.ID)); e != nil {
|
|
return e
|
|
}
|
|
if e = p.index.addParts("releaseParts", i.Release, []string{identity.ID}); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
func (p *outputSourceSpool) taskIdentity(id string) (workforcestats.OutputPartIdentity, error) {
|
|
var v workforcestats.OutputPartIdentity
|
|
raw, e := p.index.get("taskParts", id)
|
|
if e == nil && len(raw) > 0 {
|
|
e = bson.Unmarshal(raw, &v)
|
|
}
|
|
return v, e
|
|
}
|
|
func outputPartHash(part string) string {
|
|
h := sha256.Sum256([]byte(part))
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
func uniqueOutputParts(parts []string) []string {
|
|
out := make([]string, 0, len(parts))
|
|
seen := map[string]bool{}
|
|
for _, part := range parts {
|
|
if part != "" && !seen[part] {
|
|
seen[part] = true
|
|
out = append(out, part)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
func (p *outputSourceSpool) route(kind string, raw []byte) (outputEnvelope, error) {
|
|
e := outputEnvelope{Kind: kind, Data: raw}
|
|
var routeErr error
|
|
task := func(id string) {
|
|
v, err := p.taskIdentity(id)
|
|
if err != nil {
|
|
routeErr = err
|
|
}
|
|
e.Parts = append(e.Parts, v.ID)
|
|
}
|
|
item := func(id string) {
|
|
v, err := p.index.get("itemParts", id)
|
|
if err != nil {
|
|
routeErr = err
|
|
}
|
|
e.Parts = append(e.Parts, string(v))
|
|
}
|
|
refs := func(table, id string) {
|
|
v, err := p.index.parts(table, id)
|
|
if err != nil {
|
|
routeErr = err
|
|
}
|
|
e.Parts = append(e.Parts, v...)
|
|
}
|
|
switch kind {
|
|
case "annotationTasks":
|
|
var v outputTaskRef
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
identity, err := p.taskIdentity(v.ID)
|
|
if err != nil {
|
|
return e, err
|
|
}
|
|
e.Parts = append(e.Parts, identity.ID)
|
|
e.Video = identity.Video
|
|
case "humanItems":
|
|
var v outputItemRef
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
item(v.ID)
|
|
case "runs":
|
|
var v outputRunRef
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
refs("runParts", v.ID)
|
|
case "annotationWorklogEvents":
|
|
var v domain.AnnotationWorklogEventRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
task(v.TaskID)
|
|
if err := p.index.addParts("worklogParts", v.ID, e.Parts); err != nil {
|
|
return e, err
|
|
}
|
|
case "annotationReviews":
|
|
var v domain.AnnotationReviewRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
task(v.TaskID)
|
|
if err := p.index.addParts("reviewParts", v.ID, e.Parts); err != nil {
|
|
return e, err
|
|
}
|
|
case "annotationRevisions":
|
|
var v domain.AnnotationRevisionRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
task(v.TaskID)
|
|
case "humanEvents":
|
|
var v domain.HumanEventRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
for _, id := range outputStringIDs(v.Payload["sourceItemIds"]) {
|
|
item(id)
|
|
}
|
|
if op, ok := v.Payload["operationId"].(string); ok {
|
|
if err := p.index.addParts("operationParts", op, e.Parts); err != nil {
|
|
return e, err
|
|
}
|
|
}
|
|
case "reworkBaselines":
|
|
var v domain.ReworkBaselineRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
item(v.ItemID)
|
|
if err := p.index.addParts("operationParts", v.ReviewOperationID, e.Parts); err != nil {
|
|
return e, err
|
|
}
|
|
case "reworkHandlings":
|
|
var v domain.ReworkHandlingRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
item(v.ItemID)
|
|
case "reviewOperations":
|
|
var v domain.ReviewOperation
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
task(v.TaskID)
|
|
item(v.ItemID)
|
|
for _, id := range v.SourceItemIDs {
|
|
item(id)
|
|
}
|
|
for _, effect := range v.Effects {
|
|
if effect.Review != nil {
|
|
task(effect.Review.TaskID)
|
|
refs("reviewParts", effect.Review.ID)
|
|
}
|
|
if effect.Worklog != nil {
|
|
refs("worklogParts", effect.Worklog.ID)
|
|
}
|
|
}
|
|
refs("operationParts", v.ID)
|
|
refs("operationParts", v.OperationID)
|
|
case "deliveryPackages":
|
|
var v domain.DeliveryPackageRecord
|
|
if err := bson.Unmarshal(raw, &v); err != nil {
|
|
return e, err
|
|
}
|
|
for _, id := range v.ItemIDs {
|
|
item(id)
|
|
}
|
|
refs("releaseParts", v.ReleasePartitionID)
|
|
default:
|
|
return e, fmt.Errorf("unknown reporting spool collection %s", kind)
|
|
}
|
|
e.Parts = uniqueOutputParts(e.Parts)
|
|
return e, routeErr
|
|
}
|
|
func outputStringIDs(v any) []string {
|
|
switch v := v.(type) {
|
|
case bson.A:
|
|
out := []string{}
|
|
for _, x := range v {
|
|
if id, ok := x.(string); ok {
|
|
out = append(out, id)
|
|
}
|
|
}
|
|
return out
|
|
case []string:
|
|
return v
|
|
case []any:
|
|
out := []string{}
|
|
for _, x := range v {
|
|
if id, ok := x.(string); ok {
|
|
out = append(out, id)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
return nil
|
|
}
|
|
func (p *outputSourceSpool) remove(path string) {
|
|
if info, e := os.Stat(path); e == nil {
|
|
_ = os.Remove(path)
|
|
p.mu.Lock()
|
|
p.disk -= info.Size()
|
|
p.mu.Unlock()
|
|
}
|
|
}
|
|
func (p *outputSourceSpool) partition(writers map[string]*bufio.Writer, files map[string]*os.File, prefix string, e outputEnvelope) error {
|
|
groups := map[string][]string{}
|
|
for _, part := range e.Parts {
|
|
h := outputPartHash(part)
|
|
if strings.HasPrefix(h, prefix) {
|
|
key := h[:len(prefix)+1]
|
|
groups[key] = append(groups[key], part)
|
|
}
|
|
}
|
|
for key, parts := range groups {
|
|
copy := e
|
|
copy.Parts = parts
|
|
raw, err := bson.Marshal(copy)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = p.reserve(int64(len(raw) + 4)); err != nil {
|
|
return err
|
|
}
|
|
w := writers[key]
|
|
if w == nil {
|
|
f, err := os.OpenFile(filepath.Join(p.root, "part-"+key), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files[key] = f
|
|
w = bufio.NewWriterSize(f, 64<<10)
|
|
writers[key] = w
|
|
}
|
|
if err = writeOutputRecord(w, raw); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func closeOutputWriters(writers map[string]*bufio.Writer, files map[string]*os.File) error {
|
|
var first error
|
|
for key, w := range writers {
|
|
if e := w.Flush(); first == nil {
|
|
first = e
|
|
}
|
|
if e := files[key].Close(); first == nil {
|
|
first = e
|
|
}
|
|
}
|
|
return first
|
|
}
|
|
|
|
// StreamAnnotationOutput preserves the old source interpreter and API contracts,
|
|
// but replaces its global history-size cliff with per-part bounded partitions.
|
|
func (s *MongoStore) StreamAnnotationOutput(ctx context.Context, org string, projects []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return nil, err
|
|
} else if replica != nil {
|
|
return replica.StreamAnnotationOutput(ctx, org, projects, yield)
|
|
}
|
|
if len(projects) == 0 {
|
|
return &workforcestats.OutputSourceStats{}, nil
|
|
}
|
|
p, err := newOutputSourceSpool(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer p.close()
|
|
raw, err := s.loadAnnotationOutputTo(ctx, org, projects, outputSourceBudget{}, p)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, f := range p.files {
|
|
if err = f.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
p.files = map[string]*os.File{}
|
|
if err = p.identities(); err != nil {
|
|
return nil, err
|
|
}
|
|
writers, files := map[string]*bufio.Writer{}, map[string]*os.File{}
|
|
// References must be resolved before operations are routed; acceptance can
|
|
// cover parts in other projects/buckets and pending plans still fence events.
|
|
order := []string{"annotationTasks", "humanItems", "runs", "annotationWorklogEvents", "annotationReviews", "annotationRevisions", "humanEvents", "reworkBaselines", "reworkHandlings", "deliveryPackages", "reviewOperations"}
|
|
for _, kind := range order {
|
|
path := filepath.Join(p.root, kind+".raw")
|
|
err = readOutputRecords(ctx, path, func(b []byte) error {
|
|
e, err := p.route(kind, b)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return p.partition(writers, files, "", e)
|
|
})
|
|
if err != nil {
|
|
_ = closeOutputWriters(writers, files)
|
|
return nil, err
|
|
}
|
|
p.remove(path)
|
|
}
|
|
if err = closeOutputWriters(writers, files); err != nil {
|
|
return nil, err
|
|
}
|
|
// Association history stays on disk; release its mmap/transactions before
|
|
// interpreting batches. Each envelope now owns its canonical routing.
|
|
if err = p.index.flush(); err != nil {
|
|
return nil, err
|
|
}
|
|
stats := raw.SourceStats
|
|
if stats == nil {
|
|
stats = &workforcestats.OutputSourceStats{}
|
|
}
|
|
stats.DirectoryBytes = p.directoryBytes + p.index.peak
|
|
if err = p.index.close(); err != nil {
|
|
return nil, err
|
|
}
|
|
p.index = nil
|
|
p.remove(filepath.Join(p.root, "associations.db"))
|
|
var emit func(string) error
|
|
emit = func(prefix string) error {
|
|
path := filepath.Join(p.root, "part-"+prefix)
|
|
info, e := os.Stat(path)
|
|
if errors.Is(e, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
split := func() error {
|
|
// An indivisible part must fail explicitly, not spill 64 identical copies.
|
|
first := ""
|
|
multiple := errors.New("multiple canonical parts")
|
|
check := readOutputRecords(ctx, path, func(b []byte) error {
|
|
var env outputEnvelope
|
|
if err := bson.Unmarshal(b, &env); err != nil {
|
|
return err
|
|
}
|
|
for _, part := range env.Parts {
|
|
if first == "" {
|
|
first = part
|
|
} else if part != first {
|
|
return multiple
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if check != nil && !errors.Is(check, multiple) {
|
|
return check
|
|
}
|
|
if check == nil {
|
|
return fmt.Errorf("%w: one canonical part exceeds partition budget", workforcestats.ErrOutputSourceBudget)
|
|
}
|
|
if len(prefix) >= 64 {
|
|
return fmt.Errorf("%w: one canonical part exceeds partition budget", workforcestats.ErrOutputSourceBudget)
|
|
}
|
|
ws, fs := map[string]*bufio.Writer{}, map[string]*os.File{}
|
|
e := readOutputRecords(ctx, path, func(b []byte) error {
|
|
var env outputEnvelope
|
|
if e := bson.Unmarshal(b, &env); e != nil {
|
|
return e
|
|
}
|
|
return p.partition(ws, fs, prefix, env)
|
|
})
|
|
ce := closeOutputWriters(ws, fs)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if ce != nil {
|
|
return ce
|
|
}
|
|
p.remove(path)
|
|
for _, c := range "0123456789abcdef" {
|
|
if e := emit(prefix + string(c)); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if info.Size() > outputBatchTarget {
|
|
distinct := map[string]bool{}
|
|
rawBytes := 0
|
|
stop := errors.New("multiple parts")
|
|
check := readOutputRecords(ctx, path, func(b []byte) error {
|
|
var env outputEnvelope
|
|
if e := bson.Unmarshal(b, &env); e != nil {
|
|
return e
|
|
}
|
|
rawBytes += len(env.Data)
|
|
for _, part := range env.Parts {
|
|
distinct[part] = true
|
|
if len(distinct) > 1 {
|
|
return stop
|
|
}
|
|
}
|
|
if rawBytes > workforcestats.OutputSourceMaxBytes {
|
|
return fmt.Errorf("%w: one canonical part exceeds source budget", workforcestats.ErrOutputSourceBudget)
|
|
}
|
|
return nil
|
|
})
|
|
if errors.Is(check, stop) {
|
|
return split()
|
|
}
|
|
if check != nil {
|
|
return check
|
|
}
|
|
}
|
|
d := p.directories
|
|
d.Identities = map[string]workforcestats.OutputPartIdentity{}
|
|
docs, bytes := 0, 0
|
|
e = readOutputRecords(ctx, path, func(b []byte) error {
|
|
var env outputEnvelope
|
|
if e := bson.Unmarshal(b, &env); e != nil {
|
|
return e
|
|
}
|
|
docs++
|
|
bytes += len(env.Data)
|
|
if bytes > workforcestats.OutputSourceMaxBytes || docs > workforcestats.OutputSourceMaxDocuments {
|
|
return workforcestats.ErrOutputPartitionSplit
|
|
}
|
|
return appendOutputEnvelope(&d, env)
|
|
})
|
|
if errors.Is(e, workforcestats.ErrOutputPartitionSplit) {
|
|
return split()
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if bytes > workforcestats.OutputSourceMaxBytes || docs > workforcestats.OutputSourceMaxDocuments {
|
|
return split()
|
|
}
|
|
if e = yield(d); errors.Is(e, workforcestats.ErrOutputPartitionSplit) {
|
|
return split()
|
|
} else if e != nil {
|
|
return e
|
|
}
|
|
stats.Partitions++
|
|
stats.PeakBatchBytes = max(stats.PeakBatchBytes, bytes)
|
|
stats.PeakBatchDocuments = max(stats.PeakBatchDocuments, docs)
|
|
p.remove(path)
|
|
return nil
|
|
}
|
|
for _, c := range "0123456789abcdef" {
|
|
if err = emit(string(c)); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
stats.PeakSpoolBytes = p.peakDisk
|
|
return stats, nil
|
|
}
|
|
func appendOutputEnvelope(d *workforcestats.OutputSnapshot, e outputEnvelope) error {
|
|
switch e.Kind {
|
|
case "annotationTasks":
|
|
var v domain.AnnotationTaskRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Tasks = append(d.Tasks, v)
|
|
d.Identities[v.ID] = workforcestats.OutputPartIdentity{ID: e.Parts[0], Video: e.Video}
|
|
case "humanItems":
|
|
var v domain.HumanItemRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Items = append(d.Items, v)
|
|
case "runs":
|
|
var v domain.RunRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Runs = append(d.Runs, v)
|
|
case "annotationWorklogEvents":
|
|
var v domain.AnnotationWorklogEventRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Events = append(d.Events, v)
|
|
case "annotationReviews":
|
|
var v domain.AnnotationReviewRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Reviews = append(d.Reviews, v)
|
|
case "annotationRevisions":
|
|
var v domain.AnnotationRevisionRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Revisions = append(d.Revisions, v)
|
|
case "humanEvents":
|
|
var v domain.HumanEventRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.HumanEvents = append(d.HumanEvents, v)
|
|
case "reworkBaselines":
|
|
var v domain.ReworkBaselineRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Baselines = append(d.Baselines, v)
|
|
case "reworkHandlings":
|
|
var v domain.ReworkHandlingRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Handlings = append(d.Handlings, v)
|
|
case "reviewOperations":
|
|
var v domain.ReviewOperation
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Operations = append(d.Operations, v)
|
|
case "deliveryPackages":
|
|
var v domain.DeliveryPackageRecord
|
|
if err := bson.Unmarshal(e.Data, &v); err != nil {
|
|
return err
|
|
}
|
|
d.Packages = append(d.Packages, v)
|
|
default:
|
|
return fmt.Errorf("unknown reporting batch %s", e.Kind)
|
|
}
|
|
return nil
|
|
}
|