droboticflow/internal/store/annotation_output_stream.go

853 lines
21 KiB
Go

package store
import (
"bufio"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
)
const outputBatchTarget = 8 << 20
// A build spills projected metadata to private temporary files. The directory
// retains identities, not revision/decision history. Only one bounded canonical
// part partition is interpreted at a time; a part never spans partitions.
type outputSourceSpool struct {
ctx context.Context
root string
guard *os.File
mu sync.Mutex
files map[string]*os.File
disk, peakDisk int64
directoryBytes int
index *outputSpoolIndex
directories workforcestats.OutputSnapshot
}
type outputTaskRef struct {
ID string `bson:"id"`
Project string `bson:"projectId"`
Run string `bson:"runId"`
Data string `bson:"dataTaskId"`
}
type outputItemRef struct {
ID string `bson:"id"`
Task string `bson:"legacyTaskId"`
Release string `bson:"releasePartitionId"`
}
type outputRunRef struct {
ID string `bson:"id"`
Task string `bson:"taskId"`
}
type outputEnvelope struct {
Kind string `bson:"k"`
Parts []string `bson:"p"`
Video string `bson:"v,omitempty"`
Data []byte `bson:"d"`
}
func newOutputSourceSpool(ctx context.Context) (*outputSourceSpool, error) {
cleanupOutputSpools(os.TempDir(), time.Now())
root, err := os.MkdirTemp("", "droboticflow-output-")
if err != nil {
return nil, err
}
p := &outputSourceSpool{ctx: ctx, root: root, files: map[string]*os.File{}}
p.guard, err = os.OpenFile(filepath.Join(root, ".build-lock"), os.O_CREATE|os.O_RDWR, 0600)
if err == nil {
err = syscall.Flock(int(p.guard.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
}
if err != nil {
if p.guard != nil {
_ = p.guard.Close()
}
_ = os.RemoveAll(root)
return nil, err
}
p.index, err = newOutputSpoolIndex(filepath.Join(root, "associations.db"), p.reserve)
if err != nil {
_ = p.guard.Close()
_ = os.RemoveAll(root)
return nil, err
}
return p, nil
}
func (p *outputSourceSpool) close() {
for _, f := range p.files {
_ = f.Close()
}
if p.index != nil {
_ = p.index.close()
}
_ = os.RemoveAll(p.root)
if p.guard != nil {
_ = p.guard.Close()
}
}
func (p *outputSourceSpool) reserve(n int64) error {
p.mu.Lock()
defer p.mu.Unlock()
p.disk += n
if p.disk > p.peakDisk {
p.peakDisk = p.disk
}
if p.disk > workforcestats.OutputSpoolMaxBytes {
return fmt.Errorf("%w: temporary spool budget", workforcestats.ErrOutputSourceBudget)
}
return nil
}
func writeOutputRecord(w io.Writer, raw []byte) error {
var size [4]byte
binary.LittleEndian.PutUint32(size[:], uint32(len(raw)))
if _, e := w.Write(size[:]); e != nil {
return e
}
_, e := w.Write(raw)
return e
}
func readOutputRecords(ctx context.Context, path string, visit func([]byte) error) error {
f, e := os.Open(path)
if errors.Is(e, os.ErrNotExist) {
return nil
}
if e != nil {
return e
}
defer f.Close()
r := bufio.NewReaderSize(f, 64<<10)
for {
if e := ctx.Err(); e != nil {
return e
}
var size [4]byte
_, e := io.ReadFull(r, size[:])
if e == io.EOF {
return nil
}
if e != nil {
return e
}
n := int(binary.LittleEndian.Uint32(size[:]))
if n < 5 || n > 32<<20 {
return fmt.Errorf("invalid reporting spool record size")
}
raw := make([]byte, n)
if _, e = io.ReadFull(r, raw); e != nil {
return e
}
if e = visit(raw); e != nil {
return e
}
}
}
func (p *outputSourceSpool) add(kind string, raw []byte) error {
if e := p.ctx.Err(); e != nil {
return e
}
directory := false
switch kind {
case "annotationTasks":
var v outputTaskRef
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
if e := p.index.put("tasks", v.ID, raw); e != nil {
return e
}
if v.Run != "" {
if e := p.index.put("runRefs", v.Run, []byte{1}); e != nil {
return e
}
}
case "humanItems":
var v outputItemRef
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
if e := p.index.put("items", v.ID, raw); e != nil {
return e
}
case "captureRecords":
var v struct {
ID string `bson:"id"`
Run string `bson:"runId"`
}
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
if e := p.index.put("captures", v.ID, raw); e != nil {
return e
}
if v.Run != "" {
if e := p.index.put("captureByRun", v.Run, []byte(v.ID)); e != nil {
return e
}
if e := p.index.put("runRefs", v.Run, []byte{1}); e != nil {
return e
}
}
case "runs":
var v outputRunRef
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
if e := p.index.put("runs", v.ID, raw); e != nil {
return e
}
has, e := p.index.contains("captures", v.Task)
if e != nil {
return e
}
if has {
if e = p.index.put("captureByRun", v.ID, []byte(v.Task)); e != nil {
return e
}
}
case "users":
var v domain.UserRecord
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
p.directories.Users = append(p.directories.Users, v)
directory = true
case "suppliers":
var v domain.SupplierRecord
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
p.directories.Suppliers = append(p.directories.Suppliers, v)
directory = true
case "projects":
var v domain.ProjectRecord
if e := bson.Unmarshal(raw, &v); e != nil {
return e
}
p.directories.Projects = append(p.directories.Projects, v)
directory = true
}
if directory {
p.mu.Lock()
p.directoryBytes += len(raw) + 512
over := p.directoryBytes > workforcestats.OutputDirectoryMaxBytes
p.mu.Unlock()
if over {
return fmt.Errorf("%w: display directory budget", workforcestats.ErrOutputSourceBudget)
}
}
if kind == "captureRecords" || kind == "users" || kind == "suppliers" || kind == "projects" {
return nil
}
if e := p.reserve(int64(len(raw) + 4)); e != nil {
return e
}
p.mu.Lock()
defer p.mu.Unlock()
f := p.files[kind]
if f == nil {
var e error
f, e = os.OpenFile(filepath.Join(p.root, kind+".raw"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
if e != nil {
return e
}
p.files[kind] = f
}
return writeOutputRecord(f, raw)
}
func (p *outputSourceSpool) identities() error {
if e := p.index.eachKeys(p.ctx, "tasks", func(ids []string) error {
for _, id := range ids {
raw, e := p.index.get("tasks", id)
if e != nil {
return e
}
var t outputTaskRef
if e = bson.Unmarshal(raw, &t); e != nil {
return e
}
capID, e := p.index.get("captureByRun", t.Run)
if e != nil {
return e
}
cRaw, e := p.index.get("captures", string(capID))
if e != nil {
return e
}
if len(cRaw) == 0 {
cRaw, e = p.index.get("captures", t.Data)
if e != nil {
return e
}
}
var capture *domain.CaptureRecord
if len(cRaw) > 0 {
var c domain.CaptureRecord
if e = bson.Unmarshal(cRaw, &c); e != nil {
return e
}
capture = &c
}
identity := workforcestats.OutputPartIdentityForCapture(t.Project, t.ID, capture)
value, e := bson.Marshal(identity)
if e != nil {
return e
}
if e = p.index.put("taskParts", id, value); e != nil {
return e
}
if e = p.index.addParts("runParts", t.Run, []string{identity.ID}); e != nil {
return e
}
}
return nil
}); e != nil {
return e
}
return p.index.eachKeys(p.ctx, "items", func(ids []string) error {
for _, id := range ids {
raw, e := p.index.get("items", id)
if e != nil {
return e
}
var i outputItemRef
if e = bson.Unmarshal(raw, &i); e != nil {
return e
}
identity, e := p.taskIdentity(i.Task)
if e != nil {
return e
}
if identity.ID != "" {
if e = p.index.put("itemParts", id, []byte(identity.ID)); e != nil {
return e
}
if e = p.index.addParts("releaseParts", i.Release, []string{identity.ID}); e != nil {
return e
}
}
}
return nil
})
}
func (p *outputSourceSpool) taskIdentity(id string) (workforcestats.OutputPartIdentity, error) {
var v workforcestats.OutputPartIdentity
raw, e := p.index.get("taskParts", id)
if e == nil && len(raw) > 0 {
e = bson.Unmarshal(raw, &v)
}
return v, e
}
func outputPartHash(part string) string {
h := sha256.Sum256([]byte(part))
return hex.EncodeToString(h[:])
}
func uniqueOutputParts(parts []string) []string {
out := make([]string, 0, len(parts))
seen := map[string]bool{}
for _, part := range parts {
if part != "" && !seen[part] {
seen[part] = true
out = append(out, part)
}
}
return out
}
func (p *outputSourceSpool) route(kind string, raw []byte) (outputEnvelope, error) {
e := outputEnvelope{Kind: kind, Data: raw}
var routeErr error
task := func(id string) {
v, err := p.taskIdentity(id)
if err != nil {
routeErr = err
}
e.Parts = append(e.Parts, v.ID)
}
item := func(id string) {
v, err := p.index.get("itemParts", id)
if err != nil {
routeErr = err
}
e.Parts = append(e.Parts, string(v))
}
refs := func(table, id string) {
v, err := p.index.parts(table, id)
if err != nil {
routeErr = err
}
e.Parts = append(e.Parts, v...)
}
switch kind {
case "annotationTasks":
var v outputTaskRef
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
identity, err := p.taskIdentity(v.ID)
if err != nil {
return e, err
}
e.Parts = append(e.Parts, identity.ID)
e.Video = identity.Video
case "humanItems":
var v outputItemRef
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
item(v.ID)
case "runs":
var v outputRunRef
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
refs("runParts", v.ID)
case "annotationWorklogEvents":
var v domain.AnnotationWorklogEventRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
task(v.TaskID)
if err := p.index.addParts("worklogParts", v.ID, e.Parts); err != nil {
return e, err
}
case "annotationReviews":
var v domain.AnnotationReviewRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
task(v.TaskID)
if err := p.index.addParts("reviewParts", v.ID, e.Parts); err != nil {
return e, err
}
case "annotationRevisions":
var v domain.AnnotationRevisionRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
task(v.TaskID)
case "humanEvents":
var v domain.HumanEventRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
for _, id := range outputStringIDs(v.Payload["sourceItemIds"]) {
item(id)
}
if op, ok := v.Payload["operationId"].(string); ok {
if err := p.index.addParts("operationParts", op, e.Parts); err != nil {
return e, err
}
}
case "reworkBaselines":
var v domain.ReworkBaselineRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
item(v.ItemID)
if err := p.index.addParts("operationParts", v.ReviewOperationID, e.Parts); err != nil {
return e, err
}
case "reworkHandlings":
var v domain.ReworkHandlingRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
item(v.ItemID)
case "reviewOperations":
var v domain.ReviewOperation
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
task(v.TaskID)
item(v.ItemID)
for _, id := range v.SourceItemIDs {
item(id)
}
for _, effect := range v.Effects {
if effect.Review != nil {
task(effect.Review.TaskID)
refs("reviewParts", effect.Review.ID)
}
if effect.Worklog != nil {
refs("worklogParts", effect.Worklog.ID)
}
}
refs("operationParts", v.ID)
refs("operationParts", v.OperationID)
case "deliveryPackages":
var v domain.DeliveryPackageRecord
if err := bson.Unmarshal(raw, &v); err != nil {
return e, err
}
for _, id := range v.ItemIDs {
item(id)
}
refs("releaseParts", v.ReleasePartitionID)
default:
return e, fmt.Errorf("unknown reporting spool collection %s", kind)
}
e.Parts = uniqueOutputParts(e.Parts)
return e, routeErr
}
func outputStringIDs(v any) []string {
switch v := v.(type) {
case bson.A:
out := []string{}
for _, x := range v {
if id, ok := x.(string); ok {
out = append(out, id)
}
}
return out
case []string:
return v
case []any:
out := []string{}
for _, x := range v {
if id, ok := x.(string); ok {
out = append(out, id)
}
}
return out
}
return nil
}
func (p *outputSourceSpool) remove(path string) {
if info, e := os.Stat(path); e == nil {
_ = os.Remove(path)
p.mu.Lock()
p.disk -= info.Size()
p.mu.Unlock()
}
}
func (p *outputSourceSpool) partition(writers map[string]*bufio.Writer, files map[string]*os.File, prefix string, e outputEnvelope) error {
groups := map[string][]string{}
for _, part := range e.Parts {
h := outputPartHash(part)
if strings.HasPrefix(h, prefix) {
key := h[:len(prefix)+1]
groups[key] = append(groups[key], part)
}
}
for key, parts := range groups {
copy := e
copy.Parts = parts
raw, err := bson.Marshal(copy)
if err != nil {
return err
}
if err = p.reserve(int64(len(raw) + 4)); err != nil {
return err
}
w := writers[key]
if w == nil {
f, err := os.OpenFile(filepath.Join(p.root, "part-"+key), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return err
}
files[key] = f
w = bufio.NewWriterSize(f, 64<<10)
writers[key] = w
}
if err = writeOutputRecord(w, raw); err != nil {
return err
}
}
return nil
}
func closeOutputWriters(writers map[string]*bufio.Writer, files map[string]*os.File) error {
var first error
for key, w := range writers {
if e := w.Flush(); first == nil {
first = e
}
if e := files[key].Close(); first == nil {
first = e
}
}
return first
}
// StreamAnnotationOutput preserves the old source interpreter and API contracts,
// but replaces its global history-size cliff with per-part bounded partitions.
func (s *MongoStore) StreamAnnotationOutput(ctx context.Context, org string, projects []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) {
if replica, err := s.reportReplica(ctx); err != nil {
return nil, err
} else if replica != nil {
return replica.StreamAnnotationOutput(ctx, org, projects, yield)
}
if len(projects) == 0 {
return &workforcestats.OutputSourceStats{}, nil
}
p, err := newOutputSourceSpool(ctx)
if err != nil {
return nil, err
}
defer p.close()
raw, err := s.loadAnnotationOutputTo(ctx, org, projects, outputSourceBudget{}, p)
if err != nil {
return nil, err
}
for _, f := range p.files {
if err = f.Close(); err != nil {
return nil, err
}
}
p.files = map[string]*os.File{}
if err = p.identities(); err != nil {
return nil, err
}
writers, files := map[string]*bufio.Writer{}, map[string]*os.File{}
// References must be resolved before operations are routed; acceptance can
// cover parts in other projects/buckets and pending plans still fence events.
order := []string{"annotationTasks", "humanItems", "runs", "annotationWorklogEvents", "annotationReviews", "annotationRevisions", "humanEvents", "reworkBaselines", "reworkHandlings", "deliveryPackages", "reviewOperations"}
for _, kind := range order {
path := filepath.Join(p.root, kind+".raw")
err = readOutputRecords(ctx, path, func(b []byte) error {
e, err := p.route(kind, b)
if err != nil {
return err
}
return p.partition(writers, files, "", e)
})
if err != nil {
_ = closeOutputWriters(writers, files)
return nil, err
}
p.remove(path)
}
if err = closeOutputWriters(writers, files); err != nil {
return nil, err
}
// Association history stays on disk; release its mmap/transactions before
// interpreting batches. Each envelope now owns its canonical routing.
if err = p.index.flush(); err != nil {
return nil, err
}
stats := raw.SourceStats
if stats == nil {
stats = &workforcestats.OutputSourceStats{}
}
stats.DirectoryBytes = p.directoryBytes + p.index.peak
if err = p.index.close(); err != nil {
return nil, err
}
p.index = nil
p.remove(filepath.Join(p.root, "associations.db"))
var emit func(string) error
emit = func(prefix string) error {
path := filepath.Join(p.root, "part-"+prefix)
info, e := os.Stat(path)
if errors.Is(e, os.ErrNotExist) {
return nil
}
if e != nil {
return e
}
split := func() error {
// An indivisible part must fail explicitly, not spill 64 identical copies.
first := ""
multiple := errors.New("multiple canonical parts")
check := readOutputRecords(ctx, path, func(b []byte) error {
var env outputEnvelope
if err := bson.Unmarshal(b, &env); err != nil {
return err
}
for _, part := range env.Parts {
if first == "" {
first = part
} else if part != first {
return multiple
}
}
return nil
})
if check != nil && !errors.Is(check, multiple) {
return check
}
if check == nil {
return fmt.Errorf("%w: one canonical part exceeds partition budget", workforcestats.ErrOutputSourceBudget)
}
if len(prefix) >= 64 {
return fmt.Errorf("%w: one canonical part exceeds partition budget", workforcestats.ErrOutputSourceBudget)
}
ws, fs := map[string]*bufio.Writer{}, map[string]*os.File{}
e := readOutputRecords(ctx, path, func(b []byte) error {
var env outputEnvelope
if e := bson.Unmarshal(b, &env); e != nil {
return e
}
return p.partition(ws, fs, prefix, env)
})
ce := closeOutputWriters(ws, fs)
if e != nil {
return e
}
if ce != nil {
return ce
}
p.remove(path)
for _, c := range "0123456789abcdef" {
if e := emit(prefix + string(c)); e != nil {
return e
}
}
return nil
}
if info.Size() > outputBatchTarget {
distinct := map[string]bool{}
rawBytes := 0
stop := errors.New("multiple parts")
check := readOutputRecords(ctx, path, func(b []byte) error {
var env outputEnvelope
if e := bson.Unmarshal(b, &env); e != nil {
return e
}
rawBytes += len(env.Data)
for _, part := range env.Parts {
distinct[part] = true
if len(distinct) > 1 {
return stop
}
}
if rawBytes > workforcestats.OutputSourceMaxBytes {
return fmt.Errorf("%w: one canonical part exceeds source budget", workforcestats.ErrOutputSourceBudget)
}
return nil
})
if errors.Is(check, stop) {
return split()
}
if check != nil {
return check
}
}
d := p.directories
d.Identities = map[string]workforcestats.OutputPartIdentity{}
docs, bytes := 0, 0
e = readOutputRecords(ctx, path, func(b []byte) error {
var env outputEnvelope
if e := bson.Unmarshal(b, &env); e != nil {
return e
}
docs++
bytes += len(env.Data)
if bytes > workforcestats.OutputSourceMaxBytes || docs > workforcestats.OutputSourceMaxDocuments {
return workforcestats.ErrOutputPartitionSplit
}
return appendOutputEnvelope(&d, env)
})
if errors.Is(e, workforcestats.ErrOutputPartitionSplit) {
return split()
}
if e != nil {
return e
}
if bytes > workforcestats.OutputSourceMaxBytes || docs > workforcestats.OutputSourceMaxDocuments {
return split()
}
if e = yield(d); errors.Is(e, workforcestats.ErrOutputPartitionSplit) {
return split()
} else if e != nil {
return e
}
stats.Partitions++
stats.PeakBatchBytes = max(stats.PeakBatchBytes, bytes)
stats.PeakBatchDocuments = max(stats.PeakBatchDocuments, docs)
p.remove(path)
return nil
}
for _, c := range "0123456789abcdef" {
if err = emit(string(c)); err != nil {
return nil, err
}
}
stats.PeakSpoolBytes = p.peakDisk
return stats, nil
}
func appendOutputEnvelope(d *workforcestats.OutputSnapshot, e outputEnvelope) error {
switch e.Kind {
case "annotationTasks":
var v domain.AnnotationTaskRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Tasks = append(d.Tasks, v)
d.Identities[v.ID] = workforcestats.OutputPartIdentity{ID: e.Parts[0], Video: e.Video}
case "humanItems":
var v domain.HumanItemRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Items = append(d.Items, v)
case "runs":
var v domain.RunRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Runs = append(d.Runs, v)
case "annotationWorklogEvents":
var v domain.AnnotationWorklogEventRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Events = append(d.Events, v)
case "annotationReviews":
var v domain.AnnotationReviewRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Reviews = append(d.Reviews, v)
case "annotationRevisions":
var v domain.AnnotationRevisionRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Revisions = append(d.Revisions, v)
case "humanEvents":
var v domain.HumanEventRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.HumanEvents = append(d.HumanEvents, v)
case "reworkBaselines":
var v domain.ReworkBaselineRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Baselines = append(d.Baselines, v)
case "reworkHandlings":
var v domain.ReworkHandlingRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Handlings = append(d.Handlings, v)
case "reviewOperations":
var v domain.ReviewOperation
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Operations = append(d.Operations, v)
case "deliveryPackages":
var v domain.DeliveryPackageRecord
if err := bson.Unmarshal(e.Data, &v); err != nil {
return err
}
d.Packages = append(d.Packages, v)
default:
return fmt.Errorf("unknown reporting batch %s", e.Kind)
}
return nil
}