droboticflow/internal/store/query_build_plan.go
2026-11-15 08:11:09 -08:00

197 lines
7.0 KiB
Go

package store
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"sort"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const queryBuildFactPageRows = 200
func queryBuildPlanPartKey(j queryBuildJob, kind string, n int) string {
return fmt.Sprintf("%s/%s/%09d", j.Generation, kind, n)
}
func queryBuildJSONPart(j queryBuildJob, kind string, n int, v any) (queryBuildPart, error) {
var b bytes.Buffer
z := gzip.NewWriter(&b)
if err := json.NewEncoder(z).Encode(v); err != nil {
return queryBuildPart{}, err
}
if err := z.Close(); err != nil {
return queryBuildPart{}, err
}
data := b.Bytes()
sum := sha256.Sum256(data)
return queryBuildPart{ID: queryBuildPlanPartKey(j, kind, n), Job: j.Generation, Data: data, Digest: hex.EncodeToString(sum[:])}, nil
}
func queryBuildJSONDecode(p queryBuildPart, out any) error {
sum := sha256.Sum256(p.Data)
if hex.EncodeToString(sum[:]) != p.Digest {
return errors.New("query plan digest mismatch")
}
z, err := gzip.NewReader(bytes.NewReader(p.Data))
if err != nil {
return err
}
defer z.Close()
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
if err != nil {
return err
}
if len(raw) > 64<<20 {
return errors.New("query plan decode budget exceeded")
}
return json.Unmarshal(raw, out)
}
func (s *MongoStore) queryBuildSavePart(ctx context.Context, l queryModelLease, j *queryBuildJob, p queryBuildPart) error {
if err := s.queryBuildPace(ctx, len(p.Data)); err != nil {
return err
}
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
if _, err := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": p.ID}, p, options.Replace().SetUpsert(true)); err != nil {
return err
}
return s.saveQueryBuildJob(sc, *j)
})
}
func (s *MongoStore) queryBuildProductionPlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
}))
snapshot, err := s.loadProductionStats(readCtx, j.Organization, j.Projects)
if err != nil {
return err
}
snapshot.AsOf = j.AsOf.Format(time.RFC3339Nano)
// Preserve the existing JSON/gzip manifest format for all old readers.
var data bytes.Buffer
z := gzip.NewWriter(&data)
budget := &queryModelBudgetWriter{writer: z}
if err = json.NewEncoder(budget).Encode(productionstats.Build(snapshot)); err != nil {
return err
}
if err = z.Close(); err != nil {
return err
}
if data.Len() > queryModelMaxBytes {
return errors.New("production plan exceeds existing size budget")
}
payload := data.Bytes()
sum := sha256.Sum256(payload)
head := QueryModelHead{Generation: j.Generation + "-production", AsOf: j.AsOf, Bytes: len(payload), ChunkBytes: queryModelBuildChunkBytes, Chunks: (len(payload) + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes, Digest: hex.EncodeToString(sum[:]), SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
if j.Production.Generation != "" && j.Production.Digest != head.Digest {
return errors.New("sealed production input changed")
}
for n := j.ProductionSaved; n < head.Chunks; n++ {
if err = r.check(ctx); err != nil {
return err
}
p := payload[n*head.ChunkBytes : min((n+1)*head.ChunkBytes, len(payload))]
if err = s.queryBuildPace(ctx, len(p)); err != nil {
return err
}
next := *j
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
next.Production = head
next.ProductionSaved = n + 1
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
key := queryModelBlobKey(queryModelKey("production", j.Organization, j.Projects), head.Generation, n)
_, e := s.queryData().ReplaceOne(sc, bson.M{"_id": key}, bson.M{"_id": key, "generation": head.Generation, "data": p, "createdAt": time.Now().UTC()}, options.Replace().SetUpsert(true))
if e != nil {
return e
}
return s.saveQueryBuildJob(sc, next)
}); err != nil {
return err
}
*j = next
}
j.Phase = "workforce_plan"
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
}
func (s *MongoStore) queryBuildWorkforcePlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
}))
readCtx = context.WithValue(readCtx, workforceBuildBudgetKey{}, &workforceBuildBudget{})
input, err := s.LoadWorkforceStats(readCtx, j.Organization, workforcestats.Query{Mode: domain.RunModeAll})
if err != nil {
return err
}
facts := workforcestats.Materialize(input)
sort.Slice(facts, func(i, j int) bool { return facts[i].Key < facts[j].Key })
count := (len(facts) + queryBuildFactPageRows - 1) / queryBuildFactPageRows
if j.PlanRows != 0 && j.PlanRows != len(facts) {
return errors.New("workforce plan input changed")
}
// Recompute only from durable captured input, and verify the sealed prefix
// before adding any later page (including after a code deployment).
prefix := ""
for n := 0; n < j.PlanSaved; n++ {
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
if e != nil {
return e
}
var saved queryBuildPart
op, stop := queryBuildOperation(ctx)
e = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": part.ID, "job": j.Generation}).Decode(&saved)
stop()
if e != nil {
return e
}
if saved.Digest != part.Digest {
return errors.New("sealed workforce plan fingerprint changed")
}
chain := sha256.Sum256([]byte(prefix + saved.Digest))
prefix = hex.EncodeToString(chain[:])
}
if prefix != j.PlanDigest {
return errors.New("sealed workforce plan manifest changed")
}
for n := j.PlanSaved; n < count; n++ {
if err = r.check(ctx); err != nil {
return err
}
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
if e != nil {
return e
}
next := *j
next.ProgressAt = time.Now().UTC()
next.ActiveWithoutProgress = 0
next.PlanParts = count
next.PlanRows = len(facts)
next.PlanSaved = n + 1
next.PlanBytes += int64(len(part.Data))
chain := sha256.Sum256([]byte(next.PlanDigest + part.Digest))
next.PlanDigest = hex.EncodeToString(chain[:])
if err = s.queryBuildSavePart(ctx, l, &next, part); err != nil {
return err
}
*j = next
}
j.PlanRows = len(facts)
j.PlanParts = count
j.Phase = "apply"
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
}