forked from longtao.wu/droboticflow
197 lines
7.0 KiB
Go
197 lines
7.0 KiB
Go
package store
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/productionstats"
|
|
"droboticflow/internal/workforcestats"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
const queryBuildFactPageRows = 200
|
|
|
|
func queryBuildPlanPartKey(j queryBuildJob, kind string, n int) string {
|
|
return fmt.Sprintf("%s/%s/%09d", j.Generation, kind, n)
|
|
}
|
|
func queryBuildJSONPart(j queryBuildJob, kind string, n int, v any) (queryBuildPart, error) {
|
|
var b bytes.Buffer
|
|
z := gzip.NewWriter(&b)
|
|
if err := json.NewEncoder(z).Encode(v); err != nil {
|
|
return queryBuildPart{}, err
|
|
}
|
|
if err := z.Close(); err != nil {
|
|
return queryBuildPart{}, err
|
|
}
|
|
data := b.Bytes()
|
|
sum := sha256.Sum256(data)
|
|
return queryBuildPart{ID: queryBuildPlanPartKey(j, kind, n), Job: j.Generation, Data: data, Digest: hex.EncodeToString(sum[:])}, nil
|
|
}
|
|
func queryBuildJSONDecode(p queryBuildPart, out any) error {
|
|
sum := sha256.Sum256(p.Data)
|
|
if hex.EncodeToString(sum[:]) != p.Digest {
|
|
return errors.New("query plan digest mismatch")
|
|
}
|
|
z, err := gzip.NewReader(bytes.NewReader(p.Data))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer z.Close()
|
|
raw, err := io.ReadAll(io.LimitReader(z, (64<<20)+1))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(raw) > 64<<20 {
|
|
return errors.New("query plan decode budget exceeded")
|
|
}
|
|
return json.Unmarshal(raw, out)
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildSavePart(ctx context.Context, l queryModelLease, j *queryBuildJob, p queryBuildPart) error {
|
|
if err := s.queryBuildPace(ctx, len(p.Data)); err != nil {
|
|
return err
|
|
}
|
|
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
if _, err := s.db.Collection(queryBuildPartsCollection).ReplaceOne(sc, bson.M{"_id": p.ID}, p, options.Replace().SetUpsert(true)); err != nil {
|
|
return err
|
|
}
|
|
return s.saveQueryBuildJob(sc, *j)
|
|
})
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildProductionPlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
|
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
|
|
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
|
|
}))
|
|
snapshot, err := s.loadProductionStats(readCtx, j.Organization, j.Projects)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
snapshot.AsOf = j.AsOf.Format(time.RFC3339Nano)
|
|
// Preserve the existing JSON/gzip manifest format for all old readers.
|
|
var data bytes.Buffer
|
|
z := gzip.NewWriter(&data)
|
|
budget := &queryModelBudgetWriter{writer: z}
|
|
if err = json.NewEncoder(budget).Encode(productionstats.Build(snapshot)); err != nil {
|
|
return err
|
|
}
|
|
if err = z.Close(); err != nil {
|
|
return err
|
|
}
|
|
if data.Len() > queryModelMaxBytes {
|
|
return errors.New("production plan exceeds existing size budget")
|
|
}
|
|
payload := data.Bytes()
|
|
sum := sha256.Sum256(payload)
|
|
head := QueryModelHead{Generation: j.Generation + "-production", AsOf: j.AsOf, Bytes: len(payload), ChunkBytes: queryModelBuildChunkBytes, Chunks: (len(payload) + queryModelBuildChunkBytes - 1) / queryModelBuildChunkBytes, Digest: hex.EncodeToString(sum[:]), SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
|
|
if j.Production.Generation != "" && j.Production.Digest != head.Digest {
|
|
return errors.New("sealed production input changed")
|
|
}
|
|
for n := j.ProductionSaved; n < head.Chunks; n++ {
|
|
if err = r.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
p := payload[n*head.ChunkBytes : min((n+1)*head.ChunkBytes, len(payload))]
|
|
if err = s.queryBuildPace(ctx, len(p)); err != nil {
|
|
return err
|
|
}
|
|
next := *j
|
|
next.ProgressAt = time.Now().UTC()
|
|
next.ActiveWithoutProgress = 0
|
|
next.Production = head
|
|
next.ProductionSaved = n + 1
|
|
if err = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error {
|
|
key := queryModelBlobKey(queryModelKey("production", j.Organization, j.Projects), head.Generation, n)
|
|
_, e := s.queryData().ReplaceOne(sc, bson.M{"_id": key}, bson.M{"_id": key, "generation": head.Generation, "data": p, "createdAt": time.Now().UTC()}, options.Replace().SetUpsert(true))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
return s.saveQueryBuildJob(sc, next)
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
*j = next
|
|
}
|
|
j.Phase = "workforce_plan"
|
|
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
|
|
}
|
|
|
|
func (s *MongoStore) queryBuildWorkforcePlan(ctx context.Context, l queryModelLease, j *queryBuildJob, r *queryBuildRound) error {
|
|
readCtx := context.WithValue(ctx, queryBuildReadKey{}, queryBuildReadFunc(func(c context.Context, coll mongoCollection, f, p bson.M, out any) error {
|
|
return s.queryBuildCapture(c, l, j, r, coll, f, p, out)
|
|
}))
|
|
readCtx = context.WithValue(readCtx, workforceBuildBudgetKey{}, &workforceBuildBudget{})
|
|
input, err := s.LoadWorkforceStats(readCtx, j.Organization, workforcestats.Query{Mode: domain.RunModeAll})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
facts := workforcestats.Materialize(input)
|
|
sort.Slice(facts, func(i, j int) bool { return facts[i].Key < facts[j].Key })
|
|
count := (len(facts) + queryBuildFactPageRows - 1) / queryBuildFactPageRows
|
|
if j.PlanRows != 0 && j.PlanRows != len(facts) {
|
|
return errors.New("workforce plan input changed")
|
|
}
|
|
// Recompute only from durable captured input, and verify the sealed prefix
|
|
// before adding any later page (including after a code deployment).
|
|
prefix := ""
|
|
for n := 0; n < j.PlanSaved; n++ {
|
|
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var saved queryBuildPart
|
|
op, stop := queryBuildOperation(ctx)
|
|
e = s.db.Collection(queryBuildPartsCollection).FindOne(op, bson.M{"_id": part.ID, "job": j.Generation}).Decode(&saved)
|
|
stop()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if saved.Digest != part.Digest {
|
|
return errors.New("sealed workforce plan fingerprint changed")
|
|
}
|
|
chain := sha256.Sum256([]byte(prefix + saved.Digest))
|
|
prefix = hex.EncodeToString(chain[:])
|
|
}
|
|
if prefix != j.PlanDigest {
|
|
return errors.New("sealed workforce plan manifest changed")
|
|
}
|
|
for n := j.PlanSaved; n < count; n++ {
|
|
if err = r.check(ctx); err != nil {
|
|
return err
|
|
}
|
|
part, e := queryBuildJSONPart(*j, "facts", n, facts[n*queryBuildFactPageRows:min((n+1)*queryBuildFactPageRows, len(facts))])
|
|
if e != nil {
|
|
return e
|
|
}
|
|
next := *j
|
|
next.ProgressAt = time.Now().UTC()
|
|
next.ActiveWithoutProgress = 0
|
|
next.PlanParts = count
|
|
next.PlanRows = len(facts)
|
|
next.PlanSaved = n + 1
|
|
next.PlanBytes += int64(len(part.Data))
|
|
chain := sha256.Sum256([]byte(next.PlanDigest + part.Digest))
|
|
next.PlanDigest = hex.EncodeToString(chain[:])
|
|
if err = s.queryBuildSavePart(ctx, l, &next, part); err != nil {
|
|
return err
|
|
}
|
|
*j = next
|
|
}
|
|
j.PlanRows = len(facts)
|
|
j.PlanParts = count
|
|
j.Phase = "apply"
|
|
return s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, *j) })
|
|
}
|