🐛 Resume confirmed batches in cold workforce builds

This commit is contained in:
d-robotics 2026-11-15 05:47:28 -08:00
parent 84523190b6
commit e4c4a20df5
2 changed files with 363 additions and 10 deletions

View File

@ -160,7 +160,21 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
}
oldHead := control.Heads[key]
epoch := oldHead.FactEpoch + 1
if err = s.restoreWorkforceStage(ctx, l, org, oldHead.FactEpoch); err != nil {
// An unpublished first epoch can retain confirmed batches across a failed
// cold build. Legacy epoch-zero rows keep the original recovery path.
cold := oldHead.FactEpoch == 0
if cold {
err = c.FindOne(ctx, bson.M{"organizationId": org, "fromEpoch": bson.M{"$lte": int64(0)}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return err
}
cold = errors.Is(err, mongo.ErrNoDocuments)
}
restoreHead := oldHead.FactEpoch
if cold {
restoreHead = epoch
}
if err = s.restoreWorkforceStage(ctx, l, org, restoreHead); err != nil {
return err
}
var d workforcestats.Snapshot
@ -180,21 +194,37 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
facts := workforcestats.Materialize(d)
log.Printf("workforce_fact_phase stage=loaded elapsed_ms=%d events=%d slices=%d logical_rows=%d", time.Since(started).Milliseconds(), len(d.Events), len(d.Slices), len(facts))
type previous struct {
ID string `bson:"_id"`
Key string `bson:"key"`
Hash string `bson:"hash"`
ID string `bson:"_id"`
Key string `bson:"key"`
Hash string `bson:"hash"`
reusable bool
}
cur, err := c.Find(ctx, workforceVersionFilter(org, oldHead.FactEpoch), options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1}))
previousFilter := workforceVersionFilter(org, oldHead.FactEpoch)
if cold {
previousFilter = bson.M{"organizationId": org, "fromEpoch": epoch}
}
cur, err := c.Find(ctx, previousFilter, options.Find().SetProjection(bson.M{"_id": 1, "key": 1, "hash": 1, "toEpoch": 1, "retiredAt": 1, "expireAt": 1}))
if err != nil {
return err
}
old := map[string]previous{}
staged := map[string]previous{}
for cur.Next(ctx) {
var row previous
if err = cur.Decode(&row); err != nil {
break
}
old[row.Key] = row
row.reusable = true
for _, field := range []string{"toEpoch", "retiredAt", "expireAt"} {
if _, e := cur.Current.LookupErr(field); e == nil {
row.reusable = false
}
}
if cold {
staged[row.ID] = row
} else {
old[row.Key] = row
}
}
if err == nil {
err = cur.Err()
@ -205,9 +235,9 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
}
models := []mongo.WriteModel{}
batchBytes := 0
batchRows, confirmedRows, confirmedBytes := 0, 0, 0
batchRows, confirmedRows, confirmedBytes, resumedRows := 0, 0, 0, 0
defer func() {
log.Printf("workforce_fact_phase stage=write confirmed_rows=%d confirmed_bytes=%d elapsed_ms=%d", confirmedRows, confirmedBytes, time.Since(started).Milliseconds())
log.Printf("workforce_fact_phase stage=write confirmed_rows=%d confirmed_bytes=%d resumed_rows=%d elapsed_ms=%d", confirmedRows, confirmedBytes, resumedRows, time.Since(started).Milliseconds())
}()
changed := false
flush := func() error {
@ -240,6 +270,13 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
return err
}
hash := queryserving.Key(workforceFactSchema, string(raw))
id := queryserving.Key(workforceFactSchema, org, fact.Key, epoch)
pending, pendingFound := staged[id]
delete(staged, id)
if cold && pendingFound && pending.Key == fact.Key && pending.Hash == hash && pending.reusable {
resumedRows++
continue
}
prev, found := old[fact.Key]
delete(old, fact.Key)
if found && prev.Hash == hash {
@ -262,7 +299,7 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
if err != nil {
return err
}
doc["_id"], doc["organizationId"], doc["fromEpoch"], doc["hash"], doc["at"] = queryserving.Key(workforceFactSchema, org, fact.Key, epoch), org, epoch, hash, at
doc["_id"], doc["organizationId"], doc["fromEpoch"], doc["hash"], doc["at"] = id, org, epoch, hash, at
doc["atSeconds"], doc["atNanos"] = at.Unix(), at.Nanosecond()
doc["metric"] = workforcestats.MetricCounterValues(fact.Metric)
if fact.Detail != nil {
@ -281,7 +318,13 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
if fact.Detail != nil {
doc["order"] = fact.Detail.ID + fact.Detail.UserID + fact.Detail.SupplierID
}
models = append(models, mongo.NewInsertOneModel().SetDocument(doc))
if cold && pendingFound {
// Only the uncommitted first epoch is replaced. Published versions
// retain the existing MVCC retirement and recovery behavior.
models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": id, "organizationId": org, "fromEpoch": epoch}).SetReplacement(doc))
} else {
models = append(models, mongo.NewInsertOneModel().SetDocument(doc))
}
batchRows++
stored, err := bson.Marshal(doc)
if err != nil {
@ -304,6 +347,17 @@ func (s *MongoStore) rebuildWorkforceFacts(ctx context.Context, l queryModelLeas
}
}
}
// A resumed snapshot may have lost events or changed identities. Remove
// only unmatched rows of this unpublished epoch before atomic publication.
for _, pending := range staged {
models = append(models, mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": pending.ID, "organizationId": org, "fromEpoch": epoch}))
batchBytes += 256
if len(models) >= 200 || batchBytes >= queryBuildBatchBytes {
if err = flush(); err != nil {
return err
}
}
}
if err = flush(); err != nil {
return err
}

View File

@ -5,9 +5,13 @@ import (
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"encoding/json"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"golang.org/x/time/rate"
"os"
"reflect"
"sort"
@ -380,3 +384,298 @@ func TestWorkforceCompleteBuildReusesHistoryWithinOriginalBudgetLocal(t *testing
}
t.Logf("same26000 events, build reads=%d reference reads=%d sourceBytes=%d originalCap=%d elapsed=%s facts=%d", buildReads, referenceReads, budget.bytes, 64<<20, time.Since(started), len(a))
}
// The database state after all batches commit but before the head CAS is the
// same unpublished state recovered after a worker crash.
func unpublishWorkforceFixture(t *testing.T, target *MongoStore, org string) {
t.Helper()
if _, err := target.queryControl().UpdateOne(t.Context(), bson.M{"_id": "global"}, bson.M{"$unset": bson.M{"heads." + workforceFactKey(org): ""}}); err != nil {
t.Fatal(err)
}
}
func TestWorkforceColdFactResumeKeepsConfirmedRowsLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
var mu sync.Mutex
writes := 0
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "insert" || e.CommandName == "update" || e.CommandName == "delete" {
if n, ok := e.Command.Lookup(e.CommandName).StringValueOK(); ok && n == workforceFactCollection {
mu.Lock()
writes++
mu.Unlock()
}
}
}}
source, target, replica := newLocalReplica(t, nil, monitor)
ctx := t.Context()
seedWorkforceStats(t, source)
finishLocalReplica(t, replica)
l, ok, err := target.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(ok, err)
}
if err = target.rebuildWorkforceFacts(ctx, l, "workforce-test", 0); err != nil {
t.Fatal(err)
}
unpublishWorkforceFixture(t, target, "workforce-test")
// A successor owns the restart; the previous owner must not publish even
// when every matching batch can be reused without writing any facts.
if _, err = target.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); err != nil {
t.Fatal(err)
}
successor, ok, err := target.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(ok, err)
}
if err = target.rebuildWorkforceFacts(ctx, l, "workforce-test", 0); err == nil {
t.Fatal("stale owner published resumed facts")
}
if _, exists, err := target.workforceHead(ctx, "workforce-test"); err != nil || exists {
t.Fatal("stale owner changed publication", exists, err)
}
mu.Lock()
writes = 0
mu.Unlock()
if err = target.rebuildWorkforceFacts(ctx, successor, "workforce-test", 0); err != nil {
t.Fatal(err)
}
mu.Lock()
n := writes
mu.Unlock()
if n != 0 {
t.Fatalf("confirmed cold batches were rewritten: %d commands", n)
}
h, exists, err := target.workforceHead(ctx, "workforce-test")
if err != nil || !exists || h.FactEpoch != 1 {
t.Fatal(h, exists, err)
}
}
func TestWorkforceColdFactResumeReconcilesChangedAndDeletedRowsLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
source, target, replica := newLocalReplica(t)
ctx := t.Context()
seedWorkforceStats(t, source)
finishLocalReplica(t, replica)
l, ok, err := target.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(ok, err)
}
if err = target.rebuildWorkforceFacts(ctx, l, "workforce-test", 0); err != nil {
t.Fatal(err)
}
unpublishWorkforceFixture(t, target, "workforce-test")
c := target.db.Collection(workforceFactCollection)
// Changes to the next snapshot and leftovers from aborted/legacy attempts
// must never appear as duplicate, retired or unrelated facts after publish.
if _, err = target.db.Collection("annotationWorklogEvents").DeleteOne(ctx, bson.M{"organizationId": "workforce-test", "id": "submitted"}); err != nil {
t.Fatal(err)
}
if _, err = target.db.Collection("users").UpdateOne(ctx, bson.M{"id": "u"}, bson.M{"$set": bson.M{"name": "changed synthetic name"}}); err != nil {
t.Fatal(err)
}
var existing bson.M
if err = c.FindOne(ctx, bson.M{"organizationId": "workforce-test"}).Decode(&existing); err != nil {
t.Fatal(err)
}
if _, err = c.UpdateOne(ctx, bson.M{"_id": existing["_id"]}, bson.M{"$set": bson.M{"toEpoch": int64(1)}}); err != nil {
t.Fatal(err)
}
for _, doc := range []bson.M{
{"_id": "unpublished-orphan", "organizationId": "workforce-test", "fromEpoch": int64(1), "key": "removed-key"},
{"_id": "future-orphan", "organizationId": "workforce-test", "fromEpoch": int64(2), "key": "future"},
{"_id": "foreign-orphan", "organizationId": "other-organization", "fromEpoch": int64(1), "key": "foreign"},
} {
if _, err = c.InsertOne(ctx, doc); err != nil {
t.Fatal(err)
}
}
originalGate := target.queryBuildWrites
target.queryBuildWrites = &queryBuildWriteGate{limiter: rate.NewLimiter(rate.Inf, queryBuildBatchBytes), inspect: func(context.Context) (bool, error) { return true, nil }}
interrupted, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
err = target.rebuildWorkforceFacts(interrupted, l, "workforce-test", 0)
cancel()
target.queryBuildWrites = originalGate
if err == nil {
t.Fatal("interrupted reconciliation unexpectedly published")
}
if _, exists, e := target.workforceHead(ctx, "workforce-test"); e != nil || exists {
t.Fatal("partial resume became readable", exists, e)
}
if err = target.rebuildWorkforceFacts(ctx, l, "workforce-test", 0); err != nil {
t.Fatal(err)
}
for _, id := range []string{"unpublished-orphan", "future-orphan"} {
if err = c.FindOne(ctx, bson.M{"_id": id}).Err(); !errors.Is(err, mongo.ErrNoDocuments) {
t.Fatal("unpublished orphan survived", id, err)
}
}
if err = c.FindOne(ctx, bson.M{"_id": "foreign-orphan"}).Err(); err != nil {
t.Fatal("another organization's stage modified", err)
}
h, exists, err := target.workforceHead(ctx, "workforce-test")
if err != nil || !exists {
t.Fatal(exists, err)
}
for _, q := range []workforcestats.Query{{Mode: domain.RunModeAll}, {Mode: domain.RunModeProduction, GroupTemplate: true}, {Mode: domain.RunModeAll, AllowedProjects: map[string]bool{"other": true}}, {Mode: domain.RunModeAll, AllowedSuppliers: map[string]bool{}}, {Mode: domain.RunModeAll, From: time.Date(2026, 9, 28, 12, 0, 0, 1, time.UTC)}} {
snapshot, e := target.LoadWorkforceStats(ctx, "workforce-test", q)
if e != nil {
t.Fatal(e)
}
want := workforcestats.Build(snapshot, q)
got, used, e := target.computeWorkforceFactReport(ctx, "workforce-test", q, h)
// Existing metric-only loaders differ on nil versus empty slices;
// neither represents a detail/page item. Keep all nonempty data exact.
if len(got.Details) == 0 && len(want.Details) == 0 {
got.Details, want.Details = nil, nil
}
if len(got.Items) == 0 && len(want.Items) == 0 {
got.Items, want.Items = nil, nil
}
if e != nil || !used || !reflect.DeepEqual(got, want) {
t.Fatalf("resumed report changed: %v used=%t\ngot=%+v\nwant=%+v", e, used, got, want)
}
}
}
func TestWorkforceColdFactResumeLegacyEpochZeroLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
source, target, replica := newLocalReplica(t)
ctx := t.Context()
seedWorkforceStats(t, source)
finishLocalReplica(t, replica)
l, ok, err := target.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(ok, err)
}
if err = target.rebuildWorkforceFacts(ctx, l, "workforce-test", 0); err != nil {
t.Fatal(err)
}
unpublishWorkforceFixture(t, target, "workforce-test")
c := target.db.Collection(workforceFactCollection)
var first bson.M
if err = c.FindOne(ctx, bson.M{"organizationId": "workforce-test"}).Decode(&first); err != nil {
t.Fatal(err)
}
if _, err = c.UpdateOne(ctx, bson.M{"_id": first["_id"]}, bson.M{"$set": bson.M{"fromEpoch": int64(0)}}); err != nil {
t.Fatal(err)
}
if err = target.rebuildWorkforceFacts(ctx, l, "workforce-test", 0); err != nil {
t.Fatal(err)
}
h, exists, err := target.workforceHead(ctx, "workforce-test")
if err != nil || !exists {
t.Fatal(exists, err)
}
n, err := c.CountDocuments(ctx, workforceVersionFilter("workforce-test", h.FactEpoch))
if err != nil || n != int64(h.Rows) {
t.Fatal("legacy epoch-zero facts duplicated or lost", n, h.Rows, err)
}
if err = c.FindOne(ctx, bson.M{"_id": first["_id"], "fromEpoch": int64(0)}).Err(); err != nil {
t.Fatal("unchanged legacy fact rewritten", err)
}
}
func TestWorkforceColdFactResumeSameScaleLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("owned synthetic replica required")
}
count := 500
if os.Getenv("QUERY_WORKFORCE_RESUME_SCALE") == "1" {
count = 35000
}
_, target, _ := newLocalReplica(t)
ctx := t.Context()
org := "synthetic-resume-scale"
at := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
seconds := 1.0
events, tasks := make([]any, 0, count), make([]any, 0, count)
project := strings.Repeat("synthetic-project-", 4)
for i := 0; i < count; i++ {
id := fmt.Sprintf("task-%06d", i)
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("event-%06d", i), OrganizationID: org, ProjectID: project, TaskID: id, UserID: "actor", SupplierID: "supplier", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, TemplateVersion: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: at.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": project, "supplierId": "supplier", "runId": "run"})
}
for name, docs := range map[string][]any{"annotationWorklogEvents": events, "annotationTasks": tasks} {
if _, err := target.db.Collection(name).InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
}
for name, doc := range map[string]any{
"runs": bson.M{"_id": "run", "id": "run", "organizationId": org, "mode": "production"},
"users": domain.UserRecord{ID: "actor", OrganizationID: org, Name: strings.Repeat("合成用户", 16), SupplierID: "supplier", Role: domain.UserRoleAnnotator},
"suppliers": domain.SupplierRecord{ID: "supplier", OrganizationID: org, Name: "synthetic supplier", Status: "active"},
} {
if _, err := target.db.Collection(name).InsertOne(ctx, doc); err != nil {
t.Fatal(err)
}
}
l, ok, err := target.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(ok, err)
}
// Fixture creation is not a throughput claim: materialize the exact rows with
// pacing disabled only during fixture setup, then leave 89.4% durable at the
// pre-publication boundary seen in the real failed build.
target.queryBuildWrites = nil
if err = target.rebuildWorkforceFacts(ctx, l, org, 0); err != nil {
t.Fatal(err)
}
h, exists, err := target.workforceHead(ctx, org)
if err != nil || !exists || h.Rows != count*4 {
t.Fatal("representative fact count", h.Rows, exists, err)
}
unpublishWorkforceFixture(t, target, org)
c := target.db.Collection(workforceFactCollection)
missing := h.Rows - h.Rows*894/1000
cur, err := c.Find(ctx, bson.M{"organizationId": org}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(missing)))
if err != nil {
t.Fatal(err)
}
var ids []bson.M
if err = cur.All(ctx, &ids); err != nil {
t.Fatal(err)
}
cur.Close(ctx)
for start := 0; start < len(ids); start += 1000 {
v := bson.A{}
for _, id := range ids[start:min(start+1000, len(ids))] {
v = append(v, id["_id"])
}
if _, err = c.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": v}}); err != nil {
t.Fatal(err)
}
}
target.queryBuildWrites = target.newQueryBuildWriteGate()
if target.queryBuildWrites.limiter.Limit() != rate.Limit(512<<10) {
t.Fatal("write pacing changed")
}
bounded, stop := context.WithTimeout(ctx, 5*time.Minute)
defer stop()
started := time.Now()
if err = target.rebuildWorkforceFacts(bounded, l, org, 0); err != nil {
t.Fatal("resumed build failed within unchanged 5m budget", err)
}
elapsed := time.Since(started)
h, exists, err = target.workforceHead(ctx, org)
if err != nil || !exists || h.Rows != count*4 {
t.Fatal("partial publication", h.Rows, exists, err)
}
n, err := c.CountDocuments(ctx, workforceVersionFilter(org, h.FactEpoch))
if err != nil || n != int64(h.Rows) {
t.Fatal("active row parity", n, h.Rows, err)
}
q := workforcestats.Query{Mode: domain.RunModeAll, From: at, To: at.Add(24 * time.Hour)}
report, used, err := target.computeWorkforceFactReport(ctx, org, q, h)
if err != nil || !used || report.Totals.FirstSubmittedTasks != int64(count) || report.Totals.FirstVideoSeconds != float64(count) {
t.Fatal("same-scale result parity", used, err, report.Totals)
}
t.Logf("synthetic events=%d facts=%d priorConfirmed=%d missing=%d resumedElapsed=%s writeBytesPerSecond=%d timeout=5m originalMongoReads=0 productionCapacityAccepted=false", count, h.Rows, h.Rows-missing, missing, elapsed, 512<<10)
}