From 8cdd2d79ebfd5812e06d1a49817774330faed1dc Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 28 Oct 2026 08:55:50 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Reduce=20buffer=20batch=20and=20dis?= =?UTF-8?q?play=20fallback=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/poll-pool-optimization/work.md | 8 ++++ .../annotation_work_buffer_display_test.go | 33 ++++++++++++- internal/api/human_display.go | 19 +++++--- internal/api/human_display_test.go | 35 ++++++++++++++ internal/store/annotation_work_buffer.go | 48 +++++++++++++++++-- .../annotation_work_buffer_mongo_test.go | 33 ++++++++++++- 6 files changed, 163 insertions(+), 13 deletions(-) diff --git a/.scratch/poll-pool-optimization/work.md b/.scratch/poll-pool-optimization/work.md index 225d2584..a4a313ad 100644 --- a/.scratch/poll-pool-optimization/work.md +++ b/.scratch/poll-pool-optimization/work.md @@ -6,3 +6,11 @@ Base: codex/reviewer-entry-guards @ 80bdcee9, deployed version selected by user; Scope: continuation and work-buffer polling, review pool display reads, compatibility and load validation. Out of scope: production configuration, migrations and release without separate authorization. Risk: mixed-version response compatibility, permissions, in-flight workflow recovery and database load. + +## 2026-10-28 additional read optimization + +- Buffer ledger validation now deduplicates active batch IDs and fetches only scope, workflow and stage fields in one Mongo query. Missing references are cached only within the ledger attempt; exceptional slots use the same projected fallback. Capacity and CAS logic are unchanged. +- Display enrichment skips full annotation-task reads when capture/payload has already resolved the name. Legacy task fallback and request-local reuse remain covered. +- Local isolated Mongo regression passed, including deduplicated query count, foreign-organization rejection, missing batch handling, stable poll ledger version and capacity update. +- Targeted API regressions passed: continuation, annotation buffer, display cache and legacy display fallback. The varied deterministic fixture keeps the same task count; legacy/summary gzip sizes were 12,579/331 bytes (97.4% reduction). This is synthetic response evidence, not production capacity acceptance. +- Remaining: frontend summary integration and task projection, broader display batching, exact pool query measurements and the approved 81-user repeated soak comparison. No push, PR or deployment performed. diff --git a/internal/api/annotation_work_buffer_display_test.go b/internal/api/annotation_work_buffer_display_test.go index 9f66ff4c..abae6ed6 100644 --- a/internal/api/annotation_work_buffer_display_test.go +++ b/internal/api/annotation_work_buffer_display_test.go @@ -1,9 +1,12 @@ package api import ( + "bytes" + "compress/gzip" + "crypto/sha256" "encoding/json" + "fmt" "net/http" - "strings" "testing" "droboticflow/internal/domain" @@ -85,7 +88,13 @@ func TestAnnotationBufferSummaryIsExplicitAndKeepsStatus(t *testing.T) { item.Subject.Payload = map[string]any{} } item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "completed", "phase": "annotate"} - item.Subject.Payload["largeFixture"] = strings.Repeat("fixture-", 8000) + // Varied deterministic values avoid measuring only the compression of a + // repeated string. Keep the same task count in both response modes. + fixture := make([]map[string]any, 256) + for i := range fixture { + fixture[i] = map[string]any{"id": fmt.Sprintf("slice-%d", i), "digest": fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixture-%d", i)))), "startFrame": i * 30, "endFrame": (i + 1) * 30} + } + item.Subject.Payload["largeFixture"] = fixture if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil { t.Fatal(err) } @@ -116,6 +125,26 @@ func TestAnnotationBufferSummaryIsExplicitAndKeepsStatus(t *testing.T) { if row["id"] != item.ID || row["sliceStatus"] != "completed" || row["slicePhase"] != "annotate" { t.Fatalf("summary lost state: %v", row) } + if len(legacy["items"].([]any)) != len(rows) || legacy["occupied"] != summary["occupied"] || legacy["capacity"] != summary["capacity"] { + t.Fatal("summary changed buffer membership or capacity") + } + compressedSize := func(body []byte) int { + t.Helper() + var out bytes.Buffer + writer := gzip.NewWriter(&out) + if _, err := writer.Write(body); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return out.Len() + } + oldGzip, compactGzip := compressedSize(old.Body.Bytes()), compressedSize(compact.Body.Bytes()) + t.Logf("buffer bytes raw=%d/%d gzip=%d/%d (legacy/summary)", old.Body.Len(), compact.Body.Len(), oldGzip, compactGzip) + if compactGzip*2 > oldGzip { + t.Fatalf("summary gzip reduction below 50%%: %d vs %d", compactGzip, oldGzip) + } if compact.Body.Len()*2 >= old.Body.Len() { t.Fatalf("summary failed to halve fixture bytes: %d vs %d", compact.Body.Len(), old.Body.Len()) } diff --git a/internal/api/human_display.go b/internal/api/human_display.go index 0e9f386c..fa894f7d 100644 --- a/internal/api/human_display.go +++ b/internal/api/human_display.go @@ -184,13 +184,18 @@ func (resolver *humanDisplayResolver) enrichItem(ctx context.Context, item *doma appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(item.Subject.ItemKey)) humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values) } - task, err := resolver.annotationTask(ctx, item.LegacyTaskID) - if err != nil { - return err - } - if task != nil && len(values) == 0 { - appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.ArtifactID)) - appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.SourceVideo)) + // Task artifacts are a legacy fallback, not another source to merge with + // an already resolved capture. Avoid fetching the full annotation document + // (including its workspace) when its fields cannot affect the display. + if len(values) == 0 { + task, err := resolver.annotationTask(ctx, item.LegacyTaskID) + if err != nil { + return err + } + if task != nil { + appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.ArtifactID)) + appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.SourceVideo)) + } } if run != nil && len(values) == 0 { values = append(values, domain.CaptureDirectoriesFromRun(*run)...) diff --git a/internal/api/human_display_test.go b/internal/api/human_display_test.go index 436f4715..4d7e0dec 100644 --- a/internal/api/human_display_test.go +++ b/internal/api/human_display_test.go @@ -53,3 +53,38 @@ func TestHumanDisplayResolverPrefersCaptureRecordDirectoryOverFilePrefix(t *test t.Fatalf("display context = %#v", item.Display) } } + +// Instrument the real fallback store path rather than just checking JSON size. +type displayTaskReadStore struct { + store.Store + reads int +} + +func (s *displayTaskReadStore) GetAnnotationTask(ctx context.Context, id string) (*domain.AnnotationTaskRecord, error) { + s.reads++ + return s.Store.GetAnnotationTask(ctx, id) +} +func TestHumanDisplayResolverReadsTaskOnlyForMissingCapture(t *testing.T) { + ctx := context.Background() + data := &displayTaskReadStore{Store: store.NewMemoryStore()} + if err := data.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "fallback", ArtifactID: "run-a-blurred_Orbbec_Ego_CAPTURE_0001_camera_left_part0001.mp4"}); err != nil { + t.Fatal(err) + } + resolver := newHumanDisplayResolver(&server{store: data}) + item := domain.HumanItemRecord{LegacyTaskID: "fallback", Subject: domain.HumanSubjectRef{ItemKey: "Orbbec_Ego_DEVICE_20260829_camera_left_part0001.mp4"}} + if err := resolver.enrichItem(ctx, &item); err != nil { + t.Fatal(err) + } + if data.reads != 0 || item.Display == nil || len(item.Display.CaptureDirectories) == 0 { + t.Fatalf("resolved capture fetched fallback: %d %+v", data.reads, item.Display) + } + item.Subject = domain.HumanSubjectRef{} + for i := 0; i < 2; i++ { + if err := resolver.enrichItem(ctx, &item); err != nil { + t.Fatal(err) + } + if data.reads != 1 || len(item.Display.CaptureDirectories) != 1 || item.Display.CaptureDirectories[0] != "Orbbec_Ego_CAPTURE_0001" { + t.Fatalf("legacy fallback/cache changed: %d %+v", data.reads, item.Display) + } + } +} diff --git a/internal/store/annotation_work_buffer.go b/internal/store/annotation_work_buffer.go index 968aa743..34cb7eac 100644 --- a/internal/store/annotation_work_buffer.go +++ b/internal/store/annotation_work_buffer.go @@ -88,15 +88,24 @@ func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project st return old, err } occupiedIDs := map[string]bool{} - batches := map[string]*domain.HumanBatchRecord{} + batchIDs := make([]string, 0, len(items)) + for _, item := range items { + batchIDs = append(batchIDs, item.BatchID) + } + batches, err := s.annotationBufferBatches(ctx, batchIDs) + if err != nil { + return old, err + } trusted := func(item domain.HumanItemRecord) (bool, error) { batch, ok := batches[item.BatchID] if !ok { - var e error - batch, e = s.GetHumanBatch(ctx, item.BatchID) + // A ledger slot may reference an item outside the active scan. + // Resolve its batch with the same projection and request-local cache. + missing, e := s.annotationBufferBatches(ctx, []string{item.BatchID}) if e != nil { return false, e } + batch = missing[item.BatchID] batches[item.BatchID] = batch } return batch != nil && annotationBufferBatchMatches(item, *batch), nil @@ -189,3 +198,36 @@ func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project st func (s *MongoStore) AnnotationWorkBuffer(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) { return s.annotationBuffer(ctx, org, user, project, capacity, nil) } + +// annotationBufferBatches reads only fields used by annotationBufferBatchMatches. +// Cache missing IDs too, so corrupt historical references do not cause N+1 reads. +// This cache lives for one ledger attempt; permission/workflow changes remain live. +func (s *MongoStore) annotationBufferBatches(ctx context.Context, ids []string) (map[string]*domain.HumanBatchRecord, error) { + result := make(map[string]*domain.HumanBatchRecord, len(ids)) + unique := make([]string, 0, len(ids)) + for _, id := range ids { + if _, exists := result[id]; exists { + continue + } + result[id] = nil + unique = append(unique, id) + } + if len(unique) == 0 { + return result, nil + } + cursor, err := s.humanBatches.Find(ctx, bson.M{"id": bson.M{"$in": unique}}, options.Find().SetProjection(bson.M{ + "_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "processingProfile": 1, "stages": 1, + })) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + for cursor.Next(ctx) { + var batch domain.HumanBatchRecord + if err := cursor.Decode(&batch); err != nil { + return nil, err + } + result[batch.ID] = &batch + } + return result, cursor.Err() +} diff --git a/internal/store/annotation_work_buffer_mongo_test.go b/internal/store/annotation_work_buffer_mongo_test.go index 26347dad..365d0c65 100644 --- a/internal/store/annotation_work_buffer_mongo_test.go +++ b/internal/store/annotation_work_buffer_mongo_test.go @@ -5,9 +5,12 @@ import ( "droboticflow/internal/domain" "fmt" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "os" + "strings" + "sync/atomic" "testing" "time" ) @@ -20,7 +23,13 @@ func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() - c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true")) + var batchFinds atomic.Int64 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "humanBatches" { + batchFinds.Add(1) + } + }} + c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetMonitor(monitor)) if err != nil { t.Fatal(err) } @@ -37,6 +46,28 @@ func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) { org, user, project := "buffer-store-org", "buffer-store-user", "buffer-store-project" batch := domain.HumanBatchRecord{ID: "buffer-store-batch", OrganizationID: org, ProjectID: project, WorkflowID: "buffer-store-workflow", PluginID: "annotation-qa", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()} must(s.UpsertHumanBatch(ctx, batch)) + // An unknown large field represents historical metadata: the buffer must not + // materialize it just to validate the batch's scope and workflow. + _, err = db.Collection("humanBatches").UpdateOne(ctx, bson.M{"id": batch.ID}, bson.M{"$set": bson.M{"legacyMetadata": strings.Repeat("x", 1024*1024)}}) + must(err) + other := batch + other.ID = "other-batch" + other.OrganizationID = "other-org" + must(s.UpsertHumanBatch(ctx, other)) + before := batchFinds.Load() + projected, err := s.annotationBufferBatches(ctx, []string{batch.ID, batch.ID, other.ID, "missing"}) + must(err) + if batchFinds.Load()-before != 1 || len(projected) != 3 || projected["missing"] != nil { + t.Fatalf("expected one deduplicated query including missing references: %+v", projected) + } + probe := domain.HumanItemRecord{OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID} + if !annotationBufferBatchMatches(probe, *projected[batch.ID]) || annotationBufferBatchMatches(probe, *projected[other.ID]) { + t.Fatal("projected batches changed scope decisions") + } + if projected[batch.ID].PluginID != "" { + t.Fatal("batch query returned unneeded fields") + } + item := func(id, stage, assignee, author string, status domain.HumanItemStatus) domain.HumanItemRecord { return domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: "task-" + id, Subject: domain.HumanSubjectRef{ItemKey: id}, CurrentStageID: stage, Status: status, AssignedUserID: assignee, LatestAuthorUserID: author, Version: 1} }