⚡ Reduce buffer batch and display fallback reads

This commit is contained in:
d-robotics 2026-10-28 08:55:50 -07:00
parent a8c1f0cf24
commit 8cdd2d79eb
6 changed files with 163 additions and 13 deletions

View File

@ -6,3 +6,11 @@ Base: codex/reviewer-entry-guards @ 80bdcee9, deployed version selected by user;
Scope: continuation and work-buffer polling, review pool display reads, compatibility and load validation.
Out of scope: production configuration, migrations and release without separate authorization.
Risk: mixed-version response compatibility, permissions, in-flight workflow recovery and database load.
## 2026-10-28 additional read optimization
- Buffer ledger validation now deduplicates active batch IDs and fetches only scope, workflow and stage fields in one Mongo query. Missing references are cached only within the ledger attempt; exceptional slots use the same projected fallback. Capacity and CAS logic are unchanged.
- Display enrichment skips full annotation-task reads when capture/payload has already resolved the name. Legacy task fallback and request-local reuse remain covered.
- Local isolated Mongo regression passed, including deduplicated query count, foreign-organization rejection, missing batch handling, stable poll ledger version and capacity update.
- Targeted API regressions passed: continuation, annotation buffer, display cache and legacy display fallback. The varied deterministic fixture keeps the same task count; legacy/summary gzip sizes were 12,579/331 bytes (97.4% reduction). This is synthetic response evidence, not production capacity acceptance.
- Remaining: frontend summary integration and task projection, broader display batching, exact pool query measurements and the approved 81-user repeated soak comparison. No push, PR or deployment performed.

View File

@ -1,9 +1,12 @@
package api
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"strings"
"testing"
"droboticflow/internal/domain"
@ -85,7 +88,13 @@ func TestAnnotationBufferSummaryIsExplicitAndKeepsStatus(t *testing.T) {
item.Subject.Payload = map[string]any{}
}
item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "completed", "phase": "annotate"}
item.Subject.Payload["largeFixture"] = strings.Repeat("fixture-", 8000)
// Varied deterministic values avoid measuring only the compression of a
// repeated string. Keep the same task count in both response modes.
fixture := make([]map[string]any, 256)
for i := range fixture {
fixture[i] = map[string]any{"id": fmt.Sprintf("slice-%d", i), "digest": fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixture-%d", i)))), "startFrame": i * 30, "endFrame": (i + 1) * 30}
}
item.Subject.Payload["largeFixture"] = fixture
if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil {
t.Fatal(err)
}
@ -116,6 +125,26 @@ func TestAnnotationBufferSummaryIsExplicitAndKeepsStatus(t *testing.T) {
if row["id"] != item.ID || row["sliceStatus"] != "completed" || row["slicePhase"] != "annotate" {
t.Fatalf("summary lost state: %v", row)
}
if len(legacy["items"].([]any)) != len(rows) || legacy["occupied"] != summary["occupied"] || legacy["capacity"] != summary["capacity"] {
t.Fatal("summary changed buffer membership or capacity")
}
compressedSize := func(body []byte) int {
t.Helper()
var out bytes.Buffer
writer := gzip.NewWriter(&out)
if _, err := writer.Write(body); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
return out.Len()
}
oldGzip, compactGzip := compressedSize(old.Body.Bytes()), compressedSize(compact.Body.Bytes())
t.Logf("buffer bytes raw=%d/%d gzip=%d/%d (legacy/summary)", old.Body.Len(), compact.Body.Len(), oldGzip, compactGzip)
if compactGzip*2 > oldGzip {
t.Fatalf("summary gzip reduction below 50%%: %d vs %d", compactGzip, oldGzip)
}
if compact.Body.Len()*2 >= old.Body.Len() {
t.Fatalf("summary failed to halve fixture bytes: %d vs %d", compact.Body.Len(), old.Body.Len())
}

View File

@ -184,13 +184,18 @@ func (resolver *humanDisplayResolver) enrichItem(ctx context.Context, item *doma
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(item.Subject.ItemKey))
humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values)
}
task, err := resolver.annotationTask(ctx, item.LegacyTaskID)
if err != nil {
return err
}
if task != nil && len(values) == 0 {
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.ArtifactID))
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.SourceVideo))
// Task artifacts are a legacy fallback, not another source to merge with
// an already resolved capture. Avoid fetching the full annotation document
// (including its workspace) when its fields cannot affect the display.
if len(values) == 0 {
task, err := resolver.annotationTask(ctx, item.LegacyTaskID)
if err != nil {
return err
}
if task != nil {
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.ArtifactID))
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.SourceVideo))
}
}
if run != nil && len(values) == 0 {
values = append(values, domain.CaptureDirectoriesFromRun(*run)...)

View File

@ -53,3 +53,38 @@ func TestHumanDisplayResolverPrefersCaptureRecordDirectoryOverFilePrefix(t *test
t.Fatalf("display context = %#v", item.Display)
}
}
// Instrument the real fallback store path rather than just checking JSON size.
type displayTaskReadStore struct {
store.Store
reads int
}
func (s *displayTaskReadStore) GetAnnotationTask(ctx context.Context, id string) (*domain.AnnotationTaskRecord, error) {
s.reads++
return s.Store.GetAnnotationTask(ctx, id)
}
func TestHumanDisplayResolverReadsTaskOnlyForMissingCapture(t *testing.T) {
ctx := context.Background()
data := &displayTaskReadStore{Store: store.NewMemoryStore()}
if err := data.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "fallback", ArtifactID: "run-a-blurred_Orbbec_Ego_CAPTURE_0001_camera_left_part0001.mp4"}); err != nil {
t.Fatal(err)
}
resolver := newHumanDisplayResolver(&server{store: data})
item := domain.HumanItemRecord{LegacyTaskID: "fallback", Subject: domain.HumanSubjectRef{ItemKey: "Orbbec_Ego_DEVICE_20260829_camera_left_part0001.mp4"}}
if err := resolver.enrichItem(ctx, &item); err != nil {
t.Fatal(err)
}
if data.reads != 0 || item.Display == nil || len(item.Display.CaptureDirectories) == 0 {
t.Fatalf("resolved capture fetched fallback: %d %+v", data.reads, item.Display)
}
item.Subject = domain.HumanSubjectRef{}
for i := 0; i < 2; i++ {
if err := resolver.enrichItem(ctx, &item); err != nil {
t.Fatal(err)
}
if data.reads != 1 || len(item.Display.CaptureDirectories) != 1 || item.Display.CaptureDirectories[0] != "Orbbec_Ego_CAPTURE_0001" {
t.Fatalf("legacy fallback/cache changed: %d %+v", data.reads, item.Display)
}
}
}

View File

@ -88,15 +88,24 @@ func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project st
return old, err
}
occupiedIDs := map[string]bool{}
batches := map[string]*domain.HumanBatchRecord{}
batchIDs := make([]string, 0, len(items))
for _, item := range items {
batchIDs = append(batchIDs, item.BatchID)
}
batches, err := s.annotationBufferBatches(ctx, batchIDs)
if err != nil {
return old, err
}
trusted := func(item domain.HumanItemRecord) (bool, error) {
batch, ok := batches[item.BatchID]
if !ok {
var e error
batch, e = s.GetHumanBatch(ctx, item.BatchID)
// A ledger slot may reference an item outside the active scan.
// Resolve its batch with the same projection and request-local cache.
missing, e := s.annotationBufferBatches(ctx, []string{item.BatchID})
if e != nil {
return false, e
}
batch = missing[item.BatchID]
batches[item.BatchID] = batch
}
return batch != nil && annotationBufferBatchMatches(item, *batch), nil
@ -189,3 +198,36 @@ func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project st
func (s *MongoStore) AnnotationWorkBuffer(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
return s.annotationBuffer(ctx, org, user, project, capacity, nil)
}
// annotationBufferBatches reads only fields used by annotationBufferBatchMatches.
// Cache missing IDs too, so corrupt historical references do not cause N+1 reads.
// This cache lives for one ledger attempt; permission/workflow changes remain live.
func (s *MongoStore) annotationBufferBatches(ctx context.Context, ids []string) (map[string]*domain.HumanBatchRecord, error) {
result := make(map[string]*domain.HumanBatchRecord, len(ids))
unique := make([]string, 0, len(ids))
for _, id := range ids {
if _, exists := result[id]; exists {
continue
}
result[id] = nil
unique = append(unique, id)
}
if len(unique) == 0 {
return result, nil
}
cursor, err := s.humanBatches.Find(ctx, bson.M{"id": bson.M{"$in": unique}}, options.Find().SetProjection(bson.M{
"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "processingProfile": 1, "stages": 1,
}))
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var batch domain.HumanBatchRecord
if err := cursor.Decode(&batch); err != nil {
return nil, err
}
result[batch.ID] = &batch
}
return result, cursor.Err()
}

View File

@ -5,9 +5,12 @@ import (
"droboticflow/internal/domain"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"strings"
"sync/atomic"
"testing"
"time"
)
@ -20,7 +23,13 @@ func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) {
}
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
var batchFinds atomic.Int64
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "humanBatches" {
batchFinds.Add(1)
}
}}
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
@ -37,6 +46,28 @@ func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) {
org, user, project := "buffer-store-org", "buffer-store-user", "buffer-store-project"
batch := domain.HumanBatchRecord{ID: "buffer-store-batch", OrganizationID: org, ProjectID: project, WorkflowID: "buffer-store-workflow", PluginID: "annotation-qa", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()}
must(s.UpsertHumanBatch(ctx, batch))
// An unknown large field represents historical metadata: the buffer must not
// materialize it just to validate the batch's scope and workflow.
_, err = db.Collection("humanBatches").UpdateOne(ctx, bson.M{"id": batch.ID}, bson.M{"$set": bson.M{"legacyMetadata": strings.Repeat("x", 1024*1024)}})
must(err)
other := batch
other.ID = "other-batch"
other.OrganizationID = "other-org"
must(s.UpsertHumanBatch(ctx, other))
before := batchFinds.Load()
projected, err := s.annotationBufferBatches(ctx, []string{batch.ID, batch.ID, other.ID, "missing"})
must(err)
if batchFinds.Load()-before != 1 || len(projected) != 3 || projected["missing"] != nil {
t.Fatalf("expected one deduplicated query including missing references: %+v", projected)
}
probe := domain.HumanItemRecord{OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID}
if !annotationBufferBatchMatches(probe, *projected[batch.ID]) || annotationBufferBatchMatches(probe, *projected[other.ID]) {
t.Fatal("projected batches changed scope decisions")
}
if projected[batch.ID].PluginID != "" {
t.Fatal("batch query returned unneeded fields")
}
item := func(id, stage, assignee, author string, status domain.HumanItemStatus) domain.HumanItemRecord {
return domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: "task-" + id, Subject: domain.HumanSubjectRef{ItemKey: id}, CurrentStageID: stage, Status: status, AssignedUserID: assignee, LatestAuthorUserID: author, Version: 1}
}