forked from longtao.wu/droboticflow
⚡ Reduce buffer batch and display fallback reads
This commit is contained in:
parent
a8c1f0cf24
commit
8cdd2d79eb
@ -6,3 +6,11 @@ Base: codex/reviewer-entry-guards @ 80bdcee9, deployed version selected by user;
|
||||
Scope: continuation and work-buffer polling, review pool display reads, compatibility and load validation.
|
||||
Out of scope: production configuration, migrations and release without separate authorization.
|
||||
Risk: mixed-version response compatibility, permissions, in-flight workflow recovery and database load.
|
||||
|
||||
## 2026-10-28 additional read optimization
|
||||
|
||||
- Buffer ledger validation now deduplicates active batch IDs and fetches only scope, workflow and stage fields in one Mongo query. Missing references are cached only within the ledger attempt; exceptional slots use the same projected fallback. Capacity and CAS logic are unchanged.
|
||||
- Display enrichment skips full annotation-task reads when capture/payload has already resolved the name. Legacy task fallback and request-local reuse remain covered.
|
||||
- Local isolated Mongo regression passed, including deduplicated query count, foreign-organization rejection, missing batch handling, stable poll ledger version and capacity update.
|
||||
- Targeted API regressions passed: continuation, annotation buffer, display cache and legacy display fallback. The varied deterministic fixture keeps the same task count; legacy/summary gzip sizes were 12,579/331 bytes (97.4% reduction). This is synthetic response evidence, not production capacity acceptance.
|
||||
- Remaining: frontend summary integration and task projection, broader display batching, exact pool query measurements and the approved 81-user repeated soak comparison. No push, PR or deployment performed.
|
||||
|
||||
@ -1,9 +1,12 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -85,7 +88,13 @@ func TestAnnotationBufferSummaryIsExplicitAndKeepsStatus(t *testing.T) {
|
||||
item.Subject.Payload = map[string]any{}
|
||||
}
|
||||
item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "completed", "phase": "annotate"}
|
||||
item.Subject.Payload["largeFixture"] = strings.Repeat("fixture-", 8000)
|
||||
// Varied deterministic values avoid measuring only the compression of a
|
||||
// repeated string. Keep the same task count in both response modes.
|
||||
fixture := make([]map[string]any, 256)
|
||||
for i := range fixture {
|
||||
fixture[i] = map[string]any{"id": fmt.Sprintf("slice-%d", i), "digest": fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("fixture-%d", i)))), "startFrame": i * 30, "endFrame": (i + 1) * 30}
|
||||
}
|
||||
item.Subject.Payload["largeFixture"] = fixture
|
||||
if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -116,6 +125,26 @@ func TestAnnotationBufferSummaryIsExplicitAndKeepsStatus(t *testing.T) {
|
||||
if row["id"] != item.ID || row["sliceStatus"] != "completed" || row["slicePhase"] != "annotate" {
|
||||
t.Fatalf("summary lost state: %v", row)
|
||||
}
|
||||
if len(legacy["items"].([]any)) != len(rows) || legacy["occupied"] != summary["occupied"] || legacy["capacity"] != summary["capacity"] {
|
||||
t.Fatal("summary changed buffer membership or capacity")
|
||||
}
|
||||
compressedSize := func(body []byte) int {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
writer := gzip.NewWriter(&out)
|
||||
if _, err := writer.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Len()
|
||||
}
|
||||
oldGzip, compactGzip := compressedSize(old.Body.Bytes()), compressedSize(compact.Body.Bytes())
|
||||
t.Logf("buffer bytes raw=%d/%d gzip=%d/%d (legacy/summary)", old.Body.Len(), compact.Body.Len(), oldGzip, compactGzip)
|
||||
if compactGzip*2 > oldGzip {
|
||||
t.Fatalf("summary gzip reduction below 50%%: %d vs %d", compactGzip, oldGzip)
|
||||
}
|
||||
if compact.Body.Len()*2 >= old.Body.Len() {
|
||||
t.Fatalf("summary failed to halve fixture bytes: %d vs %d", compact.Body.Len(), old.Body.Len())
|
||||
}
|
||||
|
||||
@ -184,13 +184,18 @@ func (resolver *humanDisplayResolver) enrichItem(ctx context.Context, item *doma
|
||||
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(item.Subject.ItemKey))
|
||||
humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values)
|
||||
}
|
||||
task, err := resolver.annotationTask(ctx, item.LegacyTaskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task != nil && len(values) == 0 {
|
||||
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.ArtifactID))
|
||||
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.SourceVideo))
|
||||
// Task artifacts are a legacy fallback, not another source to merge with
|
||||
// an already resolved capture. Avoid fetching the full annotation document
|
||||
// (including its workspace) when its fields cannot affect the display.
|
||||
if len(values) == 0 {
|
||||
task, err := resolver.annotationTask(ctx, item.LegacyTaskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task != nil {
|
||||
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.ArtifactID))
|
||||
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(task.SourceVideo))
|
||||
}
|
||||
}
|
||||
if run != nil && len(values) == 0 {
|
||||
values = append(values, domain.CaptureDirectoriesFromRun(*run)...)
|
||||
|
||||
@ -53,3 +53,38 @@ func TestHumanDisplayResolverPrefersCaptureRecordDirectoryOverFilePrefix(t *test
|
||||
t.Fatalf("display context = %#v", item.Display)
|
||||
}
|
||||
}
|
||||
|
||||
// Instrument the real fallback store path rather than just checking JSON size.
|
||||
type displayTaskReadStore struct {
|
||||
store.Store
|
||||
reads int
|
||||
}
|
||||
|
||||
func (s *displayTaskReadStore) GetAnnotationTask(ctx context.Context, id string) (*domain.AnnotationTaskRecord, error) {
|
||||
s.reads++
|
||||
return s.Store.GetAnnotationTask(ctx, id)
|
||||
}
|
||||
func TestHumanDisplayResolverReadsTaskOnlyForMissingCapture(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := &displayTaskReadStore{Store: store.NewMemoryStore()}
|
||||
if err := data.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "fallback", ArtifactID: "run-a-blurred_Orbbec_Ego_CAPTURE_0001_camera_left_part0001.mp4"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolver := newHumanDisplayResolver(&server{store: data})
|
||||
item := domain.HumanItemRecord{LegacyTaskID: "fallback", Subject: domain.HumanSubjectRef{ItemKey: "Orbbec_Ego_DEVICE_20260829_camera_left_part0001.mp4"}}
|
||||
if err := resolver.enrichItem(ctx, &item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data.reads != 0 || item.Display == nil || len(item.Display.CaptureDirectories) == 0 {
|
||||
t.Fatalf("resolved capture fetched fallback: %d %+v", data.reads, item.Display)
|
||||
}
|
||||
item.Subject = domain.HumanSubjectRef{}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := resolver.enrichItem(ctx, &item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data.reads != 1 || len(item.Display.CaptureDirectories) != 1 || item.Display.CaptureDirectories[0] != "Orbbec_Ego_CAPTURE_0001" {
|
||||
t.Fatalf("legacy fallback/cache changed: %d %+v", data.reads, item.Display)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -88,15 +88,24 @@ func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project st
|
||||
return old, err
|
||||
}
|
||||
occupiedIDs := map[string]bool{}
|
||||
batches := map[string]*domain.HumanBatchRecord{}
|
||||
batchIDs := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
batchIDs = append(batchIDs, item.BatchID)
|
||||
}
|
||||
batches, err := s.annotationBufferBatches(ctx, batchIDs)
|
||||
if err != nil {
|
||||
return old, err
|
||||
}
|
||||
trusted := func(item domain.HumanItemRecord) (bool, error) {
|
||||
batch, ok := batches[item.BatchID]
|
||||
if !ok {
|
||||
var e error
|
||||
batch, e = s.GetHumanBatch(ctx, item.BatchID)
|
||||
// A ledger slot may reference an item outside the active scan.
|
||||
// Resolve its batch with the same projection and request-local cache.
|
||||
missing, e := s.annotationBufferBatches(ctx, []string{item.BatchID})
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
batch = missing[item.BatchID]
|
||||
batches[item.BatchID] = batch
|
||||
}
|
||||
return batch != nil && annotationBufferBatchMatches(item, *batch), nil
|
||||
@ -189,3 +198,36 @@ func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project st
|
||||
func (s *MongoStore) AnnotationWorkBuffer(ctx context.Context, org, user, project string, capacity int) (AnnotationWorkBuffer, error) {
|
||||
return s.annotationBuffer(ctx, org, user, project, capacity, nil)
|
||||
}
|
||||
|
||||
// annotationBufferBatches reads only fields used by annotationBufferBatchMatches.
|
||||
// Cache missing IDs too, so corrupt historical references do not cause N+1 reads.
|
||||
// This cache lives for one ledger attempt; permission/workflow changes remain live.
|
||||
func (s *MongoStore) annotationBufferBatches(ctx context.Context, ids []string) (map[string]*domain.HumanBatchRecord, error) {
|
||||
result := make(map[string]*domain.HumanBatchRecord, len(ids))
|
||||
unique := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if _, exists := result[id]; exists {
|
||||
continue
|
||||
}
|
||||
result[id] = nil
|
||||
unique = append(unique, id)
|
||||
}
|
||||
if len(unique) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
cursor, err := s.humanBatches.Find(ctx, bson.M{"id": bson.M{"$in": unique}}, options.Find().SetProjection(bson.M{
|
||||
"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "processingProfile": 1, "stages": 1,
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
for cursor.Next(ctx) {
|
||||
var batch domain.HumanBatchRecord
|
||||
if err := cursor.Decode(&batch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[batch.ID] = &batch
|
||||
}
|
||||
return result, cursor.Err()
|
||||
}
|
||||
|
||||
@ -5,9 +5,12 @@ import (
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -20,7 +23,13 @@ func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
|
||||
var batchFinds atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == "humanBatches" {
|
||||
batchFinds.Add(1)
|
||||
}
|
||||
}}
|
||||
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetMonitor(monitor))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -37,6 +46,28 @@ func TestMongoAnnotationWorkBufferPollIsReadOnly(t *testing.T) {
|
||||
org, user, project := "buffer-store-org", "buffer-store-user", "buffer-store-project"
|
||||
batch := domain.HumanBatchRecord{ID: "buffer-store-batch", OrganizationID: org, ProjectID: project, WorkflowID: "buffer-store-workflow", PluginID: "annotation-qa", Status: domain.HumanBatchActive, ProcessingProfile: domain.ProcessingProfileRawHuman, Stages: domain.DefaultFanyuV58HumanStages()}
|
||||
must(s.UpsertHumanBatch(ctx, batch))
|
||||
// An unknown large field represents historical metadata: the buffer must not
|
||||
// materialize it just to validate the batch's scope and workflow.
|
||||
_, err = db.Collection("humanBatches").UpdateOne(ctx, bson.M{"id": batch.ID}, bson.M{"$set": bson.M{"legacyMetadata": strings.Repeat("x", 1024*1024)}})
|
||||
must(err)
|
||||
other := batch
|
||||
other.ID = "other-batch"
|
||||
other.OrganizationID = "other-org"
|
||||
must(s.UpsertHumanBatch(ctx, other))
|
||||
before := batchFinds.Load()
|
||||
projected, err := s.annotationBufferBatches(ctx, []string{batch.ID, batch.ID, other.ID, "missing"})
|
||||
must(err)
|
||||
if batchFinds.Load()-before != 1 || len(projected) != 3 || projected["missing"] != nil {
|
||||
t.Fatalf("expected one deduplicated query including missing references: %+v", projected)
|
||||
}
|
||||
probe := domain.HumanItemRecord{OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID}
|
||||
if !annotationBufferBatchMatches(probe, *projected[batch.ID]) || annotationBufferBatchMatches(probe, *projected[other.ID]) {
|
||||
t.Fatal("projected batches changed scope decisions")
|
||||
}
|
||||
if projected[batch.ID].PluginID != "" {
|
||||
t.Fatal("batch query returned unneeded fields")
|
||||
}
|
||||
|
||||
item := func(id, stage, assignee, author string, status domain.HumanItemStatus) domain.HumanItemRecord {
|
||||
return domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, WorkflowID: batch.WorkflowID, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: "task-" + id, Subject: domain.HumanSubjectRef{ItemKey: id}, CurrentStageID: stage, Status: status, AssignedUserID: assignee, LatestAuthorUserID: author, Version: 1}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user