droboticflow/internal/api/annotation_output_snapshot_scale_test.go

190 lines
7.2 KiB
Go

package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/store"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"net/http"
"os"
"sort"
"sync"
"testing"
"time"
)
func TestAnnotationOutputSnapshotScale(t *testing.T) {
if os.Getenv("RUN_OUTPUT_SNAPSHOT_BENCHMARK") != "1" || os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" {
t.Skip("explicit isolated local timing run")
}
ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second)
defer cancel()
c, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018"))
if e != nil {
t.Fatal(e)
}
db := c.Database(fmt.Sprintf("annotation_output_scale_%d", time.Now().UnixNano()))
defer c.Disconnect(context.Background())
defer db.Drop(context.Background())
f := newCombinedDraftFixture(t)
s := store.NewMongoStore(db)
for _, u := range []domain.UserRecord{f.admin, f.annotator} {
if e = s.UpsertUser(ctx, u); e != nil {
t.Fatal(e)
}
}
if e = s.UpsertProject(ctx, domain.ProjectRecord{ID: "scale", OrganizationID: domain.DefaultOrganizationID, Name: "Synthetic scale project"}); e != nil {
t.Fatal(e)
}
if _, e = s.SaveProductionStatsScope(ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{"scale"}, Revision: 1}, 0); e != nil {
t.Fatal(e)
}
n := 5000
at := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)
collections := map[string][]any{"annotationTasks": {}, "annotationWorklogEvents": {}}
for i := 0; i < n; i++ {
id := fmt.Sprintf("scale-%d", i)
collections["annotationTasks"] = append(collections["annotationTasks"], bson.M{"id": id, "organizationId": domain.DefaultOrganizationID, "projectId": "scale", "sourceIsFullVideo": true, "sourceDurationSeconds": 900, "createdAt": at})
for j := 0; j < 5; j++ {
collections["annotationWorklogEvents"] = append(collections["annotationWorklogEvents"], bson.M{"id": fmt.Sprintf("%s-%d", id, j), "organizationId": domain.DefaultOrganizationID, "projectId": "scale", "taskId": id, "userId": f.annotator.ID, "supplierId": f.annotator.SupplierID, "revision": 1, "submissionKind": "first", "eventType": "combined_submitted", "createdAt": at})
}
}
for name, docs := range collections {
for len(docs) > 0 {
size := 1000
if size > len(docs) {
size = len(docs)
}
if _, e = db.Collection(name).InsertMany(ctx, docs[:size]); e != nil {
t.Fatal(e)
}
docs = docs[size:]
}
}
wrapped := &countingOutputStore{Store: s}
f.handler = NewHandler(wrapped, nil)
start := time.Now()
cold := outputReady(t, f, "/api/dashboard/annotation-output", f.admin)
coldMS := time.Since(start).Milliseconds()
var body struct {
Snapshot struct {
ID string `json:"id"`
} `json:"snapshot"`
Tasks struct {
Total int `json:"total"`
} `json:"tasks"`
}
if e = json.Unmarshal(cold.Body.Bytes(), &body); e != nil || body.Tasks.Total != n {
t.Fatalf("scale setup %d %v", body.Tasks.Total, e)
}
samples := []int64{}
// Four independent handlers share only Mongo, including after a restart.
for instance := 0; instance < 4; instance++ {
proxy := &countingOutputStore{Store: store.NewMongoStore(db)}
if instance == 0 {
proxy = wrapped
}
f.handler = NewHandler(proxy, nil)
for page := 1; page <= 8; page++ {
start := time.Now()
r := f.request("GET", fmt.Sprintf("/api/dashboard/annotation-output?snapshotId=%s&tasksPage=%d", body.Snapshot.ID, page), f.admin, nil)
samples = append(samples, time.Since(start).Milliseconds())
if r.Code != 200 {
t.Fatalf("warm %d", r.Code)
}
}
if instance > 0 && proxy.loads.Load() != 0 {
t.Fatal("another API reloaded source history")
}
}
sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] })
p95 := samples[(len(samples)*95-1)/100]
// Release gate runs this scenario both with race checks and without instrumentation for timing.
if !raceInstrumentationEnabled && p95 > 1000 {
t.Fatalf("warm report p95 %dms exceeds local target", p95)
}
if wrapped.loads.Load() != 1 {
t.Fatalf("%d raw source reads", wrapped.loads.Load())
}
sizes := db.Collection("annotationOutputSnapshotSlots")
count, _ := sizes.CountDocuments(ctx, bson.M{})
var slot struct {
Data []byte `bson:"data"`
}
_ = sizes.FindOne(ctx, bson.M{}).Decode(&slot)
report := map[string]any{"environment": "local isolated Mongo; synthetic metadata; no production performance claim", "parts": n, "historyEvents": n * 5, "apiInstances": 4, "warmRequests": len(samples), "coldMs": coldMS, "warmP50Ms": samples[len(samples)/2], "warmP95Ms": p95, "sourceBuilds": wrapped.loads.Load(), "storedSlots": count, "compressedBytes": len(slot.Data)}
encoded, _ := json.Marshal(report)
t.Log(string(encoded))
t.Run("production-quality-concurrent-warm-reads", func(t *testing.T) {
url := fmt.Sprintf("/api/dashboard/production-quality?from=%s&to=%s&timeZone=UTC", time.Now().UTC().AddDate(0, 0, -1).Format("2006-01-02"), time.Now().UTC().AddDate(0, 0, 1).Format("2006-01-02"))
ready := outputReady(t, f, url, f.admin)
var initial struct {
Snapshot struct {
ID string `json:"id"`
} `json:"snapshot"`
}
if err := json.Unmarshal(ready.Body.Bytes(), &initial); err != nil || initial.Snapshot.ID == "" {
t.Fatal("new report snapshot not ready")
}
url += "&snapshotId=" + initial.Snapshot.ID
readers := make([]*countingOutputStore, 4)
handlers := make([]http.Handler, len(readers))
for i := range readers {
readers[i] = &countingOutputStore{Store: store.NewMongoStore(db)}
handlers[i] = NewHandler(readers[i], nil)
}
const burst = 121
latencies := make(chan int64, burst)
start := make(chan struct{})
var wg sync.WaitGroup
for i := 0; i < burst; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
local := f
local.handler = handlers[i%len(handlers)]
<-start
at := time.Now()
r := local.request("GET", url, local.admin, nil)
latencies <- time.Since(at).Milliseconds()
var result struct {
Totals struct {
Submitted struct {
Count int `json:"count"`
} `json:"submitted"`
} `json:"totals"`
Snapshot struct {
ID string `json:"id"`
} `json:"snapshot"`
}
if err := json.Unmarshal(r.Body.Bytes(), &result); err != nil || r.Code != 200 || result.Totals.Submitted.Count != n || result.Snapshot.ID != initial.Snapshot.ID {
t.Errorf("cohort read code=%d parts=%d snapshotMatches=%t", r.Code, result.Totals.Submitted.Count, result.Snapshot.ID == initial.Snapshot.ID)
}
}(i)
}
close(start)
wg.Wait()
close(latencies)
samples := make([]int64, 0, burst)
for ms := range latencies {
samples = append(samples, ms)
}
sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] })
for _, reader := range readers {
if reader.loads.Load() != 0 {
t.Error("warm report reloaded source history")
}
}
p95 := samples[(len(samples)*95-1)/100]
t.Logf("local synthetic warm cohort burst: requests=%d apiHandlers=4 parts=%d p95_ms=%d max_ms=%d sourceBuilds=0; not full capacity acceptance", burst, n, p95, samples[len(samples)-1])
if !raceInstrumentationEnabled && p95 > 2000 {
t.Errorf("warm concurrent report p95 %dms exceeds 2s target", p95)
}
})
}