droboticflow/internal/api/annotation_source_mongo_load_test.go
d-robotics 94a840c2ad
Some checks are pending
web-ci / checks (push) Waiting to run
⚡ Parallelize project saves while fencing scope mutations
Same-project saves queued behind one project lock. Add default-off record stripes while deletion, authorization and claim operations retain exclusive protection across all stripes. Require a complete barrier rollout before striped activation. Verify local Mongo deletion, revocation, crash recovery and two 100-user HTTP runs with save p95 below 400ms.
2026-10-17 01:32:16 +08:00

283 lines
11 KiB
Go

package api
import (
"bytes"
"context"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sort"
"sync"
"testing"
"time"
)
// Opt-in actual Mongo + HTTP verification. The fixed loopback server and fresh
// database name deliberately cannot be redirected to a company database.
func TestSourceIndex100ActiveOperatorsMongoHTTP(t *testing.T) {
if os.Getenv("FANYU_SOURCE_INDEX_MONGO_LOAD") != "1" {
t.Skip("opt-in isolated local Mongo load test")
}
t.Setenv("ANNOTATION_SOURCE_INDEX", "true")
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "revision")
f, root := annotationProcessorFixture(t)
sourceLoadProductionShape(t, &f, root)
ctx, cancel := context.WithTimeout(f.ctx, 2*time.Minute)
defer cancel()
client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
database := client.Database(fmt.Sprintf("source_index_http_test_%d", time.Now().UnixNano()))
defer func() {
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
defer stop()
if e := database.Drop(cleanup); e != nil {
t.Error(e)
}
}()
db := store.NewMongoStore(database)
must := func(e error) {
t.Helper()
if e != nil {
t.Fatal(e)
}
}
must(db.EnsureIndexes(ctx))
org, _ := f.store.GetOrganization(ctx, f.task.OrganizationID)
must(db.UpsertOrganization(ctx, *org))
supplier, _ := f.store.GetSupplier(ctx, f.annotator.SupplierID)
must(db.UpsertSupplier(ctx, *supplier))
project, _ := f.store.GetProject(ctx, f.task.ProjectID)
must(db.UpsertProject(ctx, *project))
workflow, _ := f.store.GetWorkflow(ctx, f.task.WorkflowID)
must(db.UpsertWorkflow(ctx, *workflow))
template, _ := f.store.GetAnnotationTemplate(ctx, f.task.TemplateID)
must(db.UpsertAnnotationTemplate(ctx, *template))
batch, _ := f.store.GetHumanBatch(ctx, f.item.BatchID)
batch.ProcessingProfile = domain.ProcessingProfileRawHuman
must(db.UpsertHumanBatch(ctx, *batch))
run, _ := f.store.GetRun(ctx, f.task.RunID)
must(db.UpsertRun(ctx, *run))
labels, _ := f.store.ListAnnotationLabels(ctx, f.task.OrganizationID)
for _, label := range labels {
must(db.UpsertAnnotationLabel(ctx, label))
}
selection, _ := f.store.GetProjectAnnotationLabels(ctx, f.task.ProjectID)
must(db.UpsertProjectSliceLabels(ctx, *selection))
precompute := &server{store: db, workspaceMountPath: root, annotationSourceIndex: true}
baseKey, e := precompute.annotationSourceFingerprint(ctx, f.task, *run)
must(e)
baseCertificate, e := precompute.annotationSourceCertificate(ctx, f.task, *run, baseKey)
must(e)
actors := map[string]domain.UserRecord{}
tasks := make([]domain.AnnotationTaskRecord, 100)
items := make([]domain.HumanItemRecord, 100)
for n := 0; n < 100; n++ {
actor := f.annotator
actor.ID = fmt.Sprintf("source-load-user-%03d", n)
actor.Phone = actor.ID
actors[actor.ID] = actor
must(db.UpsertUser(ctx, actor))
must(db.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + actor.ID, OrganizationID: actor.OrganizationID, ProjectID: f.task.ProjectID, UserID: actor.ID, SupplierID: actor.SupplierID, Roles: domain.AccountProductionRoles(actor)}))
must(db.UpsertProjectAnnotationMembership(ctx, domain.ProjectAnnotationMembershipRecord{ID: "annotation-" + actor.ID, OrganizationID: actor.OrganizationID, ProjectID: f.task.ProjectID, UserID: actor.ID, AssignmentRole: domain.AnnotationRoleAnnotator}))
task := f.task
task.ID = fmt.Sprintf("source-load-task-%03d", n)
task.RunID = fmt.Sprintf("source-load-run-%03d", n)
task.ProcessingProfile = domain.ProcessingProfileRawHuman
newRun := *run
newRun.ID = task.RunID
must(db.UpsertRun(ctx, newRun))
key, err := precompute.annotationSourceFingerprint(ctx, task, newRun)
must(err)
cert := sourceLoadCertificateForRun(t, root, baseCertificate, task, key)
body, digest := sourceLoadStoredBody(t, cert)
must(db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: sourceCertificateID(task.OrganizationID, key), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: task.RunID, Body: body, SHA256: digest}))
body, digest = sourceLoadStoredBody(t, cert.Timeline)
must(db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: "annotation-timeline-v1:" + task.OrganizationID + ":" + key, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: task.RunID, Body: body, SHA256: digest}))
tasks[n] = task
must(db.UpsertAnnotationTask(ctx, task))
item := f.item
item.ID = fmt.Sprintf("source-load-item-%03d", n)
item.LegacyTaskID = task.ID
item.RunID = task.RunID
newBatch := *batch
newBatch.ID = fmt.Sprintf("source-load-batch-%03d", n)
newBatch.RunID = task.RunID
item.BatchID = newBatch.ID
must(db.UpsertHumanBatch(ctx, newBatch))
item.Subject.ItemKey = task.ID
item.AssignedUserID = actor.ID
item.Subject.Payload = map[string]any{"captureDirectory": "synthetic-load", "processingProfile": "raw_human"}
items[n] = item
must(db.UpsertHumanItem(ctx, item))
}
handlers := []http.Handler{}
for n := 0; n < 4; n++ {
handlers = append(handlers, NewHandlerWithConfig(store.NewMongoStore(database), nil, config.AppConfig{WorkspaceMountPath: root}))
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
actor, ok := actors[r.Header.Get("X-Synthetic-Actor")]
if !ok {
http.Error(w, "synthetic actor required", 403)
return
}
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: actor, Organization: *org}))
handlers[int(actor.ID[len(actor.ID)-1])%len(handlers)].ServeHTTP(w, r)
}))
defer server.Close()
processorCtx, stopProcessor := context.WithCancel(ctx)
done := make(chan error, 2)
for n := 0; n < 2; n++ {
go func() {
done <- RunAnnotationProcessor(processorCtx, db, config.AppConfig{WorkspaceMountPath: root}, 8, false)
}()
}
defer func() { stopProcessor(); <-done; <-done }()
type observation struct {
Kind string `json:"kind"`
Milliseconds float64 `json:"milliseconds"`
Status int `json:"status"`
}
var mu sync.Mutex
observations := []observation{}
completions := []float64{}
call := func(actor, method, path, kind string, body any) (map[string]any, error) {
b, _ := json.Marshal(body)
req, e := http.NewRequestWithContext(ctx, method, server.URL+path, bytes.NewReader(b))
if e != nil {
return nil, e
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Synthetic-Actor", actor)
req.Header.Set(csrfHeaderName, "source-load-csrf")
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "source-load-csrf"})
started := time.Now()
response, e := server.Client().Do(req)
if e != nil {
return nil, e
}
defer response.Body.Close()
data, e := io.ReadAll(io.LimitReader(response.Body, 8<<20))
elapsed := float64(time.Since(started).Microseconds()) / 1000
mu.Lock()
observations = append(observations, observation{kind, elapsed, response.StatusCode})
mu.Unlock()
if e != nil {
return nil, e
}
if response.StatusCode != 200 && response.StatusCode != 202 {
return nil, fmt.Errorf("%s %d %s", kind, response.StatusCode, string(data))
}
var out map[string]any
e = json.Unmarshal(data, &out)
return out, e
}
start := make(chan struct{})
var wg sync.WaitGroup
began := time.Now()
for n := 0; n < 100; n++ {
wg.Add(1)
go func(n int) {
defer wg.Done()
<-start
actor := fmt.Sprintf("source-load-user-%03d", n)
endpoint := "/api/plugins/annotation-qa/tasks/" + tasks[n].ID
fail := func(e error) bool {
if e != nil {
t.Error(actor, e)
return true
}
return false
}
status, e := call(actor, "GET", endpoint+"?view=status", "task-status", nil)
if fail(e) {
return
}
if _, e = call(actor, "GET", "/api/plugins/annotation-qa/work-buffer?projectId="+f.task.ProjectID, "work-buffer", nil); fail(e) {
return
}
slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: tasks[n].CoverageEndSecond}}
draft, e := call(actor, "PUT", endpoint+"/drafts", "draft-save", map[string]any{"version": intValue(status["version"]), "sliceSegments": slices, "segments": []any{}})
if fail(e) {
return
}
version := intValue(draft["version"])
if version == 0 {
current, e := db.GetHumanItem(ctx, items[n].ID)
if fail(e) {
return
}
version = current.Version
}
confirmedAt := time.Now()
_, e = call(actor, "POST", endpoint+"/slice-confirmations", "slice-confirm", map[string]any{"version": version, "sliceSegments": slices})
if fail(e) {
return
}
for time.Since(confirmedAt) < 45*time.Second {
state, e := call(actor, "GET", endpoint+"?view=status", "processing-status", nil)
if fail(e) {
return
}
if stringValue(state["sliceStatus"]) == "completed" {
mu.Lock()
completions = append(completions, float64(time.Since(confirmedAt).Microseconds())/1000)
mu.Unlock()
return
}
if stringValue(state["sliceStatus"]) == "failed" {
t.Error(actor, "slice failed", state)
return
}
time.Sleep(2 * time.Second)
}
t.Error(actor, "slice completion timed out")
}(n)
}
close(start)
wg.Wait()
if len(completions) != 100 {
t.Fatalf("only %d/100 completed", len(completions))
}
percentile := func(values []float64, p float64) float64 {
sort.Float64s(values)
return values[int(float64(len(values)-1)*p)]
}
latencies := map[string][]float64{}
for _, o := range observations {
latencies[o.Kind] = append(latencies[o.Kind], o.Milliseconds)
}
summary := map[string]any{}
for kind, values := range latencies {
summary[kind] = map[string]any{"requests": len(values), "p50Ms": percentile(values, .5), "p95Ms": percentile(values, .95), "maxMs": percentile(values, 1)}
}
report := map[string]any{"activeOperators": 100, "apiHandlerInstances": 4, "revisionReplicas": 2, "revisionConcurrencyPerReplica": 8, "sourceSecondsPerTask": 900, "sourceFramesPerCamera": 27000, "sourceIMURows": 1800000, "completed": len(completions), "wallSeconds": time.Since(began).Seconds(), "completionP95Ms": percentile(completions, .95), "requests": summary, "scope": "isolated local Mongo and real HTTP, 100 distinct run/source certificate identities with production-scale prevalidated synthetic indexes; excludes login, production BOS/network and 100 browser decoders"}
report["projectWriteLockMode"], _ = config.ProjectWriteLockMode()
b, _ := json.MarshalIndent(report, "", " ")
t.Log(string(b))
if out := os.Getenv("FANYU_SOURCE_INDEX_LOAD_REPORT"); out != "" {
if !filepath.IsAbs(out) {
t.Fatal("absolute report path required")
}
must(os.WriteFile(out, b, 0600))
}
if percentile(completions, .95) > 10000 {
t.Fatal("warm source revision completion p95 exceeds 10s")
}
if percentile(latencies["draft-save"], .95) > 750 {
t.Fatal("100-operator draft-save p95 exceeds 750ms")
}
}