package api import ( "bytes" "context" "droboticflow/internal/config" "droboticflow/internal/domain" "droboticflow/internal/store" "encoding/json" "fmt" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "io" "net/http" "net/http/httptest" "os" "path/filepath" "sort" "sync" "testing" "time" ) // Opt-in actual Mongo + HTTP verification. The fixed loopback server and fresh // database name deliberately cannot be redirected to a company database. func TestSourceIndex100ActiveOperatorsMongoHTTP(t *testing.T) { if os.Getenv("FANYU_SOURCE_INDEX_MONGO_LOAD") != "1" { t.Skip("opt-in isolated local Mongo load test") } t.Setenv("ANNOTATION_SOURCE_INDEX", "true") t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") t.Setenv("ANNOTATION_PROCESSOR_ROLE", "revision") f, root := annotationProcessorFixture(t) sourceLoadProductionShape(t, &f, root) ctx, cancel := context.WithTimeout(f.ctx, 2*time.Minute) defer cancel() client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second)) if e != nil { t.Fatal(e) } defer client.Disconnect(context.Background()) database := client.Database(fmt.Sprintf("source_index_http_test_%d", time.Now().UnixNano())) defer func() { cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second) defer stop() if e := database.Drop(cleanup); e != nil { t.Error(e) } }() db := store.NewMongoStore(database) must := func(e error) { t.Helper() if e != nil { t.Fatal(e) } } must(db.EnsureIndexes(ctx)) org, _ := f.store.GetOrganization(ctx, f.task.OrganizationID) must(db.UpsertOrganization(ctx, *org)) supplier, _ := f.store.GetSupplier(ctx, f.annotator.SupplierID) must(db.UpsertSupplier(ctx, *supplier)) project, _ := f.store.GetProject(ctx, f.task.ProjectID) must(db.UpsertProject(ctx, *project)) workflow, _ := f.store.GetWorkflow(ctx, f.task.WorkflowID) must(db.UpsertWorkflow(ctx, *workflow)) template, _ := f.store.GetAnnotationTemplate(ctx, f.task.TemplateID) must(db.UpsertAnnotationTemplate(ctx, *template)) batch, _ := f.store.GetHumanBatch(ctx, f.item.BatchID) batch.ProcessingProfile = domain.ProcessingProfileRawHuman must(db.UpsertHumanBatch(ctx, *batch)) run, _ := f.store.GetRun(ctx, f.task.RunID) must(db.UpsertRun(ctx, *run)) labels, _ := f.store.ListAnnotationLabels(ctx, f.task.OrganizationID) for _, label := range labels { must(db.UpsertAnnotationLabel(ctx, label)) } selection, _ := f.store.GetProjectAnnotationLabels(ctx, f.task.ProjectID) must(db.UpsertProjectSliceLabels(ctx, *selection)) precompute := &server{store: db, workspaceMountPath: root, annotationSourceIndex: true} baseKey, e := precompute.annotationSourceFingerprint(ctx, f.task, *run) must(e) baseCertificate, e := precompute.annotationSourceCertificate(ctx, f.task, *run, baseKey) must(e) actors := map[string]domain.UserRecord{} tasks := make([]domain.AnnotationTaskRecord, 100) items := make([]domain.HumanItemRecord, 100) for n := 0; n < 100; n++ { actor := f.annotator actor.ID = fmt.Sprintf("source-load-user-%03d", n) actor.Phone = actor.ID actors[actor.ID] = actor must(db.UpsertUser(ctx, actor)) must(db.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + actor.ID, OrganizationID: actor.OrganizationID, ProjectID: f.task.ProjectID, UserID: actor.ID, SupplierID: actor.SupplierID, Roles: domain.AccountProductionRoles(actor)})) must(db.UpsertProjectAnnotationMembership(ctx, domain.ProjectAnnotationMembershipRecord{ID: "annotation-" + actor.ID, OrganizationID: actor.OrganizationID, ProjectID: f.task.ProjectID, UserID: actor.ID, AssignmentRole: domain.AnnotationRoleAnnotator})) task := f.task task.ID = fmt.Sprintf("source-load-task-%03d", n) task.RunID = fmt.Sprintf("source-load-run-%03d", n) task.ProcessingProfile = domain.ProcessingProfileRawHuman newRun := *run newRun.ID = task.RunID must(db.UpsertRun(ctx, newRun)) key, err := precompute.annotationSourceFingerprint(ctx, task, newRun) must(err) cert := sourceLoadCertificateForRun(t, root, baseCertificate, task, key) body, digest := sourceLoadStoredBody(t, cert) must(db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: sourceCertificateID(task.OrganizationID, key), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: task.RunID, Body: body, SHA256: digest})) body, digest = sourceLoadStoredBody(t, cert.Timeline) must(db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: "annotation-timeline-v1:" + task.OrganizationID + ":" + key, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: task.RunID, Body: body, SHA256: digest})) tasks[n] = task must(db.UpsertAnnotationTask(ctx, task)) item := f.item item.ID = fmt.Sprintf("source-load-item-%03d", n) item.LegacyTaskID = task.ID item.RunID = task.RunID newBatch := *batch newBatch.ID = fmt.Sprintf("source-load-batch-%03d", n) newBatch.RunID = task.RunID item.BatchID = newBatch.ID must(db.UpsertHumanBatch(ctx, newBatch)) item.Subject.ItemKey = task.ID item.AssignedUserID = actor.ID item.Subject.Payload = map[string]any{"captureDirectory": "synthetic-load", "processingProfile": "raw_human"} items[n] = item must(db.UpsertHumanItem(ctx, item)) } handlers := []http.Handler{} for n := 0; n < 4; n++ { handlers = append(handlers, NewHandlerWithConfig(store.NewMongoStore(database), nil, config.AppConfig{WorkspaceMountPath: root})) } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { actor, ok := actors[r.Header.Get("X-Synthetic-Actor")] if !ok { http.Error(w, "synthetic actor required", 403) return } r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: actor, Organization: *org})) handlers[int(actor.ID[len(actor.ID)-1])%len(handlers)].ServeHTTP(w, r) })) defer server.Close() processorCtx, stopProcessor := context.WithCancel(ctx) done := make(chan error, 2) for n := 0; n < 2; n++ { go func() { done <- RunAnnotationProcessor(processorCtx, db, config.AppConfig{WorkspaceMountPath: root}, 8, false) }() } defer func() { stopProcessor(); <-done; <-done }() type observation struct { Kind string `json:"kind"` Milliseconds float64 `json:"milliseconds"` Status int `json:"status"` } var mu sync.Mutex observations := []observation{} completions := []float64{} call := func(actor, method, path, kind string, body any) (map[string]any, error) { b, _ := json.Marshal(body) req, e := http.NewRequestWithContext(ctx, method, server.URL+path, bytes.NewReader(b)) if e != nil { return nil, e } req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Synthetic-Actor", actor) req.Header.Set(csrfHeaderName, "source-load-csrf") req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "source-load-csrf"}) started := time.Now() response, e := server.Client().Do(req) if e != nil { return nil, e } defer response.Body.Close() data, e := io.ReadAll(io.LimitReader(response.Body, 8<<20)) elapsed := float64(time.Since(started).Microseconds()) / 1000 mu.Lock() observations = append(observations, observation{kind, elapsed, response.StatusCode}) mu.Unlock() if e != nil { return nil, e } if response.StatusCode != 200 && response.StatusCode != 202 { return nil, fmt.Errorf("%s %d %s", kind, response.StatusCode, string(data)) } var out map[string]any e = json.Unmarshal(data, &out) return out, e } start := make(chan struct{}) var wg sync.WaitGroup began := time.Now() for n := 0; n < 100; n++ { wg.Add(1) go func(n int) { defer wg.Done() <-start actor := fmt.Sprintf("source-load-user-%03d", n) endpoint := "/api/plugins/annotation-qa/tasks/" + tasks[n].ID fail := func(e error) bool { if e != nil { t.Error(actor, e) return true } return false } status, e := call(actor, "GET", endpoint+"?view=status", "task-status", nil) if fail(e) { return } if _, e = call(actor, "GET", "/api/plugins/annotation-qa/work-buffer?projectId="+f.task.ProjectID, "work-buffer", nil); fail(e) { return } slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: tasks[n].CoverageEndSecond}} draft, e := call(actor, "PUT", endpoint+"/drafts", "draft-save", map[string]any{"version": intValue(status["version"]), "sliceSegments": slices, "segments": []any{}}) if fail(e) { return } version := intValue(draft["version"]) if version == 0 { current, e := db.GetHumanItem(ctx, items[n].ID) if fail(e) { return } version = current.Version } confirmedAt := time.Now() _, e = call(actor, "POST", endpoint+"/slice-confirmations", "slice-confirm", map[string]any{"version": version, "sliceSegments": slices}) if fail(e) { return } for time.Since(confirmedAt) < 45*time.Second { state, e := call(actor, "GET", endpoint+"?view=status", "processing-status", nil) if fail(e) { return } if stringValue(state["sliceStatus"]) == "completed" { mu.Lock() completions = append(completions, float64(time.Since(confirmedAt).Microseconds())/1000) mu.Unlock() return } if stringValue(state["sliceStatus"]) == "failed" { t.Error(actor, "slice failed", state) return } time.Sleep(2 * time.Second) } t.Error(actor, "slice completion timed out") }(n) } close(start) wg.Wait() if len(completions) != 100 { t.Fatalf("only %d/100 completed", len(completions)) } percentile := func(values []float64, p float64) float64 { sort.Float64s(values) return values[int(float64(len(values)-1)*p)] } latencies := map[string][]float64{} for _, o := range observations { latencies[o.Kind] = append(latencies[o.Kind], o.Milliseconds) } summary := map[string]any{} for kind, values := range latencies { summary[kind] = map[string]any{"requests": len(values), "p50Ms": percentile(values, .5), "p95Ms": percentile(values, .95), "maxMs": percentile(values, 1)} } report := map[string]any{"activeOperators": 100, "apiHandlerInstances": 4, "revisionReplicas": 2, "revisionConcurrencyPerReplica": 8, "sourceSecondsPerTask": 900, "sourceFramesPerCamera": 27000, "sourceIMURows": 1800000, "completed": len(completions), "wallSeconds": time.Since(began).Seconds(), "completionP95Ms": percentile(completions, .95), "requests": summary, "scope": "isolated local Mongo and real HTTP, 100 distinct run/source certificate identities with production-scale prevalidated synthetic indexes; excludes login, production BOS/network and 100 browser decoders"} report["projectWriteLockMode"], _ = config.ProjectWriteLockMode() b, _ := json.MarshalIndent(report, "", " ") t.Log(string(b)) if out := os.Getenv("FANYU_SOURCE_INDEX_LOAD_REPORT"); out != "" { if !filepath.IsAbs(out) { t.Fatal("absolute report path required") } must(os.WriteFile(out, b, 0600)) } if percentile(completions, .95) > 10000 { t.Fatal("warm source revision completion p95 exceeds 10s") } if percentile(latencies["draft-save"], .95) > 750 { t.Fatal("100-operator draft-save p95 exceeds 750ms") } }