forked from longtao.wu/droboticflow
Same-project saves queued behind one project lock. Add default-off record stripes while deletion, authorization and claim operations retain exclusive protection across all stripes. Require a complete barrier rollout before striped activation. Verify local Mongo deletion, revocation, crash recovery and two 100-user HTTP runs with save p95 below 400ms.
283 lines
11 KiB
Go
283 lines
11 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"droboticflow/internal/config"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"encoding/json"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Opt-in actual Mongo + HTTP verification. The fixed loopback server and fresh
|
|
// database name deliberately cannot be redirected to a company database.
|
|
func TestSourceIndex100ActiveOperatorsMongoHTTP(t *testing.T) {
|
|
if os.Getenv("FANYU_SOURCE_INDEX_MONGO_LOAD") != "1" {
|
|
t.Skip("opt-in isolated local Mongo load test")
|
|
}
|
|
t.Setenv("ANNOTATION_SOURCE_INDEX", "true")
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "revision")
|
|
f, root := annotationProcessorFixture(t)
|
|
sourceLoadProductionShape(t, &f, root)
|
|
ctx, cancel := context.WithTimeout(f.ctx, 2*time.Minute)
|
|
defer cancel()
|
|
client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
database := client.Database(fmt.Sprintf("source_index_http_test_%d", time.Now().UnixNano()))
|
|
defer func() {
|
|
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer stop()
|
|
if e := database.Drop(cleanup); e != nil {
|
|
t.Error(e)
|
|
}
|
|
}()
|
|
db := store.NewMongoStore(database)
|
|
must := func(e error) {
|
|
t.Helper()
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
must(db.EnsureIndexes(ctx))
|
|
org, _ := f.store.GetOrganization(ctx, f.task.OrganizationID)
|
|
must(db.UpsertOrganization(ctx, *org))
|
|
supplier, _ := f.store.GetSupplier(ctx, f.annotator.SupplierID)
|
|
must(db.UpsertSupplier(ctx, *supplier))
|
|
project, _ := f.store.GetProject(ctx, f.task.ProjectID)
|
|
must(db.UpsertProject(ctx, *project))
|
|
workflow, _ := f.store.GetWorkflow(ctx, f.task.WorkflowID)
|
|
must(db.UpsertWorkflow(ctx, *workflow))
|
|
template, _ := f.store.GetAnnotationTemplate(ctx, f.task.TemplateID)
|
|
must(db.UpsertAnnotationTemplate(ctx, *template))
|
|
batch, _ := f.store.GetHumanBatch(ctx, f.item.BatchID)
|
|
batch.ProcessingProfile = domain.ProcessingProfileRawHuman
|
|
must(db.UpsertHumanBatch(ctx, *batch))
|
|
run, _ := f.store.GetRun(ctx, f.task.RunID)
|
|
must(db.UpsertRun(ctx, *run))
|
|
labels, _ := f.store.ListAnnotationLabels(ctx, f.task.OrganizationID)
|
|
for _, label := range labels {
|
|
must(db.UpsertAnnotationLabel(ctx, label))
|
|
}
|
|
selection, _ := f.store.GetProjectAnnotationLabels(ctx, f.task.ProjectID)
|
|
must(db.UpsertProjectSliceLabels(ctx, *selection))
|
|
precompute := &server{store: db, workspaceMountPath: root, annotationSourceIndex: true}
|
|
baseKey, e := precompute.annotationSourceFingerprint(ctx, f.task, *run)
|
|
must(e)
|
|
baseCertificate, e := precompute.annotationSourceCertificate(ctx, f.task, *run, baseKey)
|
|
must(e)
|
|
actors := map[string]domain.UserRecord{}
|
|
tasks := make([]domain.AnnotationTaskRecord, 100)
|
|
items := make([]domain.HumanItemRecord, 100)
|
|
for n := 0; n < 100; n++ {
|
|
actor := f.annotator
|
|
actor.ID = fmt.Sprintf("source-load-user-%03d", n)
|
|
actor.Phone = actor.ID
|
|
actors[actor.ID] = actor
|
|
must(db.UpsertUser(ctx, actor))
|
|
must(db.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + actor.ID, OrganizationID: actor.OrganizationID, ProjectID: f.task.ProjectID, UserID: actor.ID, SupplierID: actor.SupplierID, Roles: domain.AccountProductionRoles(actor)}))
|
|
must(db.UpsertProjectAnnotationMembership(ctx, domain.ProjectAnnotationMembershipRecord{ID: "annotation-" + actor.ID, OrganizationID: actor.OrganizationID, ProjectID: f.task.ProjectID, UserID: actor.ID, AssignmentRole: domain.AnnotationRoleAnnotator}))
|
|
task := f.task
|
|
task.ID = fmt.Sprintf("source-load-task-%03d", n)
|
|
task.RunID = fmt.Sprintf("source-load-run-%03d", n)
|
|
task.ProcessingProfile = domain.ProcessingProfileRawHuman
|
|
newRun := *run
|
|
newRun.ID = task.RunID
|
|
must(db.UpsertRun(ctx, newRun))
|
|
key, err := precompute.annotationSourceFingerprint(ctx, task, newRun)
|
|
must(err)
|
|
cert := sourceLoadCertificateForRun(t, root, baseCertificate, task, key)
|
|
body, digest := sourceLoadStoredBody(t, cert)
|
|
must(db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: sourceCertificateID(task.OrganizationID, key), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: task.RunID, Body: body, SHA256: digest}))
|
|
body, digest = sourceLoadStoredBody(t, cert.Timeline)
|
|
must(db.PutAnnotationTimelineIndex(ctx, domain.AnnotationTimelineIndex{ID: "annotation-timeline-v1:" + task.OrganizationID + ":" + key, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: task.RunID, Body: body, SHA256: digest}))
|
|
tasks[n] = task
|
|
must(db.UpsertAnnotationTask(ctx, task))
|
|
item := f.item
|
|
item.ID = fmt.Sprintf("source-load-item-%03d", n)
|
|
item.LegacyTaskID = task.ID
|
|
item.RunID = task.RunID
|
|
newBatch := *batch
|
|
newBatch.ID = fmt.Sprintf("source-load-batch-%03d", n)
|
|
newBatch.RunID = task.RunID
|
|
item.BatchID = newBatch.ID
|
|
must(db.UpsertHumanBatch(ctx, newBatch))
|
|
item.Subject.ItemKey = task.ID
|
|
item.AssignedUserID = actor.ID
|
|
item.Subject.Payload = map[string]any{"captureDirectory": "synthetic-load", "processingProfile": "raw_human"}
|
|
items[n] = item
|
|
must(db.UpsertHumanItem(ctx, item))
|
|
}
|
|
handlers := []http.Handler{}
|
|
for n := 0; n < 4; n++ {
|
|
handlers = append(handlers, NewHandlerWithConfig(store.NewMongoStore(database), nil, config.AppConfig{WorkspaceMountPath: root}))
|
|
}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
actor, ok := actors[r.Header.Get("X-Synthetic-Actor")]
|
|
if !ok {
|
|
http.Error(w, "synthetic actor required", 403)
|
|
return
|
|
}
|
|
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: actor, Organization: *org}))
|
|
handlers[int(actor.ID[len(actor.ID)-1])%len(handlers)].ServeHTTP(w, r)
|
|
}))
|
|
defer server.Close()
|
|
processorCtx, stopProcessor := context.WithCancel(ctx)
|
|
done := make(chan error, 2)
|
|
for n := 0; n < 2; n++ {
|
|
go func() {
|
|
done <- RunAnnotationProcessor(processorCtx, db, config.AppConfig{WorkspaceMountPath: root}, 8, false)
|
|
}()
|
|
}
|
|
defer func() { stopProcessor(); <-done; <-done }()
|
|
type observation struct {
|
|
Kind string `json:"kind"`
|
|
Milliseconds float64 `json:"milliseconds"`
|
|
Status int `json:"status"`
|
|
}
|
|
var mu sync.Mutex
|
|
observations := []observation{}
|
|
completions := []float64{}
|
|
call := func(actor, method, path, kind string, body any) (map[string]any, error) {
|
|
b, _ := json.Marshal(body)
|
|
req, e := http.NewRequestWithContext(ctx, method, server.URL+path, bytes.NewReader(b))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("X-Synthetic-Actor", actor)
|
|
req.Header.Set(csrfHeaderName, "source-load-csrf")
|
|
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "source-load-csrf"})
|
|
started := time.Now()
|
|
response, e := server.Client().Do(req)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer response.Body.Close()
|
|
data, e := io.ReadAll(io.LimitReader(response.Body, 8<<20))
|
|
elapsed := float64(time.Since(started).Microseconds()) / 1000
|
|
mu.Lock()
|
|
observations = append(observations, observation{kind, elapsed, response.StatusCode})
|
|
mu.Unlock()
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if response.StatusCode != 200 && response.StatusCode != 202 {
|
|
return nil, fmt.Errorf("%s %d %s", kind, response.StatusCode, string(data))
|
|
}
|
|
var out map[string]any
|
|
e = json.Unmarshal(data, &out)
|
|
return out, e
|
|
}
|
|
start := make(chan struct{})
|
|
var wg sync.WaitGroup
|
|
began := time.Now()
|
|
for n := 0; n < 100; n++ {
|
|
wg.Add(1)
|
|
go func(n int) {
|
|
defer wg.Done()
|
|
<-start
|
|
actor := fmt.Sprintf("source-load-user-%03d", n)
|
|
endpoint := "/api/plugins/annotation-qa/tasks/" + tasks[n].ID
|
|
fail := func(e error) bool {
|
|
if e != nil {
|
|
t.Error(actor, e)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
status, e := call(actor, "GET", endpoint+"?view=status", "task-status", nil)
|
|
if fail(e) {
|
|
return
|
|
}
|
|
if _, e = call(actor, "GET", "/api/plugins/annotation-qa/work-buffer?projectId="+f.task.ProjectID, "work-buffer", nil); fail(e) {
|
|
return
|
|
}
|
|
slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: tasks[n].CoverageEndSecond}}
|
|
draft, e := call(actor, "PUT", endpoint+"/drafts", "draft-save", map[string]any{"version": intValue(status["version"]), "sliceSegments": slices, "segments": []any{}})
|
|
if fail(e) {
|
|
return
|
|
}
|
|
version := intValue(draft["version"])
|
|
if version == 0 {
|
|
current, e := db.GetHumanItem(ctx, items[n].ID)
|
|
if fail(e) {
|
|
return
|
|
}
|
|
version = current.Version
|
|
}
|
|
confirmedAt := time.Now()
|
|
_, e = call(actor, "POST", endpoint+"/slice-confirmations", "slice-confirm", map[string]any{"version": version, "sliceSegments": slices})
|
|
if fail(e) {
|
|
return
|
|
}
|
|
for time.Since(confirmedAt) < 45*time.Second {
|
|
state, e := call(actor, "GET", endpoint+"?view=status", "processing-status", nil)
|
|
if fail(e) {
|
|
return
|
|
}
|
|
if stringValue(state["sliceStatus"]) == "completed" {
|
|
mu.Lock()
|
|
completions = append(completions, float64(time.Since(confirmedAt).Microseconds())/1000)
|
|
mu.Unlock()
|
|
return
|
|
}
|
|
if stringValue(state["sliceStatus"]) == "failed" {
|
|
t.Error(actor, "slice failed", state)
|
|
return
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
t.Error(actor, "slice completion timed out")
|
|
}(n)
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
if len(completions) != 100 {
|
|
t.Fatalf("only %d/100 completed", len(completions))
|
|
}
|
|
percentile := func(values []float64, p float64) float64 {
|
|
sort.Float64s(values)
|
|
return values[int(float64(len(values)-1)*p)]
|
|
}
|
|
latencies := map[string][]float64{}
|
|
for _, o := range observations {
|
|
latencies[o.Kind] = append(latencies[o.Kind], o.Milliseconds)
|
|
}
|
|
summary := map[string]any{}
|
|
for kind, values := range latencies {
|
|
summary[kind] = map[string]any{"requests": len(values), "p50Ms": percentile(values, .5), "p95Ms": percentile(values, .95), "maxMs": percentile(values, 1)}
|
|
}
|
|
report := map[string]any{"activeOperators": 100, "apiHandlerInstances": 4, "revisionReplicas": 2, "revisionConcurrencyPerReplica": 8, "sourceSecondsPerTask": 900, "sourceFramesPerCamera": 27000, "sourceIMURows": 1800000, "completed": len(completions), "wallSeconds": time.Since(began).Seconds(), "completionP95Ms": percentile(completions, .95), "requests": summary, "scope": "isolated local Mongo and real HTTP, 100 distinct run/source certificate identities with production-scale prevalidated synthetic indexes; excludes login, production BOS/network and 100 browser decoders"}
|
|
report["projectWriteLockMode"], _ = config.ProjectWriteLockMode()
|
|
b, _ := json.MarshalIndent(report, "", " ")
|
|
t.Log(string(b))
|
|
if out := os.Getenv("FANYU_SOURCE_INDEX_LOAD_REPORT"); out != "" {
|
|
if !filepath.IsAbs(out) {
|
|
t.Fatal("absolute report path required")
|
|
}
|
|
must(os.WriteFile(out, b, 0600))
|
|
}
|
|
if percentile(completions, .95) > 10000 {
|
|
t.Fatal("warm source revision completion p95 exceeds 10s")
|
|
}
|
|
if percentile(latencies["draft-save"], .95) > 750 {
|
|
t.Fatal("100-operator draft-save p95 exceeds 750ms")
|
|
}
|
|
}
|