1024 lines
38 KiB
Go
1024 lines
38 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"reflect"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/productionstats"
|
|
"droboticflow/internal/queryserving"
|
|
"droboticflow/internal/store"
|
|
"droboticflow/internal/testsupport"
|
|
valkey "github.com/valkey-io/valkey-go"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/event"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
|
"go.mongodb.org/mongo-driver/mongo/readpref"
|
|
)
|
|
|
|
// This harness has no production URI input. Both DBs are newly named synthetic
|
|
// fixtures, and the target must carry the independently verified node4 marker.
|
|
// HTTP uses the actual session, ACL and count/report handlers. Business recovery
|
|
// schedulers are deliberately not duplicated in the two benchmark handlers.
|
|
func TestQueryReplicaIsolatedHTTPComparisonLocal(t *testing.T) {
|
|
local := os.Getenv("QUERY_PERIODIC_HTTP_LOCAL") == "1"
|
|
if os.Getenv("QUERY_REPLICA_ISOLATED_E2E") != "1" && !local {
|
|
t.Skip("explicit node4 synthetic comparison only")
|
|
}
|
|
if local {
|
|
verifyPeriodicLocalHTTPFixture(t)
|
|
} else if os.Getenv("QUERY_REPLICA_CLUSTER_TEST") != "1" {
|
|
t.Fatal("cluster synthetic target must be explicitly selected")
|
|
}
|
|
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
|
t.Setenv("QUERY_SERVING_NAMESPACE", "isolated-http-comparison")
|
|
published := os.Getenv("QUERY_PUBLISHED_RESULTS") == "1"
|
|
cacheURL := os.Getenv("QUERY_COMPARE_VALKEY_URL")
|
|
if published && cacheURL == "" {
|
|
t.Fatal("published result comparison requires explicit isolated Valkey")
|
|
}
|
|
t.Setenv("QUERY_SERVING_READS", "1")
|
|
t.Setenv("QUERY_VALKEY_URL", cacheURL)
|
|
ctx := t.Context()
|
|
n := isolatedComparisonInt(t, "QUERY_COMPARE_TASKS", 26218, 20, 30000)
|
|
waves := isolatedComparisonInt(t, "QUERY_COMPARE_WAVES", 2, 1, 100)
|
|
maxActors := isolatedComparisonInt(t, "QUERY_COMPARE_MAX_ACTORS", 81, 81, 500)
|
|
maxInflight := isolatedComparisonInt(t, "QUERY_COMPARE_CPU_MAX_INFLIGHT", 0, 0, 8)
|
|
if maxInflight > 0 && !local {
|
|
t.Fatal("supplemental matched CPU comparison is local-only")
|
|
}
|
|
minPool := isolatedComparisonInt(t, "QUERY_COMPARE_MIN_POOL", 0, 0, 100)
|
|
compression := os.Getenv("QUERY_COMPARE_WIRE_COMPRESSION")
|
|
if compression != "" && compression != "zstd" {
|
|
t.Fatal("unsupported comparison compression")
|
|
}
|
|
var mu sync.Mutex
|
|
var phase string
|
|
inflight := map[int64]string{}
|
|
metrics := map[string]*isolatedCommands{}
|
|
var targetStatsMu sync.Mutex
|
|
var factTrace []map[string]any
|
|
factPending := map[int64]int{}
|
|
if path := os.Getenv("QUERY_COMPARE_DIAGNOSTIC_REPORT"); local && path != "" {
|
|
defer func() {
|
|
targetStatsMu.Lock()
|
|
defer targetStatsMu.Unlock()
|
|
raw, _ := json.MarshalIndent(factTrace, "", " ")
|
|
if err := os.WriteFile(path, raw, 0600); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
}
|
|
var workforceAggregations atomic.Int64
|
|
targetCommands := map[string]int64{}
|
|
targetMaxMillis := map[string]float64{}
|
|
targetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
|
|
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
|
if name == "queryWorkforceFacts" {
|
|
workforceAggregations.Add(1)
|
|
}
|
|
if !local {
|
|
return
|
|
}
|
|
if name != "queryWorkforceFacts" {
|
|
return
|
|
}
|
|
targetStatsMu.Lock()
|
|
defer targetStatsMu.Unlock()
|
|
if len(factTrace) >= 512 {
|
|
return
|
|
}
|
|
maxTime, present := e.Command.Lookup("maxTimeMS").AsInt64OK()
|
|
remaining := int64(-1)
|
|
if deadline, ok := ctx.Deadline(); ok {
|
|
remaining = time.Until(deadline).Milliseconds()
|
|
}
|
|
factPending[e.RequestID] = len(factTrace)
|
|
factTrace = append(factTrace, map[string]any{"appTimeUTC": time.Now().UTC().Format(time.RFC3339Nano), "wireMaxTimeMS": maxTime, "wireMaxTimePresent": present, "contextRemainingMS": remaining})
|
|
}, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
|
targetStatsMu.Lock()
|
|
defer targetStatsMu.Unlock()
|
|
targetCommands[e.CommandName]++
|
|
ms := float64(e.DurationNanos) / 1e6
|
|
targetMaxMillis[e.CommandName] = max(targetMaxMillis[e.CommandName], ms)
|
|
if i, ok := factPending[e.RequestID]; ok {
|
|
factTrace[i]["durationMS"] = ms
|
|
factTrace[i]["failed"] = false
|
|
delete(factPending, e.RequestID)
|
|
}
|
|
}, Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
|
targetStatsMu.Lock()
|
|
defer targetStatsMu.Unlock()
|
|
if i, ok := factPending[e.RequestID]; ok {
|
|
factTrace[i]["durationMS"] = float64(e.DurationNanos) / 1e6
|
|
factTrace[i]["failed"] = true
|
|
factTrace[i]["maxTimeFailure"] = strings.Contains(e.Failure, "MaxTimeMS")
|
|
delete(factPending, e.RequestID)
|
|
}
|
|
}}
|
|
// Separate foreground report aggregation from legitimate background builds
|
|
// and paginated detail aggregates when checking finished-result fallback.
|
|
var apiWorkforceAggregations atomic.Int64
|
|
apiTargetMonitor := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) {
|
|
name, _ := e.Command.Lookup("aggregate").StringValueOK()
|
|
if name == "queryWorkforceFacts" {
|
|
apiWorkforceAggregations.Add(1)
|
|
}
|
|
targetMonitor.Started(ctx, e)
|
|
}, Succeeded: targetMonitor.Succeeded, Failed: targetMonitor.Failed}
|
|
monitor := &event.CommandMonitor{
|
|
Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if phase != "" {
|
|
m := metrics[phase]
|
|
m.Commands[e.CommandName]++
|
|
if m.Collections == nil {
|
|
m.Collections = map[string]int64{}
|
|
}
|
|
value := e.Command.Lookup(e.CommandName)
|
|
if e.CommandName == "getMore" {
|
|
value = e.Command.Lookup("collection")
|
|
}
|
|
if collection, ok := value.StringValueOK(); ok {
|
|
m.Collections[e.CommandName+":"+collection]++
|
|
}
|
|
m.RequestBytes += int64(len(e.Command))
|
|
inflight[e.RequestID] = phase
|
|
}
|
|
},
|
|
Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if name := inflight[e.RequestID]; name != "" {
|
|
metrics[name].ReplyBytes += int64(len(e.Reply))
|
|
delete(inflight, e.RequestID)
|
|
}
|
|
},
|
|
Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
|
mu.Lock()
|
|
delete(inflight, e.RequestID)
|
|
mu.Unlock()
|
|
},
|
|
}
|
|
connect := func(uri string, monitor *event.CommandMonitor) (*mongo.Database, *store.MongoStore) {
|
|
opts := options.Client().ApplyURI(uri).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second).SetMinPoolSize(uint64(minPool))
|
|
if compression != "" {
|
|
opts.SetCompressors([]string{compression})
|
|
}
|
|
c, err := mongo.Connect(ctx, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db := c.Database(domain.NewID("query_http_synthetic"))
|
|
t.Cleanup(func() {
|
|
clean, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
if local && os.Getenv("QUERY_COMPARE_KEEP_FAILED_FIXTURES") == "1" && t.Failed() {
|
|
t.Logf("retained owned diagnostic fixture %s", db.Name())
|
|
} else if err := db.Drop(clean); err != nil {
|
|
t.Errorf("synthetic cleanup %s: %v", db.Name(), err)
|
|
}
|
|
_ = c.Disconnect(clean)
|
|
})
|
|
return db, store.NewMongoStore(db)
|
|
}
|
|
sdb, old := connect("mongodb://127.0.0.1:27119/?directConnection=true", monitor)
|
|
db, _ := connect(testsupport.QueryReplicaTargetURI(t), targetMonitor)
|
|
if !local {
|
|
var marker bson.M
|
|
if err := db.Client().Database("query25_synthetic").Collection("marker").FindOne(ctx, bson.M{"_id": "query25-confirmed"}).Decode(&marker); err != nil || marker["value"] != "preserve-query25-v1" {
|
|
t.Fatalf("owned 25Gi target identity not confirmed: %v", err)
|
|
}
|
|
}
|
|
newReader := store.NewMongoStore(sdb)
|
|
newReader.EnableQuerySourceTracking(true)
|
|
if err := old.EnsureIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture := seedIsolatedHTTPComparison(t, sdb, old, n)
|
|
annotationDigest := isolatedAnnotationDigest(t, sdb)
|
|
peer := func(uri, name string, size uint64, monitor *event.CommandMonitor, majority bool) *store.MongoStore {
|
|
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(size).SetMonitor(monitor).SetServerSelectionTimeout(5 * time.Second)
|
|
if compression != "" {
|
|
opts.SetCompressors([]string{compression})
|
|
}
|
|
if majority {
|
|
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
|
|
}
|
|
client, err := mongo.Connect(ctx, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
stop, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
_ = client.Disconnect(stop)
|
|
})
|
|
return store.NewMongoStore(client.Database(name))
|
|
}
|
|
workerSource := peer("mongodb://127.0.0.1:27119/?directConnection=true", sdb.Name(), 4, monitor, false)
|
|
workerTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 4, targetMonitor, true)
|
|
q, err := store.NewQueryReplica(workerSource, workerTarget)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, fn := range []func(context.Context) error{q.VerifyTopology, q.EnsureSourceIndexes, q.EnsureTargetIndexes, q.Acquire} {
|
|
if err := fn(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
start := time.Now()
|
|
q.BatchSize = isolatedComparisonInt(t, "QUERY_COMPARE_BOOTSTRAP_BATCH", 200, 20, 200)
|
|
bootstrapRestarts := 0
|
|
t.Logf("bootstrap starting source=%s target=%s batch=%d", sdb.Name(), db.Name(), q.BatchSize)
|
|
for i := 0; ; i++ {
|
|
ready, err := q.RunOnce(ctx, true)
|
|
if err != nil {
|
|
if errors.Is(err, store.ErrQueryReplicaLease) && bootstrapRestarts < 3 {
|
|
// Reacquire only our own/expired lease, never another live owner.
|
|
// Durable checkpoints, not a fresh database, resume slow commits.
|
|
if err := q.Acquire(ctx); err != nil {
|
|
t.Fatal("bootstrap lease recovery", err)
|
|
}
|
|
bootstrapRestarts++
|
|
t.Logf("bootstrap lease recovered count=%d", bootstrapRestarts)
|
|
continue
|
|
}
|
|
t.Fatal("bootstrap", err)
|
|
}
|
|
if ready {
|
|
break
|
|
}
|
|
if i > n/q.BatchSize+100 {
|
|
t.Fatal("bootstrap did not converge")
|
|
}
|
|
if i%10 == 0 {
|
|
t.Logf("bootstrap pass=%d seconds=%.1f", i, time.Since(start).Seconds())
|
|
}
|
|
}
|
|
t.Logf("bootstrap tasks=%d seconds=%.3f source=%s target=%s", n, time.Since(start).Seconds(), sdb.Name(), db.Name())
|
|
q.BatchSize = 200
|
|
builderTarget := peer(testsupport.QueryReplicaTargetURI(t), db.Name(), 100, targetMonitor, true)
|
|
if err := builderTarget.VerifyQueryReplicaTarget(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
closeBuilderCache, e := builderTarget.ConfigureQueryServingFromEnvironment()
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
t.Cleanup(closeBuilderCache)
|
|
if published {
|
|
cache, ok := builderTarget.SharedQueryCache().(*queryserving.ManagedValkey)
|
|
if !ok {
|
|
t.Fatal("missing dedicated shared cache")
|
|
}
|
|
until := time.Now().Add(5 * time.Second)
|
|
for !cache.Ready() && time.Now().Before(until) {
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if !cache.Ready() {
|
|
t.Fatal("dedicated cache not ready")
|
|
}
|
|
}
|
|
if err := builderTarget.ConfigureQueryRefreshMode(os.Getenv("QUERY_REPLICA_REFRESH_MODE")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := builderTarget.EnsureQueryReadModelIndexes(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
workerCtx, stopWorkers := context.WithCancel(ctx)
|
|
defer stopWorkers()
|
|
builder := store.NewQueryReadBuilder(builderTarget)
|
|
|
|
workerErrors := make(chan error, 2)
|
|
var workers sync.WaitGroup
|
|
workers.Add(2)
|
|
go func() {
|
|
defer workers.Done()
|
|
if err := builder.Run(workerCtx); err != nil && workerCtx.Err() == nil {
|
|
workerErrors <- err
|
|
}
|
|
}()
|
|
// The measured new phase includes every poll, reconcile and acknowledgement.
|
|
projectorEnabled := make(chan struct{})
|
|
go func() {
|
|
defer workers.Done()
|
|
select {
|
|
case <-projectorEnabled:
|
|
case <-workerCtx.Done():
|
|
return
|
|
}
|
|
interval := time.Second
|
|
if os.Getenv("QUERY_REPLICA_REFRESH_MODE") == "periodic" {
|
|
interval = store.QueryPeriodicSyncInterval
|
|
}
|
|
for workerCtx.Err() == nil {
|
|
if _, err := q.RunOnce(workerCtx, false); err != nil && workerCtx.Err() == nil {
|
|
workerErrors <- err
|
|
return
|
|
}
|
|
if err := q.WaitNextPass(workerCtx, interval); err != nil && workerCtx.Err() == nil {
|
|
workerErrors <- err
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
defer func() { stopWorkers(); workers.Wait(); builder.Close() }()
|
|
materializeStarted := time.Now()
|
|
readyCtx, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
|
err = builder.WaitReady(readyCtx, fixture.org, fixture.projects)
|
|
cancel()
|
|
if err != nil {
|
|
t.Fatal("query materialization", err)
|
|
}
|
|
materializeSeconds := time.Since(materializeStarted).Seconds()
|
|
// The projector is intentionally paused for the authority measurement. A
|
|
// long initial build must not leave that fixture's watermark expired before
|
|
// the first paired HTTP read, even though its source has had no changes.
|
|
if err := q.Acquire(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := q.PublishReady(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Use the actual API configuration entry point (20-connection, majority
|
|
// target client), separate from projector and builder connection pools.
|
|
t.Setenv("QUERY_REPLICA_REPORTS", "1")
|
|
t.Setenv("QUERY_REPLICA_POOLS", "0")
|
|
t.Setenv("QUERY_REPLICA_COUNTS", "1")
|
|
t.Setenv("QUERY_SOURCE_TRACKING", "1")
|
|
uri := testsupport.QueryReplicaTargetURI(t)
|
|
if compression != "" {
|
|
uri += "&compressors=" + compression
|
|
}
|
|
t.Setenv("QUERY_READ_MONGO_URI", uri)
|
|
t.Setenv("QUERY_READ_MONGO_DB", db.Name())
|
|
t.Setenv("QUERY_SERVING_READS", "1")
|
|
t.Setenv("QUERY_VALKEY_URL", cacheURL)
|
|
t.Setenv("QUERY_TYPESENSE_URL", "")
|
|
closeReplica, err := newReader.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(closeReplica)
|
|
serve := func(s store.Store) *httptest.Server {
|
|
h := &server{store: s, requireSessionAuth: true, authLimiter: newRateLimiter(20, time.Minute), externalWriteLimiter: newRateLimiter(120, time.Minute), dashboardResponses: dashboardResponseCache{now: time.Now}}
|
|
ts := httptest.NewServer(h.routes())
|
|
t.Cleanup(ts.Close)
|
|
return ts
|
|
}
|
|
before, after := serve(old), serve(newReader)
|
|
client := &http.Client{Timeout: 25 * time.Second}
|
|
request := func(url, path string, actor int) (int, []byte, error) {
|
|
r, err := http.NewRequestWithContext(ctx, http.MethodGet, url+path, nil)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
r.AddCookie(&http.Cookie{Name: sessionCookieName, Value: fixture.tokens[actor]})
|
|
if published {
|
|
r.Header.Set("Accept", "application/vnd.droboticflow.prepared+json")
|
|
}
|
|
resp, err := client.Do(r)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
body, err := io.ReadAll(resp.Body)
|
|
return resp.StatusCode, body, err
|
|
}
|
|
paths := []string{
|
|
"/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review",
|
|
"/api/dashboard/production?from=2026-11-01&to=2026-11-12",
|
|
"/api/dashboard/workforce?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&groupBy=member",
|
|
"/api/workforce-returns?from=2026-11-01T00:00:00Z&to=2026-11-13T00:00:00Z&limit=20&offset=0",
|
|
}
|
|
publicationWarmSeconds := float64(0)
|
|
publicationPending := 0
|
|
if published {
|
|
warmStart := time.Now()
|
|
deadline := materializeStarted.Add(4 * time.Minute)
|
|
pending := map[int]bool{}
|
|
for actor := 0; actor < maxActors; actor++ {
|
|
pending[actor] = true
|
|
}
|
|
for len(pending) > 0 {
|
|
for actor := range pending {
|
|
if !time.Now().Before(deadline) {
|
|
t.Fatalf("published responses did not become ready within original materialization deadline: %d", len(pending))
|
|
}
|
|
status, _, err := request(after.URL, paths[2], actor)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status == 200 {
|
|
delete(pending, actor)
|
|
} else if status == http.StatusAccepted {
|
|
publicationPending++
|
|
} else {
|
|
t.Fatalf("publication prewarm status %d", status)
|
|
}
|
|
}
|
|
if len(pending) > 0 {
|
|
time.Sleep(time.Second)
|
|
}
|
|
}
|
|
publicationWarmSeconds = time.Since(warmStart).Seconds()
|
|
t.Logf("published response startup seconds=%.3f pending_responses=%d", publicationWarmSeconds, publicationPending)
|
|
}
|
|
// First reads use the existing parity requests, with no extra query load.
|
|
// Exact business JSON equality; only generated report timestamps are volatile.
|
|
firstReads := map[string]any{}
|
|
for i, path := range paths {
|
|
start := time.Now()
|
|
a, ab, ae := request(before.URL, path, 0)
|
|
oldMs := float64(time.Since(start).Microseconds()) / 1000
|
|
start = time.Now()
|
|
b, bb, be := request(after.URL, path, 0)
|
|
firstReads[strings.Split(path, "?")[0]] = map[string]any{"oldMs": oldMs, "queryMs": float64(time.Since(start).Microseconds()) / 1000}
|
|
if ae != nil || be != nil || a != 200 || b != 200 {
|
|
t.Fatalf("route %d: old=%d/%v %.300s new=%d/%v %.300s", i, a, ae, ab, b, be, bb)
|
|
}
|
|
if !reflect.DeepEqual(isolatedBusinessJSON(t, ab), isolatedBusinessJSON(t, bb)) {
|
|
t.Fatalf("route %d business results differ: old %.500s new %.500s", i, ab, bb)
|
|
}
|
|
if i == 0 {
|
|
t.Logf("counter parity %s", ab)
|
|
}
|
|
}
|
|
// Keep output evidence independently identifiable rather than parsing t.Log.
|
|
report := map[string]any{"tasks": n, "projects": len(fixture.projects), "sourceDatabase": sdb.Name(), "targetDatabase": db.Name(), "cases": []any{}, "parity": true, "httpSessionAuth": true, "sourceMinimumPoolBothVariants": minPool, "apiQueryMaximumPool": 20, "projectorMaximumPool": 4, "builderMaximumPool": 100, "wireCompressionBothVariants": compression, "sourceReplyBytesKind": "logical BSON; not compressed wire bytes", "cache": "none; existing API snapshot/response caches unchanged"}
|
|
report["refreshMode"] = os.Getenv("QUERY_REPLICA_REFRESH_MODE")
|
|
report["localDocker"] = local
|
|
report["targetMetricsIncludeAPI"] = true
|
|
report["supplementalMaxInFlight"] = maxInflight
|
|
cases := []any{}
|
|
accepted := true
|
|
expectedResults := map[string]any{}
|
|
for _, mode := range []string{"old", "query"} {
|
|
url := before.URL
|
|
if mode == "query" {
|
|
url = after.URL
|
|
// Reacquire after the baseline measurement's idle period.
|
|
if err := q.Acquire(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := q.RunOnce(ctx, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
close(projectorEnabled)
|
|
}
|
|
for _, concurrency := range []int{81, 200, 500} {
|
|
if concurrency > maxActors {
|
|
continue
|
|
}
|
|
name := fmt.Sprintf("%s-%d", mode, concurrency)
|
|
// Each variant pays the same session lookup cost; do not let the old
|
|
// phase warm a shared authentication cache for the query phase.
|
|
principalCacheMu.Lock()
|
|
principalCache = map[string]principalCacheEntry{}
|
|
principalCacheMu.Unlock()
|
|
scansBefore := isolatedSourceScans(t, sdb)
|
|
cpuBefore := isolatedSourceCPU(t)
|
|
mu.Lock()
|
|
phase = name
|
|
metrics[name] = &isolatedCommands{Commands: map[string]int64{}}
|
|
mu.Unlock()
|
|
t.Log("phase", name, time.Now().UTC().Format(time.RFC3339Nano))
|
|
started := time.Now()
|
|
latency := make([]float64, 0, concurrency*waves)
|
|
byRoute := map[string][]float64{}
|
|
errors := map[string]int{}
|
|
var firstError string
|
|
var responses int64
|
|
matchedResults, mismatches := 0, 0
|
|
var slots chan struct{}
|
|
if maxInflight > 0 {
|
|
slots = make(chan struct{}, maxInflight)
|
|
}
|
|
waveReports := []any{}
|
|
for wave := 0; wave < waves; wave++ {
|
|
offsets := map[string]int{}
|
|
for path, samples := range byRoute {
|
|
offsets[path] = len(samples)
|
|
}
|
|
var wg sync.WaitGroup
|
|
var results sync.Mutex
|
|
barrier := make(chan struct{})
|
|
for actor := 0; actor < concurrency; actor++ {
|
|
wg.Add(1)
|
|
go func(actor int) {
|
|
defer wg.Done()
|
|
<-barrier
|
|
if slots != nil {
|
|
slots <- struct{}{}
|
|
defer func() { <-slots }()
|
|
}
|
|
path := paths[2+(actor+wave)%2]
|
|
if actor%81 == 0 {
|
|
path = paths[1]
|
|
} else if actor%81 >= 61 {
|
|
path = paths[0]
|
|
}
|
|
s := time.Now()
|
|
status, body, err := request(url, path, actor)
|
|
results.Lock()
|
|
defer results.Unlock()
|
|
ms := float64(time.Since(s).Microseconds()) / 1000
|
|
latency = append(latency, ms)
|
|
key := strings.Split(path, "?")[0]
|
|
byRoute[key] = append(byRoute[key], ms)
|
|
responses += int64(len(body))
|
|
if status == 200 && err == nil {
|
|
id := fmt.Sprintf("%d/%d/%s", wave, actor, path)
|
|
value := isolatedBusinessJSON(t, body)
|
|
if mode == "old" {
|
|
expectedResults[id] = value
|
|
} else if expected, ok := expectedResults[id]; ok {
|
|
matchedResults++
|
|
if !reflect.DeepEqual(expected, value) {
|
|
mismatches++
|
|
}
|
|
}
|
|
}
|
|
if status != 200 || err != nil {
|
|
errors[fmt.Sprintf("%d/%v", status, err)]++
|
|
if firstError == "" {
|
|
firstError = fmt.Sprintf("%s: %d/%v %.300s", key, status, err, body)
|
|
}
|
|
}
|
|
}(actor)
|
|
}
|
|
close(barrier)
|
|
wg.Wait()
|
|
waveRoutes := map[string]any{}
|
|
for path, samples := range byRoute {
|
|
values := append([]float64(nil), samples[offsets[path]:]...)
|
|
if len(values) == 0 {
|
|
continue
|
|
}
|
|
sort.Float64s(values)
|
|
waveRoutes[path] = map[string]any{"requests": len(values), "p95Ms": values[(len(values)*95+99)/100-1], "maxMs": values[len(values)-1]}
|
|
}
|
|
waveReports = append(waveReports, map[string]any{"wave": wave + 1, "routes": waveRoutes})
|
|
}
|
|
mu.Lock()
|
|
phase = ""
|
|
original := metrics[name]
|
|
m := &isolatedCommands{Commands: map[string]int64{}, Collections: map[string]int64{}, RequestBytes: original.RequestBytes, ReplyBytes: original.ReplyBytes}
|
|
for collection, count := range original.Collections {
|
|
m.Collections[collection] = count
|
|
}
|
|
for cmd, count := range original.Commands {
|
|
m.Commands[cmd] = count
|
|
}
|
|
mu.Unlock()
|
|
cpu := isolatedSourceCPU(t) - cpuBefore
|
|
scans := isolatedSourceScans(t, sdb)
|
|
for key, before := range scansBefore {
|
|
scans[key] -= before
|
|
}
|
|
sort.Float64s(latency)
|
|
routes := map[string]any{}
|
|
for path, samples := range byRoute {
|
|
sort.Float64s(samples)
|
|
p95 := samples[(len(samples)*95+99)/100-1]
|
|
routes[path] = map[string]any{"requests": len(samples), "p95Ms": p95, "maxMs": samples[len(samples)-1]}
|
|
if mode == "query" && p95 > 2000 {
|
|
t.Errorf("%s %s p95 exceeds preliminary 2s bound", name, path)
|
|
}
|
|
}
|
|
result := map[string]any{"name": name, "seconds": time.Since(started).Seconds(), "requests": len(latency), "p50Ms": latency[(len(latency)-1)/2], "p95Ms": latency[(len(latency)-1)*95/100], "errors": errors, "firstError": firstError, "routes": routes, "responseBytes": responses, "sourceCPUSeconds": cpu, "sourceScans": scans, "source": m}
|
|
result["waves"] = waveReports
|
|
result["matchedBusinessResults"], result["businessMismatches"] = matchedResults, mismatches
|
|
if mode == "query" && mismatches > 0 {
|
|
t.Errorf("%d per-actor business results differed", mismatches)
|
|
accepted = false
|
|
}
|
|
cases = append(cases, result)
|
|
encoded, _ := json.Marshal(result)
|
|
t.Log(string(encoded))
|
|
if len(errors) != 0 {
|
|
accepted = false
|
|
t.Errorf("%s has HTTP failures: %s", name, firstError)
|
|
break
|
|
}
|
|
if mode == "query" && latency[(len(latency)-1)*95/100] > 2000 {
|
|
t.Errorf("%s p95 exceeds preliminary 2s bound", name)
|
|
}
|
|
}
|
|
}
|
|
if published {
|
|
// Separate list migration from count/report rollout; formal operations
|
|
// and personal pages keep their authoritative path.
|
|
newReader.EnableQueryReplicaPoolReads(true)
|
|
poolCases := []any{}
|
|
expectedPool := map[string]any{}
|
|
for _, mode := range []string{"old", "query"} {
|
|
url := before.URL
|
|
if mode == "query" {
|
|
url = after.URL
|
|
}
|
|
for _, pool := range []string{"available", "available&search=part0001", "available&search=absent-capture", "mine", "reviewed"} {
|
|
principalCacheMu.Lock()
|
|
principalCache = map[string]principalCacheEntry{}
|
|
principalCacheMu.Unlock()
|
|
latencies := []float64{}
|
|
var guard sync.Mutex
|
|
var wg sync.WaitGroup
|
|
for actor := 61; actor < 81; actor++ {
|
|
wg.Add(1)
|
|
go func(actor int) {
|
|
defer wg.Done()
|
|
path := "/api/human-items?pluginId=annotation-qa&stageId=semantic_review&limit=20&pool=" + pool
|
|
start := time.Now()
|
|
status, body, err := request(url, path, actor)
|
|
guard.Lock()
|
|
defer guard.Unlock()
|
|
latencies = append(latencies, float64(time.Since(start).Microseconds())/1000)
|
|
if status != 200 || err != nil {
|
|
t.Errorf("pool %s/%s/%d: status=%d err=%v %.200s", mode, pool, actor, status, err, body)
|
|
return
|
|
}
|
|
key := fmt.Sprintf("%s/%d", pool, actor)
|
|
value := isolatedBusinessJSON(t, body)
|
|
if mode == "old" {
|
|
expectedPool[key] = value
|
|
} else if !reflect.DeepEqual(expectedPool[key], value) {
|
|
t.Errorf("pool business mismatch %s", key)
|
|
}
|
|
}(actor)
|
|
}
|
|
wg.Wait()
|
|
sort.Float64s(latencies)
|
|
p95 := latencies[(len(latencies)-1)*95/100]
|
|
poolCases = append(poolCases, map[string]any{"mode": mode, "pool": pool, "requests": len(latencies), "p95Ms": p95})
|
|
if mode == "query" && p95 > 2000 {
|
|
t.Errorf("%s pool p95 exceeds 2s: %f", pool, p95)
|
|
}
|
|
}
|
|
}
|
|
report["poolListCases"] = poolCases
|
|
newReader.EnableQueryReplicaPoolReads(false)
|
|
}
|
|
if published {
|
|
opts, e := valkey.ParseURL(cacheURL)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
vc, e := valkey.NewClient(opts)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer vc.Close()
|
|
if e = vc.Do(ctx, vc.B().Flushdb().Sync().Build()).Error(); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
beforeAggregate := apiWorkforceAggregations.Load()
|
|
fallback := map[string]float64{}
|
|
for _, path := range paths {
|
|
now := time.Now()
|
|
beforeReport := apiWorkforceAggregations.Load()
|
|
status, _, e := request(after.URL, path, 0)
|
|
if e != nil || status != 200 {
|
|
t.Fatalf("cache loss fallback status=%d err=%v", status, e)
|
|
}
|
|
if strings.HasPrefix(path, "/api/dashboard/workforce?") && apiWorkforceAggregations.Load() != beforeReport {
|
|
t.Error("cache loss reran report aggregation")
|
|
}
|
|
fallback[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
|
|
}
|
|
report["cacheLossFallbackMs"] = fallback
|
|
report["cacheLossDetailAndReportAggregations"] = apiWorkforceAggregations.Load() - beforeAggregate
|
|
// A separate API client with an unreachable loopback cache must still
|
|
// serve durable products; this does not stop any cluster service.
|
|
t.Setenv("QUERY_VALKEY_URL", "redis://127.0.0.1:1/0")
|
|
blocked := store.NewMongoStore(sdb)
|
|
finish, e := blocked.ConfigureQueryReplicaFromEnvironment(ctx, apiTargetMonitor)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer finish()
|
|
unavailable := serve(blocked)
|
|
unavailableMs := map[string]float64{}
|
|
for _, path := range paths {
|
|
now := time.Now()
|
|
status, _, e := request(unavailable.URL, path, 0)
|
|
if e != nil || status != 200 {
|
|
t.Fatalf("unavailable cache fallback status=%d err=%v", status, e)
|
|
}
|
|
unavailableMs[strings.Split(path, "?")[0]] = float64(time.Since(now).Microseconds()) / 1000
|
|
}
|
|
report["cacheUnavailableFallbackMs"] = unavailableMs
|
|
}
|
|
if published {
|
|
report["cache"] = "dedicated Valkey with majority-committed durable response fallback"
|
|
}
|
|
report["publishedResults"] = published
|
|
report["publicationPrewarmSeconds"] = publicationWarmSeconds
|
|
report["publicationPendingResponses"] = publicationPending
|
|
report["materializationSeconds"] = materializeSeconds
|
|
report["firstReads"] = firstReads
|
|
report["cases"] = cases
|
|
report["errorFree"] = accepted
|
|
report["capacityAccepted"] = false
|
|
report["bootstrapLeaseRecoveries"] = bootstrapRestarts
|
|
report["queryServing"] = newReader.QueryServingStatistics()
|
|
if newReader.QueryServingStatistics()["report_hits"] == 0 {
|
|
t.Error("production read did not use its background materialized report")
|
|
}
|
|
if actual := isolatedAnnotationDigest(t, sdb); actual != annotationDigest {
|
|
t.Fatal("read projection changed task status, version or annotation")
|
|
}
|
|
report["annotationsAndVersionsPreserved"] = true
|
|
targetStatsMu.Lock()
|
|
report["targetWorkerCommands"] = targetCommands
|
|
report["targetWorkerMaxRoundTripMs"] = targetMaxMillis
|
|
targetStatsMu.Unlock()
|
|
select {
|
|
case err := <-workerErrors:
|
|
t.Fatal("background worker failed", err)
|
|
default:
|
|
}
|
|
if path := os.Getenv("QUERY_COMPARE_REPORT"); path != "" {
|
|
targetStatsMu.Lock()
|
|
data, err := json.MarshalIndent(report, "", " ")
|
|
targetStatsMu.Unlock()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, data, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func isolatedAnnotationDigest(t *testing.T, db *mongo.Database) string {
|
|
t.Helper()
|
|
cur, err := db.Collection("humanItems").Find(t.Context(), bson.M{}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "version": 1, "status": 1, "assignedUserId": 1, "subject.payload.annotationDraft": 1}).SetSort(bson.D{{Key: "id", Value: 1}}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer cur.Close(t.Context())
|
|
hash := sha256.New()
|
|
for cur.Next(t.Context()) {
|
|
var row bson.M
|
|
if err := cur.Decode(&row); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data, err := json.Marshal(row)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, _ = hash.Write(data)
|
|
}
|
|
if err := cur.Err(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return fmt.Sprintf("%x", hash.Sum(nil))
|
|
}
|
|
|
|
func isolatedSourceScans(t *testing.T, db *mongo.Database) map[string]int64 {
|
|
t.Helper()
|
|
var stats struct {
|
|
Network struct {
|
|
BytesIn int64 `bson:"bytesIn"`
|
|
BytesOut int64 `bson:"bytesOut"`
|
|
PhysicalBytesIn int64 `bson:"physicalBytesIn"`
|
|
PhysicalBytesOut int64 `bson:"physicalBytesOut"`
|
|
} `bson:"network"`
|
|
Metrics struct {
|
|
QueryExecutor struct {
|
|
Scanned int64 `bson:"scanned"`
|
|
ScannedObjects int64 `bson:"scannedObjects"`
|
|
} `bson:"queryExecutor"`
|
|
} `bson:"metrics"`
|
|
}
|
|
if err := db.Client().Database("admin").RunCommand(t.Context(), bson.M{"serverStatus": 1}).Decode(&stats); err != nil {
|
|
t.Fatal("synthetic source scan counters", err)
|
|
}
|
|
return map[string]int64{"indexEntries": stats.Metrics.QueryExecutor.Scanned, "documents": stats.Metrics.QueryExecutor.ScannedObjects, "networkLogicalBytesIn": stats.Network.BytesIn, "networkLogicalBytesOut": stats.Network.BytesOut, "networkPhysicalBytesIn": stats.Network.PhysicalBytesIn, "networkPhysicalBytesOut": stats.Network.PhysicalBytesOut}
|
|
}
|
|
|
|
func isolatedSourceCPU(t *testing.T) float64 {
|
|
t.Helper()
|
|
if os.Getenv("QUERY_COMPARE_IN_CLUSTER") == "1" {
|
|
if runtime.GOOS != "linux" || (os.Getenv("NODE_NAME") != "10.189.16.4" && os.Getenv("NODE_NAME") != "10.189.16.33") || os.Getenv("POD_NAME") != "node4-query-fixture" {
|
|
t.Fatal("isolated source process identity rejected")
|
|
}
|
|
raw, err := os.ReadFile("/synthetic/mongod.pid")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pid := strings.TrimSpace(string(raw))
|
|
if _, err := strconv.ParseUint(pid, 10, 32); err != nil {
|
|
t.Fatal("invalid synthetic source pid")
|
|
}
|
|
command, err := os.ReadFile("/proc/" + pid + "/cmdline")
|
|
if err != nil || !strings.Contains(string(command), "--dbpath\x00/synthetic/source\x00") || !strings.Contains(string(command), "--port\x0027119\x00") {
|
|
t.Fatal("source PID does not own the synthetic database")
|
|
}
|
|
stat, err := os.ReadFile("/proc/" + pid + "/stat")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fields := strings.Fields(string(stat))
|
|
if len(fields) < 15 || fields[1] != "(mongod)" {
|
|
t.Fatal("unexpected synthetic process counters")
|
|
}
|
|
clock, err := exec.Command("getconf", "CLK_TCK").Output()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hz, err := strconv.ParseFloat(strings.TrimSpace(string(clock)), 64)
|
|
if err != nil || hz <= 0 {
|
|
t.Fatal("invalid process clock")
|
|
}
|
|
user, e1 := strconv.ParseFloat(fields[13], 64)
|
|
system, e2 := strconv.ParseFloat(fields[14], 64)
|
|
if e1 != nil || e2 != nil {
|
|
t.Fatal("invalid process CPU counters")
|
|
}
|
|
return (user + system) / hz
|
|
}
|
|
raw, err := exec.Command("docker", "exec", "fanyu-query-replica-local-source-1", "cat", "/sys/fs/cgroup/cpu.stat").Output()
|
|
if err != nil {
|
|
t.Fatal("isolated source cgroup unavailable", err)
|
|
}
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) == 2 && fields[0] == "usage_usec" {
|
|
value, err := strconv.ParseFloat(fields[1], 64)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return value / 1e6
|
|
}
|
|
}
|
|
t.Fatal("isolated source CPU counter missing")
|
|
return 0
|
|
}
|
|
|
|
type isolatedCommands struct {
|
|
Collections map[string]int64 `json:"collections"`
|
|
Commands map[string]int64 `json:"commands"`
|
|
RequestBytes int64 `json:"requestBytes"`
|
|
ReplyBytes int64 `json:"replyBytes"`
|
|
}
|
|
|
|
type isolatedHTTPFixture struct {
|
|
org string
|
|
projects []string
|
|
tokens []string
|
|
}
|
|
|
|
func isolatedComparisonInt(t *testing.T, key string, fallback, min, max int) int {
|
|
t.Helper()
|
|
if os.Getenv(key) == "" {
|
|
return fallback
|
|
}
|
|
n, err := strconv.Atoi(os.Getenv(key))
|
|
if err != nil || n < min || n > max {
|
|
t.Fatalf("invalid %s", key)
|
|
}
|
|
return n
|
|
}
|
|
|
|
func isolatedBusinessJSON(t *testing.T, data []byte) any {
|
|
t.Helper()
|
|
var result any
|
|
if err := json.Unmarshal(data, &result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Remove only named generation timestamps, never business dates or versions.
|
|
var walk func(any)
|
|
walk = func(v any) {
|
|
switch x := v.(type) {
|
|
case map[string]any:
|
|
delete(x, "asOf")
|
|
delete(x, "generatedAt")
|
|
for _, child := range x {
|
|
walk(child)
|
|
}
|
|
case []any:
|
|
for _, child := range x {
|
|
walk(child)
|
|
}
|
|
}
|
|
}
|
|
walk(result)
|
|
return result
|
|
}
|
|
|
|
func seedIsolatedHTTPComparison(t *testing.T, db *mongo.Database, s *store.MongoStore, n int) isolatedHTTPFixture {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
f := isolatedHTTPFixture{org: "query-synthetic-org"}
|
|
must := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: f.org, Name: "Synthetic only", Slug: "query-synthetic", Status: domain.OrganizationStatusActive}))
|
|
for i := 0; i < 3; i++ {
|
|
must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: fmt.Sprintf("v%d", i), OrganizationID: f.org, Status: "active", Name: fmt.Sprintf("Synthetic supplier %d", i)}))
|
|
}
|
|
for i := 0; i < 20; i++ {
|
|
id := fmt.Sprintf("p%02d", i)
|
|
f.projects = append(f.projects, id)
|
|
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: id, OrganizationID: f.org, Name: "Synthetic " + id, Status: domain.ProjectStatusActive}))
|
|
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "b" + id, RunID: "batch-run-" + id, GateNodeID: "gate", OrganizationID: f.org, ProjectID: id, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV5HumanStages()}))
|
|
}
|
|
_, err := s.SaveProductionStatsScope(ctx, f.org, productionstats.Scope{ProjectIDs: f.projects, Revision: 1}, 0)
|
|
must(err)
|
|
rows := map[string][]any{}
|
|
push := func(c string, d any) {
|
|
if c == "annotationTasks" {
|
|
raw, err := bson.Marshal(d)
|
|
must(err)
|
|
var doc bson.M
|
|
must(bson.Unmarshal(raw, &doc))
|
|
doc["sourceKey"] = doc["id"]
|
|
d = doc
|
|
}
|
|
rows[c] = append(rows[c], d)
|
|
}
|
|
flush := func() {
|
|
for c, docs := range rows {
|
|
if len(docs) > 0 {
|
|
_, err := db.Collection(c).InsertMany(ctx, docs)
|
|
must(err)
|
|
}
|
|
}
|
|
rows = map[string][]any{}
|
|
}
|
|
for i := 0; i < 500; i++ {
|
|
role := domain.UserRoleAnnotator
|
|
if i%81 == 0 {
|
|
role = domain.UserRoleAdmin
|
|
} else if i%81 >= 73 {
|
|
role = domain.UserRoleAcceptanceReviewer
|
|
} else if i%81 >= 61 {
|
|
role = domain.UserRoleQualityReviewer
|
|
}
|
|
uid, vendor := fmt.Sprintf("u%03d", i), fmt.Sprintf("v%d", i%3)
|
|
push("users", domain.UserRecord{ID: uid, OrganizationID: f.org, Role: role, Name: uid, Phone: fmt.Sprintf("138%08d", i), Status: domain.UserStatusActive, SupplierID: vendor})
|
|
token, err := randomSessionToken()
|
|
must(err)
|
|
f.tokens = append(f.tokens, token)
|
|
push("sessions", domain.SessionRecord{ID: "session-" + uid, UserID: uid, OrganizationID: f.org, TokenHash: sessionTokenHash(token), ExpiresAt: time.Now().Add(3 * time.Hour).UTC().Format(time.RFC3339), ExpiresAtTime: time.Now().Add(3 * time.Hour)})
|
|
for _, p := range f.projects {
|
|
push("projectMembers", domain.ProjectMemberRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, SupplierID: vendor})
|
|
ar := domain.AnnotationRoleAnnotator
|
|
if role == domain.UserRoleQualityReviewer {
|
|
ar = domain.AnnotationRoleQualityReviewer
|
|
}
|
|
push("annotationMemberships", domain.ProjectAnnotationMembershipRecord{ID: p + uid, OrganizationID: f.org, ProjectID: p, UserID: uid, WorkgroupID: vendor, VendorID: vendor, AssignmentRole: ar})
|
|
}
|
|
}
|
|
flush()
|
|
for i := 0; i < n; i++ {
|
|
id := fmt.Sprintf("t%06d", i)
|
|
p, vendor, uid := f.projects[i%len(f.projects)], fmt.Sprintf("v%d", (i%60+1)%3), fmt.Sprintf("u%03d", i%60+1)
|
|
at := fmt.Sprintf("2026-11-%02dT10:00:00Z", i%10+1)
|
|
run := domain.RunRecord{ID: "r" + id, OrganizationID: f.org, ProjectID: p, TaskID: "c" + id, Status: domain.RunStatusWaitingForHuman, CaptureDirectories: []string{"SYNTHETIC/" + id}, CreatedAt: at}
|
|
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: f.org, ProjectID: p, RunID: run.ID, ArtifactID: "SYNTHETIC/" + id + "/part0001", SourceVideo: "bos://synthetic.invalid/video.mp4", ClipEnd: 60, CurrentRevision: 1, Status: domain.AnnotationTaskStatusPendingQuality, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, TemplateVersion: 5, CreatedAt: at, UpdatedAt: at}
|
|
item := domain.HumanItemRecord{ID: "i" + id, OrganizationID: f.org, ProjectID: p, BatchID: "b" + p, RunID: run.ID, LegacyTaskID: id, PluginID: "annotation-qa", CurrentStageID: domain.FanyuV5StageSemanticReview, Status: domain.HumanItemReady, SupplierID: vendor, WorkgroupID: vendor, VendorID: vendor, LatestAuthorUserID: uid, CurrentRevision: 1, Version: 1, CreatedAt: at, UpdatedAt: at, Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"durationSeconds": 60, "annotationDraft": strings.Repeat("synthetic protected annotation;", 128)}}}
|
|
push("runs", run)
|
|
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: f.org, ProjectID: p, Prefix: "SYNTHETIC/" + id, CaptureID: "SYNTHETIC/" + id, RunID: run.ID})
|
|
push("annotationTasks", task)
|
|
push("humanItems", item)
|
|
push("annotationRevisions", domain.AnnotationRevisionRecord{ID: "rev" + id, OrganizationID: f.org, TaskID: id, Revision: 1, AuthorUserID: uid, CountsAsOutput: true})
|
|
push("sliceRevisions", domain.SliceRevisionRecord{ID: "slice" + id, OrganizationID: f.org, TaskID: id, Revision: 1, Segments: []domain.SliceSegment{{ID: "s", Start: 0, End: 60}}})
|
|
push("annotationWorklogEvents", domain.AnnotationWorklogEventRecord{ID: "event" + id, OrganizationID: f.org, ProjectID: p, TaskID: id, UserID: uid, SupplierID: vendor, EventType: "combined_submitted", Revision: 1, OutputDelta: 1, CreatedAt: at})
|
|
if i%6 == 0 {
|
|
push("reviewOperations", domain.ReviewOperation{ID: "op" + id, OrganizationID: f.org, ProjectID: p, ItemID: item.ID, TaskID: id, StageID: "quality_review", Status: "completed", Decision: "rejected", CreatedAt: at, CompletedAt: at, ReviewRound: 1, Draft: domain.ReviewDraft{Problems: []domain.ReviewProblem{{ID: "problem", Scope: "slice", SliceID: "s", Start: 0, End: 10, TimeBasis: "capture-seconds", Text: "Synthetic return detail must survive"}}}, Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: "review" + id, TaskID: id, Revision: 1}}, {BeforeTask: &task}, {BeforeItem: &item, Item: &item}}})
|
|
}
|
|
if (i+1)%500 == 0 {
|
|
flush()
|
|
}
|
|
}
|
|
flush()
|
|
return f
|
|
}
|