forked from longtao.wu/droboticflow
314 lines
11 KiB
Go
314 lines
11 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/productionstats"
|
|
"droboticflow/internal/queryserving"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type queryModelFixture struct {
|
|
s *MongoStore
|
|
cache *queryserving.ManagedValkey
|
|
search *queryserving.Typesense
|
|
org string
|
|
}
|
|
|
|
func newQueryModelFixture(t *testing.T, n int) queryModelFixture {
|
|
t.Helper()
|
|
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
|
|
t.Skip("isolated query-serving dependencies")
|
|
}
|
|
ctx := t.Context()
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true").SetServerSelectionTimeout(3*time.Second))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db := client.Database(fmt.Sprintf("query_serving_test_%d", time.Now().UnixNano()))
|
|
s := NewMongoStore(db)
|
|
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
cur, e := s.queryData().Find(cleanup, bson.M{"kind": "search_collection"}, options.Find().SetProjection(bson.M{"collection": 1}))
|
|
if e == nil {
|
|
for cur.Next(cleanup) {
|
|
var row struct {
|
|
Collection string `bson:"collection"`
|
|
}
|
|
if cur.Decode(&row) == nil {
|
|
_ = search.Delete(cleanup, row.Collection)
|
|
}
|
|
}
|
|
_ = cur.Close(cleanup)
|
|
}
|
|
_ = db.Drop(cleanup)
|
|
_ = client.Disconnect(cleanup)
|
|
})
|
|
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(cache.Close)
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for !cache.Ready() && time.Now().Before(deadline) {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if !cache.Ready() {
|
|
t.Fatal("isolated cache unavailable")
|
|
}
|
|
s.ConfigureQueryServing(cache, search)
|
|
must := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, c := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects", "humanBatches"} {
|
|
_, err := db.Collection(c).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)})
|
|
must(err)
|
|
}
|
|
must(s.EnsureQueryReadModelIndexes(ctx))
|
|
org := "query-org"
|
|
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
|
|
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "历史项目"}))
|
|
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: "p", PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}))
|
|
_, err = s.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0)
|
|
must(err)
|
|
docs := map[string][]any{}
|
|
push := func(c string, value any, legacy bool) {
|
|
raw, e := bson.Marshal(value)
|
|
must(e)
|
|
var d bson.M
|
|
must(bson.Unmarshal(raw, &d))
|
|
if !legacy {
|
|
stampPoolSearchMetadata(c, d)
|
|
}
|
|
docs[c] = append(docs[c], d)
|
|
}
|
|
flush := func() {
|
|
for c, rows := range docs {
|
|
if len(rows) > 0 {
|
|
_, e := db.Collection(c).InsertMany(ctx, rows)
|
|
must(e)
|
|
}
|
|
}
|
|
docs = map[string][]any{}
|
|
}
|
|
for i := 0; i < n; i++ {
|
|
id := fmt.Sprintf("item-%06d", i)
|
|
capture := fmt.Sprintf("FYLX_TEST_%06d", i)
|
|
if i == 0 {
|
|
capture = "FYLX_B07_AZER764008K_20260927_074440"
|
|
}
|
|
if i == 1 {
|
|
capture = "İ中文😀"
|
|
}
|
|
run := domain.RunRecord{ID: "run-" + id, OrganizationID: org, ProjectID: "p", TaskID: "capture-" + id, CaptureDirectories: []string{capture}, CreatedAt: "2026-11-10T00:00:00Z", Status: domain.RunStatusWaitingForHuman}
|
|
task := domain.AnnotationTaskRecord{ID: "task-" + id, OrganizationID: org, ProjectID: "p", RunID: run.ID, ArtifactID: capture + "/part0002/camera_left.mp4", SourceVideo: capture + "/part0002/camera_left.mp4", ClipEnd: 10}
|
|
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", BatchID: "batch", PluginID: "annotation-qa", RunID: run.ID, LegacyTaskID: task.ID, CurrentStageID: "quality_review", Status: domain.HumanItemReady, SupplierID: fmt.Sprintf("supplier-%d", i%2), Version: 1, CreatedAt: time.Date(2026, 11, 10, 0, 0, i, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"captureDirectory": capture, "annotationDraft": map[string]any{"description": "synthetic annotation must remain unchanged"}}}}
|
|
if i < 3 {
|
|
item.Subject.Payload["unusedBody"] = strings.Repeat("x", 1<<20)
|
|
}
|
|
push("runs", run, i%3 == 0)
|
|
push("annotationTasks", task, i%3 == 0)
|
|
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: org, ProjectID: "p", CaptureID: capture, RunID: run.ID}, i%3 == 0)
|
|
push("humanItems", item, i%3 == 0)
|
|
if (i+1)%1000 == 0 {
|
|
flush()
|
|
}
|
|
}
|
|
flush()
|
|
return queryModelFixture{s: s, cache: cache, search: search, org: org}
|
|
}
|
|
|
|
func queryIDs(items []domain.HumanItemRecord) []string {
|
|
out := []string{}
|
|
for _, i := range items {
|
|
out = append(out, i.ID)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestQueryReadModelsMongoEquivalenceFencingAndFailure(t *testing.T) {
|
|
f := newQueryModelFixture(t, 241)
|
|
ctx := t.Context()
|
|
s := f.s
|
|
before, err := s.GetHumanItem(ctx, "item-000000")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
old := s.queryServing
|
|
s.queryServing = nil
|
|
queries := []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "TEST_0000", "中文", "İ", "item-000001", "历史项目", "[missing]", "part0002", "camera_left", "000000/part"}
|
|
expected := map[string][]string{}
|
|
for _, q := range queries {
|
|
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
expected[q] = queryIDs(items)
|
|
}
|
|
base, e := s.LoadProductionStats(ctx, f.org, []string{"p"})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
wantReport := productionstats.Build(base)
|
|
s.queryServing = old
|
|
b := NewQueryReadBuilder(s)
|
|
defer b.Close()
|
|
if err = b.RunOnce(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, q := range queries {
|
|
candidateIDs, ready := s.querySearchCandidates(ctx, f.org, q, "")
|
|
if !ready {
|
|
t.Logf("accelerator unavailable for %q; verifying compatibility result", q)
|
|
}
|
|
set := map[string]bool{}
|
|
for _, id := range candidateIDs {
|
|
set[id] = true
|
|
}
|
|
for _, id := range expected[q] {
|
|
if ready && !set[id] {
|
|
t.Fatalf("candidate false negative query=%q id=%s", q, id)
|
|
}
|
|
}
|
|
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
|
|
if e != nil || !reflect.DeepEqual(queryIDs(items), expected[q]) {
|
|
t.Fatalf("result mismatch query=%q got=%v expected=%v err=%v", q, queryIDs(items), expected[q], e)
|
|
}
|
|
}
|
|
if stats := s.QueryServingStatistics(); stats["search_hits"] == 0 || stats["typesense_reads"] == 0 {
|
|
t.Fatal("new search path never exercised", stats)
|
|
}
|
|
report, asOf, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
|
|
if err != nil || asOf == "" || !reflect.DeepEqual(report, wantReport) {
|
|
t.Fatal("materialized production report differs", err)
|
|
}
|
|
after, err := s.GetHumanItem(ctx, before.ID)
|
|
if err != nil || !reflect.DeepEqual(before, after) {
|
|
t.Fatal("read model changed annotation, ownership or version")
|
|
}
|
|
// A late writer can stage data but cannot replace a newer owner's head.
|
|
l1, ok, err := s.acquireQueryModelLease(ctx)
|
|
if err != nil || !ok {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
l2, ok, err := s.acquireQueryModelLease(ctx)
|
|
if err != nil || !ok {
|
|
t.Fatal(err)
|
|
}
|
|
key := queryModelKey("production", f.org, []string{"p"})
|
|
head, ok := s.readQueryModelHead(ctx, key, time.Hour)
|
|
if !ok {
|
|
t.Fatal("missing production head")
|
|
}
|
|
if s.publishQueryModelHead(ctx, l1, key, head) == nil {
|
|
t.Fatal("expired writer published")
|
|
}
|
|
if err = s.publishQueryModelHead(ctx, l2, key, head); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Incomplete/corrupt derived storage never becomes zero business data.
|
|
_, err = s.queryData().UpdateOne(ctx, bson.M{"_id": queryModelBlobKey(key, head.Generation, 0)}, bson.M{"$set": bson.M{"data": []byte("corrupt")}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s.queryServing.cache = nil
|
|
got, _, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
|
|
if err != nil || !reflect.DeepEqual(got, wantReport) {
|
|
t.Fatal("corrupt materialization failed to retain compatibility path", err)
|
|
}
|
|
}
|
|
|
|
func TestQuerySearchChangeObservationIgnoresDraftsAndRecoversNamesLocal(t *testing.T) {
|
|
f := newQueryModelFixture(t, 12)
|
|
s := f.s
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
b := NewQueryReadBuilder(s)
|
|
done := make(chan error, 1)
|
|
go func() { done <- b.Run(ctx) }()
|
|
defer func() { cancel(); <-done; b.Close() }()
|
|
wait := func(check func() bool) {
|
|
t.Helper()
|
|
end := time.Now().Add(20 * time.Second)
|
|
for time.Now().Before(end) {
|
|
if check() {
|
|
return
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
t.Fatal("streamed read model did not converge")
|
|
}
|
|
wait(func() bool { _, ready := s.querySearchCandidates(ctx, f.org, "FYLX", ""); return ready })
|
|
state, err := s.querySearchState(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key := queryModelKey("search", f.org, nil)
|
|
generation := state.Heads[key].Generation
|
|
item, err := s.GetHumanItem(ctx, "item-000001")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
item.Version++
|
|
item.Status = domain.HumanItemClaimed
|
|
item.AssignedUserID = "synthetic-owner"
|
|
item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
|
if err = s.UpsertHumanItem(ctx, *item); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(1500 * time.Millisecond)
|
|
state, err = s.querySearchState(ctx)
|
|
if err != nil || state.Heads[key].Generation != generation || state.Revisions[key] != state.Heads[key].Revision {
|
|
t.Fatal("ordinary task state invalidated name search", err)
|
|
}
|
|
if err = s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: f.org, Name: "新项目名称"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wait(func() bool {
|
|
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
|
|
return ready && len(ids) == 12
|
|
})
|
|
before, err := s.GetHumanItem(ctx, "item-000000")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": before.ID}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wait(func() bool {
|
|
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
|
|
return ready && len(ids) == 11
|
|
})
|
|
// Old generations survive publication only as unreachable, bounded derived
|
|
// records; no original annotation or audit records are rewritten.
|
|
remaining, err := s.GetHumanItem(ctx, "item-000002")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, _ := json.Marshal(remaining.Subject.Payload)
|
|
if !strings.Contains(string(raw), "synthetic annotation must remain unchanged") {
|
|
t.Fatal("background rebuild lost annotation")
|
|
}
|
|
}
|