droboticflow/internal/store/query_read_models_test.go

314 lines
11 KiB
Go

package store
import (
"context"
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/productionstats"
"droboticflow/internal/queryserving"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type queryModelFixture struct {
s *MongoStore
cache *queryserving.ManagedValkey
search *queryserving.Typesense
org string
}
func newQueryModelFixture(t *testing.T, n int) queryModelFixture {
t.Helper()
if os.Getenv("QUERY_SERVING_LOCAL_TEST") != "1" {
t.Skip("isolated query-serving dependencies")
}
ctx := t.Context()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true").SetServerSelectionTimeout(3*time.Second))
if err != nil {
t.Fatal(err)
}
db := client.Database(fmt.Sprintf("query_serving_test_%d", time.Now().UnixNano()))
s := NewMongoStore(db)
search, err := queryserving.NewTypesense("http://127.0.0.1:18108", os.Getenv("QUERY_TEST_TYPESENSE_API_KEY"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cur, e := s.queryData().Find(cleanup, bson.M{"kind": "search_collection"}, options.Find().SetProjection(bson.M{"collection": 1}))
if e == nil {
for cur.Next(cleanup) {
var row struct {
Collection string `bson:"collection"`
}
if cur.Decode(&row) == nil {
_ = search.Delete(cleanup, row.Collection)
}
}
_ = cur.Close(cleanup)
}
_ = db.Drop(cleanup)
_ = client.Disconnect(cleanup)
})
cache, err := queryserving.NewManagedValkey("valkey://127.0.0.1:16379", db.Name())
if err != nil {
t.Fatal(err)
}
t.Cleanup(cache.Close)
deadline := time.Now().Add(3 * time.Second)
for !cache.Ready() && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if !cache.Ready() {
t.Fatal("isolated cache unavailable")
}
s.ConfigureQueryServing(cache, search)
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
for _, c := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects", "humanBatches"} {
_, err := db.Collection(c).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)})
must(err)
}
must(s.EnsureQueryReadModelIndexes(ctx))
org := "query-org"
must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}))
must(s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "历史项目"}))
must(s.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: "p", PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}))
_, err = s.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: []string{"p"}, Revision: 1}, 0)
must(err)
docs := map[string][]any{}
push := func(c string, value any, legacy bool) {
raw, e := bson.Marshal(value)
must(e)
var d bson.M
must(bson.Unmarshal(raw, &d))
if !legacy {
stampPoolSearchMetadata(c, d)
}
docs[c] = append(docs[c], d)
}
flush := func() {
for c, rows := range docs {
if len(rows) > 0 {
_, e := db.Collection(c).InsertMany(ctx, rows)
must(e)
}
}
docs = map[string][]any{}
}
for i := 0; i < n; i++ {
id := fmt.Sprintf("item-%06d", i)
capture := fmt.Sprintf("FYLX_TEST_%06d", i)
if i == 0 {
capture = "FYLX_B07_AZER764008K_20260927_074440"
}
if i == 1 {
capture = "İ中文😀"
}
run := domain.RunRecord{ID: "run-" + id, OrganizationID: org, ProjectID: "p", TaskID: "capture-" + id, CaptureDirectories: []string{capture}, CreatedAt: "2026-11-10T00:00:00Z", Status: domain.RunStatusWaitingForHuman}
task := domain.AnnotationTaskRecord{ID: "task-" + id, OrganizationID: org, ProjectID: "p", RunID: run.ID, ArtifactID: capture + "/part0002/camera_left.mp4", SourceVideo: capture + "/part0002/camera_left.mp4", ClipEnd: 10}
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", BatchID: "batch", PluginID: "annotation-qa", RunID: run.ID, LegacyTaskID: task.ID, CurrentStageID: "quality_review", Status: domain.HumanItemReady, SupplierID: fmt.Sprintf("supplier-%d", i%2), Version: 1, CreatedAt: time.Date(2026, 11, 10, 0, 0, i, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"captureDirectory": capture, "annotationDraft": map[string]any{"description": "synthetic annotation must remain unchanged"}}}}
if i < 3 {
item.Subject.Payload["unusedBody"] = strings.Repeat("x", 1<<20)
}
push("runs", run, i%3 == 0)
push("annotationTasks", task, i%3 == 0)
push("captureRecords", domain.CaptureRecord{ID: run.TaskID, OrganizationID: org, ProjectID: "p", CaptureID: capture, RunID: run.ID}, i%3 == 0)
push("humanItems", item, i%3 == 0)
if (i+1)%1000 == 0 {
flush()
}
}
flush()
return queryModelFixture{s: s, cache: cache, search: search, org: org}
}
func queryIDs(items []domain.HumanItemRecord) []string {
out := []string{}
for _, i := range items {
out = append(out, i.ID)
}
return out
}
func TestQueryReadModelsMongoEquivalenceFencingAndFailure(t *testing.T) {
f := newQueryModelFixture(t, 241)
ctx := t.Context()
s := f.s
before, err := s.GetHumanItem(ctx, "item-000000")
if err != nil {
t.Fatal(err)
}
old := s.queryServing
s.queryServing = nil
queries := []string{"FYLX_B07_AZER764008K_20260927_074440/part0002", "TEST_0000", "中文", "İ", "item-000001", "历史项目", "[missing]", "part0002", "camera_left", "000000/part"}
expected := map[string][]string{}
for _, q := range queries {
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
if e != nil {
t.Fatal(e)
}
expected[q] = queryIDs(items)
}
base, e := s.LoadProductionStats(ctx, f.org, []string{"p"})
if e != nil {
t.Fatal(e)
}
wantReport := productionstats.Build(base)
s.queryServing = old
b := NewQueryReadBuilder(s)
defer b.Close()
if err = b.RunOnce(ctx); err != nil {
t.Fatal(err)
}
for _, q := range queries {
candidateIDs, ready := s.querySearchCandidates(ctx, f.org, q, "")
if !ready {
t.Logf("accelerator unavailable for %q; verifying compatibility result", q)
}
set := map[string]bool{}
for _, id := range candidateIDs {
set[id] = true
}
for _, id := range expected[q] {
if ready && !set[id] {
t.Fatalf("candidate false negative query=%q id=%s", q, id)
}
}
items, e := s.ListHumanItems(ctx, f.org, domain.HumanItemQuery{PluginID: "annotation-qa", Search: q, Limit: 500})
if e != nil || !reflect.DeepEqual(queryIDs(items), expected[q]) {
t.Fatalf("result mismatch query=%q got=%v expected=%v err=%v", q, queryIDs(items), expected[q], e)
}
}
if stats := s.QueryServingStatistics(); stats["search_hits"] == 0 || stats["typesense_reads"] == 0 {
t.Fatal("new search path never exercised", stats)
}
report, asOf, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
if err != nil || asOf == "" || !reflect.DeepEqual(report, wantReport) {
t.Fatal("materialized production report differs", err)
}
after, err := s.GetHumanItem(ctx, before.ID)
if err != nil || !reflect.DeepEqual(before, after) {
t.Fatal("read model changed annotation, ownership or version")
}
// A late writer can stage data but cannot replace a newer owner's head.
l1, ok, err := s.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(err)
}
_, err = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}})
if err != nil {
t.Fatal(err)
}
l2, ok, err := s.acquireQueryModelLease(ctx)
if err != nil || !ok {
t.Fatal(err)
}
key := queryModelKey("production", f.org, []string{"p"})
head, ok := s.readQueryModelHead(ctx, key, time.Hour)
if !ok {
t.Fatal("missing production head")
}
if s.publishQueryModelHead(ctx, l1, key, head) == nil {
t.Fatal("expired writer published")
}
if err = s.publishQueryModelHead(ctx, l2, key, head); err != nil {
t.Fatal(err)
}
// Incomplete/corrupt derived storage never becomes zero business data.
_, err = s.queryData().UpdateOne(ctx, bson.M{"_id": queryModelBlobKey(key, head.Generation, 0)}, bson.M{"$set": bson.M{"data": []byte("corrupt")}})
if err != nil {
t.Fatal(err)
}
s.queryServing.cache = nil
got, _, err := s.LoadProductionReport(ctx, f.org, []string{"p"})
if err != nil || !reflect.DeepEqual(got, wantReport) {
t.Fatal("corrupt materialization failed to retain compatibility path", err)
}
}
func TestQuerySearchChangeObservationIgnoresDraftsAndRecoversNamesLocal(t *testing.T) {
f := newQueryModelFixture(t, 12)
s := f.s
ctx, cancel := context.WithCancel(t.Context())
b := NewQueryReadBuilder(s)
done := make(chan error, 1)
go func() { done <- b.Run(ctx) }()
defer func() { cancel(); <-done; b.Close() }()
wait := func(check func() bool) {
t.Helper()
end := time.Now().Add(20 * time.Second)
for time.Now().Before(end) {
if check() {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatal("streamed read model did not converge")
}
wait(func() bool { _, ready := s.querySearchCandidates(ctx, f.org, "FYLX", ""); return ready })
state, err := s.querySearchState(ctx)
if err != nil {
t.Fatal(err)
}
key := queryModelKey("search", f.org, nil)
generation := state.Heads[key].Generation
item, err := s.GetHumanItem(ctx, "item-000001")
if err != nil {
t.Fatal(err)
}
item.Version++
item.Status = domain.HumanItemClaimed
item.AssignedUserID = "synthetic-owner"
item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
if err = s.UpsertHumanItem(ctx, *item); err != nil {
t.Fatal(err)
}
time.Sleep(1500 * time.Millisecond)
state, err = s.querySearchState(ctx)
if err != nil || state.Heads[key].Generation != generation || state.Revisions[key] != state.Heads[key].Revision {
t.Fatal("ordinary task state invalidated name search", err)
}
if err = s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: f.org, Name: "新项目名称"}); err != nil {
t.Fatal(err)
}
wait(func() bool {
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
return ready && len(ids) == 12
})
before, err := s.GetHumanItem(ctx, "item-000000")
if err != nil {
t.Fatal(err)
}
if _, err = s.humanItems.DeleteOne(ctx, bson.M{"id": before.ID}); err != nil {
t.Fatal(err)
}
wait(func() bool {
ids, ready := s.querySearchCandidates(ctx, f.org, "新项目名称", "")
return ready && len(ids) == 11
})
// Old generations survive publication only as unreachable, bounded derived
// records; no original annotation or audit records are rewritten.
remaining, err := s.GetHumanItem(ctx, "item-000002")
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(remaining.Subject.Payload)
if !strings.Contains(string(raw), "synthetic annotation must remain unchanged") {
t.Fatal("background rebuild lost annotation")
}
}