droboticflow/cmd/query-replica/main_local_test.go
2026-11-14 22:07:22 -08:00

195 lines
7.2 KiB
Go

package main
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
)
func TestProjectorGuardedProcessRestartLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
t.Skip("isolated source/query lab required")
}
ctx := t.Context()
sourceClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true"))
if e != nil {
t.Fatal(e)
}
defer sourceClient.Disconnect(context.Background())
targetClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"))
if e != nil {
t.Fatal(e)
}
defer targetClient.Disconnect(context.Background())
source, target := sourceClient.Database(domain.NewID("query_sync_cmd")), targetClient.Database(domain.NewID("query_replica_cmd"))
t.Cleanup(func() { _ = source.Drop(context.Background()); _ = target.Drop(context.Background()) })
s, d := store.NewMongoStore(source), store.NewMongoStore(target)
q, e := store.NewQueryReplica(s, d)
if e != nil {
t.Fatal(e)
}
if e = q.EnsureSourceIndexes(ctx); e != nil {
t.Fatal(e)
}
if e = q.EnsureTargetIndexes(ctx); e != nil {
t.Fatal(e)
}
for _, id := range []string{"a", "b", "c"} {
_, e = source.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "status": "completed", "organizationId": "org", "privateDraft": "not projected"})
if e != nil {
t.Fatal(e)
}
}
binary := filepath.Join(t.TempDir(), "projector")
if e = exec.Command("go", "build", "-o", binary, ".").Run(); e != nil {
t.Fatal("local projector build failed")
}
now := time.Now().UTC()
recent := []store.QueryReplicaSafetyObservation{}
for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} {
recent = append(recent, store.QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1})
}
raw, _ := bson.Marshal(recent[2])
var fields bson.M
_ = bson.Unmarshal(raw, &fields)
fields["recent"] = recent
_, e = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true))
if e != nil {
t.Fatal(e)
}
env := []string{}
for _, entry := range os.Environ() {
if len(entry) >= 6 && (entry[:6] == "MONGO_" || len(entry) >= 6 && entry[:6] == "QUERY_") {
continue
}
env = append(env, entry)
}
env = append(env, "MONGO_URI=mongodb://127.0.0.1:27119/?directConnection=true", "MONGO_DB="+source.Name(), "QUERY_READ_MONGO_URI=mongodb://127.0.0.1:27129/?replicaSet=query-read-rs", "QUERY_READ_MONGO_DB="+target.Name(), "QUERY_SERVING_NAMESPACE=local:"+source.Name())
args := []string{"--writers-covered", "--bootstrap", "--batch-size=1", "--safety-run=process-test", "--max-source-records=2", "--source-records-per-second=50", "--source-bytes-per-second=262144", "--source-commands-per-second=8", "--source-cpu-baseline=85"}
start := func() (*exec.Cmd, <-chan error) {
t.Helper()
p := exec.Command(binary, args...)
p.Env = env
p.Stdout = io.Discard
p.Stderr = io.Discard
if e = p.Start(); e != nil {
t.Fatal(e)
}
done := make(chan error, 1)
go func() { done <- p.Wait() }()
t.Cleanup(func() {
_ = p.Process.Kill()
})
return p, done
}
child, done := start()
var state bson.M
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state)
if state["paused"] == true {
break
}
select {
case <-done:
t.Fatal("guarded process exited instead of parking")
default:
}
time.Sleep(50 * time.Millisecond)
}
if state["paused"] != true || state["reserved"] != int64(2) {
t.Fatal("process bypassed total budget", state)
}
_ = child.Process.Kill()
<-done
_, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
next, finished := start()
time.Sleep(600 * time.Millisecond)
select {
case <-finished:
t.Fatal("paused restart should park")
default:
}
var after bson.M
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
if after["reserved"] != state["reserved"] || after["resumeToken"] != state["resumeToken"] {
t.Fatal("process restart reset durable state")
}
_ = next.Process.Kill()
<-finished
// Operator status mode cannot connect to source, even with an invalid URI.
status := exec.Command(binary, "--safety-status")
status.Env = append(env, "MONGO_URI=mongodb://127.0.0.1:1/?directConnection=true")
out, e := status.Output()
if e != nil {
t.Fatal("target-only operator mode failed")
}
var v map[string]any
if json.Unmarshal(out, &v) != nil || v["paused"] != true {
t.Fatal("operator status invalid")
}
// Resume the same guarded run, then stop it while it is actively reading.
// SIGTERM must persist operator_stop rather than a misleading operation_error.
if e = d.RecordQueryReplicaObservation(ctx, store.QueryReplicaSafetyObservation{
At: time.Now().UTC(), CPU: 99, Ready: true, WritersCovered: true,
HostFreeBytes: 120 << 30, TargetUsedFraction: .1,
SafetySignalsVersion: 1, SourceAvailable: true,
}); e != nil {
t.Fatal("maintenance health unavailable")
}
_, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
for i, arg := range args {
if strings.HasPrefix(arg, "--max-source-records=") {
args[i] = "--max-source-records=10000"
}
}
args = append(args, "--resume-pause="+after["resumeToken"].(string), "--maintenance-initialization", "--maintenance-until="+time.Now().UTC().Add(10*time.Minute).Format(time.RFC3339))
active, stopped := start()
deadline = time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
if after["paused"] == false && after["reserved"].(int64) > 2 {
break
}
time.Sleep(50 * time.Millisecond)
}
if after["paused"] != false || after["reserved"].(int64) <= 2 {
t.Fatal("same-run maintenance resume did not preserve and advance budget")
}
if e = active.Process.Signal(syscall.SIGTERM); e != nil {
t.Fatal("could not signal isolated process")
}
select {
case e = <-stopped:
if e != nil {
t.Fatal("graceful stop failed")
}
case <-time.After(6 * time.Second):
t.Fatal("graceful stop did not finish")
}
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
if after["paused"] != true || after["reason"] != "operator_stop" || after["reserved"].(int64) < 2 {
t.Fatal("intentional stop misclassified or reset budget")
}
count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"})
if e != nil || count != 3 {
t.Fatal("source payload changed")
}
var copied bson.M
if e = target.Collection("annotationIndexJobs").FindOne(ctx, bson.M{}).Decode(&copied); e != nil || copied["privateDraft"] != nil {
t.Fatal("projection boundary failed", e)
}
}