forked from longtao.wu/droboticflow
🛡️ Add durable budgets and bounded recovery for shadow synchronization
This commit is contained in:
parent
1a6e61909e
commit
f73665cfa0
@ -19,6 +19,7 @@ import (
|
||||
func main() {
|
||||
once := flag.Bool("once", false, "build once; streaming freshness expires after exit")
|
||||
ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building")
|
||||
shadowRun := flag.String("shadow-safety-run", "", "opt-in independently monitored shadow builder; no authority gates")
|
||||
flag.Parse()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
@ -43,25 +44,58 @@ func main() {
|
||||
log.Fatal("builder accelerator configuration invalid")
|
||||
}
|
||||
defer closeServing()
|
||||
workCtx := ctx
|
||||
if *shadowRun != "" {
|
||||
guarded, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
workCtx = guarded
|
||||
if s.CheckQueryReplicaSafety(ctx, *shadowRun) != nil {
|
||||
log.Print("shadow builder safety paused")
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-guarded.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
short, stop := context.WithTimeout(guarded, 2*time.Second)
|
||||
err := s.CheckQueryReplicaSafety(short, *shadowRun)
|
||||
stop()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
if *ensure {
|
||||
if s.EnsureQueryReadModelIndexes(ctx) != nil {
|
||||
if s.EnsureQueryReadModelIndexes(workCtx) != nil {
|
||||
log.Fatal("derived index creation failed")
|
||||
}
|
||||
}
|
||||
b := store.NewQueryReadBuilder(s)
|
||||
defer b.Close()
|
||||
if *once {
|
||||
if b.RunOnce(ctx) != nil {
|
||||
if b.RunOnce(workCtx) != nil {
|
||||
log.Fatal("initial materialization incomplete")
|
||||
}
|
||||
return
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
if b.Run(ctx) != nil && ctx.Err() == nil {
|
||||
for workCtx.Err() == nil {
|
||||
if b.Run(workCtx) != nil && workCtx.Err() == nil {
|
||||
log.Print("query builder paused; retrying with full reconciliation")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-workCtx.Done():
|
||||
if ctx.Err() == nil {
|
||||
log.Print("shadow builder safety paused; explicit operator restart required")
|
||||
<-ctx.Done()
|
||||
}
|
||||
return
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
|
||||
@ -4,12 +4,17 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
@ -25,7 +30,104 @@ func main() {
|
||||
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
|
||||
batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)")
|
||||
minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)")
|
||||
safetyRun := flag.String("safety-run", "", "explicit durable shadow run identity; requires global transport and health budgets")
|
||||
maxRecords := flag.Int64("max-source-records", 1000, "durable total read reservations;0 explicitly permits full backfill")
|
||||
rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing")
|
||||
bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget")
|
||||
commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead")
|
||||
baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume")
|
||||
resume := flag.String("resume-pause", "", "exact current pause nonce after operator review")
|
||||
initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes")
|
||||
status := flag.Bool("safety-status", false, "target-only private safety status")
|
||||
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
|
||||
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
|
||||
flag.Parse()
|
||||
if *sampleSource {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if os.Getenv("MONGO_URI") == "" {
|
||||
log.Fatal("source binding required")
|
||||
}
|
||||
c, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")).SetMaxPoolSize(1))
|
||||
if e != nil {
|
||||
log.Fatal("source unavailable")
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
var info struct {
|
||||
Uptime int64 `bson:"uptimeMillis"`
|
||||
Extra struct {
|
||||
User int64 `bson:"user_time_us"`
|
||||
System int64 `bson:"system_time_us"`
|
||||
} `bson:"extra_info"`
|
||||
}
|
||||
if c.Database("admin").RunCommand(ctx, map[string]any{"serverStatus": 1}).Decode(&info) != nil {
|
||||
log.Fatal("source counters unavailable")
|
||||
}
|
||||
var hello struct {
|
||||
Topology struct {
|
||||
Process any `bson:"processId"`
|
||||
} `bson:"topologyVersion"`
|
||||
}
|
||||
if c.Database("admin").RunCommand(ctx, map[string]any{"hello": 1}).Decode(&hello) != nil {
|
||||
log.Fatal("source identity unavailable")
|
||||
}
|
||||
identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process)))
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
return
|
||||
}
|
||||
if *observation != "" || *status {
|
||||
var value store.QueryReplicaSafetyObservation
|
||||
if !*status {
|
||||
input := io.Reader(os.Stdin)
|
||||
if *observation != "-" {
|
||||
f, e := os.Open(*observation)
|
||||
if e != nil {
|
||||
log.Fatal("observation unavailable")
|
||||
}
|
||||
defer f.Close()
|
||||
input = f
|
||||
}
|
||||
decoder := json.NewDecoder(io.LimitReader(input, 16384))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&value) != nil {
|
||||
log.Fatal("invalid observation")
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB")
|
||||
if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
||||
log.Fatal("explicit independent target required")
|
||||
}
|
||||
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(1))
|
||||
if e != nil {
|
||||
log.Fatal("target unavailable")
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
if *status {
|
||||
value, e := store.NewMongoStore(c.Database(db)).QueryReplicaSafetyStatus(ctx)
|
||||
if e != nil {
|
||||
log.Fatal("safety state unavailable")
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(value)
|
||||
return
|
||||
}
|
||||
if store.NewMongoStore(c.Database(db)).RecordQueryReplicaObservation(ctx, value) != nil {
|
||||
log.Fatal("observation rejected")
|
||||
}
|
||||
return
|
||||
}
|
||||
var transport *store.QueryReplicaTransport
|
||||
if *safetyRun != "" {
|
||||
var e error
|
||||
transport, e = store.NewQueryReplicaTransport(*bytesRate, *commandsRate)
|
||||
if e != nil || *ensure {
|
||||
log.Fatal("invalid guarded source configuration")
|
||||
}
|
||||
} else if *resume != "" {
|
||||
log.Fatal("resume requires durable safety")
|
||||
}
|
||||
|
||||
if !validPacing(*batchSize, *minimumPass) {
|
||||
log.Fatal("invalid bounded projector pacing")
|
||||
}
|
||||
@ -44,6 +146,9 @@ func main() {
|
||||
*interval = store.QueryPeriodicSyncInterval
|
||||
}
|
||||
}
|
||||
if *initializeTarget && *ensure {
|
||||
log.Fatal("source index writes forbidden during target binding")
|
||||
}
|
||||
if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
||||
log.Fatal("writer coverage, namespace and bounded polling interval required")
|
||||
}
|
||||
@ -54,6 +159,9 @@ func main() {
|
||||
log.Fatal("explicit source/query connection required")
|
||||
}
|
||||
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
|
||||
if !target && transport != nil {
|
||||
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5 * time.Second)
|
||||
}
|
||||
if target {
|
||||
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
|
||||
}
|
||||
@ -78,21 +186,90 @@ func main() {
|
||||
if *ensure && q.EnsureSourceIndexes(ctx) != nil {
|
||||
log.Fatal("source index preparation failed")
|
||||
}
|
||||
if q.EnsureTargetIndexes(ctx) != nil {
|
||||
log.Fatal("query index preparation failed")
|
||||
for {
|
||||
err := q.Acquire(ctx)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if *safetyRun == "" || !errors.Is(err, store.ErrQueryReplicaLease) {
|
||||
log.Fatal("projector lease unavailable")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
if q.Acquire(ctx) != nil {
|
||||
log.Fatal("projector lease unavailable")
|
||||
if *initializeTarget {
|
||||
return
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
if *safetyRun != "" {
|
||||
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
|
||||
log.Print("shadow safety paused; inspect durable state and explicitly resume")
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
}
|
||||
workCtx := ctx
|
||||
if *safetyRun != "" {
|
||||
guarded, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
workCtx = guarded
|
||||
go func() {
|
||||
tick := time.NewTicker(5 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-guarded.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
short, stop := context.WithTimeout(guarded, 3*time.Second)
|
||||
err := q.CheckSafety(short)
|
||||
stop()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
pause := func(reason string) {
|
||||
short, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if q.PauseSafety(short, reason) != nil {
|
||||
log.Print("pause persistence unavailable; startup fencing remains required")
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if ctx.Err() != nil {
|
||||
pause("operator_stop")
|
||||
}
|
||||
}()
|
||||
if q.EnsureTargetIndexes(workCtx) != nil {
|
||||
pause("target_error")
|
||||
log.Print("query index preparation paused")
|
||||
if *safetyRun == "" {
|
||||
os.Exit(1)
|
||||
}
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
for workCtx.Err() == nil {
|
||||
started := time.Now()
|
||||
ready, e := q.RunOnce(ctx, *bootstrap)
|
||||
ready, e := q.RunOnce(workCtx, *bootstrap)
|
||||
if e != nil {
|
||||
pause("operation_error")
|
||||
log.Print("projector paused; checkpoint retained (no customer payload logged)")
|
||||
if *safetyRun != "" {
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
if *once {
|
||||
if !ready {
|
||||
pause("incomplete_pilot")
|
||||
log.Print("projection incomplete; resume required")
|
||||
os.Exit(2)
|
||||
}
|
||||
@ -101,13 +278,22 @@ func main() {
|
||||
// Optional pacing covers backfill too. Idle lease maintenance only
|
||||
// touches the independent query DB, never scans authority for keepalive.
|
||||
if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 {
|
||||
if e := q.WaitNextPass(ctx, delay); e != nil {
|
||||
if e := q.WaitNextPass(workCtx, delay); e != nil {
|
||||
if ctx.Err() == nil {
|
||||
pause("target_error")
|
||||
log.Print("projector lease unavailable; checkpoint retained")
|
||||
if *safetyRun != "" {
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if *safetyRun != "" && ctx.Err() == nil {
|
||||
pause("operation_error")
|
||||
<-ctx.Done()
|
||||
}
|
||||
}
|
||||
|
||||
149
cmd/query-replica/main_local_test.go
Normal file
149
cmd/query-replica/main_local_test.go
Normal file
@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestProjectorGuardedProcessRestartLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("isolated source/query lab required")
|
||||
}
|
||||
ctx := t.Context()
|
||||
sourceClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer sourceClient.Disconnect(context.Background())
|
||||
targetClient, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer targetClient.Disconnect(context.Background())
|
||||
source, target := sourceClient.Database(domain.NewID("query_sync_cmd")), targetClient.Database(domain.NewID("query_replica_cmd"))
|
||||
t.Cleanup(func() { _ = source.Drop(context.Background()); _ = target.Drop(context.Background()) })
|
||||
s, d := store.NewMongoStore(source), store.NewMongoStore(target)
|
||||
q, e := store.NewQueryReplica(s, d)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = q.EnsureSourceIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = q.EnsureTargetIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, id := range []string{"a", "b", "c"} {
|
||||
_, e = source.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "status": "completed", "organizationId": "org", "privateDraft": "not projected"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
binary := filepath.Join(t.TempDir(), "projector")
|
||||
if e = exec.Command("go", "build", "-o", binary, ".").Run(); e != nil {
|
||||
t.Fatal("local projector build failed")
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
recent := []store.QueryReplicaSafetyObservation{}
|
||||
for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} {
|
||||
recent = append(recent, store.QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1})
|
||||
}
|
||||
raw, _ := bson.Marshal(recent[2])
|
||||
var fields bson.M
|
||||
_ = bson.Unmarshal(raw, &fields)
|
||||
fields["recent"] = recent
|
||||
_, e = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
env := []string{}
|
||||
for _, entry := range os.Environ() {
|
||||
if len(entry) >= 6 && (entry[:6] == "MONGO_" || len(entry) >= 6 && entry[:6] == "QUERY_") {
|
||||
continue
|
||||
}
|
||||
env = append(env, entry)
|
||||
}
|
||||
env = append(env, "MONGO_URI=mongodb://127.0.0.1:27119/?directConnection=true", "MONGO_DB="+source.Name(), "QUERY_READ_MONGO_URI=mongodb://127.0.0.1:27129/?replicaSet=query-read-rs", "QUERY_READ_MONGO_DB="+target.Name(), "QUERY_SERVING_NAMESPACE=local:"+source.Name())
|
||||
args := []string{"--writers-covered", "--bootstrap", "--batch-size=1", "--safety-run=process-test", "--max-source-records=2", "--source-records-per-second=50", "--source-bytes-per-second=262144", "--source-commands-per-second=8", "--source-cpu-baseline=85"}
|
||||
start := func() (*exec.Cmd, <-chan error) {
|
||||
t.Helper()
|
||||
p := exec.Command(binary, args...)
|
||||
p.Env = env
|
||||
p.Stdout = io.Discard
|
||||
p.Stderr = io.Discard
|
||||
if e = p.Start(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- p.Wait() }()
|
||||
t.Cleanup(func() {
|
||||
_ = p.Process.Kill()
|
||||
})
|
||||
return p, done
|
||||
}
|
||||
child, done := start()
|
||||
var state bson.M
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state)
|
||||
if state["paused"] == true {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("guarded process exited instead of parking")
|
||||
default:
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
if state["paused"] != true || state["reserved"] != int64(2) {
|
||||
t.Fatal("process bypassed total budget", state)
|
||||
}
|
||||
_ = child.Process.Kill()
|
||||
<-done
|
||||
_, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
|
||||
next, finished := start()
|
||||
time.Sleep(600 * time.Millisecond)
|
||||
select {
|
||||
case <-finished:
|
||||
t.Fatal("paused restart should park")
|
||||
default:
|
||||
}
|
||||
var after bson.M
|
||||
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
|
||||
if after["reserved"] != state["reserved"] || after["resumeToken"] != state["resumeToken"] {
|
||||
t.Fatal("process restart reset durable state")
|
||||
}
|
||||
_ = next.Process.Kill()
|
||||
<-finished
|
||||
// Operator status mode cannot connect to source, even with an invalid URI.
|
||||
status := exec.Command(binary, "--safety-status")
|
||||
status.Env = append(env, "MONGO_URI=mongodb://127.0.0.1:1/?directConnection=true")
|
||||
out, e := status.Output()
|
||||
if e != nil {
|
||||
t.Fatal("target-only operator mode failed")
|
||||
}
|
||||
var v map[string]any
|
||||
if json.Unmarshal(out, &v) != nil || v["paused"] != true {
|
||||
t.Fatal("operator status invalid")
|
||||
}
|
||||
count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"})
|
||||
if e != nil || count != 3 {
|
||||
t.Fatal("source payload changed")
|
||||
}
|
||||
var copied bson.M
|
||||
if e = target.Collection("annotationIndexJobs").FindOne(ctx, bson.M{}).Decode(&copied); e != nil || copied["privateDraft"] != nil {
|
||||
t.Fatal("projection boundary failed", e)
|
||||
}
|
||||
}
|
||||
@ -195,3 +195,37 @@ v5的广范围搜索及冷暖读取对照记录于 `final-products-pool-v5`;
|
||||
最终集成报告:`local-data/test-results/verify-1794636683705497000/report.json`,源码未在运行中变化、无未映射文件。41项中39项通过,包含隔离副本/缓存/Mongo专项、源码校验不入队、标注/切片浏览器、搜索及面板、生产前端构建和静态启动。API整包原始检查仍报告已有失败,最终官方Go基线门禁确认68项均在70项既有基线中,无新增;store整包PASS,但默认跳过的外部环境用例标为未完整覆盖,不算通过。专项显式启用的本机副本/Valkey测试已执行通过,未扩大 known-failures。
|
||||
|
||||
`.4` 测后资源申请总量约6.475/7.8核(83%)、内存11,813Mi(42%),包含实验查询Pod;低实际使用率不能替代调度申请余量。后续继续扩核须重新核对该节点及生产峰值。当前仍是限时实验Pod,保留原自动结束机制和25Gi盘;后续恢复验证按已记录UID、配置及回退步骤重建,不将该单Pod直接当生产查询库。
|
||||
|
||||
## 影子同步全局预算与持久暂停(方案 8e8abb0cb64f)
|
||||
|
||||
本节替代上文仅按集合批次及轮次间隔控制的生产影子启动建议;旧命令保留兼容,但不得直接用于此次生产启动。当前仅完成本地实现与合成数据验证,未启动生产回填、未切换生产查询入口。
|
||||
|
||||
守护模式只作用于显式带 `--safety-run` 的独立投影器及带 `--shadow-safety-run` 的构建器,不进入保存、切片、送检、质检或最终验收准入。权威权限与版本判断不变。
|
||||
|
||||
| 项目 | 首轮候选值及行为 |
|
||||
| --- | --- |
|
||||
| 总读取预算 | `--max-source-records=1000`,读取前持久预留。空读、重复读及崩溃也可能消耗;不是已迁移条数。预算不能因重启、换 run ID 清零 |
|
||||
| 全局节奏 | `--source-records-per-second=10 --source-bytes-per-second=65536 --source-commands-per-second=2 --batch-size=25`;共享源连接节流,包含握手/心跳开销,不按22个集合分别给额度 |
|
||||
| 持久性 | 查询库事务 `majority+journal`,先提交投影/断点再 CAS 确认源标记,不降低写入确认,不延长超时 |
|
||||
| 单次读取 | 源查询 `maxTimeMS=1500`,整个源操作5秒上限;大回复无法在预算内完成时暂停,不放开时限 |
|
||||
| 源 CPU | 三个间隔30秒的有效样本,跨度至少60秒,CPU均≤90%、波动≤5个百分点才允许开始;基线固定,恢复不得抬高 |
|
||||
| 停止 | 源 CPU≥95%一次,或比固定基线高3个百分点连续两个独立样本;受保护 Pod IO full avg10>1%连续两次或>5%一次;客户作业异常/延迟回退、健康缺失或超过45秒、写者变化、目标盘≥70%、宿主剩余<100GiB均暂停 |
|
||||
| 调度 | 脏数据优先,正常每次历史批次前先排一个脏批次;积压超过5分钟时增加最多三个脏批次。按集合持久轮转,完整历史完成前不做全量审计,完成后首次审计延后6小时 |
|
||||
| 恢复 | 私有控制表保存原因、预算、当前恢复令牌;进程异常重启转为暂停。仅显式提供当前 `--resume-pause`、相同 run ID、评审后的预算和健康窗口才能恢复。不会自行轮询恢复 |
|
||||
|
||||
源端修改集中在受控元数据接口,只允许同步标记及私有删除意图。过期删除声明的取消与清理按有界 ID 批次处理,不重放业务删除,不回写业务正文、版本、状态、归属或标注。若预算不足以确认所有待删除目标 ID,暂停并保留目标,不把尚未探测的记录当成已删除。
|
||||
|
||||
Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予 find/update,私有删除意图集合 find/insert/update,另授 serverStatus;禁止 remove、DDL、跨库访问。字段白名单由应用强制,不能宣称源账号“只读”。构建器没有源凭证。
|
||||
|
||||
### 最终启动前逐项确认
|
||||
|
||||
1. 将本地改动核验合入并同步 `dev` 后,通过正式发布门禁,单独批准镜像上传及实际启动计划。绑定不可变镜像、二进制 SHA、两个命名空间 UID、五个写者 UID/镜像/提交/副本、影子控制器 UID、Mongo Pod UID、来源端点摘要和目标库;核对所有生产读取开关仍关闭。不要直接执行旧方案里的启动命令。
|
||||
2. 核验源47个等价索引与最小权限角色,准备独立源 Secret。Secret 配额8、源网络出口、查询库4核8Gi/WiredTiger 2Gi/25Gi容量和 loop/底层 IO 双重限额保持现行已批准值。新增账号/Secret及私有同步元数据写入须列入实际启动授权。
|
||||
3. 先执行一次 `query-replica --writers-covered --initialize-target`:核对独立拓扑并仅初始化目标绑定,不扫描历史或创建源索引。然后启动可信观察器,等待三个有效样本,避免未绑定目标造成观察器无法写入健康数据。
|
||||
4. 接通可信客户作业健康输入 `--work-health`:JSON 必须含 UTC `at`、`coverage: "save-slice-submit-review-acceptance"`、布尔 `unexpectedErrors` 与 `latencyRegressed`。必须由已核实的指标提供者更新,≤45秒;不得用人工常量、readiness 或“日志没报错”替代。**实际生产提供者、延迟基线及故障演练尚待绑定,未完成不能启动。** 观察器来源 relay 目前仅适用于已审查的私网非TLS单端点;若实际连接启用TLS,必须先验证服务器名称处理,不得关闭TLS绕过。
|
||||
5. 最终计划列明观察器 `--bindings-sha256`、`--binary-sha256` 和 `--stop-on-failure` 的授权范围。停止仅对两个影子控制器 UID/resourceVersion CAS 缩至0;不会修改 API、处理器、校验器、Worker、协调器。观察器缺失时进程5秒巡检健康,健康超过45秒会停止工作,不能无限继续扫描。
|
||||
6. 仅投影器从0升至1,使用上表1000预留预算、10条/秒、64KiB/秒、2命令/秒、批次25及固定 CPU 基线。构建器保持0,生产查询开关保持关闭。影子目标数据不完整时不发布 ready。首轮停止后核对实际读取/预留/源确认量、源 CPU/读取字节、客户操作与共享盘 IO,对照同时间负载基线。
|
||||
7. 首轮达标后再确认后续预算。25条/秒、128KiB/秒、5命令/秒、批次50,以及最高50条/秒、256KiB/秒、8命令/秒、批次100都只是后续上限,**本清单不授权自动升级**。完整复制与增量追平后才单列构建器启动、旧新结果及权限隔离对照,仍不切生产入口。
|
||||
8. 回退:先停两个影子控制器并确认 Pod 退出,再停观察器;保留目标断点、暂停记录、最后完整结果和源标记。不删除源记录/标记,不清空查询库,不恢复业务旧快照。再次启动先读 `query-replica --safety-status`,重新确认健康、绑定和当前暂停令牌。生产入口从未切换,无需改客户作业配置。
|
||||
|
||||
此阶段只能证明新增同步负载受控及影子结果正确。生产入口未切换时不能宣称原 Mongo 总 CPU 已下降;降载验收仍需后续经授权切读后的同条件对照。
|
||||
|
||||
@ -66,7 +66,12 @@ func (q *QueryReplica) WaitNextPass(ctx context.Context, interval time.Duration)
|
||||
return nil
|
||||
case <-heartbeat.C:
|
||||
short, stop := context.WithTimeout(ctx, 3*time.Second)
|
||||
err := q.transaction(short, func(_ mongo.SessionContext) error { return nil })
|
||||
var err error
|
||||
if q.safety != nil {
|
||||
_, err = q.sourcePermit(short, 0)
|
||||
} else {
|
||||
err = q.transaction(short, func(_ mongo.SessionContext) error { return nil })
|
||||
}
|
||||
stop()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@ -31,6 +31,7 @@ type QueryReplica struct {
|
||||
BatchSize int
|
||||
dirtyAfter map[string]any
|
||||
reconcileNext int
|
||||
safety *queryReplicaSafety
|
||||
}
|
||||
type queryReplicaCheckpoint struct {
|
||||
ID string `bson:"_id"`
|
||||
@ -143,7 +144,7 @@ func (q *QueryReplica) transaction(ctx context.Context, fn func(mongo.SessionCon
|
||||
return nil, ErrQueryReplicaLease
|
||||
}
|
||||
return nil, fn(sc)
|
||||
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.Majority()))
|
||||
}, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
|
||||
return e
|
||||
}
|
||||
func (q *QueryReplica) checkpoint(ctx context.Context, n string) (queryReplicaCheckpoint, error) {
|
||||
@ -240,7 +241,14 @@ func bsonEqual(a, b any) bool {
|
||||
return e == nil && string(x) == string(y)
|
||||
}
|
||||
func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) {
|
||||
opts := options.Find().SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(q.limit()).SetBatchSize(int32(q.limit()))
|
||||
parent := ctx
|
||||
limit, e := q.sourcePermit(ctx, q.limit())
|
||||
if e != nil {
|
||||
return nil, nil, e
|
||||
}
|
||||
ctx, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
opts := options.Find().SetMaxTime(1500 * time.Millisecond).SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(limit).SetBatchSize(int32(limit))
|
||||
if filter[querySyncField+".dirty"] == true {
|
||||
opts.SetHint("query_sync_dirty_v1")
|
||||
} else {
|
||||
@ -265,7 +273,13 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
|
||||
}
|
||||
records = append(records, r)
|
||||
}
|
||||
return records, last, cur.Err()
|
||||
if e := cur.Err(); e != nil {
|
||||
return nil, nil, e
|
||||
}
|
||||
if e := q.paceRows(parent, len(records)); e != nil {
|
||||
return nil, nil, e
|
||||
}
|
||||
return records, last, nil
|
||||
}
|
||||
func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []queryReplicaRecord) error {
|
||||
models := make([]mongo.WriteModel, 0, len(records))
|
||||
@ -283,8 +297,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer
|
||||
if len(models) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false))
|
||||
return e
|
||||
return q.sourceMetadataBulk(ctx, n, models)
|
||||
}
|
||||
|
||||
// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across
|
||||
@ -306,7 +319,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int,
|
||||
if e != nil {
|
||||
return false, 0, e
|
||||
}
|
||||
done := int64(len(rows)) < q.limit()
|
||||
done := len(rows) == 0
|
||||
e = q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
if e := q.apply(sc, n, rows); e != nil {
|
||||
return e
|
||||
@ -366,6 +379,13 @@ func (s *MongoStore) EnableQuerySourceTracking(enabled bool) { s.querySyncEnable
|
||||
// RunOnce performs at most one batch per collection. Backfill is resumable and
|
||||
// bounded; the process does not begin unbounded scans on an HTTP cache miss.
|
||||
func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error) {
|
||||
if q.safety != nil {
|
||||
return q.runGuardedOnce(ctx, bootstrap)
|
||||
}
|
||||
if _, e := q.sourcePermit(ctx, 0); e != nil {
|
||||
return false, e
|
||||
}
|
||||
allBootstrapped := true
|
||||
for _, n := range replicaCollections() {
|
||||
cp, e := q.checkpoint(ctx, n)
|
||||
if e != nil {
|
||||
@ -375,16 +395,20 @@ func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error
|
||||
if !bootstrap {
|
||||
return false, ErrQueryReplicaNotReady
|
||||
}
|
||||
if _, _, e = q.BootstrapStep(ctx, n); e != nil {
|
||||
var complete bool
|
||||
if complete, _, e = q.BootstrapStep(ctx, n); e != nil {
|
||||
return false, e
|
||||
}
|
||||
allBootstrapped = allBootstrapped && complete
|
||||
}
|
||||
if _, e = q.DrainStep(ctx, n); e != nil {
|
||||
return false, e
|
||||
}
|
||||
}
|
||||
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
|
||||
return false, e
|
||||
if allBootstrapped {
|
||||
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
|
||||
return false, e
|
||||
}
|
||||
}
|
||||
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
|
||||
return false, e
|
||||
|
||||
@ -13,8 +13,13 @@ import (
|
||||
// It reads only source identities, never annotator payloads. Survivors retain
|
||||
// their data; a cancelled deletion is not replayed or restored from snapshots.
|
||||
func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
|
||||
if _, e := q.sourcePermit(ctx, 1); e != nil {
|
||||
return false, e
|
||||
}
|
||||
sourceCtx, sourceCancel := q.sourceContext(ctx)
|
||||
defer sourceCancel()
|
||||
var i queryDeleteIntent
|
||||
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i)
|
||||
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(sourceCtx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetMaxTime(1500*time.Millisecond).SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i)
|
||||
if e == mongo.ErrNoDocuments {
|
||||
return false, nil
|
||||
}
|
||||
@ -24,7 +29,7 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
|
||||
if queryReplicaFields[i.Collection] == "" {
|
||||
return false, ErrQueryReplicaUnsupported
|
||||
}
|
||||
if ok, e := q.Source.queryCollection(i.Collection).recoverDelete(ctx, i, time.Now().UTC()); e != nil || !ok {
|
||||
if ok, e := q.recoverSourceClaims(ctx, i); e != nil || !ok {
|
||||
return false, e
|
||||
}
|
||||
key := "delete/" + i.ID
|
||||
@ -34,7 +39,7 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
|
||||
return false, e
|
||||
}
|
||||
if cp.Complete {
|
||||
_, e = q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}})
|
||||
_, e = q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}})
|
||||
return true, e
|
||||
}
|
||||
filter := bson.M{}
|
||||
@ -63,7 +68,19 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
|
||||
}
|
||||
present := []any{}
|
||||
if len(ids) > 0 {
|
||||
src, e := q.Source.db.Collection(i.Collection).Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1}))
|
||||
permitted, err := q.sourcePermit(ctx, int64(len(ids)))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if permitted < int64(len(ids)) {
|
||||
if err = q.PauseSafety(ctx, "pilot_budget"); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, ErrQueryReplicaPaused
|
||||
}
|
||||
sourceCtx, sourceCancel := q.sourceContext(ctx)
|
||||
defer sourceCancel()
|
||||
src, e := q.Source.db.Collection(i.Collection).Find(sourceCtx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1}).SetLimit(int64(len(ids))).SetBatchSize(int32(len(ids))).SetMaxTime(1500*time.Millisecond))
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
@ -81,6 +98,9 @@ func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) {
|
||||
return false, e
|
||||
}
|
||||
}
|
||||
if e := q.paceRows(ctx, len(present)); e != nil {
|
||||
return false, e
|
||||
}
|
||||
absent := []bson.M{}
|
||||
for _, r := range rows {
|
||||
id := deletionDoc(r["_queryReplica"])["sourceId"]
|
||||
|
||||
@ -161,7 +161,12 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error {
|
||||
if !cp.Complete {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
e = q.Source.db.Collection(n).FindOne(ctx, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1")).Err()
|
||||
if _, e = q.sourcePermit(ctx, 1); e != nil {
|
||||
return e
|
||||
}
|
||||
work, cancel := q.sourceContext(ctx)
|
||||
e = q.Source.db.Collection(n).FindOne(work, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1").SetMaxTime(1500*time.Millisecond)).Err()
|
||||
cancel()
|
||||
if e == nil {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
@ -169,7 +174,12 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error {
|
||||
return e
|
||||
}
|
||||
}
|
||||
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
|
||||
if _, e := q.sourcePermit(ctx, 1); e != nil {
|
||||
return e
|
||||
}
|
||||
work, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(work, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err()
|
||||
if e == nil {
|
||||
return ErrQueryReplicaNotReady
|
||||
}
|
||||
@ -177,7 +187,11 @@ func (q *QueryReplica) PublishReady(ctx context.Context) error {
|
||||
return e
|
||||
}
|
||||
return q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
_, e := q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true))
|
||||
_, e := q.control().UpdateOne(sc, bson.M{"_id": "state", "bootstrapAt": bson.M{"$exists": false}}, bson.M{"$set": bson.M{"bootstrapAt": time.Now().UTC()}})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
})
|
||||
}
|
||||
|
||||
@ -40,6 +40,17 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati
|
||||
if !boot.Complete {
|
||||
continue
|
||||
}
|
||||
if q.safety != nil {
|
||||
var first struct {
|
||||
At time.Time `bson:"bootstrapAt"`
|
||||
}
|
||||
if e = q.control().FindOne(ctx, bson.M{"_id": "state"}).Decode(&first); e != nil && e != mongo.ErrNoDocuments {
|
||||
return 0, e
|
||||
}
|
||||
if first.At.IsZero() || time.Since(first.At) < interval {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
key := "reconcile/" + name
|
||||
var cp queryReplicaCheckpoint
|
||||
e = q.control().FindOne(ctx, bson.M{"_id": key}).Decode(&cp)
|
||||
@ -97,7 +108,7 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati
|
||||
}); e != nil {
|
||||
return 0, e
|
||||
}
|
||||
if _, e = q.Source.db.Collection(name).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)); e != nil {
|
||||
if e = q.sourceMetadataBulk(ctx, name, models); e != nil {
|
||||
return 0, e
|
||||
}
|
||||
}
|
||||
@ -106,7 +117,7 @@ func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Durati
|
||||
// Reuse the durable identity sweeper for unreported physical deletions.
|
||||
// Audits have lower priority and do not block unrelated report readiness.
|
||||
i := queryDeleteIntent{ID: domain.NewID("query-audit"), Collection: name, Phase: "reconcile", CreatedAt: time.Now().UTC(), Audit: true}
|
||||
if _, e = q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil {
|
||||
if e = q.sourceAuditIntent(ctx, i); e != nil {
|
||||
return 0, e
|
||||
}
|
||||
}
|
||||
|
||||
331
internal/store/query_replica_safety.go
Normal file
331
internal/store/query_replica_safety.go
Normal file
@ -0,0 +1,331 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/writeconcern"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrQueryReplicaPaused = errors.New("shadow synchronization paused; explicit reviewed resume required")
|
||||
|
||||
// Safety is opt-in for the independent projector only. It never gates authority
|
||||
// business operations. The policy and pause survive process and Pod replacement.
|
||||
type QueryReplicaSafetyPolicy struct {
|
||||
BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"`
|
||||
CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"`
|
||||
MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"`
|
||||
RunID string `json:"runId" bson:"runId"`
|
||||
MaxRecords int64 `json:"maxRecords" bson:"maxRecords"`
|
||||
RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"`
|
||||
BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"`
|
||||
}
|
||||
type QueryReplicaSafetyObservation struct {
|
||||
At time.Time `json:"at" bson:"at"`
|
||||
CPU float64 `json:"cpu" bson:"cpu"`
|
||||
Ready bool `json:"ready" bson:"ready"`
|
||||
WritersCovered bool `json:"writersCovered" bson:"writersCovered"`
|
||||
WorkErrors bool `json:"workErrors" bson:"workErrors"`
|
||||
ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"`
|
||||
TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"`
|
||||
HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"`
|
||||
}
|
||||
type queryReplicaSafetyState struct {
|
||||
Owner string `bson:"owner"`
|
||||
Policy QueryReplicaSafetyPolicy `bson:"policy"`
|
||||
Digest string `bson:"digest"`
|
||||
Reserved int64 `bson:"reserved"`
|
||||
Paused bool `bson:"paused"`
|
||||
Reason string `bson:"reason"`
|
||||
ResumeToken string `bson:"resumeToken"`
|
||||
LastObservation time.Time `bson:"lastObservation"`
|
||||
High int `bson:"high"`
|
||||
HighIO int `bson:"highIO"`
|
||||
}
|
||||
type queryReplicaSafety struct {
|
||||
policy QueryReplicaSafetyPolicy
|
||||
rows *queryReplicaPacer
|
||||
}
|
||||
|
||||
func safetyPolicyDigest(p QueryReplicaSafetyPolicy) string {
|
||||
b, _ := json.Marshal(p)
|
||||
h := sha256.Sum256(b)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
func (p QueryReplicaSafetyPolicy) validate() error {
|
||||
if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || p.BaselineCPU <= 0 || p.BaselineCPU > 90 {
|
||||
return errors.New("invalid shadow safety policy")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordQueryReplicaObservation is for a trusted out-of-process observer. Only
|
||||
// complete, fresh observations are accepted; callers never submit customer data.
|
||||
func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryReplicaSafetyObservation) error {
|
||||
if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 {
|
||||
return errors.New("invalid shadow observation")
|
||||
}
|
||||
var hello bson.M
|
||||
if e := s.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&hello); e != nil || hello["setName"] == nil {
|
||||
return errors.New("independent query replica required")
|
||||
}
|
||||
var binding queryReplicaBinding
|
||||
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "source-binding"}).Decode(&binding); e != nil || binding.SourceDatabase == "" || binding.SourceDatabase == s.db.Name() || binding.Namespace != queryReplicaNamespace(binding.SourceDatabase) {
|
||||
return errors.New("reviewed query binding required before observation writes")
|
||||
}
|
||||
c := s.db.Collection(queryReplicaControl, options.Collection().SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
|
||||
// Monotonic CAS: an old observer cannot overwrite newer health evidence.
|
||||
_, e := c.UpdateOne(ctx, bson.M{"_id": "safety-health", "$or": bson.A{bson.M{"at": bson.M{"$lt": o.At}}, bson.M{"at": bson.M{"$exists": false}}}}, bson.M{"$set": o, "$push": bson.M{"recent": bson.M{"$each": bson.A{o}, "$slice": -3}}}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
}
|
||||
func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafetyPolicy, resumeToken string) error {
|
||||
if e := p.validate(); e != nil {
|
||||
return e
|
||||
}
|
||||
if q.limit() > int64(p.MaxBatchRows) {
|
||||
return errors.New("batch exceeds reviewed safety policy")
|
||||
}
|
||||
digest := safetyPolicyDigest(p)
|
||||
restarted := false
|
||||
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
var state queryReplicaSafetyState
|
||||
e := q.control().FindOne(sc, bson.M{"_id": "safety"}).Decode(&state)
|
||||
if e != nil && e != mongo.ErrNoDocuments {
|
||||
return e
|
||||
}
|
||||
if e == nil {
|
||||
if state.Paused {
|
||||
if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || p.BaselineCPU > state.Policy.BaselineCPU || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
} else {
|
||||
if state.Digest != digest || resumeToken != "" {
|
||||
return errors.New("active shadow policy cannot be replaced")
|
||||
}
|
||||
if state.Owner == q.owner {
|
||||
return nil
|
||||
}
|
||||
restarted = true
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"paused": true, "reason": "process_restarted", "resumeToken": domain.NewID("shadow-resume")}})
|
||||
if e == nil {
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true))
|
||||
}
|
||||
return e
|
||||
}
|
||||
} else if resumeToken != "" {
|
||||
return errors.New("resume requires an existing pause")
|
||||
}
|
||||
denyStart := func() error {
|
||||
if state.Paused {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
restarted = true
|
||||
_, err := q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": true, "reason": "starting_health", "resumeToken": domain.NewID("shadow-resume"), "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true))
|
||||
if err == nil {
|
||||
_, err = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true))
|
||||
}
|
||||
return err
|
||||
}
|
||||
var health struct {
|
||||
Recent []QueryReplicaSafetyObservation `bson:"recent"`
|
||||
}
|
||||
if e = q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
|
||||
return denyStart()
|
||||
}
|
||||
if len(health.Recent) != 3 {
|
||||
return denyStart()
|
||||
}
|
||||
low, high := 100., 0.
|
||||
for i, o := range health.Recent {
|
||||
if observationFailure(o, o.At) != "" || o.CPU > 90 {
|
||||
return denyStart()
|
||||
}
|
||||
if i > 0 && !o.At.After(health.Recent[i-1].At) {
|
||||
return denyStart()
|
||||
}
|
||||
low = math.Min(low, o.CPU)
|
||||
high = math.Max(high, o.CPU)
|
||||
}
|
||||
if high-low > 5 || health.Recent[2].At.Sub(health.Recent[0].At) < 60*time.Second || observationFailure(health.Recent[2], time.Now()) != "" {
|
||||
return denyStart()
|
||||
}
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": false, "reason": "", "resumeToken": "", "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if restarted {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
q.safety = &queryReplicaSafety{policy: p, rows: newQueryReplicaPacer(float64(p.RecordsPerSecond))}
|
||||
_, e = q.sourcePermit(ctx, 0)
|
||||
return e
|
||||
}
|
||||
func observationFailure(o QueryReplicaSafetyObservation, now time.Time) string {
|
||||
switch {
|
||||
case o.At.IsZero() || now.Sub(o.At) > 45*time.Second || o.At.After(now.Add(time.Second)):
|
||||
return "health_missing_or_stale"
|
||||
case !o.Ready || !o.WritersCovered || o.WorkErrors:
|
||||
return "production_health"
|
||||
case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1:
|
||||
return "invalid_health"
|
||||
case o.CPU >= 95:
|
||||
return "source_cpu_absolute"
|
||||
case o.ProtectedIOFull > 5:
|
||||
return "protected_io"
|
||||
case o.TargetUsedFraction >= 0.70 || o.HostFreeBytes < 100*1024*1024*1024:
|
||||
return "storage_pressure"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// sourcePermit reserves a bounded maximum before a read. A crash consumes the
|
||||
// reservation rather than silently resetting a pilot's total budget. Empty and
|
||||
// repeated reads can consume it too; reserved is explicitly not progress.
|
||||
func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, error) {
|
||||
if q.safety == nil {
|
||||
return want, nil
|
||||
}
|
||||
allowed := want
|
||||
paused := false
|
||||
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
var s queryReplicaSafetyState
|
||||
if e := q.control().FindOne(sc, bson.M{"_id": "safety"}).Decode(&s); e != nil {
|
||||
return e
|
||||
}
|
||||
if s.Paused {
|
||||
paused = true
|
||||
return nil
|
||||
}
|
||||
if s.Digest != safetyPolicyDigest(q.safety.policy) {
|
||||
return errors.New("shadow policy changed")
|
||||
}
|
||||
var o QueryReplicaSafetyObservation
|
||||
e := q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&o)
|
||||
if e != nil && e != mongo.ErrNoDocuments {
|
||||
return e
|
||||
}
|
||||
reason := observationFailure(o, time.Now())
|
||||
if reason == "" && !o.At.Equal(s.LastObservation) {
|
||||
if o.CPU > s.Policy.BaselineCPU+3 {
|
||||
s.High++
|
||||
} else {
|
||||
s.High = 0
|
||||
}
|
||||
if o.ProtectedIOFull > 1 {
|
||||
s.HighIO++
|
||||
} else {
|
||||
s.HighIO = 0
|
||||
}
|
||||
s.LastObservation = o.At
|
||||
if s.High >= 2 {
|
||||
reason = "source_cpu_relative"
|
||||
}
|
||||
if s.HighIO >= 2 {
|
||||
reason = "protected_io"
|
||||
}
|
||||
}
|
||||
if reason == "" && want > 0 && s.Policy.MaxRecords > 0 {
|
||||
left := s.Policy.MaxRecords - s.Reserved
|
||||
if left <= 0 {
|
||||
reason = "pilot_budget"
|
||||
} else if allowed > left {
|
||||
allowed = left
|
||||
}
|
||||
}
|
||||
fields := bson.M{"lastObservation": s.LastObservation, "high": s.High, "highIO": s.HighIO}
|
||||
if reason != "" {
|
||||
paused = true
|
||||
fields["paused"] = true
|
||||
fields["reason"] = reason
|
||||
fields["resumeToken"] = domain.NewID("shadow-resume")
|
||||
if _, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
update := bson.M{"$set": fields}
|
||||
if !paused && allowed > 0 {
|
||||
update["$inc"] = bson.M{"reserved": allowed}
|
||||
}
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, update)
|
||||
return e
|
||||
})
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
if paused {
|
||||
return 0, ErrQueryReplicaPaused
|
||||
}
|
||||
return allowed, nil
|
||||
}
|
||||
func (q *QueryReplica) PauseSafety(ctx context.Context, reason string) error {
|
||||
if q.safety == nil {
|
||||
return nil
|
||||
}
|
||||
// Only fixed non-payload reasons reach durable diagnostics.
|
||||
switch reason {
|
||||
case "source_error", "target_error", "incomplete_pilot", "observer_error", "operator_stop", "pilot_budget":
|
||||
default:
|
||||
reason = "operation_error"
|
||||
}
|
||||
return q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
_, e := q.control().UpdateOne(sc, bson.M{"_id": "safety", "paused": false}, bson.M{"$set": bson.M{"paused": true, "reason": reason, "resumeToken": domain.NewID("shadow-resume")}})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
})
|
||||
}
|
||||
func (q *QueryReplica) paceRows(ctx context.Context, n int) error {
|
||||
if q.safety == nil {
|
||||
return nil
|
||||
}
|
||||
return q.safety.rows.wait(ctx, float64(n))
|
||||
}
|
||||
func (q *QueryReplica) sourceContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(ctx, 5*time.Second)
|
||||
}
|
||||
|
||||
// QueryReplicaSafetyStatus contains no payloads or connection information.
|
||||
// The nonce is an operator CAS token, not authorization: target credentials
|
||||
// remain mandatory. It is never emitted by the running projector's logs.
|
||||
func (s *MongoStore) QueryReplicaSafetyStatus(ctx context.Context) (bson.M, error) {
|
||||
var state bson.M
|
||||
e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}, options.FindOne().SetProjection(bson.M{"_id": 0, "policy": 1, "reserved": 1, "paused": 1, "reason": 1, "resumeToken": 1, "lastObservation": 1, "high": 1})).Decode(&state)
|
||||
return state, e
|
||||
}
|
||||
|
||||
// CheckQueryReplicaSafety gates only the opt-in shadow builder, never API reads
|
||||
// or any authority operation. A lost observer leaves the builder parked.
|
||||
func (s *MongoStore) CheckQueryReplicaSafety(ctx context.Context, runID string) error {
|
||||
var state queryReplicaSafetyState
|
||||
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state); e != nil {
|
||||
return e
|
||||
}
|
||||
if state.Paused || state.Policy.RunID != runID {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
var health QueryReplicaSafetyObservation
|
||||
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
|
||||
return e
|
||||
}
|
||||
if observationFailure(health, time.Now()) != "" {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryReplica) CheckSafety(ctx context.Context) error {
|
||||
_, e := q.sourcePermit(ctx, 0)
|
||||
return e
|
||||
}
|
||||
427
internal/store/query_replica_safety_local_test.go
Normal file
427
internal/store/query_replica_safety_local_test.go
Normal file
@ -0,0 +1,427 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func seedReplicaSafetyHealth(t *testing.T, q *QueryReplica) {
|
||||
t.Helper()
|
||||
if q.BatchSize > 100 {
|
||||
q.BatchSize = 25
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
recent := []QueryReplicaSafetyObservation{}
|
||||
for _, ago := range []time.Duration{60 * time.Second, 30 * time.Second, 0} {
|
||||
recent = append(recent, QueryReplicaSafetyObservation{At: now.Add(-ago), CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1})
|
||||
}
|
||||
raw, e := bson.Marshal(recent[2])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var fields bson.M
|
||||
if e = bson.Unmarshal(raw, &fields); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
fields["recent"] = recent
|
||||
if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
func safetyPolicyForTest() QueryReplicaSafetyPolicy {
|
||||
return QueryReplicaSafetyPolicy{BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "synthetic-run", MaxRecords: 1000, RecordsPerSecond: 50, BaselineCPU: 85}
|
||||
}
|
||||
func TestQueryReplicaSafetyPilotBudgetAndRestartLocal(t *testing.T) {
|
||||
source, target, q := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
p := safetyPolicyForTest()
|
||||
p.MaxRecords = 4
|
||||
q.BatchSize = 3
|
||||
for _, n := range []string{"projects", "humanItems"} {
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, e := source.db.Collection(n).InsertOne(ctx, bson.M{"_id": domain.NewID("source"), "id": domain.NewID("logical"), "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "subject": bson.M{"payload": bson.M{"annotationDraft": "preserve"}}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
if e := q.ConfigureSafety(ctx, p, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, n, e := q.BootstrapStep(ctx, "projects"); e != nil || n != 3 {
|
||||
t.Fatal(n, e)
|
||||
}
|
||||
complete, n, e := q.BootstrapStep(ctx, "humanItems")
|
||||
if e != nil || n != 1 || complete {
|
||||
t.Fatal("short budget falsely completed collection", complete, n, e)
|
||||
}
|
||||
if _, _, e = q.BootstrapStep(ctx, "humanItems"); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("pilot exceeded durable cap", e)
|
||||
}
|
||||
var state queryReplicaSafetyState
|
||||
if e = q.control().FindOne(ctx, bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Reserved != 4 || !state.Paused || state.ResumeToken == "" {
|
||||
t.Fatal(state, e)
|
||||
}
|
||||
if _, e = q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
successor, e := NewQueryReplica(source, target)
|
||||
successor.BatchSize = q.BatchSize
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = successor.Acquire(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
p.MaxRecords = 20
|
||||
if e = successor.ConfigureSafety(ctx, p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("restart bypassed pause", e)
|
||||
}
|
||||
changed := p
|
||||
changed.RunID = "replacement-run"
|
||||
if e = successor.ConfigureSafety(ctx, changed, state.ResumeToken); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("new run reset budget", e)
|
||||
}
|
||||
seedReplicaSafetyHealth(t, successor)
|
||||
if e = successor.ConfigureSafety(ctx, p, state.ResumeToken); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cp, e := successor.checkpoint(ctx, "humanItems")
|
||||
if e != nil || !cp.HasLast || cp.Complete {
|
||||
t.Fatal(cp, e)
|
||||
}
|
||||
if _, n, e = successor.BootstrapStep(ctx, "humanItems"); e != nil || n != 2 {
|
||||
t.Fatal("resume lost remaining history", n, e)
|
||||
}
|
||||
var doc bson.M
|
||||
if e = source.db.Collection("humanItems").FindOne(ctx, bson.M{}).Decode(&doc); e != nil || doc["version"] != int32(7) || deletionDoc(deletionDoc(doc["subject"])["payload"])["annotationDraft"] != "preserve" {
|
||||
t.Fatal("source annotation altered", e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaSafetyRelativeCPUCountsDistinctObservationsLocal(t *testing.T) {
|
||||
_, _, q := newLocalReplica(t)
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
p := safetyPolicyForTest()
|
||||
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
now := time.Now()
|
||||
for i := 1; i <= 2; i++ {
|
||||
o := QueryReplicaSafetyObservation{At: now.Add(time.Duration(i) * time.Millisecond), CPU: 90, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}
|
||||
if e := q.Target.RecordQueryReplicaObservation(t.Context(), o); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, e := q.sourcePermit(t.Context(), 0)
|
||||
if i == 1 {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for j := 0; j < 3; j++ {
|
||||
if _, e = q.sourcePermit(t.Context(), 0); e != nil {
|
||||
t.Fatal("same sample counted twice", e)
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("CPU did not persist pause", e)
|
||||
}
|
||||
}
|
||||
var state queryReplicaSafetyState
|
||||
_ = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state)
|
||||
if !state.Paused || state.Reason != "source_cpu_relative" {
|
||||
t.Fatal(state)
|
||||
}
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
if _, e := q.sourcePermit(t.Context(), 0); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("cooldown silently resumed", e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaSafetyRestartWithoutPauseFailsClosedLocal(t *testing.T) {
|
||||
source, target, q := newLocalReplica(t)
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
p := safetyPolicyForTest()
|
||||
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := q.sourcePermit(t.Context(), 7); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, _ = q.control().UpdateOne(t.Context(), bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
|
||||
next, _ := NewQueryReplica(source, target)
|
||||
next.BatchSize = q.BatchSize
|
||||
if e := next.Acquire(t.Context()); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := next.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("crash bypassed operator review", e)
|
||||
}
|
||||
var s queryReplicaSafetyState
|
||||
_ = next.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&s)
|
||||
if !s.Paused || s.Reason != "process_restarted" || s.Reserved != 7 {
|
||||
t.Fatal(s)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaBoundedClaimRecoveryPreservesAnnotationsLocal(t *testing.T) {
|
||||
var many int
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "delete" {
|
||||
t.Error("projector attempted source delete")
|
||||
}
|
||||
if e.CommandName == "update" {
|
||||
arr := e.Command.Lookup("updates").Array()
|
||||
vs, _ := arr.Values()
|
||||
for _, v := range vs {
|
||||
u := v.Document()
|
||||
if u.Lookup("multi").Type != 0 && u.Lookup("multi").Boolean() {
|
||||
ids, _ := u.Lookup("q").Document().Lookup("_id").Document().Lookup("$in").Array().Values()
|
||||
if len(ids) > 2 {
|
||||
t.Error("unbounded cleanup")
|
||||
}
|
||||
many++
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
source, _, q := newLocalReplica(t, monitor)
|
||||
ctx := t.Context()
|
||||
q.BatchSize = 2
|
||||
i := queryDeleteIntent{ID: "expired-delete", Collection: "humanItems", Phase: "deleting", LeaseUntil: time.Now().Add(-time.Minute), CreatedAt: time.Now()}
|
||||
if _, e := source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for n := 0; n < 7; n++ {
|
||||
if _, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"id": domain.NewID("item"), "version": 9, "status": "claimed", "annotation": "kept", "_querySync": bson.M{"deleteClaim": i.ID}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
for n := 0; n < 6; n++ {
|
||||
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
count, e := source.db.Collection("humanItems").CountDocuments(ctx, bson.M{"version": 9, "status": "claimed", "annotation": "kept", "_querySync.deleteClaim": bson.M{"$exists": false}})
|
||||
if e != nil || count != 7 || many != 4 {
|
||||
t.Fatal(count, many, e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaSafetyMajorityJournalAndNoEarlyAuditLocal(t *testing.T) {
|
||||
var commits int
|
||||
targetMonitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "commitTransaction" {
|
||||
commits++
|
||||
wc := e.Command.Lookup("writeConcern").Document()
|
||||
if wc.Lookup("w").StringValue() != "majority" || !wc.Lookup("j").Boolean() {
|
||||
t.Error("durability weakened")
|
||||
}
|
||||
}
|
||||
}}
|
||||
_, _, q := newLocalReplica(t, nil, targetMonitor)
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
if e := q.ConfigureSafety(t.Context(), safetyPolicyForTest(), ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := q.ReconcileOneDue(t.Context(), 6*time.Hour); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if commits == 0 {
|
||||
t.Fatal("no transactional durability evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaSafetyMissingHealthCannotRestartIntoRunningLocal(t *testing.T) {
|
||||
_, _, q := newLocalReplica(t)
|
||||
q.BatchSize = 25
|
||||
p := safetyPolicyForTest()
|
||||
if e := q.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
if e := q.ConfigureSafety(t.Context(), p, ""); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("missing health latch auto-resumed", e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaDeleteBudgetCannotDeleteUnprobedTargetLocal(t *testing.T) {
|
||||
source, target, q := newLocalReplica(t)
|
||||
q.BatchSize = 2
|
||||
ctx := t.Context()
|
||||
_, e := source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, queryDeleteIntent{ID: "audit", Audit: true, Collection: "humanItems", Phase: "reconcile", CreatedAt: time.Now()})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, id := range []string{"one", "two"} {
|
||||
_, e = target.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": id, "_queryReplica": bson.M{"sourceId": id}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
p := safetyPolicyForTest()
|
||||
p.MaxRecords = 4
|
||||
if e = q.ConfigureSafety(ctx, p, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = q.ReconcileDeleteStep(ctx); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
n, e := target.db.Collection("humanItems").CountDocuments(ctx, bson.M{})
|
||||
if e != nil || n != 2 {
|
||||
t.Fatal("unprobed target facts removed", n, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaSafetyRejectsSourceObservationAndStopsBuilderLocal(t *testing.T) {
|
||||
source, target, q := newLocalReplica(t)
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
p := safetyPolicyForTest()
|
||||
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
o := QueryReplicaSafetyObservation{At: now, CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}
|
||||
if e := source.RecordQueryReplicaObservation(t.Context(), o); e == nil {
|
||||
t.Fatal("observer wrote into authority")
|
||||
}
|
||||
count, _ := source.db.Collection(queryReplicaControl).CountDocuments(t.Context(), bson.M{})
|
||||
if count != 0 {
|
||||
t.Fatal("observation mutated source")
|
||||
}
|
||||
if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
o.At = now.Add(-46 * time.Second)
|
||||
if e := target.RecordQueryReplicaObservation(t.Context(), o); e == nil {
|
||||
t.Fatal("stale health accepted")
|
||||
}
|
||||
if e := q.PauseSafety(t.Context(), "operator_stop"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("builder ignored durable pause", e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaSafetyTargetFailureRetainsDirtyAndCheckpointLocal(t *testing.T) {
|
||||
source, target, q := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
if e := q.ConfigureSafety(ctx, safetyPolicyForTest(), ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "source", "id": "item", "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "annotation": "keep", "_querySync": bson.M{"revision": 1, "dirty": true}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Failure injection is confined to this fixture's random target database.
|
||||
if e = target.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: "humanItems"}, {Key: "validator", Value: bson.M{"mustNeverExist": true}}}).Err(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, _, e = q.BootstrapStep(ctx, "humanItems"); e == nil {
|
||||
t.Fatal("invalid target write unexpectedly committed")
|
||||
}
|
||||
cp, e := q.checkpoint(ctx, "humanItems")
|
||||
if e != nil || cp.HasLast || cp.Complete {
|
||||
t.Fatal("failed transaction advanced checkpoint", cp, e)
|
||||
}
|
||||
var row bson.M
|
||||
if e = source.db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "source"}).Decode(&row); e != nil || deletionDoc(row[querySyncField])["dirty"] != true || row["version"] != int32(7) || row["annotation"] != "keep" {
|
||||
t.Fatal("failed sink damaged source", e)
|
||||
}
|
||||
if e = target.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: "humanItems"}, {Key: "validator", Value: bson.M{}}}).Err(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, n, e := q.BootstrapStep(ctx, "humanItems"); e != nil || n != 1 {
|
||||
t.Fatal("replay failed", n, e)
|
||||
}
|
||||
if n, e := q.DrainStep(ctx, "humanItems"); e != nil || n != 0 {
|
||||
t.Fatal("dirty acknowledged before persistence or not replayable", n, e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaSafetyConcurrentSavePreservesNewRevisionLocal(t *testing.T) {
|
||||
var source *MongoStore
|
||||
armed := false
|
||||
fired := false
|
||||
mon := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName != "update" {
|
||||
return
|
||||
}
|
||||
if !armed || fired {
|
||||
return
|
||||
}
|
||||
name, _ := e.Command.Lookup("update").StringValueOK()
|
||||
if name != "humanItems" {
|
||||
return
|
||||
}
|
||||
fired = true
|
||||
short, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
_, err := source.db.Collection("humanItems").UpdateOne(short, bson.M{"_id": "source"}, bson.M{"$set": bson.M{"version": 8, "status": "submitted", "annotation": "new author edit", "_querySync.revision": 2, "_querySync.dirty": true}})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}}
|
||||
var q *QueryReplica
|
||||
source, _, q = newLocalReplica(t, nil, mon)
|
||||
ctx := t.Context()
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
if e := q.ConfigureSafety(ctx, safetyPolicyForTest(), ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, e := source.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "source", "id": "item", "organizationId": "o", "projectId": "p", "version": 7, "status": "claimed", "annotation": "old author edit", "_querySync": bson.M{"revision": 1, "dirty": true}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
armed = true
|
||||
if _, _, e = q.BootstrapStep(ctx, "humanItems"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var kept bson.M
|
||||
_ = source.db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "source"}).Decode(&kept)
|
||||
if !fired || kept["version"] != int32(8) || kept["annotation"] != "new author edit" || deletionDoc(kept[querySyncField])["dirty"] != true {
|
||||
t.Fatal("old copy acknowledged or overwrote concurrent save")
|
||||
}
|
||||
if n, e := q.DrainStep(ctx, "humanItems"); e != nil || n != 1 {
|
||||
t.Fatal("new revision not recoverable", n, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaSafetyBacklogPriorityAndRoundRobinLocal(t *testing.T) {
|
||||
source, _, q := newLocalReplica(t)
|
||||
q.BatchSize = 1
|
||||
ctx := t.Context()
|
||||
seedReplicaSafetyHealth(t, q)
|
||||
p := safetyPolicyForTest()
|
||||
p.MaxRecords = 0
|
||||
if e := q.ConfigureSafety(ctx, p, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, id := range []string{"a", "b", "c", "d", "e"} {
|
||||
_, e := source.db.Collection("annotationIndexJobs").InsertOne(ctx, bson.M{"_id": id, "id": id, "organizationId": "o", "status": "completed", "_querySync": bson.M{"revision": 1, "dirty": true}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
_, e := q.control().InsertOne(ctx, bson.M{"_id": "backlog/annotationIndexJobs", "since": time.Now().Add(-6 * time.Minute)})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if ready, e := q.RunOnce(ctx, true); e != nil || ready {
|
||||
t.Fatal("partial history published", ready, e)
|
||||
}
|
||||
clean, e := source.db.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"_querySync.dirty": false})
|
||||
if e != nil || clean != 4 {
|
||||
t.Fatal("old dirty backlog did not receive4:1 priority", clean, e)
|
||||
}
|
||||
var cursor struct {
|
||||
Next int `bson:"next"`
|
||||
}
|
||||
if e = q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if cursor.Next != 0 {
|
||||
t.Fatal("round-robin checkpoint not persisted")
|
||||
}
|
||||
}
|
||||
146
internal/store/query_replica_safety_test.go
Normal file
146
internal/store/query_replica_safety_test.go
Normal file
@ -0,0 +1,146 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"math"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryReplicaMetadataWriterRejectsBusinessFields(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
model mongo.WriteModel
|
||||
ok bool
|
||||
}{
|
||||
{"ack", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}), true},
|
||||
{"repair", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": bson.M{"$exists": false}}).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(2)}), true},
|
||||
{"annotation", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"subject.payload.annotationDraft": "overwrite"}}), false},
|
||||
{"version", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$inc": bson.M{"version": 1}}), false},
|
||||
{"upsert", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}).SetUpsert(true), false},
|
||||
{"missing-CAS", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i"}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}}), false},
|
||||
{"delete", mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": "i"}), false},
|
||||
{"replace", mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": "i"}).SetReplacement(bson.M{"_querySync": bson.M{"dirty": false}}), false},
|
||||
{"pipeline", mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.A{bson.M{"$set": bson.M{"_querySync.dirty": false}}}), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
e := validateReplicaMetadataWrite("humanItems", c.model)
|
||||
if (e == nil) != c.ok {
|
||||
t.Fatalf("allowed=%v error=%v", c.ok, e)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaSafetyRejectsNonFiniteAndUnsafeHealth(t *testing.T) {
|
||||
p := QueryReplicaSafetyPolicy{BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "r", RecordsPerSecond: 10, BaselineCPU: 85}
|
||||
if p.validate() != nil {
|
||||
t.Fatal("valid policy")
|
||||
}
|
||||
p.BaselineCPU = math.NaN()
|
||||
if p.validate() == nil {
|
||||
t.Fatal("NaN policy accepted")
|
||||
}
|
||||
now := time.Now()
|
||||
good := QueryReplicaSafetyObservation{At: now, CPU: 85, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1}
|
||||
if observationFailure(good, now) != "" {
|
||||
t.Fatal("healthy observation")
|
||||
}
|
||||
for _, change := range []func(*QueryReplicaSafetyObservation){func(o *QueryReplicaSafetyObservation) { o.CPU = 95 }, func(o *QueryReplicaSafetyObservation) { o.CPU = math.NaN() }, func(o *QueryReplicaSafetyObservation) { o.WorkErrors = true }, func(o *QueryReplicaSafetyObservation) { o.At = now.Add(-46 * time.Second) }, func(o *QueryReplicaSafetyObservation) { o.TargetUsedFraction = .7 }, func(o *QueryReplicaSafetyObservation) { o.HostFreeBytes = 99 << 30 }, func(o *QueryReplicaSafetyObservation) { o.ProtectedIOFull = 6 }} {
|
||||
o := good
|
||||
change(&o)
|
||||
if observationFailure(o, now) == "" {
|
||||
t.Fatal("unsafe observation accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaGlobalPacingAndCancellation(t *testing.T) {
|
||||
p := newQueryReplicaPacer(100)
|
||||
start := time.Now()
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 4; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if e := p.wait(context.Background(), 5); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if time.Since(start) < 190*time.Millisecond {
|
||||
t.Fatal("per-caller budgets bypassed shared rate")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
start = time.Now()
|
||||
if e := p.wait(ctx, 100); !errors.Is(e, context.Canceled) || time.Since(start) > 50*time.Millisecond {
|
||||
t.Fatal("uncancelable budget", e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaTransportReadDeadlineAndClose(t *testing.T) {
|
||||
for _, closeIt := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "deadline", true: "close"}[closeIt], func(t *testing.T) {
|
||||
a, b := net.Pipe()
|
||||
defer b.Close()
|
||||
budget, _ := NewQueryReplicaTransport(1024, 2)
|
||||
c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{}, 1)}
|
||||
defer c.Close()
|
||||
if !closeIt {
|
||||
_ = c.SetReadDeadline(time.Now().Add(30 * time.Millisecond))
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { _, e := c.Read(make([]byte, 1024)); done <- e }()
|
||||
if closeIt {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
_ = c.Close()
|
||||
}
|
||||
select {
|
||||
case e := <-done:
|
||||
if e == nil {
|
||||
t.Fatal("unbounded read succeeded")
|
||||
}
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
t.Fatal("pacing ignores connection deadline/close")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaTransportByteBudgetSharedAcrossConnections(t *testing.T) {
|
||||
budget, _ := NewQueryReplicaTransport(20480, 8)
|
||||
start := time.Now()
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 2; i++ {
|
||||
a, b := net.Pipe()
|
||||
c := &queryReplicaRateConn{Conn: a, budget: budget, closed: make(chan struct{}), wake: make(chan struct{}, 1)}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer c.Close()
|
||||
defer b.Close()
|
||||
go func() { _, _ = b.Write(make([]byte, 4096)) }()
|
||||
got := 0
|
||||
buf := make([]byte, 4096)
|
||||
for got < 4096 {
|
||||
n, e := c.Read(buf)
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
return
|
||||
}
|
||||
if n > 1024 {
|
||||
t.Error("unbounded transport burst")
|
||||
}
|
||||
got += n
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if time.Since(start) < 390*time.Millisecond {
|
||||
t.Fatal("aggregate byte rate exceeded")
|
||||
}
|
||||
}
|
||||
126
internal/store/query_replica_schedule.go
Normal file
126
internal/store/query_replica_schedule.go
Normal file
@ -0,0 +1,126 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Preserve incremental progress during history copy. Full dirty batches reserve
|
||||
// at least half the row opportunities (more than the30% floor); a backlog seen
|
||||
// continuously for5min receives four batches for each history batch. This age
|
||||
// is explicitly observation age, not an invented customer event timestamp.
|
||||
func (q *QueryReplica) backlogAge(ctx context.Context, n string, rows int) (time.Duration, error) {
|
||||
var since time.Time
|
||||
e := q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
key := "backlog/" + n
|
||||
if rows < int(q.limit()) {
|
||||
_, e := q.control().DeleteOne(sc, bson.M{"_id": key})
|
||||
return e
|
||||
}
|
||||
var d struct {
|
||||
Since time.Time `bson:"since"`
|
||||
}
|
||||
e := q.control().FindOne(sc, bson.M{"_id": key}).Decode(&d)
|
||||
if e != nil && e != mongo.ErrNoDocuments {
|
||||
return e
|
||||
}
|
||||
if e == mongo.ErrNoDocuments {
|
||||
d.Since = time.Now().UTC()
|
||||
if _, e = q.control().InsertOne(sc, bson.M{"_id": key, "since": d.Since}); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
since = d.Since
|
||||
return nil
|
||||
})
|
||||
if e != nil || since.IsZero() {
|
||||
return 0, e
|
||||
}
|
||||
return time.Since(since), nil
|
||||
}
|
||||
func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool, error) {
|
||||
if _, e := q.sourcePermit(ctx, 0); e != nil {
|
||||
return false, e
|
||||
}
|
||||
names := replicaCollections()
|
||||
var cursor struct {
|
||||
Next int `bson:"next"`
|
||||
}
|
||||
e := q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor)
|
||||
if e != nil && e != mongo.ErrNoDocuments {
|
||||
return false, e
|
||||
}
|
||||
start := cursor.Next % len(names)
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
for offset := 0; offset < len(names); offset++ {
|
||||
index := (start + offset) % len(names)
|
||||
name := names[index]
|
||||
n, e := q.DrainStep(ctx, name)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
age, e := q.backlogAge(ctx, name, n)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
if age > 5*time.Minute {
|
||||
for extra := 0; extra < 3; extra++ {
|
||||
n, e = q.DrainStep(ctx, name)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
if n < int(q.limit()) {
|
||||
if _, e = q.backlogAge(ctx, name, n); e != nil {
|
||||
return false, e
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
cp, e := q.checkpoint(ctx, name)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
if !cp.Complete {
|
||||
if !bootstrap {
|
||||
return false, ErrQueryReplicaNotReady
|
||||
}
|
||||
if _, _, e = q.BootstrapStep(ctx, name); e != nil {
|
||||
return false, e
|
||||
}
|
||||
}
|
||||
if e = q.transaction(ctx, func(sc mongo.SessionContext) error {
|
||||
_, e := q.control().UpdateOne(sc, bson.M{"_id": "schedule"}, bson.M{"$set": bson.M{"next": (index + 1) % len(names)}}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
}); e != nil {
|
||||
return false, e
|
||||
}
|
||||
}
|
||||
complete := true
|
||||
for _, n := range names {
|
||||
cp, e := q.checkpoint(ctx, n)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
complete = complete && cp.Complete
|
||||
}
|
||||
if complete {
|
||||
if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil {
|
||||
return false, e
|
||||
}
|
||||
}
|
||||
if _, e := q.ReconcileDeleteStep(ctx); e != nil {
|
||||
return false, e
|
||||
}
|
||||
if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady {
|
||||
return false, nil
|
||||
} else if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
165
internal/store/query_replica_source.go
Normal file
165
internal/store/query_replica_source.go
Normal file
@ -0,0 +1,165 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// All projector source mutations pass this seam. Database grants are scoped to
|
||||
// collections, not fields; neither raw replacements nor business fields belong
|
||||
// in this writer. These checks remain on even when optional pacing is disabled.
|
||||
func validateReplicaMetadataWrite(collection string, model mongo.WriteModel) error {
|
||||
m, ok := model.(*mongo.UpdateOneModel)
|
||||
if !ok || m.Upsert != nil && *m.Upsert {
|
||||
return errors.New("source metadata write rejected")
|
||||
}
|
||||
var f, u bson.M
|
||||
b, e := bson.Marshal(m.Filter)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if e = bson.Unmarshal(b, &f); e != nil {
|
||||
return e
|
||||
}
|
||||
b, e = bson.Marshal(m.Update)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if e = bson.Unmarshal(b, &u); e != nil {
|
||||
return e
|
||||
}
|
||||
if f["_id"] == nil {
|
||||
return errors.New("source metadata identity required")
|
||||
}
|
||||
if id := deletionDoc(f["_id"]); len(id) > 0 {
|
||||
for key := range id {
|
||||
if strings.HasPrefix(key, "$") {
|
||||
return errors.New("metadata identity selector rejected")
|
||||
}
|
||||
}
|
||||
}
|
||||
for op, raw := range u {
|
||||
if op != "$set" && op != "$unset" {
|
||||
return errors.New("source metadata operator rejected")
|
||||
}
|
||||
fields := deletionDoc(raw)
|
||||
if len(fields) == 0 {
|
||||
return errors.New("empty metadata update")
|
||||
}
|
||||
for path := range fields {
|
||||
allowed := false
|
||||
if collection == querySyncDeleteCollection {
|
||||
allowed = op == "$set" && (path == "phase" || path == "expireAt")
|
||||
} else if queryReplicaFields[collection] != "" {
|
||||
switch path {
|
||||
case "_querySync.dirty", "_querySync.revision", "_querySync.signature", "_querySync.mode", "_querySync.producer":
|
||||
allowed = op == "$set"
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
return errors.New("source business field write rejected")
|
||||
}
|
||||
}
|
||||
}
|
||||
if queryReplicaFields[collection] != "" && f[querySyncField+".revision"] == nil {
|
||||
return errors.New("metadata revision CAS required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (q *QueryReplica) sourceMetadataBulk(ctx context.Context, n string, models []mongo.WriteModel) error {
|
||||
if len(models) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, m := range models {
|
||||
if e := validateReplicaMetadataWrite(n, m); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
if _, e := q.sourcePermit(ctx, 0); e != nil {
|
||||
return e
|
||||
}
|
||||
ctx, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
_, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false))
|
||||
return e
|
||||
}
|
||||
func (q *QueryReplica) sourceIntentUpdate(ctx context.Context, filter, update bson.M) (*mongo.UpdateResult, error) {
|
||||
m := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update)
|
||||
if e := validateReplicaMetadataWrite(querySyncDeleteCollection, m); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if _, e := q.sourcePermit(ctx, 0); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
ctx, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
return q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, filter, update)
|
||||
}
|
||||
func (q *QueryReplica) sourceAuditIntent(ctx context.Context, i queryDeleteIntent) error {
|
||||
if !i.Audit || i.Phase != "reconcile" || queryReplicaFields[i.Collection] == "" || i.ID == "" {
|
||||
return errors.New("invalid source audit intent")
|
||||
}
|
||||
if _, e := q.sourcePermit(ctx, 0); e != nil {
|
||||
return e
|
||||
}
|
||||
ctx, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
_, e := q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i)
|
||||
return e
|
||||
}
|
||||
|
||||
// Cleanup is bounded even for a historical DeleteMany intent. We never replay
|
||||
// the source delete. A cancelled old writer cannot claim additional rows.
|
||||
func (q *QueryReplica) recoverSourceClaims(ctx context.Context, i queryDeleteIntent) (bool, error) {
|
||||
if i.Phase == "prepared" || i.Phase == "deleting" {
|
||||
now := time.Now().UTC()
|
||||
if i.LeaseUntil.After(now) {
|
||||
return false, nil
|
||||
}
|
||||
r, e := q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": i.Phase, "leaseUntil": bson.M{"$lte": now}}, bson.M{"$set": bson.M{"phase": "cancelled"}})
|
||||
if e != nil || r.MatchedCount != 1 {
|
||||
return false, e
|
||||
}
|
||||
} else if i.Phase != "complete" && i.Phase != "cancelled" && i.Phase != "reconcile" {
|
||||
return false, nil
|
||||
}
|
||||
limit, e := q.sourcePermit(ctx, q.limit())
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
work, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
cur, e := q.Source.db.Collection(i.Collection).Find(work, bson.M{querySyncField + ".deleteClaim": i.ID}, options.Find().SetHint("query_sync_delete_claim_v1").SetProjection(bson.M{"_id": 1}).SetLimit(limit).SetBatchSize(int32(limit)).SetMaxTime(1500*time.Millisecond))
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
var rows []bson.M
|
||||
if e = cur.All(work, &rows); e != nil {
|
||||
return false, e
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
if e = q.paceRows(ctx, len(rows)); e != nil {
|
||||
return false, e
|
||||
}
|
||||
if _, e = q.sourcePermit(ctx, 0); e != nil {
|
||||
return false, e
|
||||
}
|
||||
ids := bson.A{}
|
||||
for _, r := range rows {
|
||||
ids = append(ids, r["_id"])
|
||||
}
|
||||
// This is the only UpdateMany: exact bounded identities and exact claim,
|
||||
// with only its private claim field removed; no upsert or business delete.
|
||||
work, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
_, e = q.Source.db.Collection(i.Collection).UpdateMany(work, bson.M{"_id": bson.M{"$in": ids}, querySyncField + ".deleteClaim": i.ID}, bson.M{"$unset": bson.M{querySyncField + ".deleteClaim": ""}})
|
||||
return false, e
|
||||
}
|
||||
_, e = q.sourceIntentUpdate(ctx, bson.M{"_id": i.ID, "phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"$set": bson.M{"phase": "reconcile"}})
|
||||
return e == nil, e
|
||||
}
|
||||
118
internal/store/query_replica_source_auth_local_test.go
Normal file
118
internal/store/query_replica_source_auth_local_test.go
Normal file
@ -0,0 +1,118 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// This fixture owns a separate ephemeral auth-enabled standalone Mongo. It
|
||||
// never enables authorization or creates users on the shared source/query lab.
|
||||
func TestQueryReplicaSourceAccountPermissionsLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" {
|
||||
t.Skip("isolated auth fixture required")
|
||||
}
|
||||
ctx := t.Context()
|
||||
if host := os.Getenv("DOCKER_HOST"); host != "" && !strings.HasPrefix(host, "unix://") {
|
||||
t.Fatal("remote Docker prohibited")
|
||||
}
|
||||
endpoint, e := exec.Command("docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}").Output()
|
||||
if e != nil || !strings.HasPrefix(strings.TrimSpace(string(endpoint)), "unix://") {
|
||||
t.Fatal("local Docker socket required")
|
||||
}
|
||||
user, password := domain.NewID("root"), domain.NewID("secret")
|
||||
name := strings.ToLower(domain.NewID("query-auth"))
|
||||
cmd := exec.Command("docker", "run", "--rm", "-d", "--name", name, "--label", "codex.fixture=query-source-auth", "--cpus=1", "--memory=512m", "-p", "127.0.0.1::27017", "--env", "MONGO_INITDB_ROOT_USERNAME", "--env", "MONGO_INITDB_ROOT_PASSWORD", "mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b", "--wiredTigerCacheSizeGB", "0.25", "--bind_ip_all")
|
||||
cmd.Env = append(os.Environ(), "MONGO_INITDB_ROOT_USERNAME="+user, "MONGO_INITDB_ROOT_PASSWORD="+password)
|
||||
out, e := cmd.Output()
|
||||
if e != nil {
|
||||
t.Fatal("owned authentication fixture startup failed")
|
||||
}
|
||||
container := strings.TrimSpace(string(out))
|
||||
t.Cleanup(func() { _ = exec.Command("docker", "stop", "--time=5", container).Run() })
|
||||
raw, e := exec.Command("docker", "inspect", "--format", "{{json .NetworkSettings.Ports}}", container).Output()
|
||||
if e != nil {
|
||||
t.Fatal("fixture binding unavailable")
|
||||
}
|
||||
var ports map[string][]struct{ HostIP, HostPort string }
|
||||
if json.Unmarshal(raw, &ports) != nil || len(ports["27017/tcp"]) != 1 || ports["27017/tcp"][0].HostIP != "127.0.0.1" {
|
||||
t.Fatal("fixture not loopback-only")
|
||||
}
|
||||
uri := "mongodb://127.0.0.1:" + ports["27017/tcp"][0].HostPort + "/?directConnection=true"
|
||||
admin, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetAuth(options.Credential{AuthSource: "admin", Username: user, Password: password}).SetServerSelectionTimeout(500*time.Millisecond))
|
||||
if e != nil {
|
||||
t.Fatal("fixture connection failed")
|
||||
}
|
||||
defer admin.Disconnect(context.Background())
|
||||
ready := false
|
||||
for n := 0; n < 80; n++ {
|
||||
short, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
e = admin.Ping(short, nil)
|
||||
cancel()
|
||||
if e == nil {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
if !ready {
|
||||
t.Fatal("authentication fixture readiness failed")
|
||||
}
|
||||
dbname := domain.NewID("query_sync_auth")
|
||||
db := admin.Database(dbname)
|
||||
privilege := bson.A{bson.M{"resource": bson.M{"cluster": true}, "actions": bson.A{"serverStatus"}}, bson.M{"resource": bson.M{"db": dbname, "collection": "humanItems"}, "actions": bson.A{"find", "update"}}, bson.M{"resource": bson.M{"db": dbname, "collection": querySyncDeleteCollection}, "actions": bson.A{"find", "insert", "update"}}}
|
||||
if e = admin.Database("admin").RunCommand(ctx, bson.D{{Key: "createRole", Value: "metadata-sync"}, {Key: "privileges", Value: privilege}, {Key: "roles", Value: bson.A{}}}).Err(); e != nil {
|
||||
t.Fatal("fixture role setup failed", e)
|
||||
}
|
||||
limitedPassword := domain.NewID("limited-secret")
|
||||
if e = db.RunCommand(ctx, bson.D{{Key: "createUser", Value: "projector"}, {Key: "pwd", Value: limitedPassword}, {Key: "roles", Value: bson.A{bson.M{"role": "metadata-sync", "db": "admin"}}}}).Err(); e != nil {
|
||||
t.Fatal("fixture user setup failed")
|
||||
}
|
||||
_, e = db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "i", "id": "i", "version": 7, "status": "claimed", "annotationDraft": "keep", "_querySync": bson.M{"revision": 1, "dirty": true}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetAuth(options.Credential{AuthSource: dbname, Username: "projector", Password: limitedPassword}))
|
||||
if e != nil {
|
||||
t.Fatal("fixture principal unavailable")
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
limited := client.Database(dbname)
|
||||
q := &QueryReplica{Source: NewMongoStore(limited)}
|
||||
if e = q.sourceMetadataBulk(ctx, "humanItems", []mongo.WriteModel{mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"_querySync.dirty": false}})}); e != nil {
|
||||
t.Fatal("allowed acknowledgement failed", e)
|
||||
}
|
||||
if e = q.sourceMetadataBulk(ctx, "humanItems", []mongo.WriteModel{mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": "i", "_querySync.revision": 1}).SetUpdate(bson.M{"$set": bson.M{"annotationDraft": "overwrite"}})}); e == nil {
|
||||
t.Fatal("application metadata boundary bypassed")
|
||||
}
|
||||
denied := func(label string, e error) {
|
||||
t.Helper()
|
||||
var ce mongo.CommandError
|
||||
if !errors.As(e, &ce) || ce.Code != 13 {
|
||||
t.Fatalf("%s was not denied by real authentication", label)
|
||||
}
|
||||
}
|
||||
_, e = limited.Collection("humanItems").InsertOne(ctx, bson.M{"_id": "other"})
|
||||
denied("insert", e)
|
||||
_, e = limited.Collection("humanItems").DeleteOne(ctx, bson.M{"_id": "i"})
|
||||
denied("delete", e)
|
||||
denied("DDL", limited.RunCommand(ctx, bson.M{"drop": "humanItems"}).Err())
|
||||
denied("other database", client.Database("foreign").Collection("humanItems").FindOne(ctx, bson.M{}).Err())
|
||||
if e = client.Database("admin").RunCommand(ctx, bson.M{"serverStatus": 1}).Err(); e != nil {
|
||||
t.Fatal("monitor privilege unavailable")
|
||||
}
|
||||
var kept bson.M
|
||||
if e = db.Collection("humanItems").FindOne(ctx, bson.M{"_id": "i"}).Decode(&kept); e != nil || kept["annotationDraft"] != "keep" || kept["version"] != int32(7) || kept["status"] != "claimed" || deletionDoc(kept[querySyncField])["dirty"] != false {
|
||||
t.Fatal("authority fields altered")
|
||||
}
|
||||
|
||||
}
|
||||
158
internal/store/query_replica_transport.go
Normal file
158
internal/store/query_replica_transport.go
Normal file
@ -0,0 +1,158 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A shared leaky bucket, with no accumulated idle credit. The source driver
|
||||
// shares it across application and monitoring connections; TLS overhead is
|
||||
// conservatively included. Waiting never occurs inside a target transaction.
|
||||
type queryReplicaPacer struct {
|
||||
mu sync.Mutex
|
||||
next time.Time
|
||||
rate float64
|
||||
}
|
||||
|
||||
func newQueryReplicaPacer(rate float64) *queryReplicaPacer { return &queryReplicaPacer{rate: rate} }
|
||||
func (p *queryReplicaPacer) wait(ctx context.Context, units float64) error {
|
||||
if units <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
p.mu.Lock()
|
||||
now := time.Now()
|
||||
if p.next.Before(now) {
|
||||
p.next = now
|
||||
}
|
||||
p.next = p.next.Add(time.Duration(units / p.rate * float64(time.Second)))
|
||||
until := p.next
|
||||
p.mu.Unlock()
|
||||
timer := time.NewTimer(time.Until(until))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type QueryReplicaTransport struct{ bytes, commands *queryReplicaPacer }
|
||||
|
||||
func NewQueryReplicaTransport(bytesPerSecond, commandsPerSecond int) (*QueryReplicaTransport, error) {
|
||||
if bytesPerSecond < 1024 || bytesPerSecond > 256*1024 || commandsPerSecond < 1 || commandsPerSecond > 8 {
|
||||
return nil, errors.New("invalid source transport budget")
|
||||
}
|
||||
return &QueryReplicaTransport{newQueryReplicaPacer(float64(bytesPerSecond)), newQueryReplicaPacer(float64(commandsPerSecond))}, nil
|
||||
}
|
||||
func (d *QueryReplicaTransport) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
c, e := (&net.Dialer{}).DialContext(ctx, network, address)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return &queryReplicaRateConn{Conn: c, budget: d, closed: make(chan struct{}), wake: make(chan struct{}, 1)}, nil
|
||||
}
|
||||
|
||||
type queryReplicaRateConn struct {
|
||||
net.Conn
|
||||
budget *QueryReplicaTransport
|
||||
once sync.Once
|
||||
closed chan struct{}
|
||||
mu sync.Mutex
|
||||
readDeadline, writeDeadline time.Time
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
func (c *queryReplicaRateConn) Close() error {
|
||||
c.once.Do(func() { close(c.closed) })
|
||||
return c.Conn.Close()
|
||||
}
|
||||
func (c *queryReplicaRateConn) SetDeadline(t time.Time) error {
|
||||
c.mu.Lock()
|
||||
c.readDeadline = t
|
||||
c.writeDeadline = t
|
||||
c.mu.Unlock()
|
||||
c.notify()
|
||||
return c.Conn.SetDeadline(t)
|
||||
}
|
||||
func (c *queryReplicaRateConn) SetReadDeadline(t time.Time) error {
|
||||
c.mu.Lock()
|
||||
c.readDeadline = t
|
||||
c.mu.Unlock()
|
||||
c.notify()
|
||||
return c.Conn.SetReadDeadline(t)
|
||||
}
|
||||
func (c *queryReplicaRateConn) SetWriteDeadline(t time.Time) error {
|
||||
c.mu.Lock()
|
||||
c.writeDeadline = t
|
||||
c.mu.Unlock()
|
||||
c.notify()
|
||||
return c.Conn.SetWriteDeadline(t)
|
||||
}
|
||||
func (c *queryReplicaRateConn) notify() {
|
||||
c.mu.Lock()
|
||||
old := c.wake
|
||||
c.wake = make(chan struct{})
|
||||
close(old)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
func (c *queryReplicaRateConn) wait(p *queryReplicaPacer, n int, read bool) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- p.wait(ctx, float64(n)) }()
|
||||
for {
|
||||
c.mu.Lock()
|
||||
wake := c.wake
|
||||
deadline := c.writeDeadline
|
||||
if read {
|
||||
deadline = c.readDeadline
|
||||
}
|
||||
c.mu.Unlock()
|
||||
var timer *time.Timer
|
||||
var expired <-chan time.Time
|
||||
if !deadline.IsZero() {
|
||||
timer = time.NewTimer(time.Until(deadline))
|
||||
expired = timer.C
|
||||
}
|
||||
select {
|
||||
case e := <-done:
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
return e
|
||||
case <-c.closed:
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
return net.ErrClosed
|
||||
case <-expired:
|
||||
return context.DeadlineExceeded
|
||||
case <-wake:
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func (c *queryReplicaRateConn) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if len(p) > 1024 {
|
||||
p = p[:1024]
|
||||
}
|
||||
if e := c.wait(c.budget.bytes, len(p), true); e != nil {
|
||||
return 0, e
|
||||
}
|
||||
return c.Conn.Read(p)
|
||||
}
|
||||
func (c *queryReplicaRateConn) Write(p []byte) (int, error) {
|
||||
if e := c.wait(c.budget.commands, 1, false); e != nil {
|
||||
return 0, e
|
||||
}
|
||||
return c.Conn.Write(p)
|
||||
}
|
||||
296
scripts/local/query-shadow-observer.py
Normal file
296
scripts/local/query-shadow-observer.py
Normal file
@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders.
|
||||
|
||||
Every run writes private health observations in the derived query database.
|
||||
Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the
|
||||
TWO shadow controllers in the hash-bound plan. Never changes authority services.
|
||||
Do not run against production before approving the final startup/stop scope.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
def check_bindings(b):
|
||||
if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5:
|
||||
raise ValueError('invalid namespace or writer scope')
|
||||
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
|
||||
raise ValueError('stop scope must contain only shadow controllers')
|
||||
if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90:
|
||||
raise ValueError('invalid source identity or baseline')
|
||||
|
||||
|
||||
def cpu_percent(a, z):
|
||||
if z['sourceProcess'] != a['sourceProcess']:
|
||||
raise ValueError('source identity changed')
|
||||
elapsed = z['uptimeMillis'] - a['uptimeMillis']
|
||||
used = z['cpuMicroseconds'] - a['cpuMicroseconds']
|
||||
if elapsed <= 0 or used < 0:
|
||||
raise ValueError('nonmonotonic source counters')
|
||||
return used / 1000 / elapsed / 4 * 100
|
||||
|
||||
|
||||
def full_io_pressure(text):
|
||||
for line in text.splitlines():
|
||||
if line.startswith('full '):
|
||||
return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10=')))
|
||||
raise ValueError('protected IO evidence missing')
|
||||
|
||||
|
||||
def business_health(path, now=None):
|
||||
feed = json.loads(Path(path).read_text())
|
||||
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
|
||||
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
|
||||
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
|
||||
or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool):
|
||||
raise ValueError('business-health evidence stale or incomplete')
|
||||
return feed['unexpectedErrors'] or feed['latencyRegressed']
|
||||
|
||||
|
||||
class Observer:
|
||||
def __init__(self, args, bindings):
|
||||
self.args, self.b = args, bindings
|
||||
self.k = ['kubectl', '--kubeconfig', args.kubeconfig]
|
||||
self.high = self.high_io = 0
|
||||
|
||||
def call(self, *args, data=None, timeout=15):
|
||||
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
|
||||
text=True, timeout=timeout)
|
||||
if p.returncode:
|
||||
raise RuntimeError('bounded cluster operation unavailable')
|
||||
return p.stdout
|
||||
|
||||
def get(self, ns, kind, name):
|
||||
return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json'))
|
||||
|
||||
def verify(self):
|
||||
for key in ('source', 'shadow'):
|
||||
n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace'])
|
||||
if n['metadata']['uid'] != self.b[key + 'NamespaceUID']:
|
||||
raise ValueError('namespace identity changed')
|
||||
for w in self.b['writers']:
|
||||
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
|
||||
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
|
||||
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
|
||||
or env.get('QUERY_SOURCE_TRACKING') != '1'
|
||||
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
|
||||
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
|
||||
raise ValueError('writer coverage or health changed')
|
||||
for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'):
|
||||
if env.get(flag, '0') not in ('', '0', 'false'):
|
||||
raise ValueError('production query entry changed')
|
||||
|
||||
def secret(self, name, key):
|
||||
return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode()
|
||||
|
||||
def target_environment(self, uri):
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))}
|
||||
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
|
||||
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
|
||||
return env
|
||||
|
||||
def run_binary(self, args, env, data=None):
|
||||
if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256:
|
||||
raise ValueError('observer binary changed')
|
||||
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
|
||||
text=True, timeout=10)
|
||||
if p.returncode:
|
||||
raise RuntimeError('bounded observation command unavailable')
|
||||
return p.stdout
|
||||
|
||||
@contextlib.contextmanager
|
||||
def source_relay(self, uri):
|
||||
u = urllib.parse.urlsplit(uri)
|
||||
if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc:
|
||||
raise ValueError('single reviewed source endpoint required')
|
||||
endpoint = u.hostname + ':' + str(u.port or 27017)
|
||||
if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']:
|
||||
raise ValueError('source endpoint changed')
|
||||
# No secret arguments or stderr: credentials remain in local child env.
|
||||
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
||||
pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp')
|
||||
and x['status'].get('phase') == 'Running'
|
||||
and all(c['ready'] for c in x['status'].get('containerStatuses', [])))
|
||||
listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2)
|
||||
stop = threading.Event(); children = []
|
||||
relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true'
|
||||
|
||||
def transfer(conn):
|
||||
p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay,
|
||||
'shadow-readonly-metrics', u.hostname, str(u.port or 27017)],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
children.append(p)
|
||||
def upstream():
|
||||
try:
|
||||
while not stop.is_set():
|
||||
chunk = conn.recv(65536)
|
||||
if not chunk: break
|
||||
p.stdin.write(chunk); p.stdin.flush()
|
||||
except (OSError, ValueError): pass
|
||||
finally:
|
||||
with contextlib.suppress(OSError): p.stdin.close()
|
||||
threading.Thread(target=upstream, daemon=True).start()
|
||||
try:
|
||||
while not stop.is_set():
|
||||
chunk = p.stdout.read1(65536)
|
||||
if not chunk: break
|
||||
conn.sendall(chunk)
|
||||
except (OSError, ValueError): pass
|
||||
finally:
|
||||
conn.close(); p.terminate()
|
||||
def accept():
|
||||
while not stop.is_set():
|
||||
try: conn, _ = listener.accept()
|
||||
except socket.timeout: continue
|
||||
except OSError: return
|
||||
threading.Thread(target=transfer, args=(conn,), daemon=True).start()
|
||||
thread = threading.Thread(target=accept, daemon=True); thread.start()
|
||||
params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true']
|
||||
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
|
||||
local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path,
|
||||
urllib.parse.urlencode(params, doseq=True), ''))
|
||||
# A TLS hostname is not rewritten/bypassed. This initial deployment uses
|
||||
# its reviewed non-TLS private endpoint; other topologies need a new plan.
|
||||
if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true':
|
||||
listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling')
|
||||
try: yield local
|
||||
finally:
|
||||
stop.set(); listener.close(); thread.join(timeout=1)
|
||||
for child in children:
|
||||
if child.poll() is None: child.terminate()
|
||||
for child in children:
|
||||
try: child.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired: child.kill()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def target_forward(self, uri):
|
||||
with socket.socket() as listener:
|
||||
listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1]
|
||||
process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1',
|
||||
'service/query-mongo', str(port) + ':27129'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
for _ in range(40):
|
||||
if process.poll() is not None: raise RuntimeError('query tunnel unavailable')
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', port), timeout=.1): break
|
||||
except OSError: time.sleep(.1)
|
||||
else: raise RuntimeError('query tunnel deadline')
|
||||
u = urllib.parse.urlsplit(uri)
|
||||
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
|
||||
yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, ''))
|
||||
finally:
|
||||
process.terminate()
|
||||
try: process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired: process.kill()
|
||||
|
||||
def source_sample(self):
|
||||
with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri:
|
||||
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
|
||||
return json.loads(self.run_binary(['--sample-source'], env))
|
||||
|
||||
def observation(self, cpu, sampled_at):
|
||||
self.verify()
|
||||
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
|
||||
ready = response.status == 200 and json.load(response).get('ready') is True
|
||||
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
|
||||
if pod['metadata']['uid'] != self.b['mongoPodUID']:
|
||||
raise ValueError('query Mongo rebuilt; recovery verification required')
|
||||
fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
|
||||
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
|
||||
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
|
||||
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
|
||||
io = 0.
|
||||
for p in protected:
|
||||
if p['status'].get('phase') != 'Running': continue
|
||||
uid = p['metadata']['uid'].replace('-', '_')
|
||||
# Current production workloads are Burstable; refuse unknown cgroups.
|
||||
path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure'
|
||||
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
|
||||
# A trusted business-health feed is required; readiness/log absence is
|
||||
# not evidence that saves/submits meet their latency/error baseline.
|
||||
work_errors = business_health(self.args.work_health)
|
||||
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
||||
for p in pods:
|
||||
if p['metadata'].get('deletionTimestamp'): continue
|
||||
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
|
||||
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
|
||||
return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
|
||||
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
|
||||
|
||||
def publish(self, observation):
|
||||
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
|
||||
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
|
||||
|
||||
def unsafe(self, o):
|
||||
self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0
|
||||
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
|
||||
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2
|
||||
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
|
||||
|
||||
def stop_shadow(self):
|
||||
for c in self.b['controllers']:
|
||||
d = self.get(self.b['shadowNamespace'], 'deployment', c['name'])
|
||||
if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced')
|
||||
if d['spec']['replicas'] == 0: continue
|
||||
patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']},
|
||||
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
|
||||
{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]
|
||||
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
|
||||
|
||||
def run(self):
|
||||
try: prior = self.source_sample()
|
||||
except Exception:
|
||||
if self.args.stop_on_failure: self.stop_shadow()
|
||||
raise RuntimeError('initial source observation unavailable') from None
|
||||
while True:
|
||||
try:
|
||||
for _ in range(3):
|
||||
time.sleep(10)
|
||||
with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response:
|
||||
if response.status!=200 or json.load(response).get('ready') is not True:
|
||||
raise ValueError('production readiness unavailable')
|
||||
current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current
|
||||
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
|
||||
if not 0<=age<=45: raise ValueError('source observation stale')
|
||||
o = self.observation(cpu,sampled_at); self.publish(o)
|
||||
unsafe=self.unsafe(o)
|
||||
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True)
|
||||
if unsafe:
|
||||
if self.args.stop_on_failure: self.stop_shadow()
|
||||
return
|
||||
except Exception:
|
||||
# No raw exception may print a URI, secret or customer log line.
|
||||
with contextlib.suppress(Exception):
|
||||
self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False,
|
||||
workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0))
|
||||
if self.args.stop_on_failure: self.stop_shadow()
|
||||
raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True)
|
||||
p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True)
|
||||
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
|
||||
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
|
||||
b = json.loads(raw); check_bindings(b); Observer(args, b).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except KeyboardInterrupt: raise SystemExit(0)
|
||||
except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')
|
||||
63
scripts/local/test_query_shadow_observer.py
Normal file
63
scripts/local/test_query_shadow_observer.py
Normal file
@ -0,0 +1,63 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
import tempfile
|
||||
import datetime as dt
|
||||
|
||||
spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).with_name('query-shadow-observer.py'))
|
||||
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
|
||||
class ObserverTests(unittest.TestCase):
|
||||
def bindings(self):
|
||||
return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
|
||||
baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])
|
||||
def test_cpu_and_identity(self):
|
||||
self.assertAlmostEqual(m.cpu_percent(dict(sourceProcess='s', uptimeMillis=1000, cpuMicroseconds=0),
|
||||
dict(sourceProcess='s', uptimeMillis=2000, cpuMicroseconds=2000000)), 50)
|
||||
with self.assertRaises(ValueError):
|
||||
m.cpu_percent(dict(sourceProcess='s', uptimeMillis=2, cpuMicroseconds=2), dict(sourceProcess='x', uptimeMillis=3, cpuMicroseconds=3))
|
||||
def test_stop_scope_and_uid_cas(self):
|
||||
b = self.bindings(); m.check_bindings(b)
|
||||
observer = m.Observer(SimpleNamespace(kubeconfig='unused', stop_on_failure=True), b); calls = []
|
||||
observer.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if name=='query-replica' else 'b', resourceVersion='7'), spec=dict(replicas=1))
|
||||
observer.call = lambda *args, **kwargs: calls.append(args) or ''
|
||||
observer.stop_shadow()
|
||||
self.assertEqual(len(calls), 2)
|
||||
for args in calls:
|
||||
self.assertEqual(args[:4], ('-n', 'shadow', 'patch', 'deployment'))
|
||||
payload = json.loads(args[-1]); self.assertEqual(payload[0]['op'], 'test'); self.assertEqual(payload[-1]['value'], 0)
|
||||
observer.get = lambda *args: dict(metadata=dict(uid='replaced', resourceVersion='8'), spec=dict(replicas=1))
|
||||
with self.assertRaises(ValueError): observer.stop_shadow()
|
||||
self.assertEqual(len(calls), 2)
|
||||
b['controllers'][0]['name'] = 'droboticflow-api'
|
||||
with self.assertRaises(ValueError): m.check_bindings(b)
|
||||
def test_distinct_samples_and_thresholds(self):
|
||||
o = m.Observer(SimpleNamespace(kubeconfig='unused'), self.bindings())
|
||||
good = dict(cpu=90, ready=True, writersCovered=True, workErrors=False, protectedIOFull=0, targetUsedFraction=.1, hostFreeBytes=120<<30)
|
||||
self.assertFalse(o.unsafe(good)); self.assertTrue(o.unsafe(good))
|
||||
good['cpu']=85; self.assertFalse(o.unsafe(good));good['targetUsedFraction']=.7;self.assertTrue(o.unsafe(good))
|
||||
def test_business_health_missing_stale_and_real_errors(self):
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / 'health.json'
|
||||
with self.assertRaises(FileNotFoundError): m.business_health(path, now)
|
||||
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False)
|
||||
path.write_text(json.dumps(feed)); self.assertFalse(m.business_health(path, now))
|
||||
feed['latencyRegressed'] = True
|
||||
path.write_text(json.dumps(feed)); self.assertTrue(m.business_health(path, now))
|
||||
feed['latencyRegressed'] = 'false'
|
||||
path.write_text(json.dumps(feed))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now)
|
||||
feed['latencyRegressed'] = False; feed['at'] = (now - dt.timedelta(seconds=46)).isoformat()
|
||||
path.write_text(json.dumps(feed))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now)
|
||||
feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only'
|
||||
path.write_text(json.dumps(feed))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now)
|
||||
def test_unknown_io_is_not_zero(self):
|
||||
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
|
||||
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@ -15,7 +15,11 @@ from lib.release_checks import check_environment, source_fingerprint, write_log
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PRIVATE = ROOT / 'local-data/query-replica-local'
|
||||
COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
|
||||
CASES = ('internal/store/query_replica_local_test.go',
|
||||
CASES = ('cmd/query-replica/main_local_test.go',
|
||||
'internal/store/query_replica_source_auth_local_test.go',
|
||||
'internal/store/query_replica_safety_local_test.go',
|
||||
'internal/store/query_replica_safety_test.go',
|
||||
'internal/store/query_replica_local_test.go',
|
||||
'internal/store/query_sync_local_test.go',
|
||||
'internal/store/query_replica_count_cache_test.go',
|
||||
'internal/api/query_replica_local_test.go',
|
||||
@ -91,6 +95,21 @@ def start(env):
|
||||
required = {27119: 'source', 27129: 'query'}
|
||||
if any(required[port] not in running for port in busy):
|
||||
raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports')
|
||||
if len(busy) == 2:
|
||||
# Reuse isolated random databases, never recreate a shared fixture just
|
||||
# because another authorized checkout owns its Compose metadata.
|
||||
info = json.loads(subprocess.check_output(['docker', 'inspect', 'fanyu-query-replica-local-source-1',
|
||||
'fanyu-query-replica-local-query-1'], cwd=ROOT, env=env, text=True))
|
||||
resources = [{'name': x['Name'], 'cpu': x['HostConfig']['NanoCpus'],
|
||||
'memory': x['HostConfig']['Memory']} for x in info]
|
||||
if env.get('QUERY_REPLICA_LOCAL_QUERY_BUDGET') == 'query25':
|
||||
query = next(x for x in resources if x['name'].endswith('query-1'))
|
||||
if query['cpu'] != 1_000_000_000 or query['memory'] != 1024 ** 3:
|
||||
raise RuntimeError('existing shared query fixture differs from1CPU/1Gi; do not recreate it')
|
||||
env['QUERY_REPLICA_REUSED_RESOURCES'] = json.dumps(resources)
|
||||
compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval',
|
||||
"if (!db.hello().isWritablePrimary || db.hello().setName !== 'query-read-rs') quit(2)")
|
||||
return
|
||||
compose(env, 'up', '-d')
|
||||
for _ in range(60):
|
||||
try:
|
||||
@ -135,7 +154,8 @@ def main():
|
||||
else:
|
||||
raise RuntimeError('select --native, --start or --auto explicitly')
|
||||
if args.cache:
|
||||
compose(env, 'up', '-d', 'valkey')
|
||||
if 'valkey' not in compose(env, 'ps', '--status', 'running', '--services').splitlines():
|
||||
compose(env, 'up', '-d', 'valkey')
|
||||
env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1'
|
||||
required = set()
|
||||
for name in CASES:
|
||||
@ -146,7 +166,7 @@ def main():
|
||||
required = {'TestQueryReplicaScaleLocal'}
|
||||
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
|
||||
regex = '^(' + '|'.join(sorted(required)) + ')$'
|
||||
argv = ['go', 'test', './internal/store', './internal/api', './cmd/query-cdc-probe',
|
||||
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
|
||||
'-run', regex, '-count=1', '-timeout=10m', '-v']
|
||||
if args.race:
|
||||
argv.append('-race')
|
||||
@ -159,7 +179,7 @@ def main():
|
||||
proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed')
|
||||
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
|
||||
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
|
||||
productionCapacityAccepted=False)
|
||||
productionCapacityAccepted=False, reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
|
||||
print(json.dumps(proof, ensure_ascii=False))
|
||||
return 0 if success else (code or 2)
|
||||
|
||||
|
||||
@ -1018,6 +1018,7 @@
|
||||
{
|
||||
"name": "independent-query-replica",
|
||||
"paths": [
|
||||
"^scripts/local/(query-shadow-observer|test_query_shadow_observer)\\.py$",
|
||||
"^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$",
|
||||
"^internal/api/query_replica.*\\.go$",
|
||||
"^internal/store/workforce_(report_cache|heap_diagnostic|facts|return_page|publication).*\\.go$",
|
||||
@ -1052,6 +1053,20 @@
|
||||
"./internal/testsupport": ["^TestQueryReplicaTarget"]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
"argv": [
|
||||
"python3",
|
||||
"-m",
|
||||
"unittest",
|
||||
"discover",
|
||||
"-s",
|
||||
"scripts/local",
|
||||
"-p",
|
||||
"test_query_shadow_observer.py",
|
||||
"-v"
|
||||
],
|
||||
"tests": true
|
||||
},
|
||||
{
|
||||
"argv": ["python3", "scripts/test_query_replica.py", "--auto", "--bounded-query", "--race", "--cache"],
|
||||
"tests": true
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user