forked from longtao.wu/droboticflow
🛠️ Allow guarded shadow initialization during maintenance windows
This commit is contained in:
parent
ae9e33a3bc
commit
ad1d91a9fc
@ -35,6 +35,8 @@ func main() {
|
||||
rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing")
|
||||
bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget")
|
||||
commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead")
|
||||
maintenanceUntil := flag.String("maintenance-until", "", "explicit RFC3339 deadline for the initialization-only CPU observation policy")
|
||||
maintenance := flag.Bool("maintenance-initialization", false, "explicit finite initialization run: CPU is observed, not an admission/stop gate; resource and durability guards remain")
|
||||
baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume")
|
||||
resume := flag.String("resume-pause", "", "exact current pause nonce after operator review")
|
||||
initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes")
|
||||
@ -42,6 +44,17 @@ func main() {
|
||||
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
|
||||
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
|
||||
flag.Parse()
|
||||
var maintenanceDeadline *time.Time
|
||||
if *maintenance {
|
||||
value, err := time.Parse(time.RFC3339, *maintenanceUntil)
|
||||
if err != nil || !value.After(time.Now()) {
|
||||
log.Fatal("explicit future maintenance deadline required")
|
||||
}
|
||||
maintenanceDeadline = &value
|
||||
}
|
||||
if *maintenance && (*safetyRun == "" || *maxRecords <= 0) {
|
||||
log.Fatal("maintenance initialization requires a guarded finite budget")
|
||||
}
|
||||
if *sampleSource {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
@ -55,7 +68,11 @@ func main() {
|
||||
defer c.Disconnect(context.Background())
|
||||
var info struct {
|
||||
Uptime int64 `bson:"uptimeMillis"`
|
||||
Extra struct {
|
||||
Memory struct {
|
||||
Resident int64 `bson:"resident"`
|
||||
Supported bool `bson:"supported"`
|
||||
} `bson:"mem"`
|
||||
Extra struct {
|
||||
User int64 `bson:"user_time_us"`
|
||||
System int64 `bson:"system_time_us"`
|
||||
} `bson:"extra_info"`
|
||||
@ -72,7 +89,7 @@ func main() {
|
||||
log.Fatal("source identity unavailable")
|
||||
}
|
||||
identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process)))
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano), "residentMiB": info.Memory.Resident, "memorySupported": info.Memory.Supported})
|
||||
return
|
||||
}
|
||||
if *observation != "" || *status {
|
||||
@ -204,7 +221,11 @@ func main() {
|
||||
return
|
||||
}
|
||||
if *safetyRun != "" {
|
||||
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
|
||||
mode := ""
|
||||
if *maintenance {
|
||||
mode = store.QueryReplicaMaintenanceInitialization
|
||||
}
|
||||
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{Mode: mode, MaintenanceUntil: maintenanceDeadline, BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
|
||||
log.Print("shadow safety paused; inspect durable state and explicitly resume")
|
||||
<-ctx.Done()
|
||||
return
|
||||
|
||||
@ -229,3 +229,20 @@ Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予
|
||||
8. 回退:先停两个影子控制器并确认 Pod 退出,再停观察器;保留目标断点、暂停记录、最后完整结果和源标记。不删除源记录/标记,不清空查询库,不恢复业务旧快照。再次启动先读 `query-replica --safety-status`,重新确认健康、绑定和当前暂停令牌。生产入口从未切换,无需改客户作业配置。
|
||||
|
||||
此阶段只能证明新增同步负载受控及影子结果正确。生产入口未切换时不能宣称原 Mongo 总 CPU 已下降;降载验收仍需后续经授权切读后的同条件对照。
|
||||
|
||||
|
||||
## 维护窗口初始化策略(2026-11-14,明确授权的临时覆盖)
|
||||
|
||||
本节仅覆盖上节影子初始化的 CPU 启停规则。用户已明确说明维护窗口无客户作业,并允许原 Mongo 初始化期间持续高 CPU;不再等待低 CPU 窗口。历史方案及批准摘要保留,不修改既有发布记录。
|
||||
|
||||
投影器必须同时显式传入 `--maintenance-initialization`、未来的 `--maintenance-until=<RFC3339>`、原 `--safety-run` 和有限 `--max-source-records`。策略以 `maintenance-initialization` 持久化;默认严格模式不变,既有暂停必须使用当前令牌显式恢复,累计预算不会清零。
|
||||
|
||||
- CPU 持续采集,仅观察:取消该模式的启动 CPU≤90%、波动≤5个百分点/三样本等待,以及运行 CPU≥95%和相对基线增长暂停。单份完整、新鲜的健康观测即可准入。CPU 数据缺失或非有限数仍不能伪装健康。
|
||||
- 停止条件保持:源数据库不可用、客户作业错误或延迟恶化、权限/身份/版本变化、健康过期、实际内存压力或 OOM、磁盘容量不足、持久化失败、受保护业务 IO 异常。维护期限到达或读取预算耗尽也持久暂停。停止仅限绑定的两个影子控制器,保留断点和最后完整结果。
|
||||
- 观察器记录时钟偏移与往返耗时的独立结果、Pod UID 和每次失败类型。传输/探测瞬时失败最多3次,间隔0.5、1秒;明确不健康返回、权限和身份变化直接停止,不能通过重试掩盖。不得记录连接凭证和客户正文。
|
||||
- 内存证据绑定源 `hostInfo.system.memLimitMB`、`serverStatus.mem.resident` 及目标 cgroup 的实际上限/OOM 计数;节点 MemoryPressure 缺失不是健康。目标日志的持久化错误和日志截断均须处理。观测重发仅允许相同时间和内容幂等,不允许同时间改写或乱序覆盖。
|
||||
- 限速、源查询1500ms/操作5秒、目标25Gi容量边界、`majority+journal`、权限/供应商隔离、正式操作实时校验不变。维护模式不进入标注、切片、送检、质检和验收准入。
|
||||
|
||||
现有首轮授权仍限1000条读取预留、10条/秒、64KiB/秒、2命令/秒、批次25,不因 CPU 策略更改扩大。该预算包含空读和重复探测,不能完成约76.4万源记录的初始化;完整历史预算、构建器启动及故障对照须合并列明再确认。实现核验与集群实际部署分开记录,新增镜像按正式发布规则处理。
|
||||
|
||||
成品链路验收须区分:统计事实构建成功、特定权限/筛选成品发布完成、Valkey 命中和生产读取切换是不同状态。新筛选仅登记到查询库,由后台计算并发布;不得在缺失成品时前台扫描原库。缓存丢失读取查询库最后完整成品;查询库也不可用时明确暂不可用,不集中回源。初始化完成或 Pod 就绪均不代表已分流;最终降载只能在另行批准切读后以同条件证据确认。
|
||||
|
||||
313
internal/store/query_replica_maintenance_flow_local_test.go
Normal file
313
internal/store/query_replica_maintenance_flow_local_test.go
Normal file
@ -0,0 +1,313 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/queryserving"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// Only an owned loopback synthetic container is restarted. Shared fixtures,
|
||||
// snapshots and company services are never stopped by this recovery proof.
|
||||
func maintenanceRestartTarget(t *testing.T) (*MongoStore, func(string), func() *MongoStore) {
|
||||
t.Helper()
|
||||
if value := os.Getenv("DOCKER_HOST"); value != "" && !strings.HasPrefix(value, "unix://") {
|
||||
t.Fatal("local Docker required")
|
||||
}
|
||||
run := func(args ...string) string {
|
||||
t.Helper()
|
||||
b, e := exec.Command("docker", args...).CombinedOutput()
|
||||
if e != nil {
|
||||
t.Fatalf("owned recovery fixture command failed: %v", e)
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
endpoint := run("context", "inspect", "--format", "{{.Endpoints.docker.Host}}")
|
||||
if !strings.HasPrefix(endpoint, "unix://") {
|
||||
t.Fatal("local Docker socket required")
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address := listener.Addr().String()
|
||||
_ = listener.Close()
|
||||
name := domain.NewID("query-maintenance-recovery")
|
||||
id := run("run", "-d", "--name", name, "--label", "query.fixture="+name, "--cpus", "2", "--memory", "2g", "-p", address+":27129", "mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b", "mongod", "--port", "27129", "--replSet", "maintenance-test-rs", "--bind_ip_all", "--wiredTigerCacheSizeGB", "0.25")
|
||||
t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", id).Run() })
|
||||
var info []struct {
|
||||
NetworkSettings struct {
|
||||
Ports map[string][]struct{ HostIP, HostPort string }
|
||||
}
|
||||
}
|
||||
if e := json.Unmarshal([]byte(run("inspect", id)), &info); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
port := info[0].NetworkSettings.Ports["27129/tcp"][0]
|
||||
if port.HostIP != "127.0.0.1" {
|
||||
t.Fatal("fixture exposed outside loopback")
|
||||
}
|
||||
uri := "mongodb://127.0.0.1:" + port.HostPort + "/?directConnection=true&replicaSet=maintenance-test-rs"
|
||||
connect := func() *mongo.Client {
|
||||
t.Helper()
|
||||
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI(uri).SetServerSelectionTimeout(time.Second))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return c
|
||||
}
|
||||
client := connect()
|
||||
t.Cleanup(func() { _ = client.Disconnect(context.Background()) })
|
||||
ready := func() {
|
||||
t.Helper()
|
||||
probe := connect()
|
||||
defer probe.Disconnect(context.Background())
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
c, stop := context.WithTimeout(t.Context(), time.Second)
|
||||
e := probe.Ping(c, nil)
|
||||
stop()
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Log("owned synthetic mongod recovery log", run("logs", "--tail", "12", id))
|
||||
t.Fatal("owned query replica not ready")
|
||||
}
|
||||
// Initiation is restricted to this newly-created anonymous synthetic database.
|
||||
for attempt := 0; attempt < 60; attempt++ {
|
||||
if _, e := exec.Command("docker", "exec", id, "mongosh", "--quiet", "--port", "27129", "--eval", "rs.initiate({_id:'maintenance-test-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})").Output(); e == nil {
|
||||
break
|
||||
}
|
||||
if attempt == 59 {
|
||||
t.Fatal("owned query initiation failed")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
ready()
|
||||
action := func(op string) {
|
||||
t.Helper()
|
||||
if run("inspect", "--format", "{{index .Config.Labels \"query.fixture\"}}", id) != name {
|
||||
t.Fatal("fixture identity changed")
|
||||
}
|
||||
switch op {
|
||||
case "restart":
|
||||
run("restart", "--time", "5", id)
|
||||
ready()
|
||||
case "disconnect":
|
||||
_ = client.Disconnect(t.Context())
|
||||
default:
|
||||
t.Fatal("invalid fixture action")
|
||||
}
|
||||
}
|
||||
t.Log("recovery fixture: owned2CPU/2Gi, WT0.25Gi; synthetic recovery only, not a production capacity result")
|
||||
dbname := domain.NewID("maintenance_flow_test")
|
||||
reopen := func() *MongoStore {
|
||||
t.Helper()
|
||||
c := connect()
|
||||
t.Cleanup(func() { _ = c.Disconnect(context.Background()) })
|
||||
return NewMongoStore(c.Database(dbname))
|
||||
}
|
||||
return NewMongoStore(client.Database(dbname)), action, reopen
|
||||
}
|
||||
|
||||
func TestQueryReplicaMaintenancePublicationCacheAndRestartLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" || os.Getenv("QUERY_REPLICA_VALKEY_LOCAL_TEST") != "1" {
|
||||
t.Skip("explicit owned source/query/Valkey fixtures required")
|
||||
}
|
||||
var historyReads atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" || e.CommandName == "aggregate" {
|
||||
historyReads.Add(1)
|
||||
}
|
||||
}}
|
||||
source := newQuerySyncSource(t, monitor)
|
||||
target, restart, reopen := maintenanceRestartTarget(t)
|
||||
ctx := t.Context()
|
||||
seedWorkforceStats(t, source)
|
||||
if e := source.UpsertOrganization(ctx, domain.OrganizationRecord{ID: "workforce-test", Status: domain.OrganizationStatusActive}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
query := workforcestats.Query{Mode: domain.RunModeAll}
|
||||
raw, e := source.LoadWorkforceStats(ctx, "workforce-test", query)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
want := workforcestats.Build(raw, query)
|
||||
want.Details = nil
|
||||
want.AsOf = ""
|
||||
q, e := NewQueryReplica(source, target)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
q.BatchSize = 25
|
||||
if e = q.EnsureSourceIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = q.EnsureTargetIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = q.Acquire(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
policy := safetyPolicyForTest()
|
||||
policy.Mode = QueryReplicaMaintenanceInitialization
|
||||
policy.BaselineCPU = 0
|
||||
policy.MaxRecords = 100000
|
||||
until := time.Now().Add(time.Hour)
|
||||
policy.MaintenanceUntil = &until
|
||||
maintenanceHealthLocal(t, q, 100, false)
|
||||
if e = q.ConfigureSafety(ctx, policy, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
finishLocalReplica(t, q)
|
||||
target.queryPeriodic = true
|
||||
target.ConfigureQueryPublishedResults(true)
|
||||
cache, e := queryserving.NewManagedValkey("redis://127.0.0.1:27139", domain.NewID("maintenance-published"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer cache.Close()
|
||||
target.ConfigureQueryServing(cache, nil)
|
||||
cacheReady := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
cache.Put(ctx, "probe", []byte("ready"), time.Second)
|
||||
if value, ok := cache.Get(ctx, "probe"); ok && string(value) == "ready" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(cacheReady) {
|
||||
t.Fatal("owned Valkey unavailable")
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
if e = target.EnsureQueryReadModelIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Register exact server-derived scopes before the background publication pass.
|
||||
if e = target.requestWorkforcePublication(ctx, "workforce-test", query); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
builder := NewQueryReadBuilder(target)
|
||||
defer builder.Close()
|
||||
buildCtx, stopBuild := context.WithCancel(ctx)
|
||||
defer stopBuild()
|
||||
buildDone := make(chan error, 1)
|
||||
go func() { buildDone <- builder.Run(buildCtx) }()
|
||||
await := func(read func() error) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for {
|
||||
err := read()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, ErrQueryReplicaNotReady) || time.Now().After(deadline) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if e = source.ConfigureQueryReplicaReports(target, time.Minute); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
historyReads.Store(0)
|
||||
check := func(s *MongoStore) error {
|
||||
got, e := s.LoadWorkforceReportCached(ctx, "workforce-test", query)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
got.Details = nil
|
||||
got.AsOf = ""
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
return fmt.Errorf("published generation differs from authority")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
await(func() error { return check(source) })
|
||||
hits := cache.Statistics()["hits"]
|
||||
target2 := NewMongoStore(target.db)
|
||||
target2.queryPeriodic = true
|
||||
target2.ConfigureQueryPublishedResults(true)
|
||||
target2.ConfigureQueryServing(cache, nil)
|
||||
source2 := NewMongoStore(source.db)
|
||||
if e = source2.ConfigureQueryReplicaReports(target2, time.Minute); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = check(source2); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if cache.Statistics()["hits"] <= hits {
|
||||
t.Fatal("second reader did not use Valkey")
|
||||
}
|
||||
restricted := query
|
||||
restricted.AllowedProjects = map[string]bool{"unrelated": true}
|
||||
denied, e := source2.LoadWorkforceReportCached(ctx, "workforce-test", restricted)
|
||||
if !errors.Is(e, ErrQueryReplicaNotReady) {
|
||||
t.Fatal("new permission scope reused broad publication", e)
|
||||
}
|
||||
await(func() error {
|
||||
denied, e = source2.LoadWorkforceReportCached(ctx, "workforce-test", restricted)
|
||||
if e == nil && denied.EventCount != 0 {
|
||||
return fmt.Errorf("published cache bypassed project permission")
|
||||
}
|
||||
return e
|
||||
})
|
||||
stopBuild()
|
||||
if e = <-buildDone; e != nil && !errors.Is(e, context.Canceled) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cache.Close()
|
||||
if e = check(source2); e != nil {
|
||||
t.Fatal("cache loss did not retain complete query result", e)
|
||||
}
|
||||
// Stop the target driver before the restart.81 readers must fail closed,
|
||||
// rather than issuing authority history queries during unavailability.
|
||||
restart("disconnect")
|
||||
var wg sync.WaitGroup
|
||||
for n := 0; n < 81; n++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
short, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
_, _ = source2.LoadWorkforceReportCached(short, "workforce-test", query)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if historyReads.Load() != 0 {
|
||||
t.Fatal("query/cache outage scanned source", historyReads.Load())
|
||||
}
|
||||
restart("restart")
|
||||
target3 := reopen()
|
||||
target3.queryPeriodic = true
|
||||
target3.ConfigureQueryPublishedResults(true)
|
||||
target3.ConfigureQueryServing(cache, nil)
|
||||
source3 := NewMongoStore(source.db)
|
||||
if e = source3.ConfigureQueryReplicaReports(target3, time.Minute); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = check(source3); e != nil {
|
||||
t.Fatal("query restart lost complete generation", e)
|
||||
}
|
||||
if historyReads.Load() != 0 {
|
||||
t.Fatal("recovery scanned authority history", historyReads.Load())
|
||||
}
|
||||
t.Log("high CPU -> sync -> complete publication -> Valkey -> reader; cache loss and81 disconnected readers caused0 authority history reads; real mongod restart preserved output")
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"droboticflow/internal/domain"
|
||||
@ -19,24 +20,32 @@ var ErrQueryReplicaPaused = errors.New("shadow synchronization paused; explicit
|
||||
|
||||
// Safety is opt-in for the independent projector only. It never gates authority
|
||||
// business operations. The policy and pause survive process and Pod replacement.
|
||||
const QueryReplicaMaintenanceInitialization = "maintenance-initialization"
|
||||
|
||||
type QueryReplicaSafetyPolicy struct {
|
||||
BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"`
|
||||
CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"`
|
||||
MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"`
|
||||
RunID string `json:"runId" bson:"runId"`
|
||||
MaxRecords int64 `json:"maxRecords" bson:"maxRecords"`
|
||||
RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"`
|
||||
BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"`
|
||||
MaintenanceUntil *time.Time `json:"maintenanceUntil,omitempty" bson:"maintenanceUntil,omitempty"`
|
||||
Mode string `json:"mode,omitempty" bson:"mode,omitempty"`
|
||||
BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"`
|
||||
CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"`
|
||||
MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"`
|
||||
RunID string `json:"runId" bson:"runId"`
|
||||
MaxRecords int64 `json:"maxRecords" bson:"maxRecords"`
|
||||
RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"`
|
||||
BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"`
|
||||
}
|
||||
type QueryReplicaSafetyObservation struct {
|
||||
At time.Time `json:"at" bson:"at"`
|
||||
CPU float64 `json:"cpu" bson:"cpu"`
|
||||
Ready bool `json:"ready" bson:"ready"`
|
||||
WritersCovered bool `json:"writersCovered" bson:"writersCovered"`
|
||||
WorkErrors bool `json:"workErrors" bson:"workErrors"`
|
||||
ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"`
|
||||
TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"`
|
||||
HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"`
|
||||
SafetySignalsVersion int `json:"safetySignalsVersion,omitempty" bson:"safetySignalsVersion"`
|
||||
SourceAvailable bool `json:"sourceAvailable,omitempty" bson:"sourceAvailable"`
|
||||
MemoryPressure bool `json:"memoryPressure,omitempty" bson:"memoryPressure"`
|
||||
PersistenceFailure bool `json:"persistenceFailure,omitempty" bson:"persistenceFailure"`
|
||||
At time.Time `json:"at" bson:"at"`
|
||||
CPU float64 `json:"cpu" bson:"cpu"`
|
||||
Ready bool `json:"ready" bson:"ready"`
|
||||
WritersCovered bool `json:"writersCovered" bson:"writersCovered"`
|
||||
WorkErrors bool `json:"workErrors" bson:"workErrors"`
|
||||
ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"`
|
||||
TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"`
|
||||
HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"`
|
||||
}
|
||||
type queryReplicaSafetyState struct {
|
||||
Owner string `bson:"owner"`
|
||||
@ -60,8 +69,18 @@ func safetyPolicyDigest(p QueryReplicaSafetyPolicy) string {
|
||||
h := sha256.Sum256(b)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
func (p QueryReplicaSafetyPolicy) maintenance() bool {
|
||||
return p.Mode == QueryReplicaMaintenanceInitialization
|
||||
}
|
||||
|
||||
func (p QueryReplicaSafetyPolicy) validate() error {
|
||||
if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || p.BaselineCPU <= 0 || p.BaselineCPU > 90 {
|
||||
if p.Mode != "" && !p.maintenance() {
|
||||
return errors.New("unknown shadow safety mode")
|
||||
}
|
||||
if p.maintenance() && (p.MaxRecords <= 0 || p.MaintenanceUntil == nil || !p.MaintenanceUntil.After(time.Now())) {
|
||||
return errors.New("maintenance initialization requires a finite read budget and an explicit future deadline")
|
||||
}
|
||||
if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || (!p.maintenance() && (p.BaselineCPU <= 0 || p.BaselineCPU > 90)) || (p.maintenance() && (p.BaselineCPU < 0 || p.BaselineCPU > 100)) {
|
||||
return errors.New("invalid shadow safety policy")
|
||||
}
|
||||
return nil
|
||||
@ -70,7 +89,7 @@ func (p QueryReplicaSafetyPolicy) validate() error {
|
||||
// RecordQueryReplicaObservation is for a trusted out-of-process observer. Only
|
||||
// complete, fresh observations are accepted; callers never submit customer data.
|
||||
func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryReplicaSafetyObservation) error {
|
||||
if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 {
|
||||
if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 {
|
||||
return errors.New("invalid shadow observation")
|
||||
}
|
||||
var hello bson.M
|
||||
@ -84,6 +103,16 @@ func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryR
|
||||
c := s.db.Collection(queryReplicaControl, options.Collection().SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
|
||||
// Monotonic CAS: an old observer cannot overwrite newer health evidence.
|
||||
_, e := c.UpdateOne(ctx, bson.M{"_id": "safety-health", "$or": bson.A{bson.M{"at": bson.M{"$lt": o.At}}, bson.M{"at": bson.M{"$exists": false}}}}, bson.M{"$set": o, "$push": bson.M{"recent": bson.M{"$each": bson.A{o}, "$slice": -3}}}, options.Update().SetUpsert(true))
|
||||
if mongo.IsDuplicateKeyError(e) {
|
||||
var old QueryReplicaSafetyObservation
|
||||
if readErr := c.FindOne(ctx, bson.M{"_id": "safety-health", "at": o.At}).Decode(&old); readErr == nil {
|
||||
before, first := bson.Marshal(old)
|
||||
after, second := bson.Marshal(o)
|
||||
if first == nil && second == nil && bytes.Equal(before, after) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return e
|
||||
}
|
||||
func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafetyPolicy, resumeToken string) error {
|
||||
@ -103,7 +132,7 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
|
||||
}
|
||||
if e == nil {
|
||||
if state.Paused {
|
||||
if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || p.BaselineCPU > state.Policy.BaselineCPU || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved {
|
||||
if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || (!p.maintenance() && !state.Policy.maintenance() && p.BaselineCPU > state.Policy.BaselineCPU) || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
} else {
|
||||
@ -140,12 +169,15 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
|
||||
if e = q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
|
||||
return denyStart()
|
||||
}
|
||||
if len(health.Recent) != 3 {
|
||||
if len(health.Recent) == 0 || (!p.maintenance() && len(health.Recent) != 3) {
|
||||
return denyStart()
|
||||
}
|
||||
low, high := 100., 0.
|
||||
for i, o := range health.Recent {
|
||||
if observationFailure(o, o.At) != "" || o.CPU > 90 {
|
||||
if p.maintenance() && i < len(health.Recent)-1 {
|
||||
continue
|
||||
}
|
||||
if policyObservationFailure(o, o.At, p) != "" || (!p.maintenance() && o.CPU > 90) {
|
||||
return denyStart()
|
||||
}
|
||||
if i > 0 && !o.At.After(health.Recent[i-1].At) {
|
||||
@ -154,7 +186,8 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
|
||||
low = math.Min(low, o.CPU)
|
||||
high = math.Max(high, o.CPU)
|
||||
}
|
||||
if high-low > 5 || health.Recent[2].At.Sub(health.Recent[0].At) < 60*time.Second || observationFailure(health.Recent[2], time.Now()) != "" {
|
||||
latest := health.Recent[len(health.Recent)-1]
|
||||
if (!p.maintenance() && (high-low > 5 || latest.At.Sub(health.Recent[0].At) < 60*time.Second)) || policyObservationFailure(latest, time.Now(), p) != "" {
|
||||
return denyStart()
|
||||
}
|
||||
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": false, "reason": "", "resumeToken": "", "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true))
|
||||
@ -171,14 +204,26 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
|
||||
return e
|
||||
}
|
||||
func observationFailure(o QueryReplicaSafetyObservation, now time.Time) string {
|
||||
return policyObservationFailure(o, now, QueryReplicaSafetyPolicy{})
|
||||
}
|
||||
|
||||
func policyObservationFailure(o QueryReplicaSafetyObservation, now time.Time, p QueryReplicaSafetyPolicy) string {
|
||||
switch {
|
||||
case o.At.IsZero() || now.Sub(o.At) > 45*time.Second || o.At.After(now.Add(time.Second)):
|
||||
return "health_missing_or_stale"
|
||||
case !o.Ready || !o.WritersCovered || o.WorkErrors:
|
||||
return "production_health"
|
||||
case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1:
|
||||
case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || (!p.maintenance() && o.CPU > 100) || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1:
|
||||
return "invalid_health"
|
||||
case o.CPU >= 95:
|
||||
case p.maintenance() && (p.MaintenanceUntil == nil || !now.Before(*p.MaintenanceUntil)):
|
||||
return "maintenance_window_closed"
|
||||
case p.maintenance() && (o.SafetySignalsVersion != 1 || !o.SourceAvailable):
|
||||
return "maintenance_health_incomplete"
|
||||
case o.MemoryPressure:
|
||||
return "memory_pressure"
|
||||
case o.PersistenceFailure:
|
||||
return "persistence_failure"
|
||||
case !p.maintenance() && o.CPU >= 95:
|
||||
return "source_cpu_absolute"
|
||||
case o.ProtectedIOFull > 5:
|
||||
return "protected_io"
|
||||
@ -214,9 +259,9 @@ func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, err
|
||||
if e != nil && e != mongo.ErrNoDocuments {
|
||||
return e
|
||||
}
|
||||
reason := observationFailure(o, time.Now())
|
||||
reason := policyObservationFailure(o, time.Now(), s.Policy)
|
||||
if reason == "" && !o.At.Equal(s.LastObservation) {
|
||||
if o.CPU > s.Policy.BaselineCPU+3 {
|
||||
if !s.Policy.maintenance() && o.CPU > s.Policy.BaselineCPU+3 {
|
||||
s.High++
|
||||
} else {
|
||||
s.High = 0
|
||||
@ -227,7 +272,7 @@ func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, err
|
||||
s.HighIO = 0
|
||||
}
|
||||
s.LastObservation = o.At
|
||||
if s.High >= 2 {
|
||||
if !s.Policy.maintenance() && s.High >= 2 {
|
||||
reason = "source_cpu_relative"
|
||||
}
|
||||
if s.HighIO >= 2 {
|
||||
@ -319,7 +364,7 @@ func (s *MongoStore) CheckQueryReplicaSafety(ctx context.Context, runID string)
|
||||
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
|
||||
return e
|
||||
}
|
||||
if observationFailure(health, time.Now()) != "" {
|
||||
if policyObservationFailure(health, time.Now(), state.Policy) != "" {
|
||||
return ErrQueryReplicaPaused
|
||||
}
|
||||
return nil
|
||||
|
||||
@ -425,3 +425,100 @@ func TestQueryReplicaSafetyBacklogPriorityAndRoundRobinLocal(t *testing.T) {
|
||||
t.Fatal("round-robin checkpoint not persisted")
|
||||
}
|
||||
}
|
||||
|
||||
func maintenanceHealthLocal(t *testing.T, q *QueryReplica, cpu float64, persistence bool) {
|
||||
t.Helper()
|
||||
o := QueryReplicaSafetyObservation{At: time.Now().UTC(), CPU: cpu, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1, SafetySignalsVersion: 1, SourceAvailable: true, PersistenceFailure: persistence}
|
||||
raw, e := bson.Marshal(o)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var fields bson.M
|
||||
if e = bson.Unmarshal(raw, &fields); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
fields["recent"] = []QueryReplicaSafetyObservation{o}
|
||||
if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaMaintenanceHighCPUSyncPreservesDataLocal(t *testing.T) {
|
||||
source, target, q := newLocalReplica(t)
|
||||
q.BatchSize = 2
|
||||
p := safetyPolicyForTest()
|
||||
p.Mode = QueryReplicaMaintenanceInitialization
|
||||
until := time.Now().Add(time.Hour)
|
||||
p.MaintenanceUntil = &until
|
||||
p.BaselineCPU = 0
|
||||
p.MaxRecords = 10
|
||||
maintenanceHealthLocal(t, q, 99, false)
|
||||
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
|
||||
t.Fatal("waited for low CPU", e)
|
||||
}
|
||||
if _, e := source.db.Collection("humanItems").InsertOne(t.Context(), bson.M{"_id": "legacy", "id": "legacy", "organizationId": "org", "version": 7, "status": "claimed", "subject": bson.M{"payload": bson.M{"annotationDraft": "keep rework"}}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, cpu := range []float64{100, 110, 96} {
|
||||
maintenanceHealthLocal(t, q, cpu, false)
|
||||
if _, _, e := q.BootstrapStep(t.Context(), "humanItems"); e != nil {
|
||||
t.Fatal("high CPU paused copy", e)
|
||||
}
|
||||
}
|
||||
var original bson.M
|
||||
if e := source.db.Collection("humanItems").FindOne(t.Context(), bson.M{"_id": "legacy"}).Decode(&original); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if supplierNumber(original["version"]) != 7 || original["status"] != "claimed" || deletionDoc(deletionDoc(original["subject"])["payload"])["annotationDraft"] != "keep rework" {
|
||||
t.Fatal("source annotation/version changed")
|
||||
}
|
||||
var copied bson.M
|
||||
if e := target.db.Collection("humanItems").FindOne(t.Context(), bson.M{}).Decode(&copied); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var state queryReplicaSafetyState
|
||||
if e := q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Paused || state.High != 0 {
|
||||
t.Fatal("CPU pause survived maintenance switch", state, e)
|
||||
}
|
||||
if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
maintenanceHealthLocal(t, q, 100, true)
|
||||
if _, e := q.sourcePermit(t.Context(), 1); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("persistence failure did not stop", e)
|
||||
}
|
||||
var after queryReplicaSafetyState
|
||||
_ = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&after)
|
||||
if !after.Paused || after.Reason != "persistence_failure" {
|
||||
t.Fatal("failure not durably paused", after)
|
||||
}
|
||||
maintenanceHealthLocal(t, q, 99, false)
|
||||
if _, e := q.sourcePermit(t.Context(), 1); !errors.Is(e, ErrQueryReplicaPaused) {
|
||||
t.Fatal("healthy retry bypassed durable pause")
|
||||
}
|
||||
}
|
||||
func TestQueryReplicaHealthReplayAndOutOfOrderLocal(t *testing.T) {
|
||||
_, target, q := newLocalReplica(t)
|
||||
o := QueryReplicaSafetyObservation{At: time.Now().UTC().Add(-time.Second), CPU: 99, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1, SafetySignalsVersion: 1, SourceAvailable: true}
|
||||
if e := target.RecordQueryReplicaObservation(t.Context(), o); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := target.RecordQueryReplicaObservation(t.Context(), o); e != nil {
|
||||
t.Fatal("ack retry not idempotent", e)
|
||||
}
|
||||
conflict := o
|
||||
conflict.MemoryPressure = true
|
||||
if target.RecordQueryReplicaObservation(t.Context(), conflict) == nil {
|
||||
t.Fatal("same sample timestamp overwrote health")
|
||||
}
|
||||
old := o
|
||||
old.At = old.At.Add(-time.Second)
|
||||
if target.RecordQueryReplicaObservation(t.Context(), old) == nil {
|
||||
t.Fatal("out-of-order health overwrote newer observation")
|
||||
}
|
||||
var state struct {
|
||||
Recent []QueryReplicaSafetyObservation `bson:"recent"`
|
||||
}
|
||||
if e := q.control().FindOne(t.Context(), bson.M{"_id": "safety-health"}).Decode(&state); e != nil || len(state.Recent) != 1 || state.Recent[0].MemoryPressure {
|
||||
t.Fatal("replay duplicated sample or changed result", e, state)
|
||||
}
|
||||
}
|
||||
|
||||
@ -144,3 +144,45 @@ func TestQueryReplicaTransportByteBudgetSharedAcrossConnections(t *testing.T) {
|
||||
t.Fatal("aggregate byte rate exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryReplicaMaintenancePolicyKeepsHardSafety(t *testing.T) {
|
||||
until := time.Now().Add(time.Hour)
|
||||
p := QueryReplicaSafetyPolicy{MaintenanceUntil: &until, Mode: QueryReplicaMaintenanceInitialization, BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "bounded-maintenance", MaxRecords: 1000, RecordsPerSecond: 10}
|
||||
if e := p.validate(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
unlimited := p
|
||||
unlimited.MaxRecords = 0
|
||||
if unlimited.validate() == nil {
|
||||
t.Fatal("maintenance accepted an unlimited budget")
|
||||
}
|
||||
unknown := p
|
||||
unknown.Mode = "disabled"
|
||||
if unknown.validate() == nil {
|
||||
t.Fatal("unknown policy silently disabled protections")
|
||||
}
|
||||
now := time.Now()
|
||||
healthy := QueryReplicaSafetyObservation{At: now, CPU: 100, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1, SafetySignalsVersion: 1, SourceAvailable: true}
|
||||
for _, cpu := range []float64{0, 91, 99, 100, 110} {
|
||||
o := healthy
|
||||
o.CPU = cpu
|
||||
if reason := policyObservationFailure(o, now, p); reason != "" {
|
||||
t.Fatal("CPU used as maintenance gate", cpu, reason)
|
||||
}
|
||||
}
|
||||
for _, change := range []func(*QueryReplicaSafetyObservation){func(o *QueryReplicaSafetyObservation) { o.MemoryPressure = true }, func(o *QueryReplicaSafetyObservation) { o.PersistenceFailure = true }, func(o *QueryReplicaSafetyObservation) { o.SourceAvailable = false }, func(o *QueryReplicaSafetyObservation) { o.SafetySignalsVersion = 0 }, func(o *QueryReplicaSafetyObservation) { o.WorkErrors = true }, func(o *QueryReplicaSafetyObservation) { o.Ready = false }, func(o *QueryReplicaSafetyObservation) { o.At = now.Add(-46 * time.Second) }, func(o *QueryReplicaSafetyObservation) { o.CPU = math.NaN() }, func(o *QueryReplicaSafetyObservation) { o.TargetUsedFraction = .7 }} {
|
||||
o := healthy
|
||||
change(&o)
|
||||
if policyObservationFailure(o, now, p) == "" {
|
||||
t.Fatal("hard safety disabled")
|
||||
}
|
||||
}
|
||||
expired := healthy
|
||||
expired.At = until.Add(time.Second)
|
||||
if reason := policyObservationFailure(expired, expired.At, p); reason != "maintenance_window_closed" {
|
||||
t.Fatal("expired maintenance remained active")
|
||||
}
|
||||
if observationFailure(healthy, now) == "" {
|
||||
t.Fatal("legacy strict CPU protection changed")
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,6 +13,7 @@ import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
import socket
|
||||
@ -21,6 +22,7 @@ import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
|
||||
|
||||
|
||||
def check_bindings(b):
|
||||
@ -28,7 +30,7 @@ def check_bindings(b):
|
||||
raise ValueError('invalid namespace or writer scope')
|
||||
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
|
||||
raise ValueError('stop scope must contain only shadow controllers')
|
||||
if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90:
|
||||
if b['sourceDatabase'] == b['targetDatabase'] or (not b.get('maintenanceInitialization') and not 0 < b['baselineCPU'] <= 90):
|
||||
raise ValueError('invalid source identity or baseline')
|
||||
|
||||
|
||||
@ -71,7 +73,7 @@ class Observer:
|
||||
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
|
||||
text=True, timeout=timeout)
|
||||
if p.returncode:
|
||||
raise RuntimeError('bounded cluster operation unavailable')
|
||||
raise ProbeUnavailable('bounded cluster operation unavailable')
|
||||
return p.stdout
|
||||
|
||||
def get(self, ns, kind, name):
|
||||
@ -109,7 +111,7 @@ class Observer:
|
||||
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
|
||||
text=True, timeout=10)
|
||||
if p.returncode:
|
||||
raise RuntimeError('bounded observation command unavailable')
|
||||
raise ProbeUnavailable('bounded observation command unavailable')
|
||||
return p.stdout
|
||||
|
||||
@contextlib.contextmanager
|
||||
@ -203,10 +205,18 @@ class Observer:
|
||||
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
|
||||
return json.loads(self.run_binary(['--sample-source'], env))
|
||||
|
||||
def observation(self, cpu, sampled_at):
|
||||
def readiness(self):
|
||||
try:
|
||||
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
|
||||
return response.status == 200 and json.load(response).get('ready') is True
|
||||
except urllib.error.HTTPError:
|
||||
return False
|
||||
except (OSError, ValueError) as error:
|
||||
raise ProbeUnavailable('readiness transport unavailable') from None
|
||||
|
||||
def observation(self, cpu, sampled_at, source=None):
|
||||
self.verify()
|
||||
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
|
||||
ready = response.status == 200 and json.load(response).get('ready') is True
|
||||
ready = retry_probe(self.readiness, 'readiness')
|
||||
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
|
||||
if pod['metadata']['uid'] != self.b['mongoPodUID']:
|
||||
raise ValueError('query Mongo rebuilt; recovery verification required')
|
||||
@ -229,17 +239,37 @@ class Observer:
|
||||
if p['metadata'].get('deletionTimestamp'): continue
|
||||
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
|
||||
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
|
||||
return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
|
||||
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
|
||||
result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
|
||||
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
|
||||
if self.b.get('maintenanceInitialization'):
|
||||
if not source or source.get('memorySupported') is not True or type(source.get('residentMiB')) not in (int,float) or not math.isfinite(source['residentMiB']) or source['residentMiB'] <= 0 or type(self.b.get('sourceMemoryLimitMiB')) not in (int,float) or not math.isfinite(self.b['sourceMemoryLimitMiB']) or self.b['sourceMemoryLimitMiB'] <= 0:
|
||||
raise ProbeUnavailable('source memory evidence incomplete')
|
||||
group='/hostcg/kubepods.slice/kubepods-pod'+pod['metadata']['uid'].replace('-','_')+'.slice/'
|
||||
used_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.current')
|
||||
limit_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.max')
|
||||
events=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.events')
|
||||
memory=parse_memory(used_mem,limit_mem,events)
|
||||
node=json.loads(self.call('get','node','10.189.16.4','-o','json'))
|
||||
pressure = [c['status'] for c in node['status']['conditions'] if c['type']=='MemoryPressure']
|
||||
if len(pressure)!=1 or pressure[0] not in ('True','False'): raise ProbeUnavailable('node memory evidence incomplete')
|
||||
node_pressure = pressure[0]=='True'
|
||||
mongo_logs=self.call('-n',ns,'logs','query-mongo-0','-c','mongo','--since=45s','--tail=2000')
|
||||
if len(mongo_logs.splitlines())>=2000: raise ProbeUnavailable('persistence log window truncated')
|
||||
result.update(safetySignalsVersion=1,sourceAvailable=True,
|
||||
memoryPressure=memory['pressure'] or node_pressure or source['residentMiB']>=.98*self.b['sourceMemoryLimitMiB'],
|
||||
persistenceFailure=persistence_failed(mongo_logs))
|
||||
return result
|
||||
|
||||
def publish(self, observation):
|
||||
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
|
||||
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
|
||||
|
||||
def unsafe(self, o):
|
||||
self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0
|
||||
self.high = self.high + 1 if not self.b.get('maintenanceInitialization') and o['cpu'] > self.b['baselineCPU'] + 3 else 0
|
||||
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
|
||||
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2
|
||||
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or (not self.b.get('maintenanceInitialization') and (o['cpu'] >= 95 or self.high >= 2))
|
||||
or (self.b.get('maintenanceInitialization') and (not self.b.get('maintenanceUntil') or dt.datetime.now(dt.timezone.utc)>=dt.datetime.fromisoformat(self.b['maintenanceUntil'].replace('Z','+00:00'))))
|
||||
or o.get('memoryPressure',False) or o.get('persistenceFailure',False) or (self.b.get('maintenanceInitialization') and (o.get('safetySignalsVersion')!=1 or not o.get('sourceAvailable')))
|
||||
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
|
||||
|
||||
def stop_shadow(self):
|
||||
@ -253,7 +283,11 @@ class Observer:
|
||||
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
|
||||
|
||||
def run(self):
|
||||
try: prior = self.source_sample()
|
||||
try:
|
||||
self.verify()
|
||||
pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items']
|
||||
check_clocks(self.call,self.b['sourceNamespace'],pods)
|
||||
prior = retry_probe(self.source_sample,'initial-source')
|
||||
except Exception:
|
||||
if self.args.stop_on_failure: self.stop_shadow()
|
||||
raise RuntimeError('initial source observation unavailable') from None
|
||||
@ -261,15 +295,14 @@ class Observer:
|
||||
try:
|
||||
for _ in range(3):
|
||||
time.sleep(10)
|
||||
with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response:
|
||||
if response.status!=200 or json.load(response).get('ready') is not True:
|
||||
raise ValueError('production readiness unavailable')
|
||||
current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current
|
||||
if not retry_probe(self.readiness, 'readiness'):
|
||||
raise ValueError('production readiness unavailable')
|
||||
current = retry_probe(self.source_sample,'source-sample'); cpu = cpu_percent(prior, current); prior = current
|
||||
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
|
||||
if not 0<=age<=45: raise ValueError('source observation stale')
|
||||
o = self.observation(cpu,sampled_at); self.publish(o)
|
||||
o = retry_probe(lambda: self.observation(cpu,sampled_at,current),'health-observation'); retry_probe(lambda: self.publish(o),'health-publish')
|
||||
unsafe=self.unsafe(o)
|
||||
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True)
|
||||
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe,'cpuObservedOnly':bool(self.b.get('maintenanceInitialization'))}),flush=True)
|
||||
if unsafe:
|
||||
if self.args.stop_on_failure: self.stop_shadow()
|
||||
return
|
||||
|
||||
51
scripts/local/query_shadow_health.py
Normal file
51
scripts/local/query_shadow_health.py
Normal file
@ -0,0 +1,51 @@
|
||||
"""Bounded, diagnosed read-only probes. Never retries returned unhealthy states."""
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
class ProbeUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
def emit(value):
|
||||
print(json.dumps(value, sort_keys=True), flush=True)
|
||||
|
||||
def retry_probe(probe, phase, attempts=3, delay=.5, diagnostic=emit, sleep=time.sleep):
|
||||
if not 1 <= attempts <= 3 or not 0 <= delay <= 1: raise ValueError('invalid retry budget')
|
||||
for index in range(attempts):
|
||||
try: return probe()
|
||||
except (ProbeUnavailable, subprocess.TimeoutExpired, OSError) as error:
|
||||
diagnostic(dict(event='probe_retry', phase=phase, attempt=index+1, maxAttempts=attempts, failureType=type(error).__name__))
|
||||
if index+1 == attempts: raise ProbeUnavailable('bounded probe exhausted') from None
|
||||
sleep(delay*(index+1))
|
||||
|
||||
def check_clocks(call, namespace, pods, container='api', attempts=3, diagnostic=emit,
|
||||
utcnow=lambda: dt.datetime.now(dt.timezone.utc), monotonic=time.monotonic, sleep=time.sleep):
|
||||
results=[]
|
||||
for pod in pods:
|
||||
def measure():
|
||||
start=utcnow(); tick=monotonic()
|
||||
try:
|
||||
text=call('-n', namespace, 'exec', pod['metadata']['name'], '-c', container, '--', 'date', '-u', '+%s.%N', timeout=5)
|
||||
remote=float(text.strip())
|
||||
except (ValueError, RuntimeError, subprocess.TimeoutExpired, OSError) as error:
|
||||
diagnostic(dict(event='clock_probe', node=pod['spec']['nodeName'], podUID=pod['metadata']['uid'], available=False, failureType=type(error).__name__))
|
||||
raise ProbeUnavailable('clock transport unavailable') from None
|
||||
end=utcnow();rtt=monotonic()-tick;offset=remote-(start+(end-start)/2).timestamp()
|
||||
value=dict(event='clock_probe',node=pod['spec']['nodeName'],podUID=pod['metadata']['uid'],available=True,offsetSeconds=offset,rttSeconds=rtt,offsetPassed=math.isfinite(offset) and abs(offset)<=1,rttPassed=0<=rtt<=2)
|
||||
diagnostic(value)
|
||||
if not value['offsetPassed'] or not value['rttPassed']: raise ProbeUnavailable('clock gate failed')
|
||||
return value
|
||||
results.append(retry_probe(measure,'clock',attempts=attempts,diagnostic=diagnostic,sleep=sleep))
|
||||
return results
|
||||
|
||||
def parse_memory(current, maximum, events):
|
||||
used=int(current.strip());limit=int(maximum.strip())
|
||||
counters={line.split()[0]:int(line.split()[1]) for line in events.splitlines() if len(line.split())==2}
|
||||
if used<0 or limit<=0 or not {'oom','oom_kill'} <= counters.keys(): raise ValueError('memory evidence incomplete')
|
||||
return dict(usedBytes=used,limitBytes=limit,oom=counters['oom'],oomKill=counters['oom_kill'],pressure=used>=.98*limit or counters['oom']>0 or counters['oom_kill']>0)
|
||||
|
||||
def persistence_failed(text):
|
||||
lower=text.lower()
|
||||
return any(token in lower for token in ('wt_panic','failed to fsync','input/output error','no space left on device','journal write failed','fatal assertion'))
|
||||
42
scripts/local/test_query_shadow_health.py
Normal file
42
scripts/local/test_query_shadow_health.py
Normal file
@ -0,0 +1,42 @@
|
||||
import datetime as dt
|
||||
import unittest
|
||||
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
|
||||
|
||||
class HealthProbeTests(unittest.TestCase):
|
||||
def test_bounded_retry_has_diagnostics_and_never_retries_bad_result(self):
|
||||
events=[];calls=[]
|
||||
def work():
|
||||
calls.append(1)
|
||||
if len(calls)<3: raise ProbeUnavailable('fixed reason')
|
||||
return {'healthy':False}
|
||||
self.assertEqual(retry_probe(work,'test',diagnostic=events.append,sleep=lambda _:None),{'healthy':False})
|
||||
self.assertEqual(len(calls),3);self.assertEqual([e['attempt'] for e in events],[1,2])
|
||||
with self.assertRaises(ProbeUnavailable):retry_probe(lambda:(_ for _ in ()).throw(ProbeUnavailable()),'test',diagnostic=events.append,sleep=lambda _:None)
|
||||
with self.assertRaises(ValueError):retry_probe(lambda:None,'test',attempts=4)
|
||||
def test_clock_offset_and_rtt_are_distinguished_and_bounded(self):
|
||||
now=dt.datetime.now(dt.timezone.utc);events=[]
|
||||
pods=[dict(metadata=dict(name='api',uid='uid'),spec=dict(nodeName='node'))]
|
||||
values=iter([str(now.timestamp()+2),str(now.timestamp())]);ticks=iter([0,.1,1,3.1,4,4.1])
|
||||
# first offset2s fails; second RTT2.1 fails; final sample passes.
|
||||
def call(*args,**kwargs):
|
||||
n=sum(e.get('event')=='clock_probe' for e in events)
|
||||
return str(now.timestamp()+2) if n==0 else str(now.timestamp())
|
||||
rows=check_clocks(call,'ns',pods,diagnostic=events.append,utcnow=lambda:now,monotonic=lambda:next(ticks),sleep=lambda _:None)
|
||||
samples=[e for e in events if e.get('event')=='clock_probe']
|
||||
self.assertFalse(samples[0]['offsetPassed']);self.assertTrue(samples[0]['rttPassed'])
|
||||
self.assertTrue(samples[1]['offsetPassed']);self.assertFalse(samples[1]['rttPassed'])
|
||||
self.assertTrue(rows[0]['offsetPassed']);self.assertEqual(len(samples),3)
|
||||
def test_clock_transport_failure_records_reason(self):
|
||||
events=[];pods=[dict(metadata=dict(name='api',uid='uid'),spec=dict(nodeName='node'))]
|
||||
with self.assertRaises(ProbeUnavailable):check_clocks(lambda *a,**kw:(_ for _ in ()).throw(RuntimeError()),'ns',pods,diagnostic=events.append,sleep=lambda _:None)
|
||||
self.assertEqual(sum(e.get('event')=='clock_probe' for e in events),3)
|
||||
self.assertTrue(all(e.get('available') is False for e in events if e.get('event')=='clock_probe'))
|
||||
def test_memory_and_durability_errors_remain_hard_signals(self):
|
||||
self.assertFalse(parse_memory('100','1000','oom 0\noom_kill 0')['pressure'])
|
||||
self.assertTrue(parse_memory('990','1000','oom 0\noom_kill 0')['pressure'])
|
||||
self.assertTrue(parse_memory('100','1000','oom 0\noom_kill 1')['pressure'])
|
||||
with self.assertRaises(ValueError):parse_memory('100','max','')
|
||||
self.assertTrue(persistence_failed('WiredTiger WT_PANIC'));self.assertTrue(persistence_failed('journal write failed'))
|
||||
self.assertFalse(persistence_failed('checkpoint completed'))
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@ -66,6 +66,14 @@ class ObserverTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64)
|
||||
feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed))
|
||||
with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64)
|
||||
def test_maintenance_cpu_is_observed_but_resource_failures_stop(self):
|
||||
b=self.bindings();b['maintenanceInitialization']=True;b['baselineCPU']=0;b['maintenanceUntil']=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat()
|
||||
m.check_bindings(b);o=m.Observer(SimpleNamespace(kubeconfig='unused'),b)
|
||||
good=dict(cpu=100,ready=True,writersCovered=True,workErrors=False,protectedIOFull=0,targetUsedFraction=.1,hostFreeBytes=120<<30,safetySignalsVersion=1,sourceAvailable=True,memoryPressure=False,persistenceFailure=False)
|
||||
for cpu in [95,100,110]:
|
||||
good['cpu']=cpu;self.assertFalse(o.unsafe(good))
|
||||
for field,value in [('memoryPressure',True),('persistenceFailure',True),('sourceAvailable',False),('workErrors',True),('safetySignalsVersion',0)]:
|
||||
bad=dict(good);bad[field]=value;self.assertTrue(o.unsafe(bad))
|
||||
def test_unknown_io_is_not_zero(self):
|
||||
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
|
||||
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)
|
||||
|
||||
@ -18,6 +18,7 @@ COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
|
||||
CASES = ('cmd/query-replica/main_local_test.go',
|
||||
'internal/store/query_replica_source_auth_local_test.go',
|
||||
'internal/store/query_replica_safety_local_test.go',
|
||||
'internal/store/query_replica_maintenance_flow_local_test.go',
|
||||
'internal/store/query_replica_safety_test.go',
|
||||
'internal/store/query_replica_local_test.go',
|
||||
'internal/store/query_sync_local_test.go',
|
||||
|
||||
@ -1018,7 +1018,7 @@
|
||||
{
|
||||
"name": "shadow-work-health",
|
||||
"paths": [
|
||||
"^scripts/local/(query-shadow-(observer|work-health)|test_query_shadow_(observer|work_health))\\.py$"
|
||||
"^scripts/local/(query-shadow-(observer|work-health)|query_shadow_health|test_query_shadow_(observer|work_health|health))\\.py$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user