🛠️ Allow guarded shadow initialization during maintenance windows

This commit is contained in:
d-robotics 2026-11-14 18:30:51 -08:00
parent ae9e33a3bc
commit ad1d91a9fc
12 changed files with 718 additions and 48 deletions

View File

@ -35,6 +35,8 @@ func main() {
rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing")
bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget")
commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead")
maintenanceUntil := flag.String("maintenance-until", "", "explicit RFC3339 deadline for the initialization-only CPU observation policy")
maintenance := flag.Bool("maintenance-initialization", false, "explicit finite initialization run: CPU is observed, not an admission/stop gate; resource and durability guards remain")
baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume")
resume := flag.String("resume-pause", "", "exact current pause nonce after operator review")
initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes")
@ -42,6 +44,17 @@ func main() {
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
flag.Parse()
var maintenanceDeadline *time.Time
if *maintenance {
value, err := time.Parse(time.RFC3339, *maintenanceUntil)
if err != nil || !value.After(time.Now()) {
log.Fatal("explicit future maintenance deadline required")
}
maintenanceDeadline = &value
}
if *maintenance && (*safetyRun == "" || *maxRecords <= 0) {
log.Fatal("maintenance initialization requires a guarded finite budget")
}
if *sampleSource {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@ -55,7 +68,11 @@ func main() {
defer c.Disconnect(context.Background())
var info struct {
Uptime int64 `bson:"uptimeMillis"`
Extra struct {
Memory struct {
Resident int64 `bson:"resident"`
Supported bool `bson:"supported"`
} `bson:"mem"`
Extra struct {
User int64 `bson:"user_time_us"`
System int64 `bson:"system_time_us"`
} `bson:"extra_info"`
@ -72,7 +89,7 @@ func main() {
log.Fatal("source identity unavailable")
}
identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process)))
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano)})
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano), "residentMiB": info.Memory.Resident, "memorySupported": info.Memory.Supported})
return
}
if *observation != "" || *status {
@ -204,7 +221,11 @@ func main() {
return
}
if *safetyRun != "" {
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
mode := ""
if *maintenance {
mode = store.QueryReplicaMaintenanceInitialization
}
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{Mode: mode, MaintenanceUntil: maintenanceDeadline, BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
log.Print("shadow safety paused; inspect durable state and explicitly resume")
<-ctx.Done()
return

View File

@ -229,3 +229,20 @@ Mongo 角色不能限制更新到某个字段:源账号按22个源集合授予
8. 回退:先停两个影子控制器并确认 Pod 退出,再停观察器;保留目标断点、暂停记录、最后完整结果和源标记。不删除源记录/标记,不清空查询库,不恢复业务旧快照。再次启动先读 `query-replica --safety-status`,重新确认健康、绑定和当前暂停令牌。生产入口从未切换,无需改客户作业配置。
此阶段只能证明新增同步负载受控及影子结果正确。生产入口未切换时不能宣称原 Mongo 总 CPU 已下降;降载验收仍需后续经授权切读后的同条件对照。
## 维护窗口初始化策略(2026-11-14,明确授权的临时覆盖)
本节仅覆盖上节影子初始化的 CPU 启停规则。用户已明确说明维护窗口无客户作业,并允许原 Mongo 初始化期间持续高 CPU;不再等待低 CPU 窗口。历史方案及批准摘要保留,不修改既有发布记录。
投影器必须同时显式传入 `--maintenance-initialization`、未来的 `--maintenance-until=<RFC3339>`、原 `--safety-run` 和有限 `--max-source-records`。策略以 `maintenance-initialization` 持久化;默认严格模式不变,既有暂停必须使用当前令牌显式恢复,累计预算不会清零。
- CPU 持续采集,仅观察:取消该模式的启动 CPU≤90%、波动≤5个百分点/三样本等待,以及运行 CPU≥95%和相对基线增长暂停。单份完整、新鲜的健康观测即可准入。CPU 数据缺失或非有限数仍不能伪装健康。
- 停止条件保持:源数据库不可用、客户作业错误或延迟恶化、权限/身份/版本变化、健康过期、实际内存压力或 OOM、磁盘容量不足、持久化失败、受保护业务 IO 异常。维护期限到达或读取预算耗尽也持久暂停。停止仅限绑定的两个影子控制器,保留断点和最后完整结果。
- 观察器记录时钟偏移与往返耗时的独立结果、Pod UID 和每次失败类型。传输/探测瞬时失败最多3次,间隔0.5、1秒;明确不健康返回、权限和身份变化直接停止,不能通过重试掩盖。不得记录连接凭证和客户正文。
- 内存证据绑定源 `hostInfo.system.memLimitMB`、`serverStatus.mem.resident` 及目标 cgroup 的实际上限/OOM 计数;节点 MemoryPressure 缺失不是健康。目标日志的持久化错误和日志截断均须处理。观测重发仅允许相同时间和内容幂等,不允许同时间改写或乱序覆盖。
- 限速、源查询1500ms/操作5秒、目标25Gi容量边界、`majority+journal`、权限/供应商隔离、正式操作实时校验不变。维护模式不进入标注、切片、送检、质检和验收准入。
现有首轮授权仍限1000条读取预留、10条/秒、64KiB/秒、2命令/秒、批次25,不因 CPU 策略更改扩大。该预算包含空读和重复探测,不能完成约76.4万源记录的初始化;完整历史预算、构建器启动及故障对照须合并列明再确认。实现核验与集群实际部署分开记录,新增镜像按正式发布规则处理。
成品链路验收须区分:统计事实构建成功、特定权限/筛选成品发布完成、Valkey 命中和生产读取切换是不同状态。新筛选仅登记到查询库,由后台计算并发布;不得在缺失成品时前台扫描原库。缓存丢失读取查询库最后完整成品;查询库也不可用时明确暂不可用,不集中回源。初始化完成或 Pod 就绪均不代表已分流;最终降载只能在另行批准切读后以同条件证据确认。

View File

@ -0,0 +1,313 @@
package store
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"os/exec"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/queryserving"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// Only an owned loopback synthetic container is restarted. Shared fixtures,
// snapshots and company services are never stopped by this recovery proof.
func maintenanceRestartTarget(t *testing.T) (*MongoStore, func(string), func() *MongoStore) {
t.Helper()
if value := os.Getenv("DOCKER_HOST"); value != "" && !strings.HasPrefix(value, "unix://") {
t.Fatal("local Docker required")
}
run := func(args ...string) string {
t.Helper()
b, e := exec.Command("docker", args...).CombinedOutput()
if e != nil {
t.Fatalf("owned recovery fixture command failed: %v", e)
}
return strings.TrimSpace(string(b))
}
endpoint := run("context", "inspect", "--format", "{{.Endpoints.docker.Host}}")
if !strings.HasPrefix(endpoint, "unix://") {
t.Fatal("local Docker socket required")
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
address := listener.Addr().String()
_ = listener.Close()
name := domain.NewID("query-maintenance-recovery")
id := run("run", "-d", "--name", name, "--label", "query.fixture="+name, "--cpus", "2", "--memory", "2g", "-p", address+":27129", "mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b", "mongod", "--port", "27129", "--replSet", "maintenance-test-rs", "--bind_ip_all", "--wiredTigerCacheSizeGB", "0.25")
t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", id).Run() })
var info []struct {
NetworkSettings struct {
Ports map[string][]struct{ HostIP, HostPort string }
}
}
if e := json.Unmarshal([]byte(run("inspect", id)), &info); e != nil {
t.Fatal(e)
}
port := info[0].NetworkSettings.Ports["27129/tcp"][0]
if port.HostIP != "127.0.0.1" {
t.Fatal("fixture exposed outside loopback")
}
uri := "mongodb://127.0.0.1:" + port.HostPort + "/?directConnection=true&replicaSet=maintenance-test-rs"
connect := func() *mongo.Client {
t.Helper()
c, e := mongo.Connect(t.Context(), options.Client().ApplyURI(uri).SetServerSelectionTimeout(time.Second))
if e != nil {
t.Fatal(e)
}
return c
}
client := connect()
t.Cleanup(func() { _ = client.Disconnect(context.Background()) })
ready := func() {
t.Helper()
probe := connect()
defer probe.Disconnect(context.Background())
deadline := time.Now().Add(30 * time.Second)
for time.Now().Before(deadline) {
c, stop := context.WithTimeout(t.Context(), time.Second)
e := probe.Ping(c, nil)
stop()
if e == nil {
return
}
time.Sleep(100 * time.Millisecond)
}
t.Log("owned synthetic mongod recovery log", run("logs", "--tail", "12", id))
t.Fatal("owned query replica not ready")
}
// Initiation is restricted to this newly-created anonymous synthetic database.
for attempt := 0; attempt < 60; attempt++ {
if _, e := exec.Command("docker", "exec", id, "mongosh", "--quiet", "--port", "27129", "--eval", "rs.initiate({_id:'maintenance-test-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})").Output(); e == nil {
break
}
if attempt == 59 {
t.Fatal("owned query initiation failed")
}
time.Sleep(100 * time.Millisecond)
}
ready()
action := func(op string) {
t.Helper()
if run("inspect", "--format", "{{index .Config.Labels \"query.fixture\"}}", id) != name {
t.Fatal("fixture identity changed")
}
switch op {
case "restart":
run("restart", "--time", "5", id)
ready()
case "disconnect":
_ = client.Disconnect(t.Context())
default:
t.Fatal("invalid fixture action")
}
}
t.Log("recovery fixture: owned2CPU/2Gi, WT0.25Gi; synthetic recovery only, not a production capacity result")
dbname := domain.NewID("maintenance_flow_test")
reopen := func() *MongoStore {
t.Helper()
c := connect()
t.Cleanup(func() { _ = c.Disconnect(context.Background()) })
return NewMongoStore(c.Database(dbname))
}
return NewMongoStore(client.Database(dbname)), action, reopen
}
func TestQueryReplicaMaintenancePublicationCacheAndRestartLocal(t *testing.T) {
if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" || os.Getenv("QUERY_REPLICA_VALKEY_LOCAL_TEST") != "1" {
t.Skip("explicit owned source/query/Valkey fixtures required")
}
var historyReads atomic.Int64
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" || e.CommandName == "aggregate" {
historyReads.Add(1)
}
}}
source := newQuerySyncSource(t, monitor)
target, restart, reopen := maintenanceRestartTarget(t)
ctx := t.Context()
seedWorkforceStats(t, source)
if e := source.UpsertOrganization(ctx, domain.OrganizationRecord{ID: "workforce-test", Status: domain.OrganizationStatusActive}); e != nil {
t.Fatal(e)
}
query := workforcestats.Query{Mode: domain.RunModeAll}
raw, e := source.LoadWorkforceStats(ctx, "workforce-test", query)
if e != nil {
t.Fatal(e)
}
want := workforcestats.Build(raw, query)
want.Details = nil
want.AsOf = ""
q, e := NewQueryReplica(source, target)
if e != nil {
t.Fatal(e)
}
q.BatchSize = 25
if e = q.EnsureSourceIndexes(ctx); e != nil {
t.Fatal(e)
}
if e = q.EnsureTargetIndexes(ctx); e != nil {
t.Fatal(e)
}
if e = q.Acquire(ctx); e != nil {
t.Fatal(e)
}
policy := safetyPolicyForTest()
policy.Mode = QueryReplicaMaintenanceInitialization
policy.BaselineCPU = 0
policy.MaxRecords = 100000
until := time.Now().Add(time.Hour)
policy.MaintenanceUntil = &until
maintenanceHealthLocal(t, q, 100, false)
if e = q.ConfigureSafety(ctx, policy, ""); e != nil {
t.Fatal(e)
}
finishLocalReplica(t, q)
target.queryPeriodic = true
target.ConfigureQueryPublishedResults(true)
cache, e := queryserving.NewManagedValkey("redis://127.0.0.1:27139", domain.NewID("maintenance-published"))
if e != nil {
t.Fatal(e)
}
defer cache.Close()
target.ConfigureQueryServing(cache, nil)
cacheReady := time.Now().Add(5 * time.Second)
for {
cache.Put(ctx, "probe", []byte("ready"), time.Second)
if value, ok := cache.Get(ctx, "probe"); ok && string(value) == "ready" {
break
}
if time.Now().After(cacheReady) {
t.Fatal("owned Valkey unavailable")
}
time.Sleep(25 * time.Millisecond)
}
if e = target.EnsureQueryReadModelIndexes(ctx); e != nil {
t.Fatal(e)
}
// Register exact server-derived scopes before the background publication pass.
if e = target.requestWorkforcePublication(ctx, "workforce-test", query); e != nil {
t.Fatal(e)
}
builder := NewQueryReadBuilder(target)
defer builder.Close()
buildCtx, stopBuild := context.WithCancel(ctx)
defer stopBuild()
buildDone := make(chan error, 1)
go func() { buildDone <- builder.Run(buildCtx) }()
await := func(read func() error) {
t.Helper()
deadline := time.Now().Add(20 * time.Second)
for {
err := read()
if err == nil {
return
}
if !errors.Is(err, ErrQueryReplicaNotReady) || time.Now().After(deadline) {
t.Fatal(err)
}
time.Sleep(25 * time.Millisecond)
}
}
if e = source.ConfigureQueryReplicaReports(target, time.Minute); e != nil {
t.Fatal(e)
}
historyReads.Store(0)
check := func(s *MongoStore) error {
got, e := s.LoadWorkforceReportCached(ctx, "workforce-test", query)
if e != nil {
return e
}
got.Details = nil
got.AsOf = ""
if !reflect.DeepEqual(got, want) {
return fmt.Errorf("published generation differs from authority")
}
return nil
}
await(func() error { return check(source) })
hits := cache.Statistics()["hits"]
target2 := NewMongoStore(target.db)
target2.queryPeriodic = true
target2.ConfigureQueryPublishedResults(true)
target2.ConfigureQueryServing(cache, nil)
source2 := NewMongoStore(source.db)
if e = source2.ConfigureQueryReplicaReports(target2, time.Minute); e != nil {
t.Fatal(e)
}
if e = check(source2); e != nil {
t.Fatal(e)
}
if cache.Statistics()["hits"] <= hits {
t.Fatal("second reader did not use Valkey")
}
restricted := query
restricted.AllowedProjects = map[string]bool{"unrelated": true}
denied, e := source2.LoadWorkforceReportCached(ctx, "workforce-test", restricted)
if !errors.Is(e, ErrQueryReplicaNotReady) {
t.Fatal("new permission scope reused broad publication", e)
}
await(func() error {
denied, e = source2.LoadWorkforceReportCached(ctx, "workforce-test", restricted)
if e == nil && denied.EventCount != 0 {
return fmt.Errorf("published cache bypassed project permission")
}
return e
})
stopBuild()
if e = <-buildDone; e != nil && !errors.Is(e, context.Canceled) {
t.Fatal(e)
}
cache.Close()
if e = check(source2); e != nil {
t.Fatal("cache loss did not retain complete query result", e)
}
// Stop the target driver before the restart.81 readers must fail closed,
// rather than issuing authority history queries during unavailability.
restart("disconnect")
var wg sync.WaitGroup
for n := 0; n < 81; n++ {
wg.Add(1)
go func() {
defer wg.Done()
short, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
_, _ = source2.LoadWorkforceReportCached(short, "workforce-test", query)
}()
}
wg.Wait()
if historyReads.Load() != 0 {
t.Fatal("query/cache outage scanned source", historyReads.Load())
}
restart("restart")
target3 := reopen()
target3.queryPeriodic = true
target3.ConfigureQueryPublishedResults(true)
target3.ConfigureQueryServing(cache, nil)
source3 := NewMongoStore(source.db)
if e = source3.ConfigureQueryReplicaReports(target3, time.Minute); e != nil {
t.Fatal(e)
}
if e = check(source3); e != nil {
t.Fatal("query restart lost complete generation", e)
}
if historyReads.Load() != 0 {
t.Fatal("recovery scanned authority history", historyReads.Load())
}
t.Log("high CPU -> sync -> complete publication -> Valkey -> reader; cache loss and81 disconnected readers caused0 authority history reads; real mongod restart preserved output")
}

View File

@ -1,6 +1,7 @@
package store
import (
"bytes"
"context"
"crypto/sha256"
"droboticflow/internal/domain"
@ -19,24 +20,32 @@ var ErrQueryReplicaPaused = errors.New("shadow synchronization paused; explicit
// Safety is opt-in for the independent projector only. It never gates authority
// business operations. The policy and pause survive process and Pod replacement.
const QueryReplicaMaintenanceInitialization = "maintenance-initialization"
type QueryReplicaSafetyPolicy struct {
BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"`
CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"`
MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"`
RunID string `json:"runId" bson:"runId"`
MaxRecords int64 `json:"maxRecords" bson:"maxRecords"`
RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"`
BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"`
MaintenanceUntil *time.Time `json:"maintenanceUntil,omitempty" bson:"maintenanceUntil,omitempty"`
Mode string `json:"mode,omitempty" bson:"mode,omitempty"`
BytesPerSecond int `json:"bytesPerSecond" bson:"bytesPerSecond"`
CommandsPerSecond int `json:"commandsPerSecond" bson:"commandsPerSecond"`
MaxBatchRows int `json:"maxBatchRows" bson:"maxBatchRows"`
RunID string `json:"runId" bson:"runId"`
MaxRecords int64 `json:"maxRecords" bson:"maxRecords"`
RecordsPerSecond int `json:"recordsPerSecond" bson:"recordsPerSecond"`
BaselineCPU float64 `json:"baselineCPU" bson:"baselineCPU"`
}
type QueryReplicaSafetyObservation struct {
At time.Time `json:"at" bson:"at"`
CPU float64 `json:"cpu" bson:"cpu"`
Ready bool `json:"ready" bson:"ready"`
WritersCovered bool `json:"writersCovered" bson:"writersCovered"`
WorkErrors bool `json:"workErrors" bson:"workErrors"`
ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"`
TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"`
HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"`
SafetySignalsVersion int `json:"safetySignalsVersion,omitempty" bson:"safetySignalsVersion"`
SourceAvailable bool `json:"sourceAvailable,omitempty" bson:"sourceAvailable"`
MemoryPressure bool `json:"memoryPressure,omitempty" bson:"memoryPressure"`
PersistenceFailure bool `json:"persistenceFailure,omitempty" bson:"persistenceFailure"`
At time.Time `json:"at" bson:"at"`
CPU float64 `json:"cpu" bson:"cpu"`
Ready bool `json:"ready" bson:"ready"`
WritersCovered bool `json:"writersCovered" bson:"writersCovered"`
WorkErrors bool `json:"workErrors" bson:"workErrors"`
ProtectedIOFull float64 `json:"protectedIOFull" bson:"protectedIOFull"`
TargetUsedFraction float64 `json:"targetUsedFraction" bson:"targetUsedFraction"`
HostFreeBytes int64 `json:"hostFreeBytes" bson:"hostFreeBytes"`
}
type queryReplicaSafetyState struct {
Owner string `bson:"owner"`
@ -60,8 +69,18 @@ func safetyPolicyDigest(p QueryReplicaSafetyPolicy) string {
h := sha256.Sum256(b)
return hex.EncodeToString(h[:])
}
func (p QueryReplicaSafetyPolicy) maintenance() bool {
return p.Mode == QueryReplicaMaintenanceInitialization
}
func (p QueryReplicaSafetyPolicy) validate() error {
if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || p.BaselineCPU <= 0 || p.BaselineCPU > 90 {
if p.Mode != "" && !p.maintenance() {
return errors.New("unknown shadow safety mode")
}
if p.maintenance() && (p.MaxRecords <= 0 || p.MaintenanceUntil == nil || !p.MaintenanceUntil.After(time.Now())) {
return errors.New("maintenance initialization requires a finite read budget and an explicit future deadline")
}
if p.BytesPerSecond < 1024 || p.BytesPerSecond > 256*1024 || p.CommandsPerSecond < 1 || p.CommandsPerSecond > 8 || p.MaxBatchRows < 1 || p.MaxBatchRows > 100 || p.RunID == "" || len(p.RunID) > 100 || p.MaxRecords < 0 || p.RecordsPerSecond < 1 || p.RecordsPerSecond > 50 || math.IsNaN(p.BaselineCPU) || math.IsInf(p.BaselineCPU, 0) || (!p.maintenance() && (p.BaselineCPU <= 0 || p.BaselineCPU > 90)) || (p.maintenance() && (p.BaselineCPU < 0 || p.BaselineCPU > 100)) {
return errors.New("invalid shadow safety policy")
}
return nil
@ -70,7 +89,7 @@ func (p QueryReplicaSafetyPolicy) validate() error {
// RecordQueryReplicaObservation is for a trusted out-of-process observer. Only
// complete, fresh observations are accepted; callers never submit customer data.
func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryReplicaSafetyObservation) error {
if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 {
if o.At.IsZero() || time.Since(o.At) > 45*time.Second || o.At.After(time.Now().Add(time.Second)) || math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || o.ProtectedIOFull > 100 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1 || o.HostFreeBytes < 0 {
return errors.New("invalid shadow observation")
}
var hello bson.M
@ -84,6 +103,16 @@ func (s *MongoStore) RecordQueryReplicaObservation(ctx context.Context, o QueryR
c := s.db.Collection(queryReplicaControl, options.Collection().SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))))
// Monotonic CAS: an old observer cannot overwrite newer health evidence.
_, e := c.UpdateOne(ctx, bson.M{"_id": "safety-health", "$or": bson.A{bson.M{"at": bson.M{"$lt": o.At}}, bson.M{"at": bson.M{"$exists": false}}}}, bson.M{"$set": o, "$push": bson.M{"recent": bson.M{"$each": bson.A{o}, "$slice": -3}}}, options.Update().SetUpsert(true))
if mongo.IsDuplicateKeyError(e) {
var old QueryReplicaSafetyObservation
if readErr := c.FindOne(ctx, bson.M{"_id": "safety-health", "at": o.At}).Decode(&old); readErr == nil {
before, first := bson.Marshal(old)
after, second := bson.Marshal(o)
if first == nil && second == nil && bytes.Equal(before, after) {
return nil
}
}
}
return e
}
func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafetyPolicy, resumeToken string) error {
@ -103,7 +132,7 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
}
if e == nil {
if state.Paused {
if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || p.BaselineCPU > state.Policy.BaselineCPU || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved {
if resumeToken == "" || state.ResumeToken != resumeToken || state.Policy.RunID != p.RunID || (!p.maintenance() && !state.Policy.maintenance() && p.BaselineCPU > state.Policy.BaselineCPU) || p.MaxRecords > 0 && p.MaxRecords <= state.Reserved {
return ErrQueryReplicaPaused
}
} else {
@ -140,12 +169,15 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
if e = q.control().FindOne(sc, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
return denyStart()
}
if len(health.Recent) != 3 {
if len(health.Recent) == 0 || (!p.maintenance() && len(health.Recent) != 3) {
return denyStart()
}
low, high := 100., 0.
for i, o := range health.Recent {
if observationFailure(o, o.At) != "" || o.CPU > 90 {
if p.maintenance() && i < len(health.Recent)-1 {
continue
}
if policyObservationFailure(o, o.At, p) != "" || (!p.maintenance() && o.CPU > 90) {
return denyStart()
}
if i > 0 && !o.At.After(health.Recent[i-1].At) {
@ -154,7 +186,8 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
low = math.Min(low, o.CPU)
high = math.Max(high, o.CPU)
}
if high-low > 5 || health.Recent[2].At.Sub(health.Recent[0].At) < 60*time.Second || observationFailure(health.Recent[2], time.Now()) != "" {
latest := health.Recent[len(health.Recent)-1]
if (!p.maintenance() && (high-low > 5 || latest.At.Sub(health.Recent[0].At) < 60*time.Second)) || policyObservationFailure(latest, time.Now(), p) != "" {
return denyStart()
}
_, e = q.control().UpdateOne(sc, bson.M{"_id": "safety"}, bson.M{"$set": bson.M{"policy": p, "digest": digest, "owner": q.owner, "paused": false, "reason": "", "resumeToken": "", "high": 0, "highIO": 0}, "$setOnInsert": bson.M{"reserved": int64(0)}}, options.Update().SetUpsert(true))
@ -171,14 +204,26 @@ func (q *QueryReplica) ConfigureSafety(ctx context.Context, p QueryReplicaSafety
return e
}
func observationFailure(o QueryReplicaSafetyObservation, now time.Time) string {
return policyObservationFailure(o, now, QueryReplicaSafetyPolicy{})
}
func policyObservationFailure(o QueryReplicaSafetyObservation, now time.Time, p QueryReplicaSafetyPolicy) string {
switch {
case o.At.IsZero() || now.Sub(o.At) > 45*time.Second || o.At.After(now.Add(time.Second)):
return "health_missing_or_stale"
case !o.Ready || !o.WritersCovered || o.WorkErrors:
return "production_health"
case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || o.CPU > 100 || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1:
case math.IsNaN(o.CPU) || math.IsInf(o.CPU, 0) || o.CPU < 0 || (!p.maintenance() && o.CPU > 100) || math.IsNaN(o.ProtectedIOFull) || math.IsInf(o.ProtectedIOFull, 0) || o.ProtectedIOFull < 0 || math.IsNaN(o.TargetUsedFraction) || math.IsInf(o.TargetUsedFraction, 0) || o.TargetUsedFraction < 0 || o.TargetUsedFraction > 1:
return "invalid_health"
case o.CPU >= 95:
case p.maintenance() && (p.MaintenanceUntil == nil || !now.Before(*p.MaintenanceUntil)):
return "maintenance_window_closed"
case p.maintenance() && (o.SafetySignalsVersion != 1 || !o.SourceAvailable):
return "maintenance_health_incomplete"
case o.MemoryPressure:
return "memory_pressure"
case o.PersistenceFailure:
return "persistence_failure"
case !p.maintenance() && o.CPU >= 95:
return "source_cpu_absolute"
case o.ProtectedIOFull > 5:
return "protected_io"
@ -214,9 +259,9 @@ func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, err
if e != nil && e != mongo.ErrNoDocuments {
return e
}
reason := observationFailure(o, time.Now())
reason := policyObservationFailure(o, time.Now(), s.Policy)
if reason == "" && !o.At.Equal(s.LastObservation) {
if o.CPU > s.Policy.BaselineCPU+3 {
if !s.Policy.maintenance() && o.CPU > s.Policy.BaselineCPU+3 {
s.High++
} else {
s.High = 0
@ -227,7 +272,7 @@ func (q *QueryReplica) sourcePermit(ctx context.Context, want int64) (int64, err
s.HighIO = 0
}
s.LastObservation = o.At
if s.High >= 2 {
if !s.Policy.maintenance() && s.High >= 2 {
reason = "source_cpu_relative"
}
if s.HighIO >= 2 {
@ -319,7 +364,7 @@ func (s *MongoStore) CheckQueryReplicaSafety(ctx context.Context, runID string)
if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "safety-health"}).Decode(&health); e != nil {
return e
}
if observationFailure(health, time.Now()) != "" {
if policyObservationFailure(health, time.Now(), state.Policy) != "" {
return ErrQueryReplicaPaused
}
return nil

View File

@ -425,3 +425,100 @@ func TestQueryReplicaSafetyBacklogPriorityAndRoundRobinLocal(t *testing.T) {
t.Fatal("round-robin checkpoint not persisted")
}
}
func maintenanceHealthLocal(t *testing.T, q *QueryReplica, cpu float64, persistence bool) {
t.Helper()
o := QueryReplicaSafetyObservation{At: time.Now().UTC(), CPU: cpu, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1, SafetySignalsVersion: 1, SourceAvailable: true, PersistenceFailure: persistence}
raw, e := bson.Marshal(o)
if e != nil {
t.Fatal(e)
}
var fields bson.M
if e = bson.Unmarshal(raw, &fields); e != nil {
t.Fatal(e)
}
fields["recent"] = []QueryReplicaSafetyObservation{o}
if _, e = q.control().UpdateOne(t.Context(), bson.M{"_id": "safety-health"}, bson.M{"$set": fields}, options.Update().SetUpsert(true)); e != nil {
t.Fatal(e)
}
}
func TestQueryReplicaMaintenanceHighCPUSyncPreservesDataLocal(t *testing.T) {
source, target, q := newLocalReplica(t)
q.BatchSize = 2
p := safetyPolicyForTest()
p.Mode = QueryReplicaMaintenanceInitialization
until := time.Now().Add(time.Hour)
p.MaintenanceUntil = &until
p.BaselineCPU = 0
p.MaxRecords = 10
maintenanceHealthLocal(t, q, 99, false)
if e := q.ConfigureSafety(t.Context(), p, ""); e != nil {
t.Fatal("waited for low CPU", e)
}
if _, e := source.db.Collection("humanItems").InsertOne(t.Context(), bson.M{"_id": "legacy", "id": "legacy", "organizationId": "org", "version": 7, "status": "claimed", "subject": bson.M{"payload": bson.M{"annotationDraft": "keep rework"}}}); e != nil {
t.Fatal(e)
}
for _, cpu := range []float64{100, 110, 96} {
maintenanceHealthLocal(t, q, cpu, false)
if _, _, e := q.BootstrapStep(t.Context(), "humanItems"); e != nil {
t.Fatal("high CPU paused copy", e)
}
}
var original bson.M
if e := source.db.Collection("humanItems").FindOne(t.Context(), bson.M{"_id": "legacy"}).Decode(&original); e != nil {
t.Fatal(e)
}
if supplierNumber(original["version"]) != 7 || original["status"] != "claimed" || deletionDoc(deletionDoc(original["subject"])["payload"])["annotationDraft"] != "keep rework" {
t.Fatal("source annotation/version changed")
}
var copied bson.M
if e := target.db.Collection("humanItems").FindOne(t.Context(), bson.M{}).Decode(&copied); e != nil {
t.Fatal(e)
}
var state queryReplicaSafetyState
if e := q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&state); e != nil || state.Paused || state.High != 0 {
t.Fatal("CPU pause survived maintenance switch", state, e)
}
if e := target.CheckQueryReplicaSafety(t.Context(), p.RunID); e != nil {
t.Fatal(e)
}
maintenanceHealthLocal(t, q, 100, true)
if _, e := q.sourcePermit(t.Context(), 1); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("persistence failure did not stop", e)
}
var after queryReplicaSafetyState
_ = q.control().FindOne(t.Context(), bson.M{"_id": "safety"}).Decode(&after)
if !after.Paused || after.Reason != "persistence_failure" {
t.Fatal("failure not durably paused", after)
}
maintenanceHealthLocal(t, q, 99, false)
if _, e := q.sourcePermit(t.Context(), 1); !errors.Is(e, ErrQueryReplicaPaused) {
t.Fatal("healthy retry bypassed durable pause")
}
}
func TestQueryReplicaHealthReplayAndOutOfOrderLocal(t *testing.T) {
_, target, q := newLocalReplica(t)
o := QueryReplicaSafetyObservation{At: time.Now().UTC().Add(-time.Second), CPU: 99, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1, SafetySignalsVersion: 1, SourceAvailable: true}
if e := target.RecordQueryReplicaObservation(t.Context(), o); e != nil {
t.Fatal(e)
}
if e := target.RecordQueryReplicaObservation(t.Context(), o); e != nil {
t.Fatal("ack retry not idempotent", e)
}
conflict := o
conflict.MemoryPressure = true
if target.RecordQueryReplicaObservation(t.Context(), conflict) == nil {
t.Fatal("same sample timestamp overwrote health")
}
old := o
old.At = old.At.Add(-time.Second)
if target.RecordQueryReplicaObservation(t.Context(), old) == nil {
t.Fatal("out-of-order health overwrote newer observation")
}
var state struct {
Recent []QueryReplicaSafetyObservation `bson:"recent"`
}
if e := q.control().FindOne(t.Context(), bson.M{"_id": "safety-health"}).Decode(&state); e != nil || len(state.Recent) != 1 || state.Recent[0].MemoryPressure {
t.Fatal("replay duplicated sample or changed result", e, state)
}
}

View File

@ -144,3 +144,45 @@ func TestQueryReplicaTransportByteBudgetSharedAcrossConnections(t *testing.T) {
t.Fatal("aggregate byte rate exceeded")
}
}
func TestQueryReplicaMaintenancePolicyKeepsHardSafety(t *testing.T) {
until := time.Now().Add(time.Hour)
p := QueryReplicaSafetyPolicy{MaintenanceUntil: &until, Mode: QueryReplicaMaintenanceInitialization, BytesPerSecond: 65536, CommandsPerSecond: 2, MaxBatchRows: 25, RunID: "bounded-maintenance", MaxRecords: 1000, RecordsPerSecond: 10}
if e := p.validate(); e != nil {
t.Fatal(e)
}
unlimited := p
unlimited.MaxRecords = 0
if unlimited.validate() == nil {
t.Fatal("maintenance accepted an unlimited budget")
}
unknown := p
unknown.Mode = "disabled"
if unknown.validate() == nil {
t.Fatal("unknown policy silently disabled protections")
}
now := time.Now()
healthy := QueryReplicaSafetyObservation{At: now, CPU: 100, Ready: true, WritersCovered: true, HostFreeBytes: 120 << 30, TargetUsedFraction: .1, SafetySignalsVersion: 1, SourceAvailable: true}
for _, cpu := range []float64{0, 91, 99, 100, 110} {
o := healthy
o.CPU = cpu
if reason := policyObservationFailure(o, now, p); reason != "" {
t.Fatal("CPU used as maintenance gate", cpu, reason)
}
}
for _, change := range []func(*QueryReplicaSafetyObservation){func(o *QueryReplicaSafetyObservation) { o.MemoryPressure = true }, func(o *QueryReplicaSafetyObservation) { o.PersistenceFailure = true }, func(o *QueryReplicaSafetyObservation) { o.SourceAvailable = false }, func(o *QueryReplicaSafetyObservation) { o.SafetySignalsVersion = 0 }, func(o *QueryReplicaSafetyObservation) { o.WorkErrors = true }, func(o *QueryReplicaSafetyObservation) { o.Ready = false }, func(o *QueryReplicaSafetyObservation) { o.At = now.Add(-46 * time.Second) }, func(o *QueryReplicaSafetyObservation) { o.CPU = math.NaN() }, func(o *QueryReplicaSafetyObservation) { o.TargetUsedFraction = .7 }} {
o := healthy
change(&o)
if policyObservationFailure(o, now, p) == "" {
t.Fatal("hard safety disabled")
}
}
expired := healthy
expired.At = until.Add(time.Second)
if reason := policyObservationFailure(expired, expired.At, p); reason != "maintenance_window_closed" {
t.Fatal("expired maintenance remained active")
}
if observationFailure(healthy, now) == "" {
t.Fatal("legacy strict CPU protection changed")
}
}

View File

@ -13,6 +13,7 @@ import datetime as dt
import hashlib
import json
import os
import math
import re
from pathlib import Path
import socket
@ -21,6 +22,7 @@ import threading
import time
import urllib.parse
import urllib.request
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
def check_bindings(b):
@ -28,7 +30,7 @@ def check_bindings(b):
raise ValueError('invalid namespace or writer scope')
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
raise ValueError('stop scope must contain only shadow controllers')
if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90:
if b['sourceDatabase'] == b['targetDatabase'] or (not b.get('maintenanceInitialization') and not 0 < b['baselineCPU'] <= 90):
raise ValueError('invalid source identity or baseline')
@ -71,7 +73,7 @@ class Observer:
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
text=True, timeout=timeout)
if p.returncode:
raise RuntimeError('bounded cluster operation unavailable')
raise ProbeUnavailable('bounded cluster operation unavailable')
return p.stdout
def get(self, ns, kind, name):
@ -109,7 +111,7 @@ class Observer:
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
text=True, timeout=10)
if p.returncode:
raise RuntimeError('bounded observation command unavailable')
raise ProbeUnavailable('bounded observation command unavailable')
return p.stdout
@contextlib.contextmanager
@ -203,10 +205,18 @@ class Observer:
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
return json.loads(self.run_binary(['--sample-source'], env))
def observation(self, cpu, sampled_at):
def readiness(self):
try:
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
return response.status == 200 and json.load(response).get('ready') is True
except urllib.error.HTTPError:
return False
except (OSError, ValueError) as error:
raise ProbeUnavailable('readiness transport unavailable') from None
def observation(self, cpu, sampled_at, source=None):
self.verify()
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
ready = response.status == 200 and json.load(response).get('ready') is True
ready = retry_probe(self.readiness, 'readiness')
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
if pod['metadata']['uid'] != self.b['mongoPodUID']:
raise ValueError('query Mongo rebuilt; recovery verification required')
@ -229,17 +239,37 @@ class Observer:
if p['metadata'].get('deletionTimestamp'): continue
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
if self.b.get('maintenanceInitialization'):
if not source or source.get('memorySupported') is not True or type(source.get('residentMiB')) not in (int,float) or not math.isfinite(source['residentMiB']) or source['residentMiB'] <= 0 or type(self.b.get('sourceMemoryLimitMiB')) not in (int,float) or not math.isfinite(self.b['sourceMemoryLimitMiB']) or self.b['sourceMemoryLimitMiB'] <= 0:
raise ProbeUnavailable('source memory evidence incomplete')
group='/hostcg/kubepods.slice/kubepods-pod'+pod['metadata']['uid'].replace('-','_')+'.slice/'
used_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.current')
limit_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.max')
events=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.events')
memory=parse_memory(used_mem,limit_mem,events)
node=json.loads(self.call('get','node','10.189.16.4','-o','json'))
pressure = [c['status'] for c in node['status']['conditions'] if c['type']=='MemoryPressure']
if len(pressure)!=1 or pressure[0] not in ('True','False'): raise ProbeUnavailable('node memory evidence incomplete')
node_pressure = pressure[0]=='True'
mongo_logs=self.call('-n',ns,'logs','query-mongo-0','-c','mongo','--since=45s','--tail=2000')
if len(mongo_logs.splitlines())>=2000: raise ProbeUnavailable('persistence log window truncated')
result.update(safetySignalsVersion=1,sourceAvailable=True,
memoryPressure=memory['pressure'] or node_pressure or source['residentMiB']>=.98*self.b['sourceMemoryLimitMiB'],
persistenceFailure=persistence_failed(mongo_logs))
return result
def publish(self, observation):
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
def unsafe(self, o):
self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0
self.high = self.high + 1 if not self.b.get('maintenanceInitialization') and o['cpu'] > self.b['baselineCPU'] + 3 else 0
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or (not self.b.get('maintenanceInitialization') and (o['cpu'] >= 95 or self.high >= 2))
or (self.b.get('maintenanceInitialization') and (not self.b.get('maintenanceUntil') or dt.datetime.now(dt.timezone.utc)>=dt.datetime.fromisoformat(self.b['maintenanceUntil'].replace('Z','+00:00'))))
or o.get('memoryPressure',False) or o.get('persistenceFailure',False) or (self.b.get('maintenanceInitialization') and (o.get('safetySignalsVersion')!=1 or not o.get('sourceAvailable')))
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
def stop_shadow(self):
@ -253,7 +283,11 @@ class Observer:
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
def run(self):
try: prior = self.source_sample()
try:
self.verify()
pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items']
check_clocks(self.call,self.b['sourceNamespace'],pods)
prior = retry_probe(self.source_sample,'initial-source')
except Exception:
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('initial source observation unavailable') from None
@ -261,15 +295,14 @@ class Observer:
try:
for _ in range(3):
time.sleep(10)
with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response:
if response.status!=200 or json.load(response).get('ready') is not True:
raise ValueError('production readiness unavailable')
current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current
if not retry_probe(self.readiness, 'readiness'):
raise ValueError('production readiness unavailable')
current = retry_probe(self.source_sample,'source-sample'); cpu = cpu_percent(prior, current); prior = current
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
if not 0<=age<=45: raise ValueError('source observation stale')
o = self.observation(cpu,sampled_at); self.publish(o)
o = retry_probe(lambda: self.observation(cpu,sampled_at,current),'health-observation'); retry_probe(lambda: self.publish(o),'health-publish')
unsafe=self.unsafe(o)
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True)
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe,'cpuObservedOnly':bool(self.b.get('maintenanceInitialization'))}),flush=True)
if unsafe:
if self.args.stop_on_failure: self.stop_shadow()
return

View File

@ -0,0 +1,51 @@
"""Bounded, diagnosed read-only probes. Never retries returned unhealthy states."""
import datetime as dt
import json
import math
import subprocess
import time
class ProbeUnavailable(RuntimeError):
pass
def emit(value):
print(json.dumps(value, sort_keys=True), flush=True)
def retry_probe(probe, phase, attempts=3, delay=.5, diagnostic=emit, sleep=time.sleep):
if not 1 <= attempts <= 3 or not 0 <= delay <= 1: raise ValueError('invalid retry budget')
for index in range(attempts):
try: return probe()
except (ProbeUnavailable, subprocess.TimeoutExpired, OSError) as error:
diagnostic(dict(event='probe_retry', phase=phase, attempt=index+1, maxAttempts=attempts, failureType=type(error).__name__))
if index+1 == attempts: raise ProbeUnavailable('bounded probe exhausted') from None
sleep(delay*(index+1))
def check_clocks(call, namespace, pods, container='api', attempts=3, diagnostic=emit,
utcnow=lambda: dt.datetime.now(dt.timezone.utc), monotonic=time.monotonic, sleep=time.sleep):
results=[]
for pod in pods:
def measure():
start=utcnow(); tick=monotonic()
try:
text=call('-n', namespace, 'exec', pod['metadata']['name'], '-c', container, '--', 'date', '-u', '+%s.%N', timeout=5)
remote=float(text.strip())
except (ValueError, RuntimeError, subprocess.TimeoutExpired, OSError) as error:
diagnostic(dict(event='clock_probe', node=pod['spec']['nodeName'], podUID=pod['metadata']['uid'], available=False, failureType=type(error).__name__))
raise ProbeUnavailable('clock transport unavailable') from None
end=utcnow();rtt=monotonic()-tick;offset=remote-(start+(end-start)/2).timestamp()
value=dict(event='clock_probe',node=pod['spec']['nodeName'],podUID=pod['metadata']['uid'],available=True,offsetSeconds=offset,rttSeconds=rtt,offsetPassed=math.isfinite(offset) and abs(offset)<=1,rttPassed=0<=rtt<=2)
diagnostic(value)
if not value['offsetPassed'] or not value['rttPassed']: raise ProbeUnavailable('clock gate failed')
return value
results.append(retry_probe(measure,'clock',attempts=attempts,diagnostic=diagnostic,sleep=sleep))
return results
def parse_memory(current, maximum, events):
used=int(current.strip());limit=int(maximum.strip())
counters={line.split()[0]:int(line.split()[1]) for line in events.splitlines() if len(line.split())==2}
if used<0 or limit<=0 or not {'oom','oom_kill'} <= counters.keys(): raise ValueError('memory evidence incomplete')
return dict(usedBytes=used,limitBytes=limit,oom=counters['oom'],oomKill=counters['oom_kill'],pressure=used>=.98*limit or counters['oom']>0 or counters['oom_kill']>0)
def persistence_failed(text):
lower=text.lower()
return any(token in lower for token in ('wt_panic','failed to fsync','input/output error','no space left on device','journal write failed','fatal assertion'))

View File

@ -0,0 +1,42 @@
import datetime as dt
import unittest
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
class HealthProbeTests(unittest.TestCase):
def test_bounded_retry_has_diagnostics_and_never_retries_bad_result(self):
events=[];calls=[]
def work():
calls.append(1)
if len(calls)<3: raise ProbeUnavailable('fixed reason')
return {'healthy':False}
self.assertEqual(retry_probe(work,'test',diagnostic=events.append,sleep=lambda _:None),{'healthy':False})
self.assertEqual(len(calls),3);self.assertEqual([e['attempt'] for e in events],[1,2])
with self.assertRaises(ProbeUnavailable):retry_probe(lambda:(_ for _ in ()).throw(ProbeUnavailable()),'test',diagnostic=events.append,sleep=lambda _:None)
with self.assertRaises(ValueError):retry_probe(lambda:None,'test',attempts=4)
def test_clock_offset_and_rtt_are_distinguished_and_bounded(self):
now=dt.datetime.now(dt.timezone.utc);events=[]
pods=[dict(metadata=dict(name='api',uid='uid'),spec=dict(nodeName='node'))]
values=iter([str(now.timestamp()+2),str(now.timestamp())]);ticks=iter([0,.1,1,3.1,4,4.1])
# first offset2s fails; second RTT2.1 fails; final sample passes.
def call(*args,**kwargs):
n=sum(e.get('event')=='clock_probe' for e in events)
return str(now.timestamp()+2) if n==0 else str(now.timestamp())
rows=check_clocks(call,'ns',pods,diagnostic=events.append,utcnow=lambda:now,monotonic=lambda:next(ticks),sleep=lambda _:None)
samples=[e for e in events if e.get('event')=='clock_probe']
self.assertFalse(samples[0]['offsetPassed']);self.assertTrue(samples[0]['rttPassed'])
self.assertTrue(samples[1]['offsetPassed']);self.assertFalse(samples[1]['rttPassed'])
self.assertTrue(rows[0]['offsetPassed']);self.assertEqual(len(samples),3)
def test_clock_transport_failure_records_reason(self):
events=[];pods=[dict(metadata=dict(name='api',uid='uid'),spec=dict(nodeName='node'))]
with self.assertRaises(ProbeUnavailable):check_clocks(lambda *a,**kw:(_ for _ in ()).throw(RuntimeError()),'ns',pods,diagnostic=events.append,sleep=lambda _:None)
self.assertEqual(sum(e.get('event')=='clock_probe' for e in events),3)
self.assertTrue(all(e.get('available') is False for e in events if e.get('event')=='clock_probe'))
def test_memory_and_durability_errors_remain_hard_signals(self):
self.assertFalse(parse_memory('100','1000','oom 0\noom_kill 0')['pressure'])
self.assertTrue(parse_memory('990','1000','oom 0\noom_kill 0')['pressure'])
self.assertTrue(parse_memory('100','1000','oom 0\noom_kill 1')['pressure'])
with self.assertRaises(ValueError):parse_memory('100','max','')
self.assertTrue(persistence_failed('WiredTiger WT_PANIC'));self.assertTrue(persistence_failed('journal write failed'))
self.assertFalse(persistence_failed('checkpoint completed'))
if __name__=='__main__':unittest.main()

View File

@ -66,6 +66,14 @@ class ObserverTests(unittest.TestCase):
with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64)
feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64)
def test_maintenance_cpu_is_observed_but_resource_failures_stop(self):
b=self.bindings();b['maintenanceInitialization']=True;b['baselineCPU']=0;b['maintenanceUntil']=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat()
m.check_bindings(b);o=m.Observer(SimpleNamespace(kubeconfig='unused'),b)
good=dict(cpu=100,ready=True,writersCovered=True,workErrors=False,protectedIOFull=0,targetUsedFraction=.1,hostFreeBytes=120<<30,safetySignalsVersion=1,sourceAvailable=True,memoryPressure=False,persistenceFailure=False)
for cpu in [95,100,110]:
good['cpu']=cpu;self.assertFalse(o.unsafe(good))
for field,value in [('memoryPressure',True),('persistenceFailure',True),('sourceAvailable',False),('workErrors',True),('safetySignalsVersion',0)]:
bad=dict(good);bad[field]=value;self.assertTrue(o.unsafe(bad))
def test_unknown_io_is_not_zero(self):
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)

View File

@ -18,6 +18,7 @@ COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml'
CASES = ('cmd/query-replica/main_local_test.go',
'internal/store/query_replica_source_auth_local_test.go',
'internal/store/query_replica_safety_local_test.go',
'internal/store/query_replica_maintenance_flow_local_test.go',
'internal/store/query_replica_safety_test.go',
'internal/store/query_replica_local_test.go',
'internal/store/query_sync_local_test.go',

View File

@ -1018,7 +1018,7 @@
{
"name": "shadow-work-health",
"paths": [
"^scripts/local/(query-shadow-(observer|work-health)|test_query_shadow_(observer|work_health))\\.py$"
"^scripts/local/(query-shadow-(observer|work-health)|query_shadow_health|test_query_shadow_(observer|work_health|health))\\.py$"
],
"frontend": [],
"go": {