⚡ Streamline unattended query sync recovery and evening cutover

This commit is contained in:
d-robotics 2026-11-15 22:16:30 -08:00
parent f7e4cb287e
commit 484e5d591b
11 changed files with 1308 additions and 28 deletions

View File

@ -21,6 +21,7 @@ func main() {
once := flag.Bool("once", false, "build once; streaming freshness expires after exit")
ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building")
shadowRun := flag.String("shadow-safety-run", "", "opt-in independently monitored shadow builder; no authority gates")
autoRecover := flag.Bool("auto-recover-transient", false, "automatically cool down and clear transient build failure fences in unattended mode")
flag.Parse()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
@ -90,6 +91,26 @@ func main() {
for workCtx.Err() == nil {
if err := b.Run(workCtx); err != nil && workCtx.Err() == nil {
if errors.Is(err, store.ErrQueryBuildStopped) {
if *autoRecover && *shadowRun == "" {
log.Print("query builder hit stopped fence; cooling down before checking transient recovery")
cooldown := time.NewTimer(15 * time.Second)
select {
case <-workCtx.Done():
cooldown.Stop()
return
case <-cooldown.C:
}
resetCtx, cancel := context.WithTimeout(workCtx, 5*time.Second)
cleared, clearErr := s.ResetTransientQueryBuildFailures(resetCtx)
cancel()
if clearErr == nil && cleared > 0 {
log.Print("query builder cleared transient failure fence; resuming checkpoint")
continue
}
log.Print("query builder stopped on non-transient failure; checkpoint retained; explicit recovery required")
<-workCtx.Done()
return
}
log.Print("query builder stopped; checkpoint retained; explicit recovery required")
<-workCtx.Done()
return

View File

@ -43,6 +43,7 @@ func main() {
status := flag.Bool("safety-status", false, "target-only private safety status")
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
sourceCompress := flag.Bool("source-compress", false, "negotiate snappy/zstd wire compression on unguarded source reads")
flag.Parse()
var maintenanceDeadline *time.Time
if *maintenance {
@ -176,6 +177,12 @@ func main() {
log.Fatal("explicit source/query connection required")
}
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
if !target && transport == nil {
opts.SetMaxPoolSize(2)
if *sourceCompress {
opts.SetCompressors([]string{"snappy", "zstd"})
}
}
if !target && transport != nil {
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors(sourceCompressors(*maintenance)).SetRetryReads(false).SetTimeout(5 * time.Second)
}
@ -203,12 +210,15 @@ func main() {
if *ensure && q.EnsureSourceIndexes(ctx) != nil {
log.Fatal("source index preparation failed")
}
for {
for attempts := 0; ; attempts++ {
err := q.Acquire(ctx)
if err == nil {
break
}
if *safetyRun == "" || !errors.Is(err, store.ErrQueryReplicaLease) {
if *once || !errors.Is(err, store.ErrQueryReplicaLease) {
log.Fatal("projector lease unavailable")
}
if *safetyRun == "" && attempts >= 35 {
log.Fatal("projector lease unavailable")
}
select {
@ -276,6 +286,7 @@ func main() {
<-ctx.Done()
return
}
consecutiveFailures := 0
for workCtx.Err() == nil {
started := time.Now()
ready, e := q.RunOnce(workCtx, *bootstrap)
@ -285,6 +296,21 @@ func main() {
}
d := store.QueryReplicaDiagnostic(e)
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
if *safetyRun == "" && !*once {
wait := daemonRetryDelay(consecutiveFailures)
consecutiveFailures++
timer := time.NewTimer(wait)
select {
case <-workCtx.Done():
timer.Stop()
return
case <-timer.C:
}
if errors.Is(e, store.ErrQueryReplicaLease) {
_ = q.Acquire(workCtx)
}
continue
}
short, stop := context.WithTimeout(context.Background(), 5*time.Second)
if q.PauseSafetyFailure(short, e) != nil {
log.Print("pause diagnostics persistence unavailable; startup fencing remains required")
@ -297,6 +323,7 @@ func main() {
}
os.Exit(1)
}
consecutiveFailures = 0
if *once {
if !ready {
pause("incomplete_pilot")
@ -307,9 +334,29 @@ func main() {
}
// Optional pacing covers backfill too. Idle lease maintenance only
// touches the independent query DB, never scans authority for keepalive.
if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 {
delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass)
if *safetyRun == "" {
if cool := adaptiveBackfillCooldown(ready, time.Since(started)); cool > delay {
delay = cool
}
}
if delay > 0 {
if e := q.WaitNextPass(workCtx, delay); e != nil {
if ctx.Err() == nil {
if *safetyRun == "" && !*once {
log.Print("projector lease refresh retry")
wait := daemonRetryDelay(consecutiveFailures)
consecutiveFailures++
timer := time.NewTimer(wait)
select {
case <-workCtx.Done():
timer.Stop()
return
case <-timer.C:
}
_ = q.Acquire(workCtx)
continue
}
pause("target_error")
log.Print("projector lease unavailable; checkpoint retained")
if *safetyRun != "" {

View File

@ -23,3 +23,24 @@ func passDelay(mode string, ready bool, elapsed, interval, minimum time.Duration
}
return delay
}
func daemonRetryDelay(attempt int) time.Duration {
if attempt <= 0 {
return 2 * time.Second
}
if attempt > 10 {
attempt = 10
}
delay := 2 * time.Second << attempt
if delay > 15*time.Second || delay <= 0 {
return 15 * time.Second
}
return delay
}
func adaptiveBackfillCooldown(ready bool, elapsed time.Duration) time.Duration {
if !ready && elapsed >= 2*time.Second {
return 500 * time.Millisecond
}
return 0
}

View File

@ -40,3 +40,35 @@ func TestPacingRejectsUnboundedOrInvalidConfiguration(t *testing.T) {
}
}
}
func TestDaemonRetryDelayAndAdaptiveBackfillCooldown(t *testing.T) {
for _, tc := range []struct {
attempt int
want time.Duration
}{
{-1, 2 * time.Second},
{0, 2 * time.Second},
{1, 4 * time.Second},
{2, 8 * time.Second},
{3, 15 * time.Second},
{10, 15 * time.Second},
} {
if got := daemonRetryDelay(tc.attempt); got != tc.want {
t.Fatalf("attempt=%d got=%s want=%s", tc.attempt, got, tc.want)
}
}
for _, tc := range []struct {
ready bool
elapsed time.Duration
want time.Duration
}{
{true, 5 * time.Second, 0},
{false, 1999 * time.Millisecond, 0},
{false, 2 * time.Second, 500 * time.Millisecond},
{false, 4 * time.Second, 500 * time.Millisecond},
} {
if got := adaptiveBackfillCooldown(tc.ready, tc.elapsed); got != tc.want {
t.Fatalf("ready=%v elapsed=%s got=%s want=%s", tc.ready, tc.elapsed, got, tc.want)
}
}
}

View File

@ -204,3 +204,57 @@ func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease
}
return nil
}
var transientQueryBuildFailureClasses = []string{
"source_timeout",
"target_timeout",
"timeout",
"deadline",
"network",
"lease_lost",
"lease",
"no_durable_progress",
"transient",
}
func isTransientQueryBuildFailureClass(class string) bool {
for _, candidate := range transientQueryBuildFailureClasses {
if class == candidate {
return true
}
}
return false
}
// ResetTransientQueryBuildFailures clears transient failure fences for active
// query build jobs while preserving durable checkpoints and schema/content blocks.
func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int64, error) {
if s == nil || s.db == nil {
return 0, errors.New("store unavailable")
}
classes := bson.A{}
for _, c := range transientQueryBuildFailureClasses {
classes = append(classes, c)
}
filter := bson.M{
"schema": queryBuildJobSchema,
"phase": bson.M{"$ne": "complete"},
"failureClass": bson.M{"$in": classes},
}
update := bson.M{
"$unset": bson.M{
"failurePhase": "",
"failureClass": "",
"failureCode": "",
},
"$set": bson.M{
"activeWithoutProgress": int64(0),
"progressAt": time.Now().UTC(),
},
}
res, err := s.db.Collection(queryBuildJobsCollection).UpdateMany(ctx, filter, update)
if err != nil {
return 0, err
}
return res.ModifiedCount, nil
}

View File

@ -0,0 +1,33 @@
package store
import "testing"
func TestIsTransientQueryBuildFailureClass(t *testing.T) {
for _, class := range []string{
"source_timeout",
"target_timeout",
"timeout",
"deadline",
"network",
"lease_lost",
"lease",
"no_durable_progress",
"transient",
} {
if !isTransientQueryBuildFailureClass(class) {
t.Fatalf("expected transient class %q to return true", class)
}
}
for _, class := range []string{
"",
"unsupported_record",
"unsupported_fact",
"corrupt_state",
"auth_failed",
"bson_decode",
} {
if isTransientQueryBuildFailureClass(class) {
t.Fatalf("expected non-transient class %q to return false", class)
}
}
}

View File

@ -18,7 +18,10 @@ import (
)
const queryCountMemoryBudget = 64 << 20
const queryCountTTL = 2 * time.Second
// Cache keys include factEpoch so synced mutations invalidate immediately; a
// 15s TTL avoids 2s re-fetch/decompression thrashing while epochs are unchanged.
const queryCountTTL = 15 * time.Second
var errQueryCountCacheCapacity = errors.New("count candidate cache capacity exceeded")
@ -75,6 +78,13 @@ func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, by
if bytes > queryCountMemoryBudget {
return
}
if c.entries == nil {
c.entries = map[string]queryCountEntry{}
}
if prev, ok := c.entries[key]; ok {
delete(c.entries, key)
c.bytes -= prev.bytes
}
now := time.Now()
for k, e := range c.entries {
if !now.Before(e.expires) {
@ -82,8 +92,24 @@ func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, by
c.bytes -= e.bytes
}
}
if len(c.entries) >= 32 || c.bytes+bytes > queryCountMemoryBudget {
clear(c.entries)
for len(c.entries) >= 32 || (len(c.entries) > 0 && c.bytes+bytes > queryCountMemoryBudget) {
var oldestKey string
var oldestExpires time.Time
first := true
for k, e := range c.entries {
if first || e.expires.Before(oldestExpires) {
oldestKey = k
oldestExpires = e.expires
first = false
}
}
if oldestKey == "" {
break
}
c.bytes -= c.entries[oldestKey].bytes
delete(c.entries, oldestKey)
}
if c.bytes < 0 {
c.bytes = 0
}
c.entries[key] = queryCountEntry{rows: rows, bytes: bytes, expires: now.Add(queryCountTTL)}

View File

@ -4,7 +4,9 @@ import (
"context"
"droboticflow/internal/domain"
"errors"
"fmt"
"testing"
"time"
)
func TestQueryReplicaCountOnlyLeavesListsAndCoreReadsOnAuthority(t *testing.T) {
@ -78,3 +80,50 @@ func TestQueryReplicaMissingConfigurationCannotPanicCoreCache(t *testing.T) {
t.Fatal(e)
}
}
func TestQueryCountCacheEvictsOldestInsteadOfClearingAll(t *testing.T) {
c := &queryCountCache{}
c.remember("k0", []domain.HumanItemRecord{{ID: "0"}}, 1024)
time.Sleep(2 * time.Millisecond)
for i := 1; i < 32; i++ {
c.remember(fmt.Sprintf("k%d", i), []domain.HumanItemRecord{{ID: fmt.Sprintf("%d", i)}}, 1024)
}
if len(c.entries) != 32 || c.bytes != 32*1024 {
t.Fatalf("unexpected cache size=%d bytes=%d", len(c.entries), c.bytes)
}
c.remember("k32", []domain.HumanItemRecord{{ID: "32"}}, 1024)
if len(c.entries) != 32 {
t.Fatalf("expected 32 entries after eviction, got %d", len(c.entries))
}
if c.bytes != 32*1024 {
t.Fatalf("expected bytes=%d, got %d", 32*1024, c.bytes)
}
if _, ok := c.entries["k0"]; ok {
t.Fatal("expected oldest entry k0 to be evicted")
}
if _, ok := c.entries["k1"]; !ok {
t.Fatal("expected newer entry k1 to stay cached")
}
if _, ok := c.entries["k32"]; !ok {
t.Fatal("expected newest entry k32 to be cached")
}
}
func TestFilterAuthoritativeAdminPoolPageDropsConcurrentlyClaimedItems(t *testing.T) {
q := AdminAnnotationPoolQuery{Pool: "available"}
candidates := []domain.HumanItemRecord{
{ID: "a"},
{ID: "claimed"},
{ID: "missing"},
{ID: "b"},
}
hydrated := []domain.HumanItemRecord{
{ID: "b", CurrentStageID: "semantic_annotation", Status: domain.HumanItemReady},
{ID: "claimed", CurrentStageID: "semantic_annotation", Status: domain.HumanItemClaimed, AssignedUserID: "annotator-1"},
{ID: "a", CurrentStageID: "semantic_annotation", Status: domain.HumanItemReady},
}
got := filterAuthoritativeAdminPoolPage(candidates, hydrated, q)
if len(got) != 2 || got[0].ID != "a" || got[1].ID != "b" {
t.Fatalf("expected [a b] in candidate order without failing page, got %+v", got)
}
}

View File

@ -175,29 +175,76 @@ func (s *MongoStore) loadReplicaPoolPage(ctx context.Context, key, org string, q
return page, nil
}
func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
rows, e := target.ListAdminAnnotationPool(ctx, org, q)
if e != nil || len(rows) == 0 {
return rows, e
func filterAuthoritativeAdminPoolPage(candidates, hydrated []domain.HumanItemRecord, q AdminAnnotationPoolQuery) []domain.HumanItemRecord {
byID := make(map[string]domain.HumanItemRecord, len(hydrated))
for _, item := range hydrated {
byID[item.ID] = item
}
ids := make([]string, 0, len(rows))
for _, r := range rows {
ids = append(ids, r.ID)
}
current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids)
if e != nil {
return nil, e
}
if len(current) != len(rows) {
return nil, ErrQueryReplicaNotReady
}
if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil {
return nil, e
}
for _, item := range current {
if !adminPoolScope(item) || !adminPoolSelected(item, q) {
return nil, ErrQueryReplicaNotReady
out := make([]domain.HumanItemRecord, 0, len(candidates))
for _, cand := range candidates {
item, ok := byID[cand.ID]
if !ok || !adminPoolScope(item) || !adminPoolSelected(item, q) {
continue
}
out = append(out, item)
}
return current, nil
return out
}
func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
collected := []domain.HumanItemRecord{}
seen := map[string]bool{}
stepQ := q
for pass := 0; pass < 4; pass++ {
if q.Limit > 0 {
need := q.Limit - len(collected)
if need <= 0 {
break
}
if need > stepQ.Limit {
stepQ.Limit = need
} else if stepQ.Limit < q.Limit {
stepQ.Limit = q.Limit
}
}
rows, e := target.ListAdminAnnotationPool(ctx, org, stepQ)
if e != nil {
return nil, e
}
if len(rows) == 0 {
break
}
ids := make([]string, 0, len(rows))
for _, r := range rows {
ids = append(ids, r.ID)
}
current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids)
if e != nil {
return nil, e
}
if len(current) > 0 {
if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil {
return nil, e
}
}
for _, item := range filterAuthoritativeAdminPoolPage(rows, current, q) {
if seen[item.ID] {
continue
}
seen[item.ID] = true
collected = append(collected, item)
if q.Limit > 0 && len(collected) >= q.Limit {
return collected, nil
}
}
if q.Limit <= 0 || len(rows) < stepQ.Limit {
break
}
last := rows[len(rows)-1]
if last.ID == "" {
break
}
stepQ.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return collected, nil
}

View File

@ -0,0 +1,813 @@
#!/usr/bin/env python3
"""Unattended evening shadow-sync, parity gate, API read cutover, and auto-rollback."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
from typing import Any
DEFAULT_KUBECONFIG = "/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf"
SOURCE_NAMESPACE = "droboticflow-fanyu"
SHADOW_NAMESPACE = "droboticflow-query-shadow"
QUERY_DB_NAME = "drobodata_fanyu_query_shadow_20261114"
QUERY_SERVING_NAMESPACE = "fanyu-query-shadow-20261114"
READ_CLIENT_SECRET_NAME = "droboticflow-query-read-client"
SHADOW_CONNECTIONS_SECRET = "query-shadow-query-connections"
SHADOW_CACHE_AUTH_SECRET = "query-shadow-cache-auth"
API_DEPLOYMENT = "droboticflow-api"
API_CONTAINER = "api"
REPLICA_DEPLOYMENT = "query-replica"
REPLICA_CONTAINER = "projector"
BUILDER_DEPLOYMENT = "query-read-builder"
BUILDER_CONTAINER = "builder"
SHADOW_MONGO_POD = "query-mongo-0"
SHADOW_MONGO_CONTAINER = "mongo"
SECRET_KEY_PATTERN = re.compile(
r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)",
re.IGNORECASE,
)
ERROR_LOG_PATTERN = re.compile(
r"(ErrQueryReplicaNotReady|查询暂不可用|\bstatus=(?:500|503)\b|\"\s+(?:500|503)\s+)",
)
def utc_now() -> dt.datetime:
return dt.datetime.now(dt.timezone.utc)
def default_run_id() -> str:
return utc_now().strftime("%Y%m%dT%H%M%SZ")
def parse_iso_datetime(raw: Any) -> dt.datetime | None:
if not isinstance(raw, str) or not raw.strip():
return None
text = raw.strip()
if text.endswith("Z"):
text = text[:-1] + "+00:00"
try:
parsed = dt.datetime.fromisoformat(text)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=dt.timezone.utc)
return parsed.astimezone(dt.timezone.utc)
def redact_secrets(obj: Any, parent_key: str = "") -> Any:
"""Recursively scrub secret payloads, URIs, and credentials before persistence/logging."""
if isinstance(obj, dict):
kind = obj.get("kind")
if kind == "Secret":
out = {}
for k, v in obj.items():
if k in ("data", "stringData") and isinstance(v, dict):
out[k] = {sk: "<redacted>" for sk in v.keys()}
elif k == "metadata" and isinstance(v, dict):
meta = dict(v)
if isinstance(meta.get("annotations"), dict):
anns = dict(meta["annotations"])
if "kubectl.kubernetes.io/last-applied-configuration" in anns:
anns["kubectl.kubernetes.io/last-applied-configuration"] = "<redacted>"
meta["annotations"] = anns
out[k] = meta
else:
out[k] = redact_secrets(v, k)
return out
# Redact Kubernetes env var literal values if key name looks sensitive.
if "name" in obj and "value" in obj and isinstance(obj.get("name"), str):
env_name = obj["name"]
if SECRET_KEY_PATTERN.search(env_name):
redacted = dict(obj)
redacted["value"] = "<redacted>"
return redacted
return {k: redact_secrets(v, k) for k, v in obj.items()}
if isinstance(obj, list):
return [redact_secrets(item, parent_key) for item in obj]
if isinstance(obj, str) and parent_key and SECRET_KEY_PATTERN.search(parent_key):
return "<redacted>"
return obj
def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Merge env updates into existing Kubernetes env list preserving untouched vars and order."""
by_name = {entry["name"]: dict(entry) for entry in updates}
seen: set[str] = set()
merged: list[dict[str, Any]] = []
for entry in existing_env or []:
name = entry.get("name")
if not isinstance(name, str) or name in seen:
continue
seen.add(name)
if name in by_name:
merged.append(by_name[name])
else:
merged.append(dict(entry))
for entry in updates:
name = entry["name"]
if name not in seen:
seen.add(name)
merged.append(dict(entry))
return merged
def build_deployment_update_patch(
*,
container_name: str,
env_list: list[dict[str, Any]],
strategy_recreate: bool = False,
image: str | None = None,
args: list[str] | None = None,
) -> dict[str, Any]:
"""Build a Kubernetes strategic-merge patch for a single deployment container."""
container_patch: dict[str, Any] = {
"name": container_name,
"env": env_list,
}
if image is not None:
container_patch["image"] = image
if args is not None:
container_patch["args"] = list(args)
spec_patch: dict[str, Any] = {
"template": {
"spec": {
"containers": [container_patch],
}
}
}
if strategy_recreate:
# Clearing rollingUpdate is required when switching from RollingUpdate -> Recreate.
spec_patch["strategy"] = {"type": "Recreate", "rollingUpdate": None}
return {"spec": spec_patch}
def build_shadow_replica_args() -> list[str]:
return [
"--writers-covered",
"--bootstrap",
"--source-compress",
"--batch-size=100",
"--interval=30s",
]
def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
updates = [
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "1"},
]
return merge_env_list(existing_env, updates)
def build_shadow_builder_args() -> list[str]:
return [
"--ensure-indexes",
"--auto-recover-transient",
]
def build_shadow_builder_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
updates = [
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "1"},
]
return merge_env_list(existing_env, updates)
def build_api_cutover_env_updates() -> list[dict[str, Any]]:
return [
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "1"},
{"name": "QUERY_REPLICA_REPORTS", "value": "1"},
{"name": "QUERY_REPLICA_COUNTS", "value": "1"},
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
{
"name": "QUERY_READ_MONGO_URI",
"valueFrom": {
"secretKeyRef": {
"name": READ_CLIENT_SECRET_NAME,
"key": "api-uri",
}
},
},
{
"name": "QUERY_VALKEY_URL",
"valueFrom": {
"secretKeyRef": {
"name": READ_CLIENT_SECRET_NAME,
"key": "valkey-url",
}
},
},
]
def build_api_rollback_env_updates() -> list[dict[str, Any]]:
return [
{"name": "QUERY_REPLICA_REPORTS", "value": "0"},
{"name": "QUERY_REPLICA_COUNTS", "value": "0"},
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "0"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
]
def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) -> dict[str, Any]:
return {
"apiVersion": "v1",
"kind": "Secret",
"metadata": {
"name": READ_CLIENT_SECRET_NAME,
"namespace": SOURCE_NAMESPACE,
},
"type": "Opaque",
"data": {
"api-uri": api_uri_b64,
"valkey-url": valkey_url_b64,
},
}
def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
if not isinstance(global_doc, dict):
return False, {"hasProduction": False, "hasWorkforce": False}
heads = global_doc.get("heads") or {}
if not isinstance(heads, dict):
return False, {"hasProduction": False, "hasWorkforce": False}
has_production = False
has_workforce = False
workforce_epoch = 0
workforce_as_of = None
production_as_of = None
for head in heads.values():
if not isinstance(head, dict):
continue
gen = head.get("generation") or ""
as_of = head.get("asOf")
if gen == "workforce-facts/v1":
epoch = int(head.get("factEpoch") or 0)
if epoch >= 1 and as_of:
has_workforce = True
workforce_epoch = max(workforce_epoch, epoch)
workforce_as_of = as_of
elif gen and not head.get("searchCollection") and int(head.get("chunks") or 0) >= 1 and as_of:
has_production = True
production_as_of = as_of
details = {
"hasProduction": has_production,
"hasWorkforce": has_workforce,
"workforceFactEpoch": workforce_epoch,
"workforceAsOf": workforce_as_of,
"productionAsOf": production_as_of,
}
return (has_production and has_workforce), details
def evaluate_parity_snapshot(
snapshot: dict[str, Any],
*,
now: dt.datetime | None = None,
max_observed_age_seconds: int = 300,
max_workforce_age_seconds: int = 900,
) -> tuple[bool, list[str]]:
current_time = now or utc_now()
reasons: list[str] = []
state = snapshot.get("state") or {}
if state.get("ready") is not True:
reasons.append("replica state.ready is not true")
observed_dt = parse_iso_datetime(state.get("observedAt"))
if observed_dt is None:
reasons.append("replica state.observedAt missing or invalid")
else:
obs_age = (current_time - observed_dt).total_seconds()
if obs_age < -5 or obs_age >= max_observed_age_seconds:
reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s")
heads_ok, head_info = evaluate_builder_heads(snapshot.get("global"))
if not heads_ok or int(head_info.get("workforceFactEpoch") or 0) < 1:
reasons.append("builder workforce or production head incomplete")
else:
wf_dt = parse_iso_datetime(head_info.get("workforceAsOf"))
if wf_dt is None:
reasons.append("workforce head asOf missing or invalid")
else:
wf_age = (current_time - wf_dt).total_seconds()
if wf_age < -5 or wf_age >= max_workforce_age_seconds:
reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s")
if int(snapshot.get("poolBucketsCount") or 0) <= 0:
reasons.append("queryPoolCountBuckets is empty")
if int(snapshot.get("workforceFactsCount") or 0) <= 0:
reasons.append("queryWorkforceFacts is empty")
return len(reasons) == 0, reasons
def count_error_log_lines(log_text: str) -> int:
return sum(1 for line in log_text.splitlines() if ERROR_LOG_PATTERN.search(line))
class ClusterClient:
"""Thin kubectl wrapper enforcing kubeconfig, timeouts, and non-secret output."""
def __init__(self, kubeconfig: str) -> None:
self.kubeconfig = kubeconfig
def run(
self,
args: list[str],
*,
stdin_text: str | None = None,
timeout: int = 30,
check: bool = True,
) -> subprocess.CompletedProcess[str]:
cmd = ["kubectl", "--kubeconfig", self.kubeconfig, *args]
proc = subprocess.run(
cmd,
input=stdin_text,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
if check and proc.returncode != 0:
raise RuntimeError(
f"kubectl failed rc={proc.returncode} args={' '.join(args[:4])}: {proc.stderr.strip()[:240]}"
)
return proc
def get_json(self, namespace: str | None, *resource_args: str, ignore_not_found: bool = False) -> Any:
args = []
if namespace:
args.extend(["-n", namespace])
args.extend(["get", *resource_args, "-o", "json"])
if ignore_not_found:
args.append("--ignore-not-found")
proc = self.run(args)
if ignore_not_found and not proc.stdout.strip():
return None
return json.loads(proc.stdout)
def patch_json(self, namespace: str, kind: str, name: str, patch_obj: dict[str, Any]) -> None:
self.run(
[
"-n",
namespace,
"patch",
kind,
name,
"--type=strategic",
"-p",
json.dumps(patch_obj),
]
)
def scale(self, namespace: str, kind: str, name: str, replicas: int) -> None:
self.run(["-n", namespace, "scale", f"{kind}/{name}", f"--replicas={replicas}"])
def rollout_status(self, namespace: str, deployment: str, timeout_seconds: int = 180) -> bool:
proc = self.run(
["-n", namespace, "rollout", "status", f"deployment/{deployment}", f"--timeout={timeout_seconds}s"],
timeout=timeout_seconds + 15,
check=False,
)
return proc.returncode == 0
def wait_zero_pods(self, namespace: str, label_selector: str, timeout_seconds: int = 90) -> None:
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
pods = self.get_json(namespace, "pods", "-l", label_selector)
if not pods.get("items"):
return
time.sleep(2)
raise RuntimeError(f"timed out waiting for {namespace} pods {label_selector} to scale to zero")
def query_shadow_mongo_snapshot(client: ClusterClient) -> dict[str, Any]:
"""Inspect shadow control heads & derived counts inside query-mongo-0 without exposing creds."""
js = f"""
const fs = require('fs');
const q = db.getSiblingDB('{QUERY_DB_NAME}');
if (q.auth('shadow-health', fs.readFileSync('/health-credentials/health-password', 'utf8').trim()).ok !== 1) {{
quit(2);
}}
const st = q.queryReplicaControl.findOne({{_id: 'state'}});
const gl = q.queryReadModelControl.findOne({{_id: 'global'}});
const bootDone = q.queryReplicaControl.countDocuments({{_id: /^bootstrap\\//, complete: true}});
const poolBucketsCount = q.queryPoolCountBuckets.countDocuments({{}});
const workforceFactsCount = q.queryWorkforceFacts.countDocuments({{}});
const normHead = h => {{
if (!h) return null;
const out = Object.assign({{}}, h);
if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString();
return out;
}};
const normHeads = m => {{
const res = {{}};
for (const [k, v] of Object.entries(m || {{}})) res[k] = normHead(v);
return res;
}};
print(JSON.stringify({{
state: st ? {{
ready: Boolean(st.ready),
observedAt: st.observedAt instanceof Date ? st.observedAt.toISOString() : null,
factEpoch: Number(st.factEpoch || 0),
schema: st.schema || null
}} : null,
bootstrapCompleteCount: bootDone,
poolBucketsCount: poolBucketsCount,
workforceFactsCount: workforceFactsCount,
global: gl ? {{
productionRevision: Number(gl.productionRevision || 0),
observedAt: gl.observedAt instanceof Date ? gl.observedAt.toISOString() : null,
heads: normHeads(gl.heads)
}} : null
}}));
"""
proc = client.run(
[
"-n",
SHADOW_NAMESPACE,
"exec",
SHADOW_MONGO_POD,
"-c",
SHADOW_MONGO_CONTAINER,
"--",
"mongosh",
"--quiet",
"--host",
"127.0.0.1",
"--port",
"27129",
"--eval",
js,
],
timeout=25,
)
return json.loads(proc.stdout.strip().splitlines()[-1])
def inspect_api_health(client: ClusterClient) -> dict[str, Any]:
dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
pods = client.get_json(SOURCE_NAMESPACE, "pods", "-l", f"app={API_DEPLOYMENT}")
desired = int(dep["spec"].get("replicas") or 0)
ready_replicas = int(dep.get("status", {}).get("readyReplicas") or 0)
pod_states = []
total_restarts = 0
for pod in pods.get("items", []):
statuses = pod.get("status", {}).get("containerStatuses") or []
restarts = sum(int(cs.get("restartCount") or 0) for cs in statuses)
total_restarts += restarts
ready = bool(statuses) and all(bool(cs.get("ready")) for cs in statuses)
pod_states.append(
{
"name": pod["metadata"]["name"],
"uid": pod["metadata"]["uid"],
"ready": ready,
"restarts": restarts,
}
)
return {
"desiredReplicas": desired,
"readyReplicas": ready_replicas,
"totalRestarts": total_restarts,
"pods": pod_states,
}
def write_report(out_dir: Path, filename: str, payload: Any) -> Path:
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / filename
path.write_text(json.dumps(redact_secrets(payload), ensure_ascii=False, indent=2) + "\n")
return path
def cmd_preflight(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
shadow_deploys = client.get_json(SHADOW_NAMESPACE, "deployments")
shadow_sts = client.get_json(SHADOW_NAMESPACE, "statefulsets")
shadow_quotas = client.get_json(SHADOW_NAMESPACE, "resourcequotas")
shadow_netpol = client.get_json(SHADOW_NAMESPACE, "networkpolicies")
mongo_snap = query_shadow_mongo_snapshot(client)
api_health = inspect_api_health(client)
report = {
"at": utc_now().isoformat(),
"runId": args.run_id,
"apiHealth": api_health,
"shadowMongoSnapshot": mongo_snap,
"sourceDeployments": [
{
"name": d["metadata"]["name"],
"replicas": d["spec"].get("replicas"),
"readyReplicas": d.get("status", {}).get("readyReplicas", 0),
"images": [c.get("image") for c in d["spec"]["template"]["spec"]["containers"]],
}
for d in source_deploys.get("items", [])
],
"shadowWorkloads": [
{
"kind": w["kind"],
"name": w["metadata"]["name"],
"replicas": w["spec"].get("replicas"),
"readyReplicas": w.get("status", {}).get("readyReplicas", 0),
"images": [c.get("image") for c in w["spec"]["template"]["spec"]["containers"]],
}
for w in shadow_deploys.get("items", []) + shadow_sts.get("items", [])
],
"shadowQuotas": [
{
"name": q["metadata"]["name"],
"hard": q.get("spec", {}).get("hard"),
"used": q.get("status", {}).get("used"),
}
for q in shadow_quotas.get("items", [])
],
"shadowNetworkPolicies": [p["metadata"]["name"] for p in shadow_netpol.get("items", [])],
}
path = write_report(out_dir, "preflight.json", report)
print(json.dumps({"ok": True, "phase": "preflight", "report": str(path), "apiReady": api_health["readyReplicas"]}))
return 0
def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
baseline_health = inspect_api_health(client)
if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0:
raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}")
# Scale builder to 0 first to stay well inside ResourceQuota.
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={BUILDER_CONTAINER}")
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}")
rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT)
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
rep_patch = build_deployment_update_patch(
container_name=REPLICA_CONTAINER,
strategy_recreate=True,
image=args.replica_image,
args=build_shadow_replica_args(),
env_list=build_shadow_replica_env(rep_env),
)
client.patch_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1)
if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180):
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
raise RuntimeError("query-replica failed rollout")
sync_deadline = time.monotonic() + args.sync_timeout_seconds
synced_snap = None
while time.monotonic() < sync_deadline:
health = inspect_api_health(client)
if (
health["readyReplicas"] < baseline_health["desiredReplicas"]
or health["totalRestarts"] > baseline_health["totalRestarts"]
):
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
write_report(out_dir, "shadow-sync-aborted.json", {"reason": "api_degraded", "health": health})
raise RuntimeError(f"API health degraded during shadow-sync; scaled projector to 0: {health}")
snap = query_shadow_mongo_snapshot(client)
state = snap.get("state") or {}
if state.get("ready") is True:
obs = parse_iso_datetime(state.get("observedAt"))
if obs is not None and (utc_now() - obs).total_seconds() < 300:
synced_snap = snap
break
time.sleep(15)
if synced_snap is None:
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
raise RuntimeError("timed out waiting for query-replica ready state")
bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT)
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
bld_patch = build_deployment_update_patch(
container_name=BUILDER_CONTAINER,
strategy_recreate=True,
image=args.builder_image,
args=build_shadow_builder_args(),
env_list=build_shadow_builder_env(bld_env),
)
client.patch_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1)
if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180):
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
raise RuntimeError("query-read-builder failed rollout")
build_deadline = time.monotonic() + args.build_timeout_seconds
built_snap = None
while time.monotonic() < build_deadline:
health = inspect_api_health(client)
if (
health["readyReplicas"] < baseline_health["desiredReplicas"]
or health["totalRestarts"] > baseline_health["totalRestarts"]
):
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
write_report(out_dir, "shadow-build-aborted.json", {"reason": "api_degraded", "health": health})
raise RuntimeError(f"API health degraded during builder run; scaled builder to 0: {health}")
snap = query_shadow_mongo_snapshot(client)
ok, reasons = evaluate_parity_snapshot(snap)
if ok:
built_snap = snap
break
time.sleep(15)
if built_snap is None:
raise RuntimeError("timed out waiting for query-read-builder completion")
path = write_report(out_dir, "shadow-sync.json", {"at": utc_now().isoformat(), "snapshot": built_snap})
print(json.dumps({"ok": True, "phase": "shadow-sync", "report": str(path)}))
return 0
def cmd_verify_parity(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
snap = query_shadow_mongo_snapshot(client)
ok, reasons = evaluate_parity_snapshot(snap)
report = {
"at": utc_now().isoformat(),
"passed": ok,
"reasons": reasons,
"snapshot": snap,
}
path = write_report(out_dir, "verify-parity.json", report)
print(json.dumps({"ok": ok, "phase": "verify-parity", "reasons": reasons, "report": str(path)}))
return 0 if ok else 1
def sync_read_client_secret(client: ClusterClient) -> None:
conn_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CONNECTIONS_SECRET)
cache_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CACHE_AUTH_SECRET)
api_uri_b64 = (conn_secret.get("data") or {}).get("api-uri")
valkey_url_b64 = (cache_secret.get("data") or {}).get("url")
if not api_uri_b64 or not valkey_url_b64:
raise RuntimeError("required shadow read client secret keys missing")
manifest = build_read_client_secret_manifest(api_uri_b64, valkey_url_b64)
existing = client.get_json(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, ignore_not_found=True)
verb = "replace" if existing is not None else "create"
client.run([verb, "-f", "-"], stdin_text=json.dumps(manifest))
def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates())
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=rolled_env)
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180)
health = inspect_api_health(client)
path = write_report(
out_dir,
"rollback-api-flags.json",
{"at": utc_now().isoformat(), "rolloutOk": ok, "apiHealth": health},
)
print(json.dumps({"ok": ok, "phase": "rollback-api-flags", "report": str(path)}))
return 0 if ok else 1
def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
snap = query_shadow_mongo_snapshot(client)
ok, reasons = evaluate_parity_snapshot(snap)
if not ok:
raise RuntimeError(f"parity gate blocked cutover: {reasons}")
sync_read_client_secret(client)
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env})
next_env = merge_env_list(existing_env, build_api_cutover_env_updates())
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=next_env)
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180):
cmd_rollback_api_flags(args, client, out_dir)
raise RuntimeError("droboticflow-api rollout failed after flag cutover; rolled back")
health = inspect_api_health(client)
path = write_report(
out_dir,
"cutover-api-flags.json",
{"at": utc_now().isoformat(), "env": next_env, "apiHealth": health},
)
print(json.dumps({"ok": True, "phase": "cutover-api-flags", "report": str(path)}))
return 0
def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
baseline = inspect_api_health(client)
deadline = time.monotonic() + args.watch_seconds
samples: list[dict[str, Any]] = []
rolled_back = False
trigger_reason = ""
while time.monotonic() < deadline:
time.sleep(min(20, max(1, int(deadline - time.monotonic()))))
health = inspect_api_health(client)
log_proc = client.run(
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=25s", "--tail=300"],
timeout=15,
check=False,
)
error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "")
sample = {
"at": utc_now().isoformat(),
"readyReplicas": health["readyReplicas"],
"desiredReplicas": health["desiredReplicas"],
"totalRestarts": health["totalRestarts"],
"errorLines": error_lines,
}
samples.append(sample)
if health["readyReplicas"] < baseline["desiredReplicas"]:
trigger_reason = f"readyReplicas dropped: {health['readyReplicas']}/{baseline['desiredReplicas']}"
elif health["totalRestarts"] > baseline["totalRestarts"]:
trigger_reason = f"API pod restarts increased: {baseline['totalRestarts']} -> {health['totalRestarts']}"
elif error_lines >= args.max_error_lines:
trigger_reason = f"error log spike: {error_lines} lines in 25s window"
if trigger_reason:
rolled_back = True
cmd_rollback_api_flags(args, client, out_dir)
break
report = {
"at": utc_now().isoformat(),
"passed": not rolled_back,
"rolledBack": rolled_back,
"triggerReason": trigger_reason,
"samples": samples,
}
path = write_report(out_dir, "watch-and-guard.json", report)
print(json.dumps({"ok": not rolled_back, "phase": "watch-and-guard", "rolledBack": rolled_back, "report": str(path)}))
return 1 if rolled_back else 0
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.")
parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
parser.add_argument("--run-id", default=default_run_id())
parser.add_argument("--output-dir", default="")
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("preflight")
p_sync = sub.add_parser("shadow-sync")
p_sync.add_argument("--replica-image", required=True)
p_sync.add_argument("--builder-image", required=True)
p_sync.add_argument("--sync-timeout-seconds", type=int, default=1800)
p_sync.add_argument("--build-timeout-seconds", type=int, default=1200)
sub.add_parser("verify-parity")
sub.add_parser("cutover-api-flags")
p_watch = sub.add_parser("watch-and-guard")
p_watch.add_argument("--watch-seconds", type=int, default=900)
p_watch.add_argument("--max-error-lines", type=int, default=5)
sub.add_parser("rollback-api-flags")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
out_dir = (
Path(args.output_dir)
if args.output_dir
else Path("local-data") / "diagnostics" / "evening-cutover" / args.run_id
)
client = ClusterClient(args.kubeconfig)
handlers = {
"preflight": cmd_preflight,
"shadow-sync": cmd_shadow_sync,
"verify-parity": cmd_verify_parity,
"cutover-api-flags": cmd_cutover_api_flags,
"watch-and-guard": cmd_watch_and_guard,
"rollback-api-flags": cmd_rollback_api_flags,
}
return handlers[args.command](args, client, out_dir)
if __name__ == "__main__":
sys.exit(main())

View File

@ -0,0 +1,137 @@
import datetime as dt
import importlib.util
import json
from pathlib import Path
import unittest
def load_module():
path = Path(__file__).with_name("evening-query-cutover.py")
spec = importlib.util.spec_from_file_location("evening_query_cutover", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
CUTOVER = load_module()
class EveningQueryCutoverTests(unittest.TestCase):
def test_redact_secrets_scrubs_sensitive_fields_and_secrets(self):
payload = {
"kind": "Secret",
"metadata": {
"name": "droboticflow-query-read-client",
"annotations": {
"kubectl.kubernetes.io/last-applied-configuration": "secret-json"
},
},
"data": {"api-uri": "bW9uZ29kYjovL3NlY3JldA==", "valkey-url": "dmFsa2V5"},
"env": [
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
{"name": "MONGO_URI", "value": "mongodb://user:pass@host/db"},
{"name": "QUERY_VALKEY_URL", "value": "redis://:pwd@cache:6379"},
{
"name": "QUERY_READ_MONGO_URI",
"valueFrom": {"secretKeyRef": {"name": "s", "key": "api-uri"}},
},
],
}
redacted = CUTOVER.redact_secrets(payload)
self.assertEqual(redacted["data"]["api-uri"], "<redacted>")
self.assertEqual(redacted["data"]["valkey-url"], "<redacted>")
self.assertEqual(
redacted["metadata"]["annotations"]["kubectl.kubernetes.io/last-applied-configuration"],
"<redacted>",
)
env_by_name = {e["name"]: e for e in redacted["env"]}
self.assertEqual(env_by_name["MONGO_DB"]["value"], "drobodata-fanyu")
self.assertEqual(env_by_name["MONGO_URI"]["value"], "<redacted>")
self.assertEqual(env_by_name["QUERY_VALKEY_URL"]["value"], "<redacted>")
self.assertEqual(
env_by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["key"],
"api-uri",
)
serialized = json.dumps(redacted)
self.assertNotIn("user:pass", serialized)
self.assertNotIn("bW9uZ29k", serialized)
def test_merge_env_list_and_cutover_flags_contract(self):
existing = [
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
{"name": "QUERY_SOURCE_TRACKING", "value": "1"},
{"name": "QUERY_REPLICA_POOLS", "value": "1"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "1"},
]
merged = CUTOVER.merge_env_list(existing, CUTOVER.build_api_cutover_env_updates())
by_name = {e["name"]: e for e in merged}
self.assertEqual(by_name["MONGO_DB"]["value"], "drobodata-fanyu")
self.assertEqual(by_name["QUERY_SOURCE_TRACKING"]["value"], "1")
self.assertEqual(by_name["QUERY_PUBLISHED_RESULTS"]["value"], "0")
self.assertEqual(by_name["QUERY_REPLICA_POOLS"]["value"], "0")
self.assertEqual(by_name["QUERY_REPLICA_REPORTS"]["value"], "1")
self.assertEqual(by_name["QUERY_REPLICA_COUNTS"]["value"], "1")
self.assertEqual(by_name["QUERY_SERVING_READS"]["value"], "1")
self.assertEqual(by_name["QUERY_REPLICA_REFRESH_MODE"]["value"], "periodic")
self.assertEqual(
by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["name"],
CUTOVER.READ_CLIENT_SECRET_NAME,
)
def test_deployment_patch_sets_recreate_and_clears_rolling_update(self):
patch = CUTOVER.build_deployment_update_patch(
strategy_recreate=True,
container_name="projector",
image="example/img@sha256:abc",
args=CUTOVER.build_shadow_replica_args(),
env_list=CUTOVER.build_shadow_replica_env([]),
)
strategy = patch["spec"]["strategy"]
self.assertEqual(strategy["type"], "Recreate")
self.assertIn("rollingUpdate", strategy)
self.assertIsNone(strategy["rollingUpdate"])
container = patch["spec"]["template"]["spec"]["containers"][0]
self.assertEqual(container["name"], "projector")
self.assertIn("--source-compress", container["args"])
self.assertNotIn("--safety-run", container["args"])
def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self):
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
good = {
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3},
"poolBucketsCount": 42,
"workforceFactsCount": 1200,
"global": {
"heads": {
"h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"},
"h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"},
}
},
}
passed, reasons = CUTOVER.evaluate_parity_snapshot(good, now=now)
self.assertTrue(passed, reasons)
self.assertEqual(reasons, [])
stale = json.loads(json.dumps(good))
stale["state"]["observedAt"] = "2026-11-16T11:45:00Z"
stale["workforceFactsCount"] = 0
passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now)
self.assertFalse(passed)
self.assertEqual(len(reasons), 2)
def test_rollback_env_disables_all_query_reads(self):
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())
by_name = {e["name"]: e for e in rolled}
for key in (
"QUERY_REPLICA_REPORTS",
"QUERY_REPLICA_COUNTS",
"QUERY_REPLICA_POOLS",
"QUERY_SERVING_READS",
"QUERY_PUBLISHED_RESULTS",
):
self.assertEqual(by_name[key]["value"], "0")
if __name__ == "__main__":
unittest.main()