⚡ Streamline unattended query sync recovery and evening cutover
This commit is contained in:
parent
f7e4cb287e
commit
484e5d591b
@ -21,6 +21,7 @@ func main() {
|
||||
once := flag.Bool("once", false, "build once; streaming freshness expires after exit")
|
||||
ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building")
|
||||
shadowRun := flag.String("shadow-safety-run", "", "opt-in independently monitored shadow builder; no authority gates")
|
||||
autoRecover := flag.Bool("auto-recover-transient", false, "automatically cool down and clear transient build failure fences in unattended mode")
|
||||
flag.Parse()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
@ -90,6 +91,26 @@ func main() {
|
||||
for workCtx.Err() == nil {
|
||||
if err := b.Run(workCtx); err != nil && workCtx.Err() == nil {
|
||||
if errors.Is(err, store.ErrQueryBuildStopped) {
|
||||
if *autoRecover && *shadowRun == "" {
|
||||
log.Print("query builder hit stopped fence; cooling down before checking transient recovery")
|
||||
cooldown := time.NewTimer(15 * time.Second)
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
cooldown.Stop()
|
||||
return
|
||||
case <-cooldown.C:
|
||||
}
|
||||
resetCtx, cancel := context.WithTimeout(workCtx, 5*time.Second)
|
||||
cleared, clearErr := s.ResetTransientQueryBuildFailures(resetCtx)
|
||||
cancel()
|
||||
if clearErr == nil && cleared > 0 {
|
||||
log.Print("query builder cleared transient failure fence; resuming checkpoint")
|
||||
continue
|
||||
}
|
||||
log.Print("query builder stopped on non-transient failure; checkpoint retained; explicit recovery required")
|
||||
<-workCtx.Done()
|
||||
return
|
||||
}
|
||||
log.Print("query builder stopped; checkpoint retained; explicit recovery required")
|
||||
<-workCtx.Done()
|
||||
return
|
||||
|
||||
@ -43,6 +43,7 @@ func main() {
|
||||
status := flag.Bool("safety-status", false, "target-only private safety status")
|
||||
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
|
||||
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
|
||||
sourceCompress := flag.Bool("source-compress", false, "negotiate snappy/zstd wire compression on unguarded source reads")
|
||||
flag.Parse()
|
||||
var maintenanceDeadline *time.Time
|
||||
if *maintenance {
|
||||
@ -176,6 +177,12 @@ func main() {
|
||||
log.Fatal("explicit source/query connection required")
|
||||
}
|
||||
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
|
||||
if !target && transport == nil {
|
||||
opts.SetMaxPoolSize(2)
|
||||
if *sourceCompress {
|
||||
opts.SetCompressors([]string{"snappy", "zstd"})
|
||||
}
|
||||
}
|
||||
if !target && transport != nil {
|
||||
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors(sourceCompressors(*maintenance)).SetRetryReads(false).SetTimeout(5 * time.Second)
|
||||
}
|
||||
@ -203,12 +210,15 @@ func main() {
|
||||
if *ensure && q.EnsureSourceIndexes(ctx) != nil {
|
||||
log.Fatal("source index preparation failed")
|
||||
}
|
||||
for {
|
||||
for attempts := 0; ; attempts++ {
|
||||
err := q.Acquire(ctx)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if *safetyRun == "" || !errors.Is(err, store.ErrQueryReplicaLease) {
|
||||
if *once || !errors.Is(err, store.ErrQueryReplicaLease) {
|
||||
log.Fatal("projector lease unavailable")
|
||||
}
|
||||
if *safetyRun == "" && attempts >= 35 {
|
||||
log.Fatal("projector lease unavailable")
|
||||
}
|
||||
select {
|
||||
@ -276,6 +286,7 @@ func main() {
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
consecutiveFailures := 0
|
||||
for workCtx.Err() == nil {
|
||||
started := time.Now()
|
||||
ready, e := q.RunOnce(workCtx, *bootstrap)
|
||||
@ -285,6 +296,21 @@ func main() {
|
||||
}
|
||||
d := store.QueryReplicaDiagnostic(e)
|
||||
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
|
||||
if *safetyRun == "" && !*once {
|
||||
wait := daemonRetryDelay(consecutiveFailures)
|
||||
consecutiveFailures++
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
if errors.Is(e, store.ErrQueryReplicaLease) {
|
||||
_ = q.Acquire(workCtx)
|
||||
}
|
||||
continue
|
||||
}
|
||||
short, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if q.PauseSafetyFailure(short, e) != nil {
|
||||
log.Print("pause diagnostics persistence unavailable; startup fencing remains required")
|
||||
@ -297,6 +323,7 @@ func main() {
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
consecutiveFailures = 0
|
||||
if *once {
|
||||
if !ready {
|
||||
pause("incomplete_pilot")
|
||||
@ -307,9 +334,29 @@ func main() {
|
||||
}
|
||||
// Optional pacing covers backfill too. Idle lease maintenance only
|
||||
// touches the independent query DB, never scans authority for keepalive.
|
||||
if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 {
|
||||
delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass)
|
||||
if *safetyRun == "" {
|
||||
if cool := adaptiveBackfillCooldown(ready, time.Since(started)); cool > delay {
|
||||
delay = cool
|
||||
}
|
||||
}
|
||||
if delay > 0 {
|
||||
if e := q.WaitNextPass(workCtx, delay); e != nil {
|
||||
if ctx.Err() == nil {
|
||||
if *safetyRun == "" && !*once {
|
||||
log.Print("projector lease refresh retry")
|
||||
wait := daemonRetryDelay(consecutiveFailures)
|
||||
consecutiveFailures++
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
_ = q.Acquire(workCtx)
|
||||
continue
|
||||
}
|
||||
pause("target_error")
|
||||
log.Print("projector lease unavailable; checkpoint retained")
|
||||
if *safetyRun != "" {
|
||||
|
||||
@ -23,3 +23,24 @@ func passDelay(mode string, ready bool, elapsed, interval, minimum time.Duration
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
func daemonRetryDelay(attempt int) time.Duration {
|
||||
if attempt <= 0 {
|
||||
return 2 * time.Second
|
||||
}
|
||||
if attempt > 10 {
|
||||
attempt = 10
|
||||
}
|
||||
delay := 2 * time.Second << attempt
|
||||
if delay > 15*time.Second || delay <= 0 {
|
||||
return 15 * time.Second
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
func adaptiveBackfillCooldown(ready bool, elapsed time.Duration) time.Duration {
|
||||
if !ready && elapsed >= 2*time.Second {
|
||||
return 500 * time.Millisecond
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -40,3 +40,35 @@ func TestPacingRejectsUnboundedOrInvalidConfiguration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDaemonRetryDelayAndAdaptiveBackfillCooldown(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
attempt int
|
||||
want time.Duration
|
||||
}{
|
||||
{-1, 2 * time.Second},
|
||||
{0, 2 * time.Second},
|
||||
{1, 4 * time.Second},
|
||||
{2, 8 * time.Second},
|
||||
{3, 15 * time.Second},
|
||||
{10, 15 * time.Second},
|
||||
} {
|
||||
if got := daemonRetryDelay(tc.attempt); got != tc.want {
|
||||
t.Fatalf("attempt=%d got=%s want=%s", tc.attempt, got, tc.want)
|
||||
}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
ready bool
|
||||
elapsed time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{true, 5 * time.Second, 0},
|
||||
{false, 1999 * time.Millisecond, 0},
|
||||
{false, 2 * time.Second, 500 * time.Millisecond},
|
||||
{false, 4 * time.Second, 500 * time.Millisecond},
|
||||
} {
|
||||
if got := adaptiveBackfillCooldown(tc.ready, tc.elapsed); got != tc.want {
|
||||
t.Fatalf("ready=%v elapsed=%s got=%s want=%s", tc.ready, tc.elapsed, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -204,3 +204,57 @@ func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var transientQueryBuildFailureClasses = []string{
|
||||
"source_timeout",
|
||||
"target_timeout",
|
||||
"timeout",
|
||||
"deadline",
|
||||
"network",
|
||||
"lease_lost",
|
||||
"lease",
|
||||
"no_durable_progress",
|
||||
"transient",
|
||||
}
|
||||
|
||||
func isTransientQueryBuildFailureClass(class string) bool {
|
||||
for _, candidate := range transientQueryBuildFailureClasses {
|
||||
if class == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ResetTransientQueryBuildFailures clears transient failure fences for active
|
||||
// query build jobs while preserving durable checkpoints and schema/content blocks.
|
||||
func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int64, error) {
|
||||
if s == nil || s.db == nil {
|
||||
return 0, errors.New("store unavailable")
|
||||
}
|
||||
classes := bson.A{}
|
||||
for _, c := range transientQueryBuildFailureClasses {
|
||||
classes = append(classes, c)
|
||||
}
|
||||
filter := bson.M{
|
||||
"schema": queryBuildJobSchema,
|
||||
"phase": bson.M{"$ne": "complete"},
|
||||
"failureClass": bson.M{"$in": classes},
|
||||
}
|
||||
update := bson.M{
|
||||
"$unset": bson.M{
|
||||
"failurePhase": "",
|
||||
"failureClass": "",
|
||||
"failureCode": "",
|
||||
},
|
||||
"$set": bson.M{
|
||||
"activeWithoutProgress": int64(0),
|
||||
"progressAt": time.Now().UTC(),
|
||||
},
|
||||
}
|
||||
res, err := s.db.Collection(queryBuildJobsCollection).UpdateMany(ctx, filter, update)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.ModifiedCount, nil
|
||||
}
|
||||
|
||||
33
internal/store/query_build_transient_test.go
Normal file
33
internal/store/query_build_transient_test.go
Normal file
@ -0,0 +1,33 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsTransientQueryBuildFailureClass(t *testing.T) {
|
||||
for _, class := range []string{
|
||||
"source_timeout",
|
||||
"target_timeout",
|
||||
"timeout",
|
||||
"deadline",
|
||||
"network",
|
||||
"lease_lost",
|
||||
"lease",
|
||||
"no_durable_progress",
|
||||
"transient",
|
||||
} {
|
||||
if !isTransientQueryBuildFailureClass(class) {
|
||||
t.Fatalf("expected transient class %q to return true", class)
|
||||
}
|
||||
}
|
||||
for _, class := range []string{
|
||||
"",
|
||||
"unsupported_record",
|
||||
"unsupported_fact",
|
||||
"corrupt_state",
|
||||
"auth_failed",
|
||||
"bson_decode",
|
||||
} {
|
||||
if isTransientQueryBuildFailureClass(class) {
|
||||
t.Fatalf("expected non-transient class %q to return false", class)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -18,7 +18,10 @@ import (
|
||||
)
|
||||
|
||||
const queryCountMemoryBudget = 64 << 20
|
||||
const queryCountTTL = 2 * time.Second
|
||||
|
||||
// Cache keys include factEpoch so synced mutations invalidate immediately; a
|
||||
// 15s TTL avoids 2s re-fetch/decompression thrashing while epochs are unchanged.
|
||||
const queryCountTTL = 15 * time.Second
|
||||
|
||||
var errQueryCountCacheCapacity = errors.New("count candidate cache capacity exceeded")
|
||||
|
||||
@ -75,6 +78,13 @@ func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, by
|
||||
if bytes > queryCountMemoryBudget {
|
||||
return
|
||||
}
|
||||
if c.entries == nil {
|
||||
c.entries = map[string]queryCountEntry{}
|
||||
}
|
||||
if prev, ok := c.entries[key]; ok {
|
||||
delete(c.entries, key)
|
||||
c.bytes -= prev.bytes
|
||||
}
|
||||
now := time.Now()
|
||||
for k, e := range c.entries {
|
||||
if !now.Before(e.expires) {
|
||||
@ -82,8 +92,24 @@ func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, by
|
||||
c.bytes -= e.bytes
|
||||
}
|
||||
}
|
||||
if len(c.entries) >= 32 || c.bytes+bytes > queryCountMemoryBudget {
|
||||
clear(c.entries)
|
||||
for len(c.entries) >= 32 || (len(c.entries) > 0 && c.bytes+bytes > queryCountMemoryBudget) {
|
||||
var oldestKey string
|
||||
var oldestExpires time.Time
|
||||
first := true
|
||||
for k, e := range c.entries {
|
||||
if first || e.expires.Before(oldestExpires) {
|
||||
oldestKey = k
|
||||
oldestExpires = e.expires
|
||||
first = false
|
||||
}
|
||||
}
|
||||
if oldestKey == "" {
|
||||
break
|
||||
}
|
||||
c.bytes -= c.entries[oldestKey].bytes
|
||||
delete(c.entries, oldestKey)
|
||||
}
|
||||
if c.bytes < 0 {
|
||||
c.bytes = 0
|
||||
}
|
||||
c.entries[key] = queryCountEntry{rows: rows, bytes: bytes, expires: now.Add(queryCountTTL)}
|
||||
|
||||
@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryReplicaCountOnlyLeavesListsAndCoreReadsOnAuthority(t *testing.T) {
|
||||
@ -78,3 +80,50 @@ func TestQueryReplicaMissingConfigurationCannotPanicCoreCache(t *testing.T) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryCountCacheEvictsOldestInsteadOfClearingAll(t *testing.T) {
|
||||
c := &queryCountCache{}
|
||||
c.remember("k0", []domain.HumanItemRecord{{ID: "0"}}, 1024)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
for i := 1; i < 32; i++ {
|
||||
c.remember(fmt.Sprintf("k%d", i), []domain.HumanItemRecord{{ID: fmt.Sprintf("%d", i)}}, 1024)
|
||||
}
|
||||
if len(c.entries) != 32 || c.bytes != 32*1024 {
|
||||
t.Fatalf("unexpected cache size=%d bytes=%d", len(c.entries), c.bytes)
|
||||
}
|
||||
c.remember("k32", []domain.HumanItemRecord{{ID: "32"}}, 1024)
|
||||
if len(c.entries) != 32 {
|
||||
t.Fatalf("expected 32 entries after eviction, got %d", len(c.entries))
|
||||
}
|
||||
if c.bytes != 32*1024 {
|
||||
t.Fatalf("expected bytes=%d, got %d", 32*1024, c.bytes)
|
||||
}
|
||||
if _, ok := c.entries["k0"]; ok {
|
||||
t.Fatal("expected oldest entry k0 to be evicted")
|
||||
}
|
||||
if _, ok := c.entries["k1"]; !ok {
|
||||
t.Fatal("expected newer entry k1 to stay cached")
|
||||
}
|
||||
if _, ok := c.entries["k32"]; !ok {
|
||||
t.Fatal("expected newest entry k32 to be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterAuthoritativeAdminPoolPageDropsConcurrentlyClaimedItems(t *testing.T) {
|
||||
q := AdminAnnotationPoolQuery{Pool: "available"}
|
||||
candidates := []domain.HumanItemRecord{
|
||||
{ID: "a"},
|
||||
{ID: "claimed"},
|
||||
{ID: "missing"},
|
||||
{ID: "b"},
|
||||
}
|
||||
hydrated := []domain.HumanItemRecord{
|
||||
{ID: "b", CurrentStageID: "semantic_annotation", Status: domain.HumanItemReady},
|
||||
{ID: "claimed", CurrentStageID: "semantic_annotation", Status: domain.HumanItemClaimed, AssignedUserID: "annotator-1"},
|
||||
{ID: "a", CurrentStageID: "semantic_annotation", Status: domain.HumanItemReady},
|
||||
}
|
||||
got := filterAuthoritativeAdminPoolPage(candidates, hydrated, q)
|
||||
if len(got) != 2 || got[0].ID != "a" || got[1].ID != "b" {
|
||||
t.Fatalf("expected [a b] in candidate order without failing page, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@ -175,29 +175,76 @@ func (s *MongoStore) loadReplicaPoolPage(ctx context.Context, key, org string, q
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
|
||||
rows, e := target.ListAdminAnnotationPool(ctx, org, q)
|
||||
if e != nil || len(rows) == 0 {
|
||||
return rows, e
|
||||
func filterAuthoritativeAdminPoolPage(candidates, hydrated []domain.HumanItemRecord, q AdminAnnotationPoolQuery) []domain.HumanItemRecord {
|
||||
byID := make(map[string]domain.HumanItemRecord, len(hydrated))
|
||||
for _, item := range hydrated {
|
||||
byID[item.ID] = item
|
||||
}
|
||||
ids := make([]string, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(current) != len(rows) {
|
||||
return nil, ErrQueryReplicaNotReady
|
||||
}
|
||||
if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
for _, item := range current {
|
||||
if !adminPoolScope(item) || !adminPoolSelected(item, q) {
|
||||
return nil, ErrQueryReplicaNotReady
|
||||
out := make([]domain.HumanItemRecord, 0, len(candidates))
|
||||
for _, cand := range candidates {
|
||||
item, ok := byID[cand.ID]
|
||||
if !ok || !adminPoolScope(item) || !adminPoolSelected(item, q) {
|
||||
continue
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return current, nil
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
|
||||
collected := []domain.HumanItemRecord{}
|
||||
seen := map[string]bool{}
|
||||
stepQ := q
|
||||
for pass := 0; pass < 4; pass++ {
|
||||
if q.Limit > 0 {
|
||||
need := q.Limit - len(collected)
|
||||
if need <= 0 {
|
||||
break
|
||||
}
|
||||
if need > stepQ.Limit {
|
||||
stepQ.Limit = need
|
||||
} else if stepQ.Limit < q.Limit {
|
||||
stepQ.Limit = q.Limit
|
||||
}
|
||||
}
|
||||
rows, e := target.ListAdminAnnotationPool(ctx, org, stepQ)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
ids := make([]string, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(current) > 0 {
|
||||
if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
}
|
||||
for _, item := range filterAuthoritativeAdminPoolPage(rows, current, q) {
|
||||
if seen[item.ID] {
|
||||
continue
|
||||
}
|
||||
seen[item.ID] = true
|
||||
collected = append(collected, item)
|
||||
if q.Limit > 0 && len(collected) >= q.Limit {
|
||||
return collected, nil
|
||||
}
|
||||
}
|
||||
if q.Limit <= 0 || len(rows) < stepQ.Limit {
|
||||
break
|
||||
}
|
||||
last := rows[len(rows)-1]
|
||||
if last.ID == "" {
|
||||
break
|
||||
}
|
||||
stepQ.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
||||
}
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
813
scripts/local/evening-query-cutover.py
Normal file
813
scripts/local/evening-query-cutover.py
Normal file
@ -0,0 +1,813 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unattended evening shadow-sync, parity gate, API read cutover, and auto-rollback."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_KUBECONFIG = "/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf"
|
||||
SOURCE_NAMESPACE = "droboticflow-fanyu"
|
||||
SHADOW_NAMESPACE = "droboticflow-query-shadow"
|
||||
QUERY_DB_NAME = "drobodata_fanyu_query_shadow_20261114"
|
||||
QUERY_SERVING_NAMESPACE = "fanyu-query-shadow-20261114"
|
||||
READ_CLIENT_SECRET_NAME = "droboticflow-query-read-client"
|
||||
SHADOW_CONNECTIONS_SECRET = "query-shadow-query-connections"
|
||||
SHADOW_CACHE_AUTH_SECRET = "query-shadow-cache-auth"
|
||||
|
||||
API_DEPLOYMENT = "droboticflow-api"
|
||||
API_CONTAINER = "api"
|
||||
REPLICA_DEPLOYMENT = "query-replica"
|
||||
REPLICA_CONTAINER = "projector"
|
||||
BUILDER_DEPLOYMENT = "query-read-builder"
|
||||
BUILDER_CONTAINER = "builder"
|
||||
SHADOW_MONGO_POD = "query-mongo-0"
|
||||
SHADOW_MONGO_CONTAINER = "mongo"
|
||||
|
||||
SECRET_KEY_PATTERN = re.compile(
|
||||
r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ERROR_LOG_PATTERN = re.compile(
|
||||
r"(ErrQueryReplicaNotReady|查询暂不可用|\bstatus=(?:500|503)\b|\"\s+(?:500|503)\s+)",
|
||||
)
|
||||
|
||||
|
||||
def utc_now() -> dt.datetime:
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def default_run_id() -> str:
|
||||
return utc_now().strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
|
||||
def parse_iso_datetime(raw: Any) -> dt.datetime | None:
|
||||
if not isinstance(raw, str) or not raw.strip():
|
||||
return None
|
||||
text = raw.strip()
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
try:
|
||||
parsed = dt.datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
||||
return parsed.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
def redact_secrets(obj: Any, parent_key: str = "") -> Any:
|
||||
"""Recursively scrub secret payloads, URIs, and credentials before persistence/logging."""
|
||||
if isinstance(obj, dict):
|
||||
kind = obj.get("kind")
|
||||
if kind == "Secret":
|
||||
out = {}
|
||||
for k, v in obj.items():
|
||||
if k in ("data", "stringData") and isinstance(v, dict):
|
||||
out[k] = {sk: "<redacted>" for sk in v.keys()}
|
||||
elif k == "metadata" and isinstance(v, dict):
|
||||
meta = dict(v)
|
||||
if isinstance(meta.get("annotations"), dict):
|
||||
anns = dict(meta["annotations"])
|
||||
if "kubectl.kubernetes.io/last-applied-configuration" in anns:
|
||||
anns["kubectl.kubernetes.io/last-applied-configuration"] = "<redacted>"
|
||||
meta["annotations"] = anns
|
||||
out[k] = meta
|
||||
else:
|
||||
out[k] = redact_secrets(v, k)
|
||||
return out
|
||||
# Redact Kubernetes env var literal values if key name looks sensitive.
|
||||
if "name" in obj and "value" in obj and isinstance(obj.get("name"), str):
|
||||
env_name = obj["name"]
|
||||
if SECRET_KEY_PATTERN.search(env_name):
|
||||
redacted = dict(obj)
|
||||
redacted["value"] = "<redacted>"
|
||||
return redacted
|
||||
return {k: redact_secrets(v, k) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [redact_secrets(item, parent_key) for item in obj]
|
||||
if isinstance(obj, str) and parent_key and SECRET_KEY_PATTERN.search(parent_key):
|
||||
return "<redacted>"
|
||||
return obj
|
||||
|
||||
|
||||
def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Merge env updates into existing Kubernetes env list preserving untouched vars and order."""
|
||||
by_name = {entry["name"]: dict(entry) for entry in updates}
|
||||
seen: set[str] = set()
|
||||
merged: list[dict[str, Any]] = []
|
||||
for entry in existing_env or []:
|
||||
name = entry.get("name")
|
||||
if not isinstance(name, str) or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
if name in by_name:
|
||||
merged.append(by_name[name])
|
||||
else:
|
||||
merged.append(dict(entry))
|
||||
for entry in updates:
|
||||
name = entry["name"]
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
merged.append(dict(entry))
|
||||
return merged
|
||||
|
||||
|
||||
def build_deployment_update_patch(
|
||||
*,
|
||||
container_name: str,
|
||||
env_list: list[dict[str, Any]],
|
||||
strategy_recreate: bool = False,
|
||||
image: str | None = None,
|
||||
args: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a Kubernetes strategic-merge patch for a single deployment container."""
|
||||
container_patch: dict[str, Any] = {
|
||||
"name": container_name,
|
||||
"env": env_list,
|
||||
}
|
||||
if image is not None:
|
||||
container_patch["image"] = image
|
||||
if args is not None:
|
||||
container_patch["args"] = list(args)
|
||||
spec_patch: dict[str, Any] = {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [container_patch],
|
||||
}
|
||||
}
|
||||
}
|
||||
if strategy_recreate:
|
||||
# Clearing rollingUpdate is required when switching from RollingUpdate -> Recreate.
|
||||
spec_patch["strategy"] = {"type": "Recreate", "rollingUpdate": None}
|
||||
return {"spec": spec_patch}
|
||||
|
||||
|
||||
def build_shadow_replica_args() -> list[str]:
|
||||
return [
|
||||
"--writers-covered",
|
||||
"--bootstrap",
|
||||
"--source-compress",
|
||||
"--batch-size=100",
|
||||
"--interval=30s",
|
||||
]
|
||||
|
||||
|
||||
def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
updates = [
|
||||
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
|
||||
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
|
||||
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
|
||||
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
||||
{"name": "QUERY_SERVING_READS", "value": "1"},
|
||||
]
|
||||
return merge_env_list(existing_env, updates)
|
||||
|
||||
|
||||
def build_shadow_builder_args() -> list[str]:
|
||||
return [
|
||||
"--ensure-indexes",
|
||||
"--auto-recover-transient",
|
||||
]
|
||||
|
||||
|
||||
def build_shadow_builder_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
updates = [
|
||||
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
|
||||
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
|
||||
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
|
||||
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
||||
{"name": "QUERY_SERVING_READS", "value": "1"},
|
||||
]
|
||||
return merge_env_list(existing_env, updates)
|
||||
|
||||
|
||||
def build_api_cutover_env_updates() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
|
||||
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
|
||||
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
|
||||
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
||||
{"name": "QUERY_SERVING_READS", "value": "1"},
|
||||
{"name": "QUERY_REPLICA_REPORTS", "value": "1"},
|
||||
{"name": "QUERY_REPLICA_COUNTS", "value": "1"},
|
||||
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
|
||||
{
|
||||
"name": "QUERY_READ_MONGO_URI",
|
||||
"valueFrom": {
|
||||
"secretKeyRef": {
|
||||
"name": READ_CLIENT_SECRET_NAME,
|
||||
"key": "api-uri",
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "QUERY_VALKEY_URL",
|
||||
"valueFrom": {
|
||||
"secretKeyRef": {
|
||||
"name": READ_CLIENT_SECRET_NAME,
|
||||
"key": "valkey-url",
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def build_api_rollback_env_updates() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": "QUERY_REPLICA_REPORTS", "value": "0"},
|
||||
{"name": "QUERY_REPLICA_COUNTS", "value": "0"},
|
||||
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
|
||||
{"name": "QUERY_SERVING_READS", "value": "0"},
|
||||
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
||||
]
|
||||
|
||||
|
||||
def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) -> dict[str, Any]:
|
||||
return {
|
||||
"apiVersion": "v1",
|
||||
"kind": "Secret",
|
||||
"metadata": {
|
||||
"name": READ_CLIENT_SECRET_NAME,
|
||||
"namespace": SOURCE_NAMESPACE,
|
||||
},
|
||||
"type": "Opaque",
|
||||
"data": {
|
||||
"api-uri": api_uri_b64,
|
||||
"valkey-url": valkey_url_b64,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
|
||||
if not isinstance(global_doc, dict):
|
||||
return False, {"hasProduction": False, "hasWorkforce": False}
|
||||
heads = global_doc.get("heads") or {}
|
||||
if not isinstance(heads, dict):
|
||||
return False, {"hasProduction": False, "hasWorkforce": False}
|
||||
has_production = False
|
||||
has_workforce = False
|
||||
workforce_epoch = 0
|
||||
workforce_as_of = None
|
||||
production_as_of = None
|
||||
for head in heads.values():
|
||||
if not isinstance(head, dict):
|
||||
continue
|
||||
gen = head.get("generation") or ""
|
||||
as_of = head.get("asOf")
|
||||
if gen == "workforce-facts/v1":
|
||||
epoch = int(head.get("factEpoch") or 0)
|
||||
if epoch >= 1 and as_of:
|
||||
has_workforce = True
|
||||
workforce_epoch = max(workforce_epoch, epoch)
|
||||
workforce_as_of = as_of
|
||||
elif gen and not head.get("searchCollection") and int(head.get("chunks") or 0) >= 1 and as_of:
|
||||
has_production = True
|
||||
production_as_of = as_of
|
||||
details = {
|
||||
"hasProduction": has_production,
|
||||
"hasWorkforce": has_workforce,
|
||||
"workforceFactEpoch": workforce_epoch,
|
||||
"workforceAsOf": workforce_as_of,
|
||||
"productionAsOf": production_as_of,
|
||||
}
|
||||
return (has_production and has_workforce), details
|
||||
|
||||
|
||||
def evaluate_parity_snapshot(
|
||||
snapshot: dict[str, Any],
|
||||
*,
|
||||
now: dt.datetime | None = None,
|
||||
max_observed_age_seconds: int = 300,
|
||||
max_workforce_age_seconds: int = 900,
|
||||
) -> tuple[bool, list[str]]:
|
||||
current_time = now or utc_now()
|
||||
reasons: list[str] = []
|
||||
|
||||
state = snapshot.get("state") or {}
|
||||
if state.get("ready") is not True:
|
||||
reasons.append("replica state.ready is not true")
|
||||
|
||||
observed_dt = parse_iso_datetime(state.get("observedAt"))
|
||||
if observed_dt is None:
|
||||
reasons.append("replica state.observedAt missing or invalid")
|
||||
else:
|
||||
obs_age = (current_time - observed_dt).total_seconds()
|
||||
if obs_age < -5 or obs_age >= max_observed_age_seconds:
|
||||
reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s")
|
||||
|
||||
heads_ok, head_info = evaluate_builder_heads(snapshot.get("global"))
|
||||
if not heads_ok or int(head_info.get("workforceFactEpoch") or 0) < 1:
|
||||
reasons.append("builder workforce or production head incomplete")
|
||||
else:
|
||||
wf_dt = parse_iso_datetime(head_info.get("workforceAsOf"))
|
||||
if wf_dt is None:
|
||||
reasons.append("workforce head asOf missing or invalid")
|
||||
else:
|
||||
wf_age = (current_time - wf_dt).total_seconds()
|
||||
if wf_age < -5 or wf_age >= max_workforce_age_seconds:
|
||||
reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s")
|
||||
|
||||
if int(snapshot.get("poolBucketsCount") or 0) <= 0:
|
||||
reasons.append("queryPoolCountBuckets is empty")
|
||||
if int(snapshot.get("workforceFactsCount") or 0) <= 0:
|
||||
reasons.append("queryWorkforceFacts is empty")
|
||||
|
||||
return len(reasons) == 0, reasons
|
||||
|
||||
|
||||
def count_error_log_lines(log_text: str) -> int:
|
||||
return sum(1 for line in log_text.splitlines() if ERROR_LOG_PATTERN.search(line))
|
||||
|
||||
|
||||
class ClusterClient:
|
||||
"""Thin kubectl wrapper enforcing kubeconfig, timeouts, and non-secret output."""
|
||||
|
||||
def __init__(self, kubeconfig: str) -> None:
|
||||
self.kubeconfig = kubeconfig
|
||||
|
||||
def run(
|
||||
self,
|
||||
args: list[str],
|
||||
*,
|
||||
stdin_text: str | None = None,
|
||||
timeout: int = 30,
|
||||
check: bool = True,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
cmd = ["kubectl", "--kubeconfig", self.kubeconfig, *args]
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
input=stdin_text,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"kubectl failed rc={proc.returncode} args={' '.join(args[:4])}: {proc.stderr.strip()[:240]}"
|
||||
)
|
||||
return proc
|
||||
|
||||
def get_json(self, namespace: str | None, *resource_args: str, ignore_not_found: bool = False) -> Any:
|
||||
args = []
|
||||
if namespace:
|
||||
args.extend(["-n", namespace])
|
||||
args.extend(["get", *resource_args, "-o", "json"])
|
||||
if ignore_not_found:
|
||||
args.append("--ignore-not-found")
|
||||
proc = self.run(args)
|
||||
if ignore_not_found and not proc.stdout.strip():
|
||||
return None
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
def patch_json(self, namespace: str, kind: str, name: str, patch_obj: dict[str, Any]) -> None:
|
||||
self.run(
|
||||
[
|
||||
"-n",
|
||||
namespace,
|
||||
"patch",
|
||||
kind,
|
||||
name,
|
||||
"--type=strategic",
|
||||
"-p",
|
||||
json.dumps(patch_obj),
|
||||
]
|
||||
)
|
||||
|
||||
def scale(self, namespace: str, kind: str, name: str, replicas: int) -> None:
|
||||
self.run(["-n", namespace, "scale", f"{kind}/{name}", f"--replicas={replicas}"])
|
||||
|
||||
def rollout_status(self, namespace: str, deployment: str, timeout_seconds: int = 180) -> bool:
|
||||
proc = self.run(
|
||||
["-n", namespace, "rollout", "status", f"deployment/{deployment}", f"--timeout={timeout_seconds}s"],
|
||||
timeout=timeout_seconds + 15,
|
||||
check=False,
|
||||
)
|
||||
return proc.returncode == 0
|
||||
|
||||
def wait_zero_pods(self, namespace: str, label_selector: str, timeout_seconds: int = 90) -> None:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
pods = self.get_json(namespace, "pods", "-l", label_selector)
|
||||
if not pods.get("items"):
|
||||
return
|
||||
time.sleep(2)
|
||||
raise RuntimeError(f"timed out waiting for {namespace} pods {label_selector} to scale to zero")
|
||||
|
||||
|
||||
def query_shadow_mongo_snapshot(client: ClusterClient) -> dict[str, Any]:
|
||||
"""Inspect shadow control heads & derived counts inside query-mongo-0 without exposing creds."""
|
||||
js = f"""
|
||||
const fs = require('fs');
|
||||
const q = db.getSiblingDB('{QUERY_DB_NAME}');
|
||||
if (q.auth('shadow-health', fs.readFileSync('/health-credentials/health-password', 'utf8').trim()).ok !== 1) {{
|
||||
quit(2);
|
||||
}}
|
||||
const st = q.queryReplicaControl.findOne({{_id: 'state'}});
|
||||
const gl = q.queryReadModelControl.findOne({{_id: 'global'}});
|
||||
const bootDone = q.queryReplicaControl.countDocuments({{_id: /^bootstrap\\//, complete: true}});
|
||||
const poolBucketsCount = q.queryPoolCountBuckets.countDocuments({{}});
|
||||
const workforceFactsCount = q.queryWorkforceFacts.countDocuments({{}});
|
||||
const normHead = h => {{
|
||||
if (!h) return null;
|
||||
const out = Object.assign({{}}, h);
|
||||
if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString();
|
||||
return out;
|
||||
}};
|
||||
const normHeads = m => {{
|
||||
const res = {{}};
|
||||
for (const [k, v] of Object.entries(m || {{}})) res[k] = normHead(v);
|
||||
return res;
|
||||
}};
|
||||
print(JSON.stringify({{
|
||||
state: st ? {{
|
||||
ready: Boolean(st.ready),
|
||||
observedAt: st.observedAt instanceof Date ? st.observedAt.toISOString() : null,
|
||||
factEpoch: Number(st.factEpoch || 0),
|
||||
schema: st.schema || null
|
||||
}} : null,
|
||||
bootstrapCompleteCount: bootDone,
|
||||
poolBucketsCount: poolBucketsCount,
|
||||
workforceFactsCount: workforceFactsCount,
|
||||
global: gl ? {{
|
||||
productionRevision: Number(gl.productionRevision || 0),
|
||||
observedAt: gl.observedAt instanceof Date ? gl.observedAt.toISOString() : null,
|
||||
heads: normHeads(gl.heads)
|
||||
}} : null
|
||||
}}));
|
||||
"""
|
||||
proc = client.run(
|
||||
[
|
||||
"-n",
|
||||
SHADOW_NAMESPACE,
|
||||
"exec",
|
||||
SHADOW_MONGO_POD,
|
||||
"-c",
|
||||
SHADOW_MONGO_CONTAINER,
|
||||
"--",
|
||||
"mongosh",
|
||||
"--quiet",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"27129",
|
||||
"--eval",
|
||||
js,
|
||||
],
|
||||
timeout=25,
|
||||
)
|
||||
return json.loads(proc.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def inspect_api_health(client: ClusterClient) -> dict[str, Any]:
|
||||
dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
||||
pods = client.get_json(SOURCE_NAMESPACE, "pods", "-l", f"app={API_DEPLOYMENT}")
|
||||
desired = int(dep["spec"].get("replicas") or 0)
|
||||
ready_replicas = int(dep.get("status", {}).get("readyReplicas") or 0)
|
||||
pod_states = []
|
||||
total_restarts = 0
|
||||
for pod in pods.get("items", []):
|
||||
statuses = pod.get("status", {}).get("containerStatuses") or []
|
||||
restarts = sum(int(cs.get("restartCount") or 0) for cs in statuses)
|
||||
total_restarts += restarts
|
||||
ready = bool(statuses) and all(bool(cs.get("ready")) for cs in statuses)
|
||||
pod_states.append(
|
||||
{
|
||||
"name": pod["metadata"]["name"],
|
||||
"uid": pod["metadata"]["uid"],
|
||||
"ready": ready,
|
||||
"restarts": restarts,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"desiredReplicas": desired,
|
||||
"readyReplicas": ready_replicas,
|
||||
"totalRestarts": total_restarts,
|
||||
"pods": pod_states,
|
||||
}
|
||||
|
||||
|
||||
def write_report(out_dir: Path, filename: str, payload: Any) -> Path:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / filename
|
||||
path.write_text(json.dumps(redact_secrets(payload), ensure_ascii=False, indent=2) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def cmd_preflight(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
|
||||
shadow_deploys = client.get_json(SHADOW_NAMESPACE, "deployments")
|
||||
shadow_sts = client.get_json(SHADOW_NAMESPACE, "statefulsets")
|
||||
shadow_quotas = client.get_json(SHADOW_NAMESPACE, "resourcequotas")
|
||||
shadow_netpol = client.get_json(SHADOW_NAMESPACE, "networkpolicies")
|
||||
mongo_snap = query_shadow_mongo_snapshot(client)
|
||||
api_health = inspect_api_health(client)
|
||||
|
||||
report = {
|
||||
"at": utc_now().isoformat(),
|
||||
"runId": args.run_id,
|
||||
"apiHealth": api_health,
|
||||
"shadowMongoSnapshot": mongo_snap,
|
||||
"sourceDeployments": [
|
||||
{
|
||||
"name": d["metadata"]["name"],
|
||||
"replicas": d["spec"].get("replicas"),
|
||||
"readyReplicas": d.get("status", {}).get("readyReplicas", 0),
|
||||
"images": [c.get("image") for c in d["spec"]["template"]["spec"]["containers"]],
|
||||
}
|
||||
for d in source_deploys.get("items", [])
|
||||
],
|
||||
"shadowWorkloads": [
|
||||
{
|
||||
"kind": w["kind"],
|
||||
"name": w["metadata"]["name"],
|
||||
"replicas": w["spec"].get("replicas"),
|
||||
"readyReplicas": w.get("status", {}).get("readyReplicas", 0),
|
||||
"images": [c.get("image") for c in w["spec"]["template"]["spec"]["containers"]],
|
||||
}
|
||||
for w in shadow_deploys.get("items", []) + shadow_sts.get("items", [])
|
||||
],
|
||||
"shadowQuotas": [
|
||||
{
|
||||
"name": q["metadata"]["name"],
|
||||
"hard": q.get("spec", {}).get("hard"),
|
||||
"used": q.get("status", {}).get("used"),
|
||||
}
|
||||
for q in shadow_quotas.get("items", [])
|
||||
],
|
||||
"shadowNetworkPolicies": [p["metadata"]["name"] for p in shadow_netpol.get("items", [])],
|
||||
}
|
||||
path = write_report(out_dir, "preflight.json", report)
|
||||
print(json.dumps({"ok": True, "phase": "preflight", "report": str(path), "apiReady": api_health["readyReplicas"]}))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
baseline_health = inspect_api_health(client)
|
||||
if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0:
|
||||
raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}")
|
||||
|
||||
# Scale builder to 0 first to stay well inside ResourceQuota.
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
||||
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={BUILDER_CONTAINER}")
|
||||
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}")
|
||||
|
||||
rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT)
|
||||
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
rep_patch = build_deployment_update_patch(
|
||||
container_name=REPLICA_CONTAINER,
|
||||
strategy_recreate=True,
|
||||
image=args.replica_image,
|
||||
args=build_shadow_replica_args(),
|
||||
env_list=build_shadow_replica_env(rep_env),
|
||||
)
|
||||
client.patch_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1)
|
||||
if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180):
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
||||
raise RuntimeError("query-replica failed rollout")
|
||||
|
||||
sync_deadline = time.monotonic() + args.sync_timeout_seconds
|
||||
synced_snap = None
|
||||
while time.monotonic() < sync_deadline:
|
||||
health = inspect_api_health(client)
|
||||
if (
|
||||
health["readyReplicas"] < baseline_health["desiredReplicas"]
|
||||
or health["totalRestarts"] > baseline_health["totalRestarts"]
|
||||
):
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
||||
write_report(out_dir, "shadow-sync-aborted.json", {"reason": "api_degraded", "health": health})
|
||||
raise RuntimeError(f"API health degraded during shadow-sync; scaled projector to 0: {health}")
|
||||
snap = query_shadow_mongo_snapshot(client)
|
||||
state = snap.get("state") or {}
|
||||
if state.get("ready") is True:
|
||||
obs = parse_iso_datetime(state.get("observedAt"))
|
||||
if obs is not None and (utc_now() - obs).total_seconds() < 300:
|
||||
synced_snap = snap
|
||||
break
|
||||
time.sleep(15)
|
||||
|
||||
if synced_snap is None:
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
||||
raise RuntimeError("timed out waiting for query-replica ready state")
|
||||
|
||||
bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT)
|
||||
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
bld_patch = build_deployment_update_patch(
|
||||
container_name=BUILDER_CONTAINER,
|
||||
strategy_recreate=True,
|
||||
image=args.builder_image,
|
||||
args=build_shadow_builder_args(),
|
||||
env_list=build_shadow_builder_env(bld_env),
|
||||
)
|
||||
client.patch_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1)
|
||||
if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180):
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
||||
raise RuntimeError("query-read-builder failed rollout")
|
||||
|
||||
build_deadline = time.monotonic() + args.build_timeout_seconds
|
||||
built_snap = None
|
||||
while time.monotonic() < build_deadline:
|
||||
health = inspect_api_health(client)
|
||||
if (
|
||||
health["readyReplicas"] < baseline_health["desiredReplicas"]
|
||||
or health["totalRestarts"] > baseline_health["totalRestarts"]
|
||||
):
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
||||
write_report(out_dir, "shadow-build-aborted.json", {"reason": "api_degraded", "health": health})
|
||||
raise RuntimeError(f"API health degraded during builder run; scaled builder to 0: {health}")
|
||||
snap = query_shadow_mongo_snapshot(client)
|
||||
ok, reasons = evaluate_parity_snapshot(snap)
|
||||
if ok:
|
||||
built_snap = snap
|
||||
break
|
||||
time.sleep(15)
|
||||
|
||||
if built_snap is None:
|
||||
raise RuntimeError("timed out waiting for query-read-builder completion")
|
||||
|
||||
path = write_report(out_dir, "shadow-sync.json", {"at": utc_now().isoformat(), "snapshot": built_snap})
|
||||
print(json.dumps({"ok": True, "phase": "shadow-sync", "report": str(path)}))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_verify_parity(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
snap = query_shadow_mongo_snapshot(client)
|
||||
ok, reasons = evaluate_parity_snapshot(snap)
|
||||
report = {
|
||||
"at": utc_now().isoformat(),
|
||||
"passed": ok,
|
||||
"reasons": reasons,
|
||||
"snapshot": snap,
|
||||
}
|
||||
path = write_report(out_dir, "verify-parity.json", report)
|
||||
print(json.dumps({"ok": ok, "phase": "verify-parity", "reasons": reasons, "report": str(path)}))
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def sync_read_client_secret(client: ClusterClient) -> None:
|
||||
conn_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CONNECTIONS_SECRET)
|
||||
cache_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CACHE_AUTH_SECRET)
|
||||
api_uri_b64 = (conn_secret.get("data") or {}).get("api-uri")
|
||||
valkey_url_b64 = (cache_secret.get("data") or {}).get("url")
|
||||
if not api_uri_b64 or not valkey_url_b64:
|
||||
raise RuntimeError("required shadow read client secret keys missing")
|
||||
manifest = build_read_client_secret_manifest(api_uri_b64, valkey_url_b64)
|
||||
existing = client.get_json(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, ignore_not_found=True)
|
||||
verb = "replace" if existing is not None else "create"
|
||||
client.run([verb, "-f", "-"], stdin_text=json.dumps(manifest))
|
||||
|
||||
|
||||
def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
||||
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates())
|
||||
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=rolled_env)
|
||||
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
||||
ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180)
|
||||
health = inspect_api_health(client)
|
||||
path = write_report(
|
||||
out_dir,
|
||||
"rollback-api-flags.json",
|
||||
{"at": utc_now().isoformat(), "rolloutOk": ok, "apiHealth": health},
|
||||
)
|
||||
print(json.dumps({"ok": ok, "phase": "rollback-api-flags", "report": str(path)}))
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
snap = query_shadow_mongo_snapshot(client)
|
||||
ok, reasons = evaluate_parity_snapshot(snap)
|
||||
if not ok:
|
||||
raise RuntimeError(f"parity gate blocked cutover: {reasons}")
|
||||
|
||||
sync_read_client_secret(client)
|
||||
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
||||
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env})
|
||||
|
||||
next_env = merge_env_list(existing_env, build_api_cutover_env_updates())
|
||||
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=next_env)
|
||||
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
||||
|
||||
if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180):
|
||||
cmd_rollback_api_flags(args, client, out_dir)
|
||||
raise RuntimeError("droboticflow-api rollout failed after flag cutover; rolled back")
|
||||
|
||||
health = inspect_api_health(client)
|
||||
path = write_report(
|
||||
out_dir,
|
||||
"cutover-api-flags.json",
|
||||
{"at": utc_now().isoformat(), "env": next_env, "apiHealth": health},
|
||||
)
|
||||
print(json.dumps({"ok": True, "phase": "cutover-api-flags", "report": str(path)}))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
baseline = inspect_api_health(client)
|
||||
deadline = time.monotonic() + args.watch_seconds
|
||||
samples: list[dict[str, Any]] = []
|
||||
rolled_back = False
|
||||
trigger_reason = ""
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(min(20, max(1, int(deadline - time.monotonic()))))
|
||||
health = inspect_api_health(client)
|
||||
log_proc = client.run(
|
||||
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=25s", "--tail=300"],
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "")
|
||||
sample = {
|
||||
"at": utc_now().isoformat(),
|
||||
"readyReplicas": health["readyReplicas"],
|
||||
"desiredReplicas": health["desiredReplicas"],
|
||||
"totalRestarts": health["totalRestarts"],
|
||||
"errorLines": error_lines,
|
||||
}
|
||||
samples.append(sample)
|
||||
|
||||
if health["readyReplicas"] < baseline["desiredReplicas"]:
|
||||
trigger_reason = f"readyReplicas dropped: {health['readyReplicas']}/{baseline['desiredReplicas']}"
|
||||
elif health["totalRestarts"] > baseline["totalRestarts"]:
|
||||
trigger_reason = f"API pod restarts increased: {baseline['totalRestarts']} -> {health['totalRestarts']}"
|
||||
elif error_lines >= args.max_error_lines:
|
||||
trigger_reason = f"error log spike: {error_lines} lines in 25s window"
|
||||
|
||||
if trigger_reason:
|
||||
rolled_back = True
|
||||
cmd_rollback_api_flags(args, client, out_dir)
|
||||
break
|
||||
|
||||
report = {
|
||||
"at": utc_now().isoformat(),
|
||||
"passed": not rolled_back,
|
||||
"rolledBack": rolled_back,
|
||||
"triggerReason": trigger_reason,
|
||||
"samples": samples,
|
||||
}
|
||||
path = write_report(out_dir, "watch-and-guard.json", report)
|
||||
print(json.dumps({"ok": not rolled_back, "phase": "watch-and-guard", "rolledBack": rolled_back, "report": str(path)}))
|
||||
return 1 if rolled_back else 0
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.")
|
||||
parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
|
||||
parser.add_argument("--run-id", default=default_run_id())
|
||||
parser.add_argument("--output-dir", default="")
|
||||
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("preflight")
|
||||
|
||||
p_sync = sub.add_parser("shadow-sync")
|
||||
p_sync.add_argument("--replica-image", required=True)
|
||||
p_sync.add_argument("--builder-image", required=True)
|
||||
p_sync.add_argument("--sync-timeout-seconds", type=int, default=1800)
|
||||
p_sync.add_argument("--build-timeout-seconds", type=int, default=1200)
|
||||
|
||||
sub.add_parser("verify-parity")
|
||||
sub.add_parser("cutover-api-flags")
|
||||
|
||||
p_watch = sub.add_parser("watch-and-guard")
|
||||
p_watch.add_argument("--watch-seconds", type=int, default=900)
|
||||
p_watch.add_argument("--max-error-lines", type=int, default=5)
|
||||
|
||||
sub.add_parser("rollback-api-flags")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
out_dir = (
|
||||
Path(args.output_dir)
|
||||
if args.output_dir
|
||||
else Path("local-data") / "diagnostics" / "evening-cutover" / args.run_id
|
||||
)
|
||||
client = ClusterClient(args.kubeconfig)
|
||||
handlers = {
|
||||
"preflight": cmd_preflight,
|
||||
"shadow-sync": cmd_shadow_sync,
|
||||
"verify-parity": cmd_verify_parity,
|
||||
"cutover-api-flags": cmd_cutover_api_flags,
|
||||
"watch-and-guard": cmd_watch_and_guard,
|
||||
"rollback-api-flags": cmd_rollback_api_flags,
|
||||
}
|
||||
return handlers[args.command](args, client, out_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
137
scripts/local/test_evening_query_cutover.py
Normal file
137
scripts/local/test_evening_query_cutover.py
Normal file
@ -0,0 +1,137 @@
|
||||
import datetime as dt
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
def load_module():
|
||||
path = Path(__file__).with_name("evening-query-cutover.py")
|
||||
spec = importlib.util.spec_from_file_location("evening_query_cutover", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
CUTOVER = load_module()
|
||||
|
||||
|
||||
class EveningQueryCutoverTests(unittest.TestCase):
|
||||
def test_redact_secrets_scrubs_sensitive_fields_and_secrets(self):
|
||||
payload = {
|
||||
"kind": "Secret",
|
||||
"metadata": {
|
||||
"name": "droboticflow-query-read-client",
|
||||
"annotations": {
|
||||
"kubectl.kubernetes.io/last-applied-configuration": "secret-json"
|
||||
},
|
||||
},
|
||||
"data": {"api-uri": "bW9uZ29kYjovL3NlY3JldA==", "valkey-url": "dmFsa2V5"},
|
||||
"env": [
|
||||
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
|
||||
{"name": "MONGO_URI", "value": "mongodb://user:pass@host/db"},
|
||||
{"name": "QUERY_VALKEY_URL", "value": "redis://:pwd@cache:6379"},
|
||||
{
|
||||
"name": "QUERY_READ_MONGO_URI",
|
||||
"valueFrom": {"secretKeyRef": {"name": "s", "key": "api-uri"}},
|
||||
},
|
||||
],
|
||||
}
|
||||
redacted = CUTOVER.redact_secrets(payload)
|
||||
self.assertEqual(redacted["data"]["api-uri"], "<redacted>")
|
||||
self.assertEqual(redacted["data"]["valkey-url"], "<redacted>")
|
||||
self.assertEqual(
|
||||
redacted["metadata"]["annotations"]["kubectl.kubernetes.io/last-applied-configuration"],
|
||||
"<redacted>",
|
||||
)
|
||||
env_by_name = {e["name"]: e for e in redacted["env"]}
|
||||
self.assertEqual(env_by_name["MONGO_DB"]["value"], "drobodata-fanyu")
|
||||
self.assertEqual(env_by_name["MONGO_URI"]["value"], "<redacted>")
|
||||
self.assertEqual(env_by_name["QUERY_VALKEY_URL"]["value"], "<redacted>")
|
||||
self.assertEqual(
|
||||
env_by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["key"],
|
||||
"api-uri",
|
||||
)
|
||||
serialized = json.dumps(redacted)
|
||||
self.assertNotIn("user:pass", serialized)
|
||||
self.assertNotIn("bW9uZ29k", serialized)
|
||||
|
||||
def test_merge_env_list_and_cutover_flags_contract(self):
|
||||
existing = [
|
||||
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
|
||||
{"name": "QUERY_SOURCE_TRACKING", "value": "1"},
|
||||
{"name": "QUERY_REPLICA_POOLS", "value": "1"},
|
||||
{"name": "QUERY_PUBLISHED_RESULTS", "value": "1"},
|
||||
]
|
||||
merged = CUTOVER.merge_env_list(existing, CUTOVER.build_api_cutover_env_updates())
|
||||
by_name = {e["name"]: e for e in merged}
|
||||
self.assertEqual(by_name["MONGO_DB"]["value"], "drobodata-fanyu")
|
||||
self.assertEqual(by_name["QUERY_SOURCE_TRACKING"]["value"], "1")
|
||||
self.assertEqual(by_name["QUERY_PUBLISHED_RESULTS"]["value"], "0")
|
||||
self.assertEqual(by_name["QUERY_REPLICA_POOLS"]["value"], "0")
|
||||
self.assertEqual(by_name["QUERY_REPLICA_REPORTS"]["value"], "1")
|
||||
self.assertEqual(by_name["QUERY_REPLICA_COUNTS"]["value"], "1")
|
||||
self.assertEqual(by_name["QUERY_SERVING_READS"]["value"], "1")
|
||||
self.assertEqual(by_name["QUERY_REPLICA_REFRESH_MODE"]["value"], "periodic")
|
||||
self.assertEqual(
|
||||
by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["name"],
|
||||
CUTOVER.READ_CLIENT_SECRET_NAME,
|
||||
)
|
||||
|
||||
def test_deployment_patch_sets_recreate_and_clears_rolling_update(self):
|
||||
patch = CUTOVER.build_deployment_update_patch(
|
||||
strategy_recreate=True,
|
||||
container_name="projector",
|
||||
image="example/img@sha256:abc",
|
||||
args=CUTOVER.build_shadow_replica_args(),
|
||||
env_list=CUTOVER.build_shadow_replica_env([]),
|
||||
)
|
||||
strategy = patch["spec"]["strategy"]
|
||||
self.assertEqual(strategy["type"], "Recreate")
|
||||
self.assertIn("rollingUpdate", strategy)
|
||||
self.assertIsNone(strategy["rollingUpdate"])
|
||||
container = patch["spec"]["template"]["spec"]["containers"][0]
|
||||
self.assertEqual(container["name"], "projector")
|
||||
self.assertIn("--source-compress", container["args"])
|
||||
self.assertNotIn("--safety-run", container["args"])
|
||||
|
||||
def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self):
|
||||
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
good = {
|
||||
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3},
|
||||
"poolBucketsCount": 42,
|
||||
"workforceFactsCount": 1200,
|
||||
"global": {
|
||||
"heads": {
|
||||
"h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"},
|
||||
"h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"},
|
||||
}
|
||||
},
|
||||
}
|
||||
passed, reasons = CUTOVER.evaluate_parity_snapshot(good, now=now)
|
||||
self.assertTrue(passed, reasons)
|
||||
self.assertEqual(reasons, [])
|
||||
|
||||
stale = json.loads(json.dumps(good))
|
||||
stale["state"]["observedAt"] = "2026-11-16T11:45:00Z"
|
||||
stale["workforceFactsCount"] = 0
|
||||
passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now)
|
||||
self.assertFalse(passed)
|
||||
self.assertEqual(len(reasons), 2)
|
||||
|
||||
def test_rollback_env_disables_all_query_reads(self):
|
||||
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
|
||||
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())
|
||||
by_name = {e["name"]: e for e in rolled}
|
||||
for key in (
|
||||
"QUERY_REPLICA_REPORTS",
|
||||
"QUERY_REPLICA_COUNTS",
|
||||
"QUERY_REPLICA_POOLS",
|
||||
"QUERY_SERVING_READS",
|
||||
"QUERY_PUBLISHED_RESULTS",
|
||||
):
|
||||
self.assertEqual(by_name[key]["value"], "0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user