diff --git a/cmd/query-read-builder/main.go b/cmd/query-read-builder/main.go index 29457338..499f4432 100644 --- a/cmd/query-read-builder/main.go +++ b/cmd/query-read-builder/main.go @@ -21,6 +21,7 @@ func main() { once := flag.Bool("once", false, "build once; streaming freshness expires after exit") ensure := flag.Bool("ensure-indexes", false, "explicitly create derived-data indexes before building") shadowRun := flag.String("shadow-safety-run", "", "opt-in independently monitored shadow builder; no authority gates") + autoRecover := flag.Bool("auto-recover-transient", false, "automatically cool down and clear transient build failure fences in unattended mode") flag.Parse() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -90,6 +91,26 @@ func main() { for workCtx.Err() == nil { if err := b.Run(workCtx); err != nil && workCtx.Err() == nil { if errors.Is(err, store.ErrQueryBuildStopped) { + if *autoRecover && *shadowRun == "" { + log.Print("query builder hit stopped fence; cooling down before checking transient recovery") + cooldown := time.NewTimer(15 * time.Second) + select { + case <-workCtx.Done(): + cooldown.Stop() + return + case <-cooldown.C: + } + resetCtx, cancel := context.WithTimeout(workCtx, 5*time.Second) + cleared, clearErr := s.ResetTransientQueryBuildFailures(resetCtx) + cancel() + if clearErr == nil && cleared > 0 { + log.Print("query builder cleared transient failure fence; resuming checkpoint") + continue + } + log.Print("query builder stopped on non-transient failure; checkpoint retained; explicit recovery required") + <-workCtx.Done() + return + } log.Print("query builder stopped; checkpoint retained; explicit recovery required") <-workCtx.Done() return diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go index 6416db0c..03dc4018 100644 --- a/cmd/query-replica/main.go +++ b/cmd/query-replica/main.go @@ -43,6 +43,7 @@ func main() { status := flag.Bool("safety-status", false, "target-only private safety status") sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records") observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source") + sourceCompress := flag.Bool("source-compress", false, "negotiate snappy/zstd wire compression on unguarded source reads") flag.Parse() var maintenanceDeadline *time.Time if *maintenance { @@ -176,6 +177,12 @@ func main() { log.Fatal("explicit source/query connection required") } opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second) + if !target && transport == nil { + opts.SetMaxPoolSize(2) + if *sourceCompress { + opts.SetCompressors([]string{"snappy", "zstd"}) + } + } if !target && transport != nil { opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors(sourceCompressors(*maintenance)).SetRetryReads(false).SetTimeout(5 * time.Second) } @@ -203,12 +210,15 @@ func main() { if *ensure && q.EnsureSourceIndexes(ctx) != nil { log.Fatal("source index preparation failed") } - for { + for attempts := 0; ; attempts++ { err := q.Acquire(ctx) if err == nil { break } - if *safetyRun == "" || !errors.Is(err, store.ErrQueryReplicaLease) { + if *once || !errors.Is(err, store.ErrQueryReplicaLease) { + log.Fatal("projector lease unavailable") + } + if *safetyRun == "" && attempts >= 35 { log.Fatal("projector lease unavailable") } select { @@ -276,6 +286,7 @@ func main() { <-ctx.Done() return } + consecutiveFailures := 0 for workCtx.Err() == nil { started := time.Now() ready, e := q.RunOnce(workCtx, *bootstrap) @@ -285,6 +296,21 @@ func main() { } d := store.QueryReplicaDiagnostic(e) log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode) + if *safetyRun == "" && !*once { + wait := daemonRetryDelay(consecutiveFailures) + consecutiveFailures++ + timer := time.NewTimer(wait) + select { + case <-workCtx.Done(): + timer.Stop() + return + case <-timer.C: + } + if errors.Is(e, store.ErrQueryReplicaLease) { + _ = q.Acquire(workCtx) + } + continue + } short, stop := context.WithTimeout(context.Background(), 5*time.Second) if q.PauseSafetyFailure(short, e) != nil { log.Print("pause diagnostics persistence unavailable; startup fencing remains required") @@ -297,6 +323,7 @@ func main() { } os.Exit(1) } + consecutiveFailures = 0 if *once { if !ready { pause("incomplete_pilot") @@ -307,9 +334,29 @@ func main() { } // Optional pacing covers backfill too. Idle lease maintenance only // touches the independent query DB, never scans authority for keepalive. - if delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass); delay > 0 { + delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass) + if *safetyRun == "" { + if cool := adaptiveBackfillCooldown(ready, time.Since(started)); cool > delay { + delay = cool + } + } + if delay > 0 { if e := q.WaitNextPass(workCtx, delay); e != nil { if ctx.Err() == nil { + if *safetyRun == "" && !*once { + log.Print("projector lease refresh retry") + wait := daemonRetryDelay(consecutiveFailures) + consecutiveFailures++ + timer := time.NewTimer(wait) + select { + case <-workCtx.Done(): + timer.Stop() + return + case <-timer.C: + } + _ = q.Acquire(workCtx) + continue + } pause("target_error") log.Print("projector lease unavailable; checkpoint retained") if *safetyRun != "" { diff --git a/cmd/query-replica/pacing.go b/cmd/query-replica/pacing.go index 323147b8..16779bea 100644 --- a/cmd/query-replica/pacing.go +++ b/cmd/query-replica/pacing.go @@ -23,3 +23,24 @@ func passDelay(mode string, ready bool, elapsed, interval, minimum time.Duration } return delay } + +func daemonRetryDelay(attempt int) time.Duration { + if attempt <= 0 { + return 2 * time.Second + } + if attempt > 10 { + attempt = 10 + } + delay := 2 * time.Second << attempt + if delay > 15*time.Second || delay <= 0 { + return 15 * time.Second + } + return delay +} + +func adaptiveBackfillCooldown(ready bool, elapsed time.Duration) time.Duration { + if !ready && elapsed >= 2*time.Second { + return 500 * time.Millisecond + } + return 0 +} diff --git a/cmd/query-replica/pacing_test.go b/cmd/query-replica/pacing_test.go index 64283654..edfd9456 100644 --- a/cmd/query-replica/pacing_test.go +++ b/cmd/query-replica/pacing_test.go @@ -40,3 +40,35 @@ func TestPacingRejectsUnboundedOrInvalidConfiguration(t *testing.T) { } } } + +func TestDaemonRetryDelayAndAdaptiveBackfillCooldown(t *testing.T) { + for _, tc := range []struct { + attempt int + want time.Duration + }{ + {-1, 2 * time.Second}, + {0, 2 * time.Second}, + {1, 4 * time.Second}, + {2, 8 * time.Second}, + {3, 15 * time.Second}, + {10, 15 * time.Second}, + } { + if got := daemonRetryDelay(tc.attempt); got != tc.want { + t.Fatalf("attempt=%d got=%s want=%s", tc.attempt, got, tc.want) + } + } + for _, tc := range []struct { + ready bool + elapsed time.Duration + want time.Duration + }{ + {true, 5 * time.Second, 0}, + {false, 1999 * time.Millisecond, 0}, + {false, 2 * time.Second, 500 * time.Millisecond}, + {false, 4 * time.Second, 500 * time.Millisecond}, + } { + if got := adaptiveBackfillCooldown(tc.ready, tc.elapsed); got != tc.want { + t.Fatalf("ready=%v elapsed=%s got=%s want=%s", tc.ready, tc.elapsed, got, tc.want) + } + } +} diff --git a/internal/store/query_build_runner.go b/internal/store/query_build_runner.go index 8d12f05e..b9fda435 100644 --- a/internal/store/query_build_runner.go +++ b/internal/store/query_build_runner.go @@ -204,3 +204,57 @@ func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease } return nil } + +var transientQueryBuildFailureClasses = []string{ + "source_timeout", + "target_timeout", + "timeout", + "deadline", + "network", + "lease_lost", + "lease", + "no_durable_progress", + "transient", +} + +func isTransientQueryBuildFailureClass(class string) bool { + for _, candidate := range transientQueryBuildFailureClasses { + if class == candidate { + return true + } + } + return false +} + +// ResetTransientQueryBuildFailures clears transient failure fences for active +// query build jobs while preserving durable checkpoints and schema/content blocks. +func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int64, error) { + if s == nil || s.db == nil { + return 0, errors.New("store unavailable") + } + classes := bson.A{} + for _, c := range transientQueryBuildFailureClasses { + classes = append(classes, c) + } + filter := bson.M{ + "schema": queryBuildJobSchema, + "phase": bson.M{"$ne": "complete"}, + "failureClass": bson.M{"$in": classes}, + } + update := bson.M{ + "$unset": bson.M{ + "failurePhase": "", + "failureClass": "", + "failureCode": "", + }, + "$set": bson.M{ + "activeWithoutProgress": int64(0), + "progressAt": time.Now().UTC(), + }, + } + res, err := s.db.Collection(queryBuildJobsCollection).UpdateMany(ctx, filter, update) + if err != nil { + return 0, err + } + return res.ModifiedCount, nil +} diff --git a/internal/store/query_build_transient_test.go b/internal/store/query_build_transient_test.go new file mode 100644 index 00000000..3513b222 --- /dev/null +++ b/internal/store/query_build_transient_test.go @@ -0,0 +1,33 @@ +package store + +import "testing" + +func TestIsTransientQueryBuildFailureClass(t *testing.T) { + for _, class := range []string{ + "source_timeout", + "target_timeout", + "timeout", + "deadline", + "network", + "lease_lost", + "lease", + "no_durable_progress", + "transient", + } { + if !isTransientQueryBuildFailureClass(class) { + t.Fatalf("expected transient class %q to return true", class) + } + } + for _, class := range []string{ + "", + "unsupported_record", + "unsupported_fact", + "corrupt_state", + "auth_failed", + "bson_decode", + } { + if isTransientQueryBuildFailureClass(class) { + t.Fatalf("expected non-transient class %q to return false", class) + } + } +} diff --git a/internal/store/query_replica_count_cache.go b/internal/store/query_replica_count_cache.go index 7f9f5a47..015749ae 100644 --- a/internal/store/query_replica_count_cache.go +++ b/internal/store/query_replica_count_cache.go @@ -18,7 +18,10 @@ import ( ) const queryCountMemoryBudget = 64 << 20 -const queryCountTTL = 2 * time.Second + +// Cache keys include factEpoch so synced mutations invalidate immediately; a +// 15s TTL avoids 2s re-fetch/decompression thrashing while epochs are unchanged. +const queryCountTTL = 15 * time.Second var errQueryCountCacheCapacity = errors.New("count candidate cache capacity exceeded") @@ -75,6 +78,13 @@ func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, by if bytes > queryCountMemoryBudget { return } + if c.entries == nil { + c.entries = map[string]queryCountEntry{} + } + if prev, ok := c.entries[key]; ok { + delete(c.entries, key) + c.bytes -= prev.bytes + } now := time.Now() for k, e := range c.entries { if !now.Before(e.expires) { @@ -82,8 +92,24 @@ func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, by c.bytes -= e.bytes } } - if len(c.entries) >= 32 || c.bytes+bytes > queryCountMemoryBudget { - clear(c.entries) + for len(c.entries) >= 32 || (len(c.entries) > 0 && c.bytes+bytes > queryCountMemoryBudget) { + var oldestKey string + var oldestExpires time.Time + first := true + for k, e := range c.entries { + if first || e.expires.Before(oldestExpires) { + oldestKey = k + oldestExpires = e.expires + first = false + } + } + if oldestKey == "" { + break + } + c.bytes -= c.entries[oldestKey].bytes + delete(c.entries, oldestKey) + } + if c.bytes < 0 { c.bytes = 0 } c.entries[key] = queryCountEntry{rows: rows, bytes: bytes, expires: now.Add(queryCountTTL)} diff --git a/internal/store/query_replica_count_cache_test.go b/internal/store/query_replica_count_cache_test.go index 856b9050..ddb28992 100644 --- a/internal/store/query_replica_count_cache_test.go +++ b/internal/store/query_replica_count_cache_test.go @@ -4,7 +4,9 @@ import ( "context" "droboticflow/internal/domain" "errors" + "fmt" "testing" + "time" ) func TestQueryReplicaCountOnlyLeavesListsAndCoreReadsOnAuthority(t *testing.T) { @@ -78,3 +80,50 @@ func TestQueryReplicaMissingConfigurationCannotPanicCoreCache(t *testing.T) { t.Fatal(e) } } + +func TestQueryCountCacheEvictsOldestInsteadOfClearingAll(t *testing.T) { + c := &queryCountCache{} + c.remember("k0", []domain.HumanItemRecord{{ID: "0"}}, 1024) + time.Sleep(2 * time.Millisecond) + for i := 1; i < 32; i++ { + c.remember(fmt.Sprintf("k%d", i), []domain.HumanItemRecord{{ID: fmt.Sprintf("%d", i)}}, 1024) + } + if len(c.entries) != 32 || c.bytes != 32*1024 { + t.Fatalf("unexpected cache size=%d bytes=%d", len(c.entries), c.bytes) + } + c.remember("k32", []domain.HumanItemRecord{{ID: "32"}}, 1024) + if len(c.entries) != 32 { + t.Fatalf("expected 32 entries after eviction, got %d", len(c.entries)) + } + if c.bytes != 32*1024 { + t.Fatalf("expected bytes=%d, got %d", 32*1024, c.bytes) + } + if _, ok := c.entries["k0"]; ok { + t.Fatal("expected oldest entry k0 to be evicted") + } + if _, ok := c.entries["k1"]; !ok { + t.Fatal("expected newer entry k1 to stay cached") + } + if _, ok := c.entries["k32"]; !ok { + t.Fatal("expected newest entry k32 to be cached") + } +} + +func TestFilterAuthoritativeAdminPoolPageDropsConcurrentlyClaimedItems(t *testing.T) { + q := AdminAnnotationPoolQuery{Pool: "available"} + candidates := []domain.HumanItemRecord{ + {ID: "a"}, + {ID: "claimed"}, + {ID: "missing"}, + {ID: "b"}, + } + hydrated := []domain.HumanItemRecord{ + {ID: "b", CurrentStageID: "semantic_annotation", Status: domain.HumanItemReady}, + {ID: "claimed", CurrentStageID: "semantic_annotation", Status: domain.HumanItemClaimed, AssignedUserID: "annotator-1"}, + {ID: "a", CurrentStageID: "semantic_annotation", Status: domain.HumanItemReady}, + } + got := filterAuthoritativeAdminPoolPage(candidates, hydrated, q) + if len(got) != 2 || got[0].ID != "a" || got[1].ID != "b" { + t.Fatalf("expected [a b] in candidate order without failing page, got %+v", got) + } +} diff --git a/internal/store/query_replica_pool.go b/internal/store/query_replica_pool.go index dacb2152..1072ba89 100644 --- a/internal/store/query_replica_pool.go +++ b/internal/store/query_replica_pool.go @@ -175,29 +175,76 @@ func (s *MongoStore) loadReplicaPoolPage(ctx context.Context, key, org string, q return page, nil } -func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { - rows, e := target.ListAdminAnnotationPool(ctx, org, q) - if e != nil || len(rows) == 0 { - return rows, e +func filterAuthoritativeAdminPoolPage(candidates, hydrated []domain.HumanItemRecord, q AdminAnnotationPoolQuery) []domain.HumanItemRecord { + byID := make(map[string]domain.HumanItemRecord, len(hydrated)) + for _, item := range hydrated { + byID[item.ID] = item } - ids := make([]string, 0, len(rows)) - for _, r := range rows { - ids = append(ids, r.ID) - } - current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids) - if e != nil { - return nil, e - } - if len(current) != len(rows) { - return nil, ErrQueryReplicaNotReady - } - if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil { - return nil, e - } - for _, item := range current { - if !adminPoolScope(item) || !adminPoolSelected(item, q) { - return nil, ErrQueryReplicaNotReady + out := make([]domain.HumanItemRecord, 0, len(candidates)) + for _, cand := range candidates { + item, ok := byID[cand.ID] + if !ok || !adminPoolScope(item) || !adminPoolSelected(item, q) { + continue } + out = append(out, item) } - return current, nil + return out +} + +func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { + collected := []domain.HumanItemRecord{} + seen := map[string]bool{} + stepQ := q + for pass := 0; pass < 4; pass++ { + if q.Limit > 0 { + need := q.Limit - len(collected) + if need <= 0 { + break + } + if need > stepQ.Limit { + stepQ.Limit = need + } else if stepQ.Limit < q.Limit { + stepQ.Limit = q.Limit + } + } + rows, e := target.ListAdminAnnotationPool(ctx, org, stepQ) + if e != nil { + return nil, e + } + if len(rows) == 0 { + break + } + ids := make([]string, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.ID) + } + current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids) + if e != nil { + return nil, e + } + if len(current) > 0 { + if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil { + return nil, e + } + } + for _, item := range filterAuthoritativeAdminPoolPage(rows, current, q) { + if seen[item.ID] { + continue + } + seen[item.ID] = true + collected = append(collected, item) + if q.Limit > 0 && len(collected) >= q.Limit { + return collected, nil + } + } + if q.Limit <= 0 || len(rows) < stepQ.Limit { + break + } + last := rows[len(rows)-1] + if last.ID == "" { + break + } + stepQ.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID} + } + return collected, nil } diff --git a/scripts/local/evening-query-cutover.py b/scripts/local/evening-query-cutover.py new file mode 100644 index 00000000..101fd9b2 --- /dev/null +++ b/scripts/local/evening-query-cutover.py @@ -0,0 +1,813 @@ +#!/usr/bin/env python3 +"""Unattended evening shadow-sync, parity gate, API read cutover, and auto-rollback.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +from typing import Any + +DEFAULT_KUBECONFIG = "/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf" +SOURCE_NAMESPACE = "droboticflow-fanyu" +SHADOW_NAMESPACE = "droboticflow-query-shadow" +QUERY_DB_NAME = "drobodata_fanyu_query_shadow_20261114" +QUERY_SERVING_NAMESPACE = "fanyu-query-shadow-20261114" +READ_CLIENT_SECRET_NAME = "droboticflow-query-read-client" +SHADOW_CONNECTIONS_SECRET = "query-shadow-query-connections" +SHADOW_CACHE_AUTH_SECRET = "query-shadow-cache-auth" + +API_DEPLOYMENT = "droboticflow-api" +API_CONTAINER = "api" +REPLICA_DEPLOYMENT = "query-replica" +REPLICA_CONTAINER = "projector" +BUILDER_DEPLOYMENT = "query-read-builder" +BUILDER_CONTAINER = "builder" +SHADOW_MONGO_POD = "query-mongo-0" +SHADOW_MONGO_CONTAINER = "mongo" + +SECRET_KEY_PATTERN = re.compile( + r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)", + re.IGNORECASE, +) +ERROR_LOG_PATTERN = re.compile( + r"(ErrQueryReplicaNotReady|查询暂不可用|\bstatus=(?:500|503)\b|\"\s+(?:500|503)\s+)", +) + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def default_run_id() -> str: + return utc_now().strftime("%Y%m%dT%H%M%SZ") + + +def parse_iso_datetime(raw: Any) -> dt.datetime | None: + if not isinstance(raw, str) or not raw.strip(): + return None + text = raw.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = dt.datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=dt.timezone.utc) + return parsed.astimezone(dt.timezone.utc) + + +def redact_secrets(obj: Any, parent_key: str = "") -> Any: + """Recursively scrub secret payloads, URIs, and credentials before persistence/logging.""" + if isinstance(obj, dict): + kind = obj.get("kind") + if kind == "Secret": + out = {} + for k, v in obj.items(): + if k in ("data", "stringData") and isinstance(v, dict): + out[k] = {sk: "" for sk in v.keys()} + elif k == "metadata" and isinstance(v, dict): + meta = dict(v) + if isinstance(meta.get("annotations"), dict): + anns = dict(meta["annotations"]) + if "kubectl.kubernetes.io/last-applied-configuration" in anns: + anns["kubectl.kubernetes.io/last-applied-configuration"] = "" + meta["annotations"] = anns + out[k] = meta + else: + out[k] = redact_secrets(v, k) + return out + # Redact Kubernetes env var literal values if key name looks sensitive. + if "name" in obj and "value" in obj and isinstance(obj.get("name"), str): + env_name = obj["name"] + if SECRET_KEY_PATTERN.search(env_name): + redacted = dict(obj) + redacted["value"] = "" + return redacted + return {k: redact_secrets(v, k) for k, v in obj.items()} + if isinstance(obj, list): + return [redact_secrets(item, parent_key) for item in obj] + if isinstance(obj, str) and parent_key and SECRET_KEY_PATTERN.search(parent_key): + return "" + return obj + + +def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge env updates into existing Kubernetes env list preserving untouched vars and order.""" + by_name = {entry["name"]: dict(entry) for entry in updates} + seen: set[str] = set() + merged: list[dict[str, Any]] = [] + for entry in existing_env or []: + name = entry.get("name") + if not isinstance(name, str) or name in seen: + continue + seen.add(name) + if name in by_name: + merged.append(by_name[name]) + else: + merged.append(dict(entry)) + for entry in updates: + name = entry["name"] + if name not in seen: + seen.add(name) + merged.append(dict(entry)) + return merged + + +def build_deployment_update_patch( + *, + container_name: str, + env_list: list[dict[str, Any]], + strategy_recreate: bool = False, + image: str | None = None, + args: list[str] | None = None, +) -> dict[str, Any]: + """Build a Kubernetes strategic-merge patch for a single deployment container.""" + container_patch: dict[str, Any] = { + "name": container_name, + "env": env_list, + } + if image is not None: + container_patch["image"] = image + if args is not None: + container_patch["args"] = list(args) + spec_patch: dict[str, Any] = { + "template": { + "spec": { + "containers": [container_patch], + } + } + } + if strategy_recreate: + # Clearing rollingUpdate is required when switching from RollingUpdate -> Recreate. + spec_patch["strategy"] = {"type": "Recreate", "rollingUpdate": None} + return {"spec": spec_patch} + + +def build_shadow_replica_args() -> list[str]: + return [ + "--writers-covered", + "--bootstrap", + "--source-compress", + "--batch-size=100", + "--interval=30s", + ] + + +def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]: + updates = [ + {"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE}, + {"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME}, + {"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"}, + {"name": "QUERY_PUBLISHED_RESULTS", "value": "0"}, + {"name": "QUERY_SERVING_READS", "value": "1"}, + ] + return merge_env_list(existing_env, updates) + + +def build_shadow_builder_args() -> list[str]: + return [ + "--ensure-indexes", + "--auto-recover-transient", + ] + + +def build_shadow_builder_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]: + updates = [ + {"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE}, + {"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME}, + {"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"}, + {"name": "QUERY_PUBLISHED_RESULTS", "value": "0"}, + {"name": "QUERY_SERVING_READS", "value": "1"}, + ] + return merge_env_list(existing_env, updates) + + +def build_api_cutover_env_updates() -> list[dict[str, Any]]: + return [ + {"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE}, + {"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME}, + {"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"}, + {"name": "QUERY_PUBLISHED_RESULTS", "value": "0"}, + {"name": "QUERY_SERVING_READS", "value": "1"}, + {"name": "QUERY_REPLICA_REPORTS", "value": "1"}, + {"name": "QUERY_REPLICA_COUNTS", "value": "1"}, + {"name": "QUERY_REPLICA_POOLS", "value": "0"}, + { + "name": "QUERY_READ_MONGO_URI", + "valueFrom": { + "secretKeyRef": { + "name": READ_CLIENT_SECRET_NAME, + "key": "api-uri", + } + }, + }, + { + "name": "QUERY_VALKEY_URL", + "valueFrom": { + "secretKeyRef": { + "name": READ_CLIENT_SECRET_NAME, + "key": "valkey-url", + } + }, + }, + ] + + +def build_api_rollback_env_updates() -> list[dict[str, Any]]: + return [ + {"name": "QUERY_REPLICA_REPORTS", "value": "0"}, + {"name": "QUERY_REPLICA_COUNTS", "value": "0"}, + {"name": "QUERY_REPLICA_POOLS", "value": "0"}, + {"name": "QUERY_SERVING_READS", "value": "0"}, + {"name": "QUERY_PUBLISHED_RESULTS", "value": "0"}, + ] + + +def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) -> dict[str, Any]: + return { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": READ_CLIENT_SECRET_NAME, + "namespace": SOURCE_NAMESPACE, + }, + "type": "Opaque", + "data": { + "api-uri": api_uri_b64, + "valkey-url": valkey_url_b64, + }, + } + + +def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]: + if not isinstance(global_doc, dict): + return False, {"hasProduction": False, "hasWorkforce": False} + heads = global_doc.get("heads") or {} + if not isinstance(heads, dict): + return False, {"hasProduction": False, "hasWorkforce": False} + has_production = False + has_workforce = False + workforce_epoch = 0 + workforce_as_of = None + production_as_of = None + for head in heads.values(): + if not isinstance(head, dict): + continue + gen = head.get("generation") or "" + as_of = head.get("asOf") + if gen == "workforce-facts/v1": + epoch = int(head.get("factEpoch") or 0) + if epoch >= 1 and as_of: + has_workforce = True + workforce_epoch = max(workforce_epoch, epoch) + workforce_as_of = as_of + elif gen and not head.get("searchCollection") and int(head.get("chunks") or 0) >= 1 and as_of: + has_production = True + production_as_of = as_of + details = { + "hasProduction": has_production, + "hasWorkforce": has_workforce, + "workforceFactEpoch": workforce_epoch, + "workforceAsOf": workforce_as_of, + "productionAsOf": production_as_of, + } + return (has_production and has_workforce), details + + +def evaluate_parity_snapshot( + snapshot: dict[str, Any], + *, + now: dt.datetime | None = None, + max_observed_age_seconds: int = 300, + max_workforce_age_seconds: int = 900, +) -> tuple[bool, list[str]]: + current_time = now or utc_now() + reasons: list[str] = [] + + state = snapshot.get("state") or {} + if state.get("ready") is not True: + reasons.append("replica state.ready is not true") + + observed_dt = parse_iso_datetime(state.get("observedAt")) + if observed_dt is None: + reasons.append("replica state.observedAt missing or invalid") + else: + obs_age = (current_time - observed_dt).total_seconds() + if obs_age < -5 or obs_age >= max_observed_age_seconds: + reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s") + + heads_ok, head_info = evaluate_builder_heads(snapshot.get("global")) + if not heads_ok or int(head_info.get("workforceFactEpoch") or 0) < 1: + reasons.append("builder workforce or production head incomplete") + else: + wf_dt = parse_iso_datetime(head_info.get("workforceAsOf")) + if wf_dt is None: + reasons.append("workforce head asOf missing or invalid") + else: + wf_age = (current_time - wf_dt).total_seconds() + if wf_age < -5 or wf_age >= max_workforce_age_seconds: + reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s") + + if int(snapshot.get("poolBucketsCount") or 0) <= 0: + reasons.append("queryPoolCountBuckets is empty") + if int(snapshot.get("workforceFactsCount") or 0) <= 0: + reasons.append("queryWorkforceFacts is empty") + + return len(reasons) == 0, reasons + + +def count_error_log_lines(log_text: str) -> int: + return sum(1 for line in log_text.splitlines() if ERROR_LOG_PATTERN.search(line)) + + +class ClusterClient: + """Thin kubectl wrapper enforcing kubeconfig, timeouts, and non-secret output.""" + + def __init__(self, kubeconfig: str) -> None: + self.kubeconfig = kubeconfig + + def run( + self, + args: list[str], + *, + stdin_text: str | None = None, + timeout: int = 30, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + cmd = ["kubectl", "--kubeconfig", self.kubeconfig, *args] + proc = subprocess.run( + cmd, + input=stdin_text, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + if check and proc.returncode != 0: + raise RuntimeError( + f"kubectl failed rc={proc.returncode} args={' '.join(args[:4])}: {proc.stderr.strip()[:240]}" + ) + return proc + + def get_json(self, namespace: str | None, *resource_args: str, ignore_not_found: bool = False) -> Any: + args = [] + if namespace: + args.extend(["-n", namespace]) + args.extend(["get", *resource_args, "-o", "json"]) + if ignore_not_found: + args.append("--ignore-not-found") + proc = self.run(args) + if ignore_not_found and not proc.stdout.strip(): + return None + return json.loads(proc.stdout) + + def patch_json(self, namespace: str, kind: str, name: str, patch_obj: dict[str, Any]) -> None: + self.run( + [ + "-n", + namespace, + "patch", + kind, + name, + "--type=strategic", + "-p", + json.dumps(patch_obj), + ] + ) + + def scale(self, namespace: str, kind: str, name: str, replicas: int) -> None: + self.run(["-n", namespace, "scale", f"{kind}/{name}", f"--replicas={replicas}"]) + + def rollout_status(self, namespace: str, deployment: str, timeout_seconds: int = 180) -> bool: + proc = self.run( + ["-n", namespace, "rollout", "status", f"deployment/{deployment}", f"--timeout={timeout_seconds}s"], + timeout=timeout_seconds + 15, + check=False, + ) + return proc.returncode == 0 + + def wait_zero_pods(self, namespace: str, label_selector: str, timeout_seconds: int = 90) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + pods = self.get_json(namespace, "pods", "-l", label_selector) + if not pods.get("items"): + return + time.sleep(2) + raise RuntimeError(f"timed out waiting for {namespace} pods {label_selector} to scale to zero") + + +def query_shadow_mongo_snapshot(client: ClusterClient) -> dict[str, Any]: + """Inspect shadow control heads & derived counts inside query-mongo-0 without exposing creds.""" + js = f""" +const fs = require('fs'); +const q = db.getSiblingDB('{QUERY_DB_NAME}'); +if (q.auth('shadow-health', fs.readFileSync('/health-credentials/health-password', 'utf8').trim()).ok !== 1) {{ + quit(2); +}} +const st = q.queryReplicaControl.findOne({{_id: 'state'}}); +const gl = q.queryReadModelControl.findOne({{_id: 'global'}}); +const bootDone = q.queryReplicaControl.countDocuments({{_id: /^bootstrap\\//, complete: true}}); +const poolBucketsCount = q.queryPoolCountBuckets.countDocuments({{}}); +const workforceFactsCount = q.queryWorkforceFacts.countDocuments({{}}); +const normHead = h => {{ + if (!h) return null; + const out = Object.assign({{}}, h); + if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString(); + return out; +}}; +const normHeads = m => {{ + const res = {{}}; + for (const [k, v] of Object.entries(m || {{}})) res[k] = normHead(v); + return res; +}}; +print(JSON.stringify({{ + state: st ? {{ + ready: Boolean(st.ready), + observedAt: st.observedAt instanceof Date ? st.observedAt.toISOString() : null, + factEpoch: Number(st.factEpoch || 0), + schema: st.schema || null + }} : null, + bootstrapCompleteCount: bootDone, + poolBucketsCount: poolBucketsCount, + workforceFactsCount: workforceFactsCount, + global: gl ? {{ + productionRevision: Number(gl.productionRevision || 0), + observedAt: gl.observedAt instanceof Date ? gl.observedAt.toISOString() : null, + heads: normHeads(gl.heads) + }} : null +}})); +""" + proc = client.run( + [ + "-n", + SHADOW_NAMESPACE, + "exec", + SHADOW_MONGO_POD, + "-c", + SHADOW_MONGO_CONTAINER, + "--", + "mongosh", + "--quiet", + "--host", + "127.0.0.1", + "--port", + "27129", + "--eval", + js, + ], + timeout=25, + ) + return json.loads(proc.stdout.strip().splitlines()[-1]) + + +def inspect_api_health(client: ClusterClient) -> dict[str, Any]: + dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT) + pods = client.get_json(SOURCE_NAMESPACE, "pods", "-l", f"app={API_DEPLOYMENT}") + desired = int(dep["spec"].get("replicas") or 0) + ready_replicas = int(dep.get("status", {}).get("readyReplicas") or 0) + pod_states = [] + total_restarts = 0 + for pod in pods.get("items", []): + statuses = pod.get("status", {}).get("containerStatuses") or [] + restarts = sum(int(cs.get("restartCount") or 0) for cs in statuses) + total_restarts += restarts + ready = bool(statuses) and all(bool(cs.get("ready")) for cs in statuses) + pod_states.append( + { + "name": pod["metadata"]["name"], + "uid": pod["metadata"]["uid"], + "ready": ready, + "restarts": restarts, + } + ) + return { + "desiredReplicas": desired, + "readyReplicas": ready_replicas, + "totalRestarts": total_restarts, + "pods": pod_states, + } + + +def write_report(out_dir: Path, filename: str, payload: Any) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / filename + path.write_text(json.dumps(redact_secrets(payload), ensure_ascii=False, indent=2) + "\n") + return path + + +def cmd_preflight(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments") + shadow_deploys = client.get_json(SHADOW_NAMESPACE, "deployments") + shadow_sts = client.get_json(SHADOW_NAMESPACE, "statefulsets") + shadow_quotas = client.get_json(SHADOW_NAMESPACE, "resourcequotas") + shadow_netpol = client.get_json(SHADOW_NAMESPACE, "networkpolicies") + mongo_snap = query_shadow_mongo_snapshot(client) + api_health = inspect_api_health(client) + + report = { + "at": utc_now().isoformat(), + "runId": args.run_id, + "apiHealth": api_health, + "shadowMongoSnapshot": mongo_snap, + "sourceDeployments": [ + { + "name": d["metadata"]["name"], + "replicas": d["spec"].get("replicas"), + "readyReplicas": d.get("status", {}).get("readyReplicas", 0), + "images": [c.get("image") for c in d["spec"]["template"]["spec"]["containers"]], + } + for d in source_deploys.get("items", []) + ], + "shadowWorkloads": [ + { + "kind": w["kind"], + "name": w["metadata"]["name"], + "replicas": w["spec"].get("replicas"), + "readyReplicas": w.get("status", {}).get("readyReplicas", 0), + "images": [c.get("image") for c in w["spec"]["template"]["spec"]["containers"]], + } + for w in shadow_deploys.get("items", []) + shadow_sts.get("items", []) + ], + "shadowQuotas": [ + { + "name": q["metadata"]["name"], + "hard": q.get("spec", {}).get("hard"), + "used": q.get("status", {}).get("used"), + } + for q in shadow_quotas.get("items", []) + ], + "shadowNetworkPolicies": [p["metadata"]["name"] for p in shadow_netpol.get("items", [])], + } + path = write_report(out_dir, "preflight.json", report) + print(json.dumps({"ok": True, "phase": "preflight", "report": str(path), "apiReady": api_health["readyReplicas"]})) + return 0 + + +def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + baseline_health = inspect_api_health(client) + if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0: + raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}") + + # Scale builder to 0 first to stay well inside ResourceQuota. + client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0) + client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0) + client.wait_zero_pods(SHADOW_NAMESPACE, f"component={BUILDER_CONTAINER}") + client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}") + + rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT) + rep_env = rep_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] + rep_patch = build_deployment_update_patch( + container_name=REPLICA_CONTAINER, + strategy_recreate=True, + image=args.replica_image, + args=build_shadow_replica_args(), + env_list=build_shadow_replica_env(rep_env), + ) + client.patch_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch) + client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1) + if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180): + client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0) + raise RuntimeError("query-replica failed rollout") + + sync_deadline = time.monotonic() + args.sync_timeout_seconds + synced_snap = None + while time.monotonic() < sync_deadline: + health = inspect_api_health(client) + if ( + health["readyReplicas"] < baseline_health["desiredReplicas"] + or health["totalRestarts"] > baseline_health["totalRestarts"] + ): + client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0) + write_report(out_dir, "shadow-sync-aborted.json", {"reason": "api_degraded", "health": health}) + raise RuntimeError(f"API health degraded during shadow-sync; scaled projector to 0: {health}") + snap = query_shadow_mongo_snapshot(client) + state = snap.get("state") or {} + if state.get("ready") is True: + obs = parse_iso_datetime(state.get("observedAt")) + if obs is not None and (utc_now() - obs).total_seconds() < 300: + synced_snap = snap + break + time.sleep(15) + + if synced_snap is None: + client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0) + raise RuntimeError("timed out waiting for query-replica ready state") + + bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT) + bld_env = bld_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] + bld_patch = build_deployment_update_patch( + container_name=BUILDER_CONTAINER, + strategy_recreate=True, + image=args.builder_image, + args=build_shadow_builder_args(), + env_list=build_shadow_builder_env(bld_env), + ) + client.patch_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch) + client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1) + if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180): + client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0) + raise RuntimeError("query-read-builder failed rollout") + + build_deadline = time.monotonic() + args.build_timeout_seconds + built_snap = None + while time.monotonic() < build_deadline: + health = inspect_api_health(client) + if ( + health["readyReplicas"] < baseline_health["desiredReplicas"] + or health["totalRestarts"] > baseline_health["totalRestarts"] + ): + client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0) + write_report(out_dir, "shadow-build-aborted.json", {"reason": "api_degraded", "health": health}) + raise RuntimeError(f"API health degraded during builder run; scaled builder to 0: {health}") + snap = query_shadow_mongo_snapshot(client) + ok, reasons = evaluate_parity_snapshot(snap) + if ok: + built_snap = snap + break + time.sleep(15) + + if built_snap is None: + raise RuntimeError("timed out waiting for query-read-builder completion") + + path = write_report(out_dir, "shadow-sync.json", {"at": utc_now().isoformat(), "snapshot": built_snap}) + print(json.dumps({"ok": True, "phase": "shadow-sync", "report": str(path)})) + return 0 + + +def cmd_verify_parity(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + snap = query_shadow_mongo_snapshot(client) + ok, reasons = evaluate_parity_snapshot(snap) + report = { + "at": utc_now().isoformat(), + "passed": ok, + "reasons": reasons, + "snapshot": snap, + } + path = write_report(out_dir, "verify-parity.json", report) + print(json.dumps({"ok": ok, "phase": "verify-parity", "reasons": reasons, "report": str(path)})) + return 0 if ok else 1 + + +def sync_read_client_secret(client: ClusterClient) -> None: + conn_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CONNECTIONS_SECRET) + cache_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CACHE_AUTH_SECRET) + api_uri_b64 = (conn_secret.get("data") or {}).get("api-uri") + valkey_url_b64 = (cache_secret.get("data") or {}).get("url") + if not api_uri_b64 or not valkey_url_b64: + raise RuntimeError("required shadow read client secret keys missing") + manifest = build_read_client_secret_manifest(api_uri_b64, valkey_url_b64) + existing = client.get_json(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, ignore_not_found=True) + verb = "replace" if existing is not None else "create" + client.run([verb, "-f", "-"], stdin_text=json.dumps(manifest)) + + +def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT) + existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] + rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates()) + patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=rolled_env) + client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) + ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180) + health = inspect_api_health(client) + path = write_report( + out_dir, + "rollback-api-flags.json", + {"at": utc_now().isoformat(), "rolloutOk": ok, "apiHealth": health}, + ) + print(json.dumps({"ok": ok, "phase": "rollback-api-flags", "report": str(path)})) + return 0 if ok else 1 + + +def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + snap = query_shadow_mongo_snapshot(client) + ok, reasons = evaluate_parity_snapshot(snap) + if not ok: + raise RuntimeError(f"parity gate blocked cutover: {reasons}") + + sync_read_client_secret(client) + api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT) + existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] + write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env}) + + next_env = merge_env_list(existing_env, build_api_cutover_env_updates()) + patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=next_env) + client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) + + if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180): + cmd_rollback_api_flags(args, client, out_dir) + raise RuntimeError("droboticflow-api rollout failed after flag cutover; rolled back") + + health = inspect_api_health(client) + path = write_report( + out_dir, + "cutover-api-flags.json", + {"at": utc_now().isoformat(), "env": next_env, "apiHealth": health}, + ) + print(json.dumps({"ok": True, "phase": "cutover-api-flags", "report": str(path)})) + return 0 + + +def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + baseline = inspect_api_health(client) + deadline = time.monotonic() + args.watch_seconds + samples: list[dict[str, Any]] = [] + rolled_back = False + trigger_reason = "" + + while time.monotonic() < deadline: + time.sleep(min(20, max(1, int(deadline - time.monotonic())))) + health = inspect_api_health(client) + log_proc = client.run( + ["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=25s", "--tail=300"], + timeout=15, + check=False, + ) + error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "") + sample = { + "at": utc_now().isoformat(), + "readyReplicas": health["readyReplicas"], + "desiredReplicas": health["desiredReplicas"], + "totalRestarts": health["totalRestarts"], + "errorLines": error_lines, + } + samples.append(sample) + + if health["readyReplicas"] < baseline["desiredReplicas"]: + trigger_reason = f"readyReplicas dropped: {health['readyReplicas']}/{baseline['desiredReplicas']}" + elif health["totalRestarts"] > baseline["totalRestarts"]: + trigger_reason = f"API pod restarts increased: {baseline['totalRestarts']} -> {health['totalRestarts']}" + elif error_lines >= args.max_error_lines: + trigger_reason = f"error log spike: {error_lines} lines in 25s window" + + if trigger_reason: + rolled_back = True + cmd_rollback_api_flags(args, client, out_dir) + break + + report = { + "at": utc_now().isoformat(), + "passed": not rolled_back, + "rolledBack": rolled_back, + "triggerReason": trigger_reason, + "samples": samples, + } + path = write_report(out_dir, "watch-and-guard.json", report) + print(json.dumps({"ok": not rolled_back, "phase": "watch-and-guard", "rolledBack": rolled_back, "report": str(path)})) + return 1 if rolled_back else 0 + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.") + parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG) + parser.add_argument("--run-id", default=default_run_id()) + parser.add_argument("--output-dir", default="") + + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("preflight") + + p_sync = sub.add_parser("shadow-sync") + p_sync.add_argument("--replica-image", required=True) + p_sync.add_argument("--builder-image", required=True) + p_sync.add_argument("--sync-timeout-seconds", type=int, default=1800) + p_sync.add_argument("--build-timeout-seconds", type=int, default=1200) + + sub.add_parser("verify-parity") + sub.add_parser("cutover-api-flags") + + p_watch = sub.add_parser("watch-and-guard") + p_watch.add_argument("--watch-seconds", type=int, default=900) + p_watch.add_argument("--max-error-lines", type=int, default=5) + + sub.add_parser("rollback-api-flags") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + out_dir = ( + Path(args.output_dir) + if args.output_dir + else Path("local-data") / "diagnostics" / "evening-cutover" / args.run_id + ) + client = ClusterClient(args.kubeconfig) + handlers = { + "preflight": cmd_preflight, + "shadow-sync": cmd_shadow_sync, + "verify-parity": cmd_verify_parity, + "cutover-api-flags": cmd_cutover_api_flags, + "watch-and-guard": cmd_watch_and_guard, + "rollback-api-flags": cmd_rollback_api_flags, + } + return handlers[args.command](args, client, out_dir) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/local/test_evening_query_cutover.py b/scripts/local/test_evening_query_cutover.py new file mode 100644 index 00000000..32ffcf3f --- /dev/null +++ b/scripts/local/test_evening_query_cutover.py @@ -0,0 +1,137 @@ +import datetime as dt +import importlib.util +import json +from pathlib import Path +import unittest + + +def load_module(): + path = Path(__file__).with_name("evening-query-cutover.py") + spec = importlib.util.spec_from_file_location("evening_query_cutover", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +CUTOVER = load_module() + + +class EveningQueryCutoverTests(unittest.TestCase): + def test_redact_secrets_scrubs_sensitive_fields_and_secrets(self): + payload = { + "kind": "Secret", + "metadata": { + "name": "droboticflow-query-read-client", + "annotations": { + "kubectl.kubernetes.io/last-applied-configuration": "secret-json" + }, + }, + "data": {"api-uri": "bW9uZ29kYjovL3NlY3JldA==", "valkey-url": "dmFsa2V5"}, + "env": [ + {"name": "MONGO_DB", "value": "drobodata-fanyu"}, + {"name": "MONGO_URI", "value": "mongodb://user:pass@host/db"}, + {"name": "QUERY_VALKEY_URL", "value": "redis://:pwd@cache:6379"}, + { + "name": "QUERY_READ_MONGO_URI", + "valueFrom": {"secretKeyRef": {"name": "s", "key": "api-uri"}}, + }, + ], + } + redacted = CUTOVER.redact_secrets(payload) + self.assertEqual(redacted["data"]["api-uri"], "") + self.assertEqual(redacted["data"]["valkey-url"], "") + self.assertEqual( + redacted["metadata"]["annotations"]["kubectl.kubernetes.io/last-applied-configuration"], + "", + ) + env_by_name = {e["name"]: e for e in redacted["env"]} + self.assertEqual(env_by_name["MONGO_DB"]["value"], "drobodata-fanyu") + self.assertEqual(env_by_name["MONGO_URI"]["value"], "") + self.assertEqual(env_by_name["QUERY_VALKEY_URL"]["value"], "") + self.assertEqual( + env_by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["key"], + "api-uri", + ) + serialized = json.dumps(redacted) + self.assertNotIn("user:pass", serialized) + self.assertNotIn("bW9uZ29k", serialized) + + def test_merge_env_list_and_cutover_flags_contract(self): + existing = [ + {"name": "MONGO_DB", "value": "drobodata-fanyu"}, + {"name": "QUERY_SOURCE_TRACKING", "value": "1"}, + {"name": "QUERY_REPLICA_POOLS", "value": "1"}, + {"name": "QUERY_PUBLISHED_RESULTS", "value": "1"}, + ] + merged = CUTOVER.merge_env_list(existing, CUTOVER.build_api_cutover_env_updates()) + by_name = {e["name"]: e for e in merged} + self.assertEqual(by_name["MONGO_DB"]["value"], "drobodata-fanyu") + self.assertEqual(by_name["QUERY_SOURCE_TRACKING"]["value"], "1") + self.assertEqual(by_name["QUERY_PUBLISHED_RESULTS"]["value"], "0") + self.assertEqual(by_name["QUERY_REPLICA_POOLS"]["value"], "0") + self.assertEqual(by_name["QUERY_REPLICA_REPORTS"]["value"], "1") + self.assertEqual(by_name["QUERY_REPLICA_COUNTS"]["value"], "1") + self.assertEqual(by_name["QUERY_SERVING_READS"]["value"], "1") + self.assertEqual(by_name["QUERY_REPLICA_REFRESH_MODE"]["value"], "periodic") + self.assertEqual( + by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["name"], + CUTOVER.READ_CLIENT_SECRET_NAME, + ) + + def test_deployment_patch_sets_recreate_and_clears_rolling_update(self): + patch = CUTOVER.build_deployment_update_patch( + strategy_recreate=True, + container_name="projector", + image="example/img@sha256:abc", + args=CUTOVER.build_shadow_replica_args(), + env_list=CUTOVER.build_shadow_replica_env([]), + ) + strategy = patch["spec"]["strategy"] + self.assertEqual(strategy["type"], "Recreate") + self.assertIn("rollingUpdate", strategy) + self.assertIsNone(strategy["rollingUpdate"]) + container = patch["spec"]["template"]["spec"]["containers"][0] + self.assertEqual(container["name"], "projector") + self.assertIn("--source-compress", container["args"]) + self.assertNotIn("--safety-run", container["args"]) + + def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self): + now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) + good = { + "state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3}, + "poolBucketsCount": 42, + "workforceFactsCount": 1200, + "global": { + "heads": { + "h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"}, + "h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"}, + } + }, + } + passed, reasons = CUTOVER.evaluate_parity_snapshot(good, now=now) + self.assertTrue(passed, reasons) + self.assertEqual(reasons, []) + + stale = json.loads(json.dumps(good)) + stale["state"]["observedAt"] = "2026-11-16T11:45:00Z" + stale["workforceFactsCount"] = 0 + passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now) + self.assertFalse(passed) + self.assertEqual(len(reasons), 2) + + def test_rollback_env_disables_all_query_reads(self): + existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates()) + rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates()) + by_name = {e["name"]: e for e in rolled} + for key in ( + "QUERY_REPLICA_REPORTS", + "QUERY_REPLICA_COUNTS", + "QUERY_REPLICA_POOLS", + "QUERY_SERVING_READS", + "QUERY_PUBLISHED_RESULTS", + ): + self.assertEqual(by_name[key]["value"], "0") + + +if __name__ == "__main__": + unittest.main()