droboticflow/scripts/local/evening-query-cutover.py
d-robotics 412a12cfe5
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions
🛠️ Recover expired query snapshots and bound role-change reads
2026-11-16 08:21:41 -08:00

1085 lines
46 KiB
Python

#!/usr/bin/env python3
"""Unattended evening shadow-sync, parity gate, API read cutover, and auto-rollback."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import hashlib
import os
from pathlib import Path
import re
import subprocess
import sys
import time
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
from query_cutover_guard import approved_plan, bounded_shadow_args, require_result_comparison
DEFAULT_KUBECONFIG = "/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf"
SOURCE_NAMESPACE = "droboticflow-fanyu"
SHADOW_NAMESPACE = "droboticflow-query-shadow"
QUERY_DB_NAME = "drobodata_fanyu_query_shadow_20261114"
QUERY_SERVING_NAMESPACE = "fanyu-query-shadow-20261114"
READ_CLIENT_SECRET_NAME = "droboticflow-query-read-client"
SHADOW_CONNECTIONS_SECRET = "query-shadow-query-connections"
SHADOW_CACHE_AUTH_SECRET = "query-shadow-cache-auth"
API_DEPLOYMENT = "droboticflow-api"
API_CONTAINER = "api"
REPLICA_DEPLOYMENT = "query-replica"
REPLICA_CONTAINER = "projector"
BUILDER_DEPLOYMENT = "query-read-builder"
BUILDER_CONTAINER = "builder"
MONITOR_DEPLOYMENT = "query-shadow-monitor"
SHADOW_MONGO_POD = "query-mongo-0"
SHADOW_MONGO_CONTAINER = "mongo"
PRODUCTION_BUSINESS_DEPLOYMENTS = (
"droboticflow-api",
"droboticflow-annotation-processor",
"droboticflow-annotation-validator",
"droboticflow-worker",
"drosync-coordinator",
)
SECRET_KEY_PATTERN = re.compile(
r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)",
re.IGNORECASE,
)
ERROR_LOG_PATTERN = re.compile(
r"(ErrQueryReplicaNotReady|查询暂不可用|\bstatus=(?:500|503)\b|\"\s+(?:500|503)\s+)",
)
def utc_now() -> dt.datetime:
return dt.datetime.now(dt.timezone.utc)
def default_run_id() -> str:
return utc_now().strftime("%Y%m%dT%H%M%SZ")
def parse_iso_datetime(raw: Any) -> dt.datetime | None:
if not isinstance(raw, str) or not raw.strip():
return None
text = raw.strip()
if text.endswith("Z"):
text = text[:-1] + "+00:00"
try:
parsed = dt.datetime.fromisoformat(text)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=dt.timezone.utc)
return parsed.astimezone(dt.timezone.utc)
def redact_secrets(obj: Any, parent_key: str = "") -> Any:
"""Recursively scrub secret payloads, URIs, and credentials before persistence/logging."""
if isinstance(obj, dict):
kind = obj.get("kind")
if kind == "Secret":
out = {}
for k, v in obj.items():
if k in ("data", "stringData") and isinstance(v, dict):
out[k] = {sk: "<redacted>" for sk in v.keys()}
elif k == "metadata" and isinstance(v, dict):
meta = dict(v)
if isinstance(meta.get("annotations"), dict):
anns = dict(meta["annotations"])
if "kubectl.kubernetes.io/last-applied-configuration" in anns:
anns["kubectl.kubernetes.io/last-applied-configuration"] = "<redacted>"
meta["annotations"] = anns
out[k] = meta
else:
out[k] = redact_secrets(v, k)
return out
# Redact Kubernetes env var literal values if key name looks sensitive.
if "name" in obj and "value" in obj and isinstance(obj.get("name"), str):
env_name = obj["name"]
if SECRET_KEY_PATTERN.search(env_name):
redacted = dict(obj)
redacted["value"] = "<redacted>"
return redacted
return {k: redact_secrets(v, k) for k, v in obj.items()}
if isinstance(obj, list):
return [redact_secrets(item, parent_key) for item in obj]
if isinstance(obj, str) and obj.startswith("--resume-pause="):
return "--resume-pause=<redacted>"
if isinstance(obj, str) and parent_key and SECRET_KEY_PATTERN.search(parent_key):
return "<redacted>"
return obj
def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Merge env updates into existing Kubernetes env list preserving untouched vars and order."""
by_name = {entry["name"]: dict(entry) for entry in updates}
seen: set[str] = set()
merged: list[dict[str, Any]] = []
for entry in existing_env or []:
name = entry.get("name")
if not isinstance(name, str) or name in seen:
continue
seen.add(name)
if name in by_name:
merged.append(by_name[name])
else:
merged.append(dict(entry))
for entry in updates:
name = entry["name"]
if name not in seen:
seen.add(name)
merged.append(dict(entry))
return merged
def find_container_index(dep: dict[str, Any], container_name: str) -> int:
containers = dep.get("spec", {}).get("template", {}).get("spec", {}).get("containers") or []
for idx, container in enumerate(containers):
if container.get("name") == container_name:
return idx
if len(containers) <= 1:
return 0
raise RuntimeError(f"container {container_name} not found in deployment")
def build_deployment_json_patch(
*,
container_index: int = 0,
env_list: list[dict[str, Any]],
strategy_recreate: bool = False,
image: str | None = None,
args: list[str] | None = None,
expected_deployment: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
"""Build an atomic RFC 6902 JSON patch for a single deployment container."""
ops: list[dict[str, Any]] = []
if expected_deployment:
ops.extend([{"op":"test", "path":"/metadata/uid", "value":expected_deployment["metadata"]["uid"]},
{"op":"test", "path":"/metadata/resourceVersion", "value":expected_deployment["metadata"]["resourceVersion"]}])
if strategy_recreate:
ops.append({"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
ops.append(
{
"op": "add",
"path": f"/spec/template/spec/containers/{container_index}/env",
"value": env_list,
}
)
if image is not None:
ops.append(
{
"op": "add",
"path": f"/spec/template/spec/containers/{container_index}/image",
"value": image,
}
)
if args is not None:
ops.append(
{
"op": "add",
"path": f"/spec/template/spec/containers/{container_index}/args",
"value": list(args),
}
)
return ops
def build_shadow_replica_args(plan, resume_nonce="") -> list[str]:
return bounded_shadow_args(plan, resume_nonce)
def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
updates = [
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "1"},
]
return merge_env_list(existing_env, updates)
def build_shadow_builder_args(plan) -> list[str]:
return ["--ensure-indexes", "--shadow-safety-run=" + plan["shadowPolicy"]["runID"]]
def build_shadow_builder_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
updates = [
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "1"},
]
return merge_env_list(existing_env, updates)
def build_api_cutover_env_updates() -> list[dict[str, Any]]:
return [
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "1"},
{"name": "QUERY_REPLICA_REPORTS", "value": "1"},
{"name": "QUERY_REPLICA_COUNTS", "value": "1"},
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
{
"name": "QUERY_READ_MONGO_URI",
"valueFrom": {
"secretKeyRef": {
"name": READ_CLIENT_SECRET_NAME,
"key": "api-uri",
}
},
},
{
"name": "QUERY_VALKEY_URL",
"valueFrom": {
"secretKeyRef": {
"name": READ_CLIENT_SECRET_NAME,
"key": "valkey-url",
}
},
},
]
def build_api_rollback_env_updates() -> list[dict[str, Any]]:
return [
{"name": "QUERY_REPLICA_REPORTS", "value": "0"},
{"name": "QUERY_REPLICA_COUNTS", "value": "0"},
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
{"name": "QUERY_SERVING_READS", "value": "0"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
]
def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) -> dict[str, Any]:
return {
"apiVersion": "v1",
"kind": "Secret",
"metadata": {
"name": READ_CLIENT_SECRET_NAME,
"namespace": SOURCE_NAMESPACE,
},
"type": "Opaque",
"data": {
"api-uri": api_uri_b64,
"valkey-url": valkey_url_b64,
},
}
def parse_bson_int(val: Any) -> int:
"""Parse int, float, numeric string, or mongosh Long {high, low, unsigned} objects."""
if val is None:
return 0
if isinstance(val, bool):
return int(val)
if isinstance(val, (int, float)):
return int(val)
if isinstance(val, str):
try:
return int(val.strip())
except ValueError:
return 0
if isinstance(val, dict) and ("low" in val or "high" in val):
try:
low = int(val.get("low") or 0) & 0xFFFFFFFF
high = int(val.get("high") or 0)
if val.get("unsigned"):
return ((high & 0xFFFFFFFF) << 32) | low
return (high << 32) | low
except (TypeError, ValueError):
return 0
return 0
def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
if not isinstance(global_doc, dict):
return False, {"hasProduction": False, "hasWorkforce": False}
heads = global_doc.get("heads") or {}
if not isinstance(heads, dict):
return False, {"hasProduction": False, "hasWorkforce": False}
has_production = False
has_workforce = False
workforce_epoch = 0
workforce_as_of = None
production_as_of = None
for head in heads.values():
if not isinstance(head, dict):
continue
gen = head.get("generation") or ""
as_of = head.get("asOf")
if gen == "workforce-facts/v1":
epoch = parse_bson_int(head.get("factEpoch"))
if epoch >= 1 and as_of:
has_workforce = True
workforce_epoch = max(workforce_epoch, epoch)
workforce_as_of = as_of
elif gen and not head.get("searchCollection") and parse_bson_int(head.get("chunks")) >= 1 and as_of:
has_production = True
production_as_of = as_of
details = {
"hasProduction": has_production,
"hasWorkforce": has_workforce,
"workforceFactEpoch": workforce_epoch,
"workforceAsOf": workforce_as_of,
"productionAsOf": production_as_of,
}
return (has_production and has_workforce), details
def evaluate_parity_snapshot(
snapshot: dict[str, Any],
*,
now: dt.datetime | None = None,
max_observed_age_seconds: int = 300,
max_workforce_age_seconds: int = 900,
) -> tuple[bool, list[str]]:
current_time = now or utc_now()
reasons: list[str] = []
state = snapshot.get("state") or {}
if state.get("ready") is not True:
reasons.append("replica state.ready is not true")
observed_dt = parse_iso_datetime(state.get("observedAt"))
if observed_dt is None:
reasons.append("replica state.observedAt missing or invalid")
else:
obs_age = (current_time - observed_dt).total_seconds()
if obs_age < -5 or obs_age >= max_observed_age_seconds:
reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s")
heads_ok, head_info = evaluate_builder_heads(snapshot.get("global"))
if not heads_ok or parse_bson_int(head_info.get("workforceFactEpoch")) < 1:
reasons.append("builder workforce or production head incomplete")
else:
wf_dt = parse_iso_datetime(head_info.get("workforceAsOf"))
if wf_dt is None:
reasons.append("workforce head asOf missing or invalid")
else:
wf_age = (current_time - wf_dt).total_seconds()
if wf_age < -5 or wf_age >= max_workforce_age_seconds:
reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s")
cohorts = snapshot.get("cohorts") or []
if not cohorts:
reasons.append("scoped publication cohort identities missing")
heads = (snapshot.get("global") or {}).get("heads") or {}
for cohort in cohorts:
production = heads.get(cohort.get("productionKey")) or {}
workforce = heads.get(cohort.get("workforceKey")) or {}
if not production.get("generation") or parse_bson_int(production.get("chunks")) < 1 or workforce.get("generation") != "workforce-facts/v1":
reasons.append("scoped publication incomplete")
continue
for field in ("asOf", "sourceRevision", "globalRevision"):
if production.get(field) is None or production.get(field) != workforce.get(field):
reasons.append("mixed scoped publication cohort")
break
for head in (production, workforce):
at = parse_iso_datetime(head.get("asOf"))
if at is None or not 0 <= (current_time-at).total_seconds() < max_workforce_age_seconds:
reasons.append("scoped publication stale")
if parse_bson_int(snapshot.get("poolBucketsCount")) <= 0:
reasons.append("queryPoolCountBuckets is empty")
if parse_bson_int(snapshot.get("workforceFactsCount")) <= 0:
reasons.append("queryWorkforceFacts is empty")
return len(reasons) == 0, reasons
def count_error_log_lines(log_text: str) -> int:
return sum(1 for line in log_text.splitlines() if ERROR_LOG_PATTERN.search(line))
class ClusterClient:
"""Thin kubectl wrapper enforcing kubeconfig, timeouts, and non-secret output."""
def __init__(self, kubeconfig: str) -> None:
self.kubeconfig = kubeconfig
self.approved_plan = None
def run(
self,
args: list[str],
*,
stdin_text: str | None = None,
timeout: int = 30,
check: bool = True,
) -> subprocess.CompletedProcess[str]:
cmd = ["kubectl", "--kubeconfig", self.kubeconfig, *args]
proc = subprocess.run(
cmd,
input=stdin_text,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
if check and proc.returncode != 0:
raise RuntimeError(
f"kubectl failed rc={proc.returncode} args={' '.join(args[:4])}: {proc.stderr.strip()[:240]}"
)
return proc
def get_json(self, namespace: str | None, *resource_args: str, ignore_not_found: bool = False) -> Any:
args = []
if namespace:
args.extend(["-n", namespace])
args.extend(["get", *resource_args, "-o", "json"])
if ignore_not_found:
args.append("--ignore-not-found")
for attempt in range(3):
try:
proc = self.run(args, timeout=8, check=False)
if proc.returncode == 0:
if ignore_not_found and not proc.stdout.strip():
return None
return json.loads(proc.stdout)
if any(term in proc.stderr for term in ("Forbidden", "Unauthorized", "NotFound")):
raise RuntimeError("metadata authorization/identity failure")
except (subprocess.TimeoutExpired, OSError):
pass
print(json.dumps({"event":"metadata_probe_retry","attempt":attempt+1}),file=sys.stderr)
if attempt < 2:
time.sleep(.5 * (attempt+1))
raise RuntimeError("metadata probe unavailable after bounded retries")
def patch_json_ops(self, namespace: str, kind: str, name: str, ops: list[dict[str, Any]]) -> None:
if not self.approved_plan:
raise RuntimeError("approved plan required before cluster writes")
current = self.get_json(namespace, kind, name)
expected = self.approved_plan["resourceUIDs"].get(namespace + "/" + kind + "/" + name)
if not expected or current["metadata"]["uid"] != expected:
raise RuntimeError("resource identity outside approved plan")
tests = [{"op":"test", "path":"/metadata/uid", "value":expected},
{"op":"test", "path":"/metadata/resourceVersion", "value":current["metadata"]["resourceVersion"]}]
self.run(["-n",namespace,"patch",kind,name,"--type=json","--patch-file=/dev/stdin"], stdin_text=json.dumps(tests + ops))
def scale(self, namespace: str, kind: str, name: str, replicas: int) -> None:
self.patch_json_ops(namespace, kind, name, [{"op":"replace", "path":"/spec/replicas", "value":replicas}])
def rollout_status(self, namespace: str, deployment: str, timeout_seconds: int = 180) -> bool:
proc = self.run(
["-n", namespace, "rollout", "status", f"deployment/{deployment}", f"--timeout={timeout_seconds}s"],
timeout=timeout_seconds + 15,
check=False,
)
return proc.returncode == 0
def wait_zero_pods(self, namespace: str, label_selector: str, timeout_seconds: int = 90) -> None:
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
pods = self.get_json(namespace, "pods", "-l", label_selector)
if not pods.get("items"):
return
time.sleep(2)
raise RuntimeError(f"timed out waiting for {namespace} pods {label_selector} to scale to zero")
def query_shadow_mongo_snapshot(client: ClusterClient) -> dict[str, Any]:
"""Inspect shadow control heads & derived counts inside query-mongo-0 without exposing creds."""
js = f"""
const fs = require('fs');
const q = db.getSiblingDB('{QUERY_DB_NAME}');
if (q.auth('shadow-health', fs.readFileSync('/health-credentials/health-password', 'utf8').trim()).ok !== 1) {{
quit(2);
}}
const st = q.queryReplicaControl.findOne({{_id: 'state'}});
const crypto = require('crypto');
const modelKey = (kind, org, projects) => 'query-model/v3:' + crypto.createHash('sha256').update(JSON.stringify([kind, org, projects])).digest('hex');
const cohorts = q.productionStatsScopes.find({{}}, {{projectIds:1}}).toArray().map(s => {{
const ids=q.projects.find({{id:{{$in:s.projectIds||[]}}}}, {{id:1}}).toArray().map(p=>p.id).sort();
return {{productionKey:modelKey('production',String(s._id).trim(),ids),workforceKey:modelKey('workforce-facts',String(s._id).trim(),[]) ,projects:ids.length}};
}}).filter(c=>c.projects>0);
const gl = q.queryReadModelControl.findOne({{_id: 'global'}});
const bootDone = q.queryReplicaControl.countDocuments({{_id: /^bootstrap\\//, complete: true}});
const poolBucketsCount = q.queryPoolCountBuckets.countDocuments({{}});
const workforceFactsCount = q.queryWorkforceFacts.countDocuments({{}});
const normHead = h => {{
if (!h) return null;
const out = Object.assign({{}}, h);
if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString();
if (out.factEpoch !== undefined && out.factEpoch !== null) out.factEpoch = Number(out.factEpoch);
if (out.chunks !== undefined && out.chunks !== null) out.chunks = Number(out.chunks);
if (out.rows !== undefined && out.rows !== null) out.rows = Number(out.rows);
if (out.bytes !== undefined && out.bytes !== null) out.bytes = Number(out.bytes);
return out;
}};
const normHeads = m => {{
const res = {{}};
for (const [k, v] of Object.entries(m || {{}})) res[k] = normHead(v);
return res;
}};
print(JSON.stringify({{
state: st ? {{
ready: Boolean(st.ready),
observedAt: st.observedAt instanceof Date ? st.observedAt.toISOString() : null,
factEpoch: Number(st.factEpoch || 0),
schema: st.schema || null
}} : null,
cohorts: cohorts,
bootstrapCompleteCount: bootDone,
poolBucketsCount: poolBucketsCount,
workforceFactsCount: workforceFactsCount,
global: gl ? {{
productionRevision: Number(gl.productionRevision || 0),
observedAt: gl.observedAt instanceof Date ? gl.observedAt.toISOString() : null,
heads: normHeads(gl.heads)
}} : null
}}));
"""
proc = client.run(
[
"-n",
SHADOW_NAMESPACE,
"exec",
SHADOW_MONGO_POD,
"-c",
SHADOW_MONGO_CONTAINER,
"--",
"mongosh",
"--quiet",
"--host",
"127.0.0.1",
"--port",
"27129",
"--eval",
js,
],
timeout=25,
)
return json.loads(proc.stdout.strip().splitlines()[-1])
def inspect_api_health(client: ClusterClient) -> dict[str, Any]:
dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
pods = client.get_json(SOURCE_NAMESPACE, "pods", "-l", f"app={API_DEPLOYMENT}")
desired = int(dep["spec"].get("replicas") or 0)
ready_replicas = int(dep.get("status", {}).get("readyReplicas") or 0)
pod_states = []
total_restarts = 0
for pod in pods.get("items", []):
statuses = pod.get("status", {}).get("containerStatuses") or []
restarts = sum(int(cs.get("restartCount") or 0) for cs in statuses)
total_restarts += restarts
ready = bool(statuses) and all(bool(cs.get("ready")) for cs in statuses)
pod_states.append(
{
"name": pod["metadata"]["name"],
"uid": pod["metadata"]["uid"],
"ready": ready,
"restarts": restarts,
}
)
return {
"desiredReplicas": desired,
"readyReplicas": ready_replicas,
"totalRestarts": total_restarts,
"pods": pod_states,
}
def write_report(out_dir: Path, filename: str, payload: Any) -> Path:
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / filename
path.write_text(json.dumps(redact_secrets(payload), ensure_ascii=False, indent=2) + "\n")
return path
def cmd_preflight(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
shadow_deploys = client.get_json(SHADOW_NAMESPACE, "deployments")
shadow_sts = client.get_json(SHADOW_NAMESPACE, "statefulsets")
shadow_quotas = client.get_json(SHADOW_NAMESPACE, "resourcequotas")
shadow_netpol = client.get_json(SHADOW_NAMESPACE, "networkpolicies")
mongo_snap = query_shadow_mongo_snapshot(client)
api_health = inspect_api_health(client)
report = {
"at": utc_now().isoformat(),
"runId": args.run_id,
"apiHealth": api_health,
"shadowMongoSnapshot": mongo_snap,
"sourceDeployments": [
{
"name": d["metadata"]["name"],
"replicas": d["spec"].get("replicas"),
"readyReplicas": d.get("status", {}).get("readyReplicas", 0),
"images": [c.get("image") for c in d["spec"]["template"]["spec"]["containers"]],
}
for d in source_deploys.get("items", [])
],
"shadowWorkloads": [
{
"kind": w["kind"],
"name": w["metadata"]["name"],
"replicas": w["spec"].get("replicas"),
"readyReplicas": w.get("status", {}).get("readyReplicas", 0),
"images": [c.get("image") for c in w["spec"]["template"]["spec"]["containers"]],
}
for w in shadow_deploys.get("items", []) + shadow_sts.get("items", [])
],
"shadowQuotas": [
{
"name": q["metadata"]["name"],
"hard": q.get("spec", {}).get("hard"),
"used": q.get("status", {}).get("used"),
}
for q in shadow_quotas.get("items", [])
],
"shadowNetworkPolicies": [p["metadata"]["name"] for p in shadow_netpol.get("items", [])],
}
path = write_report(out_dir, "preflight.json", report)
print(json.dumps({"ok": True, "phase": "preflight", "report": str(path), "apiReady": api_health["readyReplicas"]}))
return 0
def read_shadow_safety(client):
js = "const fs=require('fs');const q=db.getSiblingDB('" + QUERY_DB_NAME + "');if(q.auth('shadow-health',fs.readFileSync('/health-credentials/health-password','utf8').trim()).ok!==1)quit(2);print(EJSON.stringify({safety:q.queryReplicaControl.findOne({_id:'safety'}),health:q.queryReplicaControl.findOne({_id:'safety-health'})},{relaxed:true}));"
p = client.run(["-n",SHADOW_NAMESPACE,"exec",SHADOW_MONGO_POD,"-c",SHADOW_MONGO_CONTAINER,"--","mongosh","--quiet","--host","127.0.0.1","--port","27129","--eval",js], timeout=25)
return json.loads(p.stdout.strip().splitlines()[-1])
def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
baseline_health = inspect_api_health(client)
if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0:
raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}")
mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT)
if int(mon_dep.get("status", {}).get("availableReplicas") or 0) != 1:
raise RuntimeError("approved shadow observer must be healthy; it is never disabled by synchronization")
policy = client.approved_plan["shadowPolicy"]
safety = read_shadow_safety(client)
health = safety.get("health") or {}
recent = health.get("recent", [])[-2:]
if len(recent) != 2 or any(not h.get("ready") or not h.get("sourceAvailable") or h.get("workErrors") or h.get("memoryPressure") or h.get("persistenceFailure") for h in recent):
raise RuntimeError("two fresh complete health observations required")
for h in recent:
at = parse_iso_datetime(h.get("at"))
if at is None or not 0 <= (utc_now()-at).total_seconds() <= 60:
raise RuntimeError("shadow health evidence is stale")
saved = safety.get("safety") or {}
if (saved.get("policy") or {}).get("runId") != policy["runID"]:
raise RuntimeError("synchronization run changed; cannot reset checkpoint/budget")
if int(saved.get("reserved") or 0) >= policy["maxRecords"]:
raise RuntimeError("approved cumulative budget exhausted")
if saved.get("paused") and saved.get("reason") not in policy.get("resumeReasons", ["operator_stop", "pilot_budget", "maintenance_window_closed"]):
raise RuntimeError("unexpected hard pause requires diagnosis, not automatic reset")
nonce = saved.get("resumeToken", "") if saved.get("paused") else ""
# Scale builder to 0 first to stay well inside ResourceQuota.
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={BUILDER_CONTAINER}")
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}")
rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT)
rep_idx = find_container_index(rep_dep, REPLICA_CONTAINER)
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][rep_idx].get("env") or []
rep_patch = build_deployment_json_patch(
container_index=rep_idx,
expected_deployment=rep_dep,
strategy_recreate=True,
image=args.replica_image,
args=build_shadow_replica_args(client.approved_plan, nonce),
env_list=build_shadow_replica_env(rep_env),
)
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1)
if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180):
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
raise RuntimeError("query-replica failed rollout")
sync_deadline = time.monotonic() + args.sync_timeout_seconds
synced_snap = None
while time.monotonic() < sync_deadline:
health = inspect_api_health(client)
if (
health["readyReplicas"] < baseline_health["desiredReplicas"]
or health["totalRestarts"] > baseline_health["totalRestarts"]
):
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
write_report(out_dir, "shadow-sync-aborted.json", {"reason": "api_degraded", "health": health})
raise RuntimeError(f"API health degraded during shadow-sync; scaled projector to 0: {health}")
snap = query_shadow_mongo_snapshot(client)
state = snap.get("state") or {}
if state.get("ready") is True:
obs = parse_iso_datetime(state.get("observedAt"))
if obs is not None and (utc_now() - obs).total_seconds() < 300:
synced_snap = snap
break
time.sleep(15)
if synced_snap is None:
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
raise RuntimeError("timed out waiting for query-replica ready state")
bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT)
bld_idx = find_container_index(bld_dep, BUILDER_CONTAINER)
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][bld_idx].get("env") or []
bld_patch = build_deployment_json_patch(
container_index=bld_idx,
expected_deployment=bld_dep,
strategy_recreate=True,
image=args.builder_image,
args=build_shadow_builder_args(client.approved_plan),
env_list=build_shadow_builder_env(bld_env),
)
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1)
if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180):
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
raise RuntimeError("query-read-builder failed rollout")
build_deadline = time.monotonic() + args.build_timeout_seconds
built_snap = None
while time.monotonic() < build_deadline:
health = inspect_api_health(client)
if (
health["readyReplicas"] < baseline_health["desiredReplicas"]
or health["totalRestarts"] > baseline_health["totalRestarts"]
):
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
write_report(out_dir, "shadow-build-aborted.json", {"reason": "api_degraded", "health": health})
raise RuntimeError(f"API health degraded during builder run; scaled builder to 0: {health}")
snap = query_shadow_mongo_snapshot(client)
ok, reasons = evaluate_parity_snapshot(snap)
if ok:
built_snap = snap
break
time.sleep(15)
if built_snap is None:
raise RuntimeError("timed out waiting for query-read-builder completion")
path = write_report(out_dir, "shadow-sync.json", {"at": utc_now().isoformat(), "snapshot": built_snap})
print(json.dumps({"ok": True, "phase": "shadow-sync", "report": str(path)}))
return 0
def cmd_verify_parity(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
snap = query_shadow_mongo_snapshot(client)
ok, reasons = evaluate_parity_snapshot(snap)
if ok:
try:
if not getattr(args, "comparison_evidence", ""):
raise RuntimeError("result/permission comparison evidence missing")
plan = client.approved_plan or json.loads(Path(args.approved_plan).read_text())
require_result_comparison(json.loads(Path(args.comparison_evidence).read_text()), snap, plan)
except (RuntimeError, OSError, ValueError, KeyError) as error:
ok = False
reasons.append(str(error))
report = {
"at": utc_now().isoformat(),
"passed": ok,
"reasons": reasons,
"snapshot": snap,
}
path = write_report(out_dir, "verify-parity.json", report)
print(json.dumps({"ok": ok, "phase": "verify-parity", "reasons": reasons, "report": str(path)}))
return 0 if ok else 1
def sync_read_client_secret(client: ClusterClient) -> None:
conn_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CONNECTIONS_SECRET)
cache_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CACHE_AUTH_SECRET)
api_uri_b64 = (conn_secret.get("data") or {}).get("api-uri")
valkey_url_b64 = (cache_secret.get("data") or {}).get("url")
if not api_uri_b64 or not valkey_url_b64:
raise RuntimeError("required shadow read client secret keys missing")
manifest = build_read_client_secret_manifest(api_uri_b64, valkey_url_b64)
existing = client.get_json(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, ignore_not_found=True)
if existing is not None:
client.patch_json_ops(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, [{"op":"replace", "path":"/data", "value":manifest["data"]}])
else:
resource = SOURCE_NAMESPACE + "/secret/" + READ_CLIENT_SECRET_NAME
if resource not in client.approved_plan.get("createResources", []):
raise RuntimeError("read client Secret creation not authorized")
client.run(["create", "-f", "-"], stdin_text=json.dumps(manifest))
def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
api_idx = find_container_index(api_dep, API_CONTAINER)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates())
patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env, expected_deployment=api_dep)
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180)
health = inspect_api_health(client)
path = write_report(
out_dir,
"rollback-api-flags.json",
{"at": utc_now().isoformat(), "rolloutOk": ok, "apiHealth": health},
)
print(json.dumps({"ok": ok, "phase": "rollback-api-flags", "report": str(path)}))
return 0 if ok else 1
def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
snap = query_shadow_mongo_snapshot(client)
ok, reasons = evaluate_parity_snapshot(snap)
if not ok:
raise RuntimeError(f"parity gate blocked cutover: {reasons}")
if not getattr(args, "comparison_evidence", ""):
raise RuntimeError("new/old result and permission evidence required before production cutover")
require_result_comparison(json.loads(Path(args.comparison_evidence).read_text()), snap, client.approved_plan)
sync_read_client_secret(client)
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
api_idx = find_container_index(api_dep, API_CONTAINER)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env})
next_env = merge_env_list(existing_env, build_api_cutover_env_updates())
patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env, expected_deployment=api_dep)
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180):
cmd_rollback_api_flags(args, client, out_dir)
raise RuntimeError("droboticflow-api rollout failed after flag cutover; rolled back")
health = inspect_api_health(client)
path = write_report(
out_dir,
"cutover-api-flags.json",
{"at": utc_now().isoformat(), "env": next_env, "apiHealth": health},
)
print(json.dumps({"ok": True, "phase": "cutover-api-flags", "report": str(path)}))
return 0
def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
baseline = inspect_api_health(client)
deadline = time.monotonic() + args.watch_seconds
samples: list[dict[str, Any]] = []
rolled_back = False
trigger_reason = ""
while time.monotonic() < deadline:
time.sleep(min(20, max(1, int(deadline - time.monotonic()))))
health = inspect_api_health(client)
log_proc = client.run(
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=25s", "--tail=300"],
timeout=15,
check=False,
)
if log_proc.returncode != 0:
raise RuntimeError("API health log probe unavailable; no zero-error claim")
error_lines = count_error_log_lines(log_proc.stdout)
sample = {
"at": utc_now().isoformat(),
"readyReplicas": health["readyReplicas"],
"desiredReplicas": health["desiredReplicas"],
"totalRestarts": health["totalRestarts"],
"errorLines": error_lines,
}
samples.append(sample)
if health["readyReplicas"] < baseline["desiredReplicas"]:
trigger_reason = f"readyReplicas dropped: {health['readyReplicas']}/{baseline['desiredReplicas']}"
elif health["totalRestarts"] > baseline["totalRestarts"]:
trigger_reason = f"API pod restarts increased: {baseline['totalRestarts']} -> {health['totalRestarts']}"
elif error_lines >= args.max_error_lines:
trigger_reason = f"error log spike: {error_lines} lines in 25s window"
if trigger_reason:
rolled_back = True
cmd_rollback_api_flags(args, client, out_dir)
break
report = {
"at": utc_now().isoformat(),
"passed": not rolled_back,
"rolledBack": rolled_back,
"triggerReason": trigger_reason,
"samples": samples,
}
path = write_report(out_dir, "watch-and-guard.json", report)
print(json.dumps({"ok": not rolled_back, "phase": "watch-and-guard", "rolledBack": rolled_back, "report": str(path)}))
return 1 if rolled_back else 0
def api_cutover_active(api_env: list[dict[str, Any]]) -> bool:
by_name = {entry.get("name"): entry.get("value") for entry in (api_env or [])}
return by_name.get("QUERY_REPLICA_REPORTS") == "1" or by_name.get("QUERY_REPLICA_COUNTS") == "1"
def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
deploys_by_name = {d["metadata"]["name"]: d for d in source_deploys.get("items", [])}
business_health = []
unhealthy_business = []
for name in PRODUCTION_BUSINESS_DEPLOYMENTS:
dep = deploys_by_name.get(name)
if dep is None:
business_health.append({"name": name, "present": False, "desiredReplicas": 0, "readyReplicas": 0})
unhealthy_business.append(f"{name}: missing")
continue
desired = int(dep["spec"].get("replicas") or 0)
ready = int(dep.get("status", {}).get("readyReplicas") or 0)
business_health.append({"name": name, "present": True, "desiredReplicas": desired, "readyReplicas": ready})
if desired == 0 or ready < desired:
unhealthy_business.append(f"{name}: {ready}/{desired}")
api_dep = deploys_by_name.get(API_DEPLOYMENT)
if api_dep is None:
raise RuntimeError(f"missing {API_DEPLOYMENT} deployment in {SOURCE_NAMESPACE}")
api_idx = find_container_index(api_dep, API_CONTAINER)
api_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
cutover_active = api_cutover_active(api_env)
api_health = inspect_api_health(client)
api_healthy = (
api_health["desiredReplicas"] > 0
and api_health["readyReplicas"] >= api_health["desiredReplicas"]
)
log_proc = client.run(
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=12m", "--tail=500"],
timeout=15,
check=False,
)
if log_proc.returncode != 0:
raise RuntimeError("API health log probe unavailable; no zero-error claim")
error_lines = count_error_log_lines(log_proc.stdout)
snap = None
parity_ok = False
parity_reasons: list[str] = []
try:
snap = query_shadow_mongo_snapshot(client)
parity_ok, parity_reasons = evaluate_parity_snapshot(
snap,
max_observed_age_seconds=args.max_observed_age_seconds,
max_workforce_age_seconds=args.max_workforce_age_seconds,
)
except Exception as exc:
parity_ok = False
parity_reasons = [f"shadow mongo probe failed: {type(exc).__name__}"]
trigger_reasons: list[str] = []
if cutover_active:
if not api_healthy:
trigger_reasons.append(
f"API degraded: {api_health['readyReplicas']}/{api_health['desiredReplicas']}"
)
if error_lines >= args.max_error_lines:
trigger_reasons.append(f"API error log spike: {error_lines} lines in 12m")
if not parity_ok:
trigger_reasons.extend(parity_reasons)
rolled_back = False
if cutover_active and trigger_reasons:
rolled_back = True
cmd_rollback_api_flags(args, client, out_dir)
stamp = utc_now().strftime("%Y%m%dT%H%M%SZ")
report = {
"at": utc_now().isoformat(),
"cutoverActive": cutover_active,
"businessHealth": business_health,
"unhealthyBusiness": unhealthy_business,
"apiHealth": api_health,
"errorLines12m": error_lines,
"parityOk": parity_ok,
"parityReasons": parity_reasons,
"rolledBack": rolled_back,
"triggerReasons": trigger_reasons,
"shadowMongoSnapshot": snap,
}
path = write_report(out_dir, f"patrol-{stamp}.json", report)
ok = (not rolled_back) and (len(unhealthy_business) == 0) and (not cutover_active or parity_ok)
print(
json.dumps(
{
"ok": ok,
"phase": "patrol",
"cutoverActive": cutover_active,
"rolledBack": rolled_back,
"errorLines12m": error_lines,
"parityOk": parity_ok,
"unhealthyBusiness": unhealthy_business,
"report": str(path),
}
)
)
return 0 if ok else 1
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.")
parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
parser.add_argument("--approved-plan", default="")
parser.add_argument("--confirm-plan", default="")
parser.add_argument("--approval-ref", default="")
parser.add_argument("--comparison-evidence", default="")
parser.add_argument("--run-id", default=default_run_id())
parser.add_argument("--output-dir", default="")
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("preflight")
p_sync = sub.add_parser("shadow-sync")
p_sync.add_argument("--replica-image", required=True)
p_sync.add_argument("--builder-image", required=True)
p_sync.add_argument("--sync-timeout-seconds", type=int, default=1800)
p_sync.add_argument("--build-timeout-seconds", type=int, default=1200)
sub.add_parser("verify-parity")
sub.add_parser("cutover-api-flags")
p_watch = sub.add_parser("watch-and-guard")
p_watch.add_argument("--watch-seconds", type=int, default=900)
p_watch.add_argument("--max-error-lines", type=int, default=5)
p_patrol = sub.add_parser("patrol")
p_patrol.add_argument("--max-error-lines", type=int, default=5)
p_patrol.add_argument("--max-observed-age-seconds", type=int, default=600)
p_patrol.add_argument("--max-workforce-age-seconds", type=int, default=720)
sub.add_parser("rollback-api-flags")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
out_dir = (
Path(args.output_dir)
if args.output_dir
else Path(__file__).resolve().parents[2] / "local-data" / "diagnostics" / "evening-cutover" / args.run_id
)
client = ClusterClient(args.kubeconfig)
client.approved_plan = approved_plan(args, client)
handlers = {
"preflight": cmd_preflight,
"shadow-sync": cmd_shadow_sync,
"verify-parity": cmd_verify_parity,
"cutover-api-flags": cmd_cutover_api_flags,
"watch-and-guard": cmd_watch_and_guard,
"patrol": cmd_patrol,
"rollback-api-flags": cmd_rollback_api_flags,
}
try:
return handlers[args.command](args, client, out_dir)
except Exception:
if args.command == "shadow-sync" and client.approved_plan:
failures = []
for name in (REPLICA_DEPLOYMENT, BUILDER_DEPLOYMENT):
try:
client.scale(SHADOW_NAMESPACE, "deployment", name, 0)
except Exception as error:
failures.append({"controller":name, "errorClass":type(error).__name__})
write_report(out_dir, "shadow-sync-stopped.json", {"progressRetained":True,"stopFailures":failures})
raise
if __name__ == "__main__":
sys.exit(main())