1085 lines
46 KiB
Python
1085 lines
46 KiB
Python
#!/usr/bin/env python3
|
|
"""Unattended evening shadow-sync, parity gate, API read cutover, and auto-rollback."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from typing import Any
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from query_cutover_guard import approved_plan, bounded_shadow_args, require_result_comparison
|
|
|
|
DEFAULT_KUBECONFIG = "/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf"
|
|
SOURCE_NAMESPACE = "droboticflow-fanyu"
|
|
SHADOW_NAMESPACE = "droboticflow-query-shadow"
|
|
QUERY_DB_NAME = "drobodata_fanyu_query_shadow_20261114"
|
|
QUERY_SERVING_NAMESPACE = "fanyu-query-shadow-20261114"
|
|
READ_CLIENT_SECRET_NAME = "droboticflow-query-read-client"
|
|
SHADOW_CONNECTIONS_SECRET = "query-shadow-query-connections"
|
|
SHADOW_CACHE_AUTH_SECRET = "query-shadow-cache-auth"
|
|
|
|
API_DEPLOYMENT = "droboticflow-api"
|
|
API_CONTAINER = "api"
|
|
REPLICA_DEPLOYMENT = "query-replica"
|
|
REPLICA_CONTAINER = "projector"
|
|
BUILDER_DEPLOYMENT = "query-read-builder"
|
|
BUILDER_CONTAINER = "builder"
|
|
MONITOR_DEPLOYMENT = "query-shadow-monitor"
|
|
SHADOW_MONGO_POD = "query-mongo-0"
|
|
SHADOW_MONGO_CONTAINER = "mongo"
|
|
PRODUCTION_BUSINESS_DEPLOYMENTS = (
|
|
"droboticflow-api",
|
|
"droboticflow-annotation-processor",
|
|
"droboticflow-annotation-validator",
|
|
"droboticflow-worker",
|
|
"drosync-coordinator",
|
|
)
|
|
|
|
SECRET_KEY_PATTERN = re.compile(
|
|
r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)",
|
|
re.IGNORECASE,
|
|
)
|
|
ERROR_LOG_PATTERN = re.compile(
|
|
r"(ErrQueryReplicaNotReady|查询暂不可用|\bstatus=(?:500|503)\b|\"\s+(?:500|503)\s+)",
|
|
)
|
|
|
|
|
|
def utc_now() -> dt.datetime:
|
|
return dt.datetime.now(dt.timezone.utc)
|
|
|
|
|
|
def default_run_id() -> str:
|
|
return utc_now().strftime("%Y%m%dT%H%M%SZ")
|
|
|
|
|
|
def parse_iso_datetime(raw: Any) -> dt.datetime | None:
|
|
if not isinstance(raw, str) or not raw.strip():
|
|
return None
|
|
text = raw.strip()
|
|
if text.endswith("Z"):
|
|
text = text[:-1] + "+00:00"
|
|
try:
|
|
parsed = dt.datetime.fromisoformat(text)
|
|
except ValueError:
|
|
return None
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
|
return parsed.astimezone(dt.timezone.utc)
|
|
|
|
|
|
def redact_secrets(obj: Any, parent_key: str = "") -> Any:
|
|
"""Recursively scrub secret payloads, URIs, and credentials before persistence/logging."""
|
|
if isinstance(obj, dict):
|
|
kind = obj.get("kind")
|
|
if kind == "Secret":
|
|
out = {}
|
|
for k, v in obj.items():
|
|
if k in ("data", "stringData") and isinstance(v, dict):
|
|
out[k] = {sk: "<redacted>" for sk in v.keys()}
|
|
elif k == "metadata" and isinstance(v, dict):
|
|
meta = dict(v)
|
|
if isinstance(meta.get("annotations"), dict):
|
|
anns = dict(meta["annotations"])
|
|
if "kubectl.kubernetes.io/last-applied-configuration" in anns:
|
|
anns["kubectl.kubernetes.io/last-applied-configuration"] = "<redacted>"
|
|
meta["annotations"] = anns
|
|
out[k] = meta
|
|
else:
|
|
out[k] = redact_secrets(v, k)
|
|
return out
|
|
# Redact Kubernetes env var literal values if key name looks sensitive.
|
|
if "name" in obj and "value" in obj and isinstance(obj.get("name"), str):
|
|
env_name = obj["name"]
|
|
if SECRET_KEY_PATTERN.search(env_name):
|
|
redacted = dict(obj)
|
|
redacted["value"] = "<redacted>"
|
|
return redacted
|
|
return {k: redact_secrets(v, k) for k, v in obj.items()}
|
|
if isinstance(obj, list):
|
|
return [redact_secrets(item, parent_key) for item in obj]
|
|
if isinstance(obj, str) and obj.startswith("--resume-pause="):
|
|
return "--resume-pause=<redacted>"
|
|
if isinstance(obj, str) and parent_key and SECRET_KEY_PATTERN.search(parent_key):
|
|
return "<redacted>"
|
|
return obj
|
|
|
|
|
|
def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Merge env updates into existing Kubernetes env list preserving untouched vars and order."""
|
|
by_name = {entry["name"]: dict(entry) for entry in updates}
|
|
seen: set[str] = set()
|
|
merged: list[dict[str, Any]] = []
|
|
for entry in existing_env or []:
|
|
name = entry.get("name")
|
|
if not isinstance(name, str) or name in seen:
|
|
continue
|
|
seen.add(name)
|
|
if name in by_name:
|
|
merged.append(by_name[name])
|
|
else:
|
|
merged.append(dict(entry))
|
|
for entry in updates:
|
|
name = entry["name"]
|
|
if name not in seen:
|
|
seen.add(name)
|
|
merged.append(dict(entry))
|
|
return merged
|
|
|
|
|
|
def find_container_index(dep: dict[str, Any], container_name: str) -> int:
|
|
containers = dep.get("spec", {}).get("template", {}).get("spec", {}).get("containers") or []
|
|
for idx, container in enumerate(containers):
|
|
if container.get("name") == container_name:
|
|
return idx
|
|
if len(containers) <= 1:
|
|
return 0
|
|
raise RuntimeError(f"container {container_name} not found in deployment")
|
|
|
|
|
|
def build_deployment_json_patch(
|
|
*,
|
|
container_index: int = 0,
|
|
env_list: list[dict[str, Any]],
|
|
strategy_recreate: bool = False,
|
|
image: str | None = None,
|
|
args: list[str] | None = None,
|
|
expected_deployment: dict[str, Any] | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
"""Build an atomic RFC 6902 JSON patch for a single deployment container."""
|
|
ops: list[dict[str, Any]] = []
|
|
if expected_deployment:
|
|
ops.extend([{"op":"test", "path":"/metadata/uid", "value":expected_deployment["metadata"]["uid"]},
|
|
{"op":"test", "path":"/metadata/resourceVersion", "value":expected_deployment["metadata"]["resourceVersion"]}])
|
|
if strategy_recreate:
|
|
ops.append({"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
|
|
ops.append(
|
|
{
|
|
"op": "add",
|
|
"path": f"/spec/template/spec/containers/{container_index}/env",
|
|
"value": env_list,
|
|
}
|
|
)
|
|
if image is not None:
|
|
ops.append(
|
|
{
|
|
"op": "add",
|
|
"path": f"/spec/template/spec/containers/{container_index}/image",
|
|
"value": image,
|
|
}
|
|
)
|
|
if args is not None:
|
|
ops.append(
|
|
{
|
|
"op": "add",
|
|
"path": f"/spec/template/spec/containers/{container_index}/args",
|
|
"value": list(args),
|
|
}
|
|
)
|
|
return ops
|
|
|
|
|
|
def build_shadow_replica_args(plan, resume_nonce="") -> list[str]:
|
|
return bounded_shadow_args(plan, resume_nonce)
|
|
|
|
|
|
def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
updates = [
|
|
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
|
|
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
|
|
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
|
|
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
|
{"name": "QUERY_SERVING_READS", "value": "1"},
|
|
]
|
|
return merge_env_list(existing_env, updates)
|
|
|
|
|
|
def build_shadow_builder_args(plan) -> list[str]:
|
|
return ["--ensure-indexes", "--shadow-safety-run=" + plan["shadowPolicy"]["runID"]]
|
|
|
|
|
|
def build_shadow_builder_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
updates = [
|
|
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
|
|
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
|
|
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
|
|
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
|
{"name": "QUERY_SERVING_READS", "value": "1"},
|
|
]
|
|
return merge_env_list(existing_env, updates)
|
|
|
|
|
|
def build_api_cutover_env_updates() -> list[dict[str, Any]]:
|
|
return [
|
|
{"name": "QUERY_SERVING_NAMESPACE", "value": QUERY_SERVING_NAMESPACE},
|
|
{"name": "QUERY_READ_MONGO_DB", "value": QUERY_DB_NAME},
|
|
{"name": "QUERY_REPLICA_REFRESH_MODE", "value": "periodic"},
|
|
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
|
{"name": "QUERY_SERVING_READS", "value": "1"},
|
|
{"name": "QUERY_REPLICA_REPORTS", "value": "1"},
|
|
{"name": "QUERY_REPLICA_COUNTS", "value": "1"},
|
|
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
|
|
{
|
|
"name": "QUERY_READ_MONGO_URI",
|
|
"valueFrom": {
|
|
"secretKeyRef": {
|
|
"name": READ_CLIENT_SECRET_NAME,
|
|
"key": "api-uri",
|
|
}
|
|
},
|
|
},
|
|
{
|
|
"name": "QUERY_VALKEY_URL",
|
|
"valueFrom": {
|
|
"secretKeyRef": {
|
|
"name": READ_CLIENT_SECRET_NAME,
|
|
"key": "valkey-url",
|
|
}
|
|
},
|
|
},
|
|
]
|
|
|
|
|
|
def build_api_rollback_env_updates() -> list[dict[str, Any]]:
|
|
return [
|
|
{"name": "QUERY_REPLICA_REPORTS", "value": "0"},
|
|
{"name": "QUERY_REPLICA_COUNTS", "value": "0"},
|
|
{"name": "QUERY_REPLICA_POOLS", "value": "0"},
|
|
{"name": "QUERY_SERVING_READS", "value": "0"},
|
|
{"name": "QUERY_PUBLISHED_RESULTS", "value": "0"},
|
|
]
|
|
|
|
|
|
def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) -> dict[str, Any]:
|
|
return {
|
|
"apiVersion": "v1",
|
|
"kind": "Secret",
|
|
"metadata": {
|
|
"name": READ_CLIENT_SECRET_NAME,
|
|
"namespace": SOURCE_NAMESPACE,
|
|
},
|
|
"type": "Opaque",
|
|
"data": {
|
|
"api-uri": api_uri_b64,
|
|
"valkey-url": valkey_url_b64,
|
|
},
|
|
}
|
|
|
|
|
|
def parse_bson_int(val: Any) -> int:
|
|
"""Parse int, float, numeric string, or mongosh Long {high, low, unsigned} objects."""
|
|
if val is None:
|
|
return 0
|
|
if isinstance(val, bool):
|
|
return int(val)
|
|
if isinstance(val, (int, float)):
|
|
return int(val)
|
|
if isinstance(val, str):
|
|
try:
|
|
return int(val.strip())
|
|
except ValueError:
|
|
return 0
|
|
if isinstance(val, dict) and ("low" in val or "high" in val):
|
|
try:
|
|
low = int(val.get("low") or 0) & 0xFFFFFFFF
|
|
high = int(val.get("high") or 0)
|
|
if val.get("unsigned"):
|
|
return ((high & 0xFFFFFFFF) << 32) | low
|
|
return (high << 32) | low
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
return 0
|
|
|
|
|
|
def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
|
|
if not isinstance(global_doc, dict):
|
|
return False, {"hasProduction": False, "hasWorkforce": False}
|
|
heads = global_doc.get("heads") or {}
|
|
if not isinstance(heads, dict):
|
|
return False, {"hasProduction": False, "hasWorkforce": False}
|
|
has_production = False
|
|
has_workforce = False
|
|
workforce_epoch = 0
|
|
workforce_as_of = None
|
|
production_as_of = None
|
|
for head in heads.values():
|
|
if not isinstance(head, dict):
|
|
continue
|
|
gen = head.get("generation") or ""
|
|
as_of = head.get("asOf")
|
|
if gen == "workforce-facts/v1":
|
|
epoch = parse_bson_int(head.get("factEpoch"))
|
|
if epoch >= 1 and as_of:
|
|
has_workforce = True
|
|
workforce_epoch = max(workforce_epoch, epoch)
|
|
workforce_as_of = as_of
|
|
elif gen and not head.get("searchCollection") and parse_bson_int(head.get("chunks")) >= 1 and as_of:
|
|
has_production = True
|
|
production_as_of = as_of
|
|
details = {
|
|
"hasProduction": has_production,
|
|
"hasWorkforce": has_workforce,
|
|
"workforceFactEpoch": workforce_epoch,
|
|
"workforceAsOf": workforce_as_of,
|
|
"productionAsOf": production_as_of,
|
|
}
|
|
return (has_production and has_workforce), details
|
|
|
|
|
|
def evaluate_parity_snapshot(
|
|
snapshot: dict[str, Any],
|
|
*,
|
|
now: dt.datetime | None = None,
|
|
max_observed_age_seconds: int = 300,
|
|
max_workforce_age_seconds: int = 900,
|
|
) -> tuple[bool, list[str]]:
|
|
current_time = now or utc_now()
|
|
reasons: list[str] = []
|
|
|
|
state = snapshot.get("state") or {}
|
|
if state.get("ready") is not True:
|
|
reasons.append("replica state.ready is not true")
|
|
|
|
observed_dt = parse_iso_datetime(state.get("observedAt"))
|
|
if observed_dt is None:
|
|
reasons.append("replica state.observedAt missing or invalid")
|
|
else:
|
|
obs_age = (current_time - observed_dt).total_seconds()
|
|
if obs_age < -5 or obs_age >= max_observed_age_seconds:
|
|
reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s")
|
|
|
|
heads_ok, head_info = evaluate_builder_heads(snapshot.get("global"))
|
|
if not heads_ok or parse_bson_int(head_info.get("workforceFactEpoch")) < 1:
|
|
reasons.append("builder workforce or production head incomplete")
|
|
else:
|
|
wf_dt = parse_iso_datetime(head_info.get("workforceAsOf"))
|
|
if wf_dt is None:
|
|
reasons.append("workforce head asOf missing or invalid")
|
|
else:
|
|
wf_age = (current_time - wf_dt).total_seconds()
|
|
if wf_age < -5 or wf_age >= max_workforce_age_seconds:
|
|
reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s")
|
|
|
|
cohorts = snapshot.get("cohorts") or []
|
|
if not cohorts:
|
|
reasons.append("scoped publication cohort identities missing")
|
|
heads = (snapshot.get("global") or {}).get("heads") or {}
|
|
for cohort in cohorts:
|
|
production = heads.get(cohort.get("productionKey")) or {}
|
|
workforce = heads.get(cohort.get("workforceKey")) or {}
|
|
if not production.get("generation") or parse_bson_int(production.get("chunks")) < 1 or workforce.get("generation") != "workforce-facts/v1":
|
|
reasons.append("scoped publication incomplete")
|
|
continue
|
|
for field in ("asOf", "sourceRevision", "globalRevision"):
|
|
if production.get(field) is None or production.get(field) != workforce.get(field):
|
|
reasons.append("mixed scoped publication cohort")
|
|
break
|
|
for head in (production, workforce):
|
|
at = parse_iso_datetime(head.get("asOf"))
|
|
if at is None or not 0 <= (current_time-at).total_seconds() < max_workforce_age_seconds:
|
|
reasons.append("scoped publication stale")
|
|
|
|
if parse_bson_int(snapshot.get("poolBucketsCount")) <= 0:
|
|
reasons.append("queryPoolCountBuckets is empty")
|
|
if parse_bson_int(snapshot.get("workforceFactsCount")) <= 0:
|
|
reasons.append("queryWorkforceFacts is empty")
|
|
|
|
return len(reasons) == 0, reasons
|
|
|
|
|
|
def count_error_log_lines(log_text: str) -> int:
|
|
return sum(1 for line in log_text.splitlines() if ERROR_LOG_PATTERN.search(line))
|
|
|
|
|
|
class ClusterClient:
|
|
"""Thin kubectl wrapper enforcing kubeconfig, timeouts, and non-secret output."""
|
|
|
|
def __init__(self, kubeconfig: str) -> None:
|
|
self.kubeconfig = kubeconfig
|
|
self.approved_plan = None
|
|
|
|
def run(
|
|
self,
|
|
args: list[str],
|
|
*,
|
|
stdin_text: str | None = None,
|
|
timeout: int = 30,
|
|
check: bool = True,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
cmd = ["kubectl", "--kubeconfig", self.kubeconfig, *args]
|
|
proc = subprocess.run(
|
|
cmd,
|
|
input=stdin_text,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
if check and proc.returncode != 0:
|
|
raise RuntimeError(
|
|
f"kubectl failed rc={proc.returncode} args={' '.join(args[:4])}: {proc.stderr.strip()[:240]}"
|
|
)
|
|
return proc
|
|
|
|
def get_json(self, namespace: str | None, *resource_args: str, ignore_not_found: bool = False) -> Any:
|
|
args = []
|
|
if namespace:
|
|
args.extend(["-n", namespace])
|
|
args.extend(["get", *resource_args, "-o", "json"])
|
|
if ignore_not_found:
|
|
args.append("--ignore-not-found")
|
|
for attempt in range(3):
|
|
try:
|
|
proc = self.run(args, timeout=8, check=False)
|
|
if proc.returncode == 0:
|
|
if ignore_not_found and not proc.stdout.strip():
|
|
return None
|
|
return json.loads(proc.stdout)
|
|
if any(term in proc.stderr for term in ("Forbidden", "Unauthorized", "NotFound")):
|
|
raise RuntimeError("metadata authorization/identity failure")
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
pass
|
|
print(json.dumps({"event":"metadata_probe_retry","attempt":attempt+1}),file=sys.stderr)
|
|
if attempt < 2:
|
|
time.sleep(.5 * (attempt+1))
|
|
raise RuntimeError("metadata probe unavailable after bounded retries")
|
|
|
|
def patch_json_ops(self, namespace: str, kind: str, name: str, ops: list[dict[str, Any]]) -> None:
|
|
if not self.approved_plan:
|
|
raise RuntimeError("approved plan required before cluster writes")
|
|
current = self.get_json(namespace, kind, name)
|
|
expected = self.approved_plan["resourceUIDs"].get(namespace + "/" + kind + "/" + name)
|
|
if not expected or current["metadata"]["uid"] != expected:
|
|
raise RuntimeError("resource identity outside approved plan")
|
|
tests = [{"op":"test", "path":"/metadata/uid", "value":expected},
|
|
{"op":"test", "path":"/metadata/resourceVersion", "value":current["metadata"]["resourceVersion"]}]
|
|
self.run(["-n",namespace,"patch",kind,name,"--type=json","--patch-file=/dev/stdin"], stdin_text=json.dumps(tests + ops))
|
|
|
|
def scale(self, namespace: str, kind: str, name: str, replicas: int) -> None:
|
|
self.patch_json_ops(namespace, kind, name, [{"op":"replace", "path":"/spec/replicas", "value":replicas}])
|
|
|
|
def rollout_status(self, namespace: str, deployment: str, timeout_seconds: int = 180) -> bool:
|
|
proc = self.run(
|
|
["-n", namespace, "rollout", "status", f"deployment/{deployment}", f"--timeout={timeout_seconds}s"],
|
|
timeout=timeout_seconds + 15,
|
|
check=False,
|
|
)
|
|
return proc.returncode == 0
|
|
|
|
def wait_zero_pods(self, namespace: str, label_selector: str, timeout_seconds: int = 90) -> None:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while time.monotonic() < deadline:
|
|
pods = self.get_json(namespace, "pods", "-l", label_selector)
|
|
if not pods.get("items"):
|
|
return
|
|
time.sleep(2)
|
|
raise RuntimeError(f"timed out waiting for {namespace} pods {label_selector} to scale to zero")
|
|
|
|
|
|
def query_shadow_mongo_snapshot(client: ClusterClient) -> dict[str, Any]:
|
|
"""Inspect shadow control heads & derived counts inside query-mongo-0 without exposing creds."""
|
|
js = f"""
|
|
const fs = require('fs');
|
|
const q = db.getSiblingDB('{QUERY_DB_NAME}');
|
|
if (q.auth('shadow-health', fs.readFileSync('/health-credentials/health-password', 'utf8').trim()).ok !== 1) {{
|
|
quit(2);
|
|
}}
|
|
const st = q.queryReplicaControl.findOne({{_id: 'state'}});
|
|
const crypto = require('crypto');
|
|
const modelKey = (kind, org, projects) => 'query-model/v3:' + crypto.createHash('sha256').update(JSON.stringify([kind, org, projects])).digest('hex');
|
|
const cohorts = q.productionStatsScopes.find({{}}, {{projectIds:1}}).toArray().map(s => {{
|
|
const ids=q.projects.find({{id:{{$in:s.projectIds||[]}}}}, {{id:1}}).toArray().map(p=>p.id).sort();
|
|
return {{productionKey:modelKey('production',String(s._id).trim(),ids),workforceKey:modelKey('workforce-facts',String(s._id).trim(),[]) ,projects:ids.length}};
|
|
}}).filter(c=>c.projects>0);
|
|
const gl = q.queryReadModelControl.findOne({{_id: 'global'}});
|
|
const bootDone = q.queryReplicaControl.countDocuments({{_id: /^bootstrap\\//, complete: true}});
|
|
const poolBucketsCount = q.queryPoolCountBuckets.countDocuments({{}});
|
|
const workforceFactsCount = q.queryWorkforceFacts.countDocuments({{}});
|
|
const normHead = h => {{
|
|
if (!h) return null;
|
|
const out = Object.assign({{}}, h);
|
|
if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString();
|
|
if (out.factEpoch !== undefined && out.factEpoch !== null) out.factEpoch = Number(out.factEpoch);
|
|
if (out.chunks !== undefined && out.chunks !== null) out.chunks = Number(out.chunks);
|
|
if (out.rows !== undefined && out.rows !== null) out.rows = Number(out.rows);
|
|
if (out.bytes !== undefined && out.bytes !== null) out.bytes = Number(out.bytes);
|
|
return out;
|
|
}};
|
|
const normHeads = m => {{
|
|
const res = {{}};
|
|
for (const [k, v] of Object.entries(m || {{}})) res[k] = normHead(v);
|
|
return res;
|
|
}};
|
|
print(JSON.stringify({{
|
|
state: st ? {{
|
|
ready: Boolean(st.ready),
|
|
observedAt: st.observedAt instanceof Date ? st.observedAt.toISOString() : null,
|
|
factEpoch: Number(st.factEpoch || 0),
|
|
schema: st.schema || null
|
|
}} : null,
|
|
cohorts: cohorts,
|
|
bootstrapCompleteCount: bootDone,
|
|
poolBucketsCount: poolBucketsCount,
|
|
workforceFactsCount: workforceFactsCount,
|
|
global: gl ? {{
|
|
productionRevision: Number(gl.productionRevision || 0),
|
|
observedAt: gl.observedAt instanceof Date ? gl.observedAt.toISOString() : null,
|
|
heads: normHeads(gl.heads)
|
|
}} : null
|
|
}}));
|
|
"""
|
|
proc = client.run(
|
|
[
|
|
"-n",
|
|
SHADOW_NAMESPACE,
|
|
"exec",
|
|
SHADOW_MONGO_POD,
|
|
"-c",
|
|
SHADOW_MONGO_CONTAINER,
|
|
"--",
|
|
"mongosh",
|
|
"--quiet",
|
|
"--host",
|
|
"127.0.0.1",
|
|
"--port",
|
|
"27129",
|
|
"--eval",
|
|
js,
|
|
],
|
|
timeout=25,
|
|
)
|
|
return json.loads(proc.stdout.strip().splitlines()[-1])
|
|
|
|
|
|
def inspect_api_health(client: ClusterClient) -> dict[str, Any]:
|
|
dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
|
pods = client.get_json(SOURCE_NAMESPACE, "pods", "-l", f"app={API_DEPLOYMENT}")
|
|
desired = int(dep["spec"].get("replicas") or 0)
|
|
ready_replicas = int(dep.get("status", {}).get("readyReplicas") or 0)
|
|
pod_states = []
|
|
total_restarts = 0
|
|
for pod in pods.get("items", []):
|
|
statuses = pod.get("status", {}).get("containerStatuses") or []
|
|
restarts = sum(int(cs.get("restartCount") or 0) for cs in statuses)
|
|
total_restarts += restarts
|
|
ready = bool(statuses) and all(bool(cs.get("ready")) for cs in statuses)
|
|
pod_states.append(
|
|
{
|
|
"name": pod["metadata"]["name"],
|
|
"uid": pod["metadata"]["uid"],
|
|
"ready": ready,
|
|
"restarts": restarts,
|
|
}
|
|
)
|
|
return {
|
|
"desiredReplicas": desired,
|
|
"readyReplicas": ready_replicas,
|
|
"totalRestarts": total_restarts,
|
|
"pods": pod_states,
|
|
}
|
|
|
|
|
|
def write_report(out_dir: Path, filename: str, payload: Any) -> Path:
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
path = out_dir / filename
|
|
path.write_text(json.dumps(redact_secrets(payload), ensure_ascii=False, indent=2) + "\n")
|
|
return path
|
|
|
|
|
|
def cmd_preflight(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
|
|
shadow_deploys = client.get_json(SHADOW_NAMESPACE, "deployments")
|
|
shadow_sts = client.get_json(SHADOW_NAMESPACE, "statefulsets")
|
|
shadow_quotas = client.get_json(SHADOW_NAMESPACE, "resourcequotas")
|
|
shadow_netpol = client.get_json(SHADOW_NAMESPACE, "networkpolicies")
|
|
mongo_snap = query_shadow_mongo_snapshot(client)
|
|
api_health = inspect_api_health(client)
|
|
|
|
report = {
|
|
"at": utc_now().isoformat(),
|
|
"runId": args.run_id,
|
|
"apiHealth": api_health,
|
|
"shadowMongoSnapshot": mongo_snap,
|
|
"sourceDeployments": [
|
|
{
|
|
"name": d["metadata"]["name"],
|
|
"replicas": d["spec"].get("replicas"),
|
|
"readyReplicas": d.get("status", {}).get("readyReplicas", 0),
|
|
"images": [c.get("image") for c in d["spec"]["template"]["spec"]["containers"]],
|
|
}
|
|
for d in source_deploys.get("items", [])
|
|
],
|
|
"shadowWorkloads": [
|
|
{
|
|
"kind": w["kind"],
|
|
"name": w["metadata"]["name"],
|
|
"replicas": w["spec"].get("replicas"),
|
|
"readyReplicas": w.get("status", {}).get("readyReplicas", 0),
|
|
"images": [c.get("image") for c in w["spec"]["template"]["spec"]["containers"]],
|
|
}
|
|
for w in shadow_deploys.get("items", []) + shadow_sts.get("items", [])
|
|
],
|
|
"shadowQuotas": [
|
|
{
|
|
"name": q["metadata"]["name"],
|
|
"hard": q.get("spec", {}).get("hard"),
|
|
"used": q.get("status", {}).get("used"),
|
|
}
|
|
for q in shadow_quotas.get("items", [])
|
|
],
|
|
"shadowNetworkPolicies": [p["metadata"]["name"] for p in shadow_netpol.get("items", [])],
|
|
}
|
|
path = write_report(out_dir, "preflight.json", report)
|
|
print(json.dumps({"ok": True, "phase": "preflight", "report": str(path), "apiReady": api_health["readyReplicas"]}))
|
|
return 0
|
|
|
|
|
|
def read_shadow_safety(client):
|
|
js = "const fs=require('fs');const q=db.getSiblingDB('" + QUERY_DB_NAME + "');if(q.auth('shadow-health',fs.readFileSync('/health-credentials/health-password','utf8').trim()).ok!==1)quit(2);print(EJSON.stringify({safety:q.queryReplicaControl.findOne({_id:'safety'}),health:q.queryReplicaControl.findOne({_id:'safety-health'})},{relaxed:true}));"
|
|
p = client.run(["-n",SHADOW_NAMESPACE,"exec",SHADOW_MONGO_POD,"-c",SHADOW_MONGO_CONTAINER,"--","mongosh","--quiet","--host","127.0.0.1","--port","27129","--eval",js], timeout=25)
|
|
return json.loads(p.stdout.strip().splitlines()[-1])
|
|
|
|
|
|
def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
baseline_health = inspect_api_health(client)
|
|
if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0:
|
|
raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}")
|
|
|
|
mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT)
|
|
if int(mon_dep.get("status", {}).get("availableReplicas") or 0) != 1:
|
|
raise RuntimeError("approved shadow observer must be healthy; it is never disabled by synchronization")
|
|
policy = client.approved_plan["shadowPolicy"]
|
|
safety = read_shadow_safety(client)
|
|
health = safety.get("health") or {}
|
|
recent = health.get("recent", [])[-2:]
|
|
if len(recent) != 2 or any(not h.get("ready") or not h.get("sourceAvailable") or h.get("workErrors") or h.get("memoryPressure") or h.get("persistenceFailure") for h in recent):
|
|
raise RuntimeError("two fresh complete health observations required")
|
|
for h in recent:
|
|
at = parse_iso_datetime(h.get("at"))
|
|
if at is None or not 0 <= (utc_now()-at).total_seconds() <= 60:
|
|
raise RuntimeError("shadow health evidence is stale")
|
|
saved = safety.get("safety") or {}
|
|
if (saved.get("policy") or {}).get("runId") != policy["runID"]:
|
|
raise RuntimeError("synchronization run changed; cannot reset checkpoint/budget")
|
|
if int(saved.get("reserved") or 0) >= policy["maxRecords"]:
|
|
raise RuntimeError("approved cumulative budget exhausted")
|
|
if saved.get("paused") and saved.get("reason") not in policy.get("resumeReasons", ["operator_stop", "pilot_budget", "maintenance_window_closed"]):
|
|
raise RuntimeError("unexpected hard pause requires diagnosis, not automatic reset")
|
|
nonce = saved.get("resumeToken", "") if saved.get("paused") else ""
|
|
|
|
# Scale builder to 0 first to stay well inside ResourceQuota.
|
|
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
|
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
|
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={BUILDER_CONTAINER}")
|
|
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}")
|
|
|
|
rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT)
|
|
rep_idx = find_container_index(rep_dep, REPLICA_CONTAINER)
|
|
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][rep_idx].get("env") or []
|
|
rep_patch = build_deployment_json_patch(
|
|
container_index=rep_idx,
|
|
expected_deployment=rep_dep,
|
|
strategy_recreate=True,
|
|
image=args.replica_image,
|
|
args=build_shadow_replica_args(client.approved_plan, nonce),
|
|
env_list=build_shadow_replica_env(rep_env),
|
|
)
|
|
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
|
|
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1)
|
|
if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180):
|
|
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
|
raise RuntimeError("query-replica failed rollout")
|
|
|
|
sync_deadline = time.monotonic() + args.sync_timeout_seconds
|
|
synced_snap = None
|
|
while time.monotonic() < sync_deadline:
|
|
health = inspect_api_health(client)
|
|
if (
|
|
health["readyReplicas"] < baseline_health["desiredReplicas"]
|
|
or health["totalRestarts"] > baseline_health["totalRestarts"]
|
|
):
|
|
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
|
write_report(out_dir, "shadow-sync-aborted.json", {"reason": "api_degraded", "health": health})
|
|
raise RuntimeError(f"API health degraded during shadow-sync; scaled projector to 0: {health}")
|
|
snap = query_shadow_mongo_snapshot(client)
|
|
state = snap.get("state") or {}
|
|
if state.get("ready") is True:
|
|
obs = parse_iso_datetime(state.get("observedAt"))
|
|
if obs is not None and (utc_now() - obs).total_seconds() < 300:
|
|
synced_snap = snap
|
|
break
|
|
time.sleep(15)
|
|
|
|
if synced_snap is None:
|
|
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
|
raise RuntimeError("timed out waiting for query-replica ready state")
|
|
|
|
bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT)
|
|
bld_idx = find_container_index(bld_dep, BUILDER_CONTAINER)
|
|
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][bld_idx].get("env") or []
|
|
bld_patch = build_deployment_json_patch(
|
|
container_index=bld_idx,
|
|
expected_deployment=bld_dep,
|
|
strategy_recreate=True,
|
|
image=args.builder_image,
|
|
args=build_shadow_builder_args(client.approved_plan),
|
|
env_list=build_shadow_builder_env(bld_env),
|
|
)
|
|
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
|
|
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1)
|
|
if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180):
|
|
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
|
raise RuntimeError("query-read-builder failed rollout")
|
|
|
|
build_deadline = time.monotonic() + args.build_timeout_seconds
|
|
built_snap = None
|
|
while time.monotonic() < build_deadline:
|
|
health = inspect_api_health(client)
|
|
if (
|
|
health["readyReplicas"] < baseline_health["desiredReplicas"]
|
|
or health["totalRestarts"] > baseline_health["totalRestarts"]
|
|
):
|
|
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
|
write_report(out_dir, "shadow-build-aborted.json", {"reason": "api_degraded", "health": health})
|
|
raise RuntimeError(f"API health degraded during builder run; scaled builder to 0: {health}")
|
|
snap = query_shadow_mongo_snapshot(client)
|
|
ok, reasons = evaluate_parity_snapshot(snap)
|
|
if ok:
|
|
built_snap = snap
|
|
break
|
|
time.sleep(15)
|
|
|
|
if built_snap is None:
|
|
raise RuntimeError("timed out waiting for query-read-builder completion")
|
|
|
|
path = write_report(out_dir, "shadow-sync.json", {"at": utc_now().isoformat(), "snapshot": built_snap})
|
|
print(json.dumps({"ok": True, "phase": "shadow-sync", "report": str(path)}))
|
|
return 0
|
|
|
|
|
|
def cmd_verify_parity(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
snap = query_shadow_mongo_snapshot(client)
|
|
ok, reasons = evaluate_parity_snapshot(snap)
|
|
if ok:
|
|
try:
|
|
if not getattr(args, "comparison_evidence", ""):
|
|
raise RuntimeError("result/permission comparison evidence missing")
|
|
plan = client.approved_plan or json.loads(Path(args.approved_plan).read_text())
|
|
require_result_comparison(json.loads(Path(args.comparison_evidence).read_text()), snap, plan)
|
|
except (RuntimeError, OSError, ValueError, KeyError) as error:
|
|
ok = False
|
|
reasons.append(str(error))
|
|
report = {
|
|
"at": utc_now().isoformat(),
|
|
"passed": ok,
|
|
"reasons": reasons,
|
|
"snapshot": snap,
|
|
}
|
|
path = write_report(out_dir, "verify-parity.json", report)
|
|
print(json.dumps({"ok": ok, "phase": "verify-parity", "reasons": reasons, "report": str(path)}))
|
|
return 0 if ok else 1
|
|
|
|
|
|
def sync_read_client_secret(client: ClusterClient) -> None:
|
|
conn_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CONNECTIONS_SECRET)
|
|
cache_secret = client.get_json(SHADOW_NAMESPACE, "secret", SHADOW_CACHE_AUTH_SECRET)
|
|
api_uri_b64 = (conn_secret.get("data") or {}).get("api-uri")
|
|
valkey_url_b64 = (cache_secret.get("data") or {}).get("url")
|
|
if not api_uri_b64 or not valkey_url_b64:
|
|
raise RuntimeError("required shadow read client secret keys missing")
|
|
manifest = build_read_client_secret_manifest(api_uri_b64, valkey_url_b64)
|
|
existing = client.get_json(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, ignore_not_found=True)
|
|
if existing is not None:
|
|
client.patch_json_ops(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, [{"op":"replace", "path":"/data", "value":manifest["data"]}])
|
|
else:
|
|
resource = SOURCE_NAMESPACE + "/secret/" + READ_CLIENT_SECRET_NAME
|
|
if resource not in client.approved_plan.get("createResources", []):
|
|
raise RuntimeError("read client Secret creation not authorized")
|
|
client.run(["create", "-f", "-"], stdin_text=json.dumps(manifest))
|
|
|
|
|
|
def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
|
api_idx = find_container_index(api_dep, API_CONTAINER)
|
|
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
|
|
rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates())
|
|
patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env, expected_deployment=api_dep)
|
|
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
|
ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180)
|
|
health = inspect_api_health(client)
|
|
path = write_report(
|
|
out_dir,
|
|
"rollback-api-flags.json",
|
|
{"at": utc_now().isoformat(), "rolloutOk": ok, "apiHealth": health},
|
|
)
|
|
print(json.dumps({"ok": ok, "phase": "rollback-api-flags", "report": str(path)}))
|
|
return 0 if ok else 1
|
|
|
|
|
|
def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
snap = query_shadow_mongo_snapshot(client)
|
|
ok, reasons = evaluate_parity_snapshot(snap)
|
|
if not ok:
|
|
raise RuntimeError(f"parity gate blocked cutover: {reasons}")
|
|
|
|
if not getattr(args, "comparison_evidence", ""):
|
|
raise RuntimeError("new/old result and permission evidence required before production cutover")
|
|
require_result_comparison(json.loads(Path(args.comparison_evidence).read_text()), snap, client.approved_plan)
|
|
sync_read_client_secret(client)
|
|
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
|
api_idx = find_container_index(api_dep, API_CONTAINER)
|
|
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
|
|
write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env})
|
|
|
|
next_env = merge_env_list(existing_env, build_api_cutover_env_updates())
|
|
patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env, expected_deployment=api_dep)
|
|
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
|
|
|
if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180):
|
|
cmd_rollback_api_flags(args, client, out_dir)
|
|
raise RuntimeError("droboticflow-api rollout failed after flag cutover; rolled back")
|
|
|
|
health = inspect_api_health(client)
|
|
path = write_report(
|
|
out_dir,
|
|
"cutover-api-flags.json",
|
|
{"at": utc_now().isoformat(), "env": next_env, "apiHealth": health},
|
|
)
|
|
print(json.dumps({"ok": True, "phase": "cutover-api-flags", "report": str(path)}))
|
|
return 0
|
|
|
|
|
|
def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
baseline = inspect_api_health(client)
|
|
deadline = time.monotonic() + args.watch_seconds
|
|
samples: list[dict[str, Any]] = []
|
|
rolled_back = False
|
|
trigger_reason = ""
|
|
|
|
while time.monotonic() < deadline:
|
|
time.sleep(min(20, max(1, int(deadline - time.monotonic()))))
|
|
health = inspect_api_health(client)
|
|
log_proc = client.run(
|
|
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=25s", "--tail=300"],
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
if log_proc.returncode != 0:
|
|
raise RuntimeError("API health log probe unavailable; no zero-error claim")
|
|
error_lines = count_error_log_lines(log_proc.stdout)
|
|
sample = {
|
|
"at": utc_now().isoformat(),
|
|
"readyReplicas": health["readyReplicas"],
|
|
"desiredReplicas": health["desiredReplicas"],
|
|
"totalRestarts": health["totalRestarts"],
|
|
"errorLines": error_lines,
|
|
}
|
|
samples.append(sample)
|
|
|
|
if health["readyReplicas"] < baseline["desiredReplicas"]:
|
|
trigger_reason = f"readyReplicas dropped: {health['readyReplicas']}/{baseline['desiredReplicas']}"
|
|
elif health["totalRestarts"] > baseline["totalRestarts"]:
|
|
trigger_reason = f"API pod restarts increased: {baseline['totalRestarts']} -> {health['totalRestarts']}"
|
|
elif error_lines >= args.max_error_lines:
|
|
trigger_reason = f"error log spike: {error_lines} lines in 25s window"
|
|
|
|
if trigger_reason:
|
|
rolled_back = True
|
|
cmd_rollback_api_flags(args, client, out_dir)
|
|
break
|
|
|
|
report = {
|
|
"at": utc_now().isoformat(),
|
|
"passed": not rolled_back,
|
|
"rolledBack": rolled_back,
|
|
"triggerReason": trigger_reason,
|
|
"samples": samples,
|
|
}
|
|
path = write_report(out_dir, "watch-and-guard.json", report)
|
|
print(json.dumps({"ok": not rolled_back, "phase": "watch-and-guard", "rolledBack": rolled_back, "report": str(path)}))
|
|
return 1 if rolled_back else 0
|
|
|
|
|
|
def api_cutover_active(api_env: list[dict[str, Any]]) -> bool:
|
|
by_name = {entry.get("name"): entry.get("value") for entry in (api_env or [])}
|
|
return by_name.get("QUERY_REPLICA_REPORTS") == "1" or by_name.get("QUERY_REPLICA_COUNTS") == "1"
|
|
|
|
|
|
def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
|
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
|
|
deploys_by_name = {d["metadata"]["name"]: d for d in source_deploys.get("items", [])}
|
|
business_health = []
|
|
unhealthy_business = []
|
|
for name in PRODUCTION_BUSINESS_DEPLOYMENTS:
|
|
dep = deploys_by_name.get(name)
|
|
if dep is None:
|
|
business_health.append({"name": name, "present": False, "desiredReplicas": 0, "readyReplicas": 0})
|
|
unhealthy_business.append(f"{name}: missing")
|
|
continue
|
|
desired = int(dep["spec"].get("replicas") or 0)
|
|
ready = int(dep.get("status", {}).get("readyReplicas") or 0)
|
|
business_health.append({"name": name, "present": True, "desiredReplicas": desired, "readyReplicas": ready})
|
|
if desired == 0 or ready < desired:
|
|
unhealthy_business.append(f"{name}: {ready}/{desired}")
|
|
|
|
api_dep = deploys_by_name.get(API_DEPLOYMENT)
|
|
if api_dep is None:
|
|
raise RuntimeError(f"missing {API_DEPLOYMENT} deployment in {SOURCE_NAMESPACE}")
|
|
api_idx = find_container_index(api_dep, API_CONTAINER)
|
|
api_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
|
|
cutover_active = api_cutover_active(api_env)
|
|
api_health = inspect_api_health(client)
|
|
api_healthy = (
|
|
api_health["desiredReplicas"] > 0
|
|
and api_health["readyReplicas"] >= api_health["desiredReplicas"]
|
|
)
|
|
|
|
log_proc = client.run(
|
|
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=12m", "--tail=500"],
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
if log_proc.returncode != 0:
|
|
raise RuntimeError("API health log probe unavailable; no zero-error claim")
|
|
error_lines = count_error_log_lines(log_proc.stdout)
|
|
|
|
snap = None
|
|
parity_ok = False
|
|
parity_reasons: list[str] = []
|
|
try:
|
|
snap = query_shadow_mongo_snapshot(client)
|
|
parity_ok, parity_reasons = evaluate_parity_snapshot(
|
|
snap,
|
|
max_observed_age_seconds=args.max_observed_age_seconds,
|
|
max_workforce_age_seconds=args.max_workforce_age_seconds,
|
|
)
|
|
except Exception as exc:
|
|
parity_ok = False
|
|
parity_reasons = [f"shadow mongo probe failed: {type(exc).__name__}"]
|
|
|
|
trigger_reasons: list[str] = []
|
|
if cutover_active:
|
|
if not api_healthy:
|
|
trigger_reasons.append(
|
|
f"API degraded: {api_health['readyReplicas']}/{api_health['desiredReplicas']}"
|
|
)
|
|
if error_lines >= args.max_error_lines:
|
|
trigger_reasons.append(f"API error log spike: {error_lines} lines in 12m")
|
|
if not parity_ok:
|
|
trigger_reasons.extend(parity_reasons)
|
|
|
|
rolled_back = False
|
|
if cutover_active and trigger_reasons:
|
|
rolled_back = True
|
|
cmd_rollback_api_flags(args, client, out_dir)
|
|
|
|
stamp = utc_now().strftime("%Y%m%dT%H%M%SZ")
|
|
report = {
|
|
"at": utc_now().isoformat(),
|
|
"cutoverActive": cutover_active,
|
|
"businessHealth": business_health,
|
|
"unhealthyBusiness": unhealthy_business,
|
|
"apiHealth": api_health,
|
|
"errorLines12m": error_lines,
|
|
"parityOk": parity_ok,
|
|
"parityReasons": parity_reasons,
|
|
"rolledBack": rolled_back,
|
|
"triggerReasons": trigger_reasons,
|
|
"shadowMongoSnapshot": snap,
|
|
}
|
|
path = write_report(out_dir, f"patrol-{stamp}.json", report)
|
|
ok = (not rolled_back) and (len(unhealthy_business) == 0) and (not cutover_active or parity_ok)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"ok": ok,
|
|
"phase": "patrol",
|
|
"cutoverActive": cutover_active,
|
|
"rolledBack": rolled_back,
|
|
"errorLines12m": error_lines,
|
|
"parityOk": parity_ok,
|
|
"unhealthyBusiness": unhealthy_business,
|
|
"report": str(path),
|
|
}
|
|
)
|
|
)
|
|
return 0 if ok else 1
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.")
|
|
parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
|
|
parser.add_argument("--approved-plan", default="")
|
|
parser.add_argument("--confirm-plan", default="")
|
|
parser.add_argument("--approval-ref", default="")
|
|
parser.add_argument("--comparison-evidence", default="")
|
|
parser.add_argument("--run-id", default=default_run_id())
|
|
parser.add_argument("--output-dir", default="")
|
|
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
sub.add_parser("preflight")
|
|
|
|
p_sync = sub.add_parser("shadow-sync")
|
|
p_sync.add_argument("--replica-image", required=True)
|
|
p_sync.add_argument("--builder-image", required=True)
|
|
p_sync.add_argument("--sync-timeout-seconds", type=int, default=1800)
|
|
p_sync.add_argument("--build-timeout-seconds", type=int, default=1200)
|
|
|
|
sub.add_parser("verify-parity")
|
|
sub.add_parser("cutover-api-flags")
|
|
|
|
p_watch = sub.add_parser("watch-and-guard")
|
|
p_watch.add_argument("--watch-seconds", type=int, default=900)
|
|
p_watch.add_argument("--max-error-lines", type=int, default=5)
|
|
|
|
p_patrol = sub.add_parser("patrol")
|
|
p_patrol.add_argument("--max-error-lines", type=int, default=5)
|
|
p_patrol.add_argument("--max-observed-age-seconds", type=int, default=600)
|
|
p_patrol.add_argument("--max-workforce-age-seconds", type=int, default=720)
|
|
|
|
sub.add_parser("rollback-api-flags")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
out_dir = (
|
|
Path(args.output_dir)
|
|
if args.output_dir
|
|
else Path(__file__).resolve().parents[2] / "local-data" / "diagnostics" / "evening-cutover" / args.run_id
|
|
)
|
|
client = ClusterClient(args.kubeconfig)
|
|
client.approved_plan = approved_plan(args, client)
|
|
handlers = {
|
|
"preflight": cmd_preflight,
|
|
"shadow-sync": cmd_shadow_sync,
|
|
"verify-parity": cmd_verify_parity,
|
|
"cutover-api-flags": cmd_cutover_api_flags,
|
|
"watch-and-guard": cmd_watch_and_guard,
|
|
"patrol": cmd_patrol,
|
|
"rollback-api-flags": cmd_rollback_api_flags,
|
|
}
|
|
try:
|
|
return handlers[args.command](args, client, out_dir)
|
|
except Exception:
|
|
if args.command == "shadow-sync" and client.approved_plan:
|
|
failures = []
|
|
for name in (REPLICA_DEPLOYMENT, BUILDER_DEPLOYMENT):
|
|
try:
|
|
client.scale(SHADOW_NAMESPACE, "deployment", name, 0)
|
|
except Exception as error:
|
|
failures.append({"controller":name, "errorClass":type(error).__name__})
|
|
write_report(out_dir, "shadow-sync-stopped.json", {"progressRetained":True,"stopFailures":failures})
|
|
raise
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|