From 11cb09296b398770a630c45448dfd9bf667cc0b7 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sun, 15 Nov 2026 22:57:31 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Add=20proactive=20patro?= =?UTF-8?q?l=20guard=20and=20atomic=20JSON=20patching=20for=20evening=20cu?= =?UTF-8?q?tover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/local/evening-query-cutover.py | 219 ++++++++++++++++---- scripts/local/test_evening_query_cutover.py | 56 ++++- 2 files changed, 224 insertions(+), 51 deletions(-) diff --git a/scripts/local/evening-query-cutover.py b/scripts/local/evening-query-cutover.py index 101fd9b2..6c74d54e 100644 --- a/scripts/local/evening-query-cutover.py +++ b/scripts/local/evening-query-cutover.py @@ -31,6 +31,13 @@ BUILDER_DEPLOYMENT = "query-read-builder" BUILDER_CONTAINER = "builder" SHADOW_MONGO_POD = "query-mongo-0" SHADOW_MONGO_CONTAINER = "mongo" +PRODUCTION_BUSINESS_DEPLOYMENTS = ( + "droboticflow-api", + "droboticflow-annotation-processor", + "droboticflow-annotation-validator", + "droboticflow-worker", + "drosync-coordinator", +) SECRET_KEY_PATTERN = re.compile( r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)", @@ -121,34 +128,52 @@ def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, A return merged -def build_deployment_update_patch( +def find_container_index(dep: dict[str, Any], container_name: str) -> int: + containers = dep.get("spec", {}).get("template", {}).get("spec", {}).get("containers") or [] + for idx, container in enumerate(containers): + if container.get("name") == container_name: + return idx + if len(containers) <= 1: + return 0 + raise RuntimeError(f"container {container_name} not found in deployment") + + +def build_deployment_json_patch( *, - container_name: str, + container_index: int = 0, env_list: list[dict[str, Any]], strategy_recreate: bool = False, image: str | None = None, args: list[str] | None = None, -) -> dict[str, Any]: - """Build a Kubernetes strategic-merge patch for a single deployment container.""" - container_patch: dict[str, Any] = { - "name": container_name, - "env": env_list, - } - if image is not None: - container_patch["image"] = image - if args is not None: - container_patch["args"] = list(args) - spec_patch: dict[str, Any] = { - "template": { - "spec": { - "containers": [container_patch], - } - } - } +) -> list[dict[str, Any]]: + """Build an atomic RFC 6902 JSON patch for a single deployment container.""" + ops: list[dict[str, Any]] = [] if strategy_recreate: - # Clearing rollingUpdate is required when switching from RollingUpdate -> Recreate. - spec_patch["strategy"] = {"type": "Recreate", "rollingUpdate": None} - return {"spec": spec_patch} + ops.append({"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}}) + ops.append( + { + "op": "add", + "path": f"/spec/template/spec/containers/{container_index}/env", + "value": env_list, + } + ) + if image is not None: + ops.append( + { + "op": "add", + "path": f"/spec/template/spec/containers/{container_index}/image", + "value": image, + } + ) + if args is not None: + ops.append( + { + "op": "add", + "path": f"/spec/template/spec/containers/{container_index}/args", + "value": list(args), + } + ) + return ops def build_shadow_replica_args() -> list[str]: @@ -369,7 +394,7 @@ class ClusterClient: return None return json.loads(proc.stdout) - def patch_json(self, namespace: str, kind: str, name: str, patch_obj: dict[str, Any]) -> None: + def patch_json_ops(self, namespace: str, kind: str, name: str, ops: list[dict[str, Any]]) -> None: self.run( [ "-n", @@ -377,9 +402,9 @@ class ClusterClient: "patch", kind, name, - "--type=strategic", + "--type=json", "-p", - json.dumps(patch_obj), + json.dumps(ops), ] ) @@ -563,15 +588,16 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}") rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT) - rep_env = rep_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] - rep_patch = build_deployment_update_patch( - container_name=REPLICA_CONTAINER, + rep_idx = find_container_index(rep_dep, REPLICA_CONTAINER) + rep_env = rep_dep["spec"]["template"]["spec"]["containers"][rep_idx].get("env") or [] + rep_patch = build_deployment_json_patch( + container_index=rep_idx, strategy_recreate=True, image=args.replica_image, args=build_shadow_replica_args(), env_list=build_shadow_replica_env(rep_env), ) - client.patch_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch) + client.patch_json_ops(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch) client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1) if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180): client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0) @@ -602,15 +628,16 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa raise RuntimeError("timed out waiting for query-replica ready state") bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT) - bld_env = bld_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] - bld_patch = build_deployment_update_patch( - container_name=BUILDER_CONTAINER, + bld_idx = find_container_index(bld_dep, BUILDER_CONTAINER) + bld_env = bld_dep["spec"]["template"]["spec"]["containers"][bld_idx].get("env") or [] + bld_patch = build_deployment_json_patch( + container_index=bld_idx, strategy_recreate=True, image=args.builder_image, args=build_shadow_builder_args(), env_list=build_shadow_builder_env(bld_env), ) - client.patch_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch) + client.patch_json_ops(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch) client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1) if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180): client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0) @@ -671,10 +698,11 @@ def sync_read_client_secret(client: ClusterClient) -> None: def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT) - existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] + api_idx = find_container_index(api_dep, API_CONTAINER) + existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or [] rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates()) - patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=rolled_env) - client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) + patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env) + client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180) health = inspect_api_health(client) path = write_report( @@ -694,12 +722,13 @@ def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_d sync_read_client_secret(client) api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT) - existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or [] + api_idx = find_container_index(api_dep, API_CONTAINER) + existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or [] write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env}) next_env = merge_env_list(existing_env, build_api_cutover_env_updates()) - patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=next_env) - client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) + patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env) + client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180): cmd_rollback_api_flags(args, client, out_dir) @@ -764,6 +793,110 @@ def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir return 1 if rolled_back else 0 +def api_cutover_active(api_env: list[dict[str, Any]]) -> bool: + by_name = {entry.get("name"): entry.get("value") for entry in (api_env or [])} + return by_name.get("QUERY_REPLICA_REPORTS") == "1" or by_name.get("QUERY_REPLICA_COUNTS") == "1" + + +def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: + source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments") + deploys_by_name = {d["metadata"]["name"]: d for d in source_deploys.get("items", [])} + business_health = [] + unhealthy_business = [] + for name in PRODUCTION_BUSINESS_DEPLOYMENTS: + dep = deploys_by_name.get(name) + if dep is None: + business_health.append({"name": name, "present": False, "desiredReplicas": 0, "readyReplicas": 0}) + unhealthy_business.append(f"{name}: missing") + continue + desired = int(dep["spec"].get("replicas") or 0) + ready = int(dep.get("status", {}).get("readyReplicas") or 0) + business_health.append({"name": name, "present": True, "desiredReplicas": desired, "readyReplicas": ready}) + if desired == 0 or ready < desired: + unhealthy_business.append(f"{name}: {ready}/{desired}") + + api_dep = deploys_by_name.get(API_DEPLOYMENT) + if api_dep is None: + raise RuntimeError(f"missing {API_DEPLOYMENT} deployment in {SOURCE_NAMESPACE}") + api_idx = find_container_index(api_dep, API_CONTAINER) + api_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or [] + cutover_active = api_cutover_active(api_env) + api_health = inspect_api_health(client) + api_healthy = ( + api_health["desiredReplicas"] > 0 + and api_health["readyReplicas"] >= api_health["desiredReplicas"] + ) + + log_proc = client.run( + ["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=12m", "--tail=500"], + timeout=15, + check=False, + ) + error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "") + + snap = None + parity_ok = False + parity_reasons: list[str] = [] + try: + snap = query_shadow_mongo_snapshot(client) + parity_ok, parity_reasons = evaluate_parity_snapshot( + snap, + max_observed_age_seconds=args.max_observed_age_seconds, + max_workforce_age_seconds=args.max_workforce_age_seconds, + ) + except Exception as exc: + parity_ok = False + parity_reasons = [f"shadow mongo probe failed: {type(exc).__name__}"] + + trigger_reasons: list[str] = [] + if cutover_active: + if not api_healthy: + trigger_reasons.append( + f"API degraded: {api_health['readyReplicas']}/{api_health['desiredReplicas']}" + ) + if error_lines >= args.max_error_lines: + trigger_reasons.append(f"API error log spike: {error_lines} lines in 12m") + if not parity_ok: + trigger_reasons.extend(parity_reasons) + + rolled_back = False + if cutover_active and trigger_reasons: + rolled_back = True + cmd_rollback_api_flags(args, client, out_dir) + + stamp = utc_now().strftime("%Y%m%dT%H%M%SZ") + report = { + "at": utc_now().isoformat(), + "cutoverActive": cutover_active, + "businessHealth": business_health, + "unhealthyBusiness": unhealthy_business, + "apiHealth": api_health, + "errorLines12m": error_lines, + "parityOk": parity_ok, + "parityReasons": parity_reasons, + "rolledBack": rolled_back, + "triggerReasons": trigger_reasons, + "shadowMongoSnapshot": snap, + } + path = write_report(out_dir, f"patrol-{stamp}.json", report) + ok = (not rolled_back) and (len(unhealthy_business) == 0) and (not cutover_active or parity_ok) + print( + json.dumps( + { + "ok": ok, + "phase": "patrol", + "cutoverActive": cutover_active, + "rolledBack": rolled_back, + "errorLines12m": error_lines, + "parityOk": parity_ok, + "unhealthyBusiness": unhealthy_business, + "report": str(path), + } + ) + ) + return 0 if ok else 1 + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.") parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG) @@ -786,6 +919,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p_watch.add_argument("--watch-seconds", type=int, default=900) p_watch.add_argument("--max-error-lines", type=int, default=5) + p_patrol = sub.add_parser("patrol") + p_patrol.add_argument("--max-error-lines", type=int, default=5) + p_patrol.add_argument("--max-observed-age-seconds", type=int, default=600) + p_patrol.add_argument("--max-workforce-age-seconds", type=int, default=720) + sub.add_parser("rollback-api-flags") return parser.parse_args(argv) @@ -795,7 +933,7 @@ def main(argv: list[str] | None = None) -> int: out_dir = ( Path(args.output_dir) if args.output_dir - else Path("local-data") / "diagnostics" / "evening-cutover" / args.run_id + else Path(__file__).resolve().parents[2] / "local-data" / "diagnostics" / "evening-cutover" / args.run_id ) client = ClusterClient(args.kubeconfig) handlers = { @@ -804,6 +942,7 @@ def main(argv: list[str] | None = None) -> int: "verify-parity": cmd_verify_parity, "cutover-api-flags": cmd_cutover_api_flags, "watch-and-guard": cmd_watch_and_guard, + "patrol": cmd_patrol, "rollback-api-flags": cmd_rollback_api_flags, } return handlers[args.command](args, client, out_dir) diff --git a/scripts/local/test_evening_query_cutover.py b/scripts/local/test_evening_query_cutover.py index 32ffcf3f..f122bd18 100644 --- a/scripts/local/test_evening_query_cutover.py +++ b/scripts/local/test_evening_query_cutover.py @@ -78,22 +78,32 @@ class EveningQueryCutoverTests(unittest.TestCase): CUTOVER.READ_CLIENT_SECRET_NAME, ) - def test_deployment_patch_sets_recreate_and_clears_rolling_update(self): - patch = CUTOVER.build_deployment_update_patch( + def test_deployment_json_patch_and_find_container_index(self): + dep = { + "spec": { + "template": { + "spec": { + "containers": [ + {"name": "sidecar"}, + {"name": "projector"}, + ] + } + } + } + } + self.assertEqual(CUTOVER.find_container_index(dep, "projector"), 1) + patch_ops = CUTOVER.build_deployment_json_patch( + container_index=1, strategy_recreate=True, - container_name="projector", image="example/img@sha256:abc", args=CUTOVER.build_shadow_replica_args(), env_list=CUTOVER.build_shadow_replica_env([]), ) - strategy = patch["spec"]["strategy"] - self.assertEqual(strategy["type"], "Recreate") - self.assertIn("rollingUpdate", strategy) - self.assertIsNone(strategy["rollingUpdate"]) - container = patch["spec"]["template"]["spec"]["containers"][0] - self.assertEqual(container["name"], "projector") - self.assertIn("--source-compress", container["args"]) - self.assertNotIn("--safety-run", container["args"]) + by_path = {op["path"]: op for op in patch_ops} + self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}}) + self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc") + self.assertIn("--source-compress", by_path["/spec/template/spec/containers/1/args"]["value"]) + self.assertNotIn("--safety-run", by_path["/spec/template/spec/containers/1/args"]["value"]) def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self): now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) @@ -119,6 +129,30 @@ class EveningQueryCutoverTests(unittest.TestCase): self.assertFalse(passed) self.assertEqual(len(reasons), 2) + def test_proactive_patrol_thresholds_trip_before_15min_expiry(self): + now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) + near_stale = { + "state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3}, + "poolBucketsCount": 42, + "workforceFactsCount": 1200, + "global": { + "heads": { + "h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"}, + "h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"}, + } + }, + } + passed, reasons = CUTOVER.evaluate_parity_snapshot( + near_stale, + now=now, + max_observed_age_seconds=600, + max_workforce_age_seconds=720, + ) + self.assertFalse(passed) + self.assertEqual(len(reasons), 2) + self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates())) + self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates())) + def test_rollback_env_disables_all_query_reads(self): existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates()) rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())