🛡️ Add proactive patrol guard and atomic JSON patching for evening cutover

This commit is contained in:
d-robotics 2026-11-15 22:57:31 -08:00
parent 1dc9710746
commit 11cb09296b
2 changed files with 224 additions and 51 deletions

View File

@ -31,6 +31,13 @@ BUILDER_DEPLOYMENT = "query-read-builder"
BUILDER_CONTAINER = "builder"
SHADOW_MONGO_POD = "query-mongo-0"
SHADOW_MONGO_CONTAINER = "mongo"
PRODUCTION_BUSINESS_DEPLOYMENTS = (
"droboticflow-api",
"droboticflow-annotation-processor",
"droboticflow-annotation-validator",
"droboticflow-worker",
"drosync-coordinator",
)
SECRET_KEY_PATTERN = re.compile(
r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)",
@ -121,34 +128,52 @@ def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, A
return merged
def build_deployment_update_patch(
def find_container_index(dep: dict[str, Any], container_name: str) -> int:
containers = dep.get("spec", {}).get("template", {}).get("spec", {}).get("containers") or []
for idx, container in enumerate(containers):
if container.get("name") == container_name:
return idx
if len(containers) <= 1:
return 0
raise RuntimeError(f"container {container_name} not found in deployment")
def build_deployment_json_patch(
*,
container_name: str,
container_index: int = 0,
env_list: list[dict[str, Any]],
strategy_recreate: bool = False,
image: str | None = None,
args: list[str] | None = None,
) -> dict[str, Any]:
"""Build a Kubernetes strategic-merge patch for a single deployment container."""
container_patch: dict[str, Any] = {
"name": container_name,
"env": env_list,
}
if image is not None:
container_patch["image"] = image
if args is not None:
container_patch["args"] = list(args)
spec_patch: dict[str, Any] = {
"template": {
"spec": {
"containers": [container_patch],
}
}
}
) -> list[dict[str, Any]]:
"""Build an atomic RFC 6902 JSON patch for a single deployment container."""
ops: list[dict[str, Any]] = []
if strategy_recreate:
# Clearing rollingUpdate is required when switching from RollingUpdate -> Recreate.
spec_patch["strategy"] = {"type": "Recreate", "rollingUpdate": None}
return {"spec": spec_patch}
ops.append({"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
ops.append(
{
"op": "add",
"path": f"/spec/template/spec/containers/{container_index}/env",
"value": env_list,
}
)
if image is not None:
ops.append(
{
"op": "add",
"path": f"/spec/template/spec/containers/{container_index}/image",
"value": image,
}
)
if args is not None:
ops.append(
{
"op": "add",
"path": f"/spec/template/spec/containers/{container_index}/args",
"value": list(args),
}
)
return ops
def build_shadow_replica_args() -> list[str]:
@ -369,7 +394,7 @@ class ClusterClient:
return None
return json.loads(proc.stdout)
def patch_json(self, namespace: str, kind: str, name: str, patch_obj: dict[str, Any]) -> None:
def patch_json_ops(self, namespace: str, kind: str, name: str, ops: list[dict[str, Any]]) -> None:
self.run(
[
"-n",
@ -377,9 +402,9 @@ class ClusterClient:
"patch",
kind,
name,
"--type=strategic",
"--type=json",
"-p",
json.dumps(patch_obj),
json.dumps(ops),
]
)
@ -563,15 +588,16 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}")
rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT)
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
rep_patch = build_deployment_update_patch(
container_name=REPLICA_CONTAINER,
rep_idx = find_container_index(rep_dep, REPLICA_CONTAINER)
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][rep_idx].get("env") or []
rep_patch = build_deployment_json_patch(
container_index=rep_idx,
strategy_recreate=True,
image=args.replica_image,
args=build_shadow_replica_args(),
env_list=build_shadow_replica_env(rep_env),
)
client.patch_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1)
if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180):
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
@ -602,15 +628,16 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa
raise RuntimeError("timed out waiting for query-replica ready state")
bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT)
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
bld_patch = build_deployment_update_patch(
container_name=BUILDER_CONTAINER,
bld_idx = find_container_index(bld_dep, BUILDER_CONTAINER)
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][bld_idx].get("env") or []
bld_patch = build_deployment_json_patch(
container_index=bld_idx,
strategy_recreate=True,
image=args.builder_image,
args=build_shadow_builder_args(),
env_list=build_shadow_builder_env(bld_env),
)
client.patch_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1)
if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180):
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
@ -671,10 +698,11 @@ def sync_read_client_secret(client: ClusterClient) -> None:
def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
api_idx = find_container_index(api_dep, API_CONTAINER)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates())
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=rolled_env)
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env)
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180)
health = inspect_api_health(client)
path = write_report(
@ -694,12 +722,13 @@ def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_d
sync_read_client_secret(client)
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
api_idx = find_container_index(api_dep, API_CONTAINER)
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env})
next_env = merge_env_list(existing_env, build_api_cutover_env_updates())
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=next_env)
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env)
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180):
cmd_rollback_api_flags(args, client, out_dir)
@ -764,6 +793,110 @@ def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir
return 1 if rolled_back else 0
def api_cutover_active(api_env: list[dict[str, Any]]) -> bool:
by_name = {entry.get("name"): entry.get("value") for entry in (api_env or [])}
return by_name.get("QUERY_REPLICA_REPORTS") == "1" or by_name.get("QUERY_REPLICA_COUNTS") == "1"
def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
deploys_by_name = {d["metadata"]["name"]: d for d in source_deploys.get("items", [])}
business_health = []
unhealthy_business = []
for name in PRODUCTION_BUSINESS_DEPLOYMENTS:
dep = deploys_by_name.get(name)
if dep is None:
business_health.append({"name": name, "present": False, "desiredReplicas": 0, "readyReplicas": 0})
unhealthy_business.append(f"{name}: missing")
continue
desired = int(dep["spec"].get("replicas") or 0)
ready = int(dep.get("status", {}).get("readyReplicas") or 0)
business_health.append({"name": name, "present": True, "desiredReplicas": desired, "readyReplicas": ready})
if desired == 0 or ready < desired:
unhealthy_business.append(f"{name}: {ready}/{desired}")
api_dep = deploys_by_name.get(API_DEPLOYMENT)
if api_dep is None:
raise RuntimeError(f"missing {API_DEPLOYMENT} deployment in {SOURCE_NAMESPACE}")
api_idx = find_container_index(api_dep, API_CONTAINER)
api_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
cutover_active = api_cutover_active(api_env)
api_health = inspect_api_health(client)
api_healthy = (
api_health["desiredReplicas"] > 0
and api_health["readyReplicas"] >= api_health["desiredReplicas"]
)
log_proc = client.run(
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=12m", "--tail=500"],
timeout=15,
check=False,
)
error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "")
snap = None
parity_ok = False
parity_reasons: list[str] = []
try:
snap = query_shadow_mongo_snapshot(client)
parity_ok, parity_reasons = evaluate_parity_snapshot(
snap,
max_observed_age_seconds=args.max_observed_age_seconds,
max_workforce_age_seconds=args.max_workforce_age_seconds,
)
except Exception as exc:
parity_ok = False
parity_reasons = [f"shadow mongo probe failed: {type(exc).__name__}"]
trigger_reasons: list[str] = []
if cutover_active:
if not api_healthy:
trigger_reasons.append(
f"API degraded: {api_health['readyReplicas']}/{api_health['desiredReplicas']}"
)
if error_lines >= args.max_error_lines:
trigger_reasons.append(f"API error log spike: {error_lines} lines in 12m")
if not parity_ok:
trigger_reasons.extend(parity_reasons)
rolled_back = False
if cutover_active and trigger_reasons:
rolled_back = True
cmd_rollback_api_flags(args, client, out_dir)
stamp = utc_now().strftime("%Y%m%dT%H%M%SZ")
report = {
"at": utc_now().isoformat(),
"cutoverActive": cutover_active,
"businessHealth": business_health,
"unhealthyBusiness": unhealthy_business,
"apiHealth": api_health,
"errorLines12m": error_lines,
"parityOk": parity_ok,
"parityReasons": parity_reasons,
"rolledBack": rolled_back,
"triggerReasons": trigger_reasons,
"shadowMongoSnapshot": snap,
}
path = write_report(out_dir, f"patrol-{stamp}.json", report)
ok = (not rolled_back) and (len(unhealthy_business) == 0) and (not cutover_active or parity_ok)
print(
json.dumps(
{
"ok": ok,
"phase": "patrol",
"cutoverActive": cutover_active,
"rolledBack": rolled_back,
"errorLines12m": error_lines,
"parityOk": parity_ok,
"unhealthyBusiness": unhealthy_business,
"report": str(path),
}
)
)
return 0 if ok else 1
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.")
parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
@ -786,6 +919,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
p_watch.add_argument("--watch-seconds", type=int, default=900)
p_watch.add_argument("--max-error-lines", type=int, default=5)
p_patrol = sub.add_parser("patrol")
p_patrol.add_argument("--max-error-lines", type=int, default=5)
p_patrol.add_argument("--max-observed-age-seconds", type=int, default=600)
p_patrol.add_argument("--max-workforce-age-seconds", type=int, default=720)
sub.add_parser("rollback-api-flags")
return parser.parse_args(argv)
@ -795,7 +933,7 @@ def main(argv: list[str] | None = None) -> int:
out_dir = (
Path(args.output_dir)
if args.output_dir
else Path("local-data") / "diagnostics" / "evening-cutover" / args.run_id
else Path(__file__).resolve().parents[2] / "local-data" / "diagnostics" / "evening-cutover" / args.run_id
)
client = ClusterClient(args.kubeconfig)
handlers = {
@ -804,6 +942,7 @@ def main(argv: list[str] | None = None) -> int:
"verify-parity": cmd_verify_parity,
"cutover-api-flags": cmd_cutover_api_flags,
"watch-and-guard": cmd_watch_and_guard,
"patrol": cmd_patrol,
"rollback-api-flags": cmd_rollback_api_flags,
}
return handlers[args.command](args, client, out_dir)

View File

@ -78,22 +78,32 @@ class EveningQueryCutoverTests(unittest.TestCase):
CUTOVER.READ_CLIENT_SECRET_NAME,
)
def test_deployment_patch_sets_recreate_and_clears_rolling_update(self):
patch = CUTOVER.build_deployment_update_patch(
def test_deployment_json_patch_and_find_container_index(self):
dep = {
"spec": {
"template": {
"spec": {
"containers": [
{"name": "sidecar"},
{"name": "projector"},
]
}
}
}
}
self.assertEqual(CUTOVER.find_container_index(dep, "projector"), 1)
patch_ops = CUTOVER.build_deployment_json_patch(
container_index=1,
strategy_recreate=True,
container_name="projector",
image="example/img@sha256:abc",
args=CUTOVER.build_shadow_replica_args(),
env_list=CUTOVER.build_shadow_replica_env([]),
)
strategy = patch["spec"]["strategy"]
self.assertEqual(strategy["type"], "Recreate")
self.assertIn("rollingUpdate", strategy)
self.assertIsNone(strategy["rollingUpdate"])
container = patch["spec"]["template"]["spec"]["containers"][0]
self.assertEqual(container["name"], "projector")
self.assertIn("--source-compress", container["args"])
self.assertNotIn("--safety-run", container["args"])
by_path = {op["path"]: op for op in patch_ops}
self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc")
self.assertIn("--source-compress", by_path["/spec/template/spec/containers/1/args"]["value"])
self.assertNotIn("--safety-run", by_path["/spec/template/spec/containers/1/args"]["value"])
def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self):
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
@ -119,6 +129,30 @@ class EveningQueryCutoverTests(unittest.TestCase):
self.assertFalse(passed)
self.assertEqual(len(reasons), 2)
def test_proactive_patrol_thresholds_trip_before_15min_expiry(self):
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
near_stale = {
"state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3},
"poolBucketsCount": 42,
"workforceFactsCount": 1200,
"global": {
"heads": {
"h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"},
"h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"},
}
},
}
passed, reasons = CUTOVER.evaluate_parity_snapshot(
near_stale,
now=now,
max_observed_age_seconds=600,
max_workforce_age_seconds=720,
)
self.assertFalse(passed)
self.assertEqual(len(reasons), 2)
self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates()))
self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates()))
def test_rollback_env_disables_all_query_reads(self):
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())