forked from longtao.wu/droboticflow
🛡️ Add proactive patrol guard and atomic JSON patching for evening cutover
This commit is contained in:
parent
1dc9710746
commit
11cb09296b
@ -31,6 +31,13 @@ BUILDER_DEPLOYMENT = "query-read-builder"
|
||||
BUILDER_CONTAINER = "builder"
|
||||
SHADOW_MONGO_POD = "query-mongo-0"
|
||||
SHADOW_MONGO_CONTAINER = "mongo"
|
||||
PRODUCTION_BUSINESS_DEPLOYMENTS = (
|
||||
"droboticflow-api",
|
||||
"droboticflow-annotation-processor",
|
||||
"droboticflow-annotation-validator",
|
||||
"droboticflow-worker",
|
||||
"drosync-coordinator",
|
||||
)
|
||||
|
||||
SECRET_KEY_PATTERN = re.compile(
|
||||
r"(uri|url|password|passwd|secret|token|keyfile|credential|ak|sk|auth)",
|
||||
@ -121,34 +128,52 @@ def merge_env_list(existing_env: list[dict[str, Any]], updates: list[dict[str, A
|
||||
return merged
|
||||
|
||||
|
||||
def build_deployment_update_patch(
|
||||
def find_container_index(dep: dict[str, Any], container_name: str) -> int:
|
||||
containers = dep.get("spec", {}).get("template", {}).get("spec", {}).get("containers") or []
|
||||
for idx, container in enumerate(containers):
|
||||
if container.get("name") == container_name:
|
||||
return idx
|
||||
if len(containers) <= 1:
|
||||
return 0
|
||||
raise RuntimeError(f"container {container_name} not found in deployment")
|
||||
|
||||
|
||||
def build_deployment_json_patch(
|
||||
*,
|
||||
container_name: str,
|
||||
container_index: int = 0,
|
||||
env_list: list[dict[str, Any]],
|
||||
strategy_recreate: bool = False,
|
||||
image: str | None = None,
|
||||
args: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a Kubernetes strategic-merge patch for a single deployment container."""
|
||||
container_patch: dict[str, Any] = {
|
||||
"name": container_name,
|
||||
"env": env_list,
|
||||
}
|
||||
if image is not None:
|
||||
container_patch["image"] = image
|
||||
if args is not None:
|
||||
container_patch["args"] = list(args)
|
||||
spec_patch: dict[str, Any] = {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [container_patch],
|
||||
}
|
||||
}
|
||||
}
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Build an atomic RFC 6902 JSON patch for a single deployment container."""
|
||||
ops: list[dict[str, Any]] = []
|
||||
if strategy_recreate:
|
||||
# Clearing rollingUpdate is required when switching from RollingUpdate -> Recreate.
|
||||
spec_patch["strategy"] = {"type": "Recreate", "rollingUpdate": None}
|
||||
return {"spec": spec_patch}
|
||||
ops.append({"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
|
||||
ops.append(
|
||||
{
|
||||
"op": "add",
|
||||
"path": f"/spec/template/spec/containers/{container_index}/env",
|
||||
"value": env_list,
|
||||
}
|
||||
)
|
||||
if image is not None:
|
||||
ops.append(
|
||||
{
|
||||
"op": "add",
|
||||
"path": f"/spec/template/spec/containers/{container_index}/image",
|
||||
"value": image,
|
||||
}
|
||||
)
|
||||
if args is not None:
|
||||
ops.append(
|
||||
{
|
||||
"op": "add",
|
||||
"path": f"/spec/template/spec/containers/{container_index}/args",
|
||||
"value": list(args),
|
||||
}
|
||||
)
|
||||
return ops
|
||||
|
||||
|
||||
def build_shadow_replica_args() -> list[str]:
|
||||
@ -369,7 +394,7 @@ class ClusterClient:
|
||||
return None
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
def patch_json(self, namespace: str, kind: str, name: str, patch_obj: dict[str, Any]) -> None:
|
||||
def patch_json_ops(self, namespace: str, kind: str, name: str, ops: list[dict[str, Any]]) -> None:
|
||||
self.run(
|
||||
[
|
||||
"-n",
|
||||
@ -377,9 +402,9 @@ class ClusterClient:
|
||||
"patch",
|
||||
kind,
|
||||
name,
|
||||
"--type=strategic",
|
||||
"--type=json",
|
||||
"-p",
|
||||
json.dumps(patch_obj),
|
||||
json.dumps(ops),
|
||||
]
|
||||
)
|
||||
|
||||
@ -563,15 +588,16 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa
|
||||
client.wait_zero_pods(SHADOW_NAMESPACE, f"component={REPLICA_CONTAINER}")
|
||||
|
||||
rep_dep = client.get_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT)
|
||||
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
rep_patch = build_deployment_update_patch(
|
||||
container_name=REPLICA_CONTAINER,
|
||||
rep_idx = find_container_index(rep_dep, REPLICA_CONTAINER)
|
||||
rep_env = rep_dep["spec"]["template"]["spec"]["containers"][rep_idx].get("env") or []
|
||||
rep_patch = build_deployment_json_patch(
|
||||
container_index=rep_idx,
|
||||
strategy_recreate=True,
|
||||
image=args.replica_image,
|
||||
args=build_shadow_replica_args(),
|
||||
env_list=build_shadow_replica_env(rep_env),
|
||||
)
|
||||
client.patch_json(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
|
||||
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch)
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 1)
|
||||
if not client.rollout_status(SHADOW_NAMESPACE, REPLICA_DEPLOYMENT, timeout_seconds=180):
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
||||
@ -602,15 +628,16 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa
|
||||
raise RuntimeError("timed out waiting for query-replica ready state")
|
||||
|
||||
bld_dep = client.get_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT)
|
||||
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
bld_patch = build_deployment_update_patch(
|
||||
container_name=BUILDER_CONTAINER,
|
||||
bld_idx = find_container_index(bld_dep, BUILDER_CONTAINER)
|
||||
bld_env = bld_dep["spec"]["template"]["spec"]["containers"][bld_idx].get("env") or []
|
||||
bld_patch = build_deployment_json_patch(
|
||||
container_index=bld_idx,
|
||||
strategy_recreate=True,
|
||||
image=args.builder_image,
|
||||
args=build_shadow_builder_args(),
|
||||
env_list=build_shadow_builder_env(bld_env),
|
||||
)
|
||||
client.patch_json(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
|
||||
client.patch_json_ops(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch)
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 1)
|
||||
if not client.rollout_status(SHADOW_NAMESPACE, BUILDER_DEPLOYMENT, timeout_seconds=180):
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
||||
@ -671,10 +698,11 @@ def sync_read_client_secret(client: ClusterClient) -> None:
|
||||
|
||||
def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
||||
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
api_idx = find_container_index(api_dep, API_CONTAINER)
|
||||
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
|
||||
rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates())
|
||||
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=rolled_env)
|
||||
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
||||
patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env)
|
||||
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
||||
ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180)
|
||||
health = inspect_api_health(client)
|
||||
path = write_report(
|
||||
@ -694,12 +722,13 @@ def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_d
|
||||
|
||||
sync_read_client_secret(client)
|
||||
api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT)
|
||||
existing_env = api_dep["spec"]["template"]["spec"]["containers"][0].get("env") or []
|
||||
api_idx = find_container_index(api_dep, API_CONTAINER)
|
||||
existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
|
||||
write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env})
|
||||
|
||||
next_env = merge_env_list(existing_env, build_api_cutover_env_updates())
|
||||
patch = build_deployment_update_patch(container_name=API_CONTAINER, env_list=next_env)
|
||||
client.patch_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
||||
patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env)
|
||||
client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch)
|
||||
|
||||
if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180):
|
||||
cmd_rollback_api_flags(args, client, out_dir)
|
||||
@ -764,6 +793,110 @@ def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir
|
||||
return 1 if rolled_back else 0
|
||||
|
||||
|
||||
def api_cutover_active(api_env: list[dict[str, Any]]) -> bool:
|
||||
by_name = {entry.get("name"): entry.get("value") for entry in (api_env or [])}
|
||||
return by_name.get("QUERY_REPLICA_REPORTS") == "1" or by_name.get("QUERY_REPLICA_COUNTS") == "1"
|
||||
|
||||
|
||||
def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int:
|
||||
source_deploys = client.get_json(SOURCE_NAMESPACE, "deployments")
|
||||
deploys_by_name = {d["metadata"]["name"]: d for d in source_deploys.get("items", [])}
|
||||
business_health = []
|
||||
unhealthy_business = []
|
||||
for name in PRODUCTION_BUSINESS_DEPLOYMENTS:
|
||||
dep = deploys_by_name.get(name)
|
||||
if dep is None:
|
||||
business_health.append({"name": name, "present": False, "desiredReplicas": 0, "readyReplicas": 0})
|
||||
unhealthy_business.append(f"{name}: missing")
|
||||
continue
|
||||
desired = int(dep["spec"].get("replicas") or 0)
|
||||
ready = int(dep.get("status", {}).get("readyReplicas") or 0)
|
||||
business_health.append({"name": name, "present": True, "desiredReplicas": desired, "readyReplicas": ready})
|
||||
if desired == 0 or ready < desired:
|
||||
unhealthy_business.append(f"{name}: {ready}/{desired}")
|
||||
|
||||
api_dep = deploys_by_name.get(API_DEPLOYMENT)
|
||||
if api_dep is None:
|
||||
raise RuntimeError(f"missing {API_DEPLOYMENT} deployment in {SOURCE_NAMESPACE}")
|
||||
api_idx = find_container_index(api_dep, API_CONTAINER)
|
||||
api_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or []
|
||||
cutover_active = api_cutover_active(api_env)
|
||||
api_health = inspect_api_health(client)
|
||||
api_healthy = (
|
||||
api_health["desiredReplicas"] > 0
|
||||
and api_health["readyReplicas"] >= api_health["desiredReplicas"]
|
||||
)
|
||||
|
||||
log_proc = client.run(
|
||||
["-n", SOURCE_NAMESPACE, "logs", f"deployment/{API_DEPLOYMENT}", "--since=12m", "--tail=500"],
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "")
|
||||
|
||||
snap = None
|
||||
parity_ok = False
|
||||
parity_reasons: list[str] = []
|
||||
try:
|
||||
snap = query_shadow_mongo_snapshot(client)
|
||||
parity_ok, parity_reasons = evaluate_parity_snapshot(
|
||||
snap,
|
||||
max_observed_age_seconds=args.max_observed_age_seconds,
|
||||
max_workforce_age_seconds=args.max_workforce_age_seconds,
|
||||
)
|
||||
except Exception as exc:
|
||||
parity_ok = False
|
||||
parity_reasons = [f"shadow mongo probe failed: {type(exc).__name__}"]
|
||||
|
||||
trigger_reasons: list[str] = []
|
||||
if cutover_active:
|
||||
if not api_healthy:
|
||||
trigger_reasons.append(
|
||||
f"API degraded: {api_health['readyReplicas']}/{api_health['desiredReplicas']}"
|
||||
)
|
||||
if error_lines >= args.max_error_lines:
|
||||
trigger_reasons.append(f"API error log spike: {error_lines} lines in 12m")
|
||||
if not parity_ok:
|
||||
trigger_reasons.extend(parity_reasons)
|
||||
|
||||
rolled_back = False
|
||||
if cutover_active and trigger_reasons:
|
||||
rolled_back = True
|
||||
cmd_rollback_api_flags(args, client, out_dir)
|
||||
|
||||
stamp = utc_now().strftime("%Y%m%dT%H%M%SZ")
|
||||
report = {
|
||||
"at": utc_now().isoformat(),
|
||||
"cutoverActive": cutover_active,
|
||||
"businessHealth": business_health,
|
||||
"unhealthyBusiness": unhealthy_business,
|
||||
"apiHealth": api_health,
|
||||
"errorLines12m": error_lines,
|
||||
"parityOk": parity_ok,
|
||||
"parityReasons": parity_reasons,
|
||||
"rolledBack": rolled_back,
|
||||
"triggerReasons": trigger_reasons,
|
||||
"shadowMongoSnapshot": snap,
|
||||
}
|
||||
path = write_report(out_dir, f"patrol-{stamp}.json", report)
|
||||
ok = (not rolled_back) and (len(unhealthy_business) == 0) and (not cutover_active or parity_ok)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"ok": ok,
|
||||
"phase": "patrol",
|
||||
"cutoverActive": cutover_active,
|
||||
"rolledBack": rolled_back,
|
||||
"errorLines12m": error_lines,
|
||||
"parityOk": parity_ok,
|
||||
"unhealthyBusiness": unhealthy_business,
|
||||
"report": str(path),
|
||||
}
|
||||
)
|
||||
)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.")
|
||||
parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
|
||||
@ -786,6 +919,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p_watch.add_argument("--watch-seconds", type=int, default=900)
|
||||
p_watch.add_argument("--max-error-lines", type=int, default=5)
|
||||
|
||||
p_patrol = sub.add_parser("patrol")
|
||||
p_patrol.add_argument("--max-error-lines", type=int, default=5)
|
||||
p_patrol.add_argument("--max-observed-age-seconds", type=int, default=600)
|
||||
p_patrol.add_argument("--max-workforce-age-seconds", type=int, default=720)
|
||||
|
||||
sub.add_parser("rollback-api-flags")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
@ -795,7 +933,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
out_dir = (
|
||||
Path(args.output_dir)
|
||||
if args.output_dir
|
||||
else Path("local-data") / "diagnostics" / "evening-cutover" / args.run_id
|
||||
else Path(__file__).resolve().parents[2] / "local-data" / "diagnostics" / "evening-cutover" / args.run_id
|
||||
)
|
||||
client = ClusterClient(args.kubeconfig)
|
||||
handlers = {
|
||||
@ -804,6 +942,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"verify-parity": cmd_verify_parity,
|
||||
"cutover-api-flags": cmd_cutover_api_flags,
|
||||
"watch-and-guard": cmd_watch_and_guard,
|
||||
"patrol": cmd_patrol,
|
||||
"rollback-api-flags": cmd_rollback_api_flags,
|
||||
}
|
||||
return handlers[args.command](args, client, out_dir)
|
||||
|
||||
@ -78,22 +78,32 @@ class EveningQueryCutoverTests(unittest.TestCase):
|
||||
CUTOVER.READ_CLIENT_SECRET_NAME,
|
||||
)
|
||||
|
||||
def test_deployment_patch_sets_recreate_and_clears_rolling_update(self):
|
||||
patch = CUTOVER.build_deployment_update_patch(
|
||||
def test_deployment_json_patch_and_find_container_index(self):
|
||||
dep = {
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [
|
||||
{"name": "sidecar"},
|
||||
{"name": "projector"},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.assertEqual(CUTOVER.find_container_index(dep, "projector"), 1)
|
||||
patch_ops = CUTOVER.build_deployment_json_patch(
|
||||
container_index=1,
|
||||
strategy_recreate=True,
|
||||
container_name="projector",
|
||||
image="example/img@sha256:abc",
|
||||
args=CUTOVER.build_shadow_replica_args(),
|
||||
env_list=CUTOVER.build_shadow_replica_env([]),
|
||||
)
|
||||
strategy = patch["spec"]["strategy"]
|
||||
self.assertEqual(strategy["type"], "Recreate")
|
||||
self.assertIn("rollingUpdate", strategy)
|
||||
self.assertIsNone(strategy["rollingUpdate"])
|
||||
container = patch["spec"]["template"]["spec"]["containers"][0]
|
||||
self.assertEqual(container["name"], "projector")
|
||||
self.assertIn("--source-compress", container["args"])
|
||||
self.assertNotIn("--safety-run", container["args"])
|
||||
by_path = {op["path"]: op for op in patch_ops}
|
||||
self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
|
||||
self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc")
|
||||
self.assertIn("--source-compress", by_path["/spec/template/spec/containers/1/args"]["value"])
|
||||
self.assertNotIn("--safety-run", by_path["/spec/template/spec/containers/1/args"]["value"])
|
||||
|
||||
def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self):
|
||||
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
@ -119,6 +129,30 @@ class EveningQueryCutoverTests(unittest.TestCase):
|
||||
self.assertFalse(passed)
|
||||
self.assertEqual(len(reasons), 2)
|
||||
|
||||
def test_proactive_patrol_thresholds_trip_before_15min_expiry(self):
|
||||
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
near_stale = {
|
||||
"state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3},
|
||||
"poolBucketsCount": 42,
|
||||
"workforceFactsCount": 1200,
|
||||
"global": {
|
||||
"heads": {
|
||||
"h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"},
|
||||
"h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"},
|
||||
}
|
||||
},
|
||||
}
|
||||
passed, reasons = CUTOVER.evaluate_parity_snapshot(
|
||||
near_stale,
|
||||
now=now,
|
||||
max_observed_age_seconds=600,
|
||||
max_workforce_age_seconds=720,
|
||||
)
|
||||
self.assertFalse(passed)
|
||||
self.assertEqual(len(reasons), 2)
|
||||
self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates()))
|
||||
self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates()))
|
||||
|
||||
def test_rollback_env_disables_all_query_reads(self):
|
||||
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
|
||||
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user