diff --git a/scripts/local/evening-query-cutover.py b/scripts/local/evening-query-cutover.py index 6c74d54e..6ce1d66e 100644 --- a/scripts/local/evening-query-cutover.py +++ b/scripts/local/evening-query-cutover.py @@ -29,6 +29,7 @@ REPLICA_DEPLOYMENT = "query-replica" REPLICA_CONTAINER = "projector" BUILDER_DEPLOYMENT = "query-read-builder" BUILDER_CONTAINER = "builder" +MONITOR_DEPLOYMENT = "query-shadow-monitor" SHADOW_MONGO_POD = "query-mongo-0" SHADOW_MONGO_CONTAINER = "mongo" PRODUCTION_BUSINESS_DEPLOYMENTS = ( @@ -272,6 +273,31 @@ def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) -> } +def parse_bson_int(val: Any) -> int: + """Parse int, float, numeric string, or mongosh Long {high, low, unsigned} objects.""" + if val is None: + return 0 + if isinstance(val, bool): + return int(val) + if isinstance(val, (int, float)): + return int(val) + if isinstance(val, str): + try: + return int(val.strip()) + except ValueError: + return 0 + if isinstance(val, dict) and ("low" in val or "high" in val): + try: + low = int(val.get("low") or 0) & 0xFFFFFFFF + high = int(val.get("high") or 0) + if val.get("unsigned"): + return ((high & 0xFFFFFFFF) << 32) | low + return (high << 32) | low + except (TypeError, ValueError): + return 0 + return 0 + + def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]: if not isinstance(global_doc, dict): return False, {"hasProduction": False, "hasWorkforce": False} @@ -289,12 +315,12 @@ def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dic gen = head.get("generation") or "" as_of = head.get("asOf") if gen == "workforce-facts/v1": - epoch = int(head.get("factEpoch") or 0) + epoch = parse_bson_int(head.get("factEpoch")) if epoch >= 1 and as_of: has_workforce = True workforce_epoch = max(workforce_epoch, epoch) workforce_as_of = as_of - elif gen and not head.get("searchCollection") and int(head.get("chunks") or 0) >= 1 and as_of: + elif gen and not head.get("searchCollection") and parse_bson_int(head.get("chunks")) >= 1 and as_of: has_production = True production_as_of = as_of details = { @@ -330,7 +356,7 @@ def evaluate_parity_snapshot( reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s") heads_ok, head_info = evaluate_builder_heads(snapshot.get("global")) - if not heads_ok or int(head_info.get("workforceFactEpoch") or 0) < 1: + if not heads_ok or parse_bson_int(head_info.get("workforceFactEpoch")) < 1: reasons.append("builder workforce or production head incomplete") else: wf_dt = parse_iso_datetime(head_info.get("workforceAsOf")) @@ -341,9 +367,9 @@ def evaluate_parity_snapshot( if wf_age < -5 or wf_age >= max_workforce_age_seconds: reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s") - if int(snapshot.get("poolBucketsCount") or 0) <= 0: + if parse_bson_int(snapshot.get("poolBucketsCount")) <= 0: reasons.append("queryPoolCountBuckets is empty") - if int(snapshot.get("workforceFactsCount") or 0) <= 0: + if parse_bson_int(snapshot.get("workforceFactsCount")) <= 0: reasons.append("queryWorkforceFacts is empty") return len(reasons) == 0, reasons @@ -446,6 +472,10 @@ const normHead = h => {{ if (!h) return null; const out = Object.assign({{}}, h); if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString(); + if (out.factEpoch !== undefined && out.factEpoch !== null) out.factEpoch = Number(out.factEpoch); + if (out.chunks !== undefined && out.chunks !== null) out.chunks = Number(out.chunks); + if (out.rows !== undefined && out.rows !== null) out.rows = Number(out.rows); + if (out.bytes !== undefined && out.bytes !== null) out.bytes = Number(out.bytes); return out; }}; const normHeads = m => {{ @@ -581,6 +611,19 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0: raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}") + # Disarm legacy shadow monitor if running so its strict safety gate cannot scale controllers back to 0. + mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True) + if mon_dep is not None and int(mon_dep.get("spec", {}).get("replicas") or 0) > 0: + client.scale(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, 0) + mon_deadline = time.monotonic() + 60 + while time.monotonic() < mon_deadline: + d = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True) + if d is None or int(d.get("status", {}).get("replicas") or 0) == 0: + break + time.sleep(2) + else: + raise RuntimeError(f"timed out waiting for {MONITOR_DEPLOYMENT} to scale to zero") + # Scale builder to 0 first to stay well inside ResourceQuota. client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0) client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0) diff --git a/scripts/local/test_evening_query_cutover.py b/scripts/local/test_evening_query_cutover.py index f122bd18..6eb5f4a3 100644 --- a/scripts/local/test_evening_query_cutover.py +++ b/scripts/local/test_evening_query_cutover.py @@ -153,6 +153,38 @@ class EveningQueryCutoverTests(unittest.TestCase): self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates())) self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates())) + def test_parse_bson_int_handles_mongosh_long_objects(self): + self.assertEqual(CUTOVER.parse_bson_int({"high": 0, "low": 1, "unsigned": False}), 1) + self.assertEqual(CUTOVER.parse_bson_int({"high": 1, "low": 0, "unsigned": True}), 1 << 32) + self.assertEqual(CUTOVER.parse_bson_int("42"), 42) + self.assertEqual(CUTOVER.parse_bson_int(None), 0) + + now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) + snap = { + "state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"}, + "poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False}, + "workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False}, + "global": { + "heads": { + "h1": { + "generation": "prod-gen-1", + "chunks": {"high": 0, "low": 45, "unsigned": False}, + "asOf": "2026-11-16T11:57:00Z", + }, + "h2": { + "generation": "workforce-facts/v1", + "factEpoch": {"high": 0, "low": 1, "unsigned": False}, + "asOf": "2026-11-16T11:57:00Z", + }, + } + }, + } + heads_ok, details = CUTOVER.evaluate_builder_heads(snap["global"]) + self.assertTrue(heads_ok) + self.assertEqual(details["workforceFactEpoch"], 1) + passed, reasons = CUTOVER.evaluate_parity_snapshot(snap, now=now) + self.assertTrue(passed, reasons) + def test_rollback_env_disables_all_query_reads(self): existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates()) rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())