🐛 Normalize mongosh Long heads and disarm legacy shadow monitor on cutover

This commit is contained in:
d-robotics 2026-11-16 01:32:23 -08:00
parent 9dfdea9e8c
commit d0dfddaf1f
2 changed files with 80 additions and 5 deletions

View File

@ -29,6 +29,7 @@ REPLICA_DEPLOYMENT = "query-replica"
REPLICA_CONTAINER = "projector"
BUILDER_DEPLOYMENT = "query-read-builder"
BUILDER_CONTAINER = "builder"
MONITOR_DEPLOYMENT = "query-shadow-monitor"
SHADOW_MONGO_POD = "query-mongo-0"
SHADOW_MONGO_CONTAINER = "mongo"
PRODUCTION_BUSINESS_DEPLOYMENTS = (
@ -272,6 +273,31 @@ def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) ->
}
def parse_bson_int(val: Any) -> int:
"""Parse int, float, numeric string, or mongosh Long {high, low, unsigned} objects."""
if val is None:
return 0
if isinstance(val, bool):
return int(val)
if isinstance(val, (int, float)):
return int(val)
if isinstance(val, str):
try:
return int(val.strip())
except ValueError:
return 0
if isinstance(val, dict) and ("low" in val or "high" in val):
try:
low = int(val.get("low") or 0) & 0xFFFFFFFF
high = int(val.get("high") or 0)
if val.get("unsigned"):
return ((high & 0xFFFFFFFF) << 32) | low
return (high << 32) | low
except (TypeError, ValueError):
return 0
return 0
def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
if not isinstance(global_doc, dict):
return False, {"hasProduction": False, "hasWorkforce": False}
@ -289,12 +315,12 @@ def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dic
gen = head.get("generation") or ""
as_of = head.get("asOf")
if gen == "workforce-facts/v1":
epoch = int(head.get("factEpoch") or 0)
epoch = parse_bson_int(head.get("factEpoch"))
if epoch >= 1 and as_of:
has_workforce = True
workforce_epoch = max(workforce_epoch, epoch)
workforce_as_of = as_of
elif gen and not head.get("searchCollection") and int(head.get("chunks") or 0) >= 1 and as_of:
elif gen and not head.get("searchCollection") and parse_bson_int(head.get("chunks")) >= 1 and as_of:
has_production = True
production_as_of = as_of
details = {
@ -330,7 +356,7 @@ def evaluate_parity_snapshot(
reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s")
heads_ok, head_info = evaluate_builder_heads(snapshot.get("global"))
if not heads_ok or int(head_info.get("workforceFactEpoch") or 0) < 1:
if not heads_ok or parse_bson_int(head_info.get("workforceFactEpoch")) < 1:
reasons.append("builder workforce or production head incomplete")
else:
wf_dt = parse_iso_datetime(head_info.get("workforceAsOf"))
@ -341,9 +367,9 @@ def evaluate_parity_snapshot(
if wf_age < -5 or wf_age >= max_workforce_age_seconds:
reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s")
if int(snapshot.get("poolBucketsCount") or 0) <= 0:
if parse_bson_int(snapshot.get("poolBucketsCount")) <= 0:
reasons.append("queryPoolCountBuckets is empty")
if int(snapshot.get("workforceFactsCount") or 0) <= 0:
if parse_bson_int(snapshot.get("workforceFactsCount")) <= 0:
reasons.append("queryWorkforceFacts is empty")
return len(reasons) == 0, reasons
@ -446,6 +472,10 @@ const normHead = h => {{
if (!h) return null;
const out = Object.assign({{}}, h);
if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString();
if (out.factEpoch !== undefined && out.factEpoch !== null) out.factEpoch = Number(out.factEpoch);
if (out.chunks !== undefined && out.chunks !== null) out.chunks = Number(out.chunks);
if (out.rows !== undefined && out.rows !== null) out.rows = Number(out.rows);
if (out.bytes !== undefined && out.bytes !== null) out.bytes = Number(out.bytes);
return out;
}};
const normHeads = m => {{
@ -581,6 +611,19 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa
if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0:
raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}")
# Disarm legacy shadow monitor if running so its strict safety gate cannot scale controllers back to 0.
mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True)
if mon_dep is not None and int(mon_dep.get("spec", {}).get("replicas") or 0) > 0:
client.scale(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, 0)
mon_deadline = time.monotonic() + 60
while time.monotonic() < mon_deadline:
d = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True)
if d is None or int(d.get("status", {}).get("replicas") or 0) == 0:
break
time.sleep(2)
else:
raise RuntimeError(f"timed out waiting for {MONITOR_DEPLOYMENT} to scale to zero")
# Scale builder to 0 first to stay well inside ResourceQuota.
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)

View File

@ -153,6 +153,38 @@ class EveningQueryCutoverTests(unittest.TestCase):
self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates()))
self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates()))
def test_parse_bson_int_handles_mongosh_long_objects(self):
self.assertEqual(CUTOVER.parse_bson_int({"high": 0, "low": 1, "unsigned": False}), 1)
self.assertEqual(CUTOVER.parse_bson_int({"high": 1, "low": 0, "unsigned": True}), 1 << 32)
self.assertEqual(CUTOVER.parse_bson_int("42"), 42)
self.assertEqual(CUTOVER.parse_bson_int(None), 0)
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
snap = {
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"},
"poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False},
"workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False},
"global": {
"heads": {
"h1": {
"generation": "prod-gen-1",
"chunks": {"high": 0, "low": 45, "unsigned": False},
"asOf": "2026-11-16T11:57:00Z",
},
"h2": {
"generation": "workforce-facts/v1",
"factEpoch": {"high": 0, "low": 1, "unsigned": False},
"asOf": "2026-11-16T11:57:00Z",
},
}
},
}
heads_ok, details = CUTOVER.evaluate_builder_heads(snap["global"])
self.assertTrue(heads_ok)
self.assertEqual(details["workforceFactEpoch"], 1)
passed, reasons = CUTOVER.evaluate_parity_snapshot(snap, now=now)
self.assertTrue(passed, reasons)
def test_rollback_env_disables_all_query_reads(self):
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())