forked from longtao.wu/droboticflow
🐛 Normalize mongosh Long heads and disarm legacy shadow monitor on cutover
This commit is contained in:
parent
9dfdea9e8c
commit
d0dfddaf1f
@ -29,6 +29,7 @@ REPLICA_DEPLOYMENT = "query-replica"
|
||||
REPLICA_CONTAINER = "projector"
|
||||
BUILDER_DEPLOYMENT = "query-read-builder"
|
||||
BUILDER_CONTAINER = "builder"
|
||||
MONITOR_DEPLOYMENT = "query-shadow-monitor"
|
||||
SHADOW_MONGO_POD = "query-mongo-0"
|
||||
SHADOW_MONGO_CONTAINER = "mongo"
|
||||
PRODUCTION_BUSINESS_DEPLOYMENTS = (
|
||||
@ -272,6 +273,31 @@ def build_read_client_secret_manifest(api_uri_b64: str, valkey_url_b64: str) ->
|
||||
}
|
||||
|
||||
|
||||
def parse_bson_int(val: Any) -> int:
|
||||
"""Parse int, float, numeric string, or mongosh Long {high, low, unsigned} objects."""
|
||||
if val is None:
|
||||
return 0
|
||||
if isinstance(val, bool):
|
||||
return int(val)
|
||||
if isinstance(val, (int, float)):
|
||||
return int(val)
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
return int(val.strip())
|
||||
except ValueError:
|
||||
return 0
|
||||
if isinstance(val, dict) and ("low" in val or "high" in val):
|
||||
try:
|
||||
low = int(val.get("low") or 0) & 0xFFFFFFFF
|
||||
high = int(val.get("high") or 0)
|
||||
if val.get("unsigned"):
|
||||
return ((high & 0xFFFFFFFF) << 32) | low
|
||||
return (high << 32) | low
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
|
||||
if not isinstance(global_doc, dict):
|
||||
return False, {"hasProduction": False, "hasWorkforce": False}
|
||||
@ -289,12 +315,12 @@ def evaluate_builder_heads(global_doc: dict[str, Any] | None) -> tuple[bool, dic
|
||||
gen = head.get("generation") or ""
|
||||
as_of = head.get("asOf")
|
||||
if gen == "workforce-facts/v1":
|
||||
epoch = int(head.get("factEpoch") or 0)
|
||||
epoch = parse_bson_int(head.get("factEpoch"))
|
||||
if epoch >= 1 and as_of:
|
||||
has_workforce = True
|
||||
workforce_epoch = max(workforce_epoch, epoch)
|
||||
workforce_as_of = as_of
|
||||
elif gen and not head.get("searchCollection") and int(head.get("chunks") or 0) >= 1 and as_of:
|
||||
elif gen and not head.get("searchCollection") and parse_bson_int(head.get("chunks")) >= 1 and as_of:
|
||||
has_production = True
|
||||
production_as_of = as_of
|
||||
details = {
|
||||
@ -330,7 +356,7 @@ def evaluate_parity_snapshot(
|
||||
reasons.append(f"replica observedAt age out of bounds: {obs_age:.1f}s")
|
||||
|
||||
heads_ok, head_info = evaluate_builder_heads(snapshot.get("global"))
|
||||
if not heads_ok or int(head_info.get("workforceFactEpoch") or 0) < 1:
|
||||
if not heads_ok or parse_bson_int(head_info.get("workforceFactEpoch")) < 1:
|
||||
reasons.append("builder workforce or production head incomplete")
|
||||
else:
|
||||
wf_dt = parse_iso_datetime(head_info.get("workforceAsOf"))
|
||||
@ -341,9 +367,9 @@ def evaluate_parity_snapshot(
|
||||
if wf_age < -5 or wf_age >= max_workforce_age_seconds:
|
||||
reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s")
|
||||
|
||||
if int(snapshot.get("poolBucketsCount") or 0) <= 0:
|
||||
if parse_bson_int(snapshot.get("poolBucketsCount")) <= 0:
|
||||
reasons.append("queryPoolCountBuckets is empty")
|
||||
if int(snapshot.get("workforceFactsCount") or 0) <= 0:
|
||||
if parse_bson_int(snapshot.get("workforceFactsCount")) <= 0:
|
||||
reasons.append("queryWorkforceFacts is empty")
|
||||
|
||||
return len(reasons) == 0, reasons
|
||||
@ -446,6 +472,10 @@ const normHead = h => {{
|
||||
if (!h) return null;
|
||||
const out = Object.assign({{}}, h);
|
||||
if (out.asOf instanceof Date) out.asOf = out.asOf.toISOString();
|
||||
if (out.factEpoch !== undefined && out.factEpoch !== null) out.factEpoch = Number(out.factEpoch);
|
||||
if (out.chunks !== undefined && out.chunks !== null) out.chunks = Number(out.chunks);
|
||||
if (out.rows !== undefined && out.rows !== null) out.rows = Number(out.rows);
|
||||
if (out.bytes !== undefined && out.bytes !== null) out.bytes = Number(out.bytes);
|
||||
return out;
|
||||
}};
|
||||
const normHeads = m => {{
|
||||
@ -581,6 +611,19 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa
|
||||
if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0:
|
||||
raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}")
|
||||
|
||||
# Disarm legacy shadow monitor if running so its strict safety gate cannot scale controllers back to 0.
|
||||
mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True)
|
||||
if mon_dep is not None and int(mon_dep.get("spec", {}).get("replicas") or 0) > 0:
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, 0)
|
||||
mon_deadline = time.monotonic() + 60
|
||||
while time.monotonic() < mon_deadline:
|
||||
d = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True)
|
||||
if d is None or int(d.get("status", {}).get("replicas") or 0) == 0:
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise RuntimeError(f"timed out waiting for {MONITOR_DEPLOYMENT} to scale to zero")
|
||||
|
||||
# Scale builder to 0 first to stay well inside ResourceQuota.
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0)
|
||||
client.scale(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, 0)
|
||||
|
||||
@ -153,6 +153,38 @@ class EveningQueryCutoverTests(unittest.TestCase):
|
||||
self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates()))
|
||||
self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates()))
|
||||
|
||||
def test_parse_bson_int_handles_mongosh_long_objects(self):
|
||||
self.assertEqual(CUTOVER.parse_bson_int({"high": 0, "low": 1, "unsigned": False}), 1)
|
||||
self.assertEqual(CUTOVER.parse_bson_int({"high": 1, "low": 0, "unsigned": True}), 1 << 32)
|
||||
self.assertEqual(CUTOVER.parse_bson_int("42"), 42)
|
||||
self.assertEqual(CUTOVER.parse_bson_int(None), 0)
|
||||
|
||||
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
snap = {
|
||||
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"},
|
||||
"poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False},
|
||||
"workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False},
|
||||
"global": {
|
||||
"heads": {
|
||||
"h1": {
|
||||
"generation": "prod-gen-1",
|
||||
"chunks": {"high": 0, "low": 45, "unsigned": False},
|
||||
"asOf": "2026-11-16T11:57:00Z",
|
||||
},
|
||||
"h2": {
|
||||
"generation": "workforce-facts/v1",
|
||||
"factEpoch": {"high": 0, "low": 1, "unsigned": False},
|
||||
"asOf": "2026-11-16T11:57:00Z",
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
heads_ok, details = CUTOVER.evaluate_builder_heads(snap["global"])
|
||||
self.assertTrue(heads_ok)
|
||||
self.assertEqual(details["workforceFactEpoch"], 1)
|
||||
passed, reasons = CUTOVER.evaluate_parity_snapshot(snap, now=now)
|
||||
self.assertTrue(passed, reasons)
|
||||
|
||||
def test_rollback_env_disables_all_query_reads(self):
|
||||
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
|
||||
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user