droboticflow/scripts/local/test_evening_query_cutover.py

271 lines
14 KiB
Python

import datetime as dt
import importlib.util
import json
from pathlib import Path
import unittest
import tempfile
from types import SimpleNamespace
def load_module():
path = Path(__file__).with_name("evening-query-cutover.py")
spec = importlib.util.spec_from_file_location("evening_query_cutover", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
CUTOVER = load_module()
class EveningQueryCutoverTests(unittest.TestCase):
def test_redact_secrets_scrubs_sensitive_fields_and_secrets(self):
payload = {
"kind": "Secret",
"metadata": {
"name": "droboticflow-query-read-client",
"annotations": {
"kubectl.kubernetes.io/last-applied-configuration": "secret-json"
},
},
"data": {"api-uri": "bW9uZ29kYjovL3NlY3JldA==", "valkey-url": "dmFsa2V5"},
"env": [
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
{"name": "MONGO_URI", "value": "mongodb://user:pass@host/db"},
{"name": "QUERY_VALKEY_URL", "value": "redis://:pwd@cache:6379"},
{
"name": "QUERY_READ_MONGO_URI",
"valueFrom": {"secretKeyRef": {"name": "s", "key": "api-uri"}},
},
],
}
redacted = CUTOVER.redact_secrets(payload)
self.assertEqual(redacted["data"]["api-uri"], "<redacted>")
self.assertEqual(redacted["data"]["valkey-url"], "<redacted>")
self.assertEqual(
redacted["metadata"]["annotations"]["kubectl.kubernetes.io/last-applied-configuration"],
"<redacted>",
)
env_by_name = {e["name"]: e for e in redacted["env"]}
self.assertEqual(env_by_name["MONGO_DB"]["value"], "drobodata-fanyu")
self.assertEqual(env_by_name["MONGO_URI"]["value"], "<redacted>")
self.assertEqual(env_by_name["QUERY_VALKEY_URL"]["value"], "<redacted>")
self.assertEqual(
env_by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["key"],
"api-uri",
)
serialized = json.dumps(redacted)
self.assertNotIn("user:pass", serialized)
self.assertNotIn("bW9uZ29k", serialized)
def test_merge_env_list_and_cutover_flags_contract(self):
existing = [
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
{"name": "QUERY_SOURCE_TRACKING", "value": "1"},
{"name": "QUERY_REPLICA_POOLS", "value": "1"},
{"name": "QUERY_PUBLISHED_RESULTS", "value": "1"},
]
merged = CUTOVER.merge_env_list(existing, CUTOVER.build_api_cutover_env_updates())
by_name = {e["name"]: e for e in merged}
self.assertEqual(by_name["MONGO_DB"]["value"], "drobodata-fanyu")
self.assertEqual(by_name["QUERY_SOURCE_TRACKING"]["value"], "1")
self.assertEqual(by_name["QUERY_PUBLISHED_RESULTS"]["value"], "0")
self.assertEqual(by_name["QUERY_REPLICA_POOLS"]["value"], "0")
self.assertEqual(by_name["QUERY_REPLICA_REPORTS"]["value"], "1")
self.assertEqual(by_name["QUERY_REPLICA_COUNTS"]["value"], "1")
self.assertEqual(by_name["QUERY_SERVING_READS"]["value"], "1")
self.assertEqual(by_name["QUERY_REPLICA_REFRESH_MODE"]["value"], "periodic")
self.assertEqual(
by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["name"],
CUTOVER.READ_CLIENT_SECRET_NAME,
)
def test_deployment_json_patch_and_find_container_index(self):
dep = {
"spec": {
"template": {
"spec": {
"containers": [
{"name": "sidecar"},
{"name": "projector"},
]
}
}
}
}
self.assertEqual(CUTOVER.find_container_index(dep, "projector"), 1)
patch_ops = CUTOVER.build_deployment_json_patch(
container_index=1,
strategy_recreate=True,
image="example/img@sha256:abc",
args=CUTOVER.build_shadow_replica_args({"expiresAt":"2099-01-01T00:00:00Z","shadowPolicy":{"runID":"same-run","maintenanceUntil":"2099-01-01T00:00:00Z","maxRecords":15000000,"recordsPerSecond":100,"bytesPerSecond":1048576,"commandsPerSecond":16,"batchSize":100}}),
env_list=CUTOVER.build_shadow_replica_env([]),
)
by_path = {op["path"]: op for op in patch_ops}
self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc")
self.assertIn("--maintenance-initialization", by_path["/spec/template/spec/containers/1/args"]["value"])
self.assertIn("--safety-run=same-run", by_path["/spec/template/spec/containers/1/args"]["value"])
def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self):
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
good = {
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3},
"poolBucketsCount": 42,
"workforceFactsCount": 1200,
"cohorts":[{"productionKey":"h1","workforceKey":"h2"}],
"global": {
"heads": {
"h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"},
"h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"},
}
},
}
passed, reasons = CUTOVER.evaluate_parity_snapshot(good, now=now)
self.assertTrue(passed, reasons)
self.assertEqual(reasons, [])
stale = json.loads(json.dumps(good))
stale["state"]["observedAt"] = "2026-11-16T11:45:00Z"
stale["workforceFactsCount"] = 0
passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now)
self.assertFalse(passed)
self.assertGreaterEqual(len(reasons), 2)
def test_proactive_patrol_thresholds_trip_before_15min_expiry(self):
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
near_stale = {
"state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3},
"poolBucketsCount": 42,
"workforceFactsCount": 1200,
"cohorts":[{"productionKey":"h1","workforceKey":"h2"}],
"global": {
"heads": {
"h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"},
"h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"},
}
},
}
passed, reasons = CUTOVER.evaluate_parity_snapshot(
near_stale,
now=now,
max_observed_age_seconds=600,
max_workforce_age_seconds=720,
)
self.assertFalse(passed)
self.assertGreaterEqual(len(reasons), 2)
self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates()))
self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates()))
def test_parse_bson_int_handles_mongosh_long_objects(self):
self.assertEqual(CUTOVER.parse_bson_int({"high": 0, "low": 1, "unsigned": False}), 1)
self.assertEqual(CUTOVER.parse_bson_int({"high": 1, "low": 0, "unsigned": True}), 1 << 32)
self.assertEqual(CUTOVER.parse_bson_int("42"), 42)
self.assertEqual(CUTOVER.parse_bson_int(None), 0)
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
snap = {
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"},
"poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False},
"workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False},
"cohorts":[{"productionKey":"h1","workforceKey":"h2"}],
"global": {
"heads": {
"h1": {
"sourceRevision":1,"globalRevision":1,
"generation": "prod-gen-1",
"chunks": {"high": 0, "low": 45, "unsigned": False},
"asOf": "2026-11-16T11:57:00Z",
},
"h2": {
"sourceRevision":1,"globalRevision":1,
"generation": "workforce-facts/v1",
"factEpoch": {"high": 0, "low": 1, "unsigned": False},
"asOf": "2026-11-16T11:57:00Z",
},
}
},
}
heads_ok, details = CUTOVER.evaluate_builder_heads(snap["global"])
self.assertTrue(heads_ok)
self.assertEqual(details["workforceFactEpoch"], 1)
passed, reasons = CUTOVER.evaluate_parity_snapshot(snap, now=now)
self.assertTrue(passed, reasons)
def test_rollback_env_disables_all_query_reads(self):
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())
by_name = {e["name"]: e for e in rolled}
for key in (
"QUERY_REPLICA_REPORTS",
"QUERY_REPLICA_COUNTS",
"QUERY_REPLICA_POOLS",
"QUERY_SERVING_READS",
"QUERY_PUBLISHED_RESULTS",
):
self.assertEqual(by_name[key]["value"], "0")
class CutoverMutationProtectionTests(unittest.TestCase):
def test_patch_uses_uid_rv_and_stdin(self):
from unittest.mock import Mock
client = CUTOVER.ClusterClient("unused")
client.approved_plan = {"resourceUIDs": {"ns/deployment/app": "uid"}}
client.get_json = Mock(return_value={"metadata": {"uid": "uid", "resourceVersion": "10"}})
client.run = Mock()
client.patch_json_ops("ns", "deployment", "app", [{"op": "replace", "path": "/spec/replicas", "value": 0}])
argv = client.run.call_args.args[0]
ops = json.loads(client.run.call_args.kwargs["stdin_text"])
self.assertIn("--patch-file=/dev/stdin", argv)
self.assertEqual([o["path"] for o in ops[:2]], ["/metadata/uid", "/metadata/resourceVersion"])
self.assertEqual(ops[1]["value"], "10")
client.get_json.return_value["metadata"]["uid"] = "replaced"
with self.assertRaises(RuntimeError):
client.scale("ns", "deployment", "app", 1)
def test_original_read_version_is_preserved_in_patch(self):
dep = {"metadata": {"uid": "uid", "resourceVersion": "9"}}
ops = CUTOVER.build_deployment_json_patch(env_list=[], expected_deployment=dep)
self.assertEqual(ops[1], {"op": "test", "path": "/metadata/resourceVersion", "value": "9"})
def test_observer_not_disarmed_to_force_sync(self):
from unittest.mock import Mock, patch
client = Mock()
client.get_json.return_value = {"status": {"availableReplicas": 0}}
with tempfile.TemporaryDirectory() as directory, patch.object(CUTOVER,"inspect_api_health",return_value={"readyReplicas":4,"desiredReplicas":4}):
with self.assertRaisesRegex(RuntimeError,"observer"):
CUTOVER.cmd_shadow_sync(SimpleNamespace(),client,Path(directory))
client.scale.assert_not_called()
def test_nonce_never_saved_in_snapshot_reports(self):
self.assertEqual(CUTOVER.redact_secrets({"args":["--resume-pause=private-token"]}),{"args":["--resume-pause=<redacted>"]})
class ScopedPublicationTests(unittest.TestCase):
def test_unrelated_or_mixed_heads_never_mean_scope_ready(self):
now=dt.datetime(2026,11,16,12,tzinfo=dt.timezone.utc)
base={"state":{"ready":True,"observedAt":now.isoformat()},"poolBucketsCount":1,"workforceFactsCount":1,"cohorts":[{"productionKey":"p","workforceKey":"w"}],"global":{"heads":{"p":{"generation":"p","chunks":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2},"w":{"generation":"workforce-facts/v1","factEpoch":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2}}}}
self.assertTrue(CUTOVER.evaluate_parity_snapshot(base,now=now)[0])
for key in ("asOf","sourceRevision","globalRevision"):
mixed=json.loads(json.dumps(base));mixed['global']['heads']['w'][key]=None
self.assertFalse(CUTOVER.evaluate_parity_snapshot(mixed,now=now)[0])
wrong=json.loads(json.dumps(base));wrong['cohorts'][0]['productionKey']='other-org'
self.assertFalse(CUTOVER.evaluate_parity_snapshot(wrong,now=now)[0])
class BoundedMetadataProbeTests(unittest.TestCase):
def test_transient_metadata_failure_retries_but_auth_failure_does_not(self):
from unittest.mock import Mock, patch
import subprocess
client=CUTOVER.ClusterClient('unused')
client.run=Mock(side_effect=[subprocess.TimeoutExpired('get',8),SimpleNamespace(returncode=1,stderr='temporary',stdout=''),SimpleNamespace(returncode=0,stdout='{"metadata":{"uid":"u"}}')])
with patch.object(CUTOVER.time,'sleep'):
self.assertEqual(client.get_json('ns','deployment','app')['metadata']['uid'],'u')
self.assertEqual(client.run.call_count,3)
client.run=Mock(return_value=SimpleNamespace(returncode=1,stderr='Forbidden',stdout=''))
with self.assertRaises(RuntimeError):client.get_json('ns','deployment','app')
self.assertEqual(client.run.call_count,1)
if __name__ == "__main__":
unittest.main()