import datetime as dt import importlib.util import json from pathlib import Path import unittest import tempfile from types import SimpleNamespace def load_module(): path = Path(__file__).with_name("evening-query-cutover.py") spec = importlib.util.spec_from_file_location("evening_query_cutover", path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod CUTOVER = load_module() class EveningQueryCutoverTests(unittest.TestCase): def test_redact_secrets_scrubs_sensitive_fields_and_secrets(self): payload = { "kind": "Secret", "metadata": { "name": "droboticflow-query-read-client", "annotations": { "kubectl.kubernetes.io/last-applied-configuration": "secret-json" }, }, "data": {"api-uri": "bW9uZ29kYjovL3NlY3JldA==", "valkey-url": "dmFsa2V5"}, "env": [ {"name": "MONGO_DB", "value": "drobodata-fanyu"}, {"name": "MONGO_URI", "value": "mongodb://user:pass@host/db"}, {"name": "QUERY_VALKEY_URL", "value": "redis://:pwd@cache:6379"}, { "name": "QUERY_READ_MONGO_URI", "valueFrom": {"secretKeyRef": {"name": "s", "key": "api-uri"}}, }, ], } redacted = CUTOVER.redact_secrets(payload) self.assertEqual(redacted["data"]["api-uri"], "") self.assertEqual(redacted["data"]["valkey-url"], "") self.assertEqual( redacted["metadata"]["annotations"]["kubectl.kubernetes.io/last-applied-configuration"], "", ) env_by_name = {e["name"]: e for e in redacted["env"]} self.assertEqual(env_by_name["MONGO_DB"]["value"], "drobodata-fanyu") self.assertEqual(env_by_name["MONGO_URI"]["value"], "") self.assertEqual(env_by_name["QUERY_VALKEY_URL"]["value"], "") self.assertEqual( env_by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["key"], "api-uri", ) serialized = json.dumps(redacted) self.assertNotIn("user:pass", serialized) self.assertNotIn("bW9uZ29k", serialized) def test_merge_env_list_and_cutover_flags_contract(self): existing = [ {"name": "MONGO_DB", "value": "drobodata-fanyu"}, {"name": "QUERY_SOURCE_TRACKING", "value": "1"}, {"name": "QUERY_REPLICA_POOLS", "value": "1"}, {"name": "QUERY_PUBLISHED_RESULTS", "value": "1"}, ] merged = CUTOVER.merge_env_list(existing, CUTOVER.build_api_cutover_env_updates()) by_name = {e["name"]: e for e in merged} self.assertEqual(by_name["MONGO_DB"]["value"], "drobodata-fanyu") self.assertEqual(by_name["QUERY_SOURCE_TRACKING"]["value"], "1") self.assertEqual(by_name["QUERY_PUBLISHED_RESULTS"]["value"], "0") self.assertEqual(by_name["QUERY_REPLICA_POOLS"]["value"], "0") self.assertEqual(by_name["QUERY_REPLICA_REPORTS"]["value"], "1") self.assertEqual(by_name["QUERY_REPLICA_COUNTS"]["value"], "1") self.assertEqual(by_name["QUERY_SERVING_READS"]["value"], "1") self.assertEqual(by_name["QUERY_REPLICA_REFRESH_MODE"]["value"], "periodic") self.assertEqual( by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["name"], CUTOVER.READ_CLIENT_SECRET_NAME, ) def test_deployment_json_patch_and_find_container_index(self): dep = { "spec": { "template": { "spec": { "containers": [ {"name": "sidecar"}, {"name": "projector"}, ] } } } } self.assertEqual(CUTOVER.find_container_index(dep, "projector"), 1) patch_ops = CUTOVER.build_deployment_json_patch( container_index=1, strategy_recreate=True, image="example/img@sha256:abc", args=CUTOVER.build_shadow_replica_args({"expiresAt":"2099-01-01T00:00:00Z","shadowPolicy":{"runID":"same-run","maintenanceUntil":"2099-01-01T00:00:00Z","maxRecords":15000000,"recordsPerSecond":100,"bytesPerSecond":1048576,"commandsPerSecond":16,"batchSize":100}}), env_list=CUTOVER.build_shadow_replica_env([]), ) by_path = {op["path"]: op for op in patch_ops} self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}}) self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc") self.assertIn("--maintenance-initialization", by_path["/spec/template/spec/containers/1/args"]["value"]) self.assertIn("--safety-run=same-run", by_path["/spec/template/spec/containers/1/args"]["value"]) def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self): now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) good = { "state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3}, "poolBucketsCount": 42, "workforceFactsCount": 1200, "cohorts":[{"productionKey":"h1","workforceKey":"h2"}], "global": { "heads": { "h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"}, "h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"}, } }, } passed, reasons = CUTOVER.evaluate_parity_snapshot(good, now=now) self.assertTrue(passed, reasons) self.assertEqual(reasons, []) stale = json.loads(json.dumps(good)) stale["state"]["observedAt"] = "2026-11-16T11:45:00Z" stale["workforceFactsCount"] = 0 passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now) self.assertFalse(passed) self.assertGreaterEqual(len(reasons), 2) def test_proactive_patrol_thresholds_trip_before_15min_expiry(self): now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) near_stale = { "state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3}, "poolBucketsCount": 42, "workforceFactsCount": 1200, "cohorts":[{"productionKey":"h1","workforceKey":"h2"}], "global": { "heads": { "h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"}, "h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"}, } }, } passed, reasons = CUTOVER.evaluate_parity_snapshot( near_stale, now=now, max_observed_age_seconds=600, max_workforce_age_seconds=720, ) self.assertFalse(passed) self.assertGreaterEqual(len(reasons), 2) self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates())) self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates())) def test_parse_bson_int_handles_mongosh_long_objects(self): self.assertEqual(CUTOVER.parse_bson_int({"high": 0, "low": 1, "unsigned": False}), 1) self.assertEqual(CUTOVER.parse_bson_int({"high": 1, "low": 0, "unsigned": True}), 1 << 32) self.assertEqual(CUTOVER.parse_bson_int("42"), 42) self.assertEqual(CUTOVER.parse_bson_int(None), 0) now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) snap = { "state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"}, "poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False}, "workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False}, "cohorts":[{"productionKey":"h1","workforceKey":"h2"}], "global": { "heads": { "h1": { "sourceRevision":1,"globalRevision":1, "generation": "prod-gen-1", "chunks": {"high": 0, "low": 45, "unsigned": False}, "asOf": "2026-11-16T11:57:00Z", }, "h2": { "sourceRevision":1,"globalRevision":1, "generation": "workforce-facts/v1", "factEpoch": {"high": 0, "low": 1, "unsigned": False}, "asOf": "2026-11-16T11:57:00Z", }, } }, } heads_ok, details = CUTOVER.evaluate_builder_heads(snap["global"]) self.assertTrue(heads_ok) self.assertEqual(details["workforceFactEpoch"], 1) passed, reasons = CUTOVER.evaluate_parity_snapshot(snap, now=now) self.assertTrue(passed, reasons) def test_rollback_env_disables_all_query_reads(self): existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates()) rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates()) by_name = {e["name"]: e for e in rolled} for key in ( "QUERY_REPLICA_REPORTS", "QUERY_REPLICA_COUNTS", "QUERY_REPLICA_POOLS", "QUERY_SERVING_READS", "QUERY_PUBLISHED_RESULTS", ): self.assertEqual(by_name[key]["value"], "0") class CutoverMutationProtectionTests(unittest.TestCase): def test_patch_uses_uid_rv_and_stdin(self): from unittest.mock import Mock client = CUTOVER.ClusterClient("unused") client.approved_plan = {"resourceUIDs": {"ns/deployment/app": "uid"}} client.get_json = Mock(return_value={"metadata": {"uid": "uid", "resourceVersion": "10"}}) client.run = Mock() client.patch_json_ops("ns", "deployment", "app", [{"op": "replace", "path": "/spec/replicas", "value": 0}]) argv = client.run.call_args.args[0] ops = json.loads(client.run.call_args.kwargs["stdin_text"]) self.assertIn("--patch-file=/dev/stdin", argv) self.assertEqual([o["path"] for o in ops[:2]], ["/metadata/uid", "/metadata/resourceVersion"]) self.assertEqual(ops[1]["value"], "10") client.get_json.return_value["metadata"]["uid"] = "replaced" with self.assertRaises(RuntimeError): client.scale("ns", "deployment", "app", 1) def test_original_read_version_is_preserved_in_patch(self): dep = {"metadata": {"uid": "uid", "resourceVersion": "9"}} ops = CUTOVER.build_deployment_json_patch(env_list=[], expected_deployment=dep) self.assertEqual(ops[1], {"op": "test", "path": "/metadata/resourceVersion", "value": "9"}) def test_observer_not_disarmed_to_force_sync(self): from unittest.mock import Mock, patch client = Mock() client.get_json.return_value = {"status": {"availableReplicas": 0}} with tempfile.TemporaryDirectory() as directory, patch.object(CUTOVER,"inspect_api_health",return_value={"readyReplicas":4,"desiredReplicas":4}): with self.assertRaisesRegex(RuntimeError,"observer"): CUTOVER.cmd_shadow_sync(SimpleNamespace(),client,Path(directory)) client.scale.assert_not_called() def test_nonce_never_saved_in_snapshot_reports(self): self.assertEqual(CUTOVER.redact_secrets({"args":["--resume-pause=private-token"]}),{"args":["--resume-pause="]}) class ScopedPublicationTests(unittest.TestCase): def test_unrelated_or_mixed_heads_never_mean_scope_ready(self): now=dt.datetime(2026,11,16,12,tzinfo=dt.timezone.utc) base={"state":{"ready":True,"observedAt":now.isoformat()},"poolBucketsCount":1,"workforceFactsCount":1,"cohorts":[{"productionKey":"p","workforceKey":"w"}],"global":{"heads":{"p":{"generation":"p","chunks":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2},"w":{"generation":"workforce-facts/v1","factEpoch":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2}}}} self.assertTrue(CUTOVER.evaluate_parity_snapshot(base,now=now)[0]) for key in ("asOf","sourceRevision","globalRevision"): mixed=json.loads(json.dumps(base));mixed['global']['heads']['w'][key]=None self.assertFalse(CUTOVER.evaluate_parity_snapshot(mixed,now=now)[0]) wrong=json.loads(json.dumps(base));wrong['cohorts'][0]['productionKey']='other-org' self.assertFalse(CUTOVER.evaluate_parity_snapshot(wrong,now=now)[0]) class BoundedMetadataProbeTests(unittest.TestCase): def test_transient_metadata_failure_retries_but_auth_failure_does_not(self): from unittest.mock import Mock, patch import subprocess client=CUTOVER.ClusterClient('unused') client.run=Mock(side_effect=[subprocess.TimeoutExpired('get',8),SimpleNamespace(returncode=1,stderr='temporary',stdout=''),SimpleNamespace(returncode=0,stdout='{"metadata":{"uid":"u"}}')]) with patch.object(CUTOVER.time,'sleep'): self.assertEqual(client.get_json('ns','deployment','app')['metadata']['uid'],'u') self.assertEqual(client.run.call_count,3) client.run=Mock(return_value=SimpleNamespace(returncode=1,stderr='Forbidden',stdout='')) with self.assertRaises(RuntimeError):client.get_json('ns','deployment','app') self.assertEqual(client.run.call_count,1) if __name__ == "__main__": unittest.main()