forked from longtao.wu/droboticflow
271 lines
14 KiB
Python
271 lines
14 KiB
Python
import datetime as dt
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import unittest
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
|
|
|
|
def load_module():
|
|
path = Path(__file__).with_name("evening-query-cutover.py")
|
|
spec = importlib.util.spec_from_file_location("evening_query_cutover", path)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
CUTOVER = load_module()
|
|
|
|
|
|
class EveningQueryCutoverTests(unittest.TestCase):
|
|
def test_redact_secrets_scrubs_sensitive_fields_and_secrets(self):
|
|
payload = {
|
|
"kind": "Secret",
|
|
"metadata": {
|
|
"name": "droboticflow-query-read-client",
|
|
"annotations": {
|
|
"kubectl.kubernetes.io/last-applied-configuration": "secret-json"
|
|
},
|
|
},
|
|
"data": {"api-uri": "bW9uZ29kYjovL3NlY3JldA==", "valkey-url": "dmFsa2V5"},
|
|
"env": [
|
|
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
|
|
{"name": "MONGO_URI", "value": "mongodb://user:pass@host/db"},
|
|
{"name": "QUERY_VALKEY_URL", "value": "redis://:pwd@cache:6379"},
|
|
{
|
|
"name": "QUERY_READ_MONGO_URI",
|
|
"valueFrom": {"secretKeyRef": {"name": "s", "key": "api-uri"}},
|
|
},
|
|
],
|
|
}
|
|
redacted = CUTOVER.redact_secrets(payload)
|
|
self.assertEqual(redacted["data"]["api-uri"], "<redacted>")
|
|
self.assertEqual(redacted["data"]["valkey-url"], "<redacted>")
|
|
self.assertEqual(
|
|
redacted["metadata"]["annotations"]["kubectl.kubernetes.io/last-applied-configuration"],
|
|
"<redacted>",
|
|
)
|
|
env_by_name = {e["name"]: e for e in redacted["env"]}
|
|
self.assertEqual(env_by_name["MONGO_DB"]["value"], "drobodata-fanyu")
|
|
self.assertEqual(env_by_name["MONGO_URI"]["value"], "<redacted>")
|
|
self.assertEqual(env_by_name["QUERY_VALKEY_URL"]["value"], "<redacted>")
|
|
self.assertEqual(
|
|
env_by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["key"],
|
|
"api-uri",
|
|
)
|
|
serialized = json.dumps(redacted)
|
|
self.assertNotIn("user:pass", serialized)
|
|
self.assertNotIn("bW9uZ29k", serialized)
|
|
|
|
def test_merge_env_list_and_cutover_flags_contract(self):
|
|
existing = [
|
|
{"name": "MONGO_DB", "value": "drobodata-fanyu"},
|
|
{"name": "QUERY_SOURCE_TRACKING", "value": "1"},
|
|
{"name": "QUERY_REPLICA_POOLS", "value": "1"},
|
|
{"name": "QUERY_PUBLISHED_RESULTS", "value": "1"},
|
|
]
|
|
merged = CUTOVER.merge_env_list(existing, CUTOVER.build_api_cutover_env_updates())
|
|
by_name = {e["name"]: e for e in merged}
|
|
self.assertEqual(by_name["MONGO_DB"]["value"], "drobodata-fanyu")
|
|
self.assertEqual(by_name["QUERY_SOURCE_TRACKING"]["value"], "1")
|
|
self.assertEqual(by_name["QUERY_PUBLISHED_RESULTS"]["value"], "0")
|
|
self.assertEqual(by_name["QUERY_REPLICA_POOLS"]["value"], "0")
|
|
self.assertEqual(by_name["QUERY_REPLICA_REPORTS"]["value"], "1")
|
|
self.assertEqual(by_name["QUERY_REPLICA_COUNTS"]["value"], "1")
|
|
self.assertEqual(by_name["QUERY_SERVING_READS"]["value"], "1")
|
|
self.assertEqual(by_name["QUERY_REPLICA_REFRESH_MODE"]["value"], "periodic")
|
|
self.assertEqual(
|
|
by_name["QUERY_READ_MONGO_URI"]["valueFrom"]["secretKeyRef"]["name"],
|
|
CUTOVER.READ_CLIENT_SECRET_NAME,
|
|
)
|
|
|
|
def test_deployment_json_patch_and_find_container_index(self):
|
|
dep = {
|
|
"spec": {
|
|
"template": {
|
|
"spec": {
|
|
"containers": [
|
|
{"name": "sidecar"},
|
|
{"name": "projector"},
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
self.assertEqual(CUTOVER.find_container_index(dep, "projector"), 1)
|
|
patch_ops = CUTOVER.build_deployment_json_patch(
|
|
container_index=1,
|
|
strategy_recreate=True,
|
|
image="example/img@sha256:abc",
|
|
args=CUTOVER.build_shadow_replica_args({"expiresAt":"2099-01-01T00:00:00Z","shadowPolicy":{"runID":"same-run","maintenanceUntil":"2099-01-01T00:00:00Z","maxRecords":15000000,"recordsPerSecond":100,"bytesPerSecond":1048576,"commandsPerSecond":16,"batchSize":100}}),
|
|
env_list=CUTOVER.build_shadow_replica_env([]),
|
|
)
|
|
by_path = {op["path"]: op for op in patch_ops}
|
|
self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}})
|
|
self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc")
|
|
self.assertIn("--maintenance-initialization", by_path["/spec/template/spec/containers/1/args"]["value"])
|
|
self.assertIn("--safety-run=same-run", by_path["/spec/template/spec/containers/1/args"]["value"])
|
|
|
|
def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self):
|
|
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
|
good = {
|
|
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3},
|
|
"poolBucketsCount": 42,
|
|
"workforceFactsCount": 1200,
|
|
"cohorts":[{"productionKey":"h1","workforceKey":"h2"}],
|
|
"global": {
|
|
"heads": {
|
|
"h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"},
|
|
"h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"},
|
|
}
|
|
},
|
|
}
|
|
passed, reasons = CUTOVER.evaluate_parity_snapshot(good, now=now)
|
|
self.assertTrue(passed, reasons)
|
|
self.assertEqual(reasons, [])
|
|
|
|
stale = json.loads(json.dumps(good))
|
|
stale["state"]["observedAt"] = "2026-11-16T11:45:00Z"
|
|
stale["workforceFactsCount"] = 0
|
|
passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now)
|
|
self.assertFalse(passed)
|
|
self.assertGreaterEqual(len(reasons), 2)
|
|
|
|
def test_proactive_patrol_thresholds_trip_before_15min_expiry(self):
|
|
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
|
near_stale = {
|
|
"state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3},
|
|
"poolBucketsCount": 42,
|
|
"workforceFactsCount": 1200,
|
|
"cohorts":[{"productionKey":"h1","workforceKey":"h2"}],
|
|
"global": {
|
|
"heads": {
|
|
"h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"},
|
|
"h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"},
|
|
}
|
|
},
|
|
}
|
|
passed, reasons = CUTOVER.evaluate_parity_snapshot(
|
|
near_stale,
|
|
now=now,
|
|
max_observed_age_seconds=600,
|
|
max_workforce_age_seconds=720,
|
|
)
|
|
self.assertFalse(passed)
|
|
self.assertGreaterEqual(len(reasons), 2)
|
|
self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates()))
|
|
self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates()))
|
|
|
|
def test_parse_bson_int_handles_mongosh_long_objects(self):
|
|
self.assertEqual(CUTOVER.parse_bson_int({"high": 0, "low": 1, "unsigned": False}), 1)
|
|
self.assertEqual(CUTOVER.parse_bson_int({"high": 1, "low": 0, "unsigned": True}), 1 << 32)
|
|
self.assertEqual(CUTOVER.parse_bson_int("42"), 42)
|
|
self.assertEqual(CUTOVER.parse_bson_int(None), 0)
|
|
|
|
now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc)
|
|
snap = {
|
|
"state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"},
|
|
"poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False},
|
|
"workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False},
|
|
"cohorts":[{"productionKey":"h1","workforceKey":"h2"}],
|
|
"global": {
|
|
"heads": {
|
|
"h1": {
|
|
"sourceRevision":1,"globalRevision":1,
|
|
"generation": "prod-gen-1",
|
|
"chunks": {"high": 0, "low": 45, "unsigned": False},
|
|
"asOf": "2026-11-16T11:57:00Z",
|
|
},
|
|
"h2": {
|
|
"sourceRevision":1,"globalRevision":1,
|
|
"generation": "workforce-facts/v1",
|
|
"factEpoch": {"high": 0, "low": 1, "unsigned": False},
|
|
"asOf": "2026-11-16T11:57:00Z",
|
|
},
|
|
}
|
|
},
|
|
}
|
|
heads_ok, details = CUTOVER.evaluate_builder_heads(snap["global"])
|
|
self.assertTrue(heads_ok)
|
|
self.assertEqual(details["workforceFactEpoch"], 1)
|
|
passed, reasons = CUTOVER.evaluate_parity_snapshot(snap, now=now)
|
|
self.assertTrue(passed, reasons)
|
|
|
|
def test_rollback_env_disables_all_query_reads(self):
|
|
existing = CUTOVER.merge_env_list([], CUTOVER.build_api_cutover_env_updates())
|
|
rolled = CUTOVER.merge_env_list(existing, CUTOVER.build_api_rollback_env_updates())
|
|
by_name = {e["name"]: e for e in rolled}
|
|
for key in (
|
|
"QUERY_REPLICA_REPORTS",
|
|
"QUERY_REPLICA_COUNTS",
|
|
"QUERY_REPLICA_POOLS",
|
|
"QUERY_SERVING_READS",
|
|
"QUERY_PUBLISHED_RESULTS",
|
|
):
|
|
self.assertEqual(by_name[key]["value"], "0")
|
|
|
|
|
|
|
|
class CutoverMutationProtectionTests(unittest.TestCase):
|
|
def test_patch_uses_uid_rv_and_stdin(self):
|
|
from unittest.mock import Mock
|
|
client = CUTOVER.ClusterClient("unused")
|
|
client.approved_plan = {"resourceUIDs": {"ns/deployment/app": "uid"}}
|
|
client.get_json = Mock(return_value={"metadata": {"uid": "uid", "resourceVersion": "10"}})
|
|
client.run = Mock()
|
|
client.patch_json_ops("ns", "deployment", "app", [{"op": "replace", "path": "/spec/replicas", "value": 0}])
|
|
argv = client.run.call_args.args[0]
|
|
ops = json.loads(client.run.call_args.kwargs["stdin_text"])
|
|
self.assertIn("--patch-file=/dev/stdin", argv)
|
|
self.assertEqual([o["path"] for o in ops[:2]], ["/metadata/uid", "/metadata/resourceVersion"])
|
|
self.assertEqual(ops[1]["value"], "10")
|
|
client.get_json.return_value["metadata"]["uid"] = "replaced"
|
|
with self.assertRaises(RuntimeError):
|
|
client.scale("ns", "deployment", "app", 1)
|
|
|
|
def test_original_read_version_is_preserved_in_patch(self):
|
|
dep = {"metadata": {"uid": "uid", "resourceVersion": "9"}}
|
|
ops = CUTOVER.build_deployment_json_patch(env_list=[], expected_deployment=dep)
|
|
self.assertEqual(ops[1], {"op": "test", "path": "/metadata/resourceVersion", "value": "9"})
|
|
|
|
def test_observer_not_disarmed_to_force_sync(self):
|
|
from unittest.mock import Mock, patch
|
|
client = Mock()
|
|
client.get_json.return_value = {"status": {"availableReplicas": 0}}
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(CUTOVER,"inspect_api_health",return_value={"readyReplicas":4,"desiredReplicas":4}):
|
|
with self.assertRaisesRegex(RuntimeError,"observer"):
|
|
CUTOVER.cmd_shadow_sync(SimpleNamespace(),client,Path(directory))
|
|
client.scale.assert_not_called()
|
|
|
|
def test_nonce_never_saved_in_snapshot_reports(self):
|
|
self.assertEqual(CUTOVER.redact_secrets({"args":["--resume-pause=private-token"]}),{"args":["--resume-pause=<redacted>"]})
|
|
|
|
class ScopedPublicationTests(unittest.TestCase):
|
|
def test_unrelated_or_mixed_heads_never_mean_scope_ready(self):
|
|
now=dt.datetime(2026,11,16,12,tzinfo=dt.timezone.utc)
|
|
base={"state":{"ready":True,"observedAt":now.isoformat()},"poolBucketsCount":1,"workforceFactsCount":1,"cohorts":[{"productionKey":"p","workforceKey":"w"}],"global":{"heads":{"p":{"generation":"p","chunks":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2},"w":{"generation":"workforce-facts/v1","factEpoch":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2}}}}
|
|
self.assertTrue(CUTOVER.evaluate_parity_snapshot(base,now=now)[0])
|
|
for key in ("asOf","sourceRevision","globalRevision"):
|
|
mixed=json.loads(json.dumps(base));mixed['global']['heads']['w'][key]=None
|
|
self.assertFalse(CUTOVER.evaluate_parity_snapshot(mixed,now=now)[0])
|
|
wrong=json.loads(json.dumps(base));wrong['cohorts'][0]['productionKey']='other-org'
|
|
self.assertFalse(CUTOVER.evaluate_parity_snapshot(wrong,now=now)[0])
|
|
|
|
class BoundedMetadataProbeTests(unittest.TestCase):
|
|
def test_transient_metadata_failure_retries_but_auth_failure_does_not(self):
|
|
from unittest.mock import Mock, patch
|
|
import subprocess
|
|
client=CUTOVER.ClusterClient('unused')
|
|
client.run=Mock(side_effect=[subprocess.TimeoutExpired('get',8),SimpleNamespace(returncode=1,stderr='temporary',stdout=''),SimpleNamespace(returncode=0,stdout='{"metadata":{"uid":"u"}}')])
|
|
with patch.object(CUTOVER.time,'sleep'):
|
|
self.assertEqual(client.get_json('ns','deployment','app')['metadata']['uid'],'u')
|
|
self.assertEqual(client.run.call_count,3)
|
|
client.run=Mock(return_value=SimpleNamespace(returncode=1,stderr='Forbidden',stdout=''))
|
|
with self.assertRaises(RuntimeError):client.get_json('ns','deployment','app')
|
|
self.assertEqual(client.run.call_count,1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|