droboticflow/scripts/local/test_query_replica_cluster_fixture.py
2026-11-13 22:31:44 -08:00

139 lines
8.8 KiB
Python

import importlib.util
from pathlib import Path
import unittest
import tempfile
import json
spec = importlib.util.spec_from_file_location("cluster_fixture", Path(__file__).with_name("query-replica-cluster-fixture.py"))
fixture = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fixture)
compare_spec = importlib.util.spec_from_file_location("isolated_compare", Path(__file__).with_name("query-replica-isolated-compare.py"))
compare = importlib.util.module_from_spec(compare_spec)
compare_spec.loader.exec_module(compare)
valkey_spec = importlib.util.spec_from_file_location("valkey_lab", Path(__file__).with_name("query-valkey-lab.py"))
valkey_lab = importlib.util.module_from_spec(valkey_spec)
valkey_spec.loader.exec_module(valkey_lab)
class FixtureSafety(unittest.TestCase):
def test_dedicated_valkey_is_disposable_bounded_and_network_isolated(self):
objects = valkey_lab.manifests()
self.assertEqual([o["kind"] for o in objects], ["Namespace", "ResourceQuota", "NetworkPolicy", "Pod"])
policy, pod = objects[-2:]
self.assertEqual(policy["spec"]["egress"], [])
source = policy["spec"]["ingress"][0]["from"][0]
self.assertEqual(source["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], "droboticflow-query-lab")
self.assertEqual(source["podSelector"]["matchLabels"]["app"], "node4-query-fixture")
self.assertEqual(pod["spec"]["nodeName"], "10.189.16.33")
self.assertFalse(pod["spec"]["automountServiceAccountToken"])
self.assertTrue(all("hostPath" not in v and "persistentVolumeClaim" not in v for v in pod["spec"]["volumes"]))
container = pod["spec"]["containers"][0]
self.assertEqual(container["resources"]["limits"], {"cpu": "2", "memory": "4Gi"})
self.assertIn("--maxmemory", container["command"])
self.assertIn("2gb", container["command"])
self.assertIn("--appendonly", container["command"])
self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"])
def test_rebuilt_query_uid_requires_exact_storage_and_resources(self):
import copy
uid = "e1210ce7-16ee-4bda-9f8f-e3a02efefa01"
pod = {"metadata": {"name": "node4-query25", "namespace": fixture.NS, "uid": uid, "labels": {"app": "node4-query25"}}, "status": {"phase": "Running"}, "spec": {"nodeName": fixture.NODE, "containers": [{"name": "probe", "image": fixture.IMAGE, "resources": {"requests": {"cpu": "1", "memory": "1Gi"}, "limits": {"cpu": "1", "memory": "1Gi"}}}], "volumes": [{"name": "lab", "hostPath": {"path": "/var/lib/mongo-cpu-node4-query25-20261113"}}, {"name": "scripts", "configMap": {"name": "node4-query25-scripts"}}]}}
pod["spec"]["initContainers"] = [copy.deepcopy(pod["spec"]["containers"][0])]
fixture.verify_query_pod(pod, uid)
admitted = copy.deepcopy(pod)
for kind in ("requests", "limits"):
admitted["spec"]["containers"][0]["resources"][kind]["cce.baidubce.com/ip"] = "1"
fixture.verify_query_pod(admitted, uid)
admitted["spec"]["containers"][0]["resources"]["limits"]["cce.baidubce.com/ip"] = "2"
with self.assertRaises(AssertionError):
fixture.verify_query_pod(admitted, uid)
with self.assertRaises(AssertionError):
fixture.verify_query_pod(pod, fixture.QUERY_UID)
for field in ("storage", "image", "resources", "node", "namespace"):
bad = copy.deepcopy(pod)
if field == "storage": bad["spec"]["volumes"][0]["hostPath"]["path"] = "/var/lib/production"
elif field == "image": bad["spec"]["containers"][0]["image"] = "mongo:latest"
elif field == "resources": bad["spec"]["containers"][0]["resources"]["limits"]["memory"] = "8Gi"
elif field == "node": bad["spec"]["nodeName"] = "other"
else: bad["metadata"]["namespace"] = "droboticflow-fanyu"
with self.assertRaises(AssertionError, msg=field):
fixture.verify_query_pod(bad, uid)
def test_confirmed_fuse_wait_with_no_block_io_is_not_disk_degradation(self):
record = {"protectedFullAvg10": 20.95, "protectedPressure": {"api": {"avg10": 20.95, "deltaUsec": 1000000, "blockDeltaBytes": 0}, "bosfs": {"avg10": 0, "deltaUsec": 0, "blockDeltaBytes": 0}}, "awaitMs": 0.5, "busyPct": 3}
self.assertFalse(compare.protected_pressure_degraded(record, 3.92, {"api"}))
self.assertTrue(compare.protected_pressure_degraded(record, 3.92, set()))
record["protectedPressure"]["api"]["blockDeltaBytes"] = 4096
self.assertTrue(compare.protected_pressure_degraded(record, 3.92, {"api"}))
def test_empty_or_mixed_stacks_never_exempt_local_waits(self):
self.assertEqual(compare.fuse_only_waits("PROTECTED api\n"), set())
fuse = "PROTECTED api\nWAIT 123 wait_on_page_bit_common\nfilemap_read\nfuse_file_read_iter\n"
self.assertEqual(compare.fuse_only_waits(fuse), {"api"})
self.assertEqual(compare.fuse_only_waits(fuse + "WAIT 124 io_schedule\next4_sync_file\n"), set())
def test_fuse_does_not_exempt_slow_disk_or_backend_wait(self):
record = {"protectedPressure": {"api": {"avg10": 20, "deltaUsec": 100, "blockDeltaBytes": 0}, "bosfs": {"avg10": 0, "deltaUsec": 1}}, "awaitMs": .5, "busyPct": 3}
self.assertTrue(compare.protected_pressure_degraded(record, .5, {"api"}))
record["protectedPressure"]["bosfs"]["deltaUsec"] = 0
record["awaitMs"] = 2
self.assertTrue(compare.protected_pressure_degraded(record, .5, {"api"}))
def test_baseline_spans_background_cycles_before_load(self):
now = [0]
def sleep(seconds):
now[0] += seconds
rows = compare.collect_baseline(lambda: {"time": now[0]}, clock=lambda: now[0], sleep=sleep)
self.assertGreaterEqual(rows[-1]["time"] - rows[0]["time"], 120)
self.assertGreaterEqual(len(rows), 25)
with self.assertRaises(AssertionError):
compare.collect_baseline(lambda: {}, seconds=1)
def test_missing_report_cannot_count_as_successful_zero_test_run(self):
with tempfile.TemporaryDirectory() as directory:
with self.assertRaises(FileNotFoundError):
compare.validate_comparison(Path(directory) / "missing.json", 100, 2, 81)
def test_incomplete_actor_coverage_cannot_be_accepted(self):
with tempfile.TemporaryDirectory() as directory:
report = {"tasks": 100, "parity": True, "annotationsAndVersionsPreserved": True, "errorFree": True, "httpSessionAuth": True, "queryServing": {"report_hits": 1}, "cases": []}
path = Path(directory) / "report.json"
path.write_text(json.dumps(report))
with self.assertRaisesRegex(AssertionError, "coverage"):
compare.validate_comparison(path, 100, 2, 81)
def test_psi_keeps_per_pod_counters_and_excludes_test_pressure(self):
host = "PROTECTED api-uid\nsome avg10=9.00 total=800\nfull avg10=6.75 avg60=1.68 avg300=0.61 total=17137676\n252:0 wbytes=1024\nQUERY cpu.stat\nfull avg10=99 total=999999999\n"
self.assertEqual(compare.pressure_counters(host), {"api-uid": {"avg10": 6.75, "totalUsec": 17137676, "blockBytes": 1024}})
def test_source_is_memory_only_and_unprivileged(self):
pod = fixture.manifest()[-1]["spec"]
self.assertEqual(pod["nodeName"], "10.189.16.4")
self.assertFalse(pod["automountServiceAccountToken"])
self.assertFalse(pod.get("hostNetwork", False))
self.assertEqual(pod["activeDeadlineSeconds"], 7200)
for volume in pod["volumes"]:
self.assertEqual(volume["emptyDir"]["medium"], "Memory")
self.assertIn("sizeLimit", volume["emptyDir"])
self.assertNotIn("hostPath", volume)
container = pod["containers"][0]
self.assertEqual(container["resources"]["limits"]["cpu"], "4")
self.assertEqual(container["resources"]["limits"]["memory"], "4Gi")
self.assertFalse(container["securityContext"]["allowPrivilegeEscalation"])
self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"])
self.assertEqual(container["imagePullPolicy"], "Never")
def test_network_is_only_fixture_to_query_private_port(self):
ingress, egress, _ = fixture.manifest()
self.assertEqual(ingress["spec"]["podSelector"]["matchLabels"], {"app": "node4-query25"})
self.assertEqual(ingress["spec"]["ingress"], [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}])
self.assertEqual(egress["spec"]["egress"], [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}])
self.assertEqual(fixture.HARD["services"], "0")
self.assertEqual(fixture.HARD["persistentvolumeclaims"], "0")
self.assertEqual(fixture.HARD["pods"], "2")
if __name__ == "__main__":
unittest.main()