droboticflow/scripts/local/query-replica-isolated-compare.py
2026-11-13 22:31:44 -08:00

383 lines
24 KiB
Python

#!/usr/bin/env python3
"""Run only synthetic fixtures; sample both DB budgets and protect existing IO."""
import argparse
import hashlib
import gzip
import json
import os
from pathlib import Path
import signal
import subprocess
import time
import urllib.request
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
import query_lab_profiles
NODE = "10.189.16.4"
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
NAMESPACE_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
CONTAINER = "fanyu-query-replica-local-source-1"
ROOT = Path(__file__).resolve().parents[2]
TEST_NAME = "TestQueryReplicaIsolatedHTTPComparisonLocal"
def run(args, timeout=20):
return subprocess.check_output(args, text=True, timeout=timeout).strip()
def source_fingerprint():
fingerprint = hashlib.sha256()
files = run(["git", "-C", str(ROOT), "ls-files", "-c", "-o", "--exclude-standard"]).splitlines()
for name in sorted(set(files)):
if name.startswith(("internal/", "cmd/", "scripts/")) or name in ("go.mod", "go.sum", "package-lock.json"):
path = ROOT / name
if path.is_file():
fingerprint.update(name.encode() + b"\0" + path.read_bytes())
return fingerprint.hexdigest()
def cgroup(pod):
uid = pod["metadata"]["uid"].replace("-", "_")
base = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/"
qos = pod["status"]["qosClass"]
if qos == "Guaranteed":
return base + "kubepods-pod" + uid + ".slice"
if qos not in ("Burstable", "BestEffort"):
raise RuntimeError("unknown cgroup layout")
part = "kubepods-" + qos.lower()
return base + part + ".slice/" + part + "-pod" + uid + ".slice"
def pressure_counters(host):
result = {}
uid = None
for line in host.splitlines():
if line.startswith("PROTECTED "):
uid = line.split()[1]
elif line.startswith(("QUERY ", "SOURCE ")):
uid = None
elif uid and line.startswith("full "):
fields = dict(part.split("=", 1) for part in line.split()[1:])
result[uid] = {"avg10": float(fields["avg10"]), "totalUsec": int(fields["total"]), "blockBytes": 0}
elif uid and line[:1].isdigit() and uid in result:
fields = dict(part.split("=", 1) for part in line.split()[1:])
result[uid]["blockBytes"] += int(fields.get("rbytes", 0)) + int(fields.get("wbytes", 0))
return result
def protected_pressure_degraded(record, limit, fuse_only):
local_quiet = record["awaitMs"] <= 1 and record["busyPct"] <= 5
for uid, values in record["protectedPressure"].items():
if values["avg10"] <= limit or values.get("deltaUsec") == 0:
continue
unrelated_fuse = (
uid in fuse_only and local_quiet and values.get("blockDeltaBytes") == 0
and all(other == uid or v.get("deltaUsec") == 0 for other, v in record["protectedPressure"].items())
)
if not unrelated_fuse:
return True
return False
def fuse_only_waits(stacks):
groups = {}
uid = None
for line in stacks.splitlines():
if line.startswith("PROTECTED "):
uid = line.split()[1]
groups[uid] = []
elif uid and line.startswith("WAIT "):
groups[uid].append([])
elif uid and groups[uid]:
groups[uid][-1].append(line)
result = set()
for uid, waits in groups.items():
if waits and all(any("fuse_" in frame for frame in frames) and not any(token in frame for frame in frames for token in ("ext4_", "jbd2_", "submit_bio", "balance_dirty")) for frames in waits):
result.add(uid)
return result
def validate_comparison(report_path, tasks, waves, max_actors):
report = json.loads(report_path.read_text())
assert report["tasks"] == tasks and report["parity"] is True
assert report["annotationsAndVersionsPreserved"] is True and report["errorFree"] is True
assert report["httpSessionAuth"] is True and report["queryServing"]["report_hits"] > 0
expected = {f"{mode}-{actors}": actors * waves for mode in ("old", "query") for actors in (81, 200, 500) if actors <= max_actors}
actual = {case["name"]: case["requests"] for case in report["cases"]}
assert len(report["cases"]) == len(expected) and actual == expected, "incomplete comparison coverage"
for case in report["cases"]:
assert not case["errors"] and len(case["routes"]) == 4
return report
def collect_baseline(sample, seconds=120, clock=time.monotonic, sleep=time.sleep):
# A few back-to-back requests miss normal FUSE/background cycles. Observe
# the idle node before starting synthetic load; do not alter stop margins.
assert 120 <= seconds <= 300
rows = [sample()]
started = clock()
while clock() - started < seconds:
sleep(5)
rows.append(sample())
return rows
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--kubeconfig", required=True)
ap.add_argument("--output", required=True)
ap.add_argument("--tasks", type=int, default=26218, choices=range(20, 30001))
ap.add_argument("--waves", type=int, default=2, choices=range(1, 101))
ap.add_argument("--max-actors", type=int, default=81, choices=(81, 200, 500))
ap.add_argument("--fixture-plan", help="owned.json from the bounded in-cluster fixture")
ap.add_argument("--cold-write-probe", action="store_true", help="bounded 8MiB synthetic diagnostic; never HTTP capacity evidence")
ap.add_argument("--periodic-refresh", action="store_true", help="explicit periodic display policy; same load and safety limits")
ap.add_argument("--cold-pace-kib", type=int, choices=(0, 128), default=0)
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
ap.add_argument("--published-results", action="store_true")
args = ap.parse_args()
assert args.cold_write_probe or args.cold_pace_kib == 0
assert not args.cold_write_probe or args.fixture_plan, "cold diagnostic requires the owned in-cluster fixture"
test_name = "TestQueryReplicaColdWriteProbeLocal" if args.cold_write_probe else TEST_NAME
out = Path(args.output).resolve()
out.mkdir(parents=True, exist_ok=False)
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
def get(*cmd):
return json.loads(run(k + list(cmd) + ["-o", "json"]))
node = get("get", "node", NODE)
assert node["metadata"]["uid"] == NODE_UID
ns = get("get", "namespace", "droboticflow-query-lab")
assert ns["metadata"]["uid"] == NAMESPACE_UID
query = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query25")
assert query["status"]["phase"] == "Running"
assert query["spec"]["nodeName"] == NODE
query_lab_profiles.verify_resources(query, args.query_profile)
query_scripts = get("-n", "droboticflow-query-lab", "get", "configmap", "node4-query25-scripts")["data"]
query_lab_profiles.verify_scripts(query_scripts, args.query_profile)
run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query25", "-c", "probe", "--", "bash", "/scripts/verify25.sh"])
fixture = None
owned = None
source_node = NODE
cache_owned = None
if args.fixture_plan:
owned = json.loads(Path(args.fixture_plan).read_text())
source_node = owned.get("sourceNode",NODE)
assert source_node in (NODE,"10.189.16.33")
cache_owned = owned.get("valkey")
fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
assert fixture["metadata"]["uid"] == owned["fixtureUID"]
assert query["metadata"]["uid"] == owned["queryUID"]
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == source_node
budget = fixture["spec"]["containers"][0]["resources"]["limits"]
assert budget["cpu"] == "4" and budget["memory"] == "4Gi"
api_binary = owned.get("apiBinaryPath", "/synthetic/query-http.test")
assert api_binary.startswith("/synthetic/query-http") and api_binary.endswith(".test") and "/../" not in api_binary
actual = run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "sha256sum", api_binary]).split()[0]
assert actual == owned["binarySHA256"]["api"]
listed = run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", api_binary, "-test.list=^" + test_name + "$"])
assert listed.splitlines() == [test_name], "stale binary or missing comparison test; no load started"
source = {"Id": fixture["metadata"]["uid"], "Image": fixture["status"]["containerStatuses"][0]["imageID"]}
else:
source = json.loads(run(["docker", "inspect", CONTAINER]))[0]
assert source["State"]["Running"]
assert source["HostConfig"]["NanoCpus"] == 4_000_000_000
assert source["HostConfig"]["Memory"] == 4_294_967_296
socket = run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"])
assert socket.startswith("unix://"), "source must be a local Docker daemon"
protected = [p for p in get("get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE)["items"] if p["metadata"]["namespace"] != "droboticflow-query-lab"]
prod = get("-n", "droboticflow-fanyu", "get", "pods")
def pod_states(items):
return {p["metadata"]["uid"]: [s.get("restartCount", 0) for s in p["status"].get("containerStatuses", [])] for p in items}
states = pod_states(prod["items"])
# All interpolated paths are generated from validated Kubernetes UIDs/QoS,
# never arbitrary paths, credentials or user-provided command fragments.
script = "set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; "
for p in protected:
script += "echo PROTECTED " + p["metadata"]["uid"] + "; cat " + cgroup(p) + "/io.pressure; cat " + cgroup(p) + "/io.stat; "
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat", "io.max"):
script += "echo QUERY " + filename + "; cat " + cgroup(query) + "/" + filename + "; "
if fixture and source_node == NODE:
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat"):
script += "echo SOURCE " + filename + "; cat " + cgroup(fixture) + "/" + filename + "; "
previous = None
def sample():
nonlocal previous
started = time.monotonic()
with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as response:
assert response.status == 200
response.read(65536)
ready = time.monotonic() - started
current = get("-n", "droboticflow-fanyu", "get", "pods")
assert pod_states(current["items"]) == states, "existing production Pod/restart set changed"
health = get("get", "node", NODE)
assert all(c["status"] == "False" for c in health["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
host = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", script])
if fixture and source_node != NODE:
local = run(k+["-n","droboticflow-query-lab","exec","node4-query-fixture","-c","lab","--","sh","-c","cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/io.stat"])
health_other=get("get","node",source_node)
assert health_other["metadata"]["uid"]=="77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
assert all(c["status"]=="False" for c in health_other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
other_top=run(k+["top","node",source_node,"--no-headers"]).split()
assert float(other_top[2].rstrip("%"))<=60 and float(other_top[4].rstrip("%"))<=70, "source/cache node overloaded"
else:
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
if cache_owned:
cp=get("-n",cache_owned["namespace"],"get","pod",cache_owned["pod"])
assert cp["metadata"]["uid"]==cache_owned["podUID"] and cp["status"]["phase"]=="Running", "owned cache replaced/stopped"
top = run(k + ["top", "node", NODE, "--no-headers"]).split()
cpu_pct = float(top[2].rstrip("%"))
if fixture:
current_fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
assert current_fixture["metadata"]["uid"] == owned["fixtureUID"] and current_fixture["status"]["phase"] == "Running", "owned source fixture stopped or was replaced"
available = next(line.split() for line in host.splitlines() if line.startswith("AVAILABLE "))
assert int(available[1]) * int(available[2]) >= 80_530_636_800, "node free space below 75Gi"
pressure = []
inside = False
for line in host.splitlines():
if line.startswith("PROTECTED "):
inside = True
elif line.startswith(("QUERY ", "SOURCE ")):
inside = False
elif inside and line.startswith("full "):
pressure.append(float(line.split("avg10=")[1].split()[0]))
assert len(pressure) == len(protected)
disk = [int(v) for v in next(line for line in host.splitlines() if line.split()[:3] == ["252", "0", "vda"]).split()[3:]]
record = {"time": time.time(), "readySeconds": ready, "sourceCgroup": local, "host": host, "nodeCPUPct": cpu_pct, "protectedFullAvg10": max(pressure, default=0), "protectedPressure": pressure_counters(host), "disk": disk, "awaitMs": 0, "busyPct": 0}
if previous:
for uid, values in record["protectedPressure"].items():
before = previous["protectedPressure"][uid]["totalUsec"]
assert values["totalUsec"] >= before, "protected IO counter reset"
values["deltaUsec"] = values["totalUsec"] - before
values["windowPct"] = values["deltaUsec"] / max(record["time"] - previous["time"], 0.001) / 10000
values["blockDeltaBytes"] = values["blockBytes"] - previous["protectedPressure"][uid]["blockBytes"]
ops = disk[0] + disk[4] - previous["disk"][0] - previous["disk"][4]
record["awaitMs"] = (disk[3] + disk[7] - previous["disk"][3] - previous["disk"][7]) / max(ops, 1)
record["busyPct"] = (disk[9] - previous["disk"][9]) / max(record["time"] - previous["time"], 0.001) / 10
previous = record
with (out / "resources.jsonl").open("a") as f:
f.write(json.dumps(record) + "\n")
return record
baseline = collect_baseline(sample)
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": int(query_lab_profiles.profile(args.query_profile)["cpu"]), "queryMemory": query_lab_profiles.resources(args.query_profile)["memory"], "queryProfile": query_lab_profiles.profile(args.query_profile), "queryScriptSHA256": hashlib.sha256(json.dumps(query_scripts, sort_keys=True).encode()).hexdigest(), "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target loop and backing device use the recorded finite queryProfile IO caps", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
manifest["sourceNode"] = source_node
manifest["publishedResults"] = args.published_results
manifest["valkeyOwned"] = cache_owned
manifest["baselineSamples"] = len(baseline)
manifest["refreshMode"] = "periodic" if args.periodic_refresh else "continuous"
manifest["baselineSeconds"] = baseline[-1]["time"] - baseline[0]["time"]
manifest["diagnostic"] = {"kind": "cold-write-probe", "payloadBytes": 8 << 20, "paceKiB": args.cold_pace_kib} if args.cold_write_probe else None
if fixture:
manifest["limitations"][0] = "source Mongo and native HTTP/projector/builder/client share one fixed 4CPU/4Gi Pod; source storage is bounded tmpfs"
manifest["sourceCPUCounter"] = "mongod process utime+stime; whole fixture Pod cgroup sampled separately"
manifest["binarySHA256"] = owned["binarySHA256"]
(out / "manifest.json").write_text(json.dumps(manifest, indent=2))
env = dict(os.environ, QUERY_REPLICA_ISOLATED_E2E="1", QUERY_REPLICA_CLUSTER_TEST="1", QUERY_REPLICA_TEST_TARGET_URI="mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", QUERY_COMPARE_TASKS=str(args.tasks), QUERY_COMPARE_WAVES=str(args.waves), QUERY_COMPARE_MAX_ACTORS=str(args.max_actors), QUERY_COMPARE_REPORT=str(out / "comparison.json"))
env["QUERY_REPLICA_REFRESH_MODE"] = manifest["refreshMode"]
argv = ["go", "test", "./internal/api", "-run", "^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-count=1", "-timeout=45m", "-v"]
remote_report = "/synthetic/query-comparison-" + str(int(time.time())) + ".json"
if fixture:
settings = ["TMPDIR=/synthetic/tmp", "QUERY_REPLICA_ISOLATED_E2E=1", "QUERY_REPLICA_CLUSTER_TEST=1", "QUERY_REPLICA_TEST_TARGET_URI=mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "QUERY_COMPARE_IN_CLUSTER=1", "QUERY_COMPARE_TASKS=" + str(args.tasks), "QUERY_COMPARE_WAVES=" + str(args.waves), "QUERY_COMPARE_MAX_ACTORS=" + str(args.max_actors), "QUERY_COMPARE_REPORT=" + remote_report, "QUERY_COMPARE_WIRE_COMPRESSION=" + os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "QUERY_COMPARE_MIN_POOL=" + os.getenv("QUERY_COMPARE_MIN_POOL", "0")]
if args.cold_write_probe:
settings += ["QUERY_REPLICA_COLD_WRITE_PROBE=1", "QUERY_COLD_PACE_KIB=" + str(args.cold_pace_kib)]
settings += ["QUERY_REPLICA_REFRESH_MODE=" + manifest["refreshMode"]]
if args.published_results:
assert cache_owned, "published cluster comparison requires its owned dedicated cache"
settings += ["QUERY_PUBLISHED_RESULTS=1","QUERY_COMPARE_VALKEY_URL="+cache_owned["url"]]
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^" + test_name + "$", "-test.count=1", "-test.timeout=45m", "-test.v"]
with (out / "test.log").open("w") as log:
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
bad = 0
try:
while child.poll() is None:
record = sample()
stacks = ""
if record["protectedFullAvg10"] > limits["pressure"]:
probe = ""
for p in protected:
uid = p["metadata"]["uid"]
if record["protectedPressure"][uid]["avg10"] <= limits["pressure"]:
continue
probe += "echo PROTECTED " + uid + "; for f in $(find " + cgroup(p) + " -name cgroup.procs); do for pid in $(cat $f); do for t in /host/proc/$pid/task/*; do if grep -q 'State:.*D' $t/status 2>/dev/null; then read w < $t/wchan || true; echo WAIT $pid $w; cat $t/stack 2>/dev/null || true; fi; done; done; done; "
try:
stacks = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", probe or "true"])
except Exception as error:
stacks = "read-only attribution unavailable: " + str(error)
with (out / "pressure-attribution.jsonl").open("a") as attribution:
attribution.write(json.dumps({"time": record["time"], "stacks": stacks, "fuseOnly": sorted(fuse_only_waits(stacks)), "pressure": record["protectedPressure"], "awaitMs": record["awaitMs"], "busyPct": record["busyPct"]}) + "\n")
if not (out / "first-degradation-stacks.log").exists():
(out / "first-degradation-stacks.log").write_text(stacks[:32768])
degraded = record["readySeconds"] > limits["ready"] or protected_pressure_degraded(record, limits["pressure"], fuse_only_waits(stacks)) or record["awaitMs"] > limits["await"] or record["busyPct"] > 70 or record["nodeCPUPct"] > 75
bad = bad + 1 if degraded else 0
if bad >= 3:
raise RuntimeError("existing workload health/IO degraded for three samples")
time.sleep(5)
except BaseException as exc:
stopped = {"reason": str(exc), "syntheticDatabasesMayNeedOwnedCleanup": True, "cleanupErrors": []}
# Record the original stop even if the API or the child disappears
# during cleanup. Never delete a replacement Pod by name alone.
(out / "stopped.json").write_text(json.dumps(stopped))
try:
if fixture:
current = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
if current["metadata"]["uid"] == owned["fixtureUID"]:
delete = out / "delete-owned-fixture.json"
delete.write_text(json.dumps({"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": owned["fixtureUID"]}}))
run(k + ["delete", "--raw", "/api/v1/namespaces/droboticflow-query-lab/pods/node4-query-fixture", "-f", str(delete)])
except Exception as cleanup_error:
stopped["cleanupErrors"].append(str(cleanup_error))
finally:
if child.poll() is None:
try:
os.killpg(child.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
child.wait(timeout=30)
except subprocess.TimeoutExpired:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
child.wait(timeout=5)
(out / "stopped.json").write_text(json.dumps(stopped))
raise
final_health_error = None
try:
sample()
except Exception as error:
final_health_error = str(error)
if fixture:
result = subprocess.run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "gzip", "-c", remote_report], capture_output=True, timeout=30)
if result.returncode == 0:
(out / "comparison.json").write_bytes(gzip.decompress(result.stdout))
changed = source_fingerprint() != manifest["sourceFingerprint"]
exit_record = {"returncode": child.returncode, "sourceChanged": changed, "coverageComplete": False, "finalHealthError": final_health_error}
try:
if args.cold_write_probe:
probe = json.loads((out / "comparison.json").read_text())
assert probe["kind"] == "cold-write-probe" and probe["capacityAccepted"] is False
assert probe["payloadBytes"] == 8 << 20 and probe["completedBatches"] == 32
assert probe["majority"] is True and probe["journal"] is True and probe["heartbeatTimeoutMs"] == 3000
assert probe["paceKiB"] == args.cold_pace_kib
assert len(probe["samples"]) >= 34
else:
validate_comparison(out / "comparison.json", args.tasks, args.waves, args.max_actors)
exit_record["coverageComplete"] = True
except (OSError, ValueError, KeyError, AssertionError) as error:
exit_record["coverageError"] = str(error)
(out / "exit.json").write_text(json.dumps(exit_record))
raise SystemExit(child.returncode or (2 if changed or final_health_error or not exit_record["coverageComplete"] else 0))
if __name__ == "__main__":
main()