forked from longtao.wu/droboticflow
383 lines
24 KiB
Python
383 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Run only synthetic fixtures; sample both DB budgets and protect existing IO."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import gzip
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
import urllib.request
|
|
import sys
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import query_lab_profiles
|
|
|
|
NODE = "10.189.16.4"
|
|
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
|
|
NAMESPACE_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
|
|
CONTAINER = "fanyu-query-replica-local-source-1"
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
TEST_NAME = "TestQueryReplicaIsolatedHTTPComparisonLocal"
|
|
|
|
|
|
def run(args, timeout=20):
|
|
return subprocess.check_output(args, text=True, timeout=timeout).strip()
|
|
|
|
|
|
def source_fingerprint():
|
|
fingerprint = hashlib.sha256()
|
|
files = run(["git", "-C", str(ROOT), "ls-files", "-c", "-o", "--exclude-standard"]).splitlines()
|
|
for name in sorted(set(files)):
|
|
if name.startswith(("internal/", "cmd/", "scripts/")) or name in ("go.mod", "go.sum", "package-lock.json"):
|
|
path = ROOT / name
|
|
if path.is_file():
|
|
fingerprint.update(name.encode() + b"\0" + path.read_bytes())
|
|
return fingerprint.hexdigest()
|
|
|
|
|
|
def cgroup(pod):
|
|
uid = pod["metadata"]["uid"].replace("-", "_")
|
|
base = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/"
|
|
qos = pod["status"]["qosClass"]
|
|
if qos == "Guaranteed":
|
|
return base + "kubepods-pod" + uid + ".slice"
|
|
if qos not in ("Burstable", "BestEffort"):
|
|
raise RuntimeError("unknown cgroup layout")
|
|
part = "kubepods-" + qos.lower()
|
|
return base + part + ".slice/" + part + "-pod" + uid + ".slice"
|
|
|
|
|
|
def pressure_counters(host):
|
|
result = {}
|
|
uid = None
|
|
for line in host.splitlines():
|
|
if line.startswith("PROTECTED "):
|
|
uid = line.split()[1]
|
|
elif line.startswith(("QUERY ", "SOURCE ")):
|
|
uid = None
|
|
elif uid and line.startswith("full "):
|
|
fields = dict(part.split("=", 1) for part in line.split()[1:])
|
|
result[uid] = {"avg10": float(fields["avg10"]), "totalUsec": int(fields["total"]), "blockBytes": 0}
|
|
elif uid and line[:1].isdigit() and uid in result:
|
|
fields = dict(part.split("=", 1) for part in line.split()[1:])
|
|
result[uid]["blockBytes"] += int(fields.get("rbytes", 0)) + int(fields.get("wbytes", 0))
|
|
return result
|
|
|
|
|
|
def protected_pressure_degraded(record, limit, fuse_only):
|
|
local_quiet = record["awaitMs"] <= 1 and record["busyPct"] <= 5
|
|
for uid, values in record["protectedPressure"].items():
|
|
if values["avg10"] <= limit or values.get("deltaUsec") == 0:
|
|
continue
|
|
unrelated_fuse = (
|
|
uid in fuse_only and local_quiet and values.get("blockDeltaBytes") == 0
|
|
and all(other == uid or v.get("deltaUsec") == 0 for other, v in record["protectedPressure"].items())
|
|
)
|
|
if not unrelated_fuse:
|
|
return True
|
|
return False
|
|
|
|
|
|
def fuse_only_waits(stacks):
|
|
groups = {}
|
|
uid = None
|
|
for line in stacks.splitlines():
|
|
if line.startswith("PROTECTED "):
|
|
uid = line.split()[1]
|
|
groups[uid] = []
|
|
elif uid and line.startswith("WAIT "):
|
|
groups[uid].append([])
|
|
elif uid and groups[uid]:
|
|
groups[uid][-1].append(line)
|
|
result = set()
|
|
for uid, waits in groups.items():
|
|
if waits and all(any("fuse_" in frame for frame in frames) and not any(token in frame for frame in frames for token in ("ext4_", "jbd2_", "submit_bio", "balance_dirty")) for frames in waits):
|
|
result.add(uid)
|
|
return result
|
|
|
|
|
|
def validate_comparison(report_path, tasks, waves, max_actors):
|
|
report = json.loads(report_path.read_text())
|
|
assert report["tasks"] == tasks and report["parity"] is True
|
|
assert report["annotationsAndVersionsPreserved"] is True and report["errorFree"] is True
|
|
assert report["httpSessionAuth"] is True and report["queryServing"]["report_hits"] > 0
|
|
expected = {f"{mode}-{actors}": actors * waves for mode in ("old", "query") for actors in (81, 200, 500) if actors <= max_actors}
|
|
actual = {case["name"]: case["requests"] for case in report["cases"]}
|
|
assert len(report["cases"]) == len(expected) and actual == expected, "incomplete comparison coverage"
|
|
for case in report["cases"]:
|
|
assert not case["errors"] and len(case["routes"]) == 4
|
|
return report
|
|
|
|
|
|
def collect_baseline(sample, seconds=120, clock=time.monotonic, sleep=time.sleep):
|
|
# A few back-to-back requests miss normal FUSE/background cycles. Observe
|
|
# the idle node before starting synthetic load; do not alter stop margins.
|
|
assert 120 <= seconds <= 300
|
|
rows = [sample()]
|
|
started = clock()
|
|
while clock() - started < seconds:
|
|
sleep(5)
|
|
rows.append(sample())
|
|
return rows
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--kubeconfig", required=True)
|
|
ap.add_argument("--output", required=True)
|
|
ap.add_argument("--tasks", type=int, default=26218, choices=range(20, 30001))
|
|
ap.add_argument("--waves", type=int, default=2, choices=range(1, 101))
|
|
ap.add_argument("--max-actors", type=int, default=81, choices=(81, 200, 500))
|
|
ap.add_argument("--fixture-plan", help="owned.json from the bounded in-cluster fixture")
|
|
ap.add_argument("--cold-write-probe", action="store_true", help="bounded 8MiB synthetic diagnostic; never HTTP capacity evidence")
|
|
ap.add_argument("--periodic-refresh", action="store_true", help="explicit periodic display policy; same load and safety limits")
|
|
ap.add_argument("--cold-pace-kib", type=int, choices=(0, 128), default=0)
|
|
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
|
|
ap.add_argument("--published-results", action="store_true")
|
|
args = ap.parse_args()
|
|
assert args.cold_write_probe or args.cold_pace_kib == 0
|
|
assert not args.cold_write_probe or args.fixture_plan, "cold diagnostic requires the owned in-cluster fixture"
|
|
test_name = "TestQueryReplicaColdWriteProbeLocal" if args.cold_write_probe else TEST_NAME
|
|
out = Path(args.output).resolve()
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
|
|
|
|
def get(*cmd):
|
|
return json.loads(run(k + list(cmd) + ["-o", "json"]))
|
|
|
|
node = get("get", "node", NODE)
|
|
assert node["metadata"]["uid"] == NODE_UID
|
|
ns = get("get", "namespace", "droboticflow-query-lab")
|
|
assert ns["metadata"]["uid"] == NAMESPACE_UID
|
|
query = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query25")
|
|
assert query["status"]["phase"] == "Running"
|
|
assert query["spec"]["nodeName"] == NODE
|
|
query_lab_profiles.verify_resources(query, args.query_profile)
|
|
query_scripts = get("-n", "droboticflow-query-lab", "get", "configmap", "node4-query25-scripts")["data"]
|
|
query_lab_profiles.verify_scripts(query_scripts, args.query_profile)
|
|
run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query25", "-c", "probe", "--", "bash", "/scripts/verify25.sh"])
|
|
fixture = None
|
|
owned = None
|
|
source_node = NODE
|
|
cache_owned = None
|
|
if args.fixture_plan:
|
|
owned = json.loads(Path(args.fixture_plan).read_text())
|
|
source_node = owned.get("sourceNode",NODE)
|
|
assert source_node in (NODE,"10.189.16.33")
|
|
cache_owned = owned.get("valkey")
|
|
fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
|
assert fixture["metadata"]["uid"] == owned["fixtureUID"]
|
|
assert query["metadata"]["uid"] == owned["queryUID"]
|
|
assert fixture["status"]["phase"] == "Running" and fixture["spec"]["nodeName"] == source_node
|
|
budget = fixture["spec"]["containers"][0]["resources"]["limits"]
|
|
assert budget["cpu"] == "4" and budget["memory"] == "4Gi"
|
|
api_binary = owned.get("apiBinaryPath", "/synthetic/query-http.test")
|
|
assert api_binary.startswith("/synthetic/query-http") and api_binary.endswith(".test") and "/../" not in api_binary
|
|
actual = run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "sha256sum", api_binary]).split()[0]
|
|
assert actual == owned["binarySHA256"]["api"]
|
|
listed = run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", api_binary, "-test.list=^" + test_name + "$"])
|
|
assert listed.splitlines() == [test_name], "stale binary or missing comparison test; no load started"
|
|
source = {"Id": fixture["metadata"]["uid"], "Image": fixture["status"]["containerStatuses"][0]["imageID"]}
|
|
else:
|
|
source = json.loads(run(["docker", "inspect", CONTAINER]))[0]
|
|
assert source["State"]["Running"]
|
|
assert source["HostConfig"]["NanoCpus"] == 4_000_000_000
|
|
assert source["HostConfig"]["Memory"] == 4_294_967_296
|
|
socket = run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"])
|
|
assert socket.startswith("unix://"), "source must be a local Docker daemon"
|
|
protected = [p for p in get("get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE)["items"] if p["metadata"]["namespace"] != "droboticflow-query-lab"]
|
|
prod = get("-n", "droboticflow-fanyu", "get", "pods")
|
|
|
|
def pod_states(items):
|
|
return {p["metadata"]["uid"]: [s.get("restartCount", 0) for s in p["status"].get("containerStatuses", [])] for p in items}
|
|
|
|
states = pod_states(prod["items"])
|
|
# All interpolated paths are generated from validated Kubernetes UIDs/QoS,
|
|
# never arbitrary paths, credentials or user-provided command fragments.
|
|
script = "set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; "
|
|
for p in protected:
|
|
script += "echo PROTECTED " + p["metadata"]["uid"] + "; cat " + cgroup(p) + "/io.pressure; cat " + cgroup(p) + "/io.stat; "
|
|
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat", "io.max"):
|
|
script += "echo QUERY " + filename + "; cat " + cgroup(query) + "/" + filename + "; "
|
|
if fixture and source_node == NODE:
|
|
for filename in ("cpu.stat", "memory.current", "memory.events", "io.stat"):
|
|
script += "echo SOURCE " + filename + "; cat " + cgroup(fixture) + "/" + filename + "; "
|
|
previous = None
|
|
|
|
def sample():
|
|
nonlocal previous
|
|
started = time.monotonic()
|
|
with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as response:
|
|
assert response.status == 200
|
|
response.read(65536)
|
|
ready = time.monotonic() - started
|
|
current = get("-n", "droboticflow-fanyu", "get", "pods")
|
|
assert pod_states(current["items"]) == states, "existing production Pod/restart set changed"
|
|
health = get("get", "node", NODE)
|
|
assert all(c["status"] == "False" for c in health["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
|
|
host = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", script])
|
|
if fixture and source_node != NODE:
|
|
local = run(k+["-n","droboticflow-query-lab","exec","node4-query-fixture","-c","lab","--","sh","-c","cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/io.stat"])
|
|
health_other=get("get","node",source_node)
|
|
assert health_other["metadata"]["uid"]=="77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
|
|
assert all(c["status"]=="False" for c in health_other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
|
|
other_top=run(k+["top","node",source_node,"--no-headers"]).split()
|
|
assert float(other_top[2].rstrip("%"))<=60 and float(other_top[4].rstrip("%"))<=70, "source/cache node overloaded"
|
|
else:
|
|
local = "included in host SOURCE fields" if fixture else run(["docker", "exec", CONTAINER, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/io.stat"])
|
|
if cache_owned:
|
|
cp=get("-n",cache_owned["namespace"],"get","pod",cache_owned["pod"])
|
|
assert cp["metadata"]["uid"]==cache_owned["podUID"] and cp["status"]["phase"]=="Running", "owned cache replaced/stopped"
|
|
top = run(k + ["top", "node", NODE, "--no-headers"]).split()
|
|
cpu_pct = float(top[2].rstrip("%"))
|
|
if fixture:
|
|
current_fixture = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
|
assert current_fixture["metadata"]["uid"] == owned["fixtureUID"] and current_fixture["status"]["phase"] == "Running", "owned source fixture stopped or was replaced"
|
|
available = next(line.split() for line in host.splitlines() if line.startswith("AVAILABLE "))
|
|
assert int(available[1]) * int(available[2]) >= 80_530_636_800, "node free space below 75Gi"
|
|
pressure = []
|
|
inside = False
|
|
for line in host.splitlines():
|
|
if line.startswith("PROTECTED "):
|
|
inside = True
|
|
elif line.startswith(("QUERY ", "SOURCE ")):
|
|
inside = False
|
|
elif inside and line.startswith("full "):
|
|
pressure.append(float(line.split("avg10=")[1].split()[0]))
|
|
assert len(pressure) == len(protected)
|
|
disk = [int(v) for v in next(line for line in host.splitlines() if line.split()[:3] == ["252", "0", "vda"]).split()[3:]]
|
|
record = {"time": time.time(), "readySeconds": ready, "sourceCgroup": local, "host": host, "nodeCPUPct": cpu_pct, "protectedFullAvg10": max(pressure, default=0), "protectedPressure": pressure_counters(host), "disk": disk, "awaitMs": 0, "busyPct": 0}
|
|
if previous:
|
|
for uid, values in record["protectedPressure"].items():
|
|
before = previous["protectedPressure"][uid]["totalUsec"]
|
|
assert values["totalUsec"] >= before, "protected IO counter reset"
|
|
values["deltaUsec"] = values["totalUsec"] - before
|
|
values["windowPct"] = values["deltaUsec"] / max(record["time"] - previous["time"], 0.001) / 10000
|
|
values["blockDeltaBytes"] = values["blockBytes"] - previous["protectedPressure"][uid]["blockBytes"]
|
|
ops = disk[0] + disk[4] - previous["disk"][0] - previous["disk"][4]
|
|
record["awaitMs"] = (disk[3] + disk[7] - previous["disk"][3] - previous["disk"][7]) / max(ops, 1)
|
|
record["busyPct"] = (disk[9] - previous["disk"][9]) / max(record["time"] - previous["time"], 0.001) / 10
|
|
previous = record
|
|
with (out / "resources.jsonl").open("a") as f:
|
|
f.write(json.dumps(record) + "\n")
|
|
return record
|
|
|
|
baseline = collect_baseline(sample)
|
|
limits = {"ready": max(1, max(s["readySeconds"] for s in baseline) * 2), "pressure": max(0.5, max(s["protectedFullAvg10"] for s in baseline) + 0.5), "await": max(10, max(s["awaitMs"] for s in baseline) * 3)}
|
|
manifest = {"sourceCommit": run(["git", "-C", str(ROOT), "rev-parse", "HEAD"]), "sourceFingerprint": source_fingerprint(), "sourceContainer": source["Id"], "sourceImage": source["Image"], "sourceCPU": 4, "sourceMemory": 4_294_967_296, "queryPodUID": query["metadata"]["uid"], "queryImage": query["spec"]["containers"][0]["image"], "queryCPU": int(query_lab_profiles.profile(args.query_profile)["cpu"]), "queryMemory": query_lab_profiles.resources(args.query_profile)["memory"], "queryProfile": query_lab_profiles.profile(args.query_profile), "queryScriptSHA256": hashlib.sha256(json.dumps(query_scripts, sort_keys=True).encode()).hexdigest(), "tasks": args.tasks, "waves": args.waves, "maxActors": args.max_actors, "wireCompressionBothVariants": os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "minimumPoolBothVariants": os.getenv("QUERY_COMPARE_MIN_POOL", "0"), "limits": limits, "limitations": ["source Docker VM shares CPU with existing local containers", "target loop and backing device use the recorded finite queryProfile IO caps", "short comparison; not 30-minute three-round production capacity acceptance", "health probes do not establish customer workflow latency"]}
|
|
manifest["sourceNode"] = source_node
|
|
manifest["publishedResults"] = args.published_results
|
|
manifest["valkeyOwned"] = cache_owned
|
|
manifest["baselineSamples"] = len(baseline)
|
|
manifest["refreshMode"] = "periodic" if args.periodic_refresh else "continuous"
|
|
manifest["baselineSeconds"] = baseline[-1]["time"] - baseline[0]["time"]
|
|
manifest["diagnostic"] = {"kind": "cold-write-probe", "payloadBytes": 8 << 20, "paceKiB": args.cold_pace_kib} if args.cold_write_probe else None
|
|
if fixture:
|
|
manifest["limitations"][0] = "source Mongo and native HTTP/projector/builder/client share one fixed 4CPU/4Gi Pod; source storage is bounded tmpfs"
|
|
manifest["sourceCPUCounter"] = "mongod process utime+stime; whole fixture Pod cgroup sampled separately"
|
|
manifest["binarySHA256"] = owned["binarySHA256"]
|
|
(out / "manifest.json").write_text(json.dumps(manifest, indent=2))
|
|
env = dict(os.environ, QUERY_REPLICA_ISOLATED_E2E="1", QUERY_REPLICA_CLUSTER_TEST="1", QUERY_REPLICA_TEST_TARGET_URI="mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", QUERY_COMPARE_TASKS=str(args.tasks), QUERY_COMPARE_WAVES=str(args.waves), QUERY_COMPARE_MAX_ACTORS=str(args.max_actors), QUERY_COMPARE_REPORT=str(out / "comparison.json"))
|
|
env["QUERY_REPLICA_REFRESH_MODE"] = manifest["refreshMode"]
|
|
argv = ["go", "test", "./internal/api", "-run", "^TestQueryReplicaIsolatedHTTPComparisonLocal$", "-count=1", "-timeout=45m", "-v"]
|
|
remote_report = "/synthetic/query-comparison-" + str(int(time.time())) + ".json"
|
|
if fixture:
|
|
settings = ["TMPDIR=/synthetic/tmp", "QUERY_REPLICA_ISOLATED_E2E=1", "QUERY_REPLICA_CLUSTER_TEST=1", "QUERY_REPLICA_TEST_TARGET_URI=mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "QUERY_COMPARE_IN_CLUSTER=1", "QUERY_COMPARE_TASKS=" + str(args.tasks), "QUERY_COMPARE_WAVES=" + str(args.waves), "QUERY_COMPARE_MAX_ACTORS=" + str(args.max_actors), "QUERY_COMPARE_REPORT=" + remote_report, "QUERY_COMPARE_WIRE_COMPRESSION=" + os.getenv("QUERY_COMPARE_WIRE_COMPRESSION", ""), "QUERY_COMPARE_MIN_POOL=" + os.getenv("QUERY_COMPARE_MIN_POOL", "0")]
|
|
if args.cold_write_probe:
|
|
settings += ["QUERY_REPLICA_COLD_WRITE_PROBE=1", "QUERY_COLD_PACE_KIB=" + str(args.cold_pace_kib)]
|
|
settings += ["QUERY_REPLICA_REFRESH_MODE=" + manifest["refreshMode"]]
|
|
if args.published_results:
|
|
assert cache_owned, "published cluster comparison requires its owned dedicated cache"
|
|
settings += ["QUERY_PUBLISHED_RESULTS=1","QUERY_COMPARE_VALKEY_URL="+cache_owned["url"]]
|
|
argv = k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "env"] + settings + [api_binary, "-test.run=^" + test_name + "$", "-test.count=1", "-test.timeout=45m", "-test.v"]
|
|
with (out / "test.log").open("w") as log:
|
|
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
bad = 0
|
|
try:
|
|
while child.poll() is None:
|
|
record = sample()
|
|
stacks = ""
|
|
if record["protectedFullAvg10"] > limits["pressure"]:
|
|
probe = ""
|
|
for p in protected:
|
|
uid = p["metadata"]["uid"]
|
|
if record["protectedPressure"][uid]["avg10"] <= limits["pressure"]:
|
|
continue
|
|
probe += "echo PROTECTED " + uid + "; for f in $(find " + cgroup(p) + " -name cgroup.procs); do for pid in $(cat $f); do for t in /host/proc/$pid/task/*; do if grep -q 'State:.*D' $t/status 2>/dev/null; then read w < $t/wchan || true; echo WAIT $pid $w; cat $t/stack 2>/dev/null || true; fi; done; done; done; "
|
|
try:
|
|
stacks = run(k + ["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", probe or "true"])
|
|
except Exception as error:
|
|
stacks = "read-only attribution unavailable: " + str(error)
|
|
with (out / "pressure-attribution.jsonl").open("a") as attribution:
|
|
attribution.write(json.dumps({"time": record["time"], "stacks": stacks, "fuseOnly": sorted(fuse_only_waits(stacks)), "pressure": record["protectedPressure"], "awaitMs": record["awaitMs"], "busyPct": record["busyPct"]}) + "\n")
|
|
if not (out / "first-degradation-stacks.log").exists():
|
|
(out / "first-degradation-stacks.log").write_text(stacks[:32768])
|
|
degraded = record["readySeconds"] > limits["ready"] or protected_pressure_degraded(record, limits["pressure"], fuse_only_waits(stacks)) or record["awaitMs"] > limits["await"] or record["busyPct"] > 70 or record["nodeCPUPct"] > 75
|
|
bad = bad + 1 if degraded else 0
|
|
if bad >= 3:
|
|
raise RuntimeError("existing workload health/IO degraded for three samples")
|
|
time.sleep(5)
|
|
except BaseException as exc:
|
|
stopped = {"reason": str(exc), "syntheticDatabasesMayNeedOwnedCleanup": True, "cleanupErrors": []}
|
|
# Record the original stop even if the API or the child disappears
|
|
# during cleanup. Never delete a replacement Pod by name alone.
|
|
(out / "stopped.json").write_text(json.dumps(stopped))
|
|
try:
|
|
if fixture:
|
|
current = get("-n", "droboticflow-query-lab", "get", "pod", "node4-query-fixture")
|
|
if current["metadata"]["uid"] == owned["fixtureUID"]:
|
|
delete = out / "delete-owned-fixture.json"
|
|
delete.write_text(json.dumps({"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": owned["fixtureUID"]}}))
|
|
run(k + ["delete", "--raw", "/api/v1/namespaces/droboticflow-query-lab/pods/node4-query-fixture", "-f", str(delete)])
|
|
except Exception as cleanup_error:
|
|
stopped["cleanupErrors"].append(str(cleanup_error))
|
|
finally:
|
|
if child.poll() is None:
|
|
try:
|
|
os.killpg(child.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
try:
|
|
child.wait(timeout=30)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
child.wait(timeout=5)
|
|
(out / "stopped.json").write_text(json.dumps(stopped))
|
|
raise
|
|
final_health_error = None
|
|
try:
|
|
sample()
|
|
except Exception as error:
|
|
final_health_error = str(error)
|
|
if fixture:
|
|
result = subprocess.run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "gzip", "-c", remote_report], capture_output=True, timeout=30)
|
|
if result.returncode == 0:
|
|
(out / "comparison.json").write_bytes(gzip.decompress(result.stdout))
|
|
changed = source_fingerprint() != manifest["sourceFingerprint"]
|
|
exit_record = {"returncode": child.returncode, "sourceChanged": changed, "coverageComplete": False, "finalHealthError": final_health_error}
|
|
try:
|
|
if args.cold_write_probe:
|
|
probe = json.loads((out / "comparison.json").read_text())
|
|
assert probe["kind"] == "cold-write-probe" and probe["capacityAccepted"] is False
|
|
assert probe["payloadBytes"] == 8 << 20 and probe["completedBatches"] == 32
|
|
assert probe["majority"] is True and probe["journal"] is True and probe["heartbeatTimeoutMs"] == 3000
|
|
assert probe["paceKiB"] == args.cold_pace_kib
|
|
assert len(probe["samples"]) >= 34
|
|
else:
|
|
validate_comparison(out / "comparison.json", args.tasks, args.waves, args.max_actors)
|
|
exit_record["coverageComplete"] = True
|
|
except (OSError, ValueError, KeyError, AssertionError) as error:
|
|
exit_record["coverageError"] = str(error)
|
|
(out / "exit.json").write_text(json.dumps(exit_record))
|
|
raise SystemExit(child.returncode or (2 if changed or final_health_error or not exit_record["coverageComplete"] else 0))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|