#!/usr/bin/env python3 """Bounded, synthetic node4 smoke test. Does not deploy application code.""" import argparse import hashlib import json import pathlib import re import subprocess import threading import time import urllib.request import sys sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) import query_lab_profiles NODE = "10.189.16.4" NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe" PROD_UID = "d6a90ea7-b57d-4c52-b90b-b6c976cba076" NS = "droboticflow-query-lab" POD = "node4-synthetic-smoke" PATH = "/var/lib/mongo-cpu-node4-smoke-20261113" IMAGE = "docker.io/library/mongo@sha256:02a0cc7939f5ed38f30f9bc714ef5f682d49baf9350c54acf302ce833087fe8a" HERE = pathlib.Path(__file__).resolve().parent def resources(): return {"requests": {"cpu": "1", "memory": "1Gi"}, "limits": {"cpu": "1", "memory": "1Gi"}} def pod(): env = [{"name": "POD_UID", "valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}}, {"name": "NODE_NAME", "valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}}}, {"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}] return {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": POD, "namespace": NS, "labels": {"app": POD}}, "spec": {"nodeName": NODE, "restartPolicy": "Never", "activeDeadlineSeconds": 1800, "terminationGracePeriodSeconds": 60, "automountServiceAccountToken": False, "enableServiceLinks": False, "initContainers": [{"name": "isolate", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "/scripts/init.sh"], "env": env, "resources": resources(), "securityContext": {"privileged": True, "runAsUser": 0}, "volumeMounts": [{"name": "lab", "mountPath": "/lab", "mountPropagation": "Bidirectional"}, {"name": "cg", "mountPath": "/hostcg"}, {"name": "scripts", "mountPath": "/scripts", "readOnly": True}]}], "containers": [{"name": "probe", "image": IMAGE, "imagePullPolicy": "Never", "env": env, "command": ["bash", "-c", "set -e; trap 'exit 0' TERM INT; bash /scripts/guard.sh; touch /tmp/ready; for ((i=0;i<340;i++)); do sleep 5 & wait $!; done"], "resources": resources(), "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}, "readinessProbe": {"exec": {"command": ["test", "-f", "/tmp/ready"]}, "periodSeconds": 5}, "volumeMounts": [{"name": "lab", "mountPath": "/lab", "mountPropagation": "HostToContainer"}, {"name": "cg", "mountPath": "/hostcg", "readOnly": True}, {"name": "scripts", "mountPath": "/scripts", "readOnly": True}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "lab", "hostPath": {"path": PATH, "type": "DirectoryOrCreate"}}, {"name": "cg", "hostPath": {"path": "/sys/fs/cgroup", "type": "Directory"}}, {"name": "scripts", "configMap": {"name": "node4-smoke-scripts"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}} def manifests(query25=False, query_profile="baseline"): objects = [ {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": NS, "labels": {"purpose": "mongo-cpu-synthetic"}}}, {"apiVersion": "v1", "kind": "ResourceQuota", "metadata": {"name": "smoke-budget", "namespace": NS}, "spec": {"hard": {"pods": "1", "requests.cpu": "1", "limits.cpu": "1", "requests.memory": "1Gi", "limits.memory": "1Gi", "persistentvolumeclaims": "0", "services": "0"}}}, {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "deny-all", "namespace": NS}, "spec": {"podSelector": {}, "policyTypes": ["Ingress", "Egress"], "ingress": [], "egress": []}}, {"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "node4-smoke-scripts", "namespace": NS}, "data": {name: (HERE / name).read_text() for name in ["init.sh", "guard.sh", "io.sh", "mongo.sh", "attribution.sh"]}}, pod(), ] if query25: init = (HERE / "init.sh").read_text().replace("536870912", "26843545600").replace("mongo-cpu-smoke", "mongo-cpu-query").replace("node4-synthetic-pilot-v1", "node4-synthetic-query25-v1") init = init.replace("mkfs.ext4 -F -q -m 0", "mkfs.ext4 -F -q -m 0 -i 131072 -J size=32") guard = (HERE / "guard.sh").read_text().replace("131072", "6553600").replace("node4-synthetic-pilot-v1", "node4-synthetic-query25-v1") objects[-2]["metadata"]["name"] = "node4-query25-scripts" objects[-2]["data"] = {"init.sh": init, "guard.sh": guard, **{name: (HERE / name).read_text() for name in ["query25.sh", "seed25.sh", "verify25.sh"]}} obj = objects[-1] obj["metadata"]["name"] = "node4-query25" obj["metadata"]["labels"] = {"app": "node4-query25"} obj["spec"]["activeDeadlineSeconds"] = 21600 for volume in obj["spec"]["volumes"]: if volume["name"] == "lab": volume["hostPath"]["path"] = "/var/lib/mongo-cpu-node4-query25-20261113" if volume["name"] == "scripts": volume["configMap"]["name"] = "node4-query25-scripts" container = obj["spec"]["containers"][0] container["command"] = ["bash", "/scripts/query25.sh"] container["readinessProbe"] = {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27129", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5} return query_lab_profiles.render(objects, query_profile) if query25 else objects def main(): ap = argparse.ArgumentParser() ap.add_argument("--kubeconfig", required=True) ap.add_argument("--output", required=True) ap.add_argument("--execute", action="store_true") ap.add_argument("--confirm-plan") ap.add_argument("--resume-namespace-uid") ap.add_argument("--diagnose-only", action="store_true") ap.add_argument("--query25", action="store_true") ap.add_argument("--acceptance-file") ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline") args = ap.parse_args() out = pathlib.Path(args.output).resolve() out.mkdir(parents=True, exist_ok=True) assert not (args.query25 and args.diagnose_only) acceptance_sha = None if args.query25: assert args.acceptance_file, "passing small recovery evidence required" evidence_bytes = pathlib.Path(args.acceptance_file).read_bytes() evidence = json.loads(evidence_bytes) assert evidence["result"] == "small-smoke-passed" assert evidence["firstPodUID"] != evidence["secondPodUID"] acceptance_sha = hashlib.sha256(evidence_bytes).hexdigest() assert args.query25 or args.query_profile == "baseline" objects = manifests(args.query25, args.query_profile) target_pod = objects[-1]["metadata"]["name"] runner_sha = hashlib.sha256(pathlib.Path(__file__).read_bytes()).hexdigest() mode = "query25-synthetic-only" if args.query25 else "attribution-only" if args.diagnose_only else "full-synthetic-recovery" digest = hashlib.sha256(json.dumps({"objects": objects, "runner": runner_sha, "mode": mode, "acceptance": acceptance_sha}, sort_keys=True).encode()).hexdigest() (out / "objects.json").write_text(json.dumps(objects, indent=2)) (out / "plan.json").write_text(json.dumps({"sha256": digest, "node": NODE, "nodeUID": NODE_UID, "namespace": NS, "image": IMAGE, "hostPath": objects[-1]["spec"]["volumes"][0]["hostPath"]["path"], "bytes": 26843545600 if args.query25 else 536870912, "productionWrites": False, "mode": mode, "acceptanceSha256": acceptance_sha}, indent=2)) print("PLAN", digest, flush=True) if not args.execute: return if args.confirm_plan != digest: raise RuntimeError("plan confirmation mismatch") k = ["kubectl", "--kubeconfig", args.kubeconfig] def call(argv, timeout=30, payload=None): p = subprocess.run(k + argv, input=payload, text=True, capture_output=True, timeout=timeout) if p.returncode: (out / f"failed-command-{time.time_ns()}.json").write_text(json.dumps({"argv": argv, "code": p.returncode, "stdout": p.stdout[:65536], "stderr": p.stderr[:65536]}, indent=2)) raise RuntimeError(f"kubectl {argv[:3]} failed: {p.stderr[:1000]}") return p.stdout def get(argv): return json.loads(call(argv + ["-o", "json"])) node = get(["get", "node", NODE]) assert node["metadata"]["uid"] == NODE_UID assert get(["get", "namespace", "droboticflow-fanyu"])["metadata"]["uid"] == PROD_UID assert any(IMAGE in x["names"] for x in node["status"].get("images", [])) for c in node["status"]["conditions"]: if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"): assert c["status"] == "False" existing = subprocess.run(k + ["get", "namespace", NS, "--ignore-not-found", "-o", "name"], capture_output=True, text=True, check=True) config_missing = False if existing.stdout.strip(): existing_ns = get(["get", "namespace", NS]) assert args.resume_namespace_uid == existing_ns["metadata"]["uid"], "explicit recorded namespace UID required" assert existing_ns["metadata"].get("labels", {}).get("purpose") == "mongo-cpu-synthetic" assert not get(["get", "pods", "-n", NS])["items"], "stop and inspect existing pilot Pod before resuming" config_name = objects[-2]["metadata"]["name"] config_raw = call(["get", "configmap", config_name, "-n", NS, "--ignore-not-found", "-o", "json"]) if config_raw.strip(): assert json.loads(config_raw)["data"] == objects[-2]["data"] else: assert args.query25 config_missing = True policy = get(["get", "networkpolicy", "deny-all", "-n", NS])["spec"] assert policy["podSelector"] == {} and set(policy["policyTypes"]) == {"Ingress", "Egress"} assert not policy.get("ingress") and not policy.get("egress") assert get(["get", "resourcequota", "smoke-budget", "-n", NS])["spec"]["hard"] == objects[1]["spec"]["hard"] else: assert not args.resume_namespace_uid protected = {} for item in get(["get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE])["items"]: if item["metadata"]["namespace"] == NS: continue uid = item["metadata"]["uid"] assert re.fullmatch(r"[0-9a-f-]{36}", uid) qos = item["status"]["qosClass"] prefix = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/" if qos == "Guaranteed": path = prefix + "kubepods-pod" + uid.replace("-", "_") + ".slice" else: assert qos in ("Burstable", "BestEffort") part = "kubepods-" + qos.lower() path = prefix + part + ".slice/" + part + "-pod" + uid.replace("-", "_") + ".slice" protected[uid] = path protected_script = "\n".join("echo CGROUP " + uid + "; cat " + path + "/io.pressure" for uid, path in protected.items()) restarts = {} stop = threading.Event() monitor_failed = [] samples = [] def sample(): start = time.monotonic() with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as r: assert r.status == 200 r.read(65536) latency = time.monotonic() - start pods = get(["get", "pods", "-n", "droboticflow-fanyu"]) current = {p["metadata"]["uid"]: sum(c.get("restartCount", 0) for c in p["status"].get("containerStatuses", [])) for p in pods["items"]} if restarts: assert current.keys() == restarts.keys(), "production Pod set changed" assert all(current[uid] == count for uid, count in restarts.items()), "production restart observed" else: restarts.update(current) host = call(["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c", "set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; head -n 1 /host/proc/stat; cat /host/proc/pressure/io; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; " + protected_script], timeout=15) available = next(x for x in host.splitlines() if x.startswith("AVAILABLE ")).split() assert int(available[1]) * int(available[2]) >= 80530636800, "node disk below 75Gi" n = get(["get", "node", NODE]) assert all(c["status"] == "False" for c in n["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure")) per_pod = {} current_uid = None for line in host.splitlines(): if line.startswith("CGROUP "): current_uid = line.split()[1] elif current_uid and line.startswith("full "): per_pod[current_uid] = float(line.split("avg10=")[1].split()[0]) assert per_pod.keys() == protected.keys(), "protected cgroup pressure sample missing" record = {"time": time.time(), "readySeconds": latency, "host": host, "productionRestarts": current, "protectedIoFullAvg10": per_pod} disk = [int(x) for x in next(x for x in host.splitlines() if x.split()[:3] == ["252", "0", "vda"]).split()[3:]] record["diskCounters"] = disk record["diskAwaitMs"] = 0.0 record["diskBusyPct"] = 0.0 if samples: previous = samples[-1] old = previous["diskCounters"] ops = disk[0] + disk[4] - old[0] - old[4] record["diskAwaitMs"] = (disk[3] + disk[7] - old[3] - old[7]) / max(1, ops) record["diskBusyPct"] = (disk[9] - old[9]) / max(0.001, record["time"] - previous["time"]) / 10 samples.append(record) with (out / "samples.jsonl").open("a") as f: f.write(json.dumps(record) + "\n") return latency print("BASELINE sampling", flush=True) baseline = [] for _ in range(6): baseline.append(sample()) time.sleep(5) latency_limit = max(1.0, max(baseline) * 2) baseline_pressure = max(max(s["protectedIoFullAvg10"].values()) for s in samples) baseline_await = max(s["diskAwaitMs"] for s in samples) def monitor(): bad_latency = 0 bad_pressure = 0 while not stop.wait(5): try: latency = sample() bad_latency = bad_latency + 1 if latency > latency_limit else 0 if bad_latency >= 3: raise RuntimeError("readiness latency degraded for three samples") # Global PSI includes the deliberately throttled pilot itself. Protect existing # workloads with their own PSI plus actual whole-device latency/busy time. latest = samples[-1] pressure = max(latest["protectedIoFullAvg10"].values()) disk_degraded = latest["diskAwaitMs"] > max(10, baseline_await * 3) or latest["diskBusyPct"] > 70 bad_pressure = bad_pressure + 1 if pressure > max(0.5, baseline_pressure + 0.5) or disk_degraded else 0 if bad_pressure >= 3: raise RuntimeError("existing-workload IO pressure or physical disk latency increased for three samples") except Exception as e: monitor_failed.append(str(e)) (out / "monitor-failure.json").write_text(json.dumps(monitor_failed)) subprocess.run(k + ["-n", NS, "delete", "pod", target_pod, "--ignore-not-found", "--wait=false"], capture_output=True, timeout=15) return def create(obj): with (out / "intents.jsonl").open("a") as f: f.write(json.dumps({"time": time.time(), "kind": obj["kind"], "name": obj["metadata"]["name"], "plan": digest}) + "\n") print(call(["create", "-f", "-"], payload=json.dumps(obj)).strip(), flush=True) def ready(): deadline = time.monotonic() + (900 if args.query25 else 180) while time.monotonic() < deadline: if monitor_failed: raise RuntimeError(str(monitor_failed)) p = get(["get", "pod", target_pod, "-n", NS]) (out / "last-pod.json").write_text(json.dumps(p, indent=2)) if any(c["type"] == "Ready" and c["status"] == "True" for c in p["status"].get("conditions", [])): return p["metadata"]["uid"] if p["status"]["phase"] in ("Failed", "Succeeded"): raise RuntimeError("pilot stopped before readiness") time.sleep(3) raise RuntimeError("pilot readiness timed out") def probe(name, script, *extra, timeout=180): print("PROBE", name, flush=True) try: result = call(["-n", NS, "exec", target_pod, "-c", "probe", "--", "bash", "/scripts/" + script, *extra], timeout=timeout) except Exception: # Preserve bounded logs/status; never retry an unknown write result. raise (out / (name + ".txt")).write_text(result) print(result[-4000:], flush=True) assert not monitor_failed worker = None retain_query = False try: if not existing.stdout.strip(): for obj in objects[:-1]: create(obj) elif config_missing: create(objects[-2]) for _ in range(20): quota = get(["get", "resourcequota", "smoke-budget", "-n", NS]) if quota.get("status", {}).get("hard"): break time.sleep(1) else: raise RuntimeError("quota not initialized") worker = threading.Thread(target=monitor, daemon=True) worker.start() create(objects[-1]) first_uid = ready() if args.query25: probe("query25-seed", "seed25.sh", timeout=180) call(["-n", NS, "delete", "pod", target_pod, "--wait=true", "--timeout=90s"], timeout=100) create(objects[-1]) second_uid = ready() assert first_uid != second_uid probe("query25-after-rebuild", "verify25.sh", timeout=180) for _ in range(6): sample() time.sleep(5) assert not monitor_failed (out / "result.json").write_text(json.dumps({"result": "query25-created-synthetic-only", "plan": digest, "acceptanceSha256": acceptance_sha, "firstPodUID": first_uid, "secondPodUID": second_uid, "activeDeadlineSeconds": 21600, "network": "loopback-only-no-Service", "limitations": ["single-node lab", "no production sync", "not capacity acceptance"]}, indent=2)) retain_query = True print("QUERY25_CREATED_SYNTHETIC_ONLY", flush=True) return if args.diagnose_only: probe("attribution", "attribution.sh", "assert", timeout=90) (out / "result.json").write_text(json.dumps({"result": "attribution-captured-not-acceptance", "plan": digest, "podUID": first_uid}, indent=2)) return probe("io-first", "io.sh", timeout=240) probe("mongo-seed", "mongo.sh", "seed", timeout=180) call(["-n", NS, "delete", "pod", target_pod, "--wait=true", "--timeout=90s"], timeout=100) create(pod()) second_uid = ready() assert second_uid != first_uid probe("io-after-rebuild", "io.sh", timeout=240) probe("mongo-after-rebuild", "mongo.sh", "verify", timeout=180) probe("mongo-full-recovery", "mongo.sh", "full", timeout=700) print("POST sampling", flush=True) for _ in range(6): sample() time.sleep(5) assert not monitor_failed (out / "result.json").write_text(json.dumps({"result": "small-smoke-passed", "plan": digest, "firstPodUID": first_uid, "secondPodUID": second_uid, "sampleCount": len(samples), "limitations": ["short observation window", "no authenticated business workflow test", "not 25Gi or capacity acceptance"]}, indent=2)) print("SMALL_SMOKE_PASSED", flush=True) except BaseException as e: (out / "result.json").write_text(json.dumps({"result": "failed", "plan": digest, "reason": str(e)}, indent=2)) raise finally: stop.set() if worker: worker.join(timeout=30) try: logs = call(["-n", NS, "logs", target_pod, "-c", "isolate", "--tail=60"]) (out / "init.log").write_text(logs) except Exception: pass # Stop only this synthetic Pod; retain its backing image and exact mount for review. if not retain_query: subprocess.run(k + ["-n", NS, "delete", "pod", target_pod, "--ignore-not-found", "--wait=true", "--timeout=90s"], capture_output=True, timeout=100) if __name__ == "__main__": main()