368 lines
21 KiB
Python

#!/usr/bin/env python3
"""Bounded, synthetic node4 smoke test. Does not deploy application code."""
import argparse
import hashlib
import json
import pathlib
import re
import subprocess
import threading
import time
import urllib.request
NODE = "10.189.16.4"
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
PROD_UID = "d6a90ea7-b57d-4c52-b90b-b6c976cba076"
NS = "droboticflow-query-lab"
POD = "node4-synthetic-smoke"
PATH = "/var/lib/mongo-cpu-node4-smoke-20261113"
IMAGE = "docker.io/library/mongo@sha256:02a0cc7939f5ed38f30f9bc714ef5f682d49baf9350c54acf302ce833087fe8a"
HERE = pathlib.Path(__file__).resolve().parent
def resources():
return {"requests": {"cpu": "1", "memory": "1Gi"}, "limits": {"cpu": "1", "memory": "1Gi"}}
def pod():
env = [{"name": "POD_UID", "valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}},
{"name": "NODE_NAME", "valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}}},
{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}]
return {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": POD, "namespace": NS, "labels": {"app": POD}},
"spec": {"nodeName": NODE, "restartPolicy": "Never", "activeDeadlineSeconds": 1800,
"terminationGracePeriodSeconds": 60, "automountServiceAccountToken": False,
"enableServiceLinks": False,
"initContainers": [{"name": "isolate", "image": IMAGE, "imagePullPolicy": "Never",
"command": ["bash", "/scripts/init.sh"], "env": env, "resources": resources(),
"securityContext": {"privileged": True, "runAsUser": 0},
"volumeMounts": [{"name": "lab", "mountPath": "/lab", "mountPropagation": "Bidirectional"},
{"name": "cg", "mountPath": "/hostcg"},
{"name": "scripts", "mountPath": "/scripts", "readOnly": True}]}],
"containers": [{"name": "probe", "image": IMAGE, "imagePullPolicy": "Never", "env": env,
"command": ["bash", "-c", "set -e; trap 'exit 0' TERM INT; bash /scripts/guard.sh; touch /tmp/ready; for ((i=0;i<340;i++)); do sleep 5 & wait $!; done"],
"resources": resources(),
"securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True,
"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True,
"capabilities": {"drop": ["ALL"]}},
"readinessProbe": {"exec": {"command": ["test", "-f", "/tmp/ready"]}, "periodSeconds": 5},
"volumeMounts": [{"name": "lab", "mountPath": "/lab", "mountPropagation": "HostToContainer"},
{"name": "cg", "mountPath": "/hostcg", "readOnly": True},
{"name": "scripts", "mountPath": "/scripts", "readOnly": True},
{"name": "tmp", "mountPath": "/tmp"}]}],
"volumes": [{"name": "lab", "hostPath": {"path": PATH, "type": "DirectoryOrCreate"}},
{"name": "cg", "hostPath": {"path": "/sys/fs/cgroup", "type": "Directory"}},
{"name": "scripts", "configMap": {"name": "node4-smoke-scripts"}},
{"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
def manifests(query25=False):
objects = [
{"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": NS, "labels": {"purpose": "mongo-cpu-synthetic"}}},
{"apiVersion": "v1", "kind": "ResourceQuota", "metadata": {"name": "smoke-budget", "namespace": NS},
"spec": {"hard": {"pods": "1", "requests.cpu": "1", "limits.cpu": "1", "requests.memory": "1Gi", "limits.memory": "1Gi", "persistentvolumeclaims": "0", "services": "0"}}},
{"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "deny-all", "namespace": NS},
"spec": {"podSelector": {}, "policyTypes": ["Ingress", "Egress"], "ingress": [], "egress": []}},
{"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "node4-smoke-scripts", "namespace": NS},
"data": {name: (HERE / name).read_text() for name in ["init.sh", "guard.sh", "io.sh", "mongo.sh", "attribution.sh"]}},
pod(),
]
if query25:
init = (HERE / "init.sh").read_text().replace("536870912", "26843545600").replace("mongo-cpu-smoke", "mongo-cpu-query").replace("node4-synthetic-pilot-v1", "node4-synthetic-query25-v1")
init = init.replace("mkfs.ext4 -F -q -m 0", "mkfs.ext4 -F -q -m 0 -i 131072 -J size=32")
guard = (HERE / "guard.sh").read_text().replace("131072", "6553600").replace("node4-synthetic-pilot-v1", "node4-synthetic-query25-v1")
objects[-2]["metadata"]["name"] = "node4-query25-scripts"
objects[-2]["data"] = {"init.sh": init, "guard.sh": guard, **{name: (HERE / name).read_text() for name in ["query25.sh", "seed25.sh", "verify25.sh"]}}
obj = objects[-1]
obj["metadata"]["name"] = "node4-query25"
obj["metadata"]["labels"] = {"app": "node4-query25"}
obj["spec"]["activeDeadlineSeconds"] = 21600
for volume in obj["spec"]["volumes"]:
if volume["name"] == "lab":
volume["hostPath"]["path"] = "/var/lib/mongo-cpu-node4-query25-20261113"
if volume["name"] == "scripts":
volume["configMap"]["name"] = "node4-query25-scripts"
container = obj["spec"]["containers"][0]
container["command"] = ["bash", "/scripts/query25.sh"]
container["readinessProbe"] = {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27129", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}
return objects
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--kubeconfig", required=True)
ap.add_argument("--output", required=True)
ap.add_argument("--execute", action="store_true")
ap.add_argument("--confirm-plan")
ap.add_argument("--resume-namespace-uid")
ap.add_argument("--diagnose-only", action="store_true")
ap.add_argument("--query25", action="store_true")
ap.add_argument("--acceptance-file")
args = ap.parse_args()
out = pathlib.Path(args.output).resolve()
out.mkdir(parents=True, exist_ok=True)
assert not (args.query25 and args.diagnose_only)
acceptance_sha = None
if args.query25:
assert args.acceptance_file, "passing small recovery evidence required"
evidence_bytes = pathlib.Path(args.acceptance_file).read_bytes()
evidence = json.loads(evidence_bytes)
assert evidence["result"] == "small-smoke-passed"
assert evidence["firstPodUID"] != evidence["secondPodUID"]
acceptance_sha = hashlib.sha256(evidence_bytes).hexdigest()
objects = manifests(args.query25)
target_pod = objects[-1]["metadata"]["name"]
runner_sha = hashlib.sha256(pathlib.Path(__file__).read_bytes()).hexdigest()
mode = "query25-synthetic-only" if args.query25 else "attribution-only" if args.diagnose_only else "full-synthetic-recovery"
digest = hashlib.sha256(json.dumps({"objects": objects, "runner": runner_sha, "mode": mode, "acceptance": acceptance_sha}, sort_keys=True).encode()).hexdigest()
(out / "objects.json").write_text(json.dumps(objects, indent=2))
(out / "plan.json").write_text(json.dumps({"sha256": digest, "node": NODE, "nodeUID": NODE_UID,
"namespace": NS, "image": IMAGE, "hostPath": objects[-1]["spec"]["volumes"][0]["hostPath"]["path"],
"bytes": 26843545600 if args.query25 else 536870912, "productionWrites": False, "mode": mode, "acceptanceSha256": acceptance_sha}, indent=2))
print("PLAN", digest, flush=True)
if not args.execute:
return
if args.confirm_plan != digest:
raise RuntimeError("plan confirmation mismatch")
k = ["kubectl", "--kubeconfig", args.kubeconfig]
def call(argv, timeout=30, payload=None):
p = subprocess.run(k + argv, input=payload, text=True, capture_output=True, timeout=timeout)
if p.returncode:
(out / f"failed-command-{time.time_ns()}.json").write_text(json.dumps({"argv": argv, "code": p.returncode, "stdout": p.stdout[:65536], "stderr": p.stderr[:65536]}, indent=2))
raise RuntimeError(f"kubectl {argv[:3]} failed: {p.stderr[:1000]}")
return p.stdout
def get(argv):
return json.loads(call(argv + ["-o", "json"]))
node = get(["get", "node", NODE])
assert node["metadata"]["uid"] == NODE_UID
assert get(["get", "namespace", "droboticflow-fanyu"])["metadata"]["uid"] == PROD_UID
assert any(IMAGE in x["names"] for x in node["status"].get("images", []))
for c in node["status"]["conditions"]:
if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"):
assert c["status"] == "False"
existing = subprocess.run(k + ["get", "namespace", NS, "--ignore-not-found", "-o", "name"], capture_output=True, text=True, check=True)
config_missing = False
if existing.stdout.strip():
existing_ns = get(["get", "namespace", NS])
assert args.resume_namespace_uid == existing_ns["metadata"]["uid"], "explicit recorded namespace UID required"
assert existing_ns["metadata"].get("labels", {}).get("purpose") == "mongo-cpu-synthetic"
assert not get(["get", "pods", "-n", NS])["items"], "stop and inspect existing pilot Pod before resuming"
config_name = objects[-2]["metadata"]["name"]
config_raw = call(["get", "configmap", config_name, "-n", NS, "--ignore-not-found", "-o", "json"])
if config_raw.strip():
assert json.loads(config_raw)["data"] == objects[-2]["data"]
else:
assert args.query25
config_missing = True
policy = get(["get", "networkpolicy", "deny-all", "-n", NS])["spec"]
assert policy["podSelector"] == {} and set(policy["policyTypes"]) == {"Ingress", "Egress"}
assert not policy.get("ingress") and not policy.get("egress")
assert get(["get", "resourcequota", "smoke-budget", "-n", NS])["spec"]["hard"] == objects[1]["spec"]["hard"]
else:
assert not args.resume_namespace_uid
protected = {}
for item in get(["get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE])["items"]:
if item["metadata"]["namespace"] == NS:
continue
uid = item["metadata"]["uid"]
assert re.fullmatch(r"[0-9a-f-]{36}", uid)
qos = item["status"]["qosClass"]
prefix = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/"
if qos == "Guaranteed":
path = prefix + "kubepods-pod" + uid.replace("-", "_") + ".slice"
else:
assert qos in ("Burstable", "BestEffort")
part = "kubepods-" + qos.lower()
path = prefix + part + ".slice/" + part + "-pod" + uid.replace("-", "_") + ".slice"
protected[uid] = path
protected_script = "\n".join("echo CGROUP " + uid + "; cat " + path + "/io.pressure" for uid, path in protected.items())
restarts = {}
stop = threading.Event()
monitor_failed = []
samples = []
def sample():
start = time.monotonic()
with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as r:
assert r.status == 200
r.read(65536)
latency = time.monotonic() - start
pods = get(["get", "pods", "-n", "droboticflow-fanyu"])
current = {p["metadata"]["uid"]: sum(c.get("restartCount", 0) for c in p["status"].get("containerStatuses", [])) for p in pods["items"]}
if restarts:
assert current.keys() == restarts.keys(), "production Pod set changed"
assert all(current[uid] == count for uid, count in restarts.items()), "production restart observed"
else:
restarts.update(current)
host = call(["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c",
"set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; head -n 1 /host/proc/stat; cat /host/proc/pressure/io; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; " + protected_script], timeout=15)
available = next(x for x in host.splitlines() if x.startswith("AVAILABLE ")).split()
assert int(available[1]) * int(available[2]) >= 80530636800, "node disk below 75Gi"
n = get(["get", "node", NODE])
assert all(c["status"] == "False" for c in n["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
per_pod = {}
current_uid = None
for line in host.splitlines():
if line.startswith("CGROUP "):
current_uid = line.split()[1]
elif current_uid and line.startswith("full "):
per_pod[current_uid] = float(line.split("avg10=")[1].split()[0])
assert per_pod.keys() == protected.keys(), "protected cgroup pressure sample missing"
record = {"time": time.time(), "readySeconds": latency, "host": host, "productionRestarts": current, "protectedIoFullAvg10": per_pod}
disk = [int(x) for x in next(x for x in host.splitlines() if x.split()[:3] == ["252", "0", "vda"]).split()[3:]]
record["diskCounters"] = disk
record["diskAwaitMs"] = 0.0
record["diskBusyPct"] = 0.0
if samples:
previous = samples[-1]
old = previous["diskCounters"]
ops = disk[0] + disk[4] - old[0] - old[4]
record["diskAwaitMs"] = (disk[3] + disk[7] - old[3] - old[7]) / max(1, ops)
record["diskBusyPct"] = (disk[9] - old[9]) / max(0.001, record["time"] - previous["time"]) / 10
samples.append(record)
with (out / "samples.jsonl").open("a") as f:
f.write(json.dumps(record) + "\n")
return latency
print("BASELINE sampling", flush=True)
baseline = []
for _ in range(6):
baseline.append(sample())
time.sleep(5)
latency_limit = max(1.0, max(baseline) * 2)
baseline_pressure = max(max(s["protectedIoFullAvg10"].values()) for s in samples)
baseline_await = max(s["diskAwaitMs"] for s in samples)
def monitor():
bad_latency = 0
bad_pressure = 0
while not stop.wait(5):
try:
latency = sample()
bad_latency = bad_latency + 1 if latency > latency_limit else 0
if bad_latency >= 3:
raise RuntimeError("readiness latency degraded for three samples")
# Global PSI includes the deliberately throttled pilot itself. Protect existing
# workloads with their own PSI plus actual whole-device latency/busy time.
latest = samples[-1]
pressure = max(latest["protectedIoFullAvg10"].values())
disk_degraded = latest["diskAwaitMs"] > max(10, baseline_await * 3) or latest["diskBusyPct"] > 70
bad_pressure = bad_pressure + 1 if pressure > max(0.5, baseline_pressure + 0.5) or disk_degraded else 0
if bad_pressure >= 3:
raise RuntimeError("existing-workload IO pressure or physical disk latency increased for three samples")
except Exception as e:
monitor_failed.append(str(e))
(out / "monitor-failure.json").write_text(json.dumps(monitor_failed))
subprocess.run(k + ["-n", NS, "delete", "pod", target_pod, "--ignore-not-found", "--wait=false"], capture_output=True, timeout=15)
return
def create(obj):
with (out / "intents.jsonl").open("a") as f:
f.write(json.dumps({"time": time.time(), "kind": obj["kind"], "name": obj["metadata"]["name"], "plan": digest}) + "\n")
print(call(["create", "-f", "-"], payload=json.dumps(obj)).strip(), flush=True)
def ready():
deadline = time.monotonic() + (900 if args.query25 else 180)
while time.monotonic() < deadline:
if monitor_failed:
raise RuntimeError(str(monitor_failed))
p = get(["get", "pod", target_pod, "-n", NS])
(out / "last-pod.json").write_text(json.dumps(p, indent=2))
if any(c["type"] == "Ready" and c["status"] == "True" for c in p["status"].get("conditions", [])):
return p["metadata"]["uid"]
if p["status"]["phase"] in ("Failed", "Succeeded"):
raise RuntimeError("pilot stopped before readiness")
time.sleep(3)
raise RuntimeError("pilot readiness timed out")
def probe(name, script, *extra, timeout=180):
print("PROBE", name, flush=True)
try:
result = call(["-n", NS, "exec", target_pod, "-c", "probe", "--", "bash", "/scripts/" + script, *extra], timeout=timeout)
except Exception:
# Preserve bounded logs/status; never retry an unknown write result.
raise
(out / (name + ".txt")).write_text(result)
print(result[-4000:], flush=True)
assert not monitor_failed
worker = None
retain_query = False
try:
if not existing.stdout.strip():
for obj in objects[:-1]:
create(obj)
elif config_missing:
create(objects[-2])
for _ in range(20):
quota = get(["get", "resourcequota", "smoke-budget", "-n", NS])
if quota.get("status", {}).get("hard"):
break
time.sleep(1)
else:
raise RuntimeError("quota not initialized")
worker = threading.Thread(target=monitor, daemon=True)
worker.start()
create(objects[-1])
first_uid = ready()
if args.query25:
probe("query25-seed", "seed25.sh", timeout=180)
call(["-n", NS, "delete", "pod", target_pod, "--wait=true", "--timeout=90s"], timeout=100)
create(objects[-1])
second_uid = ready()
assert first_uid != second_uid
probe("query25-after-rebuild", "verify25.sh", timeout=180)
for _ in range(6):
sample()
time.sleep(5)
assert not monitor_failed
(out / "result.json").write_text(json.dumps({"result": "query25-created-synthetic-only", "plan": digest, "acceptanceSha256": acceptance_sha, "firstPodUID": first_uid, "secondPodUID": second_uid, "activeDeadlineSeconds": 21600, "network": "loopback-only-no-Service", "limitations": ["single-node lab", "no production sync", "not capacity acceptance"]}, indent=2))
retain_query = True
print("QUERY25_CREATED_SYNTHETIC_ONLY", flush=True)
return
if args.diagnose_only:
probe("attribution", "attribution.sh", "assert", timeout=90)
(out / "result.json").write_text(json.dumps({"result": "attribution-captured-not-acceptance", "plan": digest, "podUID": first_uid}, indent=2))
return
probe("io-first", "io.sh", timeout=240)
probe("mongo-seed", "mongo.sh", "seed", timeout=180)
call(["-n", NS, "delete", "pod", target_pod, "--wait=true", "--timeout=90s"], timeout=100)
create(pod())
second_uid = ready()
assert second_uid != first_uid
probe("io-after-rebuild", "io.sh", timeout=240)
probe("mongo-after-rebuild", "mongo.sh", "verify", timeout=180)
probe("mongo-full-recovery", "mongo.sh", "full", timeout=700)
print("POST sampling", flush=True)
for _ in range(6):
sample()
time.sleep(5)
assert not monitor_failed
(out / "result.json").write_text(json.dumps({"result": "small-smoke-passed", "plan": digest,
"firstPodUID": first_uid, "secondPodUID": second_uid,
"sampleCount": len(samples), "limitations": ["short observation window", "no authenticated business workflow test", "not 25Gi or capacity acceptance"]}, indent=2))
print("SMALL_SMOKE_PASSED", flush=True)
except BaseException as e:
(out / "result.json").write_text(json.dumps({"result": "failed", "plan": digest, "reason": str(e)}, indent=2))
raise
finally:
stop.set()
if worker:
worker.join(timeout=30)
try:
logs = call(["-n", NS, "logs", target_pod, "-c", "isolate", "--tail=60"])
(out / "init.log").write_text(logs)
except Exception:
pass
# Stop only this synthetic Pod; retain its backing image and exact mount for review.
if not retain_query:
subprocess.run(k + ["-n", NS, "delete", "pod", target_pod, "--ignore-not-found", "--wait=true", "--timeout=90s"], capture_output=True, timeout=100)
if __name__ == "__main__":
main()