forked from longtao.wu/droboticflow
373 lines
21 KiB
Python
373 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded, synthetic node4 smoke test. Does not deploy application code."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
import sys
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
import query_lab_profiles
|
|
|
|
|
|
NODE = "10.189.16.4"
|
|
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
|
|
PROD_UID = "d6a90ea7-b57d-4c52-b90b-b6c976cba076"
|
|
NS = "droboticflow-query-lab"
|
|
POD = "node4-synthetic-smoke"
|
|
PATH = "/var/lib/mongo-cpu-node4-smoke-20261113"
|
|
IMAGE = "docker.io/library/mongo@sha256:02a0cc7939f5ed38f30f9bc714ef5f682d49baf9350c54acf302ce833087fe8a"
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
|
|
|
|
def resources():
|
|
return {"requests": {"cpu": "1", "memory": "1Gi"}, "limits": {"cpu": "1", "memory": "1Gi"}}
|
|
|
|
|
|
def pod():
|
|
env = [{"name": "POD_UID", "valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}},
|
|
{"name": "NODE_NAME", "valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}}},
|
|
{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}]
|
|
return {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": POD, "namespace": NS, "labels": {"app": POD}},
|
|
"spec": {"nodeName": NODE, "restartPolicy": "Never", "activeDeadlineSeconds": 1800,
|
|
"terminationGracePeriodSeconds": 60, "automountServiceAccountToken": False,
|
|
"enableServiceLinks": False,
|
|
"initContainers": [{"name": "isolate", "image": IMAGE, "imagePullPolicy": "Never",
|
|
"command": ["bash", "/scripts/init.sh"], "env": env, "resources": resources(),
|
|
"securityContext": {"privileged": True, "runAsUser": 0},
|
|
"volumeMounts": [{"name": "lab", "mountPath": "/lab", "mountPropagation": "Bidirectional"},
|
|
{"name": "cg", "mountPath": "/hostcg"},
|
|
{"name": "scripts", "mountPath": "/scripts", "readOnly": True}]}],
|
|
"containers": [{"name": "probe", "image": IMAGE, "imagePullPolicy": "Never", "env": env,
|
|
"command": ["bash", "-c", "set -e; trap 'exit 0' TERM INT; bash /scripts/guard.sh; touch /tmp/ready; for ((i=0;i<340;i++)); do sleep 5 & wait $!; done"],
|
|
"resources": resources(),
|
|
"securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True,
|
|
"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True,
|
|
"capabilities": {"drop": ["ALL"]}},
|
|
"readinessProbe": {"exec": {"command": ["test", "-f", "/tmp/ready"]}, "periodSeconds": 5},
|
|
"volumeMounts": [{"name": "lab", "mountPath": "/lab", "mountPropagation": "HostToContainer"},
|
|
{"name": "cg", "mountPath": "/hostcg", "readOnly": True},
|
|
{"name": "scripts", "mountPath": "/scripts", "readOnly": True},
|
|
{"name": "tmp", "mountPath": "/tmp"}]}],
|
|
"volumes": [{"name": "lab", "hostPath": {"path": PATH, "type": "DirectoryOrCreate"}},
|
|
{"name": "cg", "hostPath": {"path": "/sys/fs/cgroup", "type": "Directory"}},
|
|
{"name": "scripts", "configMap": {"name": "node4-smoke-scripts"}},
|
|
{"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
|
|
|
|
|
|
def manifests(query25=False, query_profile="baseline"):
|
|
objects = [
|
|
{"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": NS, "labels": {"purpose": "mongo-cpu-synthetic"}}},
|
|
{"apiVersion": "v1", "kind": "ResourceQuota", "metadata": {"name": "smoke-budget", "namespace": NS},
|
|
"spec": {"hard": {"pods": "1", "requests.cpu": "1", "limits.cpu": "1", "requests.memory": "1Gi", "limits.memory": "1Gi", "persistentvolumeclaims": "0", "services": "0"}}},
|
|
{"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "deny-all", "namespace": NS},
|
|
"spec": {"podSelector": {}, "policyTypes": ["Ingress", "Egress"], "ingress": [], "egress": []}},
|
|
{"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "node4-smoke-scripts", "namespace": NS},
|
|
"data": {name: (HERE / name).read_text() for name in ["init.sh", "guard.sh", "io.sh", "mongo.sh", "attribution.sh"]}},
|
|
pod(),
|
|
]
|
|
if query25:
|
|
init = (HERE / "init.sh").read_text().replace("536870912", "26843545600").replace("mongo-cpu-smoke", "mongo-cpu-query").replace("node4-synthetic-pilot-v1", "node4-synthetic-query25-v1")
|
|
init = init.replace("mkfs.ext4 -F -q -m 0", "mkfs.ext4 -F -q -m 0 -i 131072 -J size=32")
|
|
guard = (HERE / "guard.sh").read_text().replace("131072", "6553600").replace("node4-synthetic-pilot-v1", "node4-synthetic-query25-v1")
|
|
objects[-2]["metadata"]["name"] = "node4-query25-scripts"
|
|
objects[-2]["data"] = {"init.sh": init, "guard.sh": guard, **{name: (HERE / name).read_text() for name in ["query25.sh", "seed25.sh", "verify25.sh"]}}
|
|
obj = objects[-1]
|
|
obj["metadata"]["name"] = "node4-query25"
|
|
obj["metadata"]["labels"] = {"app": "node4-query25"}
|
|
obj["spec"]["activeDeadlineSeconds"] = 21600
|
|
for volume in obj["spec"]["volumes"]:
|
|
if volume["name"] == "lab":
|
|
volume["hostPath"]["path"] = "/var/lib/mongo-cpu-node4-query25-20261113"
|
|
if volume["name"] == "scripts":
|
|
volume["configMap"]["name"] = "node4-query25-scripts"
|
|
container = obj["spec"]["containers"][0]
|
|
container["command"] = ["bash", "/scripts/query25.sh"]
|
|
container["readinessProbe"] = {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27129", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}
|
|
return query_lab_profiles.render(objects, query_profile) if query25 else objects
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--kubeconfig", required=True)
|
|
ap.add_argument("--output", required=True)
|
|
ap.add_argument("--execute", action="store_true")
|
|
ap.add_argument("--confirm-plan")
|
|
ap.add_argument("--resume-namespace-uid")
|
|
ap.add_argument("--diagnose-only", action="store_true")
|
|
ap.add_argument("--query25", action="store_true")
|
|
ap.add_argument("--acceptance-file")
|
|
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
|
|
args = ap.parse_args()
|
|
out = pathlib.Path(args.output).resolve()
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
assert not (args.query25 and args.diagnose_only)
|
|
acceptance_sha = None
|
|
if args.query25:
|
|
assert args.acceptance_file, "passing small recovery evidence required"
|
|
evidence_bytes = pathlib.Path(args.acceptance_file).read_bytes()
|
|
evidence = json.loads(evidence_bytes)
|
|
assert evidence["result"] == "small-smoke-passed"
|
|
assert evidence["firstPodUID"] != evidence["secondPodUID"]
|
|
acceptance_sha = hashlib.sha256(evidence_bytes).hexdigest()
|
|
assert args.query25 or args.query_profile == "baseline"
|
|
objects = manifests(args.query25, args.query_profile)
|
|
target_pod = objects[-1]["metadata"]["name"]
|
|
runner_sha = hashlib.sha256(pathlib.Path(__file__).read_bytes()).hexdigest()
|
|
mode = "query25-synthetic-only" if args.query25 else "attribution-only" if args.diagnose_only else "full-synthetic-recovery"
|
|
digest = hashlib.sha256(json.dumps({"objects": objects, "runner": runner_sha, "mode": mode, "acceptance": acceptance_sha}, sort_keys=True).encode()).hexdigest()
|
|
(out / "objects.json").write_text(json.dumps(objects, indent=2))
|
|
(out / "plan.json").write_text(json.dumps({"sha256": digest, "node": NODE, "nodeUID": NODE_UID,
|
|
"namespace": NS, "image": IMAGE, "hostPath": objects[-1]["spec"]["volumes"][0]["hostPath"]["path"],
|
|
"bytes": 26843545600 if args.query25 else 536870912, "productionWrites": False, "mode": mode, "acceptanceSha256": acceptance_sha}, indent=2))
|
|
print("PLAN", digest, flush=True)
|
|
if not args.execute:
|
|
return
|
|
if args.confirm_plan != digest:
|
|
raise RuntimeError("plan confirmation mismatch")
|
|
k = ["kubectl", "--kubeconfig", args.kubeconfig]
|
|
|
|
def call(argv, timeout=30, payload=None):
|
|
p = subprocess.run(k + argv, input=payload, text=True, capture_output=True, timeout=timeout)
|
|
if p.returncode:
|
|
(out / f"failed-command-{time.time_ns()}.json").write_text(json.dumps({"argv": argv, "code": p.returncode, "stdout": p.stdout[:65536], "stderr": p.stderr[:65536]}, indent=2))
|
|
raise RuntimeError(f"kubectl {argv[:3]} failed: {p.stderr[:1000]}")
|
|
return p.stdout
|
|
|
|
def get(argv):
|
|
return json.loads(call(argv + ["-o", "json"]))
|
|
|
|
node = get(["get", "node", NODE])
|
|
assert node["metadata"]["uid"] == NODE_UID
|
|
assert get(["get", "namespace", "droboticflow-fanyu"])["metadata"]["uid"] == PROD_UID
|
|
assert any(IMAGE in x["names"] for x in node["status"].get("images", []))
|
|
for c in node["status"]["conditions"]:
|
|
if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"):
|
|
assert c["status"] == "False"
|
|
existing = subprocess.run(k + ["get", "namespace", NS, "--ignore-not-found", "-o", "name"], capture_output=True, text=True, check=True)
|
|
config_missing = False
|
|
if existing.stdout.strip():
|
|
existing_ns = get(["get", "namespace", NS])
|
|
assert args.resume_namespace_uid == existing_ns["metadata"]["uid"], "explicit recorded namespace UID required"
|
|
assert existing_ns["metadata"].get("labels", {}).get("purpose") == "mongo-cpu-synthetic"
|
|
assert not get(["get", "pods", "-n", NS])["items"], "stop and inspect existing pilot Pod before resuming"
|
|
config_name = objects[-2]["metadata"]["name"]
|
|
config_raw = call(["get", "configmap", config_name, "-n", NS, "--ignore-not-found", "-o", "json"])
|
|
if config_raw.strip():
|
|
assert json.loads(config_raw)["data"] == objects[-2]["data"]
|
|
else:
|
|
assert args.query25
|
|
config_missing = True
|
|
policy = get(["get", "networkpolicy", "deny-all", "-n", NS])["spec"]
|
|
assert policy["podSelector"] == {} and set(policy["policyTypes"]) == {"Ingress", "Egress"}
|
|
assert not policy.get("ingress") and not policy.get("egress")
|
|
assert get(["get", "resourcequota", "smoke-budget", "-n", NS])["spec"]["hard"] == objects[1]["spec"]["hard"]
|
|
else:
|
|
assert not args.resume_namespace_uid
|
|
protected = {}
|
|
for item in get(["get", "pods", "-A", "--field-selector", "spec.nodeName=" + NODE])["items"]:
|
|
if item["metadata"]["namespace"] == NS:
|
|
continue
|
|
uid = item["metadata"]["uid"]
|
|
assert re.fullmatch(r"[0-9a-f-]{36}", uid)
|
|
qos = item["status"]["qosClass"]
|
|
prefix = "/host/proc/1/root/sys/fs/cgroup/kubepods.slice/"
|
|
if qos == "Guaranteed":
|
|
path = prefix + "kubepods-pod" + uid.replace("-", "_") + ".slice"
|
|
else:
|
|
assert qos in ("Burstable", "BestEffort")
|
|
part = "kubepods-" + qos.lower()
|
|
path = prefix + part + ".slice/" + part + "-pod" + uid.replace("-", "_") + ".slice"
|
|
protected[uid] = path
|
|
protected_script = "\n".join("echo CGROUP " + uid + "; cat " + path + "/io.pressure" for uid, path in protected.items())
|
|
restarts = {}
|
|
stop = threading.Event()
|
|
monitor_failed = []
|
|
samples = []
|
|
|
|
def sample():
|
|
start = time.monotonic()
|
|
with urllib.request.urlopen("http://120.48.180.80/api/ready", timeout=5) as r:
|
|
assert r.status == 200
|
|
r.read(65536)
|
|
latency = time.monotonic() - start
|
|
pods = get(["get", "pods", "-n", "droboticflow-fanyu"])
|
|
current = {p["metadata"]["uid"]: sum(c.get("restartCount", 0) for c in p["status"].get("containerStatuses", [])) for p in pods["items"]}
|
|
if restarts:
|
|
assert current.keys() == restarts.keys(), "production Pod set changed"
|
|
assert all(current[uid] == count for uid, count in restarts.items()), "production restart observed"
|
|
else:
|
|
restarts.update(current)
|
|
host = call(["-n", "kube-system", "exec", "node-problem-detector-6v59v", "-c", "node-problem-detector", "--", "sh", "-c",
|
|
"set -e; cat /host/proc/diskstats | awk '$3==\"vda\"'; head -n 1 /host/proc/stat; cat /host/proc/pressure/io; stat -f --printf='AVAILABLE %a %S\\n' /host/proc/1/root; " + protected_script], timeout=15)
|
|
available = next(x for x in host.splitlines() if x.startswith("AVAILABLE ")).split()
|
|
assert int(available[1]) * int(available[2]) >= 80530636800, "node disk below 75Gi"
|
|
n = get(["get", "node", NODE])
|
|
assert all(c["status"] == "False" for c in n["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
|
|
per_pod = {}
|
|
current_uid = None
|
|
for line in host.splitlines():
|
|
if line.startswith("CGROUP "):
|
|
current_uid = line.split()[1]
|
|
elif current_uid and line.startswith("full "):
|
|
per_pod[current_uid] = float(line.split("avg10=")[1].split()[0])
|
|
assert per_pod.keys() == protected.keys(), "protected cgroup pressure sample missing"
|
|
record = {"time": time.time(), "readySeconds": latency, "host": host, "productionRestarts": current, "protectedIoFullAvg10": per_pod}
|
|
disk = [int(x) for x in next(x for x in host.splitlines() if x.split()[:3] == ["252", "0", "vda"]).split()[3:]]
|
|
record["diskCounters"] = disk
|
|
record["diskAwaitMs"] = 0.0
|
|
record["diskBusyPct"] = 0.0
|
|
if samples:
|
|
previous = samples[-1]
|
|
old = previous["diskCounters"]
|
|
ops = disk[0] + disk[4] - old[0] - old[4]
|
|
record["diskAwaitMs"] = (disk[3] + disk[7] - old[3] - old[7]) / max(1, ops)
|
|
record["diskBusyPct"] = (disk[9] - old[9]) / max(0.001, record["time"] - previous["time"]) / 10
|
|
samples.append(record)
|
|
with (out / "samples.jsonl").open("a") as f:
|
|
f.write(json.dumps(record) + "\n")
|
|
return latency
|
|
|
|
print("BASELINE sampling", flush=True)
|
|
baseline = []
|
|
for _ in range(6):
|
|
baseline.append(sample())
|
|
time.sleep(5)
|
|
latency_limit = max(1.0, max(baseline) * 2)
|
|
baseline_pressure = max(max(s["protectedIoFullAvg10"].values()) for s in samples)
|
|
baseline_await = max(s["diskAwaitMs"] for s in samples)
|
|
|
|
def monitor():
|
|
bad_latency = 0
|
|
bad_pressure = 0
|
|
while not stop.wait(5):
|
|
try:
|
|
latency = sample()
|
|
bad_latency = bad_latency + 1 if latency > latency_limit else 0
|
|
if bad_latency >= 3:
|
|
raise RuntimeError("readiness latency degraded for three samples")
|
|
# Global PSI includes the deliberately throttled pilot itself. Protect existing
|
|
# workloads with their own PSI plus actual whole-device latency/busy time.
|
|
latest = samples[-1]
|
|
pressure = max(latest["protectedIoFullAvg10"].values())
|
|
disk_degraded = latest["diskAwaitMs"] > max(10, baseline_await * 3) or latest["diskBusyPct"] > 70
|
|
bad_pressure = bad_pressure + 1 if pressure > max(0.5, baseline_pressure + 0.5) or disk_degraded else 0
|
|
if bad_pressure >= 3:
|
|
raise RuntimeError("existing-workload IO pressure or physical disk latency increased for three samples")
|
|
except Exception as e:
|
|
monitor_failed.append(str(e))
|
|
(out / "monitor-failure.json").write_text(json.dumps(monitor_failed))
|
|
subprocess.run(k + ["-n", NS, "delete", "pod", target_pod, "--ignore-not-found", "--wait=false"], capture_output=True, timeout=15)
|
|
return
|
|
|
|
def create(obj):
|
|
with (out / "intents.jsonl").open("a") as f:
|
|
f.write(json.dumps({"time": time.time(), "kind": obj["kind"], "name": obj["metadata"]["name"], "plan": digest}) + "\n")
|
|
print(call(["create", "-f", "-"], payload=json.dumps(obj)).strip(), flush=True)
|
|
|
|
def ready():
|
|
deadline = time.monotonic() + (900 if args.query25 else 180)
|
|
while time.monotonic() < deadline:
|
|
if monitor_failed:
|
|
raise RuntimeError(str(monitor_failed))
|
|
p = get(["get", "pod", target_pod, "-n", NS])
|
|
(out / "last-pod.json").write_text(json.dumps(p, indent=2))
|
|
if any(c["type"] == "Ready" and c["status"] == "True" for c in p["status"].get("conditions", [])):
|
|
return p["metadata"]["uid"]
|
|
if p["status"]["phase"] in ("Failed", "Succeeded"):
|
|
raise RuntimeError("pilot stopped before readiness")
|
|
time.sleep(3)
|
|
raise RuntimeError("pilot readiness timed out")
|
|
|
|
def probe(name, script, *extra, timeout=180):
|
|
print("PROBE", name, flush=True)
|
|
try:
|
|
result = call(["-n", NS, "exec", target_pod, "-c", "probe", "--", "bash", "/scripts/" + script, *extra], timeout=timeout)
|
|
except Exception:
|
|
# Preserve bounded logs/status; never retry an unknown write result.
|
|
raise
|
|
(out / (name + ".txt")).write_text(result)
|
|
print(result[-4000:], flush=True)
|
|
assert not monitor_failed
|
|
|
|
worker = None
|
|
retain_query = False
|
|
try:
|
|
if not existing.stdout.strip():
|
|
for obj in objects[:-1]:
|
|
create(obj)
|
|
elif config_missing:
|
|
create(objects[-2])
|
|
for _ in range(20):
|
|
quota = get(["get", "resourcequota", "smoke-budget", "-n", NS])
|
|
if quota.get("status", {}).get("hard"):
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise RuntimeError("quota not initialized")
|
|
worker = threading.Thread(target=monitor, daemon=True)
|
|
worker.start()
|
|
create(objects[-1])
|
|
first_uid = ready()
|
|
if args.query25:
|
|
probe("query25-seed", "seed25.sh", timeout=180)
|
|
call(["-n", NS, "delete", "pod", target_pod, "--wait=true", "--timeout=90s"], timeout=100)
|
|
create(objects[-1])
|
|
second_uid = ready()
|
|
assert first_uid != second_uid
|
|
probe("query25-after-rebuild", "verify25.sh", timeout=180)
|
|
for _ in range(6):
|
|
sample()
|
|
time.sleep(5)
|
|
assert not monitor_failed
|
|
(out / "result.json").write_text(json.dumps({"result": "query25-created-synthetic-only", "plan": digest, "acceptanceSha256": acceptance_sha, "firstPodUID": first_uid, "secondPodUID": second_uid, "activeDeadlineSeconds": 21600, "network": "loopback-only-no-Service", "limitations": ["single-node lab", "no production sync", "not capacity acceptance"]}, indent=2))
|
|
retain_query = True
|
|
print("QUERY25_CREATED_SYNTHETIC_ONLY", flush=True)
|
|
return
|
|
if args.diagnose_only:
|
|
probe("attribution", "attribution.sh", "assert", timeout=90)
|
|
(out / "result.json").write_text(json.dumps({"result": "attribution-captured-not-acceptance", "plan": digest, "podUID": first_uid}, indent=2))
|
|
return
|
|
probe("io-first", "io.sh", timeout=240)
|
|
probe("mongo-seed", "mongo.sh", "seed", timeout=180)
|
|
call(["-n", NS, "delete", "pod", target_pod, "--wait=true", "--timeout=90s"], timeout=100)
|
|
create(pod())
|
|
second_uid = ready()
|
|
assert second_uid != first_uid
|
|
probe("io-after-rebuild", "io.sh", timeout=240)
|
|
probe("mongo-after-rebuild", "mongo.sh", "verify", timeout=180)
|
|
probe("mongo-full-recovery", "mongo.sh", "full", timeout=700)
|
|
print("POST sampling", flush=True)
|
|
for _ in range(6):
|
|
sample()
|
|
time.sleep(5)
|
|
assert not monitor_failed
|
|
(out / "result.json").write_text(json.dumps({"result": "small-smoke-passed", "plan": digest,
|
|
"firstPodUID": first_uid, "secondPodUID": second_uid,
|
|
"sampleCount": len(samples), "limitations": ["short observation window", "no authenticated business workflow test", "not 25Gi or capacity acceptance"]}, indent=2))
|
|
print("SMALL_SMOKE_PASSED", flush=True)
|
|
except BaseException as e:
|
|
(out / "result.json").write_text(json.dumps({"result": "failed", "plan": digest, "reason": str(e)}, indent=2))
|
|
raise
|
|
finally:
|
|
stop.set()
|
|
if worker:
|
|
worker.join(timeout=30)
|
|
try:
|
|
logs = call(["-n", NS, "logs", target_pod, "-c", "isolate", "--tail=60"])
|
|
(out / "init.log").write_text(logs)
|
|
except Exception:
|
|
pass
|
|
# Stop only this synthetic Pod; retain its backing image and exact mount for review.
|
|
if not retain_query:
|
|
subprocess.run(k + ["-n", NS, "delete", "pod", target_pod, "--ignore-not-found", "--wait=true", "--timeout=90s"], capture_output=True, timeout=100)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|