droboticflow/scripts/local/query-replica-cluster-fixture.py
2026-11-13 22:31:44 -08:00

224 lines
18 KiB
Python

#!/usr/bin/env python3
"""Create only the bounded node4 synthetic source/HTTP fixture, after preflight."""
import argparse
import hashlib
import gzip
import ipaddress
import json
from pathlib import Path
import subprocess
import uuid
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
import query_lab_profiles
NODE = "10.189.16.4"
NODE_UID = "ca3f254a-8452-4769-8929-6ec303905dbe"
NS = "droboticflow-query-lab"
NS_UID = "07d647bb-943c-472c-be0a-f6b1380d7105"
QUERY_UID = "19f5f918-9bb1-4e6a-a362-bb70afacee26"
IMAGE = "docker.io/library/mongo@sha256:02a0cc7939f5ed38f30f9bc714ef5f682d49baf9350c54acf302ce833087fe8a"
ROOT = Path(__file__).resolve().parents[2]
HARD = {"pods": "2", "requests.cpu": "2", "limits.cpu": "5", "requests.memory": "3Gi", "limits.memory": "5Gi", "services": "0", "persistentvolumeclaims": "0"}
def manifest(source_node=NODE, cache=None):
env = [{"name": name, "valueFrom": {"fieldRef": {"fieldPath": field}}} for name, field in (("POD_UID", "metadata.uid"), ("POD_NAME", "metadata.name"), ("NODE_NAME", "spec.nodeName"))]
env += [{"name": "HOME", "value": "/tmp"}, {"name": "MONGOSH_DISABLE_TELEMETRY", "value": "1"}, {"name": "GOMAXPROCS", "value": "2"}, {"name": "GOMEMLIMIT", "value": "1536MiB"}]
command = """set -euo pipefail
mkdir -p /synthetic/source /synthetic/tmp
mongod --dbpath /synthetic/source --port 27119 --bind_ip 127.0.0.1 --wiredTigerCacheSizeGB 1 --pidfilepath /synthetic/mongod.pid --logpath /synthetic/source.log --setParameter diagnosticDataCollectionEnabled=false &
pid=$!
stop() { kill -TERM "$pid" 2>/dev/null || true; wait "$pid" || true; }
trap stop EXIT
trap 'exit 0' TERM INT
while kill -0 "$pid" 2>/dev/null; do sleep 10 & wait $!; done
wait "$pid"
"""
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "node4-query-fixture", "namespace": NS, "labels": {"app": "node4-query-fixture", "purpose": "mongo-cpu-synthetic"}}, "spec": {"nodeName": source_node, "automountServiceAccountToken": False, "enableServiceLinks": False, "restartPolicy": "Never", "activeDeadlineSeconds": 7200, "terminationGracePeriodSeconds": 45, "securityContext": {"runAsUser": 999, "runAsGroup": 999, "runAsNonRoot": True, "fsGroup": 999}, "containers": [{"name": "lab", "image": IMAGE, "imagePullPolicy": "Never", "command": ["bash", "-c", command], "env": env, "resources": {"requests": {"cpu": "1", "memory": "2Gi", "ephemeral-storage": "32Mi"}, "limits": {"cpu": "4", "memory": "4Gi", "ephemeral-storage": "128Mi"}}, "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, "readinessProbe": {"exec": {"command": ["mongosh", "--quiet", "--host", "127.0.0.1", "--port", "27119", "--eval", "quit(db.adminCommand({ping:1}).ok?0:1)"]}, "periodSeconds": 15, "timeoutSeconds": 5}, "volumeMounts": [{"name": "synthetic", "mountPath": "/synthetic"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "synthetic", "emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}}, {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": "64Mi"}}]}}
ingress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-ingress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query25"}}, "policyTypes": ["Ingress"], "ingress": [{"from": [{"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
egress = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "query-fixture-egress", "namespace": NS}, "spec": {"podSelector": {"matchLabels": {"app": "node4-query-fixture"}}, "policyTypes": ["Egress"], "egress": [{"to": [{"podSelector": {"matchLabels": {"app": "node4-query25"}}}], "ports": [{"protocol": "TCP", "port": 27130}]}]}}
if source_node != NODE:
pod["spec"]["containers"][0]["imagePullPolicy"] = "IfNotPresent"
policies = [ingress, egress]
if cache:
policies.append({"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"query-fixture-cache-egress","namespace":NS},"spec":{"podSelector":{"matchLabels":{"app":"node4-query-fixture"}},"policyTypes":["Egress"],"egress":[{"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":cache["namespace"]}},"podSelector":{"matchLabels":{"app":cache["pod"]}}}],"ports":[{"protocol":"TCP","port":6379}]}]}})
return policies + [pod]
def verify_query_pod(query, expected_uid, query_profile="baseline"):
# Rebuilds change UID. Bind that explicitly into the reviewed plan, while
# preserving fixed namespace/node/image/storage/resource identity checks.
assert str(uuid.UUID(expected_uid)) == expected_uid
assert query["metadata"]["uid"] == expected_uid
assert query["metadata"]["name"] == "node4-query25" and query["metadata"]["namespace"] == NS
assert query["metadata"].get("labels", {}).get("app") == "node4-query25"
assert query["status"]["phase"] == "Running"
assert query["spec"]["nodeName"] == NODE
container = next(c for c in query["spec"]["containers"] if c["name"] == "probe")
assert container["image"] == IMAGE
for kind in ("requests", "limits"):
resources = container["resources"][kind]
assert all(resources.get(k) == v for k, v in query_lab_profiles.resources(query_profile).items())
# CCE admission injects one private-IP resource into both maps.
assert set(resources) <= {"cpu", "memory", "cce.baidubce.com/ip"}
assert resources.get("cce.baidubce.com/ip", "1") == "1"
query_lab_profiles.verify_resources(query, query_profile)
volumes = {v["name"]: v for v in query["spec"]["volumes"]}
assert volumes["lab"]["hostPath"]["path"] == "/var/lib/mongo-cpu-node4-query25-20261113"
assert volumes["scripts"]["configMap"]["name"] == "node4-query25-scripts"
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--kubeconfig", required=True)
ap.add_argument("--output", required=True)
ap.add_argument("--api-binary", required=True)
ap.add_argument("--bridge-binary", required=True)
ap.add_argument("--execute", action="store_true")
ap.add_argument("--finish-existing", action="store_true", help="resume this recorded partial fixture without replacing resources")
ap.add_argument("--update-api-test", action="store_true", help="stage a new content-addressed test binary after its previous run ended")
ap.add_argument("--confirm-plan")
ap.add_argument("--query-pod-uid", default=QUERY_UID, help="exact inspected UID after an authorized synthetic Pod rebuild")
ap.add_argument("--query-profile", choices=query_lab_profiles.PROFILES, default="baseline")
ap.add_argument("--source-node", choices=(NODE,"10.189.16.33"), default=NODE)
ap.add_argument("--valkey-owned")
args = ap.parse_args()
global HARD
HARD = query_lab_profiles.quota(args.query_profile, fixture=True)
query_uid = args.query_pod_uid
out = Path(args.output).resolve()
out.mkdir(parents=True, exist_ok=True)
k = ["kubectl", "--kubeconfig", str(Path(args.kubeconfig).resolve())]
def call(*argv, data=None, timeout=30):
p = subprocess.run(k + list(argv), input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
if p.returncode:
raise RuntimeError(p.stderr.decode()[:2000])
return p.stdout.decode().strip()
def get(*argv):
return json.loads(call(*argv, "-o", "json"))
cache = None
if args.valkey_owned:
cache = json.loads(Path(args.valkey_owned).read_text())
assert cache["namespace"] == "droboticflow-query-cache-lab" and cache["pod"] == "query-cache-synthetic"
assert get("get","namespace",cache["namespace"])["metadata"]["uid"] == cache["namespaceUID"]
cp = get("-n",cache["namespace"],"get","pod",cache["pod"])
assert cp["metadata"]["uid"] == cache["podUID"] and cp["spec"]["nodeName"] == "10.189.16.33" and cp["status"]["phase"] == "Running"
assert cp["spec"]["containers"][0]["image"] == cache["image"]
assert cache["url"] == "redis://"+cp["status"]["podIP"]+":6379/0"
if args.source_node != NODE:
other = get("get","node",args.source_node)
assert other["metadata"]["uid"] == "77747f1c-42d8-40e8-a8d0-91f0e56ba3bd"
assert all(c["status"]=="False" for c in other["status"]["conditions"] if c["type"] in ("DiskPressure","MemoryPressure","PIDPressure"))
usage=call("top","node",args.source_node,"--no-headers").split()
assert float(usage[2].rstrip("%"))<25 and float(usage[4].rstrip("%"))<60
node = get("get", "node", NODE)
assert node["metadata"]["uid"] == NODE_UID
assert all(c["status"] == "False" for c in node["status"]["conditions"] if c["type"] in ("DiskPressure", "MemoryPressure", "PIDPressure"))
top = call("top", "node", NODE, "--no-headers").split()
assert float(top[2].rstrip("%")) <= 25 and float(top[4].rstrip("%")) <= 60, "insufficient idle headroom for synthetic fixture"
assert get("get", "namespace", NS)["metadata"]["uid"] == NS_UID
query = get("-n", NS, "get", "pod", "node4-query25")
verify_query_pod(query, query_uid, args.query_profile)
query_lab_profiles.verify_scripts(get("-n", NS, "get", "configmap", "node4-query25-scripts")["data"], args.query_profile)
call("-n", NS, "exec", "node4-query25", "-c", "probe", "--", "bash", "/scripts/verify25.sh", timeout=90)
ip = query["status"]["podIP"]
assert ipaddress.IPv4Address(ip) in ipaddress.IPv4Network("10.189.16.0/24")
quota = get("-n", NS, "get", "resourcequota", "smoke-budget")
original_quota = query_lab_profiles.quota(args.query_profile)
recorded = None
if args.finish_existing:
recorded = json.loads((out / "created.json").read_text())
previous = json.loads((out / "plan.json").read_text())
previous_digest = hashlib.sha256(json.dumps(previous, sort_keys=True).encode()).hexdigest()
assert recorded["planSHA256"] == previous_digest
assert quota["spec"]["hard"] == HARD
existing = get("-n", NS, "get", "pod", "node4-query-fixture")
assert existing["metadata"]["uid"] == recorded["fixtureUID"] and existing["spec"]["nodeName"] == args.source_node
assert existing["spec"]["containers"][0]["resources"]["limits"]["cpu"] == "4"
assert existing["spec"]["containers"][0]["resources"]["limits"]["memory"] == "4Gi"
for obj in manifest(args.source_node, cache)[:-1]:
assert get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["spec"] == obj["spec"]
else:
assert quota["spec"]["hard"] == original_quota
for kind, name in (("pod", "node4-query-fixture"), ("networkpolicy", "query-fixture-ingress"), ("networkpolicy", "query-fixture-egress")):
assert not call("-n", NS, "get", kind, name, "--ignore-not-found", "-o", "name"), "existing object must be inspected, never replaced"
binaries = {"api": Path(args.api_binary).resolve(), "bridge": Path(args.bridge_binary).resolve()}
binary_root = (ROOT / "local-data/diagnostics/mongo-cpu/query25-http/bin").resolve()
for path in binaries.values():
assert path.parent == binary_root, "only this task's generated binaries may be copied"
header = path.read_bytes()[:20]
assert header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00", "Linux amd64 ELF required"
hashes = {name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in binaries.items()}
if recorded:
assert hashes["bridge"] == previous["binarySHA256"]["bridge"], "resume must keep the verified bridge"
if not args.update_api_test:
assert hashes == previous["binarySHA256"], "resume must use the recorded binaries"
else:
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", 'for p in /proc/[0-9]*/exe; do x=$(readlink "$p" 2>/dev/null || true); case "$x" in /synthetic/query-http*.test) exit 1;; esac; done')
objects = manifest(args.source_node, cache)
plan = {"kind": "synthetic-in-cluster-comparison", "sourceNode": args.source_node, "valkey": cache, "node": NODE, "namespaceUID": NS_UID, "queryUID": query_uid, "queryIP": ip, "quotaVersion": quota["metadata"]["resourceVersion"], "originalQuota": original_quota, "temporaryQuota": HARD, "objects": objects, "binarySHA256": hashes, "customerData": "none", "newDisk": "none; source is bounded tmpfs", "productionChanges": "none", "queryBridgeStorage": "bounded /tmp memory volume; database noexec retained", "stop": "three degraded readiness/IO/CPU samples; delete only owned fixture", "rollback": "stop owned bridges, delete fixture and its two policies, CAS restore original quota; keep 25Gi data"}
if recorded:
plan["resumeFrom"] = recorded
raw = json.dumps(plan, sort_keys=True).encode()
digest = hashlib.sha256(raw).hexdigest()
if args.execute:
assert args.confirm_plan == digest, "preflight identity changed; inspect a freshly rendered plan"
(out / ("finish-plan.json" if recorded else "plan.json")).write_text(json.dumps(plan, indent=2))
(out / "objects.json").write_text(json.dumps({"apiVersion": "v1", "kind": "List", "items": objects}, indent=2))
print(json.dumps({"planSHA256": digest, "fixture": "node4-query-fixture", "maxCPU": 4, "maxMemory": "4Gi", "sourceStorage": "tmpfs <=2Gi", "queryStorageAndLimits": "unchanged"}), flush=True)
if not args.execute:
return
# No application release, production credential, PVC, Service or host mount.
if not recorded:
patch = [{"op": "test", "path": "/metadata/resourceVersion", "value": plan["quotaVersion"]}, {"op": "test", "path": "/spec/hard", "value": plan["originalQuota"]}, {"op": "replace", "path": "/spec/hard", "value": HARD}]
(out / "intent.json").write_text(json.dumps({"planSHA256": digest, "action": "temporary synthetic quota then server dry-run", "queryUID": query_uid}))
call("-n", NS, "patch", "resourcequota", "smoke-budget", "--type=json", "-p", json.dumps(patch))
# Admission must see the temporary quota; the original budget is full.
call("create", "--dry-run=server", "-f", str(out / "objects.json"))
call("create", "-f", str(out / "objects.json"))
created = get("-n", NS, "get", "pod", "node4-query-fixture")
(out / "created.json").write_text(json.dumps({"fixtureUID": created["metadata"]["uid"], "planSHA256": digest}))
call("-n", NS, "wait", "--for=condition=Ready", "pod/node4-query-fixture", "--timeout=180s", timeout=190)
fixture = get("-n", NS, "get", "pod", "node4-query-fixture")
assert fixture["spec"]["nodeName"] == args.source_node
fuid = fixture["metadata"]["uid"]
assert str(uuid.UUID(fuid)) == fuid
assert get("-n", NS, "get", "pod", "node4-query25")["metadata"]["uid"] == query_uid
qdir = "/tmp/query-http-lab/" + hashes["bridge"][:16]
def copy(pod, container, local, remote):
exists = subprocess.run(k + ["-n", NS, "exec", pod, "-c", container, "--", "test", "-f", remote], capture_output=True, timeout=30).returncode == 0
if exists:
assert call("-n", NS, "exec", pod, "-c", container, "--", "sha256sum", remote).split()[0] == hashlib.sha256(local.read_bytes()).hexdigest(), "existing fixture artifact differs"
return
call("-n", NS, "exec", pod, "-c", container, "--", "mkdir", "-p", str(Path(remote).parent))
command = "set -eu; umask 077; test ! -e " + remote + "; gzip -dc > " + remote + "; chmod 700 " + remote
p = subprocess.run(k + ["-n", NS, "exec", "-i", pod, "-c", container, "--", "sh", "-c", command], input=gzip.compress(local.read_bytes(), compresslevel=1), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
if p.returncode:
raise RuntimeError("owned fixture binary transfer failed")
actual = call("-n", NS, "exec", pod, "-c", container, "--", "sha256sum", remote).split()[0]
assert actual == hashlib.sha256(local.read_bytes()).hexdigest()
copy("node4-query25", "probe", binaries["bridge"], qdir + "/bridge")
copy("node4-query-fixture", "lab", binaries["bridge"], "/synthetic/bridge")
api_path = "/synthetic/query-http-" + hashes["api"][:16] + ".test"
copy("node4-query-fixture", "lab", binaries["api"], api_path)
# Fixed, preflight-derived private addresses and UID arguments only.
qcommand = "set -eu; if test -s " + qdir + "/bridge.pid; then p=$(cat " + qdir + "/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = " + qdir + "/bridge; exit 0; fi; fi; nohup " + qdir + "/bridge -listen " + ip + ":27130 -upstream 127.0.0.1:27129 -pod-uid " + query_uid + " >" + qdir + "/bridge.log 2>&1 </dev/null & echo $! >" + qdir + "/bridge.pid"
call("-n", NS, "exec", "node4-query25", "-c", "probe", "--", "sh", "-c", qcommand)
fcommand = "set -eu; if test -s /synthetic/bridge.pid; then p=$(cat /synthetic/bridge.pid); if kill -0 \"$p\" 2>/dev/null; then test \"$(readlink /proc/$p/exe)\" = /synthetic/bridge; exit 0; fi; fi; nohup /synthetic/bridge -listen 127.0.0.1:27229 -upstream " + ip + ":27130 -pod-uid " + fuid + " >/synthetic/bridge.log 2>&1 </dev/null & echo $! >/synthetic/bridge.pid"
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "sh", "-c", fcommand)
call("-n", NS, "exec", "node4-query-fixture", "-c", "lab", "--", "mongosh", "--quiet", "mongodb://127.0.0.1:27229/?directConnection=true&replicaSet=query-lab-rs", "--eval", 'const m=db.getSiblingDB("query25_synthetic").marker.findOne({_id:"query25-confirmed"});quit(m&&m.value==="preserve-query25-v1"?0:2)')
owned = {"fixtureUID": fuid, "queryUID": query_uid, "sourceNode": args.source_node, "valkey": cache, "queryBridgeDirectory": qdir, "apiBinaryPath": api_path, "policies": {obj["metadata"]["name"]: get("-n", NS, "get", "networkpolicy", obj["metadata"]["name"])["metadata"]["uid"] for obj in objects[:-1]}, "binarySHA256": hashes, "planSHA256": digest}
(out / "owned.json").write_text(json.dumps(owned, indent=2))
print("ISOLATED_FIXTURE_READY", flush=True)
if __name__ == "__main__":
main()