droboticflow/scripts/local/sample-capacity-resources.py

87 lines
4.4 KiB
Python

#!/usr/bin/env python3
"""Read-only bounded Kubernetes sampling. Pod network counters are NOT EIP traffic."""
import argparse
import json
import math
from pathlib import Path
import re
import subprocess
import time
from urllib.parse import quote
METRICS = {
"container_cpu_usage_seconds_total", "container_cpu_cfs_periods_total",
"container_cpu_cfs_throttled_periods_total", "container_cpu_cfs_throttled_seconds_total",
"container_memory_working_set_bytes", "container_oom_events_total",
}
LINE = re.compile(r'^(\w+)\{([^}]*)\}\s+([^\s]+)(?:\s+\d+)?$')
LABEL = re.compile(r'(\w+)="((?:\\.|[^"\\])*)"')
def parse_metrics(body, namespace, pods):
rows = {}
for line in body.splitlines():
match = LINE.match(line)
if not match or match[1] not in METRICS:
continue
labels = {k: json.loads('"' + v + '"') for k, v in LABEL.findall(match[2])}
pod, container = labels.get("pod", ""), labels.get("container", "")
if labels.get("namespace") != namespace or pod not in pods or container in ("", "POD"):
continue
value = float(match[3]) # The optional trailing field is a scrape timestamp.
if not math.isfinite(value):
continue
rows.setdefault((pod, container), {})[match[1]] = value
return [{"pod": p, "container": c, **values} for (p, c), values in sorted(rows.items())]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--kubeconfig", type=Path, required=True)
parser.add_argument("--context", required=True)
parser.add_argument("--namespace", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--interval", type=int, default=30)
parser.add_argument("--samples", type=int, default=3)
args = parser.parse_args()
if not (15 <= args.interval <= 60 and 2 <= args.samples <= 1921):
parser.error("interval 15..60 seconds; samples 2..1921")
args.output.parent.mkdir(parents=True, exist_ok=True)
base = ["kubectl", "--kubeconfig", str(args.kubeconfig), "--context", args.context, "--request-timeout=15s"]
def get(*parts):
result = subprocess.run([*base, "get", *parts], capture_output=True, text=True, timeout=20)
if result.returncode:
# kube errors can contain endpoints/tokens; do not copy stderr into evidence.
raise RuntimeError("read-only Kubernetes request failed")
return result.stdout
with args.output.open("x", encoding="utf-8") as output:
for index in range(args.samples):
start = time.monotonic()
pods = json.loads(get("pods", "-n", args.namespace, "-o", "json"))["items"]
selected = [p for p in pods if p["metadata"].get("labels", {}).get("app", "") in {
"droboticflow-api", "droboticflow-worker", "drosync-coordinator",
"droboticflow-annotation-processor", "droboticflow-annotation-validator"}]
names = {p["metadata"]["name"] for p in selected}
rows, unavailable = [], []
for node in sorted({p["spec"].get("nodeName", "") for p in selected} - {""}):
try:
parsed = parse_metrics(get("--raw", f"/api/v1/nodes/{quote(node, safe='')}/proxy/metrics/cadvisor"), args.namespace, names)
rows.extend({"node": node, **row} for row in parsed)
except (RuntimeError, subprocess.TimeoutExpired):
unavailable.append(node)
status = [{"pod": p["metadata"]["name"], "node": p["spec"].get("nodeName"),
"containers": [{"name": c["name"], "resources": c.get("resources", {})} for c in p["spec"]["containers"]],
"restarts": sum(c.get("restartCount", 0) for c in p.get("status", {}).get("containerStatuses", [])),
"ready": all(c.get("ready", False) for c in p.get("status", {}).get("containerStatuses", []))} for p in selected]
output.write(json.dumps({"at": time.time(), "namespace": args.namespace, "pods": status, "metrics": rows,
"unavailable_nodes": unavailable, "eip_mbps": None, "mongo_wait_ms": None, "queue_oldest_seconds": None}) + "\n")
output.flush()
if index + 1 < args.samples:
time.sleep(max(0, args.interval - (time.monotonic() - start)))
if __name__ == "__main__":
main()