#!/usr/bin/env python3 """Read-only bounded Kubernetes sampling. Pod network counters are NOT EIP traffic.""" import argparse import json import math from pathlib import Path import re import subprocess import time from urllib.parse import quote METRICS = { "container_cpu_usage_seconds_total", "container_cpu_cfs_periods_total", "container_cpu_cfs_throttled_periods_total", "container_cpu_cfs_throttled_seconds_total", "container_memory_working_set_bytes", "container_oom_events_total", } LINE = re.compile(r'^(\w+)\{([^}]*)\}\s+([^\s]+)(?:\s+\d+)?$') LABEL = re.compile(r'(\w+)="((?:\\.|[^"\\])*)"') def parse_metrics(body, namespace, pods): rows = {} for line in body.splitlines(): match = LINE.match(line) if not match or match[1] not in METRICS: continue labels = {k: json.loads('"' + v + '"') for k, v in LABEL.findall(match[2])} pod, container = labels.get("pod", ""), labels.get("container", "") if labels.get("namespace") != namespace or pod not in pods or container in ("", "POD"): continue value = float(match[3]) # The optional trailing field is a scrape timestamp. if not math.isfinite(value): continue rows.setdefault((pod, container), {})[match[1]] = value return [{"pod": p, "container": c, **values} for (p, c), values in sorted(rows.items())] def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--kubeconfig", type=Path, required=True) parser.add_argument("--context", required=True) parser.add_argument("--namespace", required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--interval", type=int, default=30) parser.add_argument("--samples", type=int, default=3) args = parser.parse_args() if not (15 <= args.interval <= 60 and 2 <= args.samples <= 1921): parser.error("interval 15..60 seconds; samples 2..1921") args.output.parent.mkdir(parents=True, exist_ok=True) base = ["kubectl", "--kubeconfig", str(args.kubeconfig), "--context", args.context, "--request-timeout=15s"] def get(*parts): result = subprocess.run([*base, "get", *parts], capture_output=True, text=True, timeout=20) if result.returncode: # kube errors can contain endpoints/tokens; do not copy stderr into evidence. raise RuntimeError("read-only Kubernetes request failed") return result.stdout with args.output.open("x", encoding="utf-8") as output: for index in range(args.samples): start = time.monotonic() pods = json.loads(get("pods", "-n", args.namespace, "-o", "json"))["items"] selected = [p for p in pods if p["metadata"].get("labels", {}).get("app", "") in { "droboticflow-api", "droboticflow-worker", "drosync-coordinator", "droboticflow-annotation-processor", "droboticflow-annotation-validator"}] names = {p["metadata"]["name"] for p in selected} rows, unavailable = [], [] for node in sorted({p["spec"].get("nodeName", "") for p in selected} - {""}): try: parsed = parse_metrics(get("--raw", f"/api/v1/nodes/{quote(node, safe='')}/proxy/metrics/cadvisor"), args.namespace, names) rows.extend({"node": node, **row} for row in parsed) except (RuntimeError, subprocess.TimeoutExpired): unavailable.append(node) status = [{"pod": p["metadata"]["name"], "node": p["spec"].get("nodeName"), "containers": [{"name": c["name"], "resources": c.get("resources", {})} for c in p["spec"]["containers"]], "restarts": sum(c.get("restartCount", 0) for c in p.get("status", {}).get("containerStatuses", [])), "ready": all(c.get("ready", False) for c in p.get("status", {}).get("containerStatuses", []))} for p in selected] output.write(json.dumps({"at": time.time(), "namespace": args.namespace, "pods": status, "metrics": rows, "unavailable_nodes": unavailable, "eip_mbps": None, "mongo_wait_ms": None, "queue_oldest_seconds": None}) + "\n") output.flush() if index + 1 < args.samples: time.sleep(max(0, args.interval - (time.monotonic() - start))) if __name__ == "__main__": main()