87 lines
4.4 KiB
Python
87 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only bounded Kubernetes sampling. Pod network counters are NOT EIP traffic."""
|
|
import argparse
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from urllib.parse import quote
|
|
|
|
METRICS = {
|
|
"container_cpu_usage_seconds_total", "container_cpu_cfs_periods_total",
|
|
"container_cpu_cfs_throttled_periods_total", "container_cpu_cfs_throttled_seconds_total",
|
|
"container_memory_working_set_bytes", "container_oom_events_total",
|
|
}
|
|
LINE = re.compile(r'^(\w+)\{([^}]*)\}\s+([^\s]+)(?:\s+\d+)?$')
|
|
LABEL = re.compile(r'(\w+)="((?:\\.|[^"\\])*)"')
|
|
|
|
|
|
def parse_metrics(body, namespace, pods):
|
|
rows = {}
|
|
for line in body.splitlines():
|
|
match = LINE.match(line)
|
|
if not match or match[1] not in METRICS:
|
|
continue
|
|
labels = {k: json.loads('"' + v + '"') for k, v in LABEL.findall(match[2])}
|
|
pod, container = labels.get("pod", ""), labels.get("container", "")
|
|
if labels.get("namespace") != namespace or pod not in pods or container in ("", "POD"):
|
|
continue
|
|
value = float(match[3]) # The optional trailing field is a scrape timestamp.
|
|
if not math.isfinite(value):
|
|
continue
|
|
rows.setdefault((pod, container), {})[match[1]] = value
|
|
return [{"pod": p, "container": c, **values} for (p, c), values in sorted(rows.items())]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--kubeconfig", type=Path, required=True)
|
|
parser.add_argument("--context", required=True)
|
|
parser.add_argument("--namespace", required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--interval", type=int, default=30)
|
|
parser.add_argument("--samples", type=int, default=3)
|
|
args = parser.parse_args()
|
|
if not (15 <= args.interval <= 60 and 2 <= args.samples <= 1921):
|
|
parser.error("interval 15..60 seconds; samples 2..1921")
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
base = ["kubectl", "--kubeconfig", str(args.kubeconfig), "--context", args.context, "--request-timeout=15s"]
|
|
|
|
def get(*parts):
|
|
result = subprocess.run([*base, "get", *parts], capture_output=True, text=True, timeout=20)
|
|
if result.returncode:
|
|
# kube errors can contain endpoints/tokens; do not copy stderr into evidence.
|
|
raise RuntimeError("read-only Kubernetes request failed")
|
|
return result.stdout
|
|
|
|
with args.output.open("x", encoding="utf-8") as output:
|
|
for index in range(args.samples):
|
|
start = time.monotonic()
|
|
pods = json.loads(get("pods", "-n", args.namespace, "-o", "json"))["items"]
|
|
selected = [p for p in pods if p["metadata"].get("labels", {}).get("app", "") in {
|
|
"droboticflow-api", "droboticflow-worker", "drosync-coordinator",
|
|
"droboticflow-annotation-processor", "droboticflow-annotation-validator"}]
|
|
names = {p["metadata"]["name"] for p in selected}
|
|
rows, unavailable = [], []
|
|
for node in sorted({p["spec"].get("nodeName", "") for p in selected} - {""}):
|
|
try:
|
|
parsed = parse_metrics(get("--raw", f"/api/v1/nodes/{quote(node, safe='')}/proxy/metrics/cadvisor"), args.namespace, names)
|
|
rows.extend({"node": node, **row} for row in parsed)
|
|
except (RuntimeError, subprocess.TimeoutExpired):
|
|
unavailable.append(node)
|
|
status = [{"pod": p["metadata"]["name"], "node": p["spec"].get("nodeName"),
|
|
"containers": [{"name": c["name"], "resources": c.get("resources", {})} for c in p["spec"]["containers"]],
|
|
"restarts": sum(c.get("restartCount", 0) for c in p.get("status", {}).get("containerStatuses", [])),
|
|
"ready": all(c.get("ready", False) for c in p.get("status", {}).get("containerStatuses", []))} for p in selected]
|
|
output.write(json.dumps({"at": time.time(), "namespace": args.namespace, "pods": status, "metrics": rows,
|
|
"unavailable_nodes": unavailable, "eip_mbps": None, "mongo_wait_ms": None, "queue_oldest_seconds": None}) + "\n")
|
|
output.flush()
|
|
if index + 1 < args.samples:
|
|
time.sleep(max(0, args.interval - (time.monotonic() - start)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|