213 lines
7.5 KiB
Python

from __future__ import annotations
import datetime as dt
import os
import resource
import shutil
import subprocess
import sys
import threading
import time
from typing import Any
def resource_snapshot() -> tuple[float, int]:
usage = resource.getrusage(resource.RUSAGE_SELF)
cpu_seconds = usage.ru_utime + usage.ru_stime
if sys.platform == "darwin":
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
else:
memory_mib = max(1, int(usage.ru_maxrss / 1024))
return cpu_seconds, memory_mib
def parse_gpu_snapshot_output(output: str) -> dict[str, Any]:
samples: list[dict[str, Any]] = []
for line in output.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 4:
continue
try:
samples.append(
{
"gpu": int(parts[0]),
"gpuMemoryUsedMiB": int(parts[1]),
"gpuMemoryTotalMiB": int(parts[2]),
"gpuName": parts[3],
}
)
except ValueError:
continue
if not samples:
return {"gpu": 0}
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
return {
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
"gpuName": ", ".join(names),
"gpuCount": len(samples),
}
def gpu_snapshot() -> dict[str, Any]:
smi_snapshot = nvidia_smi_gpu_snapshot()
if smi_snapshot != {"gpu": 0}:
return smi_snapshot
return nvml_gpu_snapshot()
def nvidia_smi_gpu_snapshot() -> dict[str, Any]:
nvidia_smi = shutil.which("nvidia-smi")
if not nvidia_smi:
return {"gpu": 0}
try:
completed = subprocess.run(
[
nvidia_smi,
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
"--format=csv,noheader,nounits",
],
check=False,
capture_output=True,
text=True,
timeout=2,
)
except (OSError, subprocess.SubprocessError):
return {"gpu": 0}
if completed.returncode != 0:
return {"gpu": 0}
return parse_gpu_snapshot_output(completed.stdout)
def nvml_gpu_snapshot() -> dict[str, Any]:
try:
import pynvml # type: ignore
except Exception:
return {"gpu": 0}
try:
pynvml.nvmlInit()
count = pynvml.nvmlDeviceGetCount()
samples: list[dict[str, Any]] = []
for index in range(count):
handle = pynvml.nvmlDeviceGetHandleByIndex(index)
memory = pynvml.nvmlDeviceGetMemoryInfo(handle)
utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
name = pynvml.nvmlDeviceGetName(handle)
if isinstance(name, bytes):
name = name.decode("utf-8", errors="replace")
samples.append(
{
"gpu": int(getattr(utilization, "gpu", 0)),
"gpuMemoryUsedMiB": int(getattr(memory, "used", 0) / (1024 * 1024)),
"gpuMemoryTotalMiB": int(getattr(memory, "total", 0) / (1024 * 1024)),
"gpuName": str(name),
}
)
except Exception:
return {"gpu": 0}
finally:
try:
pynvml.nvmlShutdown()
except Exception:
pass
if not samples:
return {"gpu": 0}
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
return {
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
"gpuName": ", ".join(names),
"gpuCount": len(samples),
}
def resource_series(start_cpu: float, start_memory: int, started_at: str, ended_at: str) -> list[dict[str, Any]]:
return resource_series_with_samples(start_cpu, start_memory, started_at, ended_at, [])
def resource_series_with_samples(
start_cpu: float,
start_memory: int,
started_at: str,
ended_at: str,
samples: list[dict[str, Any]],
) -> list[dict[str, Any]]:
end_cpu, end_memory = resource_snapshot()
gpu_metrics = gpu_snapshot()
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
elapsed = max((ended - started).total_seconds(), 0.001)
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
series = [
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
*samples,
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, **gpu_metrics},
]
return compact_resource_series(series)
def compact_resource_series(series: list[dict[str, Any]]) -> list[dict[str, Any]]:
compacted: list[dict[str, Any]] = []
seen_timestamps: set[str] = set()
for item in series:
timestamp = str(item.get("timestamp") or "").strip()
if not timestamp or timestamp in seen_timestamps:
continue
seen_timestamps.add(timestamp)
compacted.append(item)
return compacted
def resource_sample(start_cpu: float, started_at: str) -> dict[str, Any]:
current_cpu, current_memory = resource_snapshot()
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
timestamp = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
now = dt.datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
elapsed = max((now - started).total_seconds(), 0.001)
cpu_percent = max(0, min(100, int(((current_cpu - start_cpu) / elapsed) * 100)))
return {"timestamp": timestamp, "cpu": cpu_percent, "memory": current_memory, **gpu_snapshot()}
def resource_sample_interval_seconds() -> float:
raw = os.getenv("OPERATOR_RESOURCE_SAMPLE_INTERVAL_SECONDS", "").strip()
if not raw:
return 1.0
try:
value = float(raw)
except ValueError:
return 1.0
return min(max(value, 0.2), 60.0)
class ResourceSampler:
def __init__(self, start_cpu: float, start_memory: int, started_at: str, interval_seconds: float | None = None):
self.start_cpu = start_cpu
self.start_memory = start_memory
self.started_at = started_at
self.interval_seconds = resource_sample_interval_seconds() if interval_seconds is None else interval_seconds
self._samples: list[dict[str, Any]] = []
self._stop = threading.Event()
self._lock = threading.Lock()
self._thread = threading.Thread(target=self._run, name="droboticflow-resource-sampler", daemon=True)
def start(self) -> None:
self._thread.start()
def stop(self, ended_at: str) -> list[dict[str, Any]]:
self._stop.set()
self._thread.join(timeout=max(self.interval_seconds + 1.0, 2.0))
with self._lock:
samples = list(self._samples)
return resource_series_with_samples(self.start_cpu, self.start_memory, self.started_at, ended_at, samples)
def _run(self) -> None:
while not self._stop.wait(self.interval_seconds):
sample = resource_sample(self.start_cpu, self.started_at)
with self._lock:
self._samples.append(sample)
if len(self._samples) > 7200:
self._samples = self._samples[-7200:]