droboticflow/backend/operator-runner/resource_monitor.py
2026-06-16 14:28:49 +08:00

85 lines
2.8 KiB
Python

from __future__ import annotations
import datetime as dt
import resource
import shutil
import subprocess
import sys
from typing import Any
def resource_snapshot() -> tuple[float, int]:
usage = resource.getrusage(resource.RUSAGE_SELF)
cpu_seconds = usage.ru_utime + usage.ru_stime
if sys.platform == "darwin":
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
else:
memory_mib = max(1, int(usage.ru_maxrss / 1024))
return cpu_seconds, memory_mib
def parse_gpu_snapshot_output(output: str) -> dict[str, Any]:
samples: list[dict[str, Any]] = []
for line in output.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 4:
continue
try:
samples.append(
{
"gpu": int(parts[0]),
"gpuMemoryUsedMiB": int(parts[1]),
"gpuMemoryTotalMiB": int(parts[2]),
"gpuName": parts[3],
}
)
except ValueError:
continue
if not samples:
return {"gpu": 0}
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
return {
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
"gpuName": ", ".join(names),
"gpuCount": len(samples),
}
def gpu_snapshot() -> dict[str, Any]:
nvidia_smi = shutil.which("nvidia-smi")
if not nvidia_smi:
return {"gpu": 0}
try:
completed = subprocess.run(
[
nvidia_smi,
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
"--format=csv,noheader,nounits",
],
check=False,
capture_output=True,
text=True,
timeout=2,
)
except (OSError, subprocess.SubprocessError):
return {"gpu": 0}
if completed.returncode != 0:
return {"gpu": 0}
return parse_gpu_snapshot_output(completed.stdout)
def resource_series(start_cpu: float, start_memory: int, started_at: str, ended_at: str) -> list[dict[str, Any]]:
end_cpu, end_memory = resource_snapshot()
gpu_metrics = gpu_snapshot()
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
elapsed = max((ended - started).total_seconds(), 0.001)
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
return [
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, **gpu_metrics},
]