forked from longtao.wu/droboticflow
85 lines
2.8 KiB
Python
85 lines
2.8 KiB
Python
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import resource
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
def resource_snapshot() -> tuple[float, int]:
|
|
usage = resource.getrusage(resource.RUSAGE_SELF)
|
|
cpu_seconds = usage.ru_utime + usage.ru_stime
|
|
if sys.platform == "darwin":
|
|
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
|
|
else:
|
|
memory_mib = max(1, int(usage.ru_maxrss / 1024))
|
|
return cpu_seconds, memory_mib
|
|
|
|
|
|
def parse_gpu_snapshot_output(output: str) -> dict[str, Any]:
|
|
samples: list[dict[str, Any]] = []
|
|
for line in output.splitlines():
|
|
parts = [part.strip() for part in line.split(",")]
|
|
if len(parts) < 4:
|
|
continue
|
|
try:
|
|
samples.append(
|
|
{
|
|
"gpu": int(parts[0]),
|
|
"gpuMemoryUsedMiB": int(parts[1]),
|
|
"gpuMemoryTotalMiB": int(parts[2]),
|
|
"gpuName": parts[3],
|
|
}
|
|
)
|
|
except ValueError:
|
|
continue
|
|
if not samples:
|
|
return {"gpu": 0}
|
|
|
|
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
|
|
return {
|
|
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
|
|
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
|
|
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
|
|
"gpuName": ", ".join(names),
|
|
"gpuCount": len(samples),
|
|
}
|
|
|
|
|
|
def gpu_snapshot() -> dict[str, Any]:
|
|
nvidia_smi = shutil.which("nvidia-smi")
|
|
if not nvidia_smi:
|
|
return {"gpu": 0}
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
nvidia_smi,
|
|
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
|
|
"--format=csv,noheader,nounits",
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=2,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return {"gpu": 0}
|
|
if completed.returncode != 0:
|
|
return {"gpu": 0}
|
|
return parse_gpu_snapshot_output(completed.stdout)
|
|
|
|
|
|
def resource_series(start_cpu: float, start_memory: int, started_at: str, ended_at: str) -> list[dict[str, Any]]:
|
|
end_cpu, end_memory = resource_snapshot()
|
|
gpu_metrics = gpu_snapshot()
|
|
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
|
|
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
|
|
elapsed = max((ended - started).total_seconds(), 0.001)
|
|
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
|
|
return [
|
|
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
|
|
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, **gpu_metrics},
|
|
]
|