forked from longtao.wu/droboticflow
213 lines
7.5 KiB
Python
213 lines
7.5 KiB
Python
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import os
|
|
import resource
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from typing import Any
|
|
|
|
|
|
def resource_snapshot() -> tuple[float, int]:
|
|
usage = resource.getrusage(resource.RUSAGE_SELF)
|
|
cpu_seconds = usage.ru_utime + usage.ru_stime
|
|
if sys.platform == "darwin":
|
|
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
|
|
else:
|
|
memory_mib = max(1, int(usage.ru_maxrss / 1024))
|
|
return cpu_seconds, memory_mib
|
|
|
|
|
|
def parse_gpu_snapshot_output(output: str) -> dict[str, Any]:
|
|
samples: list[dict[str, Any]] = []
|
|
for line in output.splitlines():
|
|
parts = [part.strip() for part in line.split(",")]
|
|
if len(parts) < 4:
|
|
continue
|
|
try:
|
|
samples.append(
|
|
{
|
|
"gpu": int(parts[0]),
|
|
"gpuMemoryUsedMiB": int(parts[1]),
|
|
"gpuMemoryTotalMiB": int(parts[2]),
|
|
"gpuName": parts[3],
|
|
}
|
|
)
|
|
except ValueError:
|
|
continue
|
|
if not samples:
|
|
return {"gpu": 0}
|
|
|
|
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
|
|
return {
|
|
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
|
|
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
|
|
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
|
|
"gpuName": ", ".join(names),
|
|
"gpuCount": len(samples),
|
|
}
|
|
|
|
|
|
def gpu_snapshot() -> dict[str, Any]:
|
|
smi_snapshot = nvidia_smi_gpu_snapshot()
|
|
if smi_snapshot != {"gpu": 0}:
|
|
return smi_snapshot
|
|
return nvml_gpu_snapshot()
|
|
|
|
|
|
def nvidia_smi_gpu_snapshot() -> dict[str, Any]:
|
|
nvidia_smi = shutil.which("nvidia-smi")
|
|
if not nvidia_smi:
|
|
return {"gpu": 0}
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
nvidia_smi,
|
|
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
|
|
"--format=csv,noheader,nounits",
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=2,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return {"gpu": 0}
|
|
if completed.returncode != 0:
|
|
return {"gpu": 0}
|
|
return parse_gpu_snapshot_output(completed.stdout)
|
|
|
|
|
|
def nvml_gpu_snapshot() -> dict[str, Any]:
|
|
try:
|
|
import pynvml # type: ignore
|
|
except Exception:
|
|
return {"gpu": 0}
|
|
try:
|
|
pynvml.nvmlInit()
|
|
count = pynvml.nvmlDeviceGetCount()
|
|
samples: list[dict[str, Any]] = []
|
|
for index in range(count):
|
|
handle = pynvml.nvmlDeviceGetHandleByIndex(index)
|
|
memory = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
|
|
name = pynvml.nvmlDeviceGetName(handle)
|
|
if isinstance(name, bytes):
|
|
name = name.decode("utf-8", errors="replace")
|
|
samples.append(
|
|
{
|
|
"gpu": int(getattr(utilization, "gpu", 0)),
|
|
"gpuMemoryUsedMiB": int(getattr(memory, "used", 0) / (1024 * 1024)),
|
|
"gpuMemoryTotalMiB": int(getattr(memory, "total", 0) / (1024 * 1024)),
|
|
"gpuName": str(name),
|
|
}
|
|
)
|
|
except Exception:
|
|
return {"gpu": 0}
|
|
finally:
|
|
try:
|
|
pynvml.nvmlShutdown()
|
|
except Exception:
|
|
pass
|
|
if not samples:
|
|
return {"gpu": 0}
|
|
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
|
|
return {
|
|
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
|
|
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
|
|
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
|
|
"gpuName": ", ".join(names),
|
|
"gpuCount": len(samples),
|
|
}
|
|
|
|
|
|
def resource_series(start_cpu: float, start_memory: int, started_at: str, ended_at: str) -> list[dict[str, Any]]:
|
|
return resource_series_with_samples(start_cpu, start_memory, started_at, ended_at, [])
|
|
|
|
|
|
def resource_series_with_samples(
|
|
start_cpu: float,
|
|
start_memory: int,
|
|
started_at: str,
|
|
ended_at: str,
|
|
samples: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
end_cpu, end_memory = resource_snapshot()
|
|
gpu_metrics = gpu_snapshot()
|
|
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
|
|
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
|
|
elapsed = max((ended - started).total_seconds(), 0.001)
|
|
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
|
|
series = [
|
|
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
|
|
*samples,
|
|
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, **gpu_metrics},
|
|
]
|
|
return compact_resource_series(series)
|
|
|
|
|
|
def compact_resource_series(series: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
compacted: list[dict[str, Any]] = []
|
|
seen_timestamps: set[str] = set()
|
|
for item in series:
|
|
timestamp = str(item.get("timestamp") or "").strip()
|
|
if not timestamp or timestamp in seen_timestamps:
|
|
continue
|
|
seen_timestamps.add(timestamp)
|
|
compacted.append(item)
|
|
return compacted
|
|
|
|
|
|
def resource_sample(start_cpu: float, started_at: str) -> dict[str, Any]:
|
|
current_cpu, current_memory = resource_snapshot()
|
|
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
|
|
timestamp = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
now = dt.datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
|
elapsed = max((now - started).total_seconds(), 0.001)
|
|
cpu_percent = max(0, min(100, int(((current_cpu - start_cpu) / elapsed) * 100)))
|
|
return {"timestamp": timestamp, "cpu": cpu_percent, "memory": current_memory, **gpu_snapshot()}
|
|
|
|
|
|
def resource_sample_interval_seconds() -> float:
|
|
raw = os.getenv("OPERATOR_RESOURCE_SAMPLE_INTERVAL_SECONDS", "").strip()
|
|
if not raw:
|
|
return 1.0
|
|
try:
|
|
value = float(raw)
|
|
except ValueError:
|
|
return 1.0
|
|
return min(max(value, 0.2), 60.0)
|
|
|
|
|
|
class ResourceSampler:
|
|
def __init__(self, start_cpu: float, start_memory: int, started_at: str, interval_seconds: float | None = None):
|
|
self.start_cpu = start_cpu
|
|
self.start_memory = start_memory
|
|
self.started_at = started_at
|
|
self.interval_seconds = resource_sample_interval_seconds() if interval_seconds is None else interval_seconds
|
|
self._samples: list[dict[str, Any]] = []
|
|
self._stop = threading.Event()
|
|
self._lock = threading.Lock()
|
|
self._thread = threading.Thread(target=self._run, name="droboticflow-resource-sampler", daemon=True)
|
|
|
|
def start(self) -> None:
|
|
self._thread.start()
|
|
|
|
def stop(self, ended_at: str) -> list[dict[str, Any]]:
|
|
self._stop.set()
|
|
self._thread.join(timeout=max(self.interval_seconds + 1.0, 2.0))
|
|
with self._lock:
|
|
samples = list(self._samples)
|
|
return resource_series_with_samples(self.start_cpu, self.start_memory, self.started_at, ended_at, samples)
|
|
|
|
def _run(self) -> None:
|
|
while not self._stop.wait(self.interval_seconds):
|
|
sample = resource_sample(self.start_cpu, self.started_at)
|
|
with self._lock:
|
|
self._samples.append(sample)
|
|
if len(self._samples) > 7200:
|
|
self._samples = self._samples[-7200:]
|