340 lines
14 KiB
Python
340 lines
14 KiB
Python
"""GPU stress test module — wraps gpu-burn for long-running stability tests."""
|
|
|
|
import glob
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime
|
|
|
|
from rich.console import Console
|
|
from rich.table import Table
|
|
from rich.live import Live
|
|
from rich.text import Text
|
|
|
|
from modules.gpu_specs import resolve_tools_dir
|
|
from modules.hygon_smi import run_hy_smi
|
|
|
|
|
|
class StressTest:
|
|
|
|
def __init__(self, config: dict):
|
|
self.config = config
|
|
self.console = Console()
|
|
self.stress_cfg = config.get("stress", {})
|
|
self.tools_dir = resolve_tools_dir(config)
|
|
|
|
def _find_gpu_burn(self) -> str:
|
|
p = shutil.which("gpu_burn")
|
|
if p:
|
|
return p
|
|
|
|
local = os.path.join(self.tools_dir, "gpu-burn", "gpu_burn")
|
|
if os.path.isfile(local) and os.access(local, shutil.os.X_OK):
|
|
return local
|
|
|
|
matches = glob.glob(os.path.join(self.tools_dir, "gpu-burn", "**", "gpu_burn"), recursive=True)
|
|
for m in matches:
|
|
if os.access(m, shutil.os.X_OK):
|
|
return m
|
|
return ""
|
|
|
|
def run(self) -> dict:
|
|
cfg = self.stress_cfg
|
|
duration_sec = cfg.get("duration_sec", 60)
|
|
use_doubles = cfg.get("use_doubles", False)
|
|
use_tensor_cores = cfg.get("use_tensor_cores", True)
|
|
memory_pct = cfg.get("memory_pct", 90)
|
|
target_gpus = cfg.get("gpus", "all")
|
|
|
|
gpu_burn = self._find_gpu_burn()
|
|
|
|
if gpu_burn:
|
|
# Try gpu-burn first
|
|
result = self._run_gpu_burn(gpu_burn, duration_sec, use_doubles, use_tensor_cores, target_gpus)
|
|
|
|
# If gpu-burn fails (e.g. OOM), auto-fallback to PyTorch
|
|
if not result.get("passed") and result.get("elapsed_sec", 0) < duration_sec * 0.5:
|
|
self.console.print("\n[yellow]gpu-burn exited early (possible OOM), switching to PyTorch stress test[/yellow]")
|
|
self.console.print("[dim]PyTorch mode dynamically adapts to available memory[/dim]\n")
|
|
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores)
|
|
|
|
return result
|
|
|
|
self.console.print("[yellow]gpu_burn not found, using PyTorch stress test[/yellow]")
|
|
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores)
|
|
|
|
def _run_gpu_burn(self, gpu_burn: str, duration: int,
|
|
doubles: bool, tensor_cores: bool, target_gpus: str) -> dict:
|
|
self.console.print(f"[cyan]GPU Stress Test via gpu-burn ({duration}s)[/cyan]")
|
|
|
|
cmd = [gpu_burn]
|
|
if doubles:
|
|
cmd.append("-d")
|
|
if tensor_cores:
|
|
cmd.append("-tc")
|
|
if target_gpus != "all":
|
|
cmd.extend(["-i", str(target_gpus)])
|
|
cmd.append(str(duration))
|
|
|
|
t0 = time.time()
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, text=True, timeout=duration + 120)
|
|
elapsed = round(time.time() - t0, 1)
|
|
|
|
output = r.stdout + r.stderr
|
|
passed = r.returncode == 0
|
|
|
|
gpu_results = []
|
|
for line in output.split("\n"):
|
|
line = line.strip()
|
|
if "GPU" in line and ("PASS" in line.upper() or "FAIL" in line.upper()):
|
|
gpu_results.append(line)
|
|
|
|
return {
|
|
"source": "gpu-burn",
|
|
"passed": passed,
|
|
"duration_sec": duration,
|
|
"elapsed_sec": elapsed,
|
|
"gpu_results": gpu_results,
|
|
"raw_output_tail": output[-500:] if output else "",
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
|
|
except subprocess.TimeoutExpired:
|
|
return {
|
|
"source": "gpu-burn",
|
|
"passed": False,
|
|
"duration_sec": duration,
|
|
"error": "timeout",
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"source": "gpu-burn",
|
|
"passed": False,
|
|
"error": str(e),
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
|
|
@staticmethod
|
|
def _accelerator_snapshot() -> dict:
|
|
if shutil.which("nvidia-smi"):
|
|
try:
|
|
r = subprocess.run(
|
|
["nvidia-smi", "--query-gpu=index,temperature.gpu,power.draw,"
|
|
"ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits"],
|
|
capture_output=True, text=True, timeout=15,
|
|
)
|
|
temps, powers, ecc = {}, {}, {}
|
|
if r.returncode == 0:
|
|
for line in r.stdout.splitlines():
|
|
parts = [part.strip() for part in line.split(",")]
|
|
if len(parts) < 4:
|
|
continue
|
|
idx = int(parts[0])
|
|
temps[idx] = float(parts[1])
|
|
powers[idx] = float(parts[2])
|
|
ecc[idx] = int(parts[3]) if parts[3].isdigit() else 0
|
|
return {
|
|
"temperatures_c": temps, "power_w": powers,
|
|
"uncorrected_ecc": ecc,
|
|
}
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
pass
|
|
|
|
output = run_hy_smi()
|
|
temps, powers = {}, {}
|
|
if output:
|
|
pattern = re.compile(r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W", re.MULTILINE)
|
|
for match in pattern.finditer(output):
|
|
idx = int(match.group(1))
|
|
temps[idx] = float(match.group(2))
|
|
powers[idx] = float(match.group(3))
|
|
return {"temperatures_c": temps, "power_w": powers, "uncorrected_ecc": {}}
|
|
|
|
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90,
|
|
tensor_cores: bool = True) -> dict:
|
|
try:
|
|
import torch
|
|
try:
|
|
import torch_dcu # noqa: F401
|
|
except ImportError:
|
|
pass
|
|
if not torch.cuda.is_available():
|
|
return {"error": "pytorch_not_available"}
|
|
except ImportError:
|
|
return {"error": "pytorch_not_available"}
|
|
|
|
gpu_count = torch.cuda.device_count()
|
|
self.console.print(f"[cyan]PyTorch Stress Test ({duration}s, {gpu_count} GPUs, target {memory_pct}% memory)[/cyan]")
|
|
|
|
gpu_status = {}
|
|
t0 = time.time()
|
|
snapshots = [self._accelerator_snapshot()]
|
|
|
|
try:
|
|
work_a, work_b, outputs, ballast = {}, {}, {}, {}
|
|
for i in range(gpu_count):
|
|
with torch.cuda.device(i):
|
|
# Get actual free memory (accounting for other processes)
|
|
free_mem, total_mem = torch.cuda.mem_get_info(i)
|
|
|
|
# Calculate allocation from configured memory_pct
|
|
target_mem = int(total_mem * memory_pct / 100)
|
|
|
|
# Cap at actual free memory with 5% safety margin
|
|
alloc_bytes = min(target_mem, int(free_mem * 0.95))
|
|
|
|
dtype = torch.bfloat16 if tensor_cores else torch.float32
|
|
elem_size = torch.tensor([], dtype=dtype).element_size()
|
|
mem_side = int((alloc_bytes / elem_size / 3) ** 0.5)
|
|
max_compute_side = 8192 if tensor_cores else 4096
|
|
side = min(mem_side, max_compute_side)
|
|
compute_bytes = side * side * elem_size * 3
|
|
ballast_target = max(0, alloc_bytes - compute_bytes - 1024**3)
|
|
ballast[i] = []
|
|
chunk_size = 1024**3
|
|
while ballast_target > 0:
|
|
this_chunk = min(chunk_size, ballast_target)
|
|
tensor = torch.empty(this_chunk, device=f"cuda:{i}", dtype=torch.uint8)
|
|
tensor.fill_(0x5A)
|
|
ballast[i].append(tensor)
|
|
ballast_target -= this_chunk
|
|
|
|
actual_mem_mb = (alloc_bytes - max(0, ballast_target)) / 1024 / 1024
|
|
total_mem_mb = total_mem / 1024 / 1024
|
|
free_mem_mb = free_mem / 1024 / 1024
|
|
|
|
self.console.print(
|
|
f" [dim]GPU {i}: total {total_mem_mb:.0f}MB, free {free_mem_mb:.0f}MB, "
|
|
f"alloc {actual_mem_mb:.0f}MB ({actual_mem_mb/total_mem_mb*100:.0f}%) - "
|
|
f"matrix {side}x{side}[/dim]"
|
|
)
|
|
work_a[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
|
|
work_b[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
|
|
outputs[i] = torch.empty(side, side, device=f"cuda:{i}", dtype=dtype)
|
|
|
|
self.console.print(f"\n[cyan]Starting stress test for {duration} seconds...[/cyan]")
|
|
|
|
elapsed_check = 0
|
|
while time.time() - t0 < duration:
|
|
# Dispatch matmul on all GPUs in parallel — do NOT synchronize between
|
|
# GPUs, otherwise the 8 GPUs run serially and overshoot the duration.
|
|
for i in range(gpu_count):
|
|
with torch.cuda.device(i):
|
|
outputs[i] = torch.matmul(work_a[i], work_b[i])
|
|
# Single sync per pass — waits for all 8 streams concurrently
|
|
for i in range(gpu_count):
|
|
with torch.cuda.device(i):
|
|
torch.cuda.synchronize()
|
|
|
|
# Show progress every 10 seconds
|
|
current_elapsed = time.time() - t0
|
|
if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0:
|
|
self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]")
|
|
snapshots.append(self._accelerator_snapshot())
|
|
elapsed_check = current_elapsed
|
|
|
|
for i in range(gpu_count):
|
|
gpu_status[i] = "PASS"
|
|
|
|
except RuntimeError as e:
|
|
error_msg = str(e)
|
|
self.console.print(f"\n[red]Stress test error: {error_msg}[/red]")
|
|
for i in range(gpu_count):
|
|
if i not in gpu_status:
|
|
gpu_status[i] = "FAIL"
|
|
return {
|
|
"source": "pytorch",
|
|
"passed": False,
|
|
"duration_sec": duration,
|
|
"error": error_msg,
|
|
"gpu_status": gpu_status,
|
|
}
|
|
finally:
|
|
for mapping in (work_a, work_b, outputs, ballast):
|
|
mapping.clear()
|
|
torch.cuda.empty_cache()
|
|
|
|
elapsed = round(time.time() - t0, 1)
|
|
snapshots.append(self._accelerator_snapshot())
|
|
max_temp = {}
|
|
max_power = {}
|
|
for snapshot in snapshots:
|
|
for idx, value in snapshot.get("temperatures_c", {}).items():
|
|
max_temp[idx] = max(max_temp.get(idx, value), value)
|
|
for idx, value in snapshot.get("power_w", {}).items():
|
|
max_power[idx] = max(max_power.get(idx, value), value)
|
|
first_ecc = snapshots[0].get("uncorrected_ecc", {}) if snapshots else {}
|
|
last_ecc = snapshots[-1].get("uncorrected_ecc", {}) if snapshots else {}
|
|
ecc_delta = {
|
|
idx: max(0, last_ecc.get(idx, 0) - first_ecc.get(idx, 0))
|
|
for idx in set(first_ecc) | set(last_ecc)
|
|
}
|
|
passed = not any(ecc_delta.values())
|
|
if not passed:
|
|
for idx, delta in ecc_delta.items():
|
|
if delta:
|
|
gpu_status[idx] = "FAIL"
|
|
return {
|
|
"source": "pytorch",
|
|
"passed": passed,
|
|
"duration_sec": duration,
|
|
"elapsed_sec": elapsed,
|
|
"gpu_status": gpu_status,
|
|
"memory_pct": memory_pct,
|
|
"max_temperature_c": max_temp,
|
|
"max_power_w": max_power,
|
|
"uncorrected_ecc_delta": ecc_delta,
|
|
"numerical_validation": False,
|
|
"validation_scope": "runtime stability, allocation, temperature, power, and ECC delta",
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
|
|
@staticmethod
|
|
def print_results(results: dict, console: Console = None):
|
|
c = console or Console()
|
|
if "error" in results:
|
|
c.print(f"[bold red]Error: {results['error']}[/bold red]")
|
|
return
|
|
|
|
passed = results.get("passed", False)
|
|
source = results.get("source", "unknown")
|
|
duration = results.get("duration_sec", "?")
|
|
elapsed = results.get("elapsed_sec", "?")
|
|
|
|
verdict = "[bold green]✓ Stress Test PASSED[/bold green]" if passed else "[bold red]✗ Stress Test FAILED[/bold red]"
|
|
c.print(f"\n{verdict} [dim](via {source})[/dim]")
|
|
c.print(f" Target duration: {duration}s | Actual: {elapsed}s")
|
|
|
|
gpu_results = results.get("gpu_results", [])
|
|
if gpu_results:
|
|
c.print("\n Per-GPU results:")
|
|
for line in gpu_results:
|
|
if "FAIL" in line.upper():
|
|
c.print(f" [red]{line}[/red]")
|
|
else:
|
|
c.print(f" [green]{line}[/green]")
|
|
|
|
gpu_status = results.get("gpu_status", {})
|
|
if gpu_status:
|
|
c.print("\n Per-GPU status:")
|
|
for gid, status in sorted(gpu_status.items()):
|
|
color = "green" if status == "PASS" else "red"
|
|
c.print(f" GPU {gid}: [{color}]{status}[/{color}]")
|
|
|
|
max_temp = results.get("max_temperature_c", {})
|
|
max_power = results.get("max_power_w", {})
|
|
if max_temp or max_power:
|
|
c.print("\n Peak telemetry:")
|
|
for gid in sorted(set(max_temp) | set(max_power)):
|
|
c.print(
|
|
f" GPU {gid}: {max_temp.get(gid, 'N/A')} C, "
|
|
f"{max_power.get(gid, 'N/A')} W"
|
|
)
|
|
|
|
if results.get("error"):
|
|
c.print(f" [red]Error: {results['error']}[/red]")
|