test_gpu_scripts/modules/training_sim.py

352 lines
15 KiB
Python

"""Training simulation module - LLM training workload with PyTorch."""
import time
import subprocess
import shutil
from contextlib import nullcontext
from datetime import datetime
from typing import Optional
from rich.console import Console
from rich.table import Table
from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn
TORCH_AVAILABLE = False
try:
import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available():
TORCH_AVAILABLE = True
except ImportError:
pass
class TrainingSim:
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.train_cfg = config.get("training", {})
@staticmethod
def _requires_b300_sdpa_workaround(
configured_type: str, capability, cudnn_version: int | None,
) -> bool:
is_b300 = str(configured_type).lower() == "b300" or capability == (10, 3)
# cuDNN 9.20 restores SM103 plan generation and PyTorch 2.13 correctly
# rejects unsupported cuDNN shapes (for example dropout=0.1) before
# dispatch, allowing another SDPA backend to run. Keep the workaround
# only for the affected older stack.
return is_b300 and (not cudnn_version or cudnn_version < 92000)
def _sdpa_context(self):
"""Select a reliable SDPA backend policy for B300/SM103.
PyTorch 2.11 + cuDNN 9.19 can dispatch BF16 attention on B300 to the
cuDNN frontend even though that backend cannot build an execution plan.
Keep the workaround scoped to the training operation and retain ordered
Flash/Efficient/Math fallbacks instead of changing global process state.
"""
configured_type = str(self.config.get("gpu_type", "")).lower()
capability = None
cudnn_version = None
if torch.cuda.is_available():
try:
capability = tuple(torch.cuda.get_device_capability(0))
except Exception: # noqa: BLE001 - backend discovery must not abort training
capability = None
try:
cudnn_version = torch.backends.cudnn.version()
except Exception: # noqa: BLE001 - backend discovery must not abort training
cudnn_version = None
requires_workaround = self._requires_b300_sdpa_workaround(
configured_type, capability, cudnn_version,
)
if not requires_workaround:
if (configured_type == "b300" or capability == (10, 3)) and cudnn_version:
return nullcontext(), (
f"framework_default (cuDNN {cudnn_version}; "
"unsupported SDPA shapes may fall back automatically)"
)
return nullcontext(), "framework_default"
try:
from torch.nn.attention import SDPBackend, sdpa_kernel
context = sdpa_kernel(
[
SDPBackend.FLASH_ATTENTION,
SDPBackend.EFFICIENT_ATTENTION,
SDPBackend.MATH,
],
set_priority=True,
)
return context, "flash>efficient>math (cuDNN SDPA excluded on B300)"
except (ImportError, AttributeError, TypeError):
# Compatibility path for older PyTorch versions that do not expose
# the context-manager API. Restore the global flag when leaving.
class _DisableCudnnSDPA:
def __enter__(self_inner):
self_inner.old = torch.backends.cuda.cudnn_sdp_enabled()
torch.backends.cuda.enable_cudnn_sdp(False)
def __exit__(self_inner, exc_type, exc, traceback):
torch.backends.cuda.enable_cudnn_sdp(self_inner.old)
return _DisableCudnnSDPA(), "framework fallback (cuDNN SDPA disabled on B300)"
def run(self) -> dict:
if not TORCH_AVAILABLE:
self.console.print("[yellow]PyTorch not available - skipping training simulation[/yellow]")
return {"error": "pytorch_not_available"}
gpu_count = torch.cuda.device_count()
model_name = self.train_cfg.get("model", "gpt2")
batch_size = self.train_cfg.get("batch_size", 8)
seq_length = self.train_cfg.get("seq_length", 2048)
num_steps = self.train_cfg.get("num_steps", 50)
dtype_str = self.train_cfg.get("dtype", "bf16")
dtype_map = {
"fp32": torch.float32,
"fp16": torch.float16,
"bf16": torch.bfloat16,
}
dtype = dtype_map.get(dtype_str, torch.bfloat16)
self.console.print(f"[cyan]Training Simulation[/cyan]")
self.console.print(f" Model: {model_name} | Batch: {batch_size} | Seq: {seq_length} | "
f"DType: {dtype_str} | Steps: {num_steps} | GPUs: {gpu_count}")
if model_name == "synthetic":
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype)
try:
from transformers import AutoModelForCausalLM, AutoTokenizer
except ImportError:
self.console.print("[yellow]transformers not installed - using synthetic model[/yellow]")
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype)
try:
self.console.print(f" Loading {model_name}...")
model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype=dtype,
device_map="auto" if gpu_count > 1 else None,
)
total_params = sum(p.numel() for p in model.parameters())
self.console.print(f" Parameters: {total_params / 1e6:.1f}M")
tokenizer = AutoTokenizer.from_pretrained(model_name)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
input_ids = torch.randint(0, tokenizer.vocab_size, (batch_size, seq_length))
attention_mask = torch.ones_like(input_ids)
model.train()
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)
if dtype in (torch.float16, torch.bfloat16):
scaler = torch.amp.GradScaler("cuda", enabled=(dtype == torch.float16))
step_times = []
mem_usage = []
sdpa_context, sdpa_policy = self._sdpa_context()
self.console.print(f" SDPA policy: {sdpa_policy}")
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
BarColumn(), TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(), console=self.console,
) as progress:
task = progress.add_task("Training steps...", total=num_steps)
with sdpa_context:
for step in range(num_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
input_ids = input_ids.to(model.device)
attention_mask = attention_mask.to(model.device)
if dtype in (torch.float16, torch.bfloat16):
with torch.amp.autocast("cuda", dtype=dtype):
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
else:
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
loss.backward()
optimizer.step()
optimizer.zero_grad()
torch.cuda.synchronize()
elapsed = time.perf_counter() - t0
step_times.append(elapsed)
if torch.cuda.is_available():
mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats()
progress.advance(task)
avg_step_time = sum(step_times) / len(step_times)
throughput = batch_size * seq_length / avg_step_time
return {
"model": model_name,
"total_params_m": round(total_params / 1e6, 1),
"gpu_count": gpu_count,
"dtype": dtype_str,
"batch_size": batch_size,
"seq_length": seq_length,
"num_steps": num_steps,
"avg_step_time_ms": round(avg_step_time * 1000, 1),
"throughput_tokens_per_sec": round(throughput, 0),
"throughput_samples_per_sec": round(batch_size / avg_step_time, 2),
"peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2),
"final_loss": round(loss.item(), 4) if hasattr(loss, 'item') else None,
"sdpa_backend_policy": sdpa_policy,
"timestamp": datetime.now().isoformat(),
}
except Exception as e:
self.console.print(f"[yellow]Model loading failed: {e}[/yellow]")
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype)
def _run_synthetic(self, gpu_count, batch_size, seq_length, num_steps, dtype) -> dict:
self.console.print(" Running synthetic training benchmark...")
hidden_size = int(self.train_cfg.get("hidden_size", 4096))
num_layers = int(self.train_cfg.get("num_layers", 6))
num_heads = int(self.train_cfg.get("num_heads", 32))
vocab_size = int(self.train_cfg.get("vocab_size", 32000))
class SyntheticTransformer(torch.nn.Module):
def __init__(self):
super().__init__()
self.embed = torch.nn.Embedding(vocab_size, hidden_size)
self.layers = torch.nn.ModuleList([
torch.nn.TransformerEncoderLayer(
d_model=hidden_size, nhead=num_heads,
dim_feedforward=hidden_size * 4,
batch_first=True,
dtype=dtype,
) for _ in range(num_layers)
])
self.head = torch.nn.Linear(hidden_size, vocab_size, dtype=dtype)
def forward(self, x):
h = self.embed(x).to(dtype)
for layer in self.layers:
h = layer(h)
return self.head(h)
model = SyntheticTransformer().cuda()
total_params = sum(p.numel() for p in model.parameters())
self.console.print(f" Synthetic params: {total_params / 1e6:.1f}M")
model.train()
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)
input_ids = torch.randint(0, vocab_size, (batch_size, seq_length)).cuda()
step_times = []
mem_usage = []
sdpa_context, sdpa_policy = self._sdpa_context()
self.console.print(f" SDPA policy: {sdpa_policy}")
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
BarColumn(), TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(), console=self.console,
) as progress:
task = progress.add_task("Synthetic training...", total=num_steps)
with sdpa_context:
for step in range(num_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
logits = model(input_ids)
loss = torch.nn.functional.cross_entropy(
logits.view(-1, vocab_size), input_ids.view(-1)
)
loss.backward()
optimizer.step()
optimizer.zero_grad()
torch.cuda.synchronize()
elapsed = time.perf_counter() - t0
step_times.append(elapsed)
mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats()
progress.advance(task)
avg_step_time = sum(step_times) / len(step_times)
throughput = batch_size * seq_length / avg_step_time
return {
"model": "synthetic_transformer",
"total_params_m": round(total_params / 1e6, 1),
"num_layers": num_layers,
"hidden_size": hidden_size,
"gpu_count": gpu_count,
"gpu_scope": "single_gpu:0",
"performance_verdict": "MEASURED",
"sdpa_backend_policy": sdpa_policy,
"dtype": str(dtype).replace("torch.", ""),
"batch_size": batch_size,
"seq_length": seq_length,
"num_steps": num_steps,
"avg_step_time_ms": round(avg_step_time * 1000, 1),
"throughput_tokens_per_sec": round(throughput, 0),
"throughput_samples_per_sec": round(batch_size / avg_step_time, 2),
"peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2),
"final_loss": round(loss.item(), 4),
"timestamp": datetime.now().isoformat(),
}
@staticmethod
def print_results(results: dict, console: Console = None):
c = console or Console()
if "error" in results:
c.print(f"[bold red]Error: {results['error']}[/bold red]")
return
c.print(f"\n[bold cyan]Training Simulation Results[/bold cyan]")
table = Table(box=None, padding=(0, 1))
table.add_column("Metric", style="bold")
table.add_column("Value")
metrics = [
("Model", results.get("model", "N/A")),
("Parameters", f"{results.get('total_params_m', 'N/A')}M"),
("GPU Count", str(results.get("gpu_count", "N/A"))),
("DType", results.get("dtype", "N/A")),
("Batch Size", str(results.get("batch_size", "N/A"))),
("Seq Length", str(results.get("seq_length", "N/A"))),
("Steps", str(results.get("num_steps", "N/A"))),
("Avg Step Time", f"{results.get('avg_step_time_ms', 'N/A')} ms"),
("Throughput", f"{results.get('throughput_tokens_per_sec', 'N/A')} tokens/s"),
("Samples/sec", f"{results.get('throughput_samples_per_sec', 'N/A')}"),
("Peak Memory", f"{results.get('peak_memory_gb', 'N/A')} GB"),
("Final Loss", str(results.get("final_loss", "N/A"))),
]
for label, val in metrics:
table.add_row(label, str(val))
c.print(table)