sanity: CUDA/CPU forward divergence arbitration

This commit is contained in:
dingbin.wang 2026-10-21 00:48:27 +08:00
parent 537e69d386
commit 4b80e5de46

104
sanity_cuda_cpu.py Normal file
View File

@ -0,0 +1,104 @@
#!/usr/bin/env python3
"""sanity_cuda_cpu.py — ModernBERT CUDA/CPU 前向一致性仲裁(同容器内互证, 无需训练)
矩阵: {sdpa, eager} × {GPU-fp32, GPU-bf16-autocast, CPU-fp32} × 32 对校准输入
输出: 每个 impl 的 max/mean logit diff + 自动判读到 termination-log 后主动退出(exit 1)
"""
import json, os, sys, time, urllib.request
HF = "https://hf-mirror.com/answerdotai/ModernBERT-base/resolve/main"
UA = {"User-Agent": "Mozilla/5.0"}
DEST = "/tmp/modernbert-base"
def fetch(fn, min_size=1000):
path = os.path.join(DEST, fn)
if os.path.exists(path) and os.path.getsize(path) >= min_size:
return
for attempt in range(1, 5):
try:
with urllib.request.urlopen(urllib.request.Request(f"{HF}/{fn}", headers=UA), timeout=120) as r, \
open(path + ".tmp", "wb") as f:
total = 0
while True:
b = r.read(8 * 1024 * 1024)
if not b:
break
f.write(b)
total += len(b)
os.replace(path + ".tmp", path)
print(f"[fetch] {fn}: {total//1024//1024}MB", flush=True)
return
except Exception as e:
print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:100]}", flush=True)
if attempt == 4:
raise
time.sleep(5 * attempt)
def main():
t0 = time.time()
os.makedirs(DEST, exist_ok=True)
for fn in ["config.json", "model.safetensors", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]:
fetch(fn)
print(f"[fetch] total {time.time()-t0:.0f}s", flush=True)
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification
tok = AutoTokenizer.from_pretrained(DEST)
pairs = [json.loads(l) for l in open("/tmp/jd/calibration.jsonl", encoding="utf-8")][:32]
encs = [tok(r["query"], r["doc_text"][:1200], truncation=True, max_length=512,
padding="max_length", return_tensors="pt") for r in pairs]
print(f"[sanity] {len(encs)} 对输入就绪", flush=True)
def logits(m, dev, dtype_cast=False, autocast=False):
out = []
m.to(dev)
with torch.no_grad():
for enc in encs:
ids = enc["input_ids"].to(dev)
am = enc["attention_mask"].to(dev)
if autocast:
with torch.autocast("cuda", dtype=torch.bfloat16):
lg = m(input_ids=ids, attention_mask=am).logits[0, 0]
else:
lg = m(input_ids=ids, attention_mask=am).logits[0, 0]
out.append(float(lg.float().cpu()))
m.cpu()
torch.cuda.empty_cache()
return out
report, verdicts = {}, {}
for impl in ["sdpa", "eager"]:
m = AutoModelForSequenceClassification.from_pretrained(
DEST, num_labels=1, attn_implementation=impl)
m.eval()
cpu_f32 = logits(m, "cpu")
gpu_f32 = logits(m, "cuda")
gpu_bf16 = logits(m, "cuda", autocast=True)
def diff(a, b):
ds = [abs(x - y) for x, y in zip(a, b)]
return max(ds), sum(ds) / len(ds)
d_gpu, d_bf = diff(gpu_f32, cpu_f32), diff(gpu_bf16, gpu_f32)
report[impl] = {"gpu_vs_cpu_max": d_gpu[0], "gpu_vs_cpu_mean": d_gpu[1],
"bf16_vs_f32_max": d_bf[0], "bf16_vs_f32_mean": d_bf[1]}
verdicts[impl] = "CLEAN" if d_gpu[0] < 0.05 else "BROKEN"
print(f"[sanity] {impl}: GPU-vs-CPU max|Δlogit|={d_gpu[0]:.4f} mean={d_gpu[1]:.5f} "
f"bf16-vs-f32 max={d_bf[0]:.4f} → {verdicts[impl]}", flush=True)
del m
lines = ["SANITY-MATRIX"]
for impl, rep in report.items():
lines.append(f"{impl}: {json.dumps(rep)} verdict={verdicts[impl]}")
clean = [k for k, v in verdicts.items() if v == "CLEAN"]
if verdicts.get("eager") == "CLEAN":
lines.append("DECISION: retrain_with_eager")
elif verdicts.get("sdpa") == "CLEAN":
lines.append("DECISION: sdpa_clean_bug_elsewhere_recheck_training_dynamics")
else:
lines.append("DECISION: modernbert_dead_on_this_gpu_stack")
with open("/dev/termination-log", "w") as f:
f.write("\n".join(lines))
print("\n".join(lines), flush=True)
if __name__ == "__main__":
main()
sys.exit(1) # 主动失败以回读 termination-log