sanity: CUDA/CPU forward divergence arbitration
This commit is contained in:
parent
537e69d386
commit
4b80e5de46
104
sanity_cuda_cpu.py
Normal file
104
sanity_cuda_cpu.py
Normal file
@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sanity_cuda_cpu.py — ModernBERT CUDA/CPU 前向一致性仲裁(同容器内互证, 无需训练)
|
||||
矩阵: {sdpa, eager} × {GPU-fp32, GPU-bf16-autocast, CPU-fp32} × 32 对校准输入
|
||||
输出: 每个 impl 的 max/mean logit diff + 自动判读到 termination-log 后主动退出(exit 1)
|
||||
"""
|
||||
import json, os, sys, time, urllib.request
|
||||
|
||||
HF = "https://hf-mirror.com/answerdotai/ModernBERT-base/resolve/main"
|
||||
UA = {"User-Agent": "Mozilla/5.0"}
|
||||
DEST = "/tmp/modernbert-base"
|
||||
|
||||
def fetch(fn, min_size=1000):
|
||||
path = os.path.join(DEST, fn)
|
||||
if os.path.exists(path) and os.path.getsize(path) >= min_size:
|
||||
return
|
||||
for attempt in range(1, 5):
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(f"{HF}/{fn}", headers=UA), timeout=120) as r, \
|
||||
open(path + ".tmp", "wb") as f:
|
||||
total = 0
|
||||
while True:
|
||||
b = r.read(8 * 1024 * 1024)
|
||||
if not b:
|
||||
break
|
||||
f.write(b)
|
||||
total += len(b)
|
||||
os.replace(path + ".tmp", path)
|
||||
print(f"[fetch] {fn}: {total//1024//1024}MB", flush=True)
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:100]}", flush=True)
|
||||
if attempt == 4:
|
||||
raise
|
||||
time.sleep(5 * attempt)
|
||||
|
||||
def main():
|
||||
t0 = time.time()
|
||||
os.makedirs(DEST, exist_ok=True)
|
||||
for fn in ["config.json", "model.safetensors", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]:
|
||||
fetch(fn)
|
||||
print(f"[fetch] total {time.time()-t0:.0f}s", flush=True)
|
||||
|
||||
import torch
|
||||
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(DEST)
|
||||
pairs = [json.loads(l) for l in open("/tmp/jd/calibration.jsonl", encoding="utf-8")][:32]
|
||||
encs = [tok(r["query"], r["doc_text"][:1200], truncation=True, max_length=512,
|
||||
padding="max_length", return_tensors="pt") for r in pairs]
|
||||
print(f"[sanity] {len(encs)} 对输入就绪", flush=True)
|
||||
|
||||
def logits(m, dev, dtype_cast=False, autocast=False):
|
||||
out = []
|
||||
m.to(dev)
|
||||
with torch.no_grad():
|
||||
for enc in encs:
|
||||
ids = enc["input_ids"].to(dev)
|
||||
am = enc["attention_mask"].to(dev)
|
||||
if autocast:
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
lg = m(input_ids=ids, attention_mask=am).logits[0, 0]
|
||||
else:
|
||||
lg = m(input_ids=ids, attention_mask=am).logits[0, 0]
|
||||
out.append(float(lg.float().cpu()))
|
||||
m.cpu()
|
||||
torch.cuda.empty_cache()
|
||||
return out
|
||||
|
||||
report, verdicts = {}, {}
|
||||
for impl in ["sdpa", "eager"]:
|
||||
m = AutoModelForSequenceClassification.from_pretrained(
|
||||
DEST, num_labels=1, attn_implementation=impl)
|
||||
m.eval()
|
||||
cpu_f32 = logits(m, "cpu")
|
||||
gpu_f32 = logits(m, "cuda")
|
||||
gpu_bf16 = logits(m, "cuda", autocast=True)
|
||||
def diff(a, b):
|
||||
ds = [abs(x - y) for x, y in zip(a, b)]
|
||||
return max(ds), sum(ds) / len(ds)
|
||||
d_gpu, d_bf = diff(gpu_f32, cpu_f32), diff(gpu_bf16, gpu_f32)
|
||||
report[impl] = {"gpu_vs_cpu_max": d_gpu[0], "gpu_vs_cpu_mean": d_gpu[1],
|
||||
"bf16_vs_f32_max": d_bf[0], "bf16_vs_f32_mean": d_bf[1]}
|
||||
verdicts[impl] = "CLEAN" if d_gpu[0] < 0.05 else "BROKEN"
|
||||
print(f"[sanity] {impl}: GPU-vs-CPU max|Δlogit|={d_gpu[0]:.4f} mean={d_gpu[1]:.5f} "
|
||||
f"bf16-vs-f32 max={d_bf[0]:.4f} → {verdicts[impl]}", flush=True)
|
||||
del m
|
||||
|
||||
lines = ["SANITY-MATRIX"]
|
||||
for impl, rep in report.items():
|
||||
lines.append(f"{impl}: {json.dumps(rep)} verdict={verdicts[impl]}")
|
||||
clean = [k for k, v in verdicts.items() if v == "CLEAN"]
|
||||
if verdicts.get("eager") == "CLEAN":
|
||||
lines.append("DECISION: retrain_with_eager")
|
||||
elif verdicts.get("sdpa") == "CLEAN":
|
||||
lines.append("DECISION: sdpa_clean_bug_elsewhere_recheck_training_dynamics")
|
||||
else:
|
||||
lines.append("DECISION: modernbert_dead_on_this_gpu_stack")
|
||||
with open("/dev/termination-log", "w") as f:
|
||||
f.write("\n".join(lines))
|
||||
print("\n".join(lines), flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
sys.exit(1) # 主动失败以回读 termination-log
|
||||
Loading…
x
Reference in New Issue
Block a user