148 lines
5.8 KiB
Python

#!/usr/bin/env python3
"""eval_laya_r36.py — 本地评测 r38 训练出的 Laya DecisionModel
加载 mmBERT + 重组 adapter → eval_v2 / canary / calibration 三个切片的 AUC+ECE
"""
import json, os, sys, bisect
sys.path.insert(0, "/tmp/laya_src")
BASE = sys.argv[1] if len(sys.argv) > 1 else "/tmp/mmbert_local"
ADAPTER = sys.argv[2] if len(sys.argv) > 2 else "/tmp/r36_adapter.pt"
EVAL_V2 = "/tmp/jd_push/eval_v2.jsonl"
CANARY = ["/tmp/jd_push/canary_en.jsonl", "/tmp/jd_push/canary_zh.jsonl"]
CALIB = "/tmp/jd_local/calibration.jsonl"
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
def log(m):
print(f"[eval] {m}", flush=True)
def load_jsonl(p):
return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()]
def ece(probs, labels, nbins=15):
bins = [[] for _ in range(nbins)]
for p, y in zip(probs, labels):
bins[min(int(p * nbins), nbins - 1)].append((p, y))
e = 0.0
for b in bins:
if not b:
continue
e += len(b) / len(probs) * abs(
sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
return e
def auc(probs, labels):
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
if not pos or not neg:
return float("nan")
return sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg))
def load_encoder(base):
"""mmBERT checkpoint 带 model. 前缀, AutoModel.from_pretrained 会全部 miss → 显式映射加载"""
import torch
from transformers import AutoConfig, AutoModel
cfg = AutoConfig.from_pretrained(base)
enc = AutoModel.from_config(cfg)
sd = torch.load(os.path.join(base, "pytorch_model.bin"), map_location="cpu", weights_only=False)
sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()}
missing, unexpected = enc.load_state_dict(sd2, strict=False)
# missing 应为空, unexpected = head/decoder (MLM 头, 基座不要)
core_missing = [k for k in missing if not k.startswith(("head", "decoder"))]
if core_missing:
log(f"WARN encoder missing core keys: {core_missing[:5]}")
return enc
def main():
import torch
from transformers import AutoTokenizer
from peft import LoraConfig, get_peft_model
from laya.common import DecisionModel, build_sequence
dev = "cuda" if torch.cuda.is_available() else "cpu"
log(f"device={dev}")
tok = AutoTokenizer.from_pretrained(BASE)
enc = load_encoder(BASE)
dm = DecisionModel(enc, head_layers=2, n_act=2)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
target_modules=["Wqkv", "Wo"], task_type=None)
dm.encoder = get_peft_model(dm.encoder, lora)
ad = torch.load(ADAPTER, map_location="cpu", weights_only=False)
missing, unexpected = dm.load_state_dict(ad, strict=False)
log(f"loaded adapter: missing={len(missing)} unexpected={len(unexpected)}")
if unexpected:
log(f" unexpected keys (前3): {unexpected[:3]}")
dm.to(dev)
dm.eval()
q = {"t": "noul", "ins": INSTR, "crit": None}
@torch.no_grad()
def score(rows):
probs, labels = [], []
B = 8
buf = []
probs, labels = [], []
def flush():
nonlocal buf, probs, labels
if not buf:
return
ids_l, mm_l, mp_l, lab_l, lens = [], [], [], [], []
for r in buf:
state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:2500]}"
ids, markers = build_sequence(tok, state, q, 512, 192)
pad = 512 - len(ids)
ids_l.append(ids + [0] * pad)
lens.append(len(ids))
mm_l.append([True] * len(markers) + [False] * (8 - len(markers)))
mp_l.append((markers + [0] * (8 - len(markers)))[:8])
lab_l.append(float(r["hard_label"]))
b = {
"input_ids": torch.tensor(ids_l).to(dev),
"attention_mask": torch.tensor(
[[1] * L + [0] * (512 - L) for L in lens]).to(dev),
"marker_pos": torch.tensor(mp_l).to(dev),
"marker_mask": torch.tensor(mm_l).bool().to(dev),
"qtype": torch.tensor([2] * len(ids_l)).to(dev),
}
logits, _ = dm(**b)
p = torch.softmax(logits.float(), -1)[..., 1]
probs += p.tolist()
labels += lab_l
buf = []
for r in rows:
buf.append(r)
if len(buf) >= B:
flush()
flush()
return probs, labels
def report(name, rows):
rows = [r for r in rows if r.get("hard_label") in (0, 1)]
if not rows:
log(f"{name}: no rows")
return
probs, labels = score(rows)
a, e = auc(probs, labels), ece(probs, labels)
pos_p = [p for p, y in zip(probs, labels) if y == 1]
neg_p = [p for p, y in zip(probs, labels) if y == 0]
log(f"{name}: n={len(rows)} pos={len(pos_p)} neg={len(neg_p)} "
f"AUC={a:.4f} ECE={e:.4f} "
f"mean_pos={sum(pos_p)/max(1,len(pos_p)):.3f} mean_neg={sum(neg_p)/max(1,len(neg_p)):.3f}")
return {"name": name, "n": len(rows), "auc": round(a, 4), "ece": round(e, 4)}
results = []
results.append(report("eval_v2(rdk_eval, OOD)", load_jsonl(EVAL_V2)))
for c in CANARY:
if os.path.exists(c):
results.append(report(os.path.basename(c), load_jsonl(c)))
cal = load_jsonl(CALIB)
results.append(report("calibration[:2000](ID)", cal[:2000]))
json.dump(results, open("/tmp/r36_eval_results.json", "w"), ensure_ascii=False, indent=1)
log("saved /tmp/r36_eval_results.json")
if __name__ == "__main__":
main()