v2 scripts: mmBERT/ModernBERT support (Laya同源底座)
This commit is contained in:
parent
5c07346603
commit
a81816fdc3
43
fetch_model_v2.py
Normal file
43
fetch_model_v2.py
Normal file
@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""fetch_model_v2.py — 从 hf-mirror 下载 mmBERT-base(现代Bert, Laya-multilingual 同源底座)
|
||||
用法: python3 fetch_model_v2.py <目标目录>
|
||||
"""
|
||||
import os, sys, time, urllib.request
|
||||
|
||||
BASE = "https://hf-mirror.com/jhu-clsp/mmBERT-base/resolve/main"
|
||||
FILES = ["config.json", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "pytorch_model.bin"]
|
||||
MIN_SIZE = {"pytorch_model.bin": 500_000_000}
|
||||
|
||||
dest = sys.argv[1]
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
UA = {"User-Agent": "Mozilla/5.0"}
|
||||
for fn in FILES:
|
||||
path = os.path.join(dest, fn)
|
||||
want = MIN_SIZE.get(fn, 1)
|
||||
if os.path.exists(path) and os.path.getsize(path) >= want:
|
||||
print(f"[fetch] {fn} 已存在({os.path.getsize(path)//1024//1024}MB) 跳过", flush=True)
|
||||
continue
|
||||
url = f"{BASE}/{fn}"
|
||||
for attempt in range(1, 7):
|
||||
try:
|
||||
t0 = time.time()
|
||||
req = urllib.request.Request(url, headers=UA)
|
||||
with urllib.request.urlopen(req, timeout=120) as r, open(path + ".tmp", "wb") as f:
|
||||
total = 0
|
||||
while True:
|
||||
b = r.read(8 * 1024 * 1024)
|
||||
if not b:
|
||||
break
|
||||
f.write(b)
|
||||
total += len(b)
|
||||
os.replace(path + ".tmp", path)
|
||||
print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True)
|
||||
if total >= want:
|
||||
break
|
||||
raise RuntimeError(f"size {total} < {want}")
|
||||
except Exception as e:
|
||||
print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:120]}", flush=True)
|
||||
if attempt == 6:
|
||||
sys.exit(f"FAIL-model {fn}")
|
||||
time.sleep(5 * attempt)
|
||||
print("[fetch] ALL-OK", flush=True)
|
||||
172
train_jev_v2.py
Normal file
172
train_jev_v2.py
Normal file
@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""train_jev_v2.py — jev v2 训练: 多底座判别式打分(校准概率), 支持 modernbert/xlm-roberta
|
||||
用法同 train_jev.py; --base 指向任意 encoder 底座, 架构与 LoRA 目标层自动探测
|
||||
"""
|
||||
import json, os, argparse, math, random
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_cosine_schedule_with_warmup
|
||||
from peft import LoraConfig, get_peft_model
|
||||
|
||||
def lora_targets(model):
|
||||
"""按底座架构自动探测 LoRA 目标层"""
|
||||
names = {n for n, _ in model.named_modules()}
|
||||
if any(n.endswith("Wqkv") for n in names):
|
||||
return ["Wqkv", "Wo"] # ModernBERT 融合注意力
|
||||
if any(n.endswith("query") for n in names):
|
||||
return ["query", "key", "value"] # XLM-R / BERT 系
|
||||
if any(n.endswith("q_proj") for n in names):
|
||||
return ["q_proj", "k_proj", "v_proj"] # 通用命名
|
||||
raise RuntimeError("未识别的注意力层命名, 请手动指定 --lora-targets")
|
||||
|
||||
def load_jsonl(p):
|
||||
return [json.loads(l) for l in open(p, encoding="utf-8")]
|
||||
|
||||
def self_save_live(model_dir):
|
||||
"""每个最佳 epoch 后把模型切 18MB 分片写进 git 仓库目录, 由监控循环自动推送(防僵死/抢占丢权重)"""
|
||||
import shutil
|
||||
live = os.path.join(os.environ.get("JEV_REPO", "/tmp/jd"), "model_live")
|
||||
shutil.rmtree(live, ignore_errors=True)
|
||||
os.makedirs(live, exist_ok=True)
|
||||
for fn in sorted(os.listdir(model_dir)):
|
||||
p = os.path.join(model_dir, fn)
|
||||
if os.path.getsize(p) <= 18 * 1024 * 1024:
|
||||
shutil.copy(p, os.path.join(live, fn))
|
||||
else:
|
||||
with open(p, "rb") as f:
|
||||
i = 0
|
||||
while True:
|
||||
b = f.read(18 * 1024 * 1024)
|
||||
if not b:
|
||||
break
|
||||
open(os.path.join(live, f"{fn}.part{i:03d}"), "wb").write(b)
|
||||
i += 1
|
||||
print(f"[jev] live-checkpoint 已写 {live}", flush=True)
|
||||
|
||||
class PairDS(Dataset):
|
||||
def __init__(self, rows, tok, max_len=512):
|
||||
self.rows, self.tok, self.max_len = rows, tok, max_len
|
||||
def __len__(self):
|
||||
return len(self.rows)
|
||||
def __getitem__(self, i):
|
||||
r = self.rows[i]
|
||||
enc = self.tok(r["query"], r["doc_text"][:2200], truncation="only_second",
|
||||
max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||||
soft = r.get("soft_label")
|
||||
soft = float(soft) if soft is not None else float(r["hard_label"])
|
||||
return {"input_ids": enc["input_ids"][0], "attention_mask": enc["attention_mask"][0],
|
||||
"hard": torch.tensor(float(r["hard_label"])), "soft": torch.tensor(soft)}
|
||||
|
||||
def ece(probs, labels, n_bins=15):
|
||||
bins = [[] for _ in range(n_bins)]
|
||||
for p, y in zip(probs, labels):
|
||||
bins[min(int(p * n_bins), n_bins - 1)].append((p, y))
|
||||
e = 0.0
|
||||
for b in bins:
|
||||
if not b:
|
||||
continue
|
||||
conf = sum(p for p, _ in b) / len(b)
|
||||
acc = sum(y for _, y in b) / len(b)
|
||||
e += len(b) / len(probs) * abs(conf - acc)
|
||||
return e
|
||||
|
||||
def auc(probs, labels):
|
||||
pos = sorted(p for p, y in zip(probs, labels) if y > 0.5)
|
||||
neg = sorted(p for p, y in zip(probs, labels) if y <= 0.5)
|
||||
if not pos or not neg:
|
||||
return 0.5
|
||||
import bisect
|
||||
wins = sum(bisect.bisect_left(neg, p) for p in pos)
|
||||
return wins / (len(pos) * len(neg))
|
||||
|
||||
def evaluate(model, loader, dev):
|
||||
model.eval()
|
||||
probs, labels = [], []
|
||||
with torch.no_grad():
|
||||
for b in loader:
|
||||
out = model(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].to(dev))
|
||||
probs += torch.sigmoid(out.logits[:, 0]).tolist()
|
||||
labels += b["hard"].tolist()
|
||||
model.train()
|
||||
return {"auc": round(auc(probs, labels), 4), "ece": round(ece(probs, labels), 4),
|
||||
"n": len(labels)}
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--data", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--base", default="xlm-roberta-base")
|
||||
ap.add_argument("--epochs", type=int, default=3)
|
||||
ap.add_argument("--bs", type=int, default=32)
|
||||
ap.add_argument("--lr", type=float, default=2e-5)
|
||||
ap.add_argument("--lambda-soft", type=float, default=0.5)
|
||||
ap.add_argument("--lora-targets", default=None, help="逗号分隔; 缺省自动探测")
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
dev = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
print(f"[jev] device={dev} base={args.base}", flush=True)
|
||||
try:
|
||||
tok = AutoTokenizer.from_pretrained(args.base)
|
||||
model = AutoModelForSequenceClassification.from_pretrained(args.base, num_labels=1).to(dev)
|
||||
targets = args.lora_targets.split(",") if args.lora_targets else lora_targets(model)
|
||||
print(f"[jev] LoRA targets: {targets}", flush=True)
|
||||
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
|
||||
target_modules=targets, task_type="SEQ_CLS")
|
||||
model = get_peft_model(model, lora)
|
||||
model.print_trainable_parameters()
|
||||
except Exception:
|
||||
import traceback
|
||||
tail = traceback.format_exc()[-1200:]
|
||||
try:
|
||||
with open("/dev/termination-log", "a") as f:
|
||||
f.write("\nEXC-TAIL:\n" + tail)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
train = [r for r in load_jsonl(os.path.join(args.data, "train.jsonl")) if r.get("hard_label") in (0, 1)]
|
||||
calib = load_jsonl(os.path.join(args.data, "calibration.jsonl"))
|
||||
print(f"[jev] train={len(train)} calib={len(calib)}", flush=True)
|
||||
|
||||
dl = DataLoader(PairDS(train, tok), batch_size=args.bs, shuffle=True, num_workers=2, drop_last=True)
|
||||
dl_cal = DataLoader(PairDS(calib, tok), batch_size=64, num_workers=2)
|
||||
|
||||
steps = len(dl) * args.epochs
|
||||
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.01)
|
||||
sch = get_cosine_schedule_with_warmup(opt, int(0.06 * steps), steps)
|
||||
scaler = torch.amp.GradScaler(enabled=dev == "cuda")
|
||||
best = {"auc": 0.0}
|
||||
gstep = 0
|
||||
for ep in range(args.epochs):
|
||||
for b in dl:
|
||||
with torch.autocast("cuda", enabled=dev == "cuda"):
|
||||
out = model(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].to(dev))
|
||||
logits = out.logits[:, 0].float()
|
||||
hard = b["hard"].to(dev); soft = b["soft"].to(dev)
|
||||
loss = F.binary_cross_entropy_with_logits(logits, hard) + \
|
||||
args.lambda_soft * F.mse_loss(torch.sigmoid(logits), soft)
|
||||
opt.zero_grad()
|
||||
scaler.scale(loss).backward()
|
||||
scaler.unscale_(opt)
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
||||
scaler.step(opt); scaler.update(); sch.step()
|
||||
gstep += 1
|
||||
if gstep % 100 == 0:
|
||||
print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True)
|
||||
m = evaluate(model, dl_cal, dev)
|
||||
print(f"[jev] epoch {ep} calibration: {m}", flush=True)
|
||||
if m["auc"] > best["auc"]:
|
||||
best = {"auc": m["auc"], "ece": m["ece"], "epoch": ep}
|
||||
model.save_pretrained(os.path.join(args.out, "model"))
|
||||
tok.save_pretrained(os.path.join(args.out, "model"))
|
||||
self_save_live(os.path.join(args.out, "model"))
|
||||
json.dump({"best": best, "base": args.base, "train_pairs": len(train)},
|
||||
open(os.path.join(args.out, "metrics.json"), "w"), indent=1)
|
||||
print(f"[jev] DONE best={best}", flush=True)
|
||||
with open("/dev/termination-log", "w") as f:
|
||||
f.write(f"jev training done best={best}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user