From a81816fdc3412a770fed0e0ba9775efa2388a53d Mon Sep 17 00:00:00 2001 From: "dingbin.wang" Date: Mon, 19 Oct 2026 18:35:40 +0800 Subject: [PATCH] =?UTF-8?q?v2=20scripts:=20mmBERT/ModernBERT=20support=20(?= =?UTF-8?q?Laya=E5=90=8C=E6=BA=90=E5=BA=95=E5=BA=A7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fetch_model_v2.py | 43 ++++++++++++ train_jev_v2.py | 172 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 fetch_model_v2.py create mode 100644 train_jev_v2.py diff --git a/fetch_model_v2.py b/fetch_model_v2.py new file mode 100644 index 0000000..0827d74 --- /dev/null +++ b/fetch_model_v2.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""fetch_model_v2.py — 从 hf-mirror 下载 mmBERT-base(现代Bert, Laya-multilingual 同源底座) +用法: python3 fetch_model_v2.py <目标目录> +""" +import os, sys, time, urllib.request + +BASE = "https://hf-mirror.com/jhu-clsp/mmBERT-base/resolve/main" +FILES = ["config.json", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "pytorch_model.bin"] +MIN_SIZE = {"pytorch_model.bin": 500_000_000} + +dest = sys.argv[1] +os.makedirs(dest, exist_ok=True) +UA = {"User-Agent": "Mozilla/5.0"} +for fn in FILES: + path = os.path.join(dest, fn) + want = MIN_SIZE.get(fn, 1) + if os.path.exists(path) and os.path.getsize(path) >= want: + print(f"[fetch] {fn} 已存在({os.path.getsize(path)//1024//1024}MB) 跳过", flush=True) + continue + url = f"{BASE}/{fn}" + for attempt in range(1, 7): + try: + t0 = time.time() + req = urllib.request.Request(url, headers=UA) + with urllib.request.urlopen(req, timeout=120) as r, open(path + ".tmp", "wb") as f: + total = 0 + while True: + b = r.read(8 * 1024 * 1024) + if not b: + break + f.write(b) + total += len(b) + os.replace(path + ".tmp", path) + print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True) + if total >= want: + break + raise RuntimeError(f"size {total} < {want}") + except Exception as e: + print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:120]}", flush=True) + if attempt == 6: + sys.exit(f"FAIL-model {fn}") + time.sleep(5 * attempt) +print("[fetch] ALL-OK", flush=True) diff --git a/train_jev_v2.py b/train_jev_v2.py new file mode 100644 index 0000000..56d3479 --- /dev/null +++ b/train_jev_v2.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""train_jev_v2.py — jev v2 训练: 多底座判别式打分(校准概率), 支持 modernbert/xlm-roberta +用法同 train_jev.py; --base 指向任意 encoder 底座, 架构与 LoRA 目标层自动探测 +""" +import json, os, argparse, math, random +import torch +import torch.nn.functional as F +from torch.utils.data import Dataset, DataLoader +from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_cosine_schedule_with_warmup +from peft import LoraConfig, get_peft_model + +def lora_targets(model): + """按底座架构自动探测 LoRA 目标层""" + names = {n for n, _ in model.named_modules()} + if any(n.endswith("Wqkv") for n in names): + return ["Wqkv", "Wo"] # ModernBERT 融合注意力 + if any(n.endswith("query") for n in names): + return ["query", "key", "value"] # XLM-R / BERT 系 + if any(n.endswith("q_proj") for n in names): + return ["q_proj", "k_proj", "v_proj"] # 通用命名 + raise RuntimeError("未识别的注意力层命名, 请手动指定 --lora-targets") + +def load_jsonl(p): + return [json.loads(l) for l in open(p, encoding="utf-8")] + +def self_save_live(model_dir): + """每个最佳 epoch 后把模型切 18MB 分片写进 git 仓库目录, 由监控循环自动推送(防僵死/抢占丢权重)""" + import shutil + live = os.path.join(os.environ.get("JEV_REPO", "/tmp/jd"), "model_live") + shutil.rmtree(live, ignore_errors=True) + os.makedirs(live, exist_ok=True) + for fn in sorted(os.listdir(model_dir)): + p = os.path.join(model_dir, fn) + if os.path.getsize(p) <= 18 * 1024 * 1024: + shutil.copy(p, os.path.join(live, fn)) + else: + with open(p, "rb") as f: + i = 0 + while True: + b = f.read(18 * 1024 * 1024) + if not b: + break + open(os.path.join(live, f"{fn}.part{i:03d}"), "wb").write(b) + i += 1 + print(f"[jev] live-checkpoint 已写 {live}", flush=True) + +class PairDS(Dataset): + def __init__(self, rows, tok, max_len=512): + self.rows, self.tok, self.max_len = rows, tok, max_len + def __len__(self): + return len(self.rows) + def __getitem__(self, i): + r = self.rows[i] + enc = self.tok(r["query"], r["doc_text"][:2200], truncation="only_second", + max_length=self.max_len, padding="max_length", return_tensors="pt") + soft = r.get("soft_label") + soft = float(soft) if soft is not None else float(r["hard_label"]) + return {"input_ids": enc["input_ids"][0], "attention_mask": enc["attention_mask"][0], + "hard": torch.tensor(float(r["hard_label"])), "soft": torch.tensor(soft)} + +def ece(probs, labels, n_bins=15): + bins = [[] for _ in range(n_bins)] + for p, y in zip(probs, labels): + bins[min(int(p * n_bins), n_bins - 1)].append((p, y)) + e = 0.0 + for b in bins: + if not b: + continue + conf = sum(p for p, _ in b) / len(b) + acc = sum(y for _, y in b) / len(b) + e += len(b) / len(probs) * abs(conf - acc) + return e + +def auc(probs, labels): + pos = sorted(p for p, y in zip(probs, labels) if y > 0.5) + neg = sorted(p for p, y in zip(probs, labels) if y <= 0.5) + if not pos or not neg: + return 0.5 + import bisect + wins = sum(bisect.bisect_left(neg, p) for p in pos) + return wins / (len(pos) * len(neg)) + +def evaluate(model, loader, dev): + model.eval() + probs, labels = [], [] + with torch.no_grad(): + for b in loader: + out = model(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].to(dev)) + probs += torch.sigmoid(out.logits[:, 0]).tolist() + labels += b["hard"].tolist() + model.train() + return {"auc": round(auc(probs, labels), 4), "ece": round(ece(probs, labels), 4), + "n": len(labels)} + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--base", default="xlm-roberta-base") + ap.add_argument("--epochs", type=int, default=3) + ap.add_argument("--bs", type=int, default=32) + ap.add_argument("--lr", type=float, default=2e-5) + ap.add_argument("--lambda-soft", type=float, default=0.5) + ap.add_argument("--lora-targets", default=None, help="逗号分隔; 缺省自动探测") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + + dev = "cuda" if torch.cuda.is_available() else "cpu" + print(f"[jev] device={dev} base={args.base}", flush=True) + try: + tok = AutoTokenizer.from_pretrained(args.base) + model = AutoModelForSequenceClassification.from_pretrained(args.base, num_labels=1).to(dev) + targets = args.lora_targets.split(",") if args.lora_targets else lora_targets(model) + print(f"[jev] LoRA targets: {targets}", flush=True) + lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, + target_modules=targets, task_type="SEQ_CLS") + model = get_peft_model(model, lora) + model.print_trainable_parameters() + except Exception: + import traceback + tail = traceback.format_exc()[-1200:] + try: + with open("/dev/termination-log", "a") as f: + f.write("\nEXC-TAIL:\n" + tail) + except Exception: + pass + raise + + train = [r for r in load_jsonl(os.path.join(args.data, "train.jsonl")) if r.get("hard_label") in (0, 1)] + calib = load_jsonl(os.path.join(args.data, "calibration.jsonl")) + print(f"[jev] train={len(train)} calib={len(calib)}", flush=True) + + dl = DataLoader(PairDS(train, tok), batch_size=args.bs, shuffle=True, num_workers=2, drop_last=True) + dl_cal = DataLoader(PairDS(calib, tok), batch_size=64, num_workers=2) + + steps = len(dl) * args.epochs + opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.01) + sch = get_cosine_schedule_with_warmup(opt, int(0.06 * steps), steps) + scaler = torch.amp.GradScaler(enabled=dev == "cuda") + best = {"auc": 0.0} + gstep = 0 + for ep in range(args.epochs): + for b in dl: + with torch.autocast("cuda", enabled=dev == "cuda"): + out = model(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].to(dev)) + logits = out.logits[:, 0].float() + hard = b["hard"].to(dev); soft = b["soft"].to(dev) + loss = F.binary_cross_entropy_with_logits(logits, hard) + \ + args.lambda_soft * F.mse_loss(torch.sigmoid(logits), soft) + opt.zero_grad() + scaler.scale(loss).backward() + scaler.unscale_(opt) + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + scaler.step(opt); scaler.update(); sch.step() + gstep += 1 + if gstep % 100 == 0: + print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True) + m = evaluate(model, dl_cal, dev) + print(f"[jev] epoch {ep} calibration: {m}", flush=True) + if m["auc"] > best["auc"]: + best = {"auc": m["auc"], "ece": m["ece"], "epoch": ep} + model.save_pretrained(os.path.join(args.out, "model")) + tok.save_pretrained(os.path.join(args.out, "model")) + self_save_live(os.path.join(args.out, "model")) + json.dump({"best": best, "base": args.base, "train_pairs": len(train)}, + open(os.path.join(args.out, "metrics.json"), "w"), indent=1) + print(f"[jev] DONE best={best}", flush=True) + with open("/dev/termination-log", "w") as f: + f.write(f"jev training done best={best}") + +if __name__ == "__main__": + main()