From a82f4ecf426afdc985c5912d93daf24336cdb27f Mon Sep 17 00:00:00 2001 From: "dingbin.wang" Date: Thu, 5 Nov 2026 15:38:11 +0800 Subject: [PATCH] standalone + all-in-one scripts --- run_fat.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ train_standalone.py | 151 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 run_fat.py create mode 100644 train_standalone.py diff --git a/run_fat.py b/run_fat.py new file mode 100644 index 0000000..c25f71f --- /dev/null +++ b/run_fat.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""do_all.py — 一体化: 安装依赖 + 下载 mmBERT(ModelScope) + 训练 + 评估 +在 fat 容器内运行。所有数据和配置通过 gitea clone 获取 (/tmp/jd)。 +""" +import subprocess, sys, os, json + +def sh(cmd, check=True): + r = subprocess.run(cmd, shell=True, capture_output=True, text=True) + if check and r.returncode != 0: + print(f"FAIL: {cmd[:80]}\n stderr: {r.stderr[:200]}", flush=True) + sys.exit(1) + return r + +# === Step 1: 安装依赖 === +print("=== Step 1: Install dependencies ===", flush=True) +deps = ["transformers==4.57.6", "tokenizers>=0.21", "peft", "sentencepiece", + "accelerate", "datasets", "safetensors", "huggingface_hub", "laya"] +r = subprocess.run([sys.executable, "-m", "pip", "install", "-q"] + deps, capture_output=True, text=True) +if r.returncode != 0: + r = subprocess.run([sys.executable, "-m", "pip", "install", "-q", "-i", + "https://pypi.tuna.tsinghua.edu.cn/simple"] + deps, capture_output=True, text=True) +if r.returncode != 0: + print("FAIL-pip", r.stderr[:200], flush=True); sys.exit(1) +subprocess.run([sys.executable, "-m", "pip", "install", "-q", "--no-deps", "torchaudio"], capture_output=True) +import torch +from transformers import AutoTokenizer, AutoConfig, AutoModel +from peft import LoraConfig, get_peft_model +print(f"torch={torch.__version__} cuda={torch.cuda.is_available()}", flush=True) + +# 验证 laya +from laya.common import build_sequence +print("build_sequence OK", flush=True) + +# === Step 2: 下载 mmBERT (ModelScope 直链, 无 hf-mirror 依赖) === +print("=== Step 2: Download mmBERT from ModelScope ===", flush=True) +import urllib.request +MS_BASE = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main" +os.makedirs("/tmp/mmbert", exist_ok=True) +for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "pytorch_model.bin"]: + dst = os.path.join("/tmp/mmbert", fn) + if os.path.exists(dst) and os.path.getsize(dst) > 100: + print(f" {fn} cached ({os.path.getsize(dst)//1024//1024}MB)", flush=True) + continue + with urllib.request.urlopen(f"{MS_BASE}/{fn}", timeout=180) as resp, open(dst + ".tmp", "wb") as f: + total = 0 + while True: + b = resp.read(8 * 1024 * 1024) + if not b: break + f.write(b); total += len(b) + os.replace(dst + ".tmp", dst) + print(f" {fn}: {total//1024//1024}MB", flush=True) + +# === Step 3: 读取数据 === +print("=== Step 3: Load data ===", flush=True) +data_dir = "/tmp/jd" +train = [json.loads(l) for l in open(os.path.join(data_dir, "train.jsonl"), encoding="utf-8") if json.loads(l).get("hard_label") in (0, 1)] +calib = load_jsonl_safe(os.path.join(data_dir, "calibration.jsonl")) if os.path.exists(os.path.join(data_dir, "calibration.jsonl")) else [] +print(f" train={len(train)} calib={len(calib)}", flush=True) + +def load_jsonl_safe(p): + return [json.loads(l) for l in open(p, encoding="utf-8")] + +# === Step 4: 训练 === +print("=== Step 4: Training ===", flush=True) +import torch +import torch.nn.functional as F +from torch.utils.data import Dataset, DataLoader +from peft import LoraConfig, get_peft_model +import bisect + +INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content." +PUNCT = __import__("re").compile(r"[\s,。!?;:、\"'""''()()\[\]【】《》<>\-—·…,.:;!?|\\/]+") + +def norm_bigrams(s): + t = PUNCT.sub("", s).lower() + if len(t) < 2: return {t} if t else set() + return set(t[i:i+2] for i in range(len(t)-1)) | set(__import__("re").findall(r"[a-z0-9]{2,}", t)) + +class NoulDS(Dataset): + def __init__(self, rows, tok, max_len=512): + self.rows, self.tok, self.max_len = rows, tok, max_len + self.q = {"t": "noul", "ins": INSTR, "crit": None} + def __len__(self): return len(self.rows) + def __getitem__(self, i): + r = self.rows[i] + state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:2500]}" + ids, markers = build_sequence(self.tok, state, self.q, self.max_len, 192) + pad = self.max_len - len(ids) + mm = [True]*len(markers) + [False]*(8-len(markers)) + return {"input_ids": torch.tensor(ids + [0]*pad), + "attention_mask": torch.tensor([1]*len(ids) + [0]*pad), + "marker_pos": torch.tensor(markers + [0]*(8-len(markers))), + "marker_mask": torch.tensor(mm), + "label": torch.tensor(float(r["hard_label"])), + "soft": torch.tensor(float(r.get("soft_label") or r["hard_label"]))} + +def collate_noul(batch): + keys = batch[0].keys() + return {k: torch.stack([b[k] for b in batch]) for k in keys} + +def evaluate_model(dm, loader, dev): + dm.eval() + probs, labels = [], [] + with torch.no_grad(): + for b in loader: + logits, _ = dm(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].bool().to(dev), + marker_pos=b["marker_pos"].to(dev), marker_mask=b["marker_mask"].bool().to(dev), + qtype=b["qtype"].to(dev)) + p = torch.softmax(logits.float(), -1)[..., 1] + probs += p.tolist(); labels += b["label"].tolist() + dm.train() + pos = sorted(p for p, y in zip(probs, labels) if y == 1) + neg = sorted(p for p, y in zip(probs, labels) if y == 0) + auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos)*len(neg)) if pos and neg else 0 + e = ece(probs, labels) + dm.eval() # 保持 train 状态在调用方管理 + return {"auc": round(auc, 4), "ece": round(e, 4), "n": len(labels)} + +dev = "cuda" if torch.cuda.is_available() else "cpu" +tok = AutoTokenizer.from_pretrained("/tmp/mmbert") +enc_cfg = AutoConfig.from_pretrained("/tmp/mmbert") +enc = AutoModel.from_config(enc_cfg) +from laya.common import DecisionModel +dm = DecisionModel(enc, head_layers=2, n_act=2) +lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, target_modules=["Wqkv", "Wo"], task_type=None) +dm.encoder = get_peft_model(dm.encoder, lora) +dm.to(dev); dm.train() +print(f"model on {dev}, trainable {sum(p.numel() for p in dm.parameters() if p.requires_grad):,}", flush=True) + +dl = DataLoader(NoulDS(train, tok), batch_size=16, shuffle=True, num_workers=2, drop_last=True, collate_fn=collate_noul) +dl_cal = DataLoader(NoulDS(calib, tok), batch_size=32, collate_fn=collate_noul) + +steps = len(dl) * 2 +opt = torch.optim.AdamW([p for p in dm.parameters() if p.requires_grad], lr=1e-4, weight_decay=0.01) +from torch.optim.lr_scheduler import CosineAnnealingLR +sch = CosineAnnealingLR(opt, T_max=steps) +best_auc = 0.0 +gstep = 0 +for ep in range(2): + dm.train() + for b in dl: + with torch.autocast("cuda", enabled=(dev=="cuda")): + logits, act = dm(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].bool().to(dev), + marker_pos=b["marker_pos"].to(dev), marker_mask=b["marker_mask"].bool().to(dev), + qtype=b["qtype"].to(dev)) + true_logit = logits[..., 1].float() + hard = b["label"].to(dev); soft = b["soft"].to(dev) + loss = F.binary_cross_entropy_with_logits(true_logit, hard) + 0.5*F.mse_loss(torch.sigmoid(true_logit), soft) + opt.zero_grad(); loss.backward() + torch.nn.utils.clip_grad_norm_([p for p in dm.parameters() if p.requires_grad], 1.0) + opt.step(); sch.step() + gstep += 1 + if gstep % 100 == 0: + print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True) + m = evaluate_model(dm, dl_cal, dev) + print(f"[jev] ep{ep} calib {m}", flush=True) + if m["auc"] > best_auc: + best_auc = m["auc"] + torch.save(dm.state_dict(), "/tmp/laya_out/model.pt") +print(f"[jev] DONE best_auc={best_auc:.4f}", flush=True) +print("ALL-TRAINING-DONE", flush=True) + +if __name__ == "__main__": + main() diff --git a/train_standalone.py b/train_standalone.py new file mode 100644 index 0000000..9ac8f7c --- /dev/null +++ b/train_standalone.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""train_all.py — 一体化: 下载 mmBERT + 训练 + 评估 (noul 二分类)""" +import json, os, sys, urllib.request +import torch +import torch.nn.functional as F +from torch.utils.data import Dataset, DataLoader +from transformers import AutoTokenizer, AutoConfig, AutoModel +from peft import LoraConfig, get_peft_model +import bisect + +INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content." + +def load_jsonl(p): + return [json.loads(l) for l in open(p, encoding="utf-8")] + +def build_seq(tok, state, q, max_len, head_len): + mask_tok = tok.mask_token + opts = ["false", "true"] + ins = str(q["ins"]).replace(mask_tok, " ") + head_ids = tok("%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"] + opt_ids = [] + for o in opts: + opt_ids.append([tok.mask_token_id] + tok(" "+o.replace(mask_tok," "), add_special_tokens=False)["input_ids"][:48]) + budget = head_len - sum(len(o) for o in opt_ids) + if budget < 16: + per = max(4, (head_len-16)//2) + opt_ids = [o[:per] for o in opt_ids] + budget = head_len - sum(len(o) for o in opt_ids) + head_ids = head_ids[:max(8, budget)] + ids = [tok.cls_token_id] + head_ids + [tok.sep_token_id] + markers = [len(ids)] + ids.extend(opt_ids[1]) + ids.append(tok.sep_token_id) + room = max(0, max_len - len(ids) - 1) + st = tok(state.replace(mask_tok, " "), add_special_tokens=False)["input_ids"] + st = st[:room] + ids = ids + st + [tok.sep_token_id] + return ids[:max_len], [m for m in markers if m < max_len] + +class NoulDS(Dataset): + def __init__(self, rows, tok, max_len=512): + self.rows, self.tok, self.max_len = rows, tok, max_len + self.q = {"t": "noul", "ins": INSTR, "crit": None} + def __len__(self): return len(self.rows) + def __getitem__(self, i): + r = self.rows[i] + state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:2500]}" + ids, markers = build_seq(self.tok, state, self.q, self.max_len, 192) + pad = self.max_len - len(ids) + mm = [True]*len(markers) + [False]*(8-len(markers)) + return {"input_ids": torch.tensor(ids + [0]*pad), + "attention_mask": torch.tensor([1]*len(ids) + [0]*pad), + "marker_pos": torch.tensor(markers + [0]*(8-len(markers))), + "marker_mask": torch.tensor(mm), + "label": torch.tensor(float(r["hard_label"])), + "soft": torch.tensor(float(r.get("soft_label") or r["hard_label"]))} + +def collate_noul(batch): + keys = batch[0].keys() + return {k: torch.stack([b[k] for b in batch]) for k in keys} + +def ece(probs, labels): + bins = [[] for _ in range(15)] + for p, y in zip(probs, labels): + bins[min(int(p*15), 14)].append((p, y)) + e = 0.0 + for b in bins: + if not b: continue + e += len(b)/len(probs) * abs(sum(p for p,_ in b)/len(b) - sum(y for _,y in b)/len(b)) + return e + +def main(): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--data", default="/tmp/jd") + ap.add_argument("--base", default="/tmp/mmbert") + ap.add_argument("--out", default="/tmp/laya_out") + ap.add_argument("--epochs", type=int, default=2) + ap.add_argument("--bs", type=int, default=16) + ap.add_argument("--lr", type=float, default=1e-4) + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + + dev = "cuda" if torch.cuda.is_available() else "cpu" + print(f"[jev] device={dev}", flush=True) + + tok = AutoTokenizer.from_pretrained(args.base) + enc_cfg = AutoConfig.from_pretrained(args.base) + enc = AutoModel.from_config(enc_cfg) + from laya.common import DecisionModel + dm = DecisionModel(enc, head_layers=2, n_act=2) + lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, target_modules=["Wqkv", "Wo"], task_type=None) + dm.encoder = get_peft_model(dm.encoder, lora) + dm.to(dev); dm.train() + print(f"[jev] model ready on {dev}", flush=True) + + train = [r for r in load_jsonl(os.path.join(args.data, "train.jsonl")) if r.get("hard_label") in (0, 1)] + calib = load_jsonl(os.path.join(args.data, "calibration.jsonl")) + print(f"[jev] train={len(train)} calib={len(calib)}", flush=True) + + dl = DataLoader(NoulDS(train, tok), batch_size=args.bs, shuffle=True, num_workers=2, drop_last=True, collate_fn=collate_noul) + dl_cal = DataLoader(NoulDS(calib, tok), batch_size=32, collate_fn=collate_noul) + + steps = len(dl) * args.epochs + opt = torch.optim.AdamW([p for p in dm.parameters() if p.requires_grad], lr=args.lr, weight_decay=0.01) + sch = get_cosine_schedule_with_warmup(opt, int(0.06*steps), steps) + best_auc = 0.0 + gstep = 0 + for ep in range(args.epochs): + dm.train() + for b in dl: + with torch.autocast("cuda", enabled=(dev=="cuda")): + logits, act = dm(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].to(dev), + marker_pos=b["marker_pos"].to(dev), marker_mask=b["marker_mask"].to(dev), + qtype=b["qtype"].to(dev)) + true_logit = logits[..., 1].float() + hard = b["label"].to(dev); soft = b["soft"].to(dev) + loss = F.binary_cross_entropy_with_logits(true_logit, hard) + 0.5*F.mse_loss(torch.sigmoid(true_logit), soft) + opt.zero_grad(); loss.backward() + torch.nn.utils.clip_grad_norm_([p for p in dm.parameters() if p.requires_grad], 1.0) + opt.step(); sch.step() + gstep += 1 + if gstep % 100 == 0: + print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True) + dm.eval() + ps, ls = [], [] + with torch.no_grad(): + for b in dl_cal: + lg, _ = dm(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].to(dev), + marker_pos=b["marker_pos"].to(dev), marker_mask=b["marker_mask"].to(dev), + qtype=b["qtype"].to(dev)) + p = torch.softmax(lg.float(), -1)[..., 1] + ps += p.tolist(); ls += b["label"].tolist() + pos = sorted(p for p, y in zip(ps, ls) if y == 1) + neg = sorted(p for p, y in zip(ps, ls) if y == 0) + auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos)*len(neg)) if pos and neg else 0 + e = ece(ps, ls) + print(f"[jev] ep{ep} calib AUC={auc:.4f} ECE={e:.4f}", flush=True) + if auc > best_auc: + best_auc = auc + torch.save(dm.state_dict(), os.path.join(args.out, "model.pt")) + print(f"[jev] saved best (auc={auc:.4f})", flush=True) + dm.train() + print(f"[jev] TRAINING-DONE best_auc={best_auc:.4f}", flush=True) + +def collate_noul(batch): + keys = batch[0].keys() + return {k: torch.stack([b[k] for b in batch]) for k in keys} + +if __name__ == "__main__": + main()