237 lines
9.3 KiB
Python
237 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
"""train_all.py — 一体化: 下载 mmBERT + 训练 + 评估 (noul 二分类, Laya DecisionModel)
|
|
在 fat 容器内运行。数据和 laya_repo 模块通过 gitea clone 获取 (/tmp/jd)。
|
|
用法: python3 train_all.py [--epochs N] [--lr LR] [--bs N]
|
|
"""
|
|
import json, os, sys, bisect
|
|
|
|
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
|
|
DATA_DIR = "/tmp/jd"
|
|
MMBERT_DIR = "/tmp/mmbert"
|
|
OUT_DIR = "/tmp/laya_out"
|
|
|
|
def log(msg):
|
|
print(f"[jev] {msg}", flush=True)
|
|
|
|
def load_jsonl(p):
|
|
rows = []
|
|
with open(p, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
rows.append(json.loads(line))
|
|
return rows
|
|
|
|
def ece(probs, labels, nbins=15):
|
|
bins = [[] for _ in range(nbins)]
|
|
for p, y in zip(probs, labels):
|
|
bins[min(int(p * nbins), nbins - 1)].append((p, y))
|
|
e = 0.0
|
|
for b in bins:
|
|
if not b:
|
|
continue
|
|
e += len(b) / len(probs) * abs(
|
|
sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
|
|
return e
|
|
|
|
def phase_env():
|
|
"""把 gitea clone 的 laya_repo 注入 sys.path, 优先于 pip 版"""
|
|
log("Phase 1: env setup")
|
|
repo = os.path.join(DATA_DIR, "laya_repo")
|
|
if not os.path.isdir(repo):
|
|
log(f" FATAL: {repo} 不存在")
|
|
return False
|
|
src = "/tmp/laya_src"
|
|
pkg = os.path.join(src, "laya")
|
|
os.makedirs(pkg, exist_ok=True)
|
|
for fn in os.listdir(repo):
|
|
if fn.endswith(".py"):
|
|
with open(os.path.join(repo, fn), "rb") as f:
|
|
open(os.path.join(pkg, fn), "wb").write(f.read())
|
|
init = os.path.join(pkg, "__init__.py")
|
|
if not os.path.exists(init):
|
|
open(init, "w").write("")
|
|
sys.path.insert(0, src)
|
|
from laya.common import build_sequence, DecisionModel
|
|
log(" laya.common OK")
|
|
return True
|
|
|
|
def phase_mmbert():
|
|
"""ModelScope 直链下载 mmBERT"""
|
|
import urllib.request, time
|
|
log("Phase 2: download mmBERT (ModelScope)")
|
|
MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main"
|
|
os.makedirs(MMBERT_DIR, exist_ok=True)
|
|
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json",
|
|
"special_tokens_map.json", "pytorch_model.bin"]:
|
|
dst = os.path.join(MMBERT_DIR, fn)
|
|
min_size = 300_000_000 if fn.endswith(".bin") else 100
|
|
if os.path.exists(dst) and os.path.getsize(dst) >= min_size:
|
|
log(f" {fn} cached ({os.path.getsize(dst)//1024//1024}MB)")
|
|
continue
|
|
ok = False
|
|
for attempt in range(1, 5):
|
|
try:
|
|
with urllib.request.urlopen(f"{MS}/{fn}", timeout=300) as r, \
|
|
open(dst + ".tmp", "wb") as f:
|
|
total = 0
|
|
while True:
|
|
b = r.read(8 * 1024 * 1024)
|
|
if not b:
|
|
break
|
|
f.write(b)
|
|
total += len(b)
|
|
os.replace(dst + ".tmp", dst)
|
|
log(f" {fn}: {total//1024//1024}MB")
|
|
ok = total >= min_size
|
|
break
|
|
except Exception as e:
|
|
log(f" {fn} attempt {attempt}: {str(e)[:80]}")
|
|
time.sleep(15 * attempt)
|
|
if not ok:
|
|
return False
|
|
return True
|
|
|
|
def build_model(dev):
|
|
log("Phase 3: build DecisionModel")
|
|
import torch
|
|
from transformers import AutoTokenizer, AutoConfig, AutoModel
|
|
from peft import LoraConfig, get_peft_model
|
|
from laya.common import DecisionModel
|
|
|
|
tok = AutoTokenizer.from_pretrained(MMBERT_DIR)
|
|
enc = AutoModel.from_pretrained(MMBERT_DIR)
|
|
dm = DecisionModel(enc, head_layers=2, n_act=2)
|
|
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
|
|
target_modules=["Wqkv", "Wo"], task_type=None)
|
|
dm.encoder = get_peft_model(dm.encoder, lora)
|
|
dm.to(dev)
|
|
dm.train()
|
|
n = sum(p.numel() for p in dm.parameters() if p.requires_grad)
|
|
log(f" model on {dev}, trainable {n:,}")
|
|
return dm, tok
|
|
|
|
def run(args):
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from torch.utils.data import Dataset, DataLoader
|
|
from transformers import get_cosine_schedule_with_warmup
|
|
from laya.common import build_sequence
|
|
|
|
if not phase_env():
|
|
sys.exit(1)
|
|
if not phase_mmbert():
|
|
sys.exit(1)
|
|
|
|
dev = "cuda" if torch.cuda.is_available() else "cpu"
|
|
log(f"device={dev}")
|
|
dm, tok = build_model(dev)
|
|
|
|
class NoulDS(Dataset):
|
|
def __init__(self, rows, max_len=512, head_len=192, doc_cap=2500):
|
|
self.rows, self.max_len, self.head_len, self.doc_cap = rows, max_len, head_len, doc_cap
|
|
self.q = {"t": "noul", "ins": INSTR, "crit": None}
|
|
def __len__(self):
|
|
return len(self.rows)
|
|
def __getitem__(self, i):
|
|
r = self.rows[i]
|
|
state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:self.doc_cap]}"
|
|
ids, markers = build_sequence(tok, state, self.q, self.max_len, self.head_len)
|
|
pad = self.max_len - len(ids)
|
|
mm = [True] * len(markers) + [False] * (8 - len(markers))
|
|
return {"input_ids": torch.tensor(ids + [0] * pad),
|
|
"attention_mask": torch.tensor([1] * len(ids) + [0] * pad),
|
|
"marker_pos": torch.tensor(markers + [0] * (8 - len(markers))),
|
|
"marker_mask": torch.tensor(mm),
|
|
"label": torch.tensor(float(r["hard_label"])),
|
|
"soft": torch.tensor(float(r.get("soft_label") or r["hard_label"]))}
|
|
|
|
def collate(batch):
|
|
return {k: torch.stack([b[k] for b in batch]) for k in batch[0]}
|
|
|
|
def fwd(b, no_grad=False):
|
|
ctx = torch.no_grad() if no_grad else torch.enable_grad()
|
|
with ctx:
|
|
with torch.autocast("cuda", enabled=(dev == "cuda")):
|
|
logits, _ = dm(input_ids=b["input_ids"].to(dev),
|
|
attention_mask=b["attention_mask"].bool().to(dev),
|
|
marker_pos=b["marker_pos"].to(dev),
|
|
marker_mask=b["marker_mask"].bool().to(dev),
|
|
qtype=b["qtype"].to(dev))
|
|
return logits
|
|
|
|
def evaluate(loader):
|
|
was_training = dm.training
|
|
dm.eval()
|
|
probs, labels = [], []
|
|
for b in loader:
|
|
p = torch.softmax(fwd(b, no_grad=True).float(), -1)[..., 1]
|
|
probs += p.tolist()
|
|
labels += b["label"].tolist()
|
|
if was_training:
|
|
dm.train()
|
|
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
|
|
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
|
|
auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg)) if pos and neg else 0.0
|
|
return {"auc": round(auc, 4), "ece": round(ece(probs, labels), 4), "n": len(labels)}
|
|
|
|
train = [r for r in load_jsonl(os.path.join(DATA_DIR, "train.jsonl"))
|
|
if r.get("hard_label") in (0, 1)]
|
|
calib = load_jsonl(os.path.join(DATA_DIR, "calibration.jsonl"))
|
|
log(f"train={len(train)} calib={len(calib)}")
|
|
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
dl = DataLoader(NoulDS(train), batch_size=args.bs, shuffle=True,
|
|
num_workers=2, drop_last=True, collate_fn=collate)
|
|
dl_cal = DataLoader(NoulDS(calib), batch_size=32, collate_fn=collate)
|
|
|
|
steps = len(dl) * args.epochs
|
|
opt = torch.optim.AdamW([p for p in dm.parameters() if p.requires_grad],
|
|
lr=args.lr, weight_decay=0.01)
|
|
sch = get_cosine_schedule_with_warmup(opt, int(0.06 * steps), steps)
|
|
|
|
m0 = evaluate(dl_cal)
|
|
log(f"ep-1 pretrain calib {m0}")
|
|
best_auc = m0["auc"]
|
|
|
|
gstep = 0
|
|
for ep in range(args.epochs):
|
|
dm.train()
|
|
for b in dl:
|
|
logits = fwd(b)
|
|
true_logit = logits[..., 1].float()
|
|
hard = b["label"].to(dev)
|
|
soft = b["soft"].to(dev)
|
|
loss = F.binary_cross_entropy_with_logits(true_logit, hard) + \
|
|
0.5 * F.mse_loss(torch.sigmoid(true_logit), soft)
|
|
opt.zero_grad()
|
|
loss.backward()
|
|
torch.nn.utils.clip_grad_norm_(
|
|
[p for p in dm.parameters() if p.requires_grad], 1.0)
|
|
opt.step()
|
|
sch.step()
|
|
gstep += 1
|
|
if gstep % 100 == 0:
|
|
print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True)
|
|
m = evaluate(dl_cal)
|
|
print(f"[jev] ep{ep} calib AUC={m['auc']:.4f} ECE={m['ece']:.4f}", flush=True)
|
|
if m["auc"] > best_auc:
|
|
best_auc = m["auc"]
|
|
torch.save(dm.state_dict(), os.path.join(OUT_DIR, "model.pt"))
|
|
log(f"saved best (auc={best_auc:.4f})")
|
|
|
|
log(f"TRAINING-DONE best_auc={best_auc:.4f}")
|
|
json.dump({"best_auc": best_auc, "train_pairs": len(train),
|
|
"steps": gstep, "epochs": args.epochs},
|
|
open(os.path.join(OUT_DIR, "training_log.json"), "w"),
|
|
ensure_ascii=False, indent=1)
|
|
print("ALL-DONE", flush=True)
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--epochs", type=int, default=2)
|
|
ap.add_argument("--lr", type=float, default=1e-4)
|
|
ap.add_argument("--bs", type=int, default=16)
|
|
run(ap.parse_args())
|