jev-training-data/run_fat.py

165 lines
7.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""do_all.py — 一体化: 安装依赖 + 下载 mmBERT(ModelScope) + 训练 + 评估
在 fat 容器内运行。所有数据和配置通过 gitea clone 获取 (/tmp/jd)。
"""
import subprocess, sys, os, json
def sh(cmd, check=True):
r = subprocess.run(cmd, shell=True, capture_output=True, text=True)
if check and r.returncode != 0:
print(f"FAIL: {cmd[:80]}\n stderr: {r.stderr[:200]}", flush=True)
sys.exit(1)
return r
# === Step 1: 安装依赖 ===
print("=== Step 1: Install dependencies ===", flush=True)
deps = ["transformers==4.57.6", "tokenizers>=0.21", "peft", "sentencepiece",
"accelerate", "datasets", "safetensors", "huggingface_hub", "laya"]
r = subprocess.run([sys.executable, "-m", "pip", "install", "-q"] + deps, capture_output=True, text=True)
if r.returncode != 0:
r = subprocess.run([sys.executable, "-m", "pip", "install", "-q", "-i",
"https://pypi.tuna.tsinghua.edu.cn/simple"] + deps, capture_output=True, text=True)
if r.returncode != 0:
print("FAIL-pip", r.stderr[:200], flush=True); sys.exit(1)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "--no-deps", "torchaudio"], capture_output=True)
import torch
from transformers import AutoTokenizer, AutoConfig, AutoModel
from peft import LoraConfig, get_peft_model
print(f"torch={torch.__version__} cuda={torch.cuda.is_available()}", flush=True)
# 验证 laya
from laya.common import build_sequence
print("build_sequence OK", flush=True)
# === Step 2: 下载 mmBERT (ModelScope 直链, 无 hf-mirror 依赖) ===
print("=== Step 2: Download mmBERT from ModelScope ===", flush=True)
import urllib.request
MS_BASE = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main"
os.makedirs("/tmp/mmbert", exist_ok=True)
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "pytorch_model.bin"]:
dst = os.path.join("/tmp/mmbert", fn)
if os.path.exists(dst) and os.path.getsize(dst) > 100:
print(f" {fn} cached ({os.path.getsize(dst)//1024//1024}MB)", flush=True)
continue
with urllib.request.urlopen(f"{MS_BASE}/{fn}", timeout=180) as resp, open(dst + ".tmp", "wb") as f:
total = 0
while True:
b = resp.read(8 * 1024 * 1024)
if not b: break
f.write(b); total += len(b)
os.replace(dst + ".tmp", dst)
print(f" {fn}: {total//1024//1024}MB", flush=True)
# === Step 3: 读取数据 ===
print("=== Step 3: Load data ===", flush=True)
data_dir = "/tmp/jd"
train = [json.loads(l) for l in open(os.path.join(data_dir, "train.jsonl"), encoding="utf-8") if json.loads(l).get("hard_label") in (0, 1)]
calib = load_jsonl_safe(os.path.join(data_dir, "calibration.jsonl")) if os.path.exists(os.path.join(data_dir, "calibration.jsonl")) else []
print(f" train={len(train)} calib={len(calib)}", flush=True)
def load_jsonl_safe(p):
return [json.loads(l) for l in open(p, encoding="utf-8")]
# === Step 4: 训练 ===
print("=== Step 4: Training ===", flush=True)
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from peft import LoraConfig, get_peft_model
import bisect
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
PUNCT = __import__("re").compile(r"[\s,。!?;:、\"'""''()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
def norm_bigrams(s):
t = PUNCT.sub("", s).lower()
if len(t) < 2: return {t} if t else set()
return set(t[i:i+2] for i in range(len(t)-1)) | set(__import__("re").findall(r"[a-z0-9]{2,}", t))
class NoulDS(Dataset):
def __init__(self, rows, tok, max_len=512):
self.rows, self.tok, self.max_len = rows, tok, max_len
self.q = {"t": "noul", "ins": INSTR, "crit": None}
def __len__(self): return len(self.rows)
def __getitem__(self, i):
r = self.rows[i]
state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:2500]}"
ids, markers = build_sequence(self.tok, state, self.q, self.max_len, 192)
pad = self.max_len - len(ids)
mm = [True]*len(markers) + [False]*(8-len(markers))
return {"input_ids": torch.tensor(ids + [0]*pad),
"attention_mask": torch.tensor([1]*len(ids) + [0]*pad),
"marker_pos": torch.tensor(markers + [0]*(8-len(markers))),
"marker_mask": torch.tensor(mm),
"label": torch.tensor(float(r["hard_label"])),
"soft": torch.tensor(float(r.get("soft_label") or r["hard_label"]))}
def collate_noul(batch):
keys = batch[0].keys()
return {k: torch.stack([b[k] for b in batch]) for k in keys}
def evaluate_model(dm, loader, dev):
dm.eval()
probs, labels = [], []
with torch.no_grad():
for b in loader:
logits, _ = dm(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].bool().to(dev),
marker_pos=b["marker_pos"].to(dev), marker_mask=b["marker_mask"].bool().to(dev),
qtype=b["qtype"].to(dev))
p = torch.softmax(logits.float(), -1)[..., 1]
probs += p.tolist(); labels += b["label"].tolist()
dm.train()
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos)*len(neg)) if pos and neg else 0
e = ece(probs, labels)
dm.eval() # 保持 train 状态在调用方管理
return {"auc": round(auc, 4), "ece": round(e, 4), "n": len(labels)}
dev = "cuda" if torch.cuda.is_available() else "cpu"
tok = AutoTokenizer.from_pretrained("/tmp/mmbert")
enc_cfg = AutoConfig.from_pretrained("/tmp/mmbert")
enc = AutoModel.from_config(enc_cfg)
from laya.common import DecisionModel
dm = DecisionModel(enc, head_layers=2, n_act=2)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, target_modules=["Wqkv", "Wo"], task_type=None)
dm.encoder = get_peft_model(dm.encoder, lora)
dm.to(dev); dm.train()
print(f"model on {dev}, trainable {sum(p.numel() for p in dm.parameters() if p.requires_grad):,}", flush=True)
dl = DataLoader(NoulDS(train, tok), batch_size=16, shuffle=True, num_workers=2, drop_last=True, collate_fn=collate_noul)
dl_cal = DataLoader(NoulDS(calib, tok), batch_size=32, collate_fn=collate_noul)
steps = len(dl) * 2
opt = torch.optim.AdamW([p for p in dm.parameters() if p.requires_grad], lr=1e-4, weight_decay=0.01)
from torch.optim.lr_scheduler import CosineAnnealingLR
sch = CosineAnnealingLR(opt, T_max=steps)
best_auc = 0.0
gstep = 0
for ep in range(2):
dm.train()
for b in dl:
with torch.autocast("cuda", enabled=(dev=="cuda")):
logits, act = dm(input_ids=b["input_ids"].to(dev), attention_mask=b["attention_mask"].bool().to(dev),
marker_pos=b["marker_pos"].to(dev), marker_mask=b["marker_mask"].bool().to(dev),
qtype=b["qtype"].to(dev))
true_logit = logits[..., 1].float()
hard = b["label"].to(dev); soft = b["soft"].to(dev)
loss = F.binary_cross_entropy_with_logits(true_logit, hard) + 0.5*F.mse_loss(torch.sigmoid(true_logit), soft)
opt.zero_grad(); loss.backward()
torch.nn.utils.clip_grad_norm_([p for p in dm.parameters() if p.requires_grad], 1.0)
opt.step(); sch.step()
gstep += 1
if gstep % 100 == 0:
print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True)
m = evaluate_model(dm, dl_cal, dev)
print(f"[jev] ep{ep} calib {m}", flush=True)
if m["auc"] > best_auc:
best_auc = m["auc"]
torch.save(dm.state_dict(), "/tmp/laya_out/model.pt")
print(f"[jev] DONE best_auc={best_auc:.4f}", flush=True)
print("ALL-TRAINING-DONE", flush=True)
if __name__ == "__main__":
main()