jev-training-data/train_all.py

350 lines
14 KiB
Python

#!/usr/bin/env python3
"""train_all.py — 一体化: 下载 mmBERT + 训练 + 评估 (noul 二分类, Laya DecisionModel)
在 fat 容器内运行。数据和 laya_repo 模块通过 gitea clone 获取 (/tmp/jd)。
用法: python3 train_all.py [--epochs N] [--lr LR] [--bs N]
"""
import json, os, sys, bisect
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
DATA_DIR = "/tmp/jd"
MMBERT_DIR = "/tmp/mmbert"
OUT_DIR = "/tmp/laya_out"
def log(msg):
print(f"[jev] {msg}", flush=True)
def load_jsonl(p):
rows = []
with open(p, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def ece(probs, labels, nbins=15):
bins = [[] for _ in range(nbins)]
for p, y in zip(probs, labels):
bins[min(int(p * nbins), nbins - 1)].append((p, y))
e = 0.0
for b in bins:
if not b:
continue
e += len(b) / len(probs) * abs(
sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
return e
def phase_data():
"""v05n tar.gz 分片重组 → DATA_DIR/train.jsonl"""
import glob, shutil, tarfile
dst = os.path.join(DATA_DIR, "train.jsonl")
if os.path.isfile(dst):
log(" data OK (cached)")
return True
parts = sorted(glob.glob(os.path.join(DATA_DIR, "v05n", "jev_part_*")))
if not parts:
log(f" FATAL: no train.jsonl and no v05n parts under {DATA_DIR}")
return False
log(f" reassemble {len(parts)} v05n parts")
bundle = "/tmp/bundle.tar.gz"
with open(bundle, "wb") as out:
for p in parts:
with open(p, "rb") as f:
shutil.copyfileobj(f, out)
with tarfile.open(bundle) as t:
t.extractall(DATA_DIR)
ok = os.path.isfile(dst)
log(f" extract -> {dst} {'OK' if ok else 'FAIL'}")
return ok
def phase_env():
"""把 gitea clone 的 laya_repo 注入 sys.path, 优先于 pip 版"""
log("Phase 1: env setup")
repo = os.path.join(DATA_DIR, "laya_repo")
if not os.path.isdir(repo):
log(f" FATAL: {repo} 不存在")
return False
src = "/tmp/laya_src"
pkg = os.path.join(src, "laya")
os.makedirs(pkg, exist_ok=True)
for fn in os.listdir(repo):
if fn.endswith(".py"):
with open(os.path.join(repo, fn), "rb") as f:
open(os.path.join(pkg, fn), "wb").write(f.read())
# 包级 __init__.py 会 import .hooks (vendored 集里没有) — 强制置空, 只用 common.py
open(os.path.join(pkg, "__init__.py"), "w").write("")
sys.path.insert(0, src)
from laya.common import build_sequence, DecisionModel
log(" laya.common OK")
return True
def phase_mmbert():
"""ModelScope 直链下载 mmBERT"""
import urllib.request, time
log("Phase 2: download mmBERT (ModelScope)")
MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main"
os.makedirs(MMBERT_DIR, exist_ok=True)
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json",
"special_tokens_map.json", "pytorch_model.bin"]:
dst = os.path.join(MMBERT_DIR, fn)
min_size = 300_000_000 if fn.endswith(".bin") else 100
if os.path.exists(dst) and os.path.getsize(dst) >= min_size:
log(f" {fn} cached ({os.path.getsize(dst)//1024//1024}MB)")
continue
ok = False
for attempt in range(1, 5):
try:
with urllib.request.urlopen(f"{MS}/{fn}", timeout=300) as r, \
open(dst + ".tmp", "wb") as f:
total = 0
while True:
b = r.read(8 * 1024 * 1024)
if not b:
break
f.write(b)
total += len(b)
os.replace(dst + ".tmp", dst)
log(f" {fn}: {total//1024//1024}MB")
ok = total >= min_size
break
except Exception as e:
log(f" {fn} attempt {attempt}: {str(e)[:80]}")
time.sleep(15 * attempt)
if not ok:
return False
return True
def build_model(dev, tiny=False):
log("Phase 3: build DecisionModel")
import torch
from transformers import AutoTokenizer, AutoConfig, AutoModel
from peft import LoraConfig, get_peft_model
from laya.common import DecisionModel
tok = AutoTokenizer.from_pretrained(MMBERT_DIR)
if tiny:
# 随机初始化(冒烟测试用): 不加载 590MB 权重, 只验证代码路径
enc = AutoModel.from_config(AutoConfig.from_pretrained(MMBERT_DIR))
else:
enc = AutoModel.from_pretrained(MMBERT_DIR)
dm = DecisionModel(enc, head_layers=2, n_act=2)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
target_modules=["Wqkv", "Wo"], task_type=None)
dm.encoder = get_peft_model(dm.encoder, lora)
dm.to(dev)
dm.train()
n = sum(p.numel() for p in dm.parameters() if p.requires_grad)
log(f" model on {dev}, trainable {n:,}")
return dm, tok
def run(args):
global DATA_DIR, MMBERT_DIR, OUT_DIR
if args.data:
DATA_DIR = args.data
if args.base:
MMBERT_DIR = args.base
if args.out:
OUT_DIR = args.out
if not phase_env():
sys.exit(1)
if not phase_data():
sys.exit(1)
if not (args.tiny or phase_mmbert()):
sys.exit(1)
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from transformers import get_cosine_schedule_with_warmup
from laya.common import build_sequence
dev = "cuda" if torch.cuda.is_available() else "cpu"
log(f"device={dev} tiny={args.tiny}")
dm, tok = build_model(dev, tiny=args.tiny)
class NoulDS(Dataset):
def __init__(self, rows, max_len=512, head_len=192, doc_cap=2500):
self.rows, self.max_len, self.head_len, self.doc_cap = rows, max_len, head_len, doc_cap
self.q = {"t": "noul", "ins": INSTR, "crit": None}
def __len__(self):
return len(self.rows)
def __getitem__(self, i):
r = self.rows[i]
state = f"用户问题: {r['query']}\n\n文档内容:\n{r['doc_text'][:self.doc_cap]}"
ids, markers = build_sequence(tok, state, self.q, self.max_len, self.head_len)
pad = self.max_len - len(ids)
mm = [True] * len(markers) + [False] * (8 - len(markers))
return {"input_ids": torch.tensor(ids + [0] * pad),
"attention_mask": torch.tensor([1] * len(ids) + [0] * pad),
"marker_pos": torch.tensor(markers + [0] * (8 - len(markers))),
"marker_mask": torch.tensor(mm),
"qtype": torch.tensor(2),
"label": torch.tensor(float(r["hard_label"])),
"soft": torch.tensor(float(r.get("soft_label") or r["hard_label"]))}
def collate(batch):
return {k: torch.stack([b[k] for b in batch]) for k in batch[0]}
def fwd(b, no_grad=False):
ctx = torch.no_grad() if no_grad else torch.enable_grad()
with ctx:
with torch.autocast("cuda", enabled=(dev == "cuda")):
logits, _ = dm(input_ids=b["input_ids"].to(dev),
attention_mask=b["attention_mask"].bool().to(dev),
marker_pos=b["marker_pos"].to(dev),
marker_mask=b["marker_mask"].bool().to(dev),
qtype=b["qtype"].to(dev))
return logits
def evaluate(loader):
was_training = dm.training
dm.eval()
probs, labels = [], []
for b in loader:
p = torch.softmax(fwd(b, no_grad=True).float(), -1)[..., 1]
probs += p.tolist()
labels += b["label"].tolist()
if was_training:
dm.train()
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg)) if pos and neg else 0.0
return {"auc": round(auc, 4), "ece": round(ece(probs, labels), 4), "n": len(labels)}
train = [r for r in load_jsonl(os.path.join(DATA_DIR, "train.jsonl"))
if r.get("hard_label") in (0, 1)]
calib = load_jsonl(os.path.join(DATA_DIR, "calibration.jsonl"))
if args.tiny:
train, calib = train[:64], calib[:16]
log(" TINY mode: train[:64] calib[:16]")
log(f"train={len(train)} calib={len(calib)}")
os.makedirs(OUT_DIR, exist_ok=True)
nw = 0 if args.tiny else 2
dl = DataLoader(NoulDS(train), batch_size=args.bs, shuffle=True,
num_workers=nw, drop_last=True, collate_fn=collate)
dl_cal = DataLoader(NoulDS(calib), batch_size=32, collate_fn=collate)
steps = len(dl) * args.epochs
opt = torch.optim.AdamW([p for p in dm.parameters() if p.requires_grad],
lr=args.lr, weight_decay=0.01)
sch = get_cosine_schedule_with_warmup(opt, int(0.06 * steps), steps)
m0 = evaluate(dl_cal)
log(f"ep-1 pretrain calib {m0}")
best_auc = m0["auc"]
gstep = 0
for ep in range(args.epochs):
dm.train()
for b in dl:
logits = fwd(b)
true_logit = logits[..., 1].float()
hard = b["label"].to(dev)
soft = b["soft"].to(dev)
loss = F.binary_cross_entropy_with_logits(true_logit, hard) + \
0.5 * F.mse_loss(torch.sigmoid(true_logit), soft)
opt.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(
[p for p in dm.parameters() if p.requires_grad], 1.0)
opt.step()
sch.step()
gstep += 1
if gstep % 100 == 0:
print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True)
m = evaluate(dl_cal)
print(f"[jev] ep{ep} calib AUC={m['auc']:.4f} ECE={m['ece']:.4f}", flush=True)
if m["auc"] > best_auc:
best_auc = m["auc"]
torch.save(dm.state_dict(), os.path.join(OUT_DIR, "model.pt"))
log(f"saved best (auc={best_auc:.4f})")
log(f"TRAINING-DONE best_auc={best_auc:.4f}")
json.dump({"best_auc": best_auc, "train_pairs": len(train),
"steps": gstep, "epochs": args.epochs,
"pretrain_calib": m0, "final_calib": m},
open(os.path.join(OUT_DIR, "training_log.json"), "w"),
ensure_ascii=False, indent=1)
os.makedirs(os.path.join(DATA_DIR, "logs"), exist_ok=True)
json.dump({"best_auc": best_auc, "train_pairs": len(train), "steps": gstep},
open(os.path.join(DATA_DIR, "logs", f"{args.tag}_result.json"), "w"),
ensure_ascii=False, indent=1)
if not args.tiny:
phase_push(tag=args.tag)
print("ALL-DONE", flush=True)
def phase_push(tag="r36"):
"""回传: adapter 分片 + 日志 → gitea。每个分片一个 commit, push 带认证头。"""
import subprocess, glob, pickle
log("Phase 5: push back")
GITEA_URL = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git"
AUTH = None
for cand in ("/tmp/jd_auth",):
if os.path.exists(cand):
AUTH = open(cand).read().strip()
break
if not AUTH:
log(" no /tmp/jd_auth, skip push")
return
logs_dir = os.path.join(DATA_DIR, "logs")
os.makedirs(logs_dir, exist_ok=True)
# 1) adapter (trainable params only, fp32) 分片
import torch
import torch as _t
sd = _t.load(os.path.join(OUT_DIR, "model.pt"), map_location="cpu")
ad = {k: v for k, v in sd.items() if any(
s in k for s in ("lora_", "head.", "type_emb", "scorer", "act_head", "temperature"))}
n_ad = sum(v.numel() for v in ad.values())
log(f" adapter params={n_ad:,}")
raw_path = os.path.join(OUT_DIR, "adapter.pt")
_t.save(ad, raw_path)
data = open(raw_path, "rb").read()
PART = 18 * 1024 * 1024
n_parts = max(1, (len(data) + PART - 1) // PART)
for i in range(n_parts):
open(os.path.join(logs_dir, f"{tag}_adapter.pt.part{i:02d}"), "wb").write(
data[i*PART:(i+1)*PART])
open(os.path.join(logs_dir, f"{tag}_adapter_manifest.json"), "w").write(json.dumps({
"tag": tag, "parts": n_parts, "bytes": len(data), "params": n_ad,
"format": "torch.save(DecisionModel trainable state_dict fp32)"}))
# 训练 stdout 也带回来
for src in ("/tmp/train_stdout.log",):
if os.path.exists(src):
open(os.path.join(logs_dir, f"{tag}_train_stdout.log"), "wb").write(open(src, "rb").read())
# 2) git 逐分片 commit+push (注意: 这版 gitea 对 -c http.extraHeader 命令行传法返回 400,
# 必须 git config 写入仓库配置后裸 push)
def git(*a, cwd=DATA_DIR):
return subprocess.run(["git"] + list(a), cwd=cwd,
capture_output=True, text=True)
git("config", "user.email", "dingbin.wang@d-robotics.cc")
git("config", "user.name", "dingbin.wang")
git("config", "http.extraHeader", f"Authorization: Basic {AUTH}")
files = sorted(glob.glob(os.path.join(logs_dir, f"{tag}_*")))
for f in files:
git("add", os.path.relpath(f, DATA_DIR))
c = git("commit", "-m", f"{tag} {os.path.basename(f)}")
p = git("push", "origin", "main")
ok = p.returncode == 0
log(f" push {os.path.basename(f)} ({os.path.getsize(f)//1024}KB): {'OK' if ok else 'FAIL ' + (p.stderr or c.stderr)[-120:]}")
import time
time.sleep(2)
print("PUSH-BACK-DONE", flush=True)
if __name__ == "__main__":
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--epochs", type=int, default=2)
ap.add_argument("--lr", type=float, default=1e-4)
ap.add_argument("--bs", type=int, default=16)
ap.add_argument("--data", default=None, help="覆盖 DATA_DIR")
ap.add_argument("--base", default=None, help="覆盖 MMBERT_DIR")
ap.add_argument("--out", default=None, help="覆盖 OUT_DIR")
ap.add_argument("--tiny", action="store_true", help="冒烟模式: 随机初始化 + 64 条数据")
ap.add_argument("--tag", default="r36", help="产物回传标签")
run(ap.parse_args())