608 lines
27 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""train_all.py — 一体化: 下载 mmBERT + 训练 + 评估 (noul 二分类, Laya DecisionModel)
在 fat 容器内运行。数据和 laya_repo 模块通过 gitea clone 获取 (/tmp/jd)。
用法: python3 train_all.py [--epochs N] [--lr LR] [--bs N]
"""
import json, os, sys, bisect, random
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
DATA_DIR = "/tmp/jd"
MMBERT_DIR = "/tmp/mmbert"
OUT_DIR = "/tmp/laya_out"
def log(msg):
line = f"[jev] {msg}"
print(line, flush=True)
try:
with open(os.path.join(OUT_DIR, "train_self.log"), "a", encoding="utf-8") as f:
f.write(line + "\n")
except Exception:
pass
def load_jsonl(p):
rows = []
with open(p, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def ece(probs, labels, nbins=15):
bins = [[] for _ in range(nbins)]
for p, y in zip(probs, labels):
bins[min(int(p * nbins), nbins - 1)].append((p, y))
e = 0.0
for b in bins:
if not b:
continue
e += len(b) / len(probs) * abs(
sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
return e
def bisect_auc(probs, labels):
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
if not pos or not neg:
return 0.0
return sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg))
def _lang(s):
return "zh" if any("一" <= c <= "鿿" for c in s) else "en"
def augment_crosspair_negs(rows, win_tokens, max_new=3200):
"""长文档 × 随机同语言其他 query → 长负例。
训练负例几乎都是短文档而 eval_v2 负例全是长文档——模型没学过
『长而无关 → 每窗都低分』。跨配对直接补齐 长度×标签 的失衡象限。
canary_en 诊断: 同域 hard negative(同产品不同主题)是英文难点 → en 候选优先填满 50%。"""
rng = random.Random(20261106)
nch_thr = {"zh": win_tokens, "en": win_tokens * 3} # 字符≈token 粗估
cand = [i for i, r in enumerate(rows)
if len(r.get("doc_text") or "") > nch_thr[_lang(r.get("doc_text") or "")]]
rng.shuffle(cand)
cand_en = [i for i in cand if _lang(rows[i].get("doc_text") or "") == "en"]
cand_zh = [i for i in cand if _lang(rows[i].get("doc_text") or "") == "zh"]
# en 优先: en 填一半, 剩余 zh (en 不足时 zh 补齐)
n_en = min(len(cand_en), max_new // 2)
n_zh = min(len(cand_zh), max_new - n_en)
picked = cand_en[:n_en] + cand_zh[:n_zh]
if n_en + n_zh < max_new:
rest = cand_en[n_en:] + cand_zh[n_zh:]
picked += rest[:max_new - n_en - n_zh]
pools = {}
for i, r in enumerate(rows):
pools.setdefault(_lang(r["query"]), []).append(i)
added = []
for i in picked:
L = _lang(rows[i].get("doc_text") or "")
pool = pools[L]
j = rng.choice(pool)
if j == i:
continue
added.append({"pair_id": f"xp-{rows[i].get('pair_id', i)}",
"query": rows[j]["query"],
"doc_text": rows[i]["doc_text"],
"hard_label": 0, "soft_label": 0.0})
return added
def oversample_en_pos(rows, factor=1.5):
"""canary_en 弱判别: en 正例语义学习欠充分 → 过采样 ×factor (粗放但便宜)"""
en_pos = [r for r in rows
if r.get("hard_label") == 1 and _lang(r["query"]) == "en"]
extra_n = int(len(en_pos) * (factor - 1))
rng = random.Random(42)
extra = []
for r in rng.sample(en_pos, min(extra_n, len(en_pos))):
d = dict(r)
d["pair_id"] = f"os-{d.get('pair_id', '?')}"
extra.append(d)
return extra
def phase_data():
"""v05n tar.gz 分片重组 → DATA_DIR/train.jsonl"""
import glob, shutil, tarfile
dst = os.path.join(DATA_DIR, "train.jsonl")
if os.path.isfile(dst):
log(" data OK (cached)")
return True
parts = sorted(glob.glob(os.path.join(DATA_DIR, "v05n", "jev_part_*")))
if not parts:
log(f" FATAL: no train.jsonl and no v05n parts under {DATA_DIR}")
return False
log(f" reassemble {len(parts)} v05n parts")
bundle = "/tmp/bundle.tar.gz"
with open(bundle, "wb") as out:
for p in parts:
with open(p, "rb") as f:
shutil.copyfileobj(f, out)
with tarfile.open(bundle) as t:
t.extractall(DATA_DIR)
ok = os.path.isfile(dst)
log(f" extract -> {dst} {'OK' if ok else 'FAIL'}")
return ok
def phase_env():
"""把 gitea clone 的 laya_repo 注入 sys.path, 优先于 pip 版"""
log("Phase 1: env setup")
repo = os.path.join(DATA_DIR, "laya_repo")
if not os.path.isdir(repo):
log(f" FATAL: {repo} 不存在")
return False
src = "/tmp/laya_src"
pkg = os.path.join(src, "laya")
os.makedirs(pkg, exist_ok=True)
for fn in os.listdir(repo):
if fn.endswith(".py"):
with open(os.path.join(repo, fn), "rb") as f:
open(os.path.join(pkg, fn), "wb").write(f.read())
# 包级 __init__.py 会 import .hooks (vendored 集里没有) — 强制置空, 只用 common.py
open(os.path.join(pkg, "__init__.py"), "w").write("")
sys.path.insert(0, src)
from laya.common import build_sequence, DecisionModel
log(" laya.common OK")
return True
def phase_mmbert():
"""ModelScope 直链下载 mmBERT"""
import urllib.request, time
log("Phase 2: download mmBERT (ModelScope)")
MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main"
os.makedirs(MMBERT_DIR, exist_ok=True)
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json",
"special_tokens_map.json", "pytorch_model.bin"]:
dst = os.path.join(MMBERT_DIR, fn)
min_size = 300_000_000 if fn.endswith(".bin") else 100
if os.path.exists(dst) and os.path.getsize(dst) >= min_size:
log(f" {fn} cached ({os.path.getsize(dst)//1024//1024}MB)")
continue
ok = False
for attempt in range(1, 5):
try:
with urllib.request.urlopen(f"{MS}/{fn}", timeout=300) as r, \
open(dst + ".tmp", "wb") as f:
total = 0
while True:
b = r.read(8 * 1024 * 1024)
if not b:
break
f.write(b)
total += len(b)
os.replace(dst + ".tmp", dst)
log(f" {fn}: {total//1024//1024}MB")
ok = total >= min_size
break
except Exception as e:
log(f" {fn} attempt {attempt}: {str(e)[:80]}")
time.sleep(15 * attempt)
if not ok:
return False
return True
def build_model(dev, tiny=False):
log("Phase 3: build DecisionModel")
import torch
from transformers import AutoTokenizer, AutoConfig, AutoModel
from peft import LoraConfig, get_peft_model
from laya.common import DecisionModel
tok = AutoTokenizer.from_pretrained(MMBERT_DIR)
if tiny:
# 随机初始化(冒烟测试用): 不加载 590MB 权重, 只验证代码路径
enc = AutoModel.from_config(AutoConfig.from_pretrained(MMBERT_DIR))
else:
# mmBERT checkpoint 的 key 带 model. 前缀, from_pretrained 会全部 miss(=随机初始化)!
# 必须显式 strip 前缀后 load_state_dict; matched=0 直接熔断
cfg = AutoConfig.from_pretrained(MMBERT_DIR)
enc = AutoModel.from_config(cfg)
import torch as _t
sd = _t.load(os.path.join(MMBERT_DIR, "pytorch_model.bin"),
map_location="cpu", weights_only=False)
sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()}
missing, unexpected = enc.load_state_dict(sd2, strict=False)
core_missing = [k for k in missing if not k.startswith(("head", "decoder"))]
matched = len(sd2) - len(unexpected)
log(f" encoder load: ckpt={len(sd)} matched={matched} "
f"unexpected={len(unexpected)} core_missing={len(core_missing)}")
if core_missing:
log(f" WARN core missing (前5): {core_missing[:5]}")
if matched == 0:
log(" FATAL: encoder checkpoint matched 0 keys — 加载失败, 熔断")
sys.exit(1)
dm = DecisionModel(enc, head_layers=2, n_act=2)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
target_modules=["Wqkv", "Wo"], task_type=None)
dm.encoder = get_peft_model(dm.encoder, lora)
dm.to(dev)
dm.train()
n = sum(p.numel() for p in dm.parameters() if p.requires_grad)
log(f" model on {dev}, trainable {n:,}")
return dm, tok
def run(args):
global DATA_DIR, MMBERT_DIR, OUT_DIR
if args.data:
DATA_DIR = args.data
if args.base:
MMBERT_DIR = args.base
if args.out:
OUT_DIR = args.out
os.makedirs(OUT_DIR, exist_ok=True) # 让早期 log 能落盘
if not phase_env():
sys.exit(1)
if not phase_data():
sys.exit(1)
if not (args.tiny or phase_mmbert()):
sys.exit(1)
import random
import math
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from transformers import get_cosine_schedule_with_warmup
from laya.common import build_sequence
dev = "cuda" if torch.cuda.is_available() else "cpu"
log(f"device={dev} tiny={args.tiny} max_len={args.max_len} head_len={args.head_len} "
f"win={args.win_tokens} aug_p={args.aug_p}")
dm, tok = build_model(dev, tiny=args.tiny)
class NoulDS(Dataset):
def __init__(self, rows, max_len=None, head_len=None, train_aug=False):
self.rows = rows
self.max_len = max_len or args.max_len
self.head_len = head_len or args.head_len
self.win_tokens = args.win_tokens
self.aug_p = args.aug_p
self.train_aug = train_aug
self.q = {"t": "noul", "ins": INSTR, "crit": None}
self.rng = random.Random(20261105)
def _doc_window(self, text):
"""返回 (window_text, aug_flag)。aug_p 概率对长文档随机取窗(答案可能不在窗内)"""
if not self.train_aug:
return text, False
ids = tok(text[:12000], add_special_tokens=False)["input_ids"]
if len(ids) <= self.win_tokens:
return text, False
if self.rng.random() < self.aug_p:
s = self.rng.randrange(0, len(ids) - self.win_tokens + 1)
return tok.decode(ids[s:s + self.win_tokens]), True
return text, False
def __len__(self):
return len(self.rows)
def __getitem__(self, i):
r = self.rows[i]
doc_win, aug = self._doc_window(r["doc_text"])
state = f"用户问题: {r['query']}\n\n文档内容:\n{doc_win}"
ids, markers = build_sequence(tok, state, self.q, self.max_len, self.head_len)
pad = self.max_len - len(ids)
mm = [True] * len(markers) + [False] * (8 - len(markers))
soft = r.get("soft_label")
if soft is None:
soft = r["hard_label"]
soft_clean = float(soft) if soft in (0, 1) else -1.0
# 窗口标签平滑: 随机窗口的正例, 答案可能不在窗内 → BCE 目标 0.85
label = float(r["hard_label"])
if aug and label == 1.0:
label = 0.85
if soft_clean < 0:
soft_clean = 0.85
return {"input_ids": torch.tensor(ids + [0] * pad),
"attention_mask": torch.tensor([1] * len(ids) + [0] * pad),
"marker_pos": torch.tensor(markers + [0] * (8 - len(markers))),
"marker_mask": torch.tensor(mm),
"qtype": torch.tensor(2),
"label": torch.tensor(label),
"soft": torch.tensor(float(soft)),
"soft_clean": torch.tensor(soft_clean)}
def collate(batch):
return {k: torch.stack([b[k] for b in batch]) for k in batch[0]}
def fwd(b, no_grad=False):
ctx = torch.no_grad() if no_grad else torch.enable_grad()
with ctx:
with torch.autocast("cuda", enabled=(dev == "cuda")):
logits, _ = dm(input_ids=b["input_ids"].to(dev),
attention_mask=b["attention_mask"].bool().to(dev),
marker_pos=b["marker_pos"].to(dev),
marker_mask=b["marker_mask"].bool().to(dev),
qtype=b["qtype"].to(dev))
return logits
def evaluate(loader):
was_training = dm.training
dm.eval()
probs, labels = [], []
for b in loader:
p = torch.softmax(fwd(b, no_grad=True).float(), -1)[..., 1]
probs += p.tolist()
labels += b["label"].tolist()
if was_training:
dm.train()
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
auc = sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg)) if pos and neg else 0.0
return {"auc": round(auc, 4), "ece": round(ece(probs, labels), 4), "n": len(labels)}
train = [r for r in load_jsonl(os.path.join(DATA_DIR, "train.jsonl"))
if r.get("hard_label") in (0, 1)]
calib = load_jsonl(os.path.join(DATA_DIR, "calibration.jsonl"))
if args.tiny:
train, calib = train[:64], calib[:16]
log(" TINY mode: train[:64] calib[:16]")
else:
xp = augment_crosspair_negs(train, args.win_tokens, max_new=3200)
train = train + xp
os_rows = oversample_en_pos(train, factor=args.en_boost)
train = train + os_rows
log(f"crosspair long negs: +{len(xp)}, en_pos oversample: +{len(os_rows)} "
f"(total {len(train)})")
log(f"train={len(train)} calib={len(calib)}")
os.makedirs(OUT_DIR, exist_ok=True)
nw = 0 if args.tiny else 2
dl = DataLoader(NoulDS(train, train_aug=True), batch_size=args.bs, shuffle=True,
num_workers=nw, drop_last=True, collate_fn=collate)
dl_cal = DataLoader(NoulDS(calib), batch_size=32, collate_fn=collate)
steps = len(dl) * args.epochs
opt = torch.optim.AdamW([p for p in dm.parameters() if p.requires_grad],
lr=args.lr, weight_decay=0.01)
sch = get_cosine_schedule_with_warmup(opt, int(0.06 * steps), steps)
m0 = evaluate(dl_cal)
log(f"ep-1 pretrain calib {m0}")
best_auc = m0["auc"]
gstep = 0
for ep in range(args.epochs):
dm.train()
for b in dl:
logits = fwd(b)
true_logit = logits[..., 1].float()
hard = b["label"].to(dev)
# MSE 校准项只作用于 soft∈{0,1} 的干净行 (47% 正例 soft=0.5 是标注缺失
# 的填充值, 0.5*MSE 会把概率系统性拉向 0.5 → r39 校准差的总根因)
soft_clean = b["soft_clean"].to(dev)
mask = soft_clean >= 0
loss = F.binary_cross_entropy_with_logits(true_logit, hard)
if mask.any():
loss = loss + 0.2 * F.mse_loss(torch.sigmoid(true_logit)[mask], soft_clean[mask])
opt.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(
[p for p in dm.parameters() if p.requires_grad], 1.0)
opt.step()
sch.step()
gstep += 1
if gstep % 100 == 0:
print(f"[jev] ep{ep} step{gstep}/{steps} loss={loss.item():.4f}", flush=True)
m = evaluate(dl_cal)
print(f"[jev] ep{ep} calib AUC={m['auc']:.4f} ECE={m['ece']:.4f}", flush=True)
if m["auc"] > best_auc:
best_auc = m["auc"]
torch.save(dm.state_dict(), os.path.join(OUT_DIR, "model.pt"))
log(f"saved best (auc={best_auc:.4f})")
log(f"TRAINING-DONE best_auc={best_auc:.4f}")
# ===== 温度校准: best 模型在 calib 全集上 grid-search T (min NLL) =====
calib_info = {"best_auc": best_auc}
best_path = os.path.join(OUT_DIR, "model.pt")
if os.path.exists(best_path):
dm.load_state_dict(torch.load(best_path, map_location=dev, weights_only=False))
was = dm.training
dm.eval()
z_all, y_all = [], []
with torch.no_grad():
for b in dl_cal:
lg = fwd(b, no_grad=True)
z_all += lg[..., 1].float().tolist()
y_all += b["label"].tolist()
if was:
dm.train()
def nll_at(T):
tot = 0.0
for z, y in zip(z_all, y_all):
p = 1.0 / (1.0 + math.exp(-z / T))
p = min(max(p, 1e-6), 1 - 1e-6)
tot -= math.log(p) if y == 1 else math.log(1 - p)
return tot / len(z_all)
probs_raw = [1 / (1 + math.exp(-z)) for z in z_all]
best_T, n_at_best = 1.0, nll_at(1.0)
T = 0.30
while T <= 4.001:
n = nll_at(T)
if n < n_at_best:
best_T, n_at_best = T, n
T += 0.05
probs_T = [1 / (1 + math.exp(-z / best_T)) for z in z_all]
calib_info = {
"best_auc": best_auc, "calib_n": len(z_all),
"auc_raw": round(bisect_auc(probs_raw, y_all), 4),
"ece_raw": round(ece(probs_raw, y_all), 4),
"temperature": round(best_T, 2),
"auc_calibrated": round(bisect_auc(probs_T, y_all), 4),
"ece_calibrated": round(ece(probs_T, y_all), 4),
}
log(f"CALIBRATION T={best_T:.2f} raw(ece={calib_info['ece_raw']}) "
f"calibrated(ece={calib_info['ece_calibrated']}) auc={calib_info['auc_calibrated']}")
# ===== 聚合后温度 T_agg: 合成 4-doc 长文档跑滑窗 max (生产聚合口径) =====
def agg_calib():
rng = random.Random(7)
pos_rows = [r for r in calib if r.get("hard_label") == 1][:150]
neg_rows = [r for r in calib if r.get("hard_label") == 0][:150]
neg_pool = [r for r in calib if r.get("hard_label") == 0]
if not pos_rows or not neg_rows:
return None
def windows_of(row):
docs = [row["doc_text"]] + [o["doc_text"] for o in rng.sample(neg_pool, 3)]
rng.shuffle(docs)
return [d[:4000] for d in docs]
zs, ys = [], []
B = 8
buf_ids, buf_lab, buf_idx = [], [], []
def flush_buf():
if not buf_ids:
return
b = {
"input_ids": torch.tensor([x[0] for x in buf_ids]).to(dev),
"attention_mask": torch.tensor(
[[1] * x[2] + [0] * (args.max_len - x[2]) for x in buf_ids]).to(dev),
"marker_pos": torch.tensor([x[1] for x in buf_ids]).to(dev),
"marker_mask": torch.tensor(
[[True] * x[3] + [False] * (8 - x[3]) for x in buf_ids]).bool().to(dev),
"qtype": torch.tensor([2] * len(buf_ids)).to(dev),
}
with torch.no_grad():
with torch.autocast("cuda", enabled=(dev == "cuda")):
lg, _ = dm(input_ids=b["input_ids"],
attention_mask=b["attention_mask"],
marker_pos=b["marker_pos"],
marker_mask=b["marker_mask"],
qtype=b["qtype"])
p1 = torch.softmax(lg.float(), -1)[..., 1].tolist()
for k in range(len(buf_ids)):
zs[buf_idx[k]][1].append(p1[k])
buf_ids.clear(); buf_lab.clear(); buf_idx.clear()
# zs: list of [label, [p_win...]]
zs = []
for r in pos_rows + neg_rows:
y = float(r["hard_label"])
row_pos = len(zs)
zs.append([y, []])
for d in windows_of(r):
state = f"用户问题: {r['query']}\n\n文档内容:\n{d}"
ids, markers = build_sequence(tok, state, {"t": "noul", "ins": INSTR, "crit": None},
args.max_len, args.head_len)
ids = ids + [0] * (args.max_len - len(ids))
buf_ids.append((ids, (markers + [0] * (8 - len(markers)))[:8],
len(ids), len(markers)))
buf_lab.append(y)
buf_idx.append(row_pos)
if len(buf_ids) >= B:
flush_buf()
flush_buf()
max_p = [max(ps) if ps else 0.5 for _, ps in zs]
labels = [y for y, _ in zs]
z_agg = [math.log(max(p, 1e-6) / max(1 - p, 1e-6)) for p in max_p]
def nll_z(zlist, T):
tot = 0.0
for z, y in zip(zlist, labels):
p = 1 / (1 + math.exp(-z / T))
p = min(max(p, 1e-6), 1 - 1e-6)
tot -= math.log(p) if y == 1 else math.log(1 - p)
return tot / len(zlist)
t_best, n_best = 1.0, nll_z(z_agg, 1.0)
T = 0.30
while T <= 6.001:
n = nll_z(z_agg, T)
if n < n_best:
t_best, n_best = T, n
T += 0.05
p_cal = [1 / (1 + math.exp(-z / t_best)) for z in z_agg]
return {"n": len(labels),
"auc_max_raw": round(bisect_auc(max_p, labels), 4),
"ece_max_raw": round(ece(max_p, labels), 4),
"T_agg": round(t_best, 2),
"ece_max_calibrated": round(ece(p_cal, labels), 4)}
agg = agg_calib()
if agg:
calib_info["agg"] = agg
log(f"AGG-CALIB {agg}")
json.dump({"best_auc": best_auc, "train_pairs": len(train),
"steps": gstep, "epochs": args.epochs,
"pretrain_calib": m0, "final_calib": m, "calibration": calib_info},
open(os.path.join(OUT_DIR, "training_log.json"), "w"),
ensure_ascii=False, indent=1)
os.makedirs(os.path.join(DATA_DIR, "logs"), exist_ok=True)
json.dump({"best_auc": best_auc, "train_pairs": len(train), "steps": gstep,
"calibration": calib_info},
open(os.path.join(DATA_DIR, "logs", f"{args.tag}_result.json"), "w"),
ensure_ascii=False, indent=1)
if not args.tiny:
phase_push(tag=args.tag)
print("ALL-DONE", flush=True)
def phase_push(tag="r36"):
"""回传: adapter 分片 + 日志 → gitea。每个分片一个 commit, push 带认证头。"""
import subprocess, glob, pickle
log("Phase 5: push back")
GITEA_URL = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git"
AUTH = None
for cand in ("/tmp/jd_auth",):
if os.path.exists(cand):
AUTH = open(cand).read().strip()
break
if not AUTH:
log(" no /tmp/jd_auth, skip push")
return
logs_dir = os.path.join(DATA_DIR, "logs")
os.makedirs(logs_dir, exist_ok=True)
# 1) adapter (trainable params only, fp32) 分片
import torch
import torch as _t
sd = _t.load(os.path.join(OUT_DIR, "model.pt"), map_location="cpu")
ad = {k: v for k, v in sd.items() if any(
s in k for s in ("lora_", "head.", "type_emb", "scorer", "act_head", "temperature"))}
n_ad = sum(v.numel() for v in ad.values())
log(f" adapter params={n_ad:,}")
raw_path = os.path.join(OUT_DIR, "adapter.pt")
_t.save(ad, raw_path)
data = open(raw_path, "rb").read()
PART = 18 * 1024 * 1024
n_parts = max(1, (len(data) + PART - 1) // PART)
for i in range(n_parts):
open(os.path.join(logs_dir, f"{tag}_adapter.pt.part{i:02d}"), "wb").write(
data[i*PART:(i+1)*PART])
open(os.path.join(logs_dir, f"{tag}_adapter_manifest.json"), "w").write(json.dumps({
"tag": tag, "parts": n_parts, "bytes": len(data), "params": n_ad,
"format": "torch.save(DecisionModel trainable state_dict fp32)"}))
# 训练 stdout 也带回来 (bootstrap 捕获版可能不存在——自写日志是主回传)
for src in ("/tmp/train_stdout.log", os.path.join(OUT_DIR, "train_self.log")):
if os.path.exists(src):
dstname = f"{tag}_train_stdout.log" if "stdout" in src else f"{tag}_train_self.log"
open(os.path.join(logs_dir, dstname), "wb").write(open(src, "rb").read())
# 2) git 逐分片 commit+push (注意: 这版 gitea 对 -c http.extraHeader 命令行传法返回 400,
# 必须 git config 写入仓库配置后裸 push)
def git(*a, cwd=DATA_DIR):
return subprocess.run(["git"] + list(a), cwd=cwd,
capture_output=True, text=True)
git("config", "user.email", "dingbin.wang@d-robotics.cc")
git("config", "user.name", "dingbin.wang")
git("config", "http.extraHeader", f"Authorization: Basic {AUTH}")
files = sorted(glob.glob(os.path.join(logs_dir, f"{tag}_*")))
for f in files:
git("add", os.path.relpath(f, DATA_DIR))
c = git("commit", "-m", f"{tag} {os.path.basename(f)}")
p = git("push", "origin", "main")
ok = p.returncode == 0
log(f" push {os.path.basename(f)} ({os.path.getsize(f)//1024}KB): {'OK' if ok else 'FAIL ' + (p.stderr or c.stderr)[-120:]}")
import time
time.sleep(2)
print("PUSH-BACK-DONE", flush=True)
if __name__ == "__main__":
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--epochs", type=int, default=2)
ap.add_argument("--lr", type=float, default=1e-4)
ap.add_argument("--bs", type=int, default=16)
ap.add_argument("--data", default=None, help="覆盖 DATA_DIR")
ap.add_argument("--base", default=None, help="覆盖 MMBERT_DIR")
ap.add_argument("--out", default=None, help="覆盖 OUT_DIR")
ap.add_argument("--tiny", action="store_true", help="冒烟模式: 随机初始化 + 64 条数据")
ap.add_argument("--tag", default="r36", help="产物回传标签")
ap.add_argument("--max_len", type=int, default=512, help="序列总长 (r40=1024)")
ap.add_argument("--head_len", type=int, default=192, help="question 头部预算")
ap.add_argument("--win_tokens", type=int, default=800, help="随机窗口 token 数")
ap.add_argument("--aug_p", type=float, default=0.4, help="随机窗口采样概率")
ap.add_argument("--en_boost", type=float, default=1.5, help="en 正例过采样倍数 (1=关)")
run(ap.parse_args())