train_all: explicit model.-prefix weight loading + matched=0 circuit breaker + self log

This commit is contained in:
dingbin.wang 2026-11-05 19:02:53 +08:00
parent d0bfe36591
commit afec1412d4

View File

@ -11,7 +11,13 @@ MMBERT_DIR = "/tmp/mmbert"
OUT_DIR = "/tmp/laya_out"
def log(msg):
print(f"[jev] {msg}", flush=True)
line = f"[jev] {msg}"
print(line, flush=True)
try:
with open(os.path.join(OUT_DIR, "train_self.log"), "a", encoding="utf-8") as f:
f.write(line + "\n")
except Exception:
pass
def load_jsonl(p):
rows = []
@ -126,7 +132,24 @@ def build_model(dev, tiny=False):
# 随机初始化(冒烟测试用): 不加载 590MB 权重, 只验证代码路径
enc = AutoModel.from_config(AutoConfig.from_pretrained(MMBERT_DIR))
else:
enc = AutoModel.from_pretrained(MMBERT_DIR)
# mmBERT checkpoint 的 key 带 model. 前缀, from_pretrained 会全部 miss(=随机初始化)!
# 必须显式 strip 前缀后 load_state_dict; matched=0 直接熔断
cfg = AutoConfig.from_pretrained(MMBERT_DIR)
enc = AutoModel.from_config(cfg)
import torch as _t
sd = _t.load(os.path.join(MMBERT_DIR, "pytorch_model.bin"),
map_location="cpu", weights_only=False)
sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()}
missing, unexpected = enc.load_state_dict(sd2, strict=False)
core_missing = [k for k in missing if not k.startswith(("head", "decoder"))]
matched = len(sd2) - len(unexpected)
log(f" encoder load: ckpt={len(sd)} matched={matched} "
f"unexpected={len(unexpected)} core_missing={len(core_missing)}")
if core_missing:
log(f" WARN core missing (前5): {core_missing[:5]}")
if matched == 0:
log(" FATAL: encoder checkpoint matched 0 keys — 加载失败, 熔断")
sys.exit(1)
dm = DecisionModel(enc, head_layers=2, n_act=2)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
target_modules=["Wqkv", "Wo"], task_type=None)
@ -311,10 +334,11 @@ def phase_push(tag="r36"):
open(os.path.join(logs_dir, f"{tag}_adapter_manifest.json"), "w").write(json.dumps({
"tag": tag, "parts": n_parts, "bytes": len(data), "params": n_ad,
"format": "torch.save(DecisionModel trainable state_dict fp32)"}))
# 训练 stdout 也带回来
for src in ("/tmp/train_stdout.log",):
# 训练 stdout 也带回来 (bootstrap 捕获版可能不存在——自写日志是主回传)
for src in ("/tmp/train_stdout.log", os.path.join(OUT_DIR, "train_self.log")):
if os.path.exists(src):
open(os.path.join(logs_dir, f"{tag}_train_stdout.log"), "wb").write(open(src, "rb").read())
dstname = f"{tag}_train_stdout.log" if "stdout" in src else f"{tag}_train_self.log"
open(os.path.join(logs_dir, dstname), "wb").write(open(src, "rb").read())
# 2) git 逐分片 commit+push (注意: 这版 gitea 对 -c http.extraHeader 命令行传法返回 400,
# 必须 git config 写入仓库配置后裸 push)