diff --git a/train_all.py b/train_all.py index 42c2cc3..561a726 100644 --- a/train_all.py +++ b/train_all.py @@ -11,7 +11,13 @@ MMBERT_DIR = "/tmp/mmbert" OUT_DIR = "/tmp/laya_out" def log(msg): - print(f"[jev] {msg}", flush=True) + line = f"[jev] {msg}" + print(line, flush=True) + try: + with open(os.path.join(OUT_DIR, "train_self.log"), "a", encoding="utf-8") as f: + f.write(line + "\n") + except Exception: + pass def load_jsonl(p): rows = [] @@ -126,7 +132,24 @@ def build_model(dev, tiny=False): # 随机初始化(冒烟测试用): 不加载 590MB 权重, 只验证代码路径 enc = AutoModel.from_config(AutoConfig.from_pretrained(MMBERT_DIR)) else: - enc = AutoModel.from_pretrained(MMBERT_DIR) + # mmBERT checkpoint 的 key 带 model. 前缀, from_pretrained 会全部 miss(=随机初始化)! + # 必须显式 strip 前缀后 load_state_dict; matched=0 直接熔断 + cfg = AutoConfig.from_pretrained(MMBERT_DIR) + enc = AutoModel.from_config(cfg) + import torch as _t + sd = _t.load(os.path.join(MMBERT_DIR, "pytorch_model.bin"), + map_location="cpu", weights_only=False) + sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()} + missing, unexpected = enc.load_state_dict(sd2, strict=False) + core_missing = [k for k in missing if not k.startswith(("head", "decoder"))] + matched = len(sd2) - len(unexpected) + log(f" encoder load: ckpt={len(sd)} matched={matched} " + f"unexpected={len(unexpected)} core_missing={len(core_missing)}") + if core_missing: + log(f" WARN core missing (前5): {core_missing[:5]}") + if matched == 0: + log(" FATAL: encoder checkpoint matched 0 keys — 加载失败, 熔断") + sys.exit(1) dm = DecisionModel(enc, head_layers=2, n_act=2) lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, target_modules=["Wqkv", "Wo"], task_type=None) @@ -311,10 +334,11 @@ def phase_push(tag="r36"): open(os.path.join(logs_dir, f"{tag}_adapter_manifest.json"), "w").write(json.dumps({ "tag": tag, "parts": n_parts, "bytes": len(data), "params": n_ad, "format": "torch.save(DecisionModel trainable state_dict fp32)"})) - # 训练 stdout 也带回来 - for src in ("/tmp/train_stdout.log",): + # 训练 stdout 也带回来 (bootstrap 捕获版可能不存在——自写日志是主回传) + for src in ("/tmp/train_stdout.log", os.path.join(OUT_DIR, "train_self.log")): if os.path.exists(src): - open(os.path.join(logs_dir, f"{tag}_train_stdout.log"), "wb").write(open(src, "rb").read()) + dstname = f"{tag}_train_stdout.log" if "stdout" in src else f"{tag}_train_self.log" + open(os.path.join(logs_dir, dstname), "wb").write(open(src, "rb").read()) # 2) git 逐分片 commit+push (注意: 这版 gitea 对 -c http.extraHeader 命令行传法返回 400, # 必须 git config 写入仓库配置后裸 push)