73 lines
2.6 KiB
Python
73 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""bootstrap_r36.py — fat 容器内引导脚本 (经 python3 -c b64 exec 运行)
|
|
pip 依赖 → gitea clone → train_all.py → 失败时 crash 日志回传
|
|
全部 subprocess 列表参数, 无 shell。
|
|
"""
|
|
import subprocess, sys, os
|
|
|
|
AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM="
|
|
REPO = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git"
|
|
TL = "/dev/termination-log"
|
|
|
|
def log(m):
|
|
print("[boot]", m, flush=True)
|
|
|
|
def sh(args, cwd=None):
|
|
return subprocess.run(args, capture_output=True, text=True, cwd=cwd)
|
|
|
|
log("T1-pip")
|
|
DEPS = ["transformers==4.57.6", "peft", "sentencepiece", "safetensors"]
|
|
r = sh([sys.executable, "-m", "pip", "install", "-q"] + DEPS)
|
|
if r.returncode != 0:
|
|
r = sh([sys.executable, "-m", "pip", "install", "-q", "-i",
|
|
"https://pypi.tuna.tsinghua.edu.cn/simple"] + DEPS)
|
|
pip_err = (r.stderr or "")[-1500:]
|
|
sh([sys.executable, "-m", "pip", "install", "-q", "--no-deps", "torchaudio"])
|
|
|
|
log("T2-clone")
|
|
open("/tmp/jd_auth", "w").write(AUTH)
|
|
r = sh(["git", "-c", "http.extraHeader=Authorization: Basic " + AUTH,
|
|
"clone", REPO, "/tmp/jd"])
|
|
clone_ok = r.returncode == 0
|
|
if not clone_ok:
|
|
pip_err += "\nCLONE:" + (r.stderr or "")[-400:]
|
|
|
|
log("T3-train")
|
|
train = subprocess.run(
|
|
[sys.executable, "/tmp/jd/train_all.py",
|
|
"--epochs", "3", "--bs", "16", "--lr", "1e-4",
|
|
"--max_len", "1024", "--head_len", "160",
|
|
"--win_tokens", "800", "--aug_p", "0.2", "--en_boost", "1.5",
|
|
"--tag", "r42"],
|
|
capture_output=True, text=True)
|
|
rc = train.returncode
|
|
out = (train.stdout or "") + "\n===STDERR===\n" + (train.stderr or "")
|
|
try:
|
|
open("/tmp/train_stdout.log", "w").write(out)
|
|
except Exception:
|
|
pass
|
|
log("T4-rc=%d" % rc)
|
|
|
|
# 失败摘要写 termination-log (K8s 只取尾部 2KB)
|
|
tail = out.splitlines()[-20:]
|
|
summary = "RC=%d\n" % rc + "\n".join(tail) + "\nPIP/CLONE:" + pip_err[-250:]
|
|
try:
|
|
open(TL, "w").write(summary[-2000:])
|
|
except Exception:
|
|
pass
|
|
|
|
if rc != 0 and clone_ok:
|
|
log("T5-crash-push")
|
|
os.makedirs("/tmp/jd/logs", exist_ok=True)
|
|
open("/tmp/jd/logs/r42_crash_stdout.log", "w").write(out)
|
|
sh(["git", "config", "user.email", "dingbin.wang@d-robotics.cc"], cwd="/tmp/jd")
|
|
sh(["git", "config", "user.name", "dingbin.wang"], cwd="/tmp/jd")
|
|
sh(["git", "config", "http.extraHeader", "Authorization: Basic " + AUTH], cwd="/tmp/jd")
|
|
sh(["git", "add", "logs"], cwd="/tmp/jd")
|
|
sh(["git", "commit", "-m", "r42 crash logs"], cwd="/tmp/jd")
|
|
p = sh(["git", "push", "origin", "main"], cwd="/tmp/jd")
|
|
log("T6-push-rc=%d" % p.returncode)
|
|
|
|
log("DONE rc=%d" % rc)
|
|
sys.exit(0 if rc == 0 else 1)
|