73 lines
2.6 KiB
Python

#!/usr/bin/env python3
"""bootstrap_r36.py — fat 容器内引导脚本 (经 python3 -c b64 exec 运行)
pip 依赖 → gitea clone → train_all.py → 失败时 crash 日志回传
全部 subprocess 列表参数, 无 shell。
"""
import subprocess, sys, os
AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM="
REPO = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git"
TL = "/dev/termination-log"
def log(m):
print("[boot]", m, flush=True)
def sh(args, cwd=None):
return subprocess.run(args, capture_output=True, text=True, cwd=cwd)
log("T1-pip")
DEPS = ["transformers==4.57.6", "peft", "sentencepiece", "safetensors"]
r = sh([sys.executable, "-m", "pip", "install", "-q"] + DEPS)
if r.returncode != 0:
r = sh([sys.executable, "-m", "pip", "install", "-q", "-i",
"https://pypi.tuna.tsinghua.edu.cn/simple"] + DEPS)
pip_err = (r.stderr or "")[-1500:]
sh([sys.executable, "-m", "pip", "install", "-q", "--no-deps", "torchaudio"])
log("T2-clone")
open("/tmp/jd_auth", "w").write(AUTH)
r = sh(["git", "-c", "http.extraHeader=Authorization: Basic " + AUTH,
"clone", REPO, "/tmp/jd"])
clone_ok = r.returncode == 0
if not clone_ok:
pip_err += "\nCLONE:" + (r.stderr or "")[-400:]
log("T3-train")
train = subprocess.run(
[sys.executable, "/tmp/jd/train_all.py",
"--epochs", "3", "--bs", "16", "--lr", "1e-4",
"--max_len", "1024", "--head_len", "160",
"--win_tokens", "800", "--aug_p", "0.2", "--en_boost", "1.5",
"--tag", "r42"],
capture_output=True, text=True)
rc = train.returncode
out = (train.stdout or "") + "\n===STDERR===\n" + (train.stderr or "")
try:
open("/tmp/train_stdout.log", "w").write(out)
except Exception:
pass
log("T4-rc=%d" % rc)
# 失败摘要写 termination-log (K8s 只取尾部 2KB)
tail = out.splitlines()[-20:]
summary = "RC=%d\n" % rc + "\n".join(tail) + "\nPIP/CLONE:" + pip_err[-250:]
try:
open(TL, "w").write(summary[-2000:])
except Exception:
pass
if rc != 0 and clone_ok:
log("T5-crash-push")
os.makedirs("/tmp/jd/logs", exist_ok=True)
open("/tmp/jd/logs/r42_crash_stdout.log", "w").write(out)
sh(["git", "config", "user.email", "dingbin.wang@d-robotics.cc"], cwd="/tmp/jd")
sh(["git", "config", "user.name", "dingbin.wang"], cwd="/tmp/jd")
sh(["git", "config", "http.extraHeader", "Authorization: Basic " + AUTH], cwd="/tmp/jd")
sh(["git", "add", "logs"], cwd="/tmp/jd")
sh(["git", "commit", "-m", "r42 crash logs"], cwd="/tmp/jd")
p = sh(["git", "push", "origin", "main"], cwd="/tmp/jd")
log("T6-push-rc=%d" % p.returncode)
log("DONE rc=%d" % rc)
sys.exit(0 if rc == 0 else 1)