#!/usr/bin/env python3 """bootstrap_pack.py — fat 容器内: 组装 sidecar 资产并写入数据空间 gitea clone(serve.py + adapter 分片) + ModelScope mmBERT → dataSpace/jev_sidecar_r41/ """ import subprocess, sys, os, shutil, glob, json AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM=" REPO = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git" MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main" TL = "/dev/termination-log" def log(m): print("[pack]", m, flush=True) def sh(args, cwd=None): return subprocess.run(args, capture_output=True, text=True, cwd=cwd) notes = [] def finish(rc): try: open(TL, "w").write("PACK-RC=%d\n" % rc + "\n".join(notes)[-1700:]) except Exception: pass sys.exit(0 if rc == 0 else 1) log("T1-clone") r = sh(["git", "-c", "http.extraHeader=Authorization: Basic " + AUTH, "clone", REPO, "/tmp/jd"]) if r.returncode != 0: notes.append("CLONE-FAIL " + (r.stderr or "")[-300:]) finish(1) log("T2-mmbert") os.makedirs("/tmp/mmbert", exist_ok=True) import urllib.request, time for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "pytorch_model.bin"]: dst = os.path.join("/tmp/mmbert", fn) min_size = 300_000_000 if fn.endswith(".bin") else 100 if os.path.exists(dst) and os.path.getsize(dst) >= min_size: notes.append(f"{fn} cached {os.path.getsize(dst)//1024//1024}MB") continue ok = False for attempt in range(1, 4): try: with urllib.request.urlopen(f"{MS}/{fn}", timeout=600) as resp, \ open(dst + ".tmp", "wb") as f: total = 0 while True: b = resp.read(8 * 1024 * 1024) if not b: break f.write(b) total += len(b) os.replace(dst + ".tmp", dst) notes.append(f"{fn} {total//1024//1024}MB") ok = total >= min_size break except Exception as e: notes.append(f"{fn} att{attempt} {str(e)[:60]}") time.sleep(10 * attempt) if not ok: finish(1) log("T3-adapter") parts = sorted(glob.glob("/tmp/jd/logs/r41_adapter.pt.part*")) if len(parts) != 4: notes.append(f"adapter parts={len(parts)} expect 4") finish(1) with open("/tmp/adapter.pt", "wb") as out: for p in parts: with open(p, "rb") as f: shutil.copyfileobj(f, out) notes.append(f"adapter {os.path.getsize('/tmp/adapter.pt')//1024//1024}MB") log("T4-dataspace") # hostpath 卷根挂载: /var/lib/juicefs/volume//dcloud/dataSpace/... vol_root = "/var/lib/juicefs/volume" pvs = [d for d in os.listdir(vol_root) if "dcloudfs" in d] if os.path.isdir(vol_root) else [] if not pvs: notes.append(f"no dcloudfs PV under {vol_root}: {os.listdir(vol_root)[:6] if os.path.isdir(vol_root) else 'NODIR'}") finish(1) ds_root = None for pv in sorted(pvs): cand = os.path.join(vol_root, pv, "dcloud", "dataSpace") if os.path.isdir(cand): ds_root = cand break if not ds_root: notes.append("no dataSpace under PVs") finish(1) dst_dir = os.path.join(ds_root, "jev_sidecar_r41") os.makedirs(os.path.join(dst_dir, "mmbert"), exist_ok=True) for fn in os.listdir("/tmp/mmbert"): shutil.copy(os.path.join("/tmp/mmbert", fn), os.path.join(dst_dir, "mmbert", fn)) shutil.copy("/tmp/adapter.pt", os.path.join(dst_dir, "adapter.pt")) shutil.copy("/tmp/jd/serve.py", os.path.join(dst_dir, "serve.py")) # vendored laya 模块 (serve.py 依赖, 推理容器挂到 /srv/laya) os.makedirs(os.path.join(dst_dir, "laya"), exist_ok=True) for fn in os.listdir("/tmp/jd/laya_repo"): if fn.endswith(".py"): shutil.copy(os.path.join("/tmp/jd/laya_repo", fn), os.path.join(dst_dir, "laya", fn)) open(os.path.join(dst_dir, "laya", "__init__.py"), "w").write("") json.dump({"tag": "r41", "job": "job-ee6f0c6f16fa", "T_WIN": 1.35, "T_AGG": 1.3, "protocol": {"max_len": 1024, "head_len": 160, "win_tokens": 800, "max_win": 4}, "adapter_bytes": os.path.getsize("/tmp/adapter.pt")}, open(os.path.join(dst_dir, "manifest.json"), "w"), ensure_ascii=False, indent=1) listing = subprocess.run(["ls", "-la", dst_dir], capture_output=True, text=True).stdout notes.append("DST=" + dst_dir) notes.append(listing[-600:]) finish(0)