diff --git a/bootstrap_pack.py b/bootstrap_pack.py new file mode 100644 index 0000000..b3c832b --- /dev/null +++ b/bootstrap_pack.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""bootstrap_pack.py — fat 容器内: 组装 sidecar 资产并写入数据空间 +gitea clone(serve.py + adapter 分片) + ModelScope mmBERT → dataSpace/jev_sidecar_r41/ +""" +import subprocess, sys, os, shutil, glob, json + +AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM=" +REPO = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git" +MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main" +TL = "/dev/termination-log" + +def log(m): + print("[pack]", m, flush=True) + +def sh(args, cwd=None): + return subprocess.run(args, capture_output=True, text=True, cwd=cwd) + +notes = [] + +def finish(rc): + try: + open(TL, "w").write("PACK-RC=%d\n" % rc + "\n".join(notes)[-1700:]) + except Exception: + pass + sys.exit(0 if rc == 0 else 1) + +log("T1-clone") +r = sh(["git", "-c", "http.extraHeader=Authorization: Basic " + AUTH, "clone", REPO, "/tmp/jd"]) +if r.returncode != 0: + notes.append("CLONE-FAIL " + (r.stderr or "")[-300:]) + finish(1) + +log("T2-mmbert") +os.makedirs("/tmp/mmbert", exist_ok=True) +import urllib.request, time +for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", + "special_tokens_map.json", "pytorch_model.bin"]: + dst = os.path.join("/tmp/mmbert", fn) + min_size = 300_000_000 if fn.endswith(".bin") else 100 + if os.path.exists(dst) and os.path.getsize(dst) >= min_size: + notes.append(f"{fn} cached {os.path.getsize(dst)//1024//1024}MB") + continue + ok = False + for attempt in range(1, 4): + try: + with urllib.request.urlopen(f"{MS}/{fn}", timeout=600) as resp, \ + open(dst + ".tmp", "wb") as f: + total = 0 + while True: + b = resp.read(8 * 1024 * 1024) + if not b: + break + f.write(b) + total += len(b) + os.replace(dst + ".tmp", dst) + notes.append(f"{fn} {total//1024//1024}MB") + ok = total >= min_size + break + except Exception as e: + notes.append(f"{fn} att{attempt} {str(e)[:60]}") + time.sleep(10 * attempt) + if not ok: + finish(1) + +log("T3-adapter") +parts = sorted(glob.glob("/tmp/jd/logs/r41_adapter.pt.part*")) +if len(parts) != 4: + notes.append(f"adapter parts={len(parts)} expect 4") + finish(1) +with open("/tmp/adapter.pt", "wb") as out: + for p in parts: + with open(p, "rb") as f: + shutil.copyfileobj(f, out) +notes.append(f"adapter {os.path.getsize('/tmp/adapter.pt')//1024//1024}MB") + +log("T4-dataspace") +# hostpath 卷根挂载: /var/lib/juicefs/volume//dcloud/dataSpace/... +vol_root = "/var/lib/juicefs/volume" +pvs = [d for d in os.listdir(vol_root) if "dcloudfs" in d] if os.path.isdir(vol_root) else [] +if not pvs: + notes.append(f"no dcloudfs PV under {vol_root}: {os.listdir(vol_root)[:6] if os.path.isdir(vol_root) else 'NODIR'}") + finish(1) +ds_root = None +for pv in sorted(pvs): + cand = os.path.join(vol_root, pv, "dcloud", "dataSpace") + if os.path.isdir(cand): + ds_root = cand + break +if not ds_root: + notes.append("no dataSpace under PVs") + finish(1) +dst_dir = os.path.join(ds_root, "jev_sidecar_r41") +os.makedirs(os.path.join(dst_dir, "mmbert"), exist_ok=True) +for fn in os.listdir("/tmp/mmbert"): + shutil.copy(os.path.join("/tmp/mmbert", fn), os.path.join(dst_dir, "mmbert", fn)) +shutil.copy("/tmp/adapter.pt", os.path.join(dst_dir, "adapter.pt")) +shutil.copy("/tmp/jd/serve.py", os.path.join(dst_dir, "serve.py")) +json.dump({"tag": "r41", "job": "job-ee6f0c6f16fa", "T_WIN": 1.35, "T_AGG": 1.3, + "protocol": {"max_len": 1024, "head_len": 160, "win_tokens": 800, "max_win": 4}, + "adapter_bytes": os.path.getsize("/tmp/adapter.pt")}, + open(os.path.join(dst_dir, "manifest.json"), "w"), ensure_ascii=False, indent=1) +listing = subprocess.run(["ls", "-la", dst_dir], capture_output=True, text=True).stdout +notes.append("DST=" + dst_dir) +notes.append(listing[-600:]) +finish(0) diff --git a/shadow_eval.py b/shadow_eval.py new file mode 100644 index 0000000..d284fdf --- /dev/null +++ b/shadow_eval.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""shadow_eval.py — 影子评测: 通过 HTTP 调 sidecar, 在 eval_v2/canary 上算 AUC/ECE +用法: python3 shadow_eval.py +结果: termination-log 尾部 + gitea logs/r41_shadow_*.json +""" +import json, os, sys, time, math, bisect, urllib.request + +SVC = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else "" +DATA = "/tmp/jd" +TL = "/dev/termination-log" +AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM=" + +def log(m): + print("[shadow]", m, flush=True) + +def load_jsonl(p): + return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] + +def ece(probs, labels, nbins=15): + bins = [[] for _ in range(nbins)] + for p, y in zip(probs, labels): + bins[min(int(p * nbins), nbins - 1)].append((p, y)) + e = 0.0 + for b in bins: + if not b: + continue + e += len(b) / len(probs) * abs( + sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b)) + return e + +def auc(probs, labels): + pos = sorted(p for p, y in zip(probs, labels) if y == 1) + neg = sorted(p for p, y in zip(probs, labels) if y == 0) + if not pos or not neg: + return float("nan") + return sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg)) + +def call_score(query, doc_text, retries=2): + body = json.dumps({"query": query, "doc_text": doc_text[:12000]}).encode() + req = urllib.request.Request(SVC + "/score", data=body, + headers={"Content-Type": "application/json"}) + for a in range(retries + 1): + try: + with urllib.request.urlopen(req, timeout=120) as r: + return json.loads(r.read()) + except Exception as e: + if a == retries: + raise + time.sleep(3 * (a + 1)) + +def main(): + notes = [] + def finish(rc): + try: + open(TL, "w").write("SHADOW-RC=%d\n" % rc + "\n".join(notes)[-1700:]) + except Exception: + pass + sys.exit(0 if rc == 0 else 1) + + # 0) 健康检查 + try: + with urllib.request.urlopen(SVC + "/health", timeout=15) as r: + h = json.loads(r.read()) + notes.append("HEALTH " + json.dumps(h)[:150]) + except Exception as e: + notes.append(f"HEALTH-FAIL {str(e)[:120]}") + finish(1) + + slices = [("eval_v2(rdk_eval,OOD)", os.path.join(DATA, "eval_v2.jsonl")), + ("canary_en", os.path.join(DATA, "canary_en.jsonl")), + ("canary_zh", os.path.join(DATA, "canary_zh.jsonl"))] + report = {} + lat = [] + for name, path in slices: + if not os.path.exists(path): + continue + rows = [r for r in load_jsonl(path) if r.get("hard_label") in (0, 1)] + probs, labels = [], [] + for i, r in enumerate(rows): + t0 = time.time() + try: + out = call_score(r["query"], r["doc_text"]) + except Exception as e: + notes.append(f"{name} row{i} FAIL {str(e)[:80]}") + finish(1) + lat.append(time.time() - t0) + probs.append(out["p_final"]) + probs_mean = out["p_mean"] + labels.append(float(r["hard_label"])) + if (i + 1) % 50 == 0: + log(f"{name} {i+1}/{len(rows)} last={out['p_final']:.3f} lat={lat[-1]:.2f}s") + a, e = auc(probs, labels), ece(probs, labels) + report[name] = {"n": len(rows), "auc_pfinal": round(a, 4), "ece_pfinal": round(e, 4), + "mean_pos": round(sum(p for p, y in zip(probs, labels) if y == 1) / max(1, sum(labels)), 4), + "mean_neg": round(sum(p for p, y in zip(probs, labels) if y == 0) / max(1, len(labels) - sum(labels)), 4)} + log(f"{name}: AUC={a:.4f} ECE={e:.4f}") + + lat_sorted = sorted(lat) + report["latency_s"] = {"p50": round(lat_sorted[len(lat_sorted)//2], 3), + "p90": round(lat_sorted[int(len(lat_sorted)*0.9)], 3), + "mean": round(sum(lat)/len(lat), 3)} + report["service"] = SVC + log("REPORT " + json.dumps(report)) + notes.append("REPORT " + json.dumps(report)[:1200]) + + # 回传 gitea + os.makedirs(os.path.join(DATA, "logs"), exist_ok=True) + out_path = os.path.join(DATA, "logs", "r41_shadow_eval.json") + json.dump(report, open(out_path, "w"), ensure_ascii=False, indent=1) + def sh(args, cwd=None): + return subprocess_run(args, cwd) + import subprocess + def subprocess_run(args, cwd=None): + return subprocess.run(args, capture_output=True, text=True, cwd=cwd) + subprocess_run(["git", "config", "user.email", "dingbin.wang@d-robotics.cc"], cwd=DATA) + subprocess_run(["git", "config", "user.name", "dingbin.wang"], cwd=DATA) + subprocess_run(["git", "config", "http.extraHeader", "Authorization: Basic " + AUTH], cwd=DATA) + subprocess_run(["git", "add", "logs/r41_shadow_eval.json"], cwd=DATA) + subprocess_run(["git", "commit", "-m", "r41 shadow eval results"], cwd=DATA) + p = subprocess_run(["git", "push", "origin", "main"], cwd=DATA) + notes.append(f"PUSH-rc={p.returncode}") + finish(0) + +if __name__ == "__main__": + main()