sidecar: pack bootstrap (dataspace write) + shadow eval HTTP client
This commit is contained in:
parent
e12fed76c9
commit
f3cabcce28
105
bootstrap_pack.py
Normal file
105
bootstrap_pack.py
Normal file
@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""bootstrap_pack.py — fat 容器内: 组装 sidecar 资产并写入数据空间
|
||||
gitea clone(serve.py + adapter 分片) + ModelScope mmBERT → dataSpace/jev_sidecar_r41/
|
||||
"""
|
||||
import subprocess, sys, os, shutil, glob, json
|
||||
|
||||
AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM="
|
||||
REPO = "https://gitea.d-robotics.cc/dingbin.wang/jev-training-data.git"
|
||||
MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/main"
|
||||
TL = "/dev/termination-log"
|
||||
|
||||
def log(m):
|
||||
print("[pack]", m, flush=True)
|
||||
|
||||
def sh(args, cwd=None):
|
||||
return subprocess.run(args, capture_output=True, text=True, cwd=cwd)
|
||||
|
||||
notes = []
|
||||
|
||||
def finish(rc):
|
||||
try:
|
||||
open(TL, "w").write("PACK-RC=%d\n" % rc + "\n".join(notes)[-1700:])
|
||||
except Exception:
|
||||
pass
|
||||
sys.exit(0 if rc == 0 else 1)
|
||||
|
||||
log("T1-clone")
|
||||
r = sh(["git", "-c", "http.extraHeader=Authorization: Basic " + AUTH, "clone", REPO, "/tmp/jd"])
|
||||
if r.returncode != 0:
|
||||
notes.append("CLONE-FAIL " + (r.stderr or "")[-300:])
|
||||
finish(1)
|
||||
|
||||
log("T2-mmbert")
|
||||
os.makedirs("/tmp/mmbert", exist_ok=True)
|
||||
import urllib.request, time
|
||||
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json",
|
||||
"special_tokens_map.json", "pytorch_model.bin"]:
|
||||
dst = os.path.join("/tmp/mmbert", fn)
|
||||
min_size = 300_000_000 if fn.endswith(".bin") else 100
|
||||
if os.path.exists(dst) and os.path.getsize(dst) >= min_size:
|
||||
notes.append(f"{fn} cached {os.path.getsize(dst)//1024//1024}MB")
|
||||
continue
|
||||
ok = False
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
with urllib.request.urlopen(f"{MS}/{fn}", timeout=600) as resp, \
|
||||
open(dst + ".tmp", "wb") as f:
|
||||
total = 0
|
||||
while True:
|
||||
b = resp.read(8 * 1024 * 1024)
|
||||
if not b:
|
||||
break
|
||||
f.write(b)
|
||||
total += len(b)
|
||||
os.replace(dst + ".tmp", dst)
|
||||
notes.append(f"{fn} {total//1024//1024}MB")
|
||||
ok = total >= min_size
|
||||
break
|
||||
except Exception as e:
|
||||
notes.append(f"{fn} att{attempt} {str(e)[:60]}")
|
||||
time.sleep(10 * attempt)
|
||||
if not ok:
|
||||
finish(1)
|
||||
|
||||
log("T3-adapter")
|
||||
parts = sorted(glob.glob("/tmp/jd/logs/r41_adapter.pt.part*"))
|
||||
if len(parts) != 4:
|
||||
notes.append(f"adapter parts={len(parts)} expect 4")
|
||||
finish(1)
|
||||
with open("/tmp/adapter.pt", "wb") as out:
|
||||
for p in parts:
|
||||
with open(p, "rb") as f:
|
||||
shutil.copyfileobj(f, out)
|
||||
notes.append(f"adapter {os.path.getsize('/tmp/adapter.pt')//1024//1024}MB")
|
||||
|
||||
log("T4-dataspace")
|
||||
# hostpath 卷根挂载: /var/lib/juicefs/volume/<PV>/dcloud/dataSpace/...
|
||||
vol_root = "/var/lib/juicefs/volume"
|
||||
pvs = [d for d in os.listdir(vol_root) if "dcloudfs" in d] if os.path.isdir(vol_root) else []
|
||||
if not pvs:
|
||||
notes.append(f"no dcloudfs PV under {vol_root}: {os.listdir(vol_root)[:6] if os.path.isdir(vol_root) else 'NODIR'}")
|
||||
finish(1)
|
||||
ds_root = None
|
||||
for pv in sorted(pvs):
|
||||
cand = os.path.join(vol_root, pv, "dcloud", "dataSpace")
|
||||
if os.path.isdir(cand):
|
||||
ds_root = cand
|
||||
break
|
||||
if not ds_root:
|
||||
notes.append("no dataSpace under PVs")
|
||||
finish(1)
|
||||
dst_dir = os.path.join(ds_root, "jev_sidecar_r41")
|
||||
os.makedirs(os.path.join(dst_dir, "mmbert"), exist_ok=True)
|
||||
for fn in os.listdir("/tmp/mmbert"):
|
||||
shutil.copy(os.path.join("/tmp/mmbert", fn), os.path.join(dst_dir, "mmbert", fn))
|
||||
shutil.copy("/tmp/adapter.pt", os.path.join(dst_dir, "adapter.pt"))
|
||||
shutil.copy("/tmp/jd/serve.py", os.path.join(dst_dir, "serve.py"))
|
||||
json.dump({"tag": "r41", "job": "job-ee6f0c6f16fa", "T_WIN": 1.35, "T_AGG": 1.3,
|
||||
"protocol": {"max_len": 1024, "head_len": 160, "win_tokens": 800, "max_win": 4},
|
||||
"adapter_bytes": os.path.getsize("/tmp/adapter.pt")},
|
||||
open(os.path.join(dst_dir, "manifest.json"), "w"), ensure_ascii=False, indent=1)
|
||||
listing = subprocess.run(["ls", "-la", dst_dir], capture_output=True, text=True).stdout
|
||||
notes.append("DST=" + dst_dir)
|
||||
notes.append(listing[-600:])
|
||||
finish(0)
|
||||
125
shadow_eval.py
Normal file
125
shadow_eval.py
Normal file
@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""shadow_eval.py — 影子评测: 通过 HTTP 调 sidecar, 在 eval_v2/canary 上算 AUC/ECE
|
||||
用法: python3 shadow_eval.py <service_url>
|
||||
结果: termination-log 尾部 + gitea logs/r41_shadow_*.json
|
||||
"""
|
||||
import json, os, sys, time, math, bisect, urllib.request
|
||||
|
||||
SVC = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else ""
|
||||
DATA = "/tmp/jd"
|
||||
TL = "/dev/termination-log"
|
||||
AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM="
|
||||
|
||||
def log(m):
|
||||
print("[shadow]", m, flush=True)
|
||||
|
||||
def load_jsonl(p):
|
||||
return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()]
|
||||
|
||||
def ece(probs, labels, nbins=15):
|
||||
bins = [[] for _ in range(nbins)]
|
||||
for p, y in zip(probs, labels):
|
||||
bins[min(int(p * nbins), nbins - 1)].append((p, y))
|
||||
e = 0.0
|
||||
for b in bins:
|
||||
if not b:
|
||||
continue
|
||||
e += len(b) / len(probs) * abs(
|
||||
sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
|
||||
return e
|
||||
|
||||
def auc(probs, labels):
|
||||
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
|
||||
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
|
||||
if not pos or not neg:
|
||||
return float("nan")
|
||||
return sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg))
|
||||
|
||||
def call_score(query, doc_text, retries=2):
|
||||
body = json.dumps({"query": query, "doc_text": doc_text[:12000]}).encode()
|
||||
req = urllib.request.Request(SVC + "/score", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
for a in range(retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
return json.loads(r.read())
|
||||
except Exception as e:
|
||||
if a == retries:
|
||||
raise
|
||||
time.sleep(3 * (a + 1))
|
||||
|
||||
def main():
|
||||
notes = []
|
||||
def finish(rc):
|
||||
try:
|
||||
open(TL, "w").write("SHADOW-RC=%d\n" % rc + "\n".join(notes)[-1700:])
|
||||
except Exception:
|
||||
pass
|
||||
sys.exit(0 if rc == 0 else 1)
|
||||
|
||||
# 0) 健康检查
|
||||
try:
|
||||
with urllib.request.urlopen(SVC + "/health", timeout=15) as r:
|
||||
h = json.loads(r.read())
|
||||
notes.append("HEALTH " + json.dumps(h)[:150])
|
||||
except Exception as e:
|
||||
notes.append(f"HEALTH-FAIL {str(e)[:120]}")
|
||||
finish(1)
|
||||
|
||||
slices = [("eval_v2(rdk_eval,OOD)", os.path.join(DATA, "eval_v2.jsonl")),
|
||||
("canary_en", os.path.join(DATA, "canary_en.jsonl")),
|
||||
("canary_zh", os.path.join(DATA, "canary_zh.jsonl"))]
|
||||
report = {}
|
||||
lat = []
|
||||
for name, path in slices:
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
rows = [r for r in load_jsonl(path) if r.get("hard_label") in (0, 1)]
|
||||
probs, labels = [], []
|
||||
for i, r in enumerate(rows):
|
||||
t0 = time.time()
|
||||
try:
|
||||
out = call_score(r["query"], r["doc_text"])
|
||||
except Exception as e:
|
||||
notes.append(f"{name} row{i} FAIL {str(e)[:80]}")
|
||||
finish(1)
|
||||
lat.append(time.time() - t0)
|
||||
probs.append(out["p_final"])
|
||||
probs_mean = out["p_mean"]
|
||||
labels.append(float(r["hard_label"]))
|
||||
if (i + 1) % 50 == 0:
|
||||
log(f"{name} {i+1}/{len(rows)} last={out['p_final']:.3f} lat={lat[-1]:.2f}s")
|
||||
a, e = auc(probs, labels), ece(probs, labels)
|
||||
report[name] = {"n": len(rows), "auc_pfinal": round(a, 4), "ece_pfinal": round(e, 4),
|
||||
"mean_pos": round(sum(p for p, y in zip(probs, labels) if y == 1) / max(1, sum(labels)), 4),
|
||||
"mean_neg": round(sum(p for p, y in zip(probs, labels) if y == 0) / max(1, len(labels) - sum(labels)), 4)}
|
||||
log(f"{name}: AUC={a:.4f} ECE={e:.4f}")
|
||||
|
||||
lat_sorted = sorted(lat)
|
||||
report["latency_s"] = {"p50": round(lat_sorted[len(lat_sorted)//2], 3),
|
||||
"p90": round(lat_sorted[int(len(lat_sorted)*0.9)], 3),
|
||||
"mean": round(sum(lat)/len(lat), 3)}
|
||||
report["service"] = SVC
|
||||
log("REPORT " + json.dumps(report))
|
||||
notes.append("REPORT " + json.dumps(report)[:1200])
|
||||
|
||||
# 回传 gitea
|
||||
os.makedirs(os.path.join(DATA, "logs"), exist_ok=True)
|
||||
out_path = os.path.join(DATA, "logs", "r41_shadow_eval.json")
|
||||
json.dump(report, open(out_path, "w"), ensure_ascii=False, indent=1)
|
||||
def sh(args, cwd=None):
|
||||
return subprocess_run(args, cwd)
|
||||
import subprocess
|
||||
def subprocess_run(args, cwd=None):
|
||||
return subprocess.run(args, capture_output=True, text=True, cwd=cwd)
|
||||
subprocess_run(["git", "config", "user.email", "dingbin.wang@d-robotics.cc"], cwd=DATA)
|
||||
subprocess_run(["git", "config", "user.name", "dingbin.wang"], cwd=DATA)
|
||||
subprocess_run(["git", "config", "http.extraHeader", "Authorization: Basic " + AUTH], cwd=DATA)
|
||||
subprocess_run(["git", "add", "logs/r41_shadow_eval.json"], cwd=DATA)
|
||||
subprocess_run(["git", "commit", "-m", "r41 shadow eval results"], cwd=DATA)
|
||||
p = subprocess_run(["git", "push", "origin", "main"], cwd=DATA)
|
||||
notes.append(f"PUSH-rc={p.returncode}")
|
||||
finish(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user