#!/usr/bin/env python3 """shadow_eval.py — 影子评测: 通过 HTTP 调 sidecar, 在 eval_v2/canary 上算 AUC/ECE 用法: python3 shadow_eval.py 结果: termination-log 尾部 + gitea logs/r41_shadow_*.json """ import json, os, sys, time, math, bisect, urllib.request SVC = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else "" DATA = "/tmp/jd" TL = "/dev/termination-log" AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM=" def log(m): print("[shadow]", m, flush=True) def load_jsonl(p): return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] def ece(probs, labels, nbins=15): bins = [[] for _ in range(nbins)] for p, y in zip(probs, labels): bins[min(int(p * nbins), nbins - 1)].append((p, y)) e = 0.0 for b in bins: if not b: continue e += len(b) / len(probs) * abs( sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b)) return e def auc(probs, labels): pos = sorted(p for p, y in zip(probs, labels) if y == 1) neg = sorted(p for p, y in zip(probs, labels) if y == 0) if not pos or not neg: return float("nan") return sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg)) def call_score(query, doc_text, retries=2): body = json.dumps({"query": query, "doc_text": doc_text[:12000]}).encode() req = urllib.request.Request(SVC + "/score", data=body, headers={"Content-Type": "application/json"}) for a in range(retries + 1): try: with urllib.request.urlopen(req, timeout=120) as r: return json.loads(r.read()) except Exception as e: if a == retries: raise time.sleep(3 * (a + 1)) def main(): notes = [] def finish(rc): try: open(TL, "w").write("SHADOW-RC=%d\n" % rc + "\n".join(notes)[-1700:]) except Exception: pass sys.exit(0 if rc == 0 else 1) # 0) 健康检查 try: with urllib.request.urlopen(SVC + "/health", timeout=15) as r: h = json.loads(r.read()) notes.append("HEALTH " + json.dumps(h)[:150]) except Exception as e: notes.append(f"HEALTH-FAIL {str(e)[:120]}") finish(1) slices = [("eval_v2(rdk_eval,OOD)", os.path.join(DATA, "eval_v2.jsonl")), ("canary_en", os.path.join(DATA, "canary_en.jsonl")), ("canary_zh", os.path.join(DATA, "canary_zh.jsonl"))] report = {} lat = [] for name, path in slices: if not os.path.exists(path): continue rows = [r for r in load_jsonl(path) if r.get("hard_label") in (0, 1)] probs, labels = [], [] for i, r in enumerate(rows): t0 = time.time() try: out = call_score(r["query"], r["doc_text"]) except Exception as e: notes.append(f"{name} row{i} FAIL {str(e)[:80]}") finish(1) lat.append(time.time() - t0) probs.append(out["p_final"]) probs_mean = out["p_mean"] labels.append(float(r["hard_label"])) if (i + 1) % 50 == 0: log(f"{name} {i+1}/{len(rows)} last={out['p_final']:.3f} lat={lat[-1]:.2f}s") a, e = auc(probs, labels), ece(probs, labels) report[name] = {"n": len(rows), "auc_pfinal": round(a, 4), "ece_pfinal": round(e, 4), "mean_pos": round(sum(p for p, y in zip(probs, labels) if y == 1) / max(1, sum(labels)), 4), "mean_neg": round(sum(p for p, y in zip(probs, labels) if y == 0) / max(1, len(labels) - sum(labels)), 4)} log(f"{name}: AUC={a:.4f} ECE={e:.4f}") lat_sorted = sorted(lat) report["latency_s"] = {"p50": round(lat_sorted[len(lat_sorted)//2], 3), "p90": round(lat_sorted[int(len(lat_sorted)*0.9)], 3), "mean": round(sum(lat)/len(lat), 3)} report["service"] = SVC log("REPORT " + json.dumps(report)) notes.append("REPORT " + json.dumps(report)[:1200]) # 回传 gitea os.makedirs(os.path.join(DATA, "logs"), exist_ok=True) out_path = os.path.join(DATA, "logs", "r41_shadow_eval.json") json.dump(report, open(out_path, "w"), ensure_ascii=False, indent=1) def sh(args, cwd=None): return subprocess_run(args, cwd) import subprocess def subprocess_run(args, cwd=None): return subprocess.run(args, capture_output=True, text=True, cwd=cwd) subprocess_run(["git", "config", "user.email", "dingbin.wang@d-robotics.cc"], cwd=DATA) subprocess_run(["git", "config", "user.name", "dingbin.wang"], cwd=DATA) subprocess_run(["git", "config", "http.extraHeader", "Authorization: Basic " + AUTH], cwd=DATA) subprocess_run(["git", "add", "logs/r41_shadow_eval.json"], cwd=DATA) subprocess_run(["git", "commit", "-m", "r41 shadow eval results"], cwd=DATA) p = subprocess_run(["git", "push", "origin", "main"], cwd=DATA) notes.append(f"PUSH-rc={p.returncode}") finish(0) if __name__ == "__main__": main()