126 lines
5.0 KiB
Python

#!/usr/bin/env python3
"""shadow_eval.py — 影子评测: 通过 HTTP 调 sidecar, 在 eval_v2/canary 上算 AUC/ECE
用法: python3 shadow_eval.py <service_url>
结果: termination-log 尾部 + gitea logs/r41_shadow_*.json
"""
import json, os, sys, time, math, bisect, urllib.request
SVC = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else ""
DATA = "/tmp/jd"
TL = "/dev/termination-log"
AUTH = "ZGluZ2Jpbi53YW5nOm5OcnZVd3MzTkBRTnM="
def log(m):
print("[shadow]", m, flush=True)
def load_jsonl(p):
return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()]
def ece(probs, labels, nbins=15):
bins = [[] for _ in range(nbins)]
for p, y in zip(probs, labels):
bins[min(int(p * nbins), nbins - 1)].append((p, y))
e = 0.0
for b in bins:
if not b:
continue
e += len(b) / len(probs) * abs(
sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
return e
def auc(probs, labels):
pos = sorted(p for p, y in zip(probs, labels) if y == 1)
neg = sorted(p for p, y in zip(probs, labels) if y == 0)
if not pos or not neg:
return float("nan")
return sum(bisect.bisect_left(neg, p) for p in pos) / (len(pos) * len(neg))
def call_score(query, doc_text, retries=2):
body = json.dumps({"query": query, "doc_text": doc_text[:12000]}).encode()
req = urllib.request.Request(SVC + "/score", data=body,
headers={"Content-Type": "application/json"})
for a in range(retries + 1):
try:
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read())
except Exception as e:
if a == retries:
raise
time.sleep(3 * (a + 1))
def main():
notes = []
def finish(rc):
try:
open(TL, "w").write("SHADOW-RC=%d\n" % rc + "\n".join(notes)[-1700:])
except Exception:
pass
sys.exit(0 if rc == 0 else 1)
# 0) 健康检查
try:
with urllib.request.urlopen(SVC + "/health", timeout=15) as r:
h = json.loads(r.read())
notes.append("HEALTH " + json.dumps(h)[:150])
except Exception as e:
notes.append(f"HEALTH-FAIL {str(e)[:120]}")
finish(1)
slices = [("eval_v2(rdk_eval,OOD)", os.path.join(DATA, "eval_v2.jsonl")),
("canary_en", os.path.join(DATA, "canary_en.jsonl")),
("canary_zh", os.path.join(DATA, "canary_zh.jsonl"))]
report = {}
lat = []
for name, path in slices:
if not os.path.exists(path):
continue
rows = [r for r in load_jsonl(path) if r.get("hard_label") in (0, 1)]
probs, labels = [], []
for i, r in enumerate(rows):
t0 = time.time()
try:
out = call_score(r["query"], r["doc_text"])
except Exception as e:
notes.append(f"{name} row{i} FAIL {str(e)[:80]}")
finish(1)
lat.append(time.time() - t0)
probs.append(out["p_final"])
probs_mean = out["p_mean"]
labels.append(float(r["hard_label"]))
if (i + 1) % 50 == 0:
log(f"{name} {i+1}/{len(rows)} last={out['p_final']:.3f} lat={lat[-1]:.2f}s")
a, e = auc(probs, labels), ece(probs, labels)
report[name] = {"n": len(rows), "auc_pfinal": round(a, 4), "ece_pfinal": round(e, 4),
"mean_pos": round(sum(p for p, y in zip(probs, labels) if y == 1) / max(1, sum(labels)), 4),
"mean_neg": round(sum(p for p, y in zip(probs, labels) if y == 0) / max(1, len(labels) - sum(labels)), 4)}
log(f"{name}: AUC={a:.4f} ECE={e:.4f}")
lat_sorted = sorted(lat)
report["latency_s"] = {"p50": round(lat_sorted[len(lat_sorted)//2], 3),
"p90": round(lat_sorted[int(len(lat_sorted)*0.9)], 3),
"mean": round(sum(lat)/len(lat), 3)}
report["service"] = SVC
log("REPORT " + json.dumps(report))
notes.append("REPORT " + json.dumps(report)[:1200])
# 回传 gitea
os.makedirs(os.path.join(DATA, "logs"), exist_ok=True)
out_path = os.path.join(DATA, "logs", "r41_shadow_eval.json")
json.dump(report, open(out_path, "w"), ensure_ascii=False, indent=1)
def sh(args, cwd=None):
return subprocess_run(args, cwd)
import subprocess
def subprocess_run(args, cwd=None):
return subprocess.run(args, capture_output=True, text=True, cwd=cwd)
subprocess_run(["git", "config", "user.email", "dingbin.wang@d-robotics.cc"], cwd=DATA)
subprocess_run(["git", "config", "user.name", "dingbin.wang"], cwd=DATA)
subprocess_run(["git", "config", "http.extraHeader", "Authorization: Basic " + AUTH], cwd=DATA)
subprocess_run(["git", "add", "logs/r41_shadow_eval.json"], cwd=DATA)
subprocess_run(["git", "commit", "-m", "r41 shadow eval results"], cwd=DATA)
p = subprocess_run(["git", "push", "origin", "main"], cwd=DATA)
notes.append(f"PUSH-rc={p.returncode}")
finish(0)
if __name__ == "__main__":
main()