data-gen-bpu/harness/report.py

66 lines
4.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""bench report:从 jobs.sqlite 生成静态 HTML 到 reports/<run_id>.html,给不上机器的人看。"""
from __future__ import annotations
import html
import time
from . import paths, db
CSS = """
body{font:14px/1.5 system-ui,sans-serif;margin:24px;color:#111;background:#fafafa}
h1{font-size:20px}h2{font-size:16px;margin-top:28px}
table{border-collapse:collapse;background:#fff;font-variant-numeric:tabular-nums}
th,td{border:1px solid #ddd;padding:4px 8px;text-align:left;font-size:13px}
th{background:#f0f0f0}code{font-family:ui-monospace,monospace;font-size:12px}
.PASS{color:#2f6b46}.FAIL{color:#9e3223}.UNVERIFIED{color:#8a6516}
"""
def render(run_id: str) -> str:
with db.connect() as con:
run = db.get_run(con, run_id)
jobs = db.list_jobs(con, run_id, mode="oracle")
ec = db.error_class_counts(con, run_id)
ro = con.execute("SELECT * FROM rollouts WHERE run_id=? ORDER BY job_id, kk", (run_id,)).fetchall()
by_state: dict = {}
for j in jobs:
by_state[j["state"]] = by_state.get(j["state"], 0) + 1
out = [f"<!doctype html><meta charset=utf-8><title>bench {run_id}</title><style>{CSS}</style>",
f"<h1>bench · {html.escape(run_id)}</h1>",
f"<p>generated {time.strftime('%Y-%m-%d %H:%M:%S')} · jobs {len(jobs)} · "
+ " · ".join(f"<span class={s}>{s} {n}</span>" for s, n in by_state.items()) + "</p>"]
if ec:
out.append("<h2>failures by error_class</h2><table><tr><th>error_class</th><th>stage</th><th>n</th></tr>")
out += [f"<tr><td>{html.escape(str(r['error_class']))}</td><td>{html.escape(str(r['stage_failed'] or ''))}</td><td>{r['n']}</td></tr>" for r in ec]
out.append("</table>")
out.append("<h2>jobs</h2><table><tr><th>job_id</th><th>pri</th><th>state</th><th>error_class</th><th>stage</th><th>est s</th><th>G3 Δ</th><th>lat ms</th></tr>")
import json
for j in jobs:
r = json.loads(j["result"]) if j.get("result") else {}
g = r.get("gates") or {}
d = (g.get("G3") or {}).get("delta")
lat = (g.get("G4") or {}).get("latency_ms_mean")
out.append(f"<tr><td><code>{html.escape(j['job_id'])}</code></td><td>{j['priority']}</td>"
f"<td class={j['state']}>{j['state']}</td><td>{html.escape(j['error_class'] or '')}</td>"
f"<td>{html.escape(j['stage_failed'] or '')}</td><td>{int(j['estimate_s'] or 0)}</td>"
f"<td>{'' if d is None else f'{d:+.2f}'}</td><td>{'' if lat is None else f'{lat:.2f}'}</td></tr>")
out.append("</table>")
if ro:
from .verify import completion_summary
out.append("<h2>rollouts</h2><p>实测结果、上板完成与 Agent 终止原因独立统计;超预算不抹掉已经完成的阶段。</p><table><tr><th>job_id</th><th>kk</th><th>mode</th><th>policy</th><th>实测结果</th><th>上板完成</th><th>精度达标</th><th>Agent 终止原因</th><th>claim</th><th>consistent</th><th>review</th><th>turns</th><th>tokens</th></tr>")
for r in ro:
v = json.loads(r["verdict"]) if r["verdict"] else {}
comp = v.get("completion") or completion_summary(v.get("gates") or {}, v.get("stages") or {})
stop = v.get("execution_terminal") or ("TIMEOUT" if r["state"] == "TIMEOUT" else "")
out.append(f"<tr><td><code>{html.escape(r['job_id'])}</code></td><td>{r['kk']:02d}</td><td>{r['mode']}</td><td>{r['policy'] or ''}</td>"
f"<td class={r['state']}>{r['state']}</td><td>{'是' if comp['board_completed'] else '未完成'}</td>"
f"<td>{'是' if comp['accuracy_passed'] else ('否' if comp['accuracy_evaluated'] else '未评测')}</td>"
f"<td>{html.escape(stop)}</td><td>{r['claim_terminal'] or ''}</td><td>{r['claim_consistent']}</td>"
f"<td>{'⚠' if r['review'] else ''}</td><td>{r['turns'] or ''}</td><td>{r['tokens'] or ''}</td></tr>")
out.append("</table>")
html_text = "\n".join(out)
paths.REPORTS.mkdir(parents=True, exist_ok=True)
p = paths.REPORTS / f"{run_id}.html"
p.write_text(html_text)
return str(p)