66 lines
4.1 KiB
Python
66 lines
4.1 KiB
Python
"""bench report:从 jobs.sqlite 生成静态 HTML 到 reports/<run_id>.html,给不上机器的人看。"""
|
||
from __future__ import annotations
|
||
|
||
import html
|
||
import time
|
||
|
||
from . import paths, db
|
||
|
||
CSS = """
|
||
body{font:14px/1.5 system-ui,sans-serif;margin:24px;color:#111;background:#fafafa}
|
||
h1{font-size:20px}h2{font-size:16px;margin-top:28px}
|
||
table{border-collapse:collapse;background:#fff;font-variant-numeric:tabular-nums}
|
||
th,td{border:1px solid #ddd;padding:4px 8px;text-align:left;font-size:13px}
|
||
th{background:#f0f0f0}code{font-family:ui-monospace,monospace;font-size:12px}
|
||
.PASS{color:#2f6b46}.FAIL{color:#9e3223}.UNVERIFIED{color:#8a6516}
|
||
"""
|
||
|
||
|
||
def render(run_id: str) -> str:
|
||
with db.connect() as con:
|
||
run = db.get_run(con, run_id)
|
||
jobs = db.list_jobs(con, run_id, mode="oracle")
|
||
ec = db.error_class_counts(con, run_id)
|
||
ro = con.execute("SELECT * FROM rollouts WHERE run_id=? ORDER BY job_id, kk", (run_id,)).fetchall()
|
||
by_state: dict = {}
|
||
for j in jobs:
|
||
by_state[j["state"]] = by_state.get(j["state"], 0) + 1
|
||
out = [f"<!doctype html><meta charset=utf-8><title>bench {run_id}</title><style>{CSS}</style>",
|
||
f"<h1>bench · {html.escape(run_id)}</h1>",
|
||
f"<p>generated {time.strftime('%Y-%m-%d %H:%M:%S')} · jobs {len(jobs)} · "
|
||
+ " · ".join(f"<span class={s}>{s} {n}</span>" for s, n in by_state.items()) + "</p>"]
|
||
if ec:
|
||
out.append("<h2>failures by error_class</h2><table><tr><th>error_class</th><th>stage</th><th>n</th></tr>")
|
||
out += [f"<tr><td>{html.escape(str(r['error_class']))}</td><td>{html.escape(str(r['stage_failed'] or ''))}</td><td>{r['n']}</td></tr>" for r in ec]
|
||
out.append("</table>")
|
||
out.append("<h2>jobs</h2><table><tr><th>job_id</th><th>pri</th><th>state</th><th>error_class</th><th>stage</th><th>est s</th><th>G3 Δ</th><th>lat ms</th></tr>")
|
||
import json
|
||
for j in jobs:
|
||
r = json.loads(j["result"]) if j.get("result") else {}
|
||
g = r.get("gates") or {}
|
||
d = (g.get("G3") or {}).get("delta")
|
||
lat = (g.get("G4") or {}).get("latency_ms_mean")
|
||
out.append(f"<tr><td><code>{html.escape(j['job_id'])}</code></td><td>{j['priority']}</td>"
|
||
f"<td class={j['state']}>{j['state']}</td><td>{html.escape(j['error_class'] or '')}</td>"
|
||
f"<td>{html.escape(j['stage_failed'] or '')}</td><td>{int(j['estimate_s'] or 0)}</td>"
|
||
f"<td>{'' if d is None else f'{d:+.2f}'}</td><td>{'' if lat is None else f'{lat:.2f}'}</td></tr>")
|
||
out.append("</table>")
|
||
if ro:
|
||
from .verify import completion_summary
|
||
out.append("<h2>rollouts</h2><p>实测结果、上板完成与 Agent 终止原因独立统计;超预算不抹掉已经完成的阶段。</p><table><tr><th>job_id</th><th>kk</th><th>mode</th><th>policy</th><th>实测结果</th><th>上板完成</th><th>精度达标</th><th>Agent 终止原因</th><th>claim</th><th>consistent</th><th>review</th><th>turns</th><th>tokens</th></tr>")
|
||
for r in ro:
|
||
v = json.loads(r["verdict"]) if r["verdict"] else {}
|
||
comp = v.get("completion") or completion_summary(v.get("gates") or {}, v.get("stages") or {})
|
||
stop = v.get("execution_terminal") or ("TIMEOUT" if r["state"] == "TIMEOUT" else "")
|
||
out.append(f"<tr><td><code>{html.escape(r['job_id'])}</code></td><td>{r['kk']:02d}</td><td>{r['mode']}</td><td>{r['policy'] or ''}</td>"
|
||
f"<td class={r['state']}>{r['state']}</td><td>{'是' if comp['board_completed'] else '未完成'}</td>"
|
||
f"<td>{'是' if comp['accuracy_passed'] else ('否' if comp['accuracy_evaluated'] else '未评测')}</td>"
|
||
f"<td>{html.escape(stop)}</td><td>{r['claim_terminal'] or ''}</td><td>{r['claim_consistent']}</td>"
|
||
f"<td>{'⚠' if r['review'] else ''}</td><td>{r['turns'] or ''}</td><td>{r['tokens'] or ''}</td></tr>")
|
||
out.append("</table>")
|
||
html_text = "\n".join(out)
|
||
paths.REPORTS.mkdir(parents=True, exist_ok=True)
|
||
p = paths.REPORTS / f"{run_id}.html"
|
||
p.write_text(html_text)
|
||
return str(p)
|