460 lines
26 KiB
Python
460 lines
26 KiB
Python
"""bench coverage — 338 个模型里,哪些做了、哪些没做、哪些在跑。
|
||
|
||
bench coverage 总览 + 按任务分组
|
||
bench coverage --missing 只列"还没做"的
|
||
bench coverage --task cls --detail 逐模型一行
|
||
bench coverage --csv cov.csv 导出给别人看
|
||
|
||
三个层次,别混:
|
||
registry models.csv 里 338 行,status=ready 的才会被 matrix 展开(needs_adapter/needs_eval/skip 是"还没接进来")
|
||
oracle 每个 (model × march × backend × qcfg) 一个 job:PENDING 没开始 / ORACLE_RUNNING 在跑 / PASS·FAIL·UNVERIFIED 已结束
|
||
rollout 每个 job 计划 k 条轨迹:QUEUED 没采 / RUNNING 在采 / PASS·FAIL·BLOCKED·TIMEOUT 已采
|
||
"是不是还在跑"看顶部那行 LIVE:有没有调度进程、池占了几个槽、最近一次阶段心跳多久以前。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import csv as csvmod
|
||
import json
|
||
import time
|
||
from collections import Counter, defaultdict
|
||
from pathlib import Path
|
||
|
||
from . import paths, db
|
||
from .registry import models, expand_matrix
|
||
from .scheduler import pool_usage, pool_size
|
||
|
||
DONE = ("PASS", "FAIL", "UNVERIFIED")
|
||
ROLLOUT_DONE = ("PASS", "FAIL", "BLOCKED", "TIMEOUT", "UNVERIFIED")
|
||
|
||
|
||
def _fmt_age(s: float | None) -> str:
|
||
if s is None:
|
||
return "从未"
|
||
s = int(s)
|
||
if s < 90:
|
||
return f"{s}s 前"
|
||
if s < 5400:
|
||
return f"{s // 60}m 前"
|
||
return f"{s // 3600}h{(s % 3600) // 60:02d}m 前"
|
||
|
||
|
||
def live_line() -> list[str]:
|
||
"""是不是还在跑:调度进程 + 池占用 + 最近心跳。"""
|
||
from .pipeline import scheduler_alive
|
||
out = []
|
||
alive = scheduler_alive()
|
||
pools = pool_usage()
|
||
busy = sum(pools.values())
|
||
with db.connect() as con:
|
||
row = con.execute("SELECT MAX(COALESCE(finished_at, started_at)) FROM stages").fetchone()
|
||
last = row[0] if row and row[0] else None
|
||
n_run = con.execute("SELECT COUNT(*) FROM stages WHERE state='RUNNING'").fetchone()[0]
|
||
age = (time.time() - last) if last else None
|
||
if alive or busy or n_run:
|
||
state = "🟢 在跑"
|
||
elif age is not None and age < 300:
|
||
state = "🟡 刚停(进程没了但 5 分钟内还有活动)"
|
||
else:
|
||
state = "🔴 没在跑"
|
||
out.append(f"LIVE {state} 调度进程={'有' if alive else '无'} "
|
||
+ " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pools.items())
|
||
+ f" RUNNING 阶段={n_run} 最近心跳={_fmt_age(age)}")
|
||
if not alive and not busy:
|
||
out.append(" 没有调度进程:要继续跑就 scripts/start_run.sh --resume(先 reset-stale 再接着跑 PENDING)")
|
||
return out
|
||
|
||
|
||
def exported_keys() -> set[tuple]:
|
||
"""已经进过 datasets/ 的 (run_id, job_id, kk)。"""
|
||
import json
|
||
keys = set()
|
||
files = sorted(paths.DATASETS.glob("*/teacher.jsonl")) + sorted(paths.DATASETS.glob("*/student.jsonl"))
|
||
for f in files:
|
||
if (f.parent / "VOID").exists():
|
||
continue
|
||
try:
|
||
with f.open(errors="replace") as stream:
|
||
for line in stream:
|
||
r = json.loads(line)
|
||
if (r.get("source") in (None, "agent_events") and isinstance(r.get("messages"), list)
|
||
and isinstance(r.get("rollout"), int)):
|
||
keys.add((r.get("run_id"), r.get("job_id"), r["rollout"]))
|
||
except (OSError, json.JSONDecodeError):
|
||
continue
|
||
return keys
|
||
|
||
|
||
def collect(run_id: str | None) -> dict:
|
||
"""汇总指定 run 的实际计划;run_id=None 时只展示 registry 的预期。"""
|
||
rows = models()
|
||
by_model = {r["model_id"]: r for r in rows}
|
||
jobs, ro = [], []
|
||
if run_id is not None:
|
||
with db.connect() as con:
|
||
jobs = db.list_jobs(con, run_id, mode="oracle")
|
||
ro = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=?", (run_id,))]
|
||
planned_rollouts = Counter(r["job_id"] for r in ro)
|
||
exported = exported_keys() if run_id is not None else set()
|
||
|
||
per_model: dict[str, dict] = {}
|
||
for m, r in by_model.items():
|
||
per_model[m] = {"model": m, "task": r["task"], "priority": r["priority"], "status": r.get("status") or "ready",
|
||
"jobs": [], "oracle": Counter(), "rollout": Counter(),
|
||
"planned_rollouts": 0, "exported": 0}
|
||
if run_id is None:
|
||
for spec in expand_matrix():
|
||
if spec.mode != "oracle" and spec.model in per_model:
|
||
per_model[spec.model]["planned_rollouts"] += spec.k
|
||
for j in jobs:
|
||
d = per_model.get(j["model"])
|
||
if d is None:
|
||
continue
|
||
d["jobs"].append(j)
|
||
d["oracle"][j["state"]] += 1
|
||
d["planned_rollouts"] += planned_rollouts.get(j["job_id"], 0)
|
||
for r in ro:
|
||
jrow = next((x for x in jobs if x["job_id"] == r["job_id"]), None)
|
||
if jrow:
|
||
model = jrow["model"]
|
||
else:
|
||
jp = paths.RUNS / run_id / r["job_id"] / "job.json"
|
||
model = json.loads(jp.read_text()).get("model") if jp.exists() else None
|
||
d = per_model.get(model)
|
||
if d is None:
|
||
continue
|
||
if not jrow:
|
||
d["planned_rollouts"] += 1
|
||
d["rollout"][r["state"]] += 1
|
||
if (run_id, r["job_id"], r["kk"]) in exported:
|
||
d["exported"] += 1
|
||
return per_model
|
||
|
||
|
||
def render(run_id: str, task: str | None = None, missing: bool = False, detail: bool = False) -> str:
|
||
per_model = collect(run_id)
|
||
lines = [f"bench coverage · run {run_id} · {time.strftime('%Y-%m-%d %H:%M:%S')}"]
|
||
lines += live_line()
|
||
lines.append("")
|
||
|
||
# ---- registry 层
|
||
st = Counter(d["status"] for d in per_model.values())
|
||
ready = [d for d in per_model.values() if d["status"] == "ready"]
|
||
lines.append(f"registry: 共 {len(per_model)} 个模型 · " + " · ".join(f"{k}={v}" for k, v in st.most_common())
|
||
+ " (只有 ready 会被展开成 job;其余是「还没接进来」,不是「失败」)")
|
||
|
||
# ---- oracle 层
|
||
oc = Counter()
|
||
for d in ready:
|
||
oc.update(d["oracle"])
|
||
total_jobs = sum(oc.values())
|
||
done_jobs = sum(oc[k] for k in DONE)
|
||
lines.append(f"oracle: {total_jobs} 个 job · 已结束 {done_jobs}"
|
||
+ (f"(PASS {oc['PASS']} / FAIL {oc['FAIL']} / UNVERIFIED {oc['UNVERIFIED']})" if done_jobs else "")
|
||
+ f" · 在跑 {oc['ORACLE_RUNNING']} · 没开始 {oc['PENDING']}")
|
||
|
||
# ---- rollout 层
|
||
rc = Counter()
|
||
planned = exported = 0
|
||
for d in ready:
|
||
rc.update(d["rollout"]); planned += d["planned_rollouts"]; exported += d["exported"]
|
||
got = sum(rc[k] for k in ROLLOUT_DONE)
|
||
lines.append(f"rollout: 计划 {planned} 条 · 已采 {got}"
|
||
+ (f"(PASS {rc['PASS']} / FAIL {rc['FAIL']} / BLOCKED {rc['BLOCKED']} / TIMEOUT {rc['TIMEOUT']} / UNVERIFIED {rc['UNVERIFIED']})" if got else "")
|
||
+ f" · 在采 {rc['RUNNING'] + rc['VERIFYING']}(RUNNING {rc['RUNNING']} / VERIFYING {rc['VERIFYING']})"
|
||
+ f" · 没采 {rc['QUEUED']} · 已导出 {exported}")
|
||
lines.append("")
|
||
|
||
# ---- 按任务分组
|
||
groups: dict[str, list[dict]] = defaultdict(list)
|
||
for d in per_model.values():
|
||
groups[d["task"]].append(d)
|
||
lines.append(f" {'task':<7} {'模型':>5} {'ready':>6} | {'job':>5} {'已结束':>6} {'PASS':>5} {'FAIL':>5} {'在跑':>4} {'没开始':>6} | "
|
||
f"{'轨迹计划':>8} {'已采':>5} {'PASS':>5} {'没采':>5}")
|
||
for t in sorted(groups):
|
||
g = groups[t]
|
||
gr = [d for d in g if d["status"] == "ready"]
|
||
o, r = Counter(), Counter()
|
||
pl = 0
|
||
for d in gr:
|
||
o.update(d["oracle"]); r.update(d["rollout"]); pl += d["planned_rollouts"]
|
||
lines.append(f" {t:<7} {len(g):>5} {len(gr):>6} | {sum(o.values()):>5} {sum(o[k] for k in DONE):>6} "
|
||
f"{o['PASS']:>5} {o['FAIL']:>5} {o['ORACLE_RUNNING']:>4} {o['PENDING']:>6} | "
|
||
f"{pl:>8} {sum(r[k] for k in ROLLOUT_DONE):>5} {r['PASS']:>5} {r['QUEUED']:>5}")
|
||
|
||
# ---- 逐模型
|
||
sel = [d for d in per_model.values() if (not task or d["task"] == task)]
|
||
if missing:
|
||
sel = [d for d in sel if d["status"] != "ready"
|
||
or d["oracle"].get("PENDING") or not d["jobs"]
|
||
or d["rollout"].get("QUEUED")]
|
||
lines.append("")
|
||
lines.append(f" 还没做的({len(sel)}):status≠ready,或 oracle 有 PENDING,或轨迹有 QUEUED")
|
||
if missing or detail:
|
||
lines.append("")
|
||
lines.append(f" {'model':<26} {'task':<6} {'pri':<4} {'status':<14} {'oracle':<34} {'rollout':<26}")
|
||
for d in sorted(sel, key=lambda x: (x["task"], x["priority"], x["model"])):
|
||
o = " ".join(f"{k}={v}" for k, v in sorted(d["oracle"].items())) or ("—(未展开)" if d["status"] != "ready" else "—")
|
||
r = " ".join(f"{k}={v}" for k, v in sorted(d["rollout"].items())) or "—"
|
||
if d["exported"]:
|
||
r += f" 导出={d['exported']}"
|
||
lines.append(f" {d['model']:<26} {d['task']:<6} {d['priority']:<4} {d['status']:<14} {o:<34} {r:<26}")
|
||
else:
|
||
lines.append("")
|
||
lines.append(" 逐模型明细:--detail(可加 --task cls);只看没做的:--missing;导出表格:--csv cov.csv")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def to_csv(run_id: str, out: str) -> str:
|
||
per_model = collect(run_id)
|
||
fields = ["model", "task", "priority", "status", "jobs", "oracle_done", "oracle_pass", "oracle_fail",
|
||
"oracle_unverified", "oracle_running", "oracle_pending", "rollout_planned", "rollout_done",
|
||
"rollout_pass", "rollout_queued", "exported"]
|
||
with open(out, "w", newline="", encoding="utf-8") as f:
|
||
w = csvmod.DictWriter(f, fieldnames=fields)
|
||
w.writeheader()
|
||
for d in sorted(per_model.values(), key=lambda x: (x["task"], x["priority"], x["model"])):
|
||
o, r = d["oracle"], d["rollout"]
|
||
w.writerow({"model": d["model"], "task": d["task"], "priority": d["priority"], "status": d["status"],
|
||
"jobs": len(d["jobs"]), "oracle_done": sum(o[k] for k in DONE), "oracle_pass": o["PASS"],
|
||
"oracle_fail": o["FAIL"], "oracle_unverified": o["UNVERIFIED"],
|
||
"oracle_running": o["ORACLE_RUNNING"], "oracle_pending": o["PENDING"],
|
||
"rollout_planned": d["planned_rollouts"], "rollout_done": sum(r[k] for k in ROLLOUT_DONE),
|
||
"rollout_pass": r["PASS"], "rollout_queued": r["QUEUED"], "exported": d["exported"]})
|
||
return out
|
||
|
||
|
||
def model_blockers(row: dict, cache: dict | None = None, evaluation: dict | None = None) -> list[str]:
|
||
"""Check concrete local prerequisites, independently of CSV status labels.
|
||
|
||
This is admission checking, not a claim of successful conversion or accuracy.
|
||
The latter can only come from stages and verdicts.
|
||
"""
|
||
from .eval import TASK_MODULES
|
||
from .registry import datasets
|
||
from .adapters import model_spec, calibration_dir
|
||
row = model_spec(row)
|
||
cache = {} if cache is None else cache
|
||
def once(key, read):
|
||
if key not in cache:
|
||
cache[key] = read()
|
||
return cache[key]
|
||
def exists(p):
|
||
return once(("file", str(p)), p.is_file)
|
||
def read_json(p):
|
||
return once(("json", str(p)), lambda: json.loads(p.read_text()))
|
||
task, mid, source = row["task"], row["model_id"], row.get("source")
|
||
if source == "opencv":
|
||
return ["non_neural_algorithm: no neural weights/PTQ target; retain in inventory"]
|
||
reasons = []
|
||
uri = row.get("weights", "")
|
||
if uri.startswith(("https://github.com/", "https://docs.", "https://huggingface.co/")) and not any(
|
||
part in uri for part in ("/releases/download/", "/resolve/")):
|
||
reasons.append("weights_is_catalog_page: concrete checkpoint/revision missing")
|
||
elif not uri.startswith(("torchvision://", "ultralytics://", "hf://", "file://", "https://", "http://")):
|
||
reasons.append("weights_uri_unsupported_or_missing")
|
||
adapter = paths.BENCH_ROOT / "harness" / "adapters" / (mid.replace(".", "_").replace("-", "_") + ".py")
|
||
adapters = once("adapter_files", lambda: {p.name for p in adapter.parent.glob("*.py")})
|
||
if source not in ("torchvision", "ultralytics", "recipe") and adapter.name not in adapters:
|
||
reasons.append(f"model_builder_export_adapter_missing: {adapter.relative_to(paths.BENCH_ROOT)}")
|
||
if mid.startswith("fastsam."):
|
||
reasons.append("metric_semantics: FastSAM class-agnostic masks cannot use 80-class COCO mask AP")
|
||
if task not in TASK_MODULES:
|
||
reasons.append(f"task_evaluator_missing: {task}")
|
||
if source == "ultralytics" and task in ("iseg", "pose") and not mid.startswith("fastsam."):
|
||
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
|
||
proof = read_json(probe) if exists(probe) else {}
|
||
if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid:
|
||
reasons.append(f"real_checkpoint_export_probe_missing_or_failed: {probe}")
|
||
if task == "pose":
|
||
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
|
||
proof = read_json(probe) if exists(probe) else {}
|
||
if (proof.get("pose_export_shift", {}).get("status") != "measured"
|
||
or proof["pose_export_shift"]["keypoint_mean_shift_px"] > 2):
|
||
reasons.append("auxiliary_gate_probe_missing: actual FP32/ONNX keypoint correspondence and <=2px shift required")
|
||
evidence = proof.get("pose_export_evidence") or {}
|
||
if not evidence.get("sha256") or not exists(Path(evidence.get("path", "__missing__"))):
|
||
reasons.append("auxiliary_gate_raw_evidence_missing: replayable matched keypoint coordinates required")
|
||
if source not in ("torchvision", "ultralytics", "opencv", "recipe") and adapter.name in adapters:
|
||
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
|
||
proof = read_json(probe) if exists(probe) else {}
|
||
if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid:
|
||
reasons.append("custom_adapter_unvalidated: builder present; environment/preprocessing/real forward/export proof still required")
|
||
if row.get("preprocess") and json.dumps(proof.get("preprocess"), sort_keys=True) != json.dumps(row["preprocess"], sort_keys=True):
|
||
reasons.append("preprocessing_recipe_not_validated: probe must use the current model recipe")
|
||
if task in ("stereo", "flow", "feat", "occ"):
|
||
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
|
||
proof = read_json(probe) if exists(probe) else {}
|
||
contract = proof.get("input_contract") or {}
|
||
if (proof.get("status") != "validated_fp32_export_only" or contract.get("status") != "validated"
|
||
or contract.get("input_spec") != row.get("input_spec")):
|
||
reasons.append("input_contract_unverified: real pair/multi-input/camera metadata and export proof required")
|
||
ds = row.get("dataset", "")
|
||
ds_spec = datasets().get("datasets", {}).get(ds, {})
|
||
if task not in ds_spec.get("tasks", []):
|
||
reasons.append(f"dataset_task_mismatch: {ds} is not registered for {task}")
|
||
if evaluation is None:
|
||
probe_path = paths.CACHE / "adapter-probes" / mid / "probe.json"
|
||
proof = read_json(probe_path) if exists(probe_path) else {}
|
||
if proof.get("status") == "validated_fp32_export_only":
|
||
evaluation = proof.get("eval_set")
|
||
manifest = (Path(evaluation["path"]).parent / "manifest.json" if evaluation else
|
||
paths.DATA / "t1" / ds / "manifest.json")
|
||
if not exists(manifest):
|
||
reasons.append(f"t1_missing: {manifest}")
|
||
else:
|
||
m = read_json(manifest)
|
||
if not m.get("sha256") or not exists(manifest.parent / m.get("tar", "__missing__")):
|
||
reasons.append("t1_package_or_identity_missing")
|
||
elif evaluation and (m.get("dataset") != ds or m.get("n") != evaluation.get("n")
|
||
or m.get("sha256") != evaluation.get("manifest_sha")
|
||
or m.get("content_sha256") != evaluation.get("content_sha")):
|
||
reasons.append("t1_explicit_identity_mismatch")
|
||
elif task in ("iseg", "pose") and ds == "coco2017":
|
||
import tarfile
|
||
def annotations():
|
||
with tarfile.open(manifest.parent / m["tar"]) as archive:
|
||
return json.load(archive.extractfile("annotations.json"))
|
||
ann = once(("annotations", ds), annotations)
|
||
if task == "pose" and not ann.get("keypoints", {}).get("annotations"):
|
||
reasons.append("person_keypoint_annotations_missing")
|
||
if task == "iseg" and not any(a.get("segmentation") for a in ann.get("annotations", [])):
|
||
reasons.append("instance_mask_annotations_missing")
|
||
calib = calibration_dir(row)
|
||
if not exists(calib / "manifest.json"):
|
||
reasons.append(f"calibration_missing: {calib}")
|
||
else:
|
||
cm = read_json(calib / "manifest.json")
|
||
count = once(("calibration_count", str(calib)), lambda: len(list((calib / cm.get("npy_dir", "npy_f32")).glob("*.npy"))))
|
||
if not cm.get("sha256") or count < cm.get("count", 128) or not count:
|
||
reasons.append("calibration_samples_or_identity_missing")
|
||
return reasons
|
||
|
||
|
||
def master_inventory() -> dict:
|
||
"""All CSV models, all runs, with evidence joins. Never changes task state."""
|
||
from .verify import completion_summary
|
||
with db.readonly() as con:
|
||
jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE mode='oracle'")]
|
||
rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts")]
|
||
stages = [dict(r) for r in con.execute("SELECT * FROM stages")]
|
||
by_job = {(j["run_id"], j["job_id"]): j for j in jobs}
|
||
exports = {}
|
||
for f in sorted(paths.DATASETS.glob("*/teacher.jsonl")):
|
||
if (f.parent / "VOID").exists():
|
||
continue
|
||
for line in f.open():
|
||
record = json.loads(line)
|
||
if record.get("source") == "agent_events" and record.get("messages"):
|
||
key = (record["run_id"], record["job_id"], record["rollout"])
|
||
exports[key] = {"path": str(f), "id": record["id"], "model": record.get("model"),
|
||
"sft_eligible": record.get("sft_eligible", False)}
|
||
per_model, prerequisite_cache = {}, {}
|
||
for row in models():
|
||
from .adapters import model_spec
|
||
row = model_spec(row)
|
||
mid = row["model_id"]
|
||
per_model[mid] = {"model": mid, "name": row.get("name"), "task": row["task"],
|
||
"priority": row["priority"], "registry_status": row.get("status"),
|
||
"source": row.get("source"), "weights": row.get("weights"),
|
||
"blockers": model_blockers(row, prerequisite_cache), "jobs": [], "rounds": []}
|
||
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
|
||
per_model[mid]["adapter_probe"] = {"path": str(probe), **json.loads(probe.read_text())} if probe.is_file() else None
|
||
for j in jobs:
|
||
d = per_model.get(j["model"])
|
||
if d is None:
|
||
continue
|
||
result = json.loads(j["result"]) if j.get("result") else {}
|
||
directory = paths.RUNS / j["run_id"] / j["job_id"]
|
||
d["jobs"].append({"run_id": j["run_id"], "job_id": j["job_id"], "state": j["state"],
|
||
"error_class": j["error_class"], "stage_failed": j["stage_failed"],
|
||
"path": str(directory / "oracle"),
|
||
"completion": result.get("completion") or completion_summary(result.get("gates", {}), result.get("stages", {})),
|
||
"gates": result.get("gates", {}), "eval_set": result.get("eval_set"),
|
||
"stages": [s for s in stages if s["run_id"] == j["run_id"] and s["job_id"] == j["job_id"] and s["rollout"] == -1]})
|
||
for r in rollouts:
|
||
job = by_job.get((r["run_id"], r["job_id"]))
|
||
mid = job["model"] if job else r["job_id"].split("--")[1]
|
||
if mid not in per_model:
|
||
continue
|
||
rd = paths.RUNS / r["run_id"] / r["job_id"] / "rollouts" / f"{r['kk']:02d}"
|
||
# Hundreds of QUEUED rows have no directory yet. Avoid FUSE lookups for
|
||
# each absent file; pending plans are not execution evidence.
|
||
started = r["state"] != "QUEUED" or r.get("started_at") is not None
|
||
meta = json.loads((rd / "rollout.json").read_text()) if started and (rd / "rollout.json").is_file() else {}
|
||
verdict = json.loads(r["verdict"]) if r.get("verdict") else {}
|
||
raw = {str(p): p.stat().st_size for p in (rd / "agent" / "events.jsonl", rd / "agent" / "session.jsonl") if started and p.is_file()}
|
||
observed_models = set()
|
||
events = rd / "agent" / "events.jsonl"
|
||
if str(events) in raw:
|
||
with events.open() as stream:
|
||
for line in stream:
|
||
try:
|
||
event = json.loads(line)
|
||
except json.JSONDecodeError: # an active writer's last partial line
|
||
continue
|
||
message = event.get("message") or {}
|
||
if (event.get("type") == "message_end" and isinstance(message, dict)
|
||
and message.get("role") == "assistant" and message.get("model") and message.get("content")):
|
||
observed_models.add(message["model"])
|
||
break # first real response identity; raw full stream stays linked
|
||
exported = exports.get((r["run_id"], r["job_id"], r["kk"]))
|
||
per_model[mid]["rounds"].append({"run_id": r["run_id"], "job_id": r["job_id"], "kk": r["kk"],
|
||
"state": r["state"], "mode": r["mode"], "policy": r["policy"], "model": meta.get("model"),
|
||
"raw_traces": raw, "observed_models": sorted(observed_models), "path": str(rd), "export": exported,
|
||
"completion": verdict.get("completion") or completion_summary(verdict.get("gates", {}), verdict.get("stages", {})),
|
||
"gates": verdict.get("gates", {}),
|
||
"eval_set": verdict.get("eval_set"), "error_class": verdict.get("error_class")})
|
||
return {"schema": "bench.coverage/2", "captured_at": time.time(), "models": list(per_model.values())}
|
||
|
||
|
||
def write_master(out: str) -> str:
|
||
"""CSV summary + adjacent JSON containing every run/job/round/evidence link."""
|
||
import os
|
||
import tempfile
|
||
inventory = master_inventory()
|
||
destination = Path(out)
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
fields = ["model", "name", "task", "priority", "registry_status", "source", "weights", "admission",
|
||
"blockers", "adapter_validation", "adapter_evidence", "job_ids", "latest_run", "latest_oracle_state", "latest_stage", "latest_error",
|
||
"oracle_states", "calibration", "quantization", "compile", "hbm_generated", "board", "accuracy", "accuracy_passed",
|
||
"first_glm53_trace", "glm53_exported", "sft_eligible", "round_states", "runtime_errors", "evidence_json"]
|
||
fd, tmp = tempfile.mkstemp(dir=destination.parent, prefix=destination.name + ".")
|
||
try:
|
||
with os.fdopen(fd, "w") as stream:
|
||
writer = csvmod.DictWriter(stream, fieldnames=fields)
|
||
writer.writeheader()
|
||
for d in inventory["models"]:
|
||
all_results = d["jobs"] + d["rounds"]
|
||
def done(key):
|
||
return sum(bool(x.get("completion", {}).get(key)) for x in all_results)
|
||
glm = [r for r in d["rounds"] if r["model"] == "glm-5.3" and "glm-5.3" in r["observed_models"]]
|
||
exported = [r for r in glm if r["export"] and r["export"]["model"] == "glm-5.3"]
|
||
latest = max(d["jobs"], key=lambda j: j["run_id"], default={})
|
||
latest_stage = max(latest.get("stages", []), key=lambda s: s.get("started_at") or 0, default={})
|
||
writer.writerow({**{k: d.get(k) for k in fields if k in d and k != "blockers"},
|
||
"admission": "blocked" if d["blockers"] else "prerequisites_present",
|
||
"blockers": " | ".join(d["blockers"]),
|
||
"adapter_validation": (d.get("adapter_probe") or {}).get("status", ""),
|
||
"adapter_evidence": (d.get("adapter_probe") or {}).get("path", ""),
|
||
"job_ids": " | ".join(j["run_id"] + "/" + j["job_id"] for j in d["jobs"]),
|
||
"latest_run": latest.get("run_id", ""), "latest_oracle_state": latest.get("state", "not_queued"),
|
||
"latest_stage": latest_stage.get("stage", ""), "latest_error": latest.get("error_class", ""),
|
||
"oracle_states": json.dumps(dict(Counter(j["state"] for j in d["jobs"]))),
|
||
"calibration": sum(any(s["stage"] == "04-calibrate" and s["state"] in ("OK", "SKIPPED") for s in j["stages"]) for j in d["jobs"]),
|
||
"quantization": done("compiled"), "compile": done("compiled"), "hbm_generated": done("hbm_generated"),
|
||
"board": done("board_completed"), "accuracy": done("accuracy_evaluated"), "accuracy_passed": done("accuracy_passed"),
|
||
"first_glm53_trace": glm[0]["path"] if glm else "", "glm53_exported": len(exported),
|
||
"sft_eligible": sum(r["export"]["sft_eligible"] for r in exported),
|
||
"round_states": json.dumps(dict(Counter(r["state"] for r in d["rounds"]))),
|
||
"runtime_errors": " | ".join(sorted({str(x.get("stage_failed") or "verdict") + ":" + x["error_class"] for x in all_results if x.get("error_class")})),
|
||
"evidence_json": str(destination.with_suffix(".json"))})
|
||
os.replace(tmp, destination)
|
||
fdj, tmpj = tempfile.mkstemp(dir=destination.parent, prefix=destination.stem + ".json.")
|
||
with os.fdopen(fdj, "w") as stream:
|
||
json.dump(inventory, stream, ensure_ascii=False, indent=2)
|
||
os.replace(tmpj, destination.with_suffix(".json"))
|
||
finally:
|
||
if os.path.exists(tmp):
|
||
os.unlink(tmp)
|
||
return str(destination)
|