"""bench coverage — 338 个模型里,哪些做了、哪些没做、哪些在跑。 bench coverage 总览 + 按任务分组 bench coverage --missing 只列"还没做"的 bench coverage --task cls --detail 逐模型一行 bench coverage --csv cov.csv 导出给别人看 三个层次,别混: registry models.csv 里 338 行,status=ready 的才会被 matrix 展开(needs_adapter/needs_eval/skip 是"还没接进来") oracle 每个 (model × march × backend × qcfg) 一个 job:PENDING 没开始 / ORACLE_RUNNING 在跑 / PASS·FAIL·UNVERIFIED 已结束 rollout 每个 job 计划 k 条轨迹:QUEUED 没采 / RUNNING 在采 / PASS·FAIL·BLOCKED·TIMEOUT 已采 "是不是还在跑"看顶部那行 LIVE:有没有调度进程、池占了几个槽、最近一次阶段心跳多久以前。 """ from __future__ import annotations import csv as csvmod import json import time from collections import Counter, defaultdict from pathlib import Path from . import paths, db from .registry import models, expand_matrix from .scheduler import pool_usage, pool_size DONE = ("PASS", "FAIL", "UNVERIFIED") ROLLOUT_DONE = ("PASS", "FAIL", "BLOCKED", "TIMEOUT", "UNVERIFIED") def _fmt_age(s: float | None) -> str: if s is None: return "从未" s = int(s) if s < 90: return f"{s}s 前" if s < 5400: return f"{s // 60}m 前" return f"{s // 3600}h{(s % 3600) // 60:02d}m 前" def live_line() -> list[str]: """是不是还在跑:调度进程 + 池占用 + 最近心跳。""" from .pipeline import scheduler_alive out = [] alive = scheduler_alive() pools = pool_usage() busy = sum(pools.values()) with db.connect() as con: row = con.execute("SELECT MAX(COALESCE(finished_at, started_at)) FROM stages").fetchone() last = row[0] if row and row[0] else None n_run = con.execute("SELECT COUNT(*) FROM stages WHERE state='RUNNING'").fetchone()[0] age = (time.time() - last) if last else None if alive or busy or n_run: state = "🟢 在跑" elif age is not None and age < 300: state = "🟡 刚停(进程没了但 5 分钟内还有活动)" else: state = "🔴 没在跑" out.append(f"LIVE {state} 调度进程={'有' if alive else '无'} " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pools.items()) + f" RUNNING 阶段={n_run} 最近心跳={_fmt_age(age)}") if not alive and not busy: out.append(" 没有调度进程:要继续跑就 scripts/start_run.sh --resume(先 reset-stale 再接着跑 PENDING)") return out def exported_keys() -> set[tuple]: """已经进过 datasets/ 的 (run_id, job_id, kk)。""" import json keys = set() files = sorted(paths.DATASETS.glob("*/teacher.jsonl")) + sorted(paths.DATASETS.glob("*/student.jsonl")) for f in files: if (f.parent / "VOID").exists(): continue try: with f.open(errors="replace") as stream: for line in stream: r = json.loads(line) if (r.get("source") in (None, "agent_events") and isinstance(r.get("messages"), list) and isinstance(r.get("rollout"), int)): keys.add((r.get("run_id"), r.get("job_id"), r["rollout"])) except (OSError, json.JSONDecodeError): continue return keys def collect(run_id: str | None) -> dict: """汇总指定 run 的实际计划;run_id=None 时只展示 registry 的预期。""" rows = models() by_model = {r["model_id"]: r for r in rows} jobs, ro = [], [] if run_id is not None: with db.connect() as con: jobs = db.list_jobs(con, run_id, mode="oracle") ro = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=?", (run_id,))] planned_rollouts = Counter(r["job_id"] for r in ro) exported = exported_keys() if run_id is not None else set() per_model: dict[str, dict] = {} for m, r in by_model.items(): per_model[m] = {"model": m, "task": r["task"], "priority": r["priority"], "status": r.get("status") or "ready", "jobs": [], "oracle": Counter(), "rollout": Counter(), "planned_rollouts": 0, "exported": 0} if run_id is None: for spec in expand_matrix(): if spec.mode != "oracle" and spec.model in per_model: per_model[spec.model]["planned_rollouts"] += spec.k for j in jobs: d = per_model.get(j["model"]) if d is None: continue d["jobs"].append(j) d["oracle"][j["state"]] += 1 d["planned_rollouts"] += planned_rollouts.get(j["job_id"], 0) for r in ro: jrow = next((x for x in jobs if x["job_id"] == r["job_id"]), None) if jrow: model = jrow["model"] else: jp = paths.RUNS / run_id / r["job_id"] / "job.json" model = json.loads(jp.read_text()).get("model") if jp.exists() else None d = per_model.get(model) if d is None: continue if not jrow: d["planned_rollouts"] += 1 d["rollout"][r["state"]] += 1 if (run_id, r["job_id"], r["kk"]) in exported: d["exported"] += 1 return per_model def render(run_id: str, task: str | None = None, missing: bool = False, detail: bool = False) -> str: per_model = collect(run_id) lines = [f"bench coverage · run {run_id} · {time.strftime('%Y-%m-%d %H:%M:%S')}"] lines += live_line() lines.append("") # ---- registry 层 st = Counter(d["status"] for d in per_model.values()) ready = [d for d in per_model.values() if d["status"] == "ready"] lines.append(f"registry: 共 {len(per_model)} 个模型 · " + " · ".join(f"{k}={v}" for k, v in st.most_common()) + " (只有 ready 会被展开成 job;其余是「还没接进来」,不是「失败」)") # ---- oracle 层 oc = Counter() for d in ready: oc.update(d["oracle"]) total_jobs = sum(oc.values()) done_jobs = sum(oc[k] for k in DONE) lines.append(f"oracle: {total_jobs} 个 job · 已结束 {done_jobs}" + (f"(PASS {oc['PASS']} / FAIL {oc['FAIL']} / UNVERIFIED {oc['UNVERIFIED']})" if done_jobs else "") + f" · 在跑 {oc['ORACLE_RUNNING']} · 没开始 {oc['PENDING']}") # ---- rollout 层 rc = Counter() planned = exported = 0 for d in ready: rc.update(d["rollout"]); planned += d["planned_rollouts"]; exported += d["exported"] got = sum(rc[k] for k in ROLLOUT_DONE) lines.append(f"rollout: 计划 {planned} 条 · 已采 {got}" + (f"(PASS {rc['PASS']} / FAIL {rc['FAIL']} / BLOCKED {rc['BLOCKED']} / TIMEOUT {rc['TIMEOUT']} / UNVERIFIED {rc['UNVERIFIED']})" if got else "") + f" · 在采 {rc['RUNNING'] + rc['VERIFYING']}(RUNNING {rc['RUNNING']} / VERIFYING {rc['VERIFYING']})" + f" · 没采 {rc['QUEUED']} · 已导出 {exported}") lines.append("") # ---- 按任务分组 groups: dict[str, list[dict]] = defaultdict(list) for d in per_model.values(): groups[d["task"]].append(d) lines.append(f" {'task':<7} {'模型':>5} {'ready':>6} | {'job':>5} {'已结束':>6} {'PASS':>5} {'FAIL':>5} {'在跑':>4} {'没开始':>6} | " f"{'轨迹计划':>8} {'已采':>5} {'PASS':>5} {'没采':>5}") for t in sorted(groups): g = groups[t] gr = [d for d in g if d["status"] == "ready"] o, r = Counter(), Counter() pl = 0 for d in gr: o.update(d["oracle"]); r.update(d["rollout"]); pl += d["planned_rollouts"] lines.append(f" {t:<7} {len(g):>5} {len(gr):>6} | {sum(o.values()):>5} {sum(o[k] for k in DONE):>6} " f"{o['PASS']:>5} {o['FAIL']:>5} {o['ORACLE_RUNNING']:>4} {o['PENDING']:>6} | " f"{pl:>8} {sum(r[k] for k in ROLLOUT_DONE):>5} {r['PASS']:>5} {r['QUEUED']:>5}") # ---- 逐模型 sel = [d for d in per_model.values() if (not task or d["task"] == task)] if missing: sel = [d for d in sel if d["status"] != "ready" or d["oracle"].get("PENDING") or not d["jobs"] or d["rollout"].get("QUEUED")] lines.append("") lines.append(f" 还没做的({len(sel)}):status≠ready,或 oracle 有 PENDING,或轨迹有 QUEUED") if missing or detail: lines.append("") lines.append(f" {'model':<26} {'task':<6} {'pri':<4} {'status':<14} {'oracle':<34} {'rollout':<26}") for d in sorted(sel, key=lambda x: (x["task"], x["priority"], x["model"])): o = " ".join(f"{k}={v}" for k, v in sorted(d["oracle"].items())) or ("—(未展开)" if d["status"] != "ready" else "—") r = " ".join(f"{k}={v}" for k, v in sorted(d["rollout"].items())) or "—" if d["exported"]: r += f" 导出={d['exported']}" lines.append(f" {d['model']:<26} {d['task']:<6} {d['priority']:<4} {d['status']:<14} {o:<34} {r:<26}") else: lines.append("") lines.append(" 逐模型明细:--detail(可加 --task cls);只看没做的:--missing;导出表格:--csv cov.csv") return "\n".join(lines) def to_csv(run_id: str, out: str) -> str: per_model = collect(run_id) fields = ["model", "task", "priority", "status", "jobs", "oracle_done", "oracle_pass", "oracle_fail", "oracle_unverified", "oracle_running", "oracle_pending", "rollout_planned", "rollout_done", "rollout_pass", "rollout_queued", "exported"] with open(out, "w", newline="", encoding="utf-8") as f: w = csvmod.DictWriter(f, fieldnames=fields) w.writeheader() for d in sorted(per_model.values(), key=lambda x: (x["task"], x["priority"], x["model"])): o, r = d["oracle"], d["rollout"] w.writerow({"model": d["model"], "task": d["task"], "priority": d["priority"], "status": d["status"], "jobs": len(d["jobs"]), "oracle_done": sum(o[k] for k in DONE), "oracle_pass": o["PASS"], "oracle_fail": o["FAIL"], "oracle_unverified": o["UNVERIFIED"], "oracle_running": o["ORACLE_RUNNING"], "oracle_pending": o["PENDING"], "rollout_planned": d["planned_rollouts"], "rollout_done": sum(r[k] for k in ROLLOUT_DONE), "rollout_pass": r["PASS"], "rollout_queued": r["QUEUED"], "exported": d["exported"]}) return out def model_blockers(row: dict, cache: dict | None = None, evaluation: dict | None = None) -> list[str]: """Check concrete local prerequisites, independently of CSV status labels. This is admission checking, not a claim of successful conversion or accuracy. The latter can only come from stages and verdicts. """ from .eval import TASK_MODULES from .registry import datasets from .adapters import model_spec, calibration_dir row = model_spec(row) cache = {} if cache is None else cache def once(key, read): if key not in cache: cache[key] = read() return cache[key] def exists(p): return once(("file", str(p)), p.is_file) def read_json(p): return once(("json", str(p)), lambda: json.loads(p.read_text())) task, mid, source = row["task"], row["model_id"], row.get("source") if source == "opencv": return ["non_neural_algorithm: no neural weights/PTQ target; retain in inventory"] reasons = [] uri = row.get("weights", "") if uri.startswith(("https://github.com/", "https://docs.", "https://huggingface.co/")) and not any( part in uri for part in ("/releases/download/", "/resolve/")): reasons.append("weights_is_catalog_page: concrete checkpoint/revision missing") elif not uri.startswith(("torchvision://", "ultralytics://", "hf://", "file://", "https://", "http://")): reasons.append("weights_uri_unsupported_or_missing") adapter = paths.BENCH_ROOT / "harness" / "adapters" / (mid.replace(".", "_").replace("-", "_") + ".py") adapters = once("adapter_files", lambda: {p.name for p in adapter.parent.glob("*.py")}) if source not in ("torchvision", "ultralytics", "recipe") and adapter.name not in adapters: reasons.append(f"model_builder_export_adapter_missing: {adapter.relative_to(paths.BENCH_ROOT)}") if mid.startswith("fastsam."): reasons.append("metric_semantics: FastSAM class-agnostic masks cannot use 80-class COCO mask AP") if task not in TASK_MODULES: reasons.append(f"task_evaluator_missing: {task}") if source == "ultralytics" and task in ("iseg", "pose") and not mid.startswith("fastsam."): probe = paths.CACHE / "adapter-probes" / mid / "probe.json" proof = read_json(probe) if exists(probe) else {} if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid: reasons.append(f"real_checkpoint_export_probe_missing_or_failed: {probe}") if task == "pose": probe = paths.CACHE / "adapter-probes" / mid / "probe.json" proof = read_json(probe) if exists(probe) else {} if (proof.get("pose_export_shift", {}).get("status") != "measured" or proof["pose_export_shift"]["keypoint_mean_shift_px"] > 2): reasons.append("auxiliary_gate_probe_missing: actual FP32/ONNX keypoint correspondence and <=2px shift required") evidence = proof.get("pose_export_evidence") or {} if not evidence.get("sha256") or not exists(Path(evidence.get("path", "__missing__"))): reasons.append("auxiliary_gate_raw_evidence_missing: replayable matched keypoint coordinates required") if source not in ("torchvision", "ultralytics", "opencv", "recipe") and adapter.name in adapters: probe = paths.CACHE / "adapter-probes" / mid / "probe.json" proof = read_json(probe) if exists(probe) else {} if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid: reasons.append("custom_adapter_unvalidated: builder present; environment/preprocessing/real forward/export proof still required") if row.get("preprocess") and json.dumps(proof.get("preprocess"), sort_keys=True) != json.dumps(row["preprocess"], sort_keys=True): reasons.append("preprocessing_recipe_not_validated: probe must use the current model recipe") if task in ("stereo", "flow", "feat", "occ"): probe = paths.CACHE / "adapter-probes" / mid / "probe.json" proof = read_json(probe) if exists(probe) else {} contract = proof.get("input_contract") or {} if (proof.get("status") != "validated_fp32_export_only" or contract.get("status") != "validated" or contract.get("input_spec") != row.get("input_spec")): reasons.append("input_contract_unverified: real pair/multi-input/camera metadata and export proof required") ds = row.get("dataset", "") ds_spec = datasets().get("datasets", {}).get(ds, {}) if task not in ds_spec.get("tasks", []): reasons.append(f"dataset_task_mismatch: {ds} is not registered for {task}") if evaluation is None: probe_path = paths.CACHE / "adapter-probes" / mid / "probe.json" proof = read_json(probe_path) if exists(probe_path) else {} if proof.get("status") == "validated_fp32_export_only": evaluation = proof.get("eval_set") manifest = (Path(evaluation["path"]).parent / "manifest.json" if evaluation else paths.DATA / "t1" / ds / "manifest.json") if not exists(manifest): reasons.append(f"t1_missing: {manifest}") else: m = read_json(manifest) if not m.get("sha256") or not exists(manifest.parent / m.get("tar", "__missing__")): reasons.append("t1_package_or_identity_missing") elif evaluation and (m.get("dataset") != ds or m.get("n") != evaluation.get("n") or m.get("sha256") != evaluation.get("manifest_sha") or m.get("content_sha256") != evaluation.get("content_sha")): reasons.append("t1_explicit_identity_mismatch") elif task in ("iseg", "pose") and ds == "coco2017": import tarfile def annotations(): with tarfile.open(manifest.parent / m["tar"]) as archive: return json.load(archive.extractfile("annotations.json")) ann = once(("annotations", ds), annotations) if task == "pose" and not ann.get("keypoints", {}).get("annotations"): reasons.append("person_keypoint_annotations_missing") if task == "iseg" and not any(a.get("segmentation") for a in ann.get("annotations", [])): reasons.append("instance_mask_annotations_missing") calib = calibration_dir(row) if not exists(calib / "manifest.json"): reasons.append(f"calibration_missing: {calib}") else: cm = read_json(calib / "manifest.json") count = once(("calibration_count", str(calib)), lambda: len(list((calib / cm.get("npy_dir", "npy_f32")).glob("*.npy")))) if not cm.get("sha256") or count < cm.get("count", 128) or not count: reasons.append("calibration_samples_or_identity_missing") return reasons def master_inventory() -> dict: """All CSV models, all runs, with evidence joins. Never changes task state.""" from .verify import completion_summary with db.readonly() as con: jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE mode='oracle'")] rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts")] stages = [dict(r) for r in con.execute("SELECT * FROM stages")] by_job = {(j["run_id"], j["job_id"]): j for j in jobs} exports = {} for f in sorted(paths.DATASETS.glob("*/teacher.jsonl")): if (f.parent / "VOID").exists(): continue for line in f.open(): record = json.loads(line) if record.get("source") == "agent_events" and record.get("messages"): key = (record["run_id"], record["job_id"], record["rollout"]) exports[key] = {"path": str(f), "id": record["id"], "model": record.get("model"), "sft_eligible": record.get("sft_eligible", False)} per_model, prerequisite_cache = {}, {} for row in models(): from .adapters import model_spec row = model_spec(row) mid = row["model_id"] per_model[mid] = {"model": mid, "name": row.get("name"), "task": row["task"], "priority": row["priority"], "registry_status": row.get("status"), "source": row.get("source"), "weights": row.get("weights"), "blockers": model_blockers(row, prerequisite_cache), "jobs": [], "rounds": []} probe = paths.CACHE / "adapter-probes" / mid / "probe.json" per_model[mid]["adapter_probe"] = {"path": str(probe), **json.loads(probe.read_text())} if probe.is_file() else None for j in jobs: d = per_model.get(j["model"]) if d is None: continue result = json.loads(j["result"]) if j.get("result") else {} directory = paths.RUNS / j["run_id"] / j["job_id"] d["jobs"].append({"run_id": j["run_id"], "job_id": j["job_id"], "state": j["state"], "error_class": j["error_class"], "stage_failed": j["stage_failed"], "path": str(directory / "oracle"), "completion": result.get("completion") or completion_summary(result.get("gates", {}), result.get("stages", {})), "gates": result.get("gates", {}), "eval_set": result.get("eval_set"), "stages": [s for s in stages if s["run_id"] == j["run_id"] and s["job_id"] == j["job_id"] and s["rollout"] == -1]}) for r in rollouts: job = by_job.get((r["run_id"], r["job_id"])) mid = job["model"] if job else r["job_id"].split("--")[1] if mid not in per_model: continue rd = paths.RUNS / r["run_id"] / r["job_id"] / "rollouts" / f"{r['kk']:02d}" # Hundreds of QUEUED rows have no directory yet. Avoid FUSE lookups for # each absent file; pending plans are not execution evidence. started = r["state"] != "QUEUED" or r.get("started_at") is not None meta = json.loads((rd / "rollout.json").read_text()) if started and (rd / "rollout.json").is_file() else {} verdict = json.loads(r["verdict"]) if r.get("verdict") else {} raw = {str(p): p.stat().st_size for p in (rd / "agent" / "events.jsonl", rd / "agent" / "session.jsonl") if started and p.is_file()} observed_models = set() events = rd / "agent" / "events.jsonl" if str(events) in raw: with events.open() as stream: for line in stream: try: event = json.loads(line) except json.JSONDecodeError: # an active writer's last partial line continue message = event.get("message") or {} if (event.get("type") == "message_end" and isinstance(message, dict) and message.get("role") == "assistant" and message.get("model") and message.get("content")): observed_models.add(message["model"]) break # first real response identity; raw full stream stays linked exported = exports.get((r["run_id"], r["job_id"], r["kk"])) per_model[mid]["rounds"].append({"run_id": r["run_id"], "job_id": r["job_id"], "kk": r["kk"], "state": r["state"], "mode": r["mode"], "policy": r["policy"], "model": meta.get("model"), "raw_traces": raw, "observed_models": sorted(observed_models), "path": str(rd), "export": exported, "completion": verdict.get("completion") or completion_summary(verdict.get("gates", {}), verdict.get("stages", {})), "gates": verdict.get("gates", {}), "eval_set": verdict.get("eval_set"), "error_class": verdict.get("error_class")}) return {"schema": "bench.coverage/2", "captured_at": time.time(), "models": list(per_model.values())} def write_master(out: str) -> str: """CSV summary + adjacent JSON containing every run/job/round/evidence link.""" import os import tempfile inventory = master_inventory() destination = Path(out) destination.parent.mkdir(parents=True, exist_ok=True) fields = ["model", "name", "task", "priority", "registry_status", "source", "weights", "admission", "blockers", "adapter_validation", "adapter_evidence", "job_ids", "latest_run", "latest_oracle_state", "latest_stage", "latest_error", "oracle_states", "calibration", "quantization", "compile", "hbm_generated", "board", "accuracy", "accuracy_passed", "first_glm53_trace", "glm53_exported", "sft_eligible", "round_states", "runtime_errors", "evidence_json"] fd, tmp = tempfile.mkstemp(dir=destination.parent, prefix=destination.name + ".") try: with os.fdopen(fd, "w") as stream: writer = csvmod.DictWriter(stream, fieldnames=fields) writer.writeheader() for d in inventory["models"]: all_results = d["jobs"] + d["rounds"] def done(key): return sum(bool(x.get("completion", {}).get(key)) for x in all_results) glm = [r for r in d["rounds"] if r["model"] == "glm-5.3" and "glm-5.3" in r["observed_models"]] exported = [r for r in glm if r["export"] and r["export"]["model"] == "glm-5.3"] latest = max(d["jobs"], key=lambda j: j["run_id"], default={}) latest_stage = max(latest.get("stages", []), key=lambda s: s.get("started_at") or 0, default={}) writer.writerow({**{k: d.get(k) for k in fields if k in d and k != "blockers"}, "admission": "blocked" if d["blockers"] else "prerequisites_present", "blockers": " | ".join(d["blockers"]), "adapter_validation": (d.get("adapter_probe") or {}).get("status", ""), "adapter_evidence": (d.get("adapter_probe") or {}).get("path", ""), "job_ids": " | ".join(j["run_id"] + "/" + j["job_id"] for j in d["jobs"]), "latest_run": latest.get("run_id", ""), "latest_oracle_state": latest.get("state", "not_queued"), "latest_stage": latest_stage.get("stage", ""), "latest_error": latest.get("error_class", ""), "oracle_states": json.dumps(dict(Counter(j["state"] for j in d["jobs"]))), "calibration": sum(any(s["stage"] == "04-calibrate" and s["state"] in ("OK", "SKIPPED") for s in j["stages"]) for j in d["jobs"]), "quantization": done("compiled"), "compile": done("compiled"), "hbm_generated": done("hbm_generated"), "board": done("board_completed"), "accuracy": done("accuracy_evaluated"), "accuracy_passed": done("accuracy_passed"), "first_glm53_trace": glm[0]["path"] if glm else "", "glm53_exported": len(exported), "sft_eligible": sum(r["export"]["sft_eligible"] for r in exported), "round_states": json.dumps(dict(Counter(r["state"] for r in d["rounds"]))), "runtime_errors": " | ".join(sorted({str(x.get("stage_failed") or "verdict") + ":" + x["error_class"] for x in all_results if x.get("error_class")})), "evidence_json": str(destination.with_suffix(".json"))}) os.replace(tmp, destination) fdj, tmpj = tempfile.mkstemp(dir=destination.parent, prefix=destination.stem + ".json.") with os.fdopen(fdj, "w") as stream: json.dump(inventory, stream, ensure_ascii=False, indent=2) os.replace(tmpj, destination.with_suffix(".json")) finally: if os.path.exists(tmp): os.unlink(tmp) return str(destination)