amos 6166b7624f Add the read-only live dashboard for the task catalog and rollout progress
rollout.dashboard serves catalog counts, attempts and complete trajectories, and the public
prompt and template of each task. catalog.controllers()/complete() are split out so the
dashboard can reuse receipts it has already read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:45:28 +08:00

249 lines
14 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Build the usable-task catalog from task packages, calibrations, holds and verified controller receipts."""
import argparse
import collections
import hashlib
import json
import os
from pathlib import Path
import sys
import time
from . import core
QWEN = "qwen3.8-27b-nvfp4"
def _language(names):
suffixes = {Path(name).suffix for name in names}
if suffixes & {".cpp", ".hpp", ".cc"}:
return "C++"
if suffixes & {".c", ".h"}:
return "C"
return "Python" if ".py" in suffixes else ",".join(sorted(suffixes))
def calibrations(roots, image_id):
"""Map task digest -> newest PASS Docker calibration on this image."""
found = {}
for root in roots:
for path in sorted(core._path(root).glob("calibration-*/receipt.json")):
raw = path.read_bytes()
receipt = json.loads(raw)
if (receipt.get("status") == "PASS" and receipt.get("cleanup") == "confirmed" and
receipt.get("trusted_local") is False and receipt.get("image_id") == image_id):
found[receipt["task_digest"]] = dict(path=str(path.parent), trials=len(receipt.get("trials", [])),
receipt_sha256=hashlib.sha256(raw).hexdigest())
return found
def controllers(roots):
"""Yield (controller directory, receipt or None while it has none) once per controller name."""
seen = set()
for root in roots:
for folder, dirs, files in os.walk(root):
name = os.path.basename(folder)
if name.startswith("pi-"):
dirs[:] = []
# A batch copied to cold storage before its hot copy is removed holds the same attempt twice.
if name in seen:
continue
try:
receipt = json.loads(Path(folder, "receipt.json").read_bytes())
except (OSError, ValueError):
yield Path(folder), None # Still running, or the controller died before writing.
continue
seen.add(name)
yield Path(folder), receipt
elif name in ("workspace", "candidate", "trace", "grader") or name.startswith(("calibration-", "regrade-")):
dirs[:] = []
def complete(receipt):
"""A real task outcome with VERIFIED native evidence, a clean Agent exit and confirmed cleanup."""
return (receipt.get("status") in ("PASS", "FAIL") and receipt.get("cleanup") == "confirmed" and
(receipt.get("pi_evidence") or {}).get("status") == "VERIFIED" and receipt.get("agent_exit") == 0)
def outcomes(found):
"""Count complete trajectories per task digest and model from (controller, receipt) pairs."""
counts = collections.defaultdict(lambda: collections.defaultdict(collections.Counter))
for _, receipt in found:
if receipt and complete(receipt):
counts[receipt.get("task_digest")][receipt.get("model_id", "glm-5.3")][receipt["status"]] += 1
return counts
def difficulty(qwen):
passed, failed = qwen.get("PASS", 0), qwen.get("FAIL", 0)
total = passed + failed
if total == 0:
return "UNMEASURED"
if failed == 0 and total >= 2:
return "ROUTINE"
if total >= 3:
return "HARD" if 3 * passed <= total else "MEDIUM"
return "PARTIAL"
def build(tasks_root, calibration_roots, runs_roots, holds_path, image_id, reviews_path=None, found=None,
calibrated=None):
"""`found` takes (controller, receipt) pairs already read from `runs_roots`; `calibrated` adds calibrations
already read from slow roots."""
holds = json.loads(core._read(holds_path.parent, holds_path.name)) if holds_path else {}
reviews = json.loads(core._read(reviews_path.parent, reviews_path.name)) if reviews_path else {}
calibrated = {**(calibrated or {}), **calibrations(calibration_roots, image_id)}
measured = outcomes(controllers(runs_roots) if found is None else found)
rows = []
for root in sorted(core._path(tasks_root).iterdir()):
if not root.joinpath("task.json").is_file():
continue
task = core.load_task(root)
results = {model: dict(PASS=c.get("PASS", 0), FAIL=c.get("FAIL", 0))
for model, c in sorted(measured.get(task["_digest"], {}).items())}
row = dict(id=task["id"], version=task["version"], family=task["family"], source=task["source"],
language=_language(task["candidate_files"]), candidate_files=task["candidate_files"],
fixtures=len(task["fixtures"]), task_digest=task["_digest"], results=results,
difficulty=difficulty(results.get(QWEN, {})), difficulty_source="measured",
calibration=calibrated.get(task["_digest"]))
review = reviews.get(task["id"])
# A review judged one exact task identity; it lapses when the task changes.
if review and review.get("task_digest") == task["_digest"]:
row.update(difficulty=review["difficulty"], difficulty_source="reviewed", review=review)
elif review:
row["stale_review"] = review
if root.joinpath("isolation-blocked.txt").exists():
row.update(status="BLOCKED", reason="判据会在候选进程内运行隐藏测试,需按 RUNNER 方式重写后重新准入")
elif task["id"] in holds:
row.update(status="HOLD", reason=holds[task["id"]]["reason"], hold=holds[task["id"]])
elif row["calibration"] is None:
row.update(status="UNCALIBRATED", reason="没有同摘要、同镜像的 PASS 正式校准")
else:
row.update(status="USABLE", reason=None)
rows.append(row)
usable = [row for row in rows if row["status"] == "USABLE"]
return dict(schema="embedded-rollout.task-catalog/1", generated_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
image_id=image_id, qwen_model=QWEN, total=len(rows),
status=dict(collections.Counter(row["status"] for row in rows)),
usable_difficulty=dict(collections.Counter(row["difficulty"] for row in usable)),
usable_families=dict(collections.Counter(row["family"] for row in usable)),
usable_languages=dict(collections.Counter(row["language"] for row in usable)),
tasks=rows)
USAGE = """## 使用方法
在 36 的项目根目录执行。`IMAGE` 用上面的镜像 digest,`PRIVATE_CREDENTIAL_FILE` 是 host 私有凭据文件路径。
```sh
# 单题:可直接复用题库里的校准目录,省略 --calibration 则先自动完整校准
python3 -m rollout.pi_run tasks/TASK_ID --image IMAGE --model qwen3.8-27b-nvfp4 --credential-file PRIVATE_CREDENTIAL_FILE --calibration CALIBRATION --output runs/BATCH
# 批量:按状态/难度筛出“任务 校准目录”,交给 batch 并发;基础设施失败自动补采,中断后重跑同一命令即续跑
python3 -m rollout.catalog jobs docs/task-catalog.json --status USABLE --difficulty HARD MEDIUM > JOBS
python3 -m rollout.batch JOBS --image IMAGE --model qwen3.8-27b-nvfp4 --credential-file PRIVATE_CREDENTIAL_FILE --output runs/BATCH --parallel 2
# 刷新题库:新增或修改题目、换镜像、补采后重新生成
python3 -m rollout.catalog build --calibrations runs/CALIBRATION_DIR --runs runs /mnt/data/ming01.wang/embedded-rollout/runs --holds docs/task-holds.json --reviews docs/task-reviews.json --image-id IMAGE_ID --output NEW.json --markdown NEW.md
```
状态:USABLE 可直接使用;HOLD 有已登记的题面或来源问题;BLOCKED 判据不能隔离候选代码;UNCALIBRATED 当前镜像下没有 PASS 校准。带“(评审)”的难度来自 docs/task-reviews.json 中针对同一 task digest 的人工复核(例如 Agent 自测死锁超时被复核为能力失败),否则只统计与当前 task digest 一致的 Qwen 完整轨迹:0 条为 UNMEASURED,至少 2 条全 PASS 为 ROUTINE,至少 3 条且通过率不超过 1/3 为 HARD,其余 3 条以上为 MEDIUM,样本不足为 PARTIAL。题目改版(digest 变化)后旧结果不计入。
"""
RELEASE = dict(BLOCKED="按 RUNNER 方式重写判据(候选只在子进程里运行、只拿输入;参考 progress.md 2026-10-15 重写试点),升版本后重新校准",
UNCALIBRATED="用当前镜像重新正式校准,再重新生成题库")
def _cell(text):
return str(text).replace("|", "\\|").replace("\n", " ")
def markdown(catalog):
lines = ["# 题库", "",
"生成于 {},由 `python3 -m rollout.catalog build` 从任务包、校准回执和已核对的 controller 回执生成;"
"请勿手改,改动后重新生成。正式镜像 `{}`。".format(catalog["generated_at"], catalog["image_id"]), "",
"共 {} 道:{}。可用题难度:{};任务族:{};语言:{}。".format(
catalog["total"], "、".join("{} {}".format(k, v) for k, v in sorted(catalog["status"].items())),
"、".join("{} {}".format(k, v) for k, v in sorted(catalog["usable_difficulty"].items())),
"、".join("{} {}".format(k, v) for k, v in sorted(catalog["usable_families"].items())),
"、".join("{} {}".format(k, v) for k, v in sorted(catalog["usable_languages"].items()))), "", USAGE]
order = dict(HARD=0, MEDIUM=1, PARTIAL=2, UNMEASURED=3, ROUTINE=4)
usable = sorted((row for row in catalog["tasks"] if row["status"] == "USABLE"),
key=lambda row: (order[row["difficulty"]], row["family"], row["id"]))
lines += ["## 可用题({} 道)".format(len(usable)), "",
"| 任务 | 难度 | Qwen P/F | GLM P/F | 族 | 语言 | 样例类 | 来源 |", "| --- | --- | --- | --- | --- | --- | --- | --- |"]
for row in usable:
q, g = row["results"].get(QWEN, {}), row["results"].get("glm-5.3", {})
lines.append("| {} | {} | {} | {} | {} | {} | {} | {} |".format(
row["id"], row["difficulty"] + ("(评审)" if row["difficulty_source"] == "reviewed" else ""), "{}/{}".format(q.get("PASS", 0), q.get("FAIL", 0)),
"{}/{}".format(g.get("PASS", 0), g.get("FAIL", 0)), row["family"], row["language"], row["fixtures"],
_cell(row["source"])))
reviewed = [row for row in usable if row["difficulty_source"] == "reviewed" and row["difficulty"] == "HARD"]
if reviewed:
lines += ["", "## 难题的评审依据", "", "| 任务 | 评审结果 | 依据 |", "| --- | --- | --- |"]
for row in reviewed:
lines.append("| {} | {} | {} |".format(row["id"], _cell(row["review"].get("results", "-")),
_cell(row["review"].get("basis", "-"))))
for status, title in (("HOLD", "暂缓使用"), ("UNCALIBRATED", "未校准"), ("BLOCKED", "判据待重写")):
rows = [row for row in catalog["tasks"] if row["status"] == status]
if not rows:
continue
lines += ["", "## {}:{}({} 道)".format(status, title, len(rows)), "", "| 任务 | 族 | 原因 | 解除条件 |",
"| --- | --- | --- | --- |"]
for row in sorted(rows, key=lambda row: row["id"]):
lines.append("| {} | {} | {} | {} |".format(row["id"], row["family"], _cell(row["reason"]),
_cell((row.get("hold") or {}).get("release") or RELEASE.get(status, "-"))))
return "\n".join(lines) + "\n"
def jobs(catalog, statuses, difficulties):
for row in catalog["tasks"]:
if row["status"] in statuses and (not difficulties or row["difficulty"] in difficulties):
yield "{} {}".format(row["id"], row["calibration"]["path"] if row["calibration"] else "-")
def main(argv=None):
parser = argparse.ArgumentParser(description="Build or query the usable-task catalog")
commands = parser.add_subparsers(dest="command", required=True)
make = commands.add_parser("build", help="Write a new catalog JSON")
make.add_argument("--tasks", type=Path, default=Path("tasks"))
make.add_argument("--calibrations", type=Path, nargs="+", required=True)
make.add_argument("--runs", type=Path, nargs="*", default=[])
make.add_argument("--holds", type=Path)
make.add_argument("--reviews", type=Path)
make.add_argument("--image-id", required=True)
make.add_argument("--output", type=Path, required=True)
make.add_argument("--markdown", type=Path)
render = commands.add_parser("render", help="Write the Markdown view of an existing catalog JSON")
render.add_argument("catalog", type=Path)
render.add_argument("--output", type=Path, required=True)
query = commands.add_parser("jobs", help="Print 'task calibration' lines for rollout.batch")
query.add_argument("catalog", type=Path)
query.add_argument("--status", nargs="+", default=["USABLE"])
query.add_argument("--difficulty", nargs="*", default=[])
args = parser.parse_args(argv)
try:
if args.command in ("jobs", "render"):
catalog = json.loads(core._read(core._path(args.catalog).parent, args.catalog.name, core.MAX_TRACE_BYTES))
if args.command == "render":
core._create(core._path(args.output), markdown(catalog).encode("utf-8"))
else:
print("\n".join(jobs(catalog, set(args.status), set(args.difficulty))))
return 0
catalog = build(args.tasks, args.calibrations, args.runs, args.holds and core._path(args.holds), args.image_id,
args.reviews and core._path(args.reviews))
core._create(core._path(args.output), (json.dumps(catalog, ensure_ascii=False, indent=1, sort_keys=True)
+ "\n").encode("utf-8"))
if args.markdown:
core._create(core._path(args.markdown), markdown(catalog).encode("utf-8"))
print(json.dumps({key: catalog[key] for key in ("total", "status", "usable_difficulty")}, ensure_ascii=False))
return 0
except (ValueError, OSError, KeyError) as error:
print("catalog rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())