rollout.dashboard serves catalog counts, attempts and complete trajectories, and the public prompt and template of each task. catalog.controllers()/complete() are split out so the dashboard can reuse receipts it has already read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
249 lines
14 KiB
Python
249 lines
14 KiB
Python
"""Build the usable-task catalog from task packages, calibrations, holds and verified controller receipts."""
|
||
|
||
import argparse
|
||
import collections
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import sys
|
||
import time
|
||
|
||
from . import core
|
||
|
||
QWEN = "qwen3.8-27b-nvfp4"
|
||
|
||
|
||
def _language(names):
|
||
suffixes = {Path(name).suffix for name in names}
|
||
if suffixes & {".cpp", ".hpp", ".cc"}:
|
||
return "C++"
|
||
if suffixes & {".c", ".h"}:
|
||
return "C"
|
||
return "Python" if ".py" in suffixes else ",".join(sorted(suffixes))
|
||
|
||
|
||
def calibrations(roots, image_id):
|
||
"""Map task digest -> newest PASS Docker calibration on this image."""
|
||
found = {}
|
||
for root in roots:
|
||
for path in sorted(core._path(root).glob("calibration-*/receipt.json")):
|
||
raw = path.read_bytes()
|
||
receipt = json.loads(raw)
|
||
if (receipt.get("status") == "PASS" and receipt.get("cleanup") == "confirmed" and
|
||
receipt.get("trusted_local") is False and receipt.get("image_id") == image_id):
|
||
found[receipt["task_digest"]] = dict(path=str(path.parent), trials=len(receipt.get("trials", [])),
|
||
receipt_sha256=hashlib.sha256(raw).hexdigest())
|
||
return found
|
||
|
||
|
||
def controllers(roots):
|
||
"""Yield (controller directory, receipt or None while it has none) once per controller name."""
|
||
seen = set()
|
||
for root in roots:
|
||
for folder, dirs, files in os.walk(root):
|
||
name = os.path.basename(folder)
|
||
if name.startswith("pi-"):
|
||
dirs[:] = []
|
||
# A batch copied to cold storage before its hot copy is removed holds the same attempt twice.
|
||
if name in seen:
|
||
continue
|
||
try:
|
||
receipt = json.loads(Path(folder, "receipt.json").read_bytes())
|
||
except (OSError, ValueError):
|
||
yield Path(folder), None # Still running, or the controller died before writing.
|
||
continue
|
||
seen.add(name)
|
||
yield Path(folder), receipt
|
||
elif name in ("workspace", "candidate", "trace", "grader") or name.startswith(("calibration-", "regrade-")):
|
||
dirs[:] = []
|
||
|
||
|
||
def complete(receipt):
|
||
"""A real task outcome with VERIFIED native evidence, a clean Agent exit and confirmed cleanup."""
|
||
return (receipt.get("status") in ("PASS", "FAIL") and receipt.get("cleanup") == "confirmed" and
|
||
(receipt.get("pi_evidence") or {}).get("status") == "VERIFIED" and receipt.get("agent_exit") == 0)
|
||
|
||
|
||
def outcomes(found):
|
||
"""Count complete trajectories per task digest and model from (controller, receipt) pairs."""
|
||
counts = collections.defaultdict(lambda: collections.defaultdict(collections.Counter))
|
||
for _, receipt in found:
|
||
if receipt and complete(receipt):
|
||
counts[receipt.get("task_digest")][receipt.get("model_id", "glm-5.3")][receipt["status"]] += 1
|
||
return counts
|
||
|
||
|
||
def difficulty(qwen):
|
||
passed, failed = qwen.get("PASS", 0), qwen.get("FAIL", 0)
|
||
total = passed + failed
|
||
if total == 0:
|
||
return "UNMEASURED"
|
||
if failed == 0 and total >= 2:
|
||
return "ROUTINE"
|
||
if total >= 3:
|
||
return "HARD" if 3 * passed <= total else "MEDIUM"
|
||
return "PARTIAL"
|
||
|
||
|
||
def build(tasks_root, calibration_roots, runs_roots, holds_path, image_id, reviews_path=None, found=None,
|
||
calibrated=None):
|
||
"""`found` takes (controller, receipt) pairs already read from `runs_roots`; `calibrated` adds calibrations
|
||
already read from slow roots."""
|
||
holds = json.loads(core._read(holds_path.parent, holds_path.name)) if holds_path else {}
|
||
reviews = json.loads(core._read(reviews_path.parent, reviews_path.name)) if reviews_path else {}
|
||
calibrated = {**(calibrated or {}), **calibrations(calibration_roots, image_id)}
|
||
measured = outcomes(controllers(runs_roots) if found is None else found)
|
||
rows = []
|
||
for root in sorted(core._path(tasks_root).iterdir()):
|
||
if not root.joinpath("task.json").is_file():
|
||
continue
|
||
task = core.load_task(root)
|
||
results = {model: dict(PASS=c.get("PASS", 0), FAIL=c.get("FAIL", 0))
|
||
for model, c in sorted(measured.get(task["_digest"], {}).items())}
|
||
row = dict(id=task["id"], version=task["version"], family=task["family"], source=task["source"],
|
||
language=_language(task["candidate_files"]), candidate_files=task["candidate_files"],
|
||
fixtures=len(task["fixtures"]), task_digest=task["_digest"], results=results,
|
||
difficulty=difficulty(results.get(QWEN, {})), difficulty_source="measured",
|
||
calibration=calibrated.get(task["_digest"]))
|
||
review = reviews.get(task["id"])
|
||
# A review judged one exact task identity; it lapses when the task changes.
|
||
if review and review.get("task_digest") == task["_digest"]:
|
||
row.update(difficulty=review["difficulty"], difficulty_source="reviewed", review=review)
|
||
elif review:
|
||
row["stale_review"] = review
|
||
if root.joinpath("isolation-blocked.txt").exists():
|
||
row.update(status="BLOCKED", reason="判据会在候选进程内运行隐藏测试,需按 RUNNER 方式重写后重新准入")
|
||
elif task["id"] in holds:
|
||
row.update(status="HOLD", reason=holds[task["id"]]["reason"], hold=holds[task["id"]])
|
||
elif row["calibration"] is None:
|
||
row.update(status="UNCALIBRATED", reason="没有同摘要、同镜像的 PASS 正式校准")
|
||
else:
|
||
row.update(status="USABLE", reason=None)
|
||
rows.append(row)
|
||
usable = [row for row in rows if row["status"] == "USABLE"]
|
||
return dict(schema="embedded-rollout.task-catalog/1", generated_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
image_id=image_id, qwen_model=QWEN, total=len(rows),
|
||
status=dict(collections.Counter(row["status"] for row in rows)),
|
||
usable_difficulty=dict(collections.Counter(row["difficulty"] for row in usable)),
|
||
usable_families=dict(collections.Counter(row["family"] for row in usable)),
|
||
usable_languages=dict(collections.Counter(row["language"] for row in usable)),
|
||
tasks=rows)
|
||
|
||
|
||
USAGE = """## 使用方法
|
||
|
||
在 36 的项目根目录执行。`IMAGE` 用上面的镜像 digest,`PRIVATE_CREDENTIAL_FILE` 是 host 私有凭据文件路径。
|
||
|
||
```sh
|
||
# 单题:可直接复用题库里的校准目录,省略 --calibration 则先自动完整校准
|
||
python3 -m rollout.pi_run tasks/TASK_ID --image IMAGE --model qwen3.8-27b-nvfp4 --credential-file PRIVATE_CREDENTIAL_FILE --calibration CALIBRATION --output runs/BATCH
|
||
|
||
# 批量:按状态/难度筛出“任务 校准目录”,交给 batch 并发;基础设施失败自动补采,中断后重跑同一命令即续跑
|
||
python3 -m rollout.catalog jobs docs/task-catalog.json --status USABLE --difficulty HARD MEDIUM > JOBS
|
||
python3 -m rollout.batch JOBS --image IMAGE --model qwen3.8-27b-nvfp4 --credential-file PRIVATE_CREDENTIAL_FILE --output runs/BATCH --parallel 2
|
||
|
||
# 刷新题库:新增或修改题目、换镜像、补采后重新生成
|
||
python3 -m rollout.catalog build --calibrations runs/CALIBRATION_DIR --runs runs /mnt/data/ming01.wang/embedded-rollout/runs --holds docs/task-holds.json --reviews docs/task-reviews.json --image-id IMAGE_ID --output NEW.json --markdown NEW.md
|
||
```
|
||
|
||
状态:USABLE 可直接使用;HOLD 有已登记的题面或来源问题;BLOCKED 判据不能隔离候选代码;UNCALIBRATED 当前镜像下没有 PASS 校准。带“(评审)”的难度来自 docs/task-reviews.json 中针对同一 task digest 的人工复核(例如 Agent 自测死锁超时被复核为能力失败),否则只统计与当前 task digest 一致的 Qwen 完整轨迹:0 条为 UNMEASURED,至少 2 条全 PASS 为 ROUTINE,至少 3 条且通过率不超过 1/3 为 HARD,其余 3 条以上为 MEDIUM,样本不足为 PARTIAL。题目改版(digest 变化)后旧结果不计入。
|
||
"""
|
||
|
||
|
||
RELEASE = dict(BLOCKED="按 RUNNER 方式重写判据(候选只在子进程里运行、只拿输入;参考 progress.md 2026-10-15 重写试点),升版本后重新校准",
|
||
UNCALIBRATED="用当前镜像重新正式校准,再重新生成题库")
|
||
|
||
|
||
def _cell(text):
|
||
return str(text).replace("|", "\\|").replace("\n", " ")
|
||
|
||
|
||
def markdown(catalog):
|
||
lines = ["# 题库", "",
|
||
"生成于 {},由 `python3 -m rollout.catalog build` 从任务包、校准回执和已核对的 controller 回执生成;"
|
||
"请勿手改,改动后重新生成。正式镜像 `{}`。".format(catalog["generated_at"], catalog["image_id"]), "",
|
||
"共 {} 道:{}。可用题难度:{};任务族:{};语言:{}。".format(
|
||
catalog["total"], "、".join("{} {}".format(k, v) for k, v in sorted(catalog["status"].items())),
|
||
"、".join("{} {}".format(k, v) for k, v in sorted(catalog["usable_difficulty"].items())),
|
||
"、".join("{} {}".format(k, v) for k, v in sorted(catalog["usable_families"].items())),
|
||
"、".join("{} {}".format(k, v) for k, v in sorted(catalog["usable_languages"].items()))), "", USAGE]
|
||
order = dict(HARD=0, MEDIUM=1, PARTIAL=2, UNMEASURED=3, ROUTINE=4)
|
||
usable = sorted((row for row in catalog["tasks"] if row["status"] == "USABLE"),
|
||
key=lambda row: (order[row["difficulty"]], row["family"], row["id"]))
|
||
lines += ["## 可用题({} 道)".format(len(usable)), "",
|
||
"| 任务 | 难度 | Qwen P/F | GLM P/F | 族 | 语言 | 样例类 | 来源 |", "| --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||
for row in usable:
|
||
q, g = row["results"].get(QWEN, {}), row["results"].get("glm-5.3", {})
|
||
lines.append("| {} | {} | {} | {} | {} | {} | {} | {} |".format(
|
||
row["id"], row["difficulty"] + ("(评审)" if row["difficulty_source"] == "reviewed" else ""), "{}/{}".format(q.get("PASS", 0), q.get("FAIL", 0)),
|
||
"{}/{}".format(g.get("PASS", 0), g.get("FAIL", 0)), row["family"], row["language"], row["fixtures"],
|
||
_cell(row["source"])))
|
||
reviewed = [row for row in usable if row["difficulty_source"] == "reviewed" and row["difficulty"] == "HARD"]
|
||
if reviewed:
|
||
lines += ["", "## 难题的评审依据", "", "| 任务 | 评审结果 | 依据 |", "| --- | --- | --- |"]
|
||
for row in reviewed:
|
||
lines.append("| {} | {} | {} |".format(row["id"], _cell(row["review"].get("results", "-")),
|
||
_cell(row["review"].get("basis", "-"))))
|
||
for status, title in (("HOLD", "暂缓使用"), ("UNCALIBRATED", "未校准"), ("BLOCKED", "判据待重写")):
|
||
rows = [row for row in catalog["tasks"] if row["status"] == status]
|
||
if not rows:
|
||
continue
|
||
lines += ["", "## {}:{}({} 道)".format(status, title, len(rows)), "", "| 任务 | 族 | 原因 | 解除条件 |",
|
||
"| --- | --- | --- | --- |"]
|
||
for row in sorted(rows, key=lambda row: row["id"]):
|
||
lines.append("| {} | {} | {} | {} |".format(row["id"], row["family"], _cell(row["reason"]),
|
||
_cell((row.get("hold") or {}).get("release") or RELEASE.get(status, "-"))))
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def jobs(catalog, statuses, difficulties):
|
||
for row in catalog["tasks"]:
|
||
if row["status"] in statuses and (not difficulties or row["difficulty"] in difficulties):
|
||
yield "{} {}".format(row["id"], row["calibration"]["path"] if row["calibration"] else "-")
|
||
|
||
|
||
def main(argv=None):
|
||
parser = argparse.ArgumentParser(description="Build or query the usable-task catalog")
|
||
commands = parser.add_subparsers(dest="command", required=True)
|
||
make = commands.add_parser("build", help="Write a new catalog JSON")
|
||
make.add_argument("--tasks", type=Path, default=Path("tasks"))
|
||
make.add_argument("--calibrations", type=Path, nargs="+", required=True)
|
||
make.add_argument("--runs", type=Path, nargs="*", default=[])
|
||
make.add_argument("--holds", type=Path)
|
||
make.add_argument("--reviews", type=Path)
|
||
make.add_argument("--image-id", required=True)
|
||
make.add_argument("--output", type=Path, required=True)
|
||
make.add_argument("--markdown", type=Path)
|
||
render = commands.add_parser("render", help="Write the Markdown view of an existing catalog JSON")
|
||
render.add_argument("catalog", type=Path)
|
||
render.add_argument("--output", type=Path, required=True)
|
||
query = commands.add_parser("jobs", help="Print 'task calibration' lines for rollout.batch")
|
||
query.add_argument("catalog", type=Path)
|
||
query.add_argument("--status", nargs="+", default=["USABLE"])
|
||
query.add_argument("--difficulty", nargs="*", default=[])
|
||
args = parser.parse_args(argv)
|
||
try:
|
||
if args.command in ("jobs", "render"):
|
||
catalog = json.loads(core._read(core._path(args.catalog).parent, args.catalog.name, core.MAX_TRACE_BYTES))
|
||
if args.command == "render":
|
||
core._create(core._path(args.output), markdown(catalog).encode("utf-8"))
|
||
else:
|
||
print("\n".join(jobs(catalog, set(args.status), set(args.difficulty))))
|
||
return 0
|
||
catalog = build(args.tasks, args.calibrations, args.runs, args.holds and core._path(args.holds), args.image_id,
|
||
args.reviews and core._path(args.reviews))
|
||
core._create(core._path(args.output), (json.dumps(catalog, ensure_ascii=False, indent=1, sort_keys=True)
|
||
+ "\n").encode("utf-8"))
|
||
if args.markdown:
|
||
core._create(core._path(args.markdown), markdown(catalog).encode("utf-8"))
|
||
print(json.dumps({key: catalog[key] for key in ("total", "status", "usable_difficulty")}, ensure_ascii=False))
|
||
return 0
|
||
except (ValueError, OSError, KeyError) as error:
|
||
print("catalog rejected: " + str(error), file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|