data-gen-ebd/rollout/report.py
amos 94eaf64261 fix: private grade diagnostics and a public-contract admission check
- bos_verify still runs the original check.py unchanged, but failing
  child processes now copy their stderr tail into grade.stderr; report
  rows carry fail_detail (that line, else the grade reason).
- bos_import rejects envs whose hidden tests use candidate-module names
  the public prompt/template never mention; a repo test applies the same
  check to every admitted task.
- S500_n3_152 v2 publishes process(request): its v1 FAILs came from that
  hidden requirement, not from the models. All Bos tasks carry the new
  wrapper as version 2; 21 tasks recalibrated on trace5 (805 trials).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 20:27:29 +08:00

140 lines
7.6 KiB
Python

"""Summarize one batch of Pi controllers: outcomes, evidence, usage and failures."""
import argparse
import collections
import json
from pathlib import Path
import statistics
import sys
from . import core
def _jsonl(root, name, limit=core.MAX_FILE_BYTES):
return [json.loads(line) for line in core._read(root, name, limit).decode("utf-8").splitlines() if line]
def attempt(controller):
"""One row per controller, read only from host-held receipts and native output."""
try:
control = json.loads(core._read(controller, "receipt.json"))
except ValueError:
control = dict(status="NO_RECEIPT", task_id="unknown") # Still running, or the controller died before writing.
evidence = control.get("pi_evidence", {})
row = dict(controller=controller.name, task_id=control.get("task_id"), task_digest=control.get("task_digest"),
model_id=control.get("model_id", "glm-5.3"), status=control.get("status"),
task_verdict=control.get("task_verdict"), evidence=evidence.get("status"),
evidence_reason=evidence.get("reason"), cleanup=control.get("cleanup"),
agent_exit=control.get("agent_exit"), agent_termination=control.get("agent_termination"),
error=control.get("error_type"), error_phase=control.get("error_phase"),
model_requests=evidence.get("model_requests"), tool_calls=None, tokens=None, seconds=None,
stop_reason=None, cost="unknown", fail_detail=None)
# Archivable: a real task outcome with verified native evidence and confirmed cleanup.
row["complete"] = (row["status"] in ("PASS", "FAIL") and row["evidence"] == "VERIFIED" and
row["cleanup"] == "confirmed" and row["agent_exit"] == 0)
if control.get("attempt"):
run = core._path(control["attempt"])
events = _jsonl(run, "events.jsonl")
times = [event["time"] for event in events if event.get("event") in ("created", "finished")]
row["seconds"] = round(times[-1] - times[0], 3) if len(times) >= 2 else None
try:
trace = _jsonl(run, "trace/native.jsonl", core.MAX_TRACE_BYTES)
except ValueError:
trace = []
usage = [event["message"].get("usage", {}) for event in trace if event.get("type") == "message_end"
and event.get("message", {}).get("role") == "assistant"]
row["tokens"] = {key: sum(item.get(key, 0) for item in usage)
for key in ("input", "output", "cacheRead", "totalTokens")} if usage else None
row["tool_calls"] = sum(event.get("type") == "tool_execution_end" for event in trace)
if row["task_verdict"] == "FAIL":
# A crashed hidden test leaves its exception in grade.stderr; a checked mismatch only in the reason.
lines = core._read(run, "grade.stderr").decode("utf-8", "replace").strip().splitlines() if (
run / "grade.stderr").is_file() else []
if not lines and (run / "receipt.json").is_file():
lines = [json.loads(core._read(run, "receipt.json")).get("grade", {}).get("verdict", {}).get("reason") or ""]
row["fail_detail"] = lines[-1][:300] if lines and lines[-1] else None
ends = [event for event in trace if event.get("type") == "session_end"]
if ends:
row["stop_reason"] = ends[-1].get("final_stop_reason") or ends[-1].get("reason")
row["cost"] = "available" if ends[-1].get("cost_available") else "unknown"
return row
def _spread(values):
values = sorted(value for value in values if value is not None)
if not values:
return None
return dict(n=len(values), mean=round(statistics.mean(values), 3), p50=values[len(values) // 2],
p90=values[min(len(values) - 1, int(len(values) * 0.9))], max=values[-1])
def report(batch, tasks_root):
batch = core._path(batch)
rows = [attempt(path) for path in sorted(batch.iterdir()) if path.is_dir() and path.name.startswith("pi-")]
regraded = {}
if (batch / "regrade").is_dir():
for path in sorted((batch / "regrade").iterdir()):
if path.name.startswith("regrade-"):
receipt = json.loads(core._read(path, "receipt.json"))
if receipt.get("cleanup") == "confirmed" and receipt.get("consistent") is not None:
regraded[receipt["controller"]] = receipt["consistent"]
families = {}
for path in sorted(core._path(tasks_root).iterdir()):
if path.joinpath("task.json").is_file():
manifest = json.loads(core._read(path, "task.json"))
families[manifest["id"]] = manifest["family"]
per_task = collections.defaultdict(collections.Counter)
for row in rows:
row["family"] = families.get(row["task_id"], "unknown")
row["regrade_consistent"] = regraded.get(row["controller"])
per_task[row["task_id"]][row["status"] if row["complete"] else "incomplete"] += 1
per_family = collections.defaultdict(lambda: dict(tasks=set(), attempts=0, complete=0, passed=0))
for row in rows:
family = per_family[row["family"]]
family["tasks"].add(row["task_id"])
family["attempts"] += 1
family["complete"] += row["complete"]
family["passed"] += row["complete"] and row["status"] == "PASS"
complete = [row for row in rows if row["complete"]]
return dict(
schema="embedded-rollout.batch-report/1", batch=str(batch), attempts=len(rows),
complete_trajectories=len(complete), passed=sum(row["status"] == "PASS" for row in complete),
failed=sum(row["status"] == "FAIL" for row in complete),
excluded=collections.Counter("{}/{}".format(row["status"], row["evidence"]) for row in rows
if not row["complete"]),
models=sorted({row["model_id"] for row in rows}),
tasks=dict(sorted((task, dict(counts)) for task, counts in per_task.items())),
families={name: dict(value, tasks=len(value["tasks"])) for name, value in sorted(per_family.items())},
seconds=_spread(row["seconds"] for row in rows),
model_requests=_spread(row["model_requests"] for row in rows),
tool_calls=_spread(row["tool_calls"] for row in rows),
tokens={key: sum((row["tokens"] or {}).get(key, 0) for row in rows)
for key in ("input", "output", "cacheRead", "totalTokens")},
cost="unknown" if any(row["cost"] != "available" for row in rows) else "available",
regraded=sum(row["regrade_consistent"] is not None for row in complete),
regrade_consistent=sum(row["regrade_consistent"] is True for row in complete),
rows=rows)
def main(argv=None):
parser = argparse.ArgumentParser(description="Summarize a batch directory of Pi controllers")
parser.add_argument("batch", type=Path)
parser.add_argument("--tasks", type=Path, default=Path("tasks"))
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args(argv)
try:
result = report(args.batch, args.tasks)
core._create(core._path(args.output), (json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False)
+ "\n").encode("utf-8"))
print(json.dumps({key: result[key] for key in ("attempts", "complete_trajectories", "passed", "failed",
"excluded", "regraded", "regrade_consistent", "families",
"cost")}, ensure_ascii=False))
return 0
except (ValueError, OSError, KeyError) as error:
print("report rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())