- bos_verify still runs the original check.py unchanged, but failing child processes now copy their stderr tail into grade.stderr; report rows carry fail_detail (that line, else the grade reason). - bos_import rejects envs whose hidden tests use candidate-module names the public prompt/template never mention; a repo test applies the same check to every admitted task. - S500_n3_152 v2 publishes process(request): its v1 FAILs came from that hidden requirement, not from the models. All Bos tasks carry the new wrapper as version 2; 21 tasks recalibrated on trace5 (805 trials). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
140 lines
7.6 KiB
Python
140 lines
7.6 KiB
Python
"""Summarize one batch of Pi controllers: outcomes, evidence, usage and failures."""
|
|
|
|
import argparse
|
|
import collections
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import sys
|
|
|
|
from . import core
|
|
|
|
|
|
def _jsonl(root, name, limit=core.MAX_FILE_BYTES):
|
|
return [json.loads(line) for line in core._read(root, name, limit).decode("utf-8").splitlines() if line]
|
|
|
|
|
|
def attempt(controller):
|
|
"""One row per controller, read only from host-held receipts and native output."""
|
|
try:
|
|
control = json.loads(core._read(controller, "receipt.json"))
|
|
except ValueError:
|
|
control = dict(status="NO_RECEIPT", task_id="unknown") # Still running, or the controller died before writing.
|
|
evidence = control.get("pi_evidence", {})
|
|
row = dict(controller=controller.name, task_id=control.get("task_id"), task_digest=control.get("task_digest"),
|
|
model_id=control.get("model_id", "glm-5.3"), status=control.get("status"),
|
|
task_verdict=control.get("task_verdict"), evidence=evidence.get("status"),
|
|
evidence_reason=evidence.get("reason"), cleanup=control.get("cleanup"),
|
|
agent_exit=control.get("agent_exit"), agent_termination=control.get("agent_termination"),
|
|
error=control.get("error_type"), error_phase=control.get("error_phase"),
|
|
model_requests=evidence.get("model_requests"), tool_calls=None, tokens=None, seconds=None,
|
|
stop_reason=None, cost="unknown", fail_detail=None)
|
|
# Archivable: a real task outcome with verified native evidence and confirmed cleanup.
|
|
row["complete"] = (row["status"] in ("PASS", "FAIL") and row["evidence"] == "VERIFIED" and
|
|
row["cleanup"] == "confirmed" and row["agent_exit"] == 0)
|
|
if control.get("attempt"):
|
|
run = core._path(control["attempt"])
|
|
events = _jsonl(run, "events.jsonl")
|
|
times = [event["time"] for event in events if event.get("event") in ("created", "finished")]
|
|
row["seconds"] = round(times[-1] - times[0], 3) if len(times) >= 2 else None
|
|
try:
|
|
trace = _jsonl(run, "trace/native.jsonl", core.MAX_TRACE_BYTES)
|
|
except ValueError:
|
|
trace = []
|
|
usage = [event["message"].get("usage", {}) for event in trace if event.get("type") == "message_end"
|
|
and event.get("message", {}).get("role") == "assistant"]
|
|
row["tokens"] = {key: sum(item.get(key, 0) for item in usage)
|
|
for key in ("input", "output", "cacheRead", "totalTokens")} if usage else None
|
|
row["tool_calls"] = sum(event.get("type") == "tool_execution_end" for event in trace)
|
|
if row["task_verdict"] == "FAIL":
|
|
# A crashed hidden test leaves its exception in grade.stderr; a checked mismatch only in the reason.
|
|
lines = core._read(run, "grade.stderr").decode("utf-8", "replace").strip().splitlines() if (
|
|
run / "grade.stderr").is_file() else []
|
|
if not lines and (run / "receipt.json").is_file():
|
|
lines = [json.loads(core._read(run, "receipt.json")).get("grade", {}).get("verdict", {}).get("reason") or ""]
|
|
row["fail_detail"] = lines[-1][:300] if lines and lines[-1] else None
|
|
ends = [event for event in trace if event.get("type") == "session_end"]
|
|
if ends:
|
|
row["stop_reason"] = ends[-1].get("final_stop_reason") or ends[-1].get("reason")
|
|
row["cost"] = "available" if ends[-1].get("cost_available") else "unknown"
|
|
return row
|
|
|
|
|
|
def _spread(values):
|
|
values = sorted(value for value in values if value is not None)
|
|
if not values:
|
|
return None
|
|
return dict(n=len(values), mean=round(statistics.mean(values), 3), p50=values[len(values) // 2],
|
|
p90=values[min(len(values) - 1, int(len(values) * 0.9))], max=values[-1])
|
|
|
|
|
|
def report(batch, tasks_root):
|
|
batch = core._path(batch)
|
|
rows = [attempt(path) for path in sorted(batch.iterdir()) if path.is_dir() and path.name.startswith("pi-")]
|
|
regraded = {}
|
|
if (batch / "regrade").is_dir():
|
|
for path in sorted((batch / "regrade").iterdir()):
|
|
if path.name.startswith("regrade-"):
|
|
receipt = json.loads(core._read(path, "receipt.json"))
|
|
if receipt.get("cleanup") == "confirmed" and receipt.get("consistent") is not None:
|
|
regraded[receipt["controller"]] = receipt["consistent"]
|
|
families = {}
|
|
for path in sorted(core._path(tasks_root).iterdir()):
|
|
if path.joinpath("task.json").is_file():
|
|
manifest = json.loads(core._read(path, "task.json"))
|
|
families[manifest["id"]] = manifest["family"]
|
|
per_task = collections.defaultdict(collections.Counter)
|
|
for row in rows:
|
|
row["family"] = families.get(row["task_id"], "unknown")
|
|
row["regrade_consistent"] = regraded.get(row["controller"])
|
|
per_task[row["task_id"]][row["status"] if row["complete"] else "incomplete"] += 1
|
|
per_family = collections.defaultdict(lambda: dict(tasks=set(), attempts=0, complete=0, passed=0))
|
|
for row in rows:
|
|
family = per_family[row["family"]]
|
|
family["tasks"].add(row["task_id"])
|
|
family["attempts"] += 1
|
|
family["complete"] += row["complete"]
|
|
family["passed"] += row["complete"] and row["status"] == "PASS"
|
|
complete = [row for row in rows if row["complete"]]
|
|
return dict(
|
|
schema="embedded-rollout.batch-report/1", batch=str(batch), attempts=len(rows),
|
|
complete_trajectories=len(complete), passed=sum(row["status"] == "PASS" for row in complete),
|
|
failed=sum(row["status"] == "FAIL" for row in complete),
|
|
excluded=collections.Counter("{}/{}".format(row["status"], row["evidence"]) for row in rows
|
|
if not row["complete"]),
|
|
models=sorted({row["model_id"] for row in rows}),
|
|
tasks=dict(sorted((task, dict(counts)) for task, counts in per_task.items())),
|
|
families={name: dict(value, tasks=len(value["tasks"])) for name, value in sorted(per_family.items())},
|
|
seconds=_spread(row["seconds"] for row in rows),
|
|
model_requests=_spread(row["model_requests"] for row in rows),
|
|
tool_calls=_spread(row["tool_calls"] for row in rows),
|
|
tokens={key: sum((row["tokens"] or {}).get(key, 0) for row in rows)
|
|
for key in ("input", "output", "cacheRead", "totalTokens")},
|
|
cost="unknown" if any(row["cost"] != "available" for row in rows) else "available",
|
|
regraded=sum(row["regrade_consistent"] is not None for row in complete),
|
|
regrade_consistent=sum(row["regrade_consistent"] is True for row in complete),
|
|
rows=rows)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Summarize a batch directory of Pi controllers")
|
|
parser.add_argument("batch", type=Path)
|
|
parser.add_argument("--tasks", type=Path, default=Path("tasks"))
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
result = report(args.batch, args.tasks)
|
|
core._create(core._path(args.output), (json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False)
|
|
+ "\n").encode("utf-8"))
|
|
print(json.dumps({key: result[key] for key in ("attempts", "complete_trajectories", "passed", "failed",
|
|
"excluded", "regraded", "regrade_consistent", "families",
|
|
"cost")}, ensure_ascii=False))
|
|
return 0
|
|
except (ValueError, OSError, KeyError) as error:
|
|
print("report rejected: " + str(error), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|