- bos_verify still runs the original check.py unchanged, but failing child processes now copy their stderr tail into grade.stderr; report rows carry fail_detail (that line, else the grade reason). - bos_import rejects envs whose hidden tests use candidate-module names the public prompt/template never mention; a repo test applies the same check to every admitted task. - S500_n3_152 v2 publishes process(request): its v1 FAILs came from that hidden requirement, not from the models. All Bos tasks carry the new wrapper as version 2; 21 tasks recalibrated on trace5 (805 trials). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
4.7 KiB
Python
75 lines
4.7 KiB
Python
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from rollout import report
|
|
|
|
|
|
class ReportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.batch, self.tasks = self.root / "batch", self.root / "tasks"
|
|
for task, family in (("a", "abi"), ("b", "firmware")):
|
|
self.tasks.joinpath(task).mkdir(parents=True)
|
|
self.tasks.joinpath(task, "task.json").write_text(json.dumps(dict(id=task, family=family)))
|
|
|
|
def controller(self, name, task, status, evidence="VERIFIED", cleanup="confirmed", tokens=(10, 5)):
|
|
control = self.batch / name
|
|
run = control / "run-x"
|
|
(run / "trace").mkdir(parents=True)
|
|
run.joinpath("events.jsonl").write_text("".join(json.dumps(dict(event=kind, time=time)) + "\n" for kind, time in
|
|
(("created", 100.0), ("graded", 150.0), ("finished", 160.5))))
|
|
trace = [dict(type="message_end", message=dict(role="assistant", usage=dict(input=tokens[0], output=tokens[1],
|
|
cacheRead=1, totalTokens=sum(tokens) + 1))),
|
|
dict(type="message_end", message=dict(role="toolResult")),
|
|
dict(type="tool_execution_end"), dict(type="tool_execution_end"),
|
|
dict(type="session_end", final_stop_reason="stop", cost_available=False)]
|
|
run.joinpath("trace/native.jsonl").write_text("".join(json.dumps(event) + "\n" for event in trace))
|
|
control.joinpath("receipt.json").write_text(json.dumps(dict(
|
|
task_id=task, task_digest="d", model_id="glm-5.3", status=status, task_verdict=status, cleanup=cleanup,
|
|
agent_exit=0, agent_termination="exit", attempt=str(run),
|
|
pi_evidence=dict(status=evidence, model_requests=3))))
|
|
|
|
def test_only_verified_cleaned_task_outcomes_count_as_complete(self):
|
|
self.controller("pi-1", "a", "PASS")
|
|
self.controller("pi-2", "a", "FAIL")
|
|
self.batch.joinpath("pi-2/run-x/grade.stderr").write_text("Traceback ...\nAttributeError: no attribute 'process'\n\n")
|
|
self.controller("pi-6", "a", "FAIL")
|
|
self.batch.joinpath("pi-6/run-x/receipt.json").write_text(json.dumps(dict(grade=dict(verdict=dict(
|
|
status="FAIL", checks=1, reason="FAIL: AssertionError case 4 mismatch")))))
|
|
self.controller("pi-3", "b", "UNVERIFIED", evidence="UNVERIFIED")
|
|
self.controller("pi-4", "b", "INFRA_ERROR", cleanup="unknown")
|
|
self.batch.joinpath("calibration-x").mkdir()
|
|
self.batch.joinpath("pi-5-running").mkdir()
|
|
for name, controller, consistent, cleanup in (("regrade-1", "pi-1", True, "confirmed"),
|
|
("regrade-2", "pi-2", False, "confirmed"),
|
|
("regrade-3", "pi-2", True, "unknown")):
|
|
self.batch.joinpath("regrade", name).mkdir(parents=True)
|
|
self.batch.joinpath("regrade", name, "receipt.json").write_text(json.dumps(dict(
|
|
controller=controller, consistent=consistent, cleanup=cleanup)))
|
|
result = report.report(self.batch, self.tasks)
|
|
self.assertEqual((result["regraded"], result["regrade_consistent"]), (2, 1))
|
|
self.assertEqual((result["attempts"], result["complete_trajectories"], result["passed"], result["failed"]),
|
|
(6, 3, 1, 2))
|
|
self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1,
|
|
"NO_RECEIPT/None": 1})
|
|
self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 2}, "b": {"incomplete": 2}, "unknown": {"incomplete": 1}})
|
|
self.assertEqual(result["families"]["abi"], dict(tasks=1, attempts=3, complete=3, passed=1))
|
|
self.assertEqual(result["tokens"], dict(input=50, output=25, cacheRead=5, totalTokens=80))
|
|
self.assertEqual(result["tool_calls"]["p50"], 2)
|
|
self.assertEqual(result["seconds"]["max"], 60.5)
|
|
self.assertEqual(result["cost"], "unknown")
|
|
details = {row["controller"]: row["fail_detail"] for row in result["rows"]}
|
|
self.assertEqual((details["pi-2"], details["pi-6"]), ("AttributeError: no attribute 'process'",
|
|
"FAIL: AssertionError case 4 mismatch"))
|
|
output = self.root / "report.json"
|
|
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 0)
|
|
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|