import json from pathlib import Path import tempfile import unittest from rollout import report class ReportTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() self.batch, self.tasks = self.root / "batch", self.root / "tasks" for task, family in (("a", "abi"), ("b", "firmware")): self.tasks.joinpath(task).mkdir(parents=True) self.tasks.joinpath(task, "task.json").write_text(json.dumps(dict(id=task, family=family))) def controller(self, name, task, status, evidence="VERIFIED", cleanup="confirmed", tokens=(10, 5)): control = self.batch / name run = control / "run-x" (run / "trace").mkdir(parents=True) run.joinpath("events.jsonl").write_text("".join(json.dumps(dict(event=kind, time=time)) + "\n" for kind, time in (("created", 100.0), ("graded", 150.0), ("finished", 160.5)))) trace = [dict(type="message_end", message=dict(role="assistant", usage=dict(input=tokens[0], output=tokens[1], cacheRead=1, totalTokens=sum(tokens) + 1))), dict(type="message_end", message=dict(role="toolResult")), dict(type="tool_execution_end"), dict(type="tool_execution_end"), dict(type="session_end", final_stop_reason="stop", cost_available=False)] run.joinpath("trace/native.jsonl").write_text("".join(json.dumps(event) + "\n" for event in trace)) control.joinpath("receipt.json").write_text(json.dumps(dict( task_id=task, task_digest="d", model_id="glm-5.3", status=status, task_verdict=status, cleanup=cleanup, agent_exit=0, agent_termination="exit", attempt=str(run), pi_evidence=dict(status=evidence, model_requests=3)))) def test_only_verified_cleaned_task_outcomes_count_as_complete(self): self.controller("pi-1", "a", "PASS") self.controller("pi-2", "a", "FAIL") self.batch.joinpath("pi-2/run-x/grade.stderr").write_text("Traceback ...\nAttributeError: no attribute 'process'\n\n") self.controller("pi-6", "a", "FAIL") self.batch.joinpath("pi-6/run-x/receipt.json").write_text(json.dumps(dict(grade=dict(verdict=dict( status="FAIL", checks=1, reason="FAIL: AssertionError case 4 mismatch"))))) self.controller("pi-3", "b", "UNVERIFIED", evidence="UNVERIFIED") self.controller("pi-4", "b", "INFRA_ERROR", cleanup="unknown") self.batch.joinpath("calibration-x").mkdir() self.batch.joinpath("pi-5-running").mkdir() for name, controller, consistent, cleanup in (("regrade-1", "pi-1", True, "confirmed"), ("regrade-2", "pi-2", False, "confirmed"), ("regrade-3", "pi-2", True, "unknown")): self.batch.joinpath("regrade", name).mkdir(parents=True) self.batch.joinpath("regrade", name, "receipt.json").write_text(json.dumps(dict( controller=controller, consistent=consistent, cleanup=cleanup))) result = report.report(self.batch, self.tasks) self.assertEqual((result["regraded"], result["regrade_consistent"]), (2, 1)) self.assertEqual((result["attempts"], result["complete_trajectories"], result["passed"], result["failed"]), (6, 3, 1, 2)) self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1, "NO_RECEIPT/None": 1}) self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 2}, "b": {"incomplete": 2}, "unknown": {"incomplete": 1}}) self.assertEqual(result["families"]["abi"], dict(tasks=1, attempts=3, complete=3, passed=1)) self.assertEqual(result["tokens"], dict(input=50, output=25, cacheRead=5, totalTokens=80)) self.assertEqual(result["tool_calls"]["p50"], 2) self.assertEqual(result["seconds"]["max"], 60.5) self.assertEqual(result["cost"], "unknown") details = {row["controller"]: row["fail_detail"] for row in result["rows"]} self.assertEqual((details["pi-2"], details["pi-6"]), ("AttributeError: no attribute 'process'", "FAIL: AssertionError case 4 mismatch")) output = self.root / "report.json" self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 0) self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 2) if __name__ == "__main__": unittest.main()