data-gen-ebd/tests/test_report.py
amos 94eaf64261 fix: private grade diagnostics and a public-contract admission check
- bos_verify still runs the original check.py unchanged, but failing
  child processes now copy their stderr tail into grade.stderr; report
  rows carry fail_detail (that line, else the grade reason).
- bos_import rejects envs whose hidden tests use candidate-module names
  the public prompt/template never mention; a repo test applies the same
  check to every admitted task.
- S500_n3_152 v2 publishes process(request): its v1 FAILs came from that
  hidden requirement, not from the models. All Bos tasks carry the new
  wrapper as version 2; 21 tasks recalibrated on trace5 (805 trials).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 20:27:29 +08:00

75 lines
4.7 KiB
Python

import json
from pathlib import Path
import tempfile
import unittest
from rollout import report
class ReportTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.batch, self.tasks = self.root / "batch", self.root / "tasks"
for task, family in (("a", "abi"), ("b", "firmware")):
self.tasks.joinpath(task).mkdir(parents=True)
self.tasks.joinpath(task, "task.json").write_text(json.dumps(dict(id=task, family=family)))
def controller(self, name, task, status, evidence="VERIFIED", cleanup="confirmed", tokens=(10, 5)):
control = self.batch / name
run = control / "run-x"
(run / "trace").mkdir(parents=True)
run.joinpath("events.jsonl").write_text("".join(json.dumps(dict(event=kind, time=time)) + "\n" for kind, time in
(("created", 100.0), ("graded", 150.0), ("finished", 160.5))))
trace = [dict(type="message_end", message=dict(role="assistant", usage=dict(input=tokens[0], output=tokens[1],
cacheRead=1, totalTokens=sum(tokens) + 1))),
dict(type="message_end", message=dict(role="toolResult")),
dict(type="tool_execution_end"), dict(type="tool_execution_end"),
dict(type="session_end", final_stop_reason="stop", cost_available=False)]
run.joinpath("trace/native.jsonl").write_text("".join(json.dumps(event) + "\n" for event in trace))
control.joinpath("receipt.json").write_text(json.dumps(dict(
task_id=task, task_digest="d", model_id="glm-5.3", status=status, task_verdict=status, cleanup=cleanup,
agent_exit=0, agent_termination="exit", attempt=str(run),
pi_evidence=dict(status=evidence, model_requests=3))))
def test_only_verified_cleaned_task_outcomes_count_as_complete(self):
self.controller("pi-1", "a", "PASS")
self.controller("pi-2", "a", "FAIL")
self.batch.joinpath("pi-2/run-x/grade.stderr").write_text("Traceback ...\nAttributeError: no attribute 'process'\n\n")
self.controller("pi-6", "a", "FAIL")
self.batch.joinpath("pi-6/run-x/receipt.json").write_text(json.dumps(dict(grade=dict(verdict=dict(
status="FAIL", checks=1, reason="FAIL: AssertionError case 4 mismatch")))))
self.controller("pi-3", "b", "UNVERIFIED", evidence="UNVERIFIED")
self.controller("pi-4", "b", "INFRA_ERROR", cleanup="unknown")
self.batch.joinpath("calibration-x").mkdir()
self.batch.joinpath("pi-5-running").mkdir()
for name, controller, consistent, cleanup in (("regrade-1", "pi-1", True, "confirmed"),
("regrade-2", "pi-2", False, "confirmed"),
("regrade-3", "pi-2", True, "unknown")):
self.batch.joinpath("regrade", name).mkdir(parents=True)
self.batch.joinpath("regrade", name, "receipt.json").write_text(json.dumps(dict(
controller=controller, consistent=consistent, cleanup=cleanup)))
result = report.report(self.batch, self.tasks)
self.assertEqual((result["regraded"], result["regrade_consistent"]), (2, 1))
self.assertEqual((result["attempts"], result["complete_trajectories"], result["passed"], result["failed"]),
(6, 3, 1, 2))
self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1,
"NO_RECEIPT/None": 1})
self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 2}, "b": {"incomplete": 2}, "unknown": {"incomplete": 1}})
self.assertEqual(result["families"]["abi"], dict(tasks=1, attempts=3, complete=3, passed=1))
self.assertEqual(result["tokens"], dict(input=50, output=25, cacheRead=5, totalTokens=80))
self.assertEqual(result["tool_calls"]["p50"], 2)
self.assertEqual(result["seconds"]["max"], 60.5)
self.assertEqual(result["cost"], "unknown")
details = {row["controller"]: row["fail_detail"] for row in result["rows"]}
self.assertEqual((details["pi-2"], details["pi-6"]), ("AttributeError: no attribute 'process'",
"FAIL: AssertionError case 4 mismatch"))
output = self.root / "report.json"
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 0)
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 2)
if __name__ == "__main__":
unittest.main()